Streaming WARC 1.1 parser, writer, validator, indexer and audit toolkit for MoonBit.
moon add xiaojing012/moon-warcimport {
"xiaojing012/moon-warc" @warc,
}match @warc.parse_archive(data, @warc.Limits::default()) {
Ok(archive) => println("records: \{archive.record_count()}")
Err(error) => println(error.to_string())
}warc-record = header CRLF block CRLF CRLF
header = version warc-fields
version = "WARC/1.1" CRLF
warc-fields = *named-field CRLF
block = *OCTETmoon run cli help
moon run cli parse | validate | inspect | stats | audit
moon run cli query http://example.org/a
moon run cli buildmoon run examples parse | build | validate | index | auditpub struct Limits {
max_archive_bytes : Int64
max_record_bytes : Int64
max_header_bytes : Int64
max_header_count : Int
max_field_name_bytes : Int
max_field_value_bytes : Int
max_block_bytes : Int64
max_records : Int
max_content_length_digits : Int
}type PendingRecordpub struct SegmentInfo {
number : Int64?
origin_id : String?
total_length : Int64?
}pub struct WarcDecoder {
limits : Limits
bytes : Bytes
start : Int
pos : Int64
records_done : Int
total_fed : Int64
pending : PendingRecord?
error : WarcError?
}pub struct WarcDigest {
algorithm : String
value : String
}pub struct WarcError {
stage : WarcErrorStage
kind : WarcErrorKind
byte_offset : Int64
record_index : Int64
context : String
}fn WarcError::new(stage : WarcErrorStage, kind : WarcErrorKind, byte_offset : Int64, record_index : Int64, context : String) -> WarcErrorpub enum WarcErrorKind {
UnexpectedEof
InvalidVersion
MissingColon
InvalidFieldName
InvalidUtf8
InvalidContentLength
IntegerOverflow
ContentLengthMismatch
MissingRequiredField
DuplicateField
InvalidDate
InvalidUri
InvalidDigest
InvalidRecordType
InvalidSeparator
TrailingGarbage
LimitExceeded
InvalidFieldValue
MisplacedField
InvalidIpAddress
} derive(Eq, Debug)pub struct WarcField {
name : String
value : String
}pub struct WarcFinding {
severity : FindingSeverity
code : String
record_index : Int64
context : String
}fn WarcFinding::new(severity : FindingSeverity, code : String, record_index : Int64, context : String) -> WarcFindingfn WarcRecord::require_field(self : WarcRecord, name : String, record_index : Int64) -> Result[String, WarcError]let WARC_FIELDS_MIME_TYPE : Stringlet WARC_MIME_TYPE : Stringlet WARC_VERSION_LINE : Stringfn all_digits(data : Bytes, start : Int, end : Int) -> Boolfn append_decimal_digit(value : Int64, d : Int) -> Int64?fn digit_count(data : Bytes, start : Int, end : Int) -> Intfn digit_value(b : Byte) -> Intfn eq_bytes(data : Bytes, start : Int, end : Int, needle : String) -> Boolfn eq_ignore_ascii_case(data : Bytes, start : Int, end : Int, needle : String) -> Boolfn find_crlf(data : Bytes, from : Int) -> Intfn index_of_byte(data : Bytes, from : Int, needle : Byte) -> Intfn is_separator(b : Byte) -> Boolfn is_token_byte(b : Byte) -> Boolfn lower_ascii(b : Byte) -> Bytefn parse_decimal(data : Bytes, start : Int, end : Int, max_digits : Int, context : String) -> Result[Int64, WarcError]fn parse_record(data : Bytes, start : Int, limits : Limits, record_index : Int64) -> Result[(WarcRecord, Int), WarcError]fn parse_version_line(data : Bytes, start : Int, end : Int, record_index : Int64) -> Result[String, WarcError]fn starts_with(data : Bytes, prefix : Bytes) -> Boolfn valid_field_name(data : Bytes, start : Int, end : Int) -> Boolfn valid_ip_address(s : String) -> BoolStreaming WARC 1.1 parser, writer, validator, indexer and audit toolkit for MoonBit.