mooncakes.io

cmd
→
codec
→
examples
→
json
→
ocf
→
schema
→
single_object
→
README
U
CodecError
U
CodecLimits
U
CompressionCodec
U
Datum
U
Decoder
U
Encoder
U
Field
U
JsonError
U
JsonLimits
U
NamedDefinition
U
OcfBlockConfig
U
OcfError
U
OcfHeader
U
OcfLimits
U
OcfReader
U
OcfWriter
U
Primitive
U
Schema
U
SchemaError
U
SchemaKind
U
SingleObjectError
V
decode
V
decode_container
V
decode_from
V
decode_header
V
decode_single_object
V
decode_single_object_resolved
V
default_block_config
V
default_limits
V
encode
V
encode_container
V
encode_container_blocks
V
encode_header
V
encode_into
V
encode_single_object
V
from_json
V
is_ocf
V
magic
V
parse
V
parse_json
V
single_object_fingerprint
V
stringify
V
to_json

README

yugonlian/moon-avro/ocf does not have a README file

#
OcfError

pub(all) suberror OcfError {
  InvalidMagic
  InvalidSyncMarkerLength(actual~ : Int)
  UnsupportedCodec(String)
  InvalidMetadataKey(String)
  Truncated(offset~ : Int)
  InvalidBlockCount(count~ : Int64)
  BlockTooLarge(length~ : Int, limit~ : Int)
  TooManyRecords(count~ : Int64, limit~ : Int)
  SyncMismatch(block_index~ : Int)
  HeaderSchema(@yugonlian/moon-avro/schema.SchemaError
SchemaError)
  Datum(@yugonlian/moon-avro/codec.CodecError
CodecError)
  Compression(codec~ : String, message~ : String)
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Failures produced while reading or writing an Avro Object Container File.

All offsets are byte offsets in the container. Block indices are zero based.

#
CompressionCodec

source

pub(all) enum CompressionCodec {
  Null
  Deflate
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

OCF block compression codecs supported by this release line.

#
CompressionCodec::name

source

fn CompressionCodec::name(self : CompressionCodec) -> String

#
CompressionCodec::parse

source

fn CompressionCodec::parse(name : String) -> CompressionCodec raise OcfError

#
OcfBlockConfig

source

pub struct OcfBlockConfig {
  max_records : Int
  target_bytes : Int
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Controls how an in-memory OCF writer groups records into data blocks.

max_records bounds the number of records in one block. target_bytes is a soft limit for the uncompressed datum payload: a single record larger than the target is still emitted in its own block as long as it satisfies the hard OcfLimits.max_block_bytes bound.

#
OcfBlockConfig::max_records

source

fn OcfBlockConfig::max_records(self : OcfBlockConfig) -> Int

#
OcfBlockConfig::new

source

fn OcfBlockConfig::new(max_records? : Int, target_bytes? : Int) -> OcfBlockConfig raise OcfError

Construct a block policy for multi-block OCF output.

#
OcfBlockConfig::target_bytes

source

fn OcfBlockConfig::target_bytes(self : OcfBlockConfig) -> Int

#
OcfHeader

source

pub struct OcfHeader {
  schema : @yugonlian/moon-avro/schema.Schema
Schema
  codec : CompressionCodec
  metadata : Map[String, Bytes]
  sync_marker : Bytes
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Parsed OCF header. The metadata map is kept intact so applications do not lose vendor-specific fields.

#
OcfHeader::codec

source

fn OcfHeader::codec(self : OcfHeader) -> CompressionCodec

#
OcfHeader::metadata

source

fn OcfHeader::metadata(self : OcfHeader) -> Map[String, Bytes]

#
OcfHeader::new

source

fn OcfHeader::new(schema : @yugonlian/moon-avro/schema.Schema
Schema, codec? : CompressionCodec, metadata? : Map[String, Bytes], sync_marker : Bytes) -> OcfHeader raise OcfError

#
OcfHeader::schema

source

fn OcfHeader::schema(self : OcfHeader) -> @yugonlian/moon-avro/schema.Schema
Schema

#
OcfHeader::sync_marker

source

fn OcfHeader::sync_marker(self : OcfHeader) -> Bytes

#
OcfLimits

source

pub(all) struct OcfLimits {
  max_metadata_entries : Int
  max_metadata_bytes : Int
  max_block_bytes : Int
  max_records_per_block : Int
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Bounds applied before OCF decoding allocates memory or visits records.

#
OcfLimits::new

source

fn OcfLimits::new(max_metadata_entries~ : Int, max_metadata_bytes~ : Int, max_block_bytes~ : Int, max_records_per_block~ : Int) -> OcfLimits raise OcfError

Build explicit OCF bounds. All bounds must be positive.

#
OcfReader

source

pub struct OcfReader {
  header : OcfHeader
  records : Array[@yugonlian/moon-avro/codec.Datum
Datum]
  index : Int
} derive(@moonbitlang/core/debug.Debug
Debug)

Ordered in-memory OCF reader. next returns records in block/file order.

#
OcfReader::header

source

fn OcfReader::header(self : OcfReader) -> OcfHeader

#
OcfReader::next

source

fn OcfReader::next(self : OcfReader) -> @yugonlian/moon-avro/codec.Datum
Datum?

#
OcfReader::open

source

fn OcfReader::open(input : Bytes, limits? : OcfLimits) -> OcfReader raise OcfError

#
OcfReader::remaining

source

fn OcfReader::remaining(self : OcfReader) -> Int

#
OcfWriter

source

pub struct OcfWriter {
  header : OcfHeader
  limits : OcfLimits
  block_config : OcfBlockConfig
  records : Array[@yugonlian/moon-avro/codec.Datum
Datum]
  finished : Bool
} derive(@moonbitlang/core/debug.Debug
Debug)

Ordered in-memory OCF writer. Records are appended in schema order and partitioned into bounded data blocks when finish is called.

#
OcfWriter::append

source

fn OcfWriter::append(self : OcfWriter, record : @yugonlian/moon-avro/codec.Datum
Datum) -> Unit raise OcfError

#
OcfWriter::finish

source

fn OcfWriter::finish(self : OcfWriter) -> Bytes raise OcfError

#
OcfWriter::new

source

fn OcfWriter::new(header : OcfHeader, limits? : OcfLimits, block_config? : OcfBlockConfig) -> OcfWriter

#
OcfWriter::record_count

source

fn OcfWriter::record_count(self : OcfWriter) -> Int

#
decode_container

source

fn decode_container(input : Bytes, limits? : OcfLimits) -> (OcfHeader, Array[@yugonlian/moon-avro/codec.Datum
Datum]) raise OcfError

Decode an entire OCF file, checking block sizes, record counts, sync markers and all datum payloads against the schema carried by its header.

#
decode_header

source

fn decode_header(input : Bytes, limits? : OcfLimits) -> (OcfHeader, Int) raise OcfError

Decode only an OCF header. bytes_consumed is the first block offset.

#
default_block_config

source

fn default_block_config() -> OcfBlockConfig

A balanced default for ordinary in-memory files: at most 1,000 records or approximately 256 KiB of uncompressed payload per data block.

#
default_limits

source

fn default_limits() -> OcfLimits

Conservative defaults suitable for untrusted in-memory OCF input.

#
encode_container

source

fn encode_container(header : OcfHeader, records : Array[@yugonlian/moon-avro/codec.Datum
Datum], limits? : OcfLimits) -> Bytes raise OcfError

Encode one complete Avro Object Container File as a single data block. Use encode_container_blocks when output should be partitioned for bounded decompression and incremental interoperability.

#
encode_container_blocks

source

fn encode_container_blocks(header : OcfHeader, records : Array[@yugonlian/moon-avro/codec.Datum
Datum], config? : OcfBlockConfig, limits? : OcfLimits) -> Bytes raise OcfError

Encode one complete OCF file using independently compressed data blocks. Blocks are closed when either configured record count or target uncompressed byte size is reached. Record order is preserved.

#
encode_header

source

fn encode_header(header : OcfHeader) -> Bytes raise OcfError

Serialize an OCF header (magic, metadata and 16-byte sync marker).

#
is_ocf

source

fn is_ocf(data : Bytes) -> Bool

#
magic

source

fn magic() -> Bytes

Apache Avro Object Container File magic bytes.

Source Files

Site source Report issue Packages Build queue Skills Statistics

#OcfError

#CompressionCodec

#CompressionCodec::name

#CompressionCodec::parse

#OcfBlockConfig

#OcfBlockConfig::max_records

#OcfBlockConfig::new

#OcfBlockConfig::target_bytes

#OcfHeader

#OcfHeader::codec

#OcfHeader::metadata

#OcfHeader::new

#OcfHeader::schema

#OcfHeader::sync_marker

#OcfLimits

#OcfLimits::new

#OcfReader

#OcfReader::header

#OcfReader::next

#OcfReader::open

#OcfReader::remaining

#OcfWriter

#OcfWriter::append

#OcfWriter::finish

#OcfWriter::new

#OcfWriter::record_count

#decode_container

#decode_header

#default_block_config

#default_limits

#encode_container

#encode_container_blocks

#encode_header

#is_ocf

#magic

Source Files

#
OcfError

#
CompressionCodec

#
CompressionCodec::name

#
CompressionCodec::parse

#
OcfBlockConfig

#
OcfBlockConfig::max_records

#
OcfBlockConfig::new

#
OcfBlockConfig::target_bytes

#
OcfHeader

#
OcfHeader::codec

#
OcfHeader::metadata

#
OcfHeader::new

#
OcfHeader::schema

#
OcfHeader::sync_marker

#
OcfLimits

#
OcfLimits::new

#
OcfReader

#
OcfReader::header

#
OcfReader::next

#
OcfReader::open

#
OcfReader::remaining

#
OcfWriter

#
OcfWriter::append

#
OcfWriter::finish

#
OcfWriter::new

#
OcfWriter::record_count

#
decode_container

#
decode_header

#
default_block_config

#
default_limits

#
encode_container

#
encode_container_blocks

#
encode_header

#
is_ocf

#
magic