The Fory cross-language binary format for MoonBit.
let bytes = @moonfory.dumps(42) // 01 ff 04 2a 00 00 00
let back : Int = @moonfory.loads(bytes)moon add moonbitstack/moonforyimport {
"moonbitstack/moonfory",
}// One value, all of it.
let bytes : Bytes = @moonfory.dumps(value)
let value : T = @moonfory.loads(bytes)
// The same, without allocating a fresh buffer.
@moonfory.dump(value, buffer)
let value : T = @moonfory.load(reader)| Function | Takes | Gives |
|---|---|---|
| dumps(value, ...) | any Codec value | the whole message as Bytes |
| loads(bytes, schemas?, ...) | a message | the value, or a Refused |
| dump(value, buffer, ...) | a value and a buffer | writes into the buffer |
| load(reader) | a reader | one value |
| write_message(w, value) | a writer and a value | head, value, nothing else |
| read_message(r) | a reader | the one value the head announces |
| write_value(w, v) / read_value(r) | a Value | the dynamic form |
pub impl Codec for UserId with fn ty(_self) {
TypeId::Int64
}
pub impl Codec for UserId with fn write_data(self, w) {
w.write_i64(self.0)
}
pub impl Codec for UserId with fn read_data(r, _ty) {
UserId(r.read_i64())
}| Fory type | Code | MoonBit | Written as |
|---|---|---|---|
| BOOL | 1 | Bool | one byte, 00 or 01 |
| INT8 / INT16 / INT32 | 2 / 3 / 4 | Byte / Int | two's complement, little-endian |
| INT64 | 6 | Int64 | eight bytes |
| VARINT32 / VARINT64 | 5 / 7 | any signed width | zig-zag, base-128 |
| TAGGED_INT64 / UINT64 | 8 / 15 | Int64 / UInt64 | four bytes when small, else nine |
| FLOAT32 / FLOAT64 | 19 / 20 | Float / Double | IEEE 754, little-endian |
| STRING | 21 | String | length-and-encoding header, then UTF-8 |
| LIST / SET | 22 / 23 | Array[T] | count, element header, elements |
| MAP | 24 | Map[K, V] | total, then key/value chunks |
| STRUCT | 27 | StructMeta + fields | user id, schema fingerprint, then fields in Fory order |
| ENUM | 25 | ordinal | a varint |
| DURATION / TIMESTAMP / DATE | 37 / 38 / 39 | @moondate.Span / Moment / Date | see below |
| DECIMAL | 40 | (Int, BigInt) | scale, then the unscaled integer |
| BINARY | 41 | Bytes | length, then the bytes |
| BOOL_ARRAY … FLOAT64_ARRAY | 43 … 56 | Array[T] | raw little-endian, no header per element |
| Knob | Default | What it means |
|---|---|---|
| xlang | true | the header carries the cross-language flag |
| track_ref | false | values written twice are sent once and pointed at |
| check_string | true | string bytes are checked as UTF-8 before becoming a String |
| depth | 5 | how deeply values may nest before the message is refused |
| fields | 512 | how many fields a struct may declare |
| graph | 134_217_728 | how many variable-length bytes one message may declare |
// The preset, with one field changed.
@moonfory.dumps(v, track_ref=true)
// A whole config, then one field overridden on top of it.
@moonfory.dumps(v, config=my_config, depth=8)
// Refuse rather than pick when the two disagree.
@moonfory.dumps(v, config=my_config, xlang=false, clash=Panic)let point = @moonfory.StructMeta::new("Point", 100, [
@moonfory.FieldMeta::new("x", @moonfory.TypeId::VarInt32),
@moonfory.FieldMeta::new("y", @moonfory.TypeId::VarInt32),
])
let person = @moonfory.StructMeta::new("Person", 101, [
@moonfory.FieldMeta::new("name", @moonfory.TypeId::String),
@moonfory.FieldMeta::new("age", @moonfory.TypeId::VarInt32),
// A list says what it holds, because the writer may leave the element type
// out of the header.
@moonfory.FieldMeta::new(
"tags", @moonfory.TypeId::List, element=Some(@moonfory.TypeId::String),
),
// A field whose type is declared sends no type ID, so the schema has to say
// which struct it is.
@moonfory.FieldMeta::new(
"home", @moonfory.TypeId::Struct, nullable=true, child=Some(point),
),
])let meta = @moonfory.StructMeta::of(
model, user=101, children=[point], refs=["home"],
)let node = @moonfory.StructMeta::new("Node", 300, [
@moonfory.FieldMeta::new("val", @moonfory.TypeId::VarInt32),
@moonfory.FieldMeta::new("next", @moonfory.TypeId::Struct, track_ref=true),
])
// A node whose `next` is itself. Reference zero is the root, which this is.
let value = @moonfory.Value::Struct(node, [
("val", @moonfory.Value::Int(1L)),
("next", @moonfory.Value::Pointer(0)),
])
@moonfory.dumps(value, track_ref=true) // 01 00 1b ac02 eecb234f 02 fe 00let once = @moonfory.shared(7, "the same string")
let twice = [once, once] // two references, one body
let bytes = @moonfory.dumps(twice, track_ref=true)let schema = @moonfory.RowSchema::new([
{ name: "id", ty: @moonfory.RowType::Int64, nullable: false },
{ name: "name", ty: @moonfory.RowType::String, nullable: false },
{ name: "age", ty: @moonfory.RowType::Int32, nullable: true },
])
let row = @moonfory.Row::new(schema)
row.set(0, @moonfory.RowValue::Int64(7L))
row.set(1, @moonfory.RowValue::String("ada"))
row.set(2, @moonfory.RowValue::Null)
let bytes = row.finish()
let cur = @moonfory.RowCursor::new(bytes[:], schema)
cur.get_field(1) // RowValue::String("ada"), field 0 never read
cur.is_null(2) // truelet date = @moonfory.date_of_days(19_000L) // a Date, as DATE
let moment = @moonfory.moment_of_parts(1_700_000_000L, 0) // a Moment, as TIMESTAMP
let span = @moonfory.span_of_parts(3L, 500_000_000) // a Span, as DURATION
@moonfory.dumps(moment) // and back through loads| Thing | Why not |
|---|---|
| Compatible struct mode (28, 30) | It needs a schema registry to reconcile two type definitions; that belongs with moonmodel, not inside a codec |
| FLOAT8 / FLOAT8_ARRAY (16, 56) | No peer in reach writes them; when one does, the table has the code |
| Reading a struct without its schema | The schema is the user id's meaning; loads(schemas=[...]) supplies it |
| inferring which value is shared | Identity is not in a MoonBit value; say Shared/Pointer and the wire follows |
| Variant | When |
|---|---|
| Truncated(at, want, got) | the buffer ended before the value did |
| TooLarge(at, limit, got) | a declared length or count crossed a limit |
| TooDeep(at, limit) | nesting crossed depth |
| BadType(at, got) / WantType(at, want, got) | the type ID is unknown, or not the one asked for |
| BadHeader(at, got) | the header's flag bits are not this format's |
| BadRef(at, got) | a back-reference named an id never written |
| BadEncoding(at, got) | a string's encoding tag is not one of the three |
| Cyclic(at, got) | a cycle that cannot be resolved |
| Trailing(at, got) | bytes remain after the one message |
| Unsupported(kind) | the type has no counterpart here |
python3 scripts/interop.py # writes src/fixtures_pyfory_test.mbt
(cd scripts && go run .) # writes src/fixtures_go_test.mbt
moon fmt # the generated files are formatted by the toolchainpython3 scripts/api.py # after changing an interface or a doc commentpub(all) suberror Refused {
Truncated(at~ : Int, want~ : Int, got~ : Int)
TooLarge(at~ : Int, limit~ : Int64, got~ : Int64)
TooDeep(at~ : Int, limit~ : Int)
BadType(at~ : Int, got~ : Int)
WantType(at~ : Int, want~ : Int, got~ : Int)
BadHeader(at~ : Int, got~ : Int)
BadRef(at~ : Int, got~ : Int)
BadEncoding(at~ : Int, got~ : Int)
Cyclic(at~ : Int, got~ : Int)
Trailing(at~ : Int, got~ : Int)
Unsupported(kind~ : String)
} derive(Eq, Debug)pub(all) struct Config {
xlang : Bool
track_ref : Bool
check_string : Bool
depth : Int
fields : Int
graph : Int
}fn FieldMeta::from(desc : Field, tag? : Int?, track_ref? : Bool, child? : StructMeta?, element? : TypeId?) -> FieldMeta raisefn FieldMeta::new(name : String, ty : TypeId, tag? : Int?, nullable? : Bool, track_ref? : Bool, child? : StructMeta?, element? : TypeId?) -> FieldMetapub(all) enum OnClash[T] {
Ignore
Panic
Handle((T, T) -> T)
}pub struct RowCursor {
view : BytesView
schema : RowSchema
bitmap_size : Int
fixed : Int
base : Int
}impl Show for StructMetafn StructMeta::of(model : Model, user~ : Int, tags? : Map[String, Int], refs? : Array[String], children? : Array[StructMeta]) -> StructMeta raisepub(all) enum TypeId {
Unknown
Bool
Int8
Int16
Int32
VarInt32
Int64
VarInt64
TaggedInt64
UInt8
UInt16
UInt32
VarUInt32
UInt64
VarUInt64
TaggedUInt64
Float8
Float16
BFloat16
Float32
Float64
String
List
Set
Map
Enum
NamedEnum
Struct
CompatibleStruct
NamedStruct
NamedCompatibleStruct
Ext
NamedExt
Union
TypedUnion
NamedUnion
NoneType
Duration
Timestamp
Date
Decimal
Binary
Array
BoolArray
Int8Array
Int16Array
Int32Array
Int64Array
UInt8Array
UInt16Array
UInt32Array
UInt64Array
Float8Array
Float16Array
BFloat16Array
Float32Array
Float64Array
} derive(Eq, Debug)pub(all) enum Value {
Null
Bool(Bool)
Int(Int64)
UInt(UInt64)
Float(Double)
Str(String)
Binary(Bytes)
List(TypeId, Array[Value])
Map(Array[(Value, Value)])
Struct(StructMeta, Array[(String, Value)])
Enum(Int, Int)
Timestamp(Int64, Int)
Date(Int64)
Duration(Int64, Int)
Decimal(Int, BigInt)
Shared(Int, Value)
Pointer(Int)
} derive(Eq, Debug)let root_key : Int