The Fory cross-language binary format for MoonBit.
let bytes : Bytes = @moonfory.dumps(42)
assert_eq(bytes, b"\x01\xff\x04\x2a\x00\x00\x00")
let back : Int = @moonfory.loads(bytes)
assert_eq(back, 42)moon add moonbitstack/moonforyimport {
"moonbitstack/moonfory",
}// One value, all of it.
let message : Bytes = @moonfory.dumps(42)
let value : Int = @moonfory.loads(message)
assert_eq(value, 42)
// The same, without allocating a fresh buffer.
let sink = Buffer()
@moonfory.dump(value, sink)
let source = @moonfory.Reader::new(sink.to_bytes())
source.read_head()
let replayed : Int = @moonfory.load(source)
assert_eq(replayed, 42)| Function | Takes | Gives |
|---|---|---|
| dumps(value, ...) | any Codec value | the whole message as Bytes |
| loads(bytes, schemas?, ...) | a message | the value, or a Refused |
| dump(value, buffer, ...) | a value and a buffer | writes into the buffer |
| load(reader) | a reader | one value |
| write_message(w, value) | a writer and a value | head, value, nothing else |
| read_message(r) | a reader | the one value the head announces |
| write_value(w, v) / read_value(r) | a Value | the dynamic form |
struct UserId(Int64)
pub impl @moonfory.Codec for UserId with fn ty(_self) -> @moonfory.TypeId {
@moonfory.TypeId::Int64
}
pub impl @moonfory.Codec for UserId with fn write_data(
self,
w : @moonfory.Writer,
) -> Unit raise {
w.write_i64(self.0)
}
pub impl @moonfory.Codec for UserId with fn read_data(
r : @moonfory.Reader,
_ty : @moonfory.TypeId,
) -> UserId raise {
UserId(r.read_i64())
}let user = UserId(7L)
let id_bytes = @moonfory.dumps(user)
assert_eq(id_bytes, b"\x01\xff\x06\x07\x00\x00\x00\x00\x00\x00\x00")
let same : UserId = @moonfory.loads(id_bytes)
assert_eq(same.0, 7L)| Fory type | Code | MoonBit | Written as |
|---|---|---|---|
| BOOL | 1 | Bool | one byte, 00 or 01 |
| INT8 / INT16 / INT32 | 2 / 3 / 4 | Byte / Int | two's complement, little-endian |
| INT64 | 6 | Int64 | eight bytes |
| VARINT32 / VARINT64 | 5 / 7 | any signed width | zig-zag, base-128 |
| UINT8 / UINT16 / UINT32 | 9 / 10 / 11 | UInt | unsigned, little-endian |
| UINT64 | 13 | UInt64 | eight bytes |
| TAGGED_INT64 / TAGGED_UINT64 | 8 / 15 | — | four bytes when small, else nine |
| FLOAT32 / FLOAT64 | 19 / 20 | Float / Double | IEEE 754, little-endian |
| STRING | 21 | String | length-and-encoding header, then UTF-8 |
| LIST / SET | 22 / 23 | Array[T] | count, element header, elements |
| MAP | 24 | Map[K, V] | total, then key/value chunks |
| STRUCT | 27 | StructMeta + fields | user id, schema fingerprint, then fields in Fory order |
| ENUM | 25 | ordinal | a varint |
| DURATION / TIMESTAMP / DATE | 37 / 38 / 39 | @moondate.Span / Moment / Date | see below |
| DECIMAL | 40 | (Int, BigInt) | scale, then the unscaled integer |
| BINARY | 41 | Bytes | length, then the bytes |
| BOOL_ARRAY … FLOAT64_ARRAY | 43 … 56 | Array[T] | raw little-endian, no header per element |
| Knob | Default | What it means |
|---|---|---|
| xlang | true | the header carries the cross-language flag |
| track_ref | false | values written twice are sent once and pointed at |
| check_string | true | string bytes are checked as UTF-8 before becoming a String |
| depth | 5 | how deeply values may nest before the message is refused |
| fields | 512 | how many fields a struct may declare |
| graph | 134_217_728 | how many variable-length bytes one message may declare |
// The preset, with one field changed.
let tracked = @moonfory.dumps(@moonfory.Value::Int(1L), track_ref=true)
// A whole config, with one field overridden on top of it.
let base = @moonfory.Config::new(depth=9)
let deeper = @moonfory.dumps(@moonfory.Value::Int(1L), config=base, depth=8)
// Two settings that disagree: refused, rather than silently picked.
let refused = try {
@moonfory.dumps(
@moonfory.Value::Int(1L),
config=base,
xlang=false,
clash=@moonfory.OnClash::Panic,
)
} catch {
@moonfory.Clashed(_) => b""
failure => raise failure
}
assert_eq(refused.length(), 0)let point = @moonfory.StructMeta::new("Point", 100, [
@moonfory.FieldMeta::new("x", @moonfory.TypeId::VarInt32),
@moonfory.FieldMeta::new("y", @moonfory.TypeId::VarInt32),
])
let person = @moonfory.StructMeta::new("Person", 101, [
@moonfory.FieldMeta::new("name", @moonfory.TypeId::String),
@moonfory.FieldMeta::new("age", @moonfory.TypeId::VarInt32),
// A list says what it holds, because the writer may leave the element type
// out of the header.
@moonfory.FieldMeta::new(
"tags", @moonfory.TypeId::List, element=Some(@moonfory.TypeId::String),
),
// A field whose type is declared sends no type ID, so the schema has to say
// which struct it is.
@moonfory.FieldMeta::new(
"home", @moonfory.TypeId::Struct, nullable=true, child=Some(point),
),
])let ada = @moonfory.Value::Struct(person, [
("name", @moonfory.Value::Str("ada")),
("age", @moonfory.Value::Int(36L)),
(
"tags",
@moonfory.Value::List(@moonfory.TypeId::List, [
@moonfory.Value::Str("math"),
]),
),
("home", @moonfory.Value::Struct(point, [
("x", @moonfory.Value::Int(1L)),
("y", @moonfory.Value::Int(2L)),
])),
])
let ada_bytes = @moonfory.dumps(ada)
let read_back : @moonfory.Value = @moonfory.loads(
ada_bytes, schemas=[person, point],
)
assert_eq(read_back, ada)
// And writing it again is the bytes it came from: the writer sorts, the reader
// sorts back.
assert_eq(@moonfory.dumps(read_back), ada_bytes)let account = @moonmodel.Model::new("Account", [
@moonmodel.Field::new("id", @moonmodel.Int),
@moonmodel.Field::new("nick", @moonmodel.Text),
])
let account_meta = @moonfory.StructMeta::of(account, user=102, refs=["nick"])
assert_eq(account_meta.name(), "Account")
assert_eq(account_meta.user(), 102)
assert_eq(account_meta.find("id").unwrap().ty, @moonfory.TypeId::VarInt32)let node = @moonfory.StructMeta::new("Node", 300, [
@moonfory.FieldMeta::new("val", @moonfory.TypeId::VarInt32),
@moonfory.FieldMeta::new("next", @moonfory.TypeId::Struct, track_ref=true),
])
// A node whose `next` is itself. Reference zero is the root, which this is.
let cyclic = @moonfory.Value::Struct(node, [
("val", @moonfory.Value::Int(1L)),
("next", @moonfory.Value::Pointer(0)),
])
let cyclic_bytes = @moonfory.dumps(cyclic, track_ref=true)
assert_eq(
cyclic_bytes, b"\x01\x00\x1b\xac\x02\xee\xcb\x23\x4f\x02\xfe\x00",
)let once = @moonfory.shared(7, "the same string")
let twice = [once, once] // two references, one body
let shared_bytes = @moonfory.dumps(twice, track_ref=true)
let shared_back : Array[@moonfory.Shared[String]] = @moonfory.loads(
shared_bytes, track_ref=true,
)
assert_eq(shared_back.length(), 2)
assert_eq(shared_back[1].value, "the same string")let schema = @moonfory.RowSchema::new([
{ name: "id", ty: @moonfory.RowType::Int64, nullable: false },
{ name: "name", ty: @moonfory.RowType::String, nullable: false },
{ name: "age", ty: @moonfory.RowType::Int32, nullable: true },
])
let row = @moonfory.Row::new(schema)
row.set(0, @moonfory.RowValue::Int64(7L))
row.set(1, @moonfory.RowValue::String("ada"))
row.set(2, @moonfory.RowValue::Null)
let row_bytes = row.finish()
let cur = @moonfory.RowCursor::new(row_bytes[:], schema)
assert_eq(cur.get_field(1), @moonfory.RowValue::String("ada"))
assert_eq(cur.is_null(2), true)let date = @moonfory.date_of_days(19_000L)
let moment = @moonfory.moment_of_parts(1_700_000_000L, 0)
let span = @moonfory.span_of_parts(3L, 500_000_000)
assert_eq(
@moonfory.days_of_date(@moonfory.loads(@moonfory.dumps(date))), 19_000L,
)
let (seconds, nanos) = @moonfory.parts_of_span(
@moonfory.loads(@moonfory.dumps(span)),
)
assert_eq(seconds, 3L)
assert_eq(nanos, 500_000_000)
let (epoch, _) = @moonfory.parts_of_moment(
@moonfory.loads(@moonfory.dumps(moment)),
)
assert_eq(epoch, 1_700_000_000L)| Thing | Why not |
|---|---|
| Named and compatible structs (26, 28, 29, 30) | They need a schema registry to reconcile two type definitions; that belongs with moonmodel, not inside a codec |
| Union, typed union, named union (33, 34, 35) | No MoonBit type maps to one, and which branch a peer meant cannot be recovered without the registry the named form expects |
| Ext (31, 32) | Another language's object graph, which needs a decoder this package does not have |
| FLOAT8 / FLOAT8_ARRAY (16, 52) | No peer in reach writes them; when one does, the table has the code |
| Reading a struct without its schema | The schema is the user id's meaning; loads(schemas=[...]) supplies it |
| inferring which value is shared | Identity is not in a MoonBit value; say Shared/Pointer and the wire follows |
| Variant | When |
|---|---|
| Truncated(at, want, got) | the buffer ended before the value did |
| TooLarge(at, limit, got) | a declared length or count crossed a limit |
| TooDeep(at, limit) | nesting crossed depth |
| BadType(at, got) / WantType(at, want, got) | the type ID is unknown, or not the one asked for |
| BadHeader(at, got) | the header's flag bits are not this format's |
| BadRef(at, got) | a back-reference named an id never written |
| BadEncoding(at, got) | a string's encoding tag is not one of the three |
| Cyclic(at, got) | a cycle that cannot be resolved |
| Trailing(at, got) | bytes remain after the one message |
| Unsupported(kind) | the type has no counterpart here |
python3 scripts/interop.py # writes src/fixtures_pyfory_test.mbt
(cd scripts && go run .) # writes src/fixtures_go_test.mbt
moon fmt # the generated files are formatted by the toolchainpython3 scripts/api.py # after changing an interface or a doc commentpub(all) suberror Refused {
Truncated(at~ : Int, want~ : Int, got~ : Int)
TooLarge(at~ : Int, limit~ : Int64, got~ : Int64)
TooDeep(at~ : Int, limit~ : Int)
BadType(at~ : Int, got~ : Int)
WantType(at~ : Int, want~ : Int, got~ : Int)
BadHeader(at~ : Int, got~ : Int)
BadRef(at~ : Int, got~ : Int)
BadEncoding(at~ : Int, got~ : Int)
Cyclic(at~ : Int, got~ : Int)
Trailing(at~ : Int, got~ : Int)
Unsupported(kind~ : String)
} derive(Eq, Debug)pub(all) struct Config {
xlang : Bool
track_ref : Bool
check_string : Bool
depth : Int
fields : Int
graph : Int
}fn FieldMeta::from(desc : Field, tag? : Int?, track_ref? : Bool, child? : StructMeta?, element? : TypeId?) -> FieldMeta raisefn FieldMeta::new(name : String, ty : TypeId, tag? : Int?, nullable? : Bool, track_ref? : Bool, child? : StructMeta?, element? : TypeId?) -> FieldMetapub(all) enum OnClash[T] {
Ignore
Panic
Handle((T, T) -> T)
}pub struct RowCursor {
view : BytesView
schema : RowSchema
bitmap_size : Int
fixed : Int
base : Int
}impl Show for StructMetafn StructMeta::of(model : Model, user~ : Int, tags? : Map[String, Int], refs? : Array[String], children? : Array[StructMeta]) -> StructMeta raisepub(all) enum TypeId {
Unknown
Bool
Int8
Int16
Int32
VarInt32
Int64
VarInt64
TaggedInt64
UInt8
UInt16
UInt32
VarUInt32
UInt64
VarUInt64
TaggedUInt64
Float8
Float16
BFloat16
Float32
Float64
String
List
Set
Map
Enum
NamedEnum
Struct
CompatibleStruct
NamedStruct
NamedCompatibleStruct
Ext
NamedExt
Union
TypedUnion
NamedUnion
NoneType
Duration
Timestamp
Date
Decimal
Binary
Array
BoolArray
Int8Array
Int16Array
Int32Array
Int64Array
UInt8Array
UInt16Array
UInt32Array
UInt64Array
Float8Array
Float16Array
BFloat16Array
Float32Array
Float64Array
} derive(Eq, Debug)pub(all) enum Value {
Null
Bool(Bool)
Int(Int64)
UInt(UInt64)
Float(Double)
Str(String)
Binary(Bytes)
List(TypeId, Array[Value])
Map(Array[(Value, Value)])
Struct(StructMeta, Array[(String, Value)])
Enum(Int, Int)
Timestamp(Int64, Int)
Date(Int64)
Duration(Int64, Int)
Decimal(Int, BigInt)
Shared(Int, Value)
Pointer(Int)
} derive(Eq, Debug)let root_key : IntInstall
Download zipThe Fory cross-language binary format for MoonBit.