README
- ::equalParquetError::equal
- ::not_equalParquetError::not_equal
- ::outputParquetError::output
- ::to_reprParquetError::to_repr
- ::to_stringParquetError::to_string
- ::column_typeColumn::column_type
- ::equalColumn::equal
- ::nameColumn::name
- ::not_equalColumn::not_equal
- ::outputColumn::output
- ::repetitionColumn::repetition
- ::to_reprColumn::to_repr
- ::to_stringColumn::to_string
- ::equalColumnType::equal
- ::not_equalColumnType::not_equal
- ::outputColumnType::output
- ::to_reprColumnType::to_repr
- ::to_stringColumnType::to_string
- ::equalParquetColumnData::equal
- ::lengthParquetColumnData::length
- ::not_equalParquetColumnData::not_equal
- ::outputParquetColumnData::output
- ::to_reprParquetColumnData::to_repr
- ::to_stringParquetColumnData::to_string
- ::column_dataParquetColumnarFile::column_data
- ::column_valuesParquetColumnarFile::column_values
- ::columnsParquetColumnarFile::columns
- ::created_byParquetColumnarFile::created_by
- ::equalParquetColumnarFile::equal
- ::not_equalParquetColumnarFile::not_equal
- ::outputParquetColumnarFile::output
- ::row_countParquetColumnarFile::row_count
- ::to_reprParquetColumnarFile::to_repr
- ::to_stringParquetColumnarFile::to_string
- ::column_dataParquetFile::column_data
- ::columnsParquetFile::columns
- ::created_byParquetFile::created_by
- ::equalParquetFile::equal
- ::not_equalParquetFile::not_equal
- ::outputParquetFile::output
- ::row_countParquetFile::row_count
- ::rowsParquetFile::rows
- ::to_reprParquetFile::to_repr
- ::to_stringParquetFile::to_string
- ::equalRepetition::equal
- ::not_equalRepetition::not_equal
- ::outputRepetition::output
- ::to_reprRepetition::to_repr
- ::to_stringRepetition::to_string
- ::equalValue::equal
- ::not_equalValue::not_equal
- ::outputValue::output
- ::to_reprValue::to_repr
- ::to_stringValue::to_string
V
new_column
V
new_parquet_columnar_file
V
new_parquet_file
V
read_bytes
V
read_bytes_columnar
V
read_file
V
read_file_columnar
V
write_bytes
V
write_file

parquet

Parquet reader/writer for MoonBit.

moonbit

parquet

Download zip

Author

mizchi

Version

0.2.2

License

Apache-2.0

Repository

github.com/mizchi/parquet

Last updated

6 days ago

Downloads

Dependencies

moonbitlang/x0.5.5
mizchi/duckdb0.6.4

#mizchi/parquet

Parquet reader/writer for MoonBit.

#Status

Reader passes the vendored apache-parquet-testing fixtures used in this repo:
- delta_binary_packed
- delta_byte_array
- delta_encoding_optional_column
- delta_encoding_required_column
- int32_with_null_pages
- fixed_length_byte_array
- datapage_v2_empty_datapage.snappy.parquet
- int96_from_spark
- alltypes_plain
- alltypes_dictionary
Writer currently supports flat schemas with Int32, Int64, String, and Binary, with Required / Optional repetition.
DuckDB interoperability is checked both ways:
- DuckDB-written parquet can be read by this implementation.
- MoonBit-written parquet can be read by DuckDB.

#Benchmark

Measured on 2026-03-09 on Apple M5, macOS 26.2, Darwin arm64.

Reproduce:

just bench-compare-all
just bench-compare-rust

Comparison table from just bench-compare-all:

benchmark	moon(js)	moon(wasm-gc)	moon(native)	rust
read delta binary packed benchmark	512.22 us	127.01 us	645.99 us	203.85 us
read int32 null pages benchmark	17.34 us	10.85 us	43.60 us	7.61 us
read fixed length byte array benchmark	87.17 us	10.41 us	54.68 us	unsupported
read alltypes plain benchmark	20.49 us	12.02 us	45.42 us	19.57 us
read alltypes dictionary benchmark	16.83 us	11.04 us	42.53 us	17.31 us
read int96 from spark benchmark	10.90 us	3.52 us	16.19 us	3.28 us
read empty snappy datapage v2 benchmark	3.46 us	1.72 us	6.27 us	2.58 us

Notes:

Rust is the local baseline in tools/rust-bench, built on top of Apache's parquet crate.
fixed_length_byte_array is marked unsupported because the current Rust benchmark tool does not read that fixture.
On this machine, wasm-gc is the fastest MoonBit target for the read-side microbenchmarks above.

#Development

just                  # check + test
just target=js bench
just target=wasm-gc bench
just target=native bench
just bench-compare-all
just e2e-duckdb
moon info

#Browser Playground

The repository includes a browser playground that generates parquet on DuckDB WASM and lets you try SQL against it immediately.

just playground-install
just playground-dev

Production build:

just playground-build
just playground-build-pages

In the UI, you can edit schema JSON / rows JSON / SQL, generate parquet bytes with mizchi/parquet, and execute read_parquet('playground.parquet') on the DuckDB WASM backend (mizchi/duckdb, a fork of f4ah6o/duckdb that builds on current MoonBit).

GitHub Pages:

https://mizchi.github.io/parquet/

#License

Apache-2.0

ParquetError

source

pub(all) suberror ParquetError {
  InvalidData(String)
  Unsupported(String)
  Io(String)
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for ParquetError

ParquetError::equal

source

fn ParquetError::equal(ParquetError, ParquetError) -> Bool

ParquetError::not_equal

source

fn ParquetError::not_equal(x : ParquetError, y : ParquetError) -> Bool

ParquetError::output

source

fn ParquetError::output(self : ParquetError, logger : &Logger) -> Unit

ParquetError::to_repr

source

fn ParquetError::to_repr(ParquetError) -> @moonbitlang/core/debug.Repr
Repr

ParquetError::to_string

source

fn ParquetError::to_string(self : ParquetError) -> String

Column

source

pub struct Column {
  name : String
  column_type : ColumnType
  repetition : Repetition
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for Column

Column::column_type

source

fn Column::column_type(self : Column) -> ColumnType

Column::equal

source

fn Column::equal(Column, Column) -> Bool

Column::name

source

fn Column::name(self : Column) -> String

Column::not_equal

source

fn Column::not_equal(x : Column, y : Column) -> Bool

Column::output

source

fn Column::output(self : Column, logger : &Logger) -> Unit

Column::repetition

source

fn Column::repetition(self : Column) -> Repetition

Column::to_repr

source

fn Column::to_repr(Column) -> @moonbitlang/core/debug.Repr
Repr

Column::to_string

source

fn Column::to_string(self : Column) -> String

ColumnType

source

pub(all) enum ColumnType {
  Boolean
  Int32
  Int64
  TimestampMicros
  Float
  Double
  String
  Binary
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for ColumnType

ColumnType::equal

source

fn ColumnType::equal(ColumnType, ColumnType) -> Bool

ColumnType::not_equal

source

fn ColumnType::not_equal(x : ColumnType, y : ColumnType) -> Bool

ColumnType::output

source

fn ColumnType::output(self : ColumnType, logger : &Logger) -> Unit

ColumnType::to_repr

source

fn ColumnType::to_repr(ColumnType) -> @moonbitlang/core/debug.Repr
Repr

ColumnType::to_string

source

fn ColumnType::to_string(self : ColumnType) -> String

ParquetColumnData

source

pub(all) enum ParquetColumnData {
  Values(Array[Value])
  BooleanValues(Array[Bool])
  Int32Values(Array[Int])
  NullableInt32Values(Array[Int?])
  Int64Values(Array[Int64])
  NullableInt64Values(Array[Int64?])
  TimestampMicrosValues(Array[Int64])
  NullableTimestampMicrosValues(Array[Int64?])
  FloatValues(Array[Float])
  DoubleValues(Array[Double])
  Utf8Values(Array[Bytes])
  NullableUtf8Values(Array[Bytes?])
  BinaryValues(Array[Bytes])
  NullableBinaryValues(Array[Bytes?])
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for ParquetColumnData

ParquetColumnData::equal

source

fn ParquetColumnData::equal(ParquetColumnData, ParquetColumnData) -> Bool

ParquetColumnData::length

source

fn ParquetColumnData::length(self : ParquetColumnData) -> Int

ParquetColumnData::not_equal

source

fn ParquetColumnData::not_equal(x : ParquetColumnData, y : ParquetColumnData) -> Bool

ParquetColumnData::output

source

fn ParquetColumnData::output(self : ParquetColumnData, logger : &Logger) -> Unit

ParquetColumnData::to_repr

source

fn ParquetColumnData::to_repr(ParquetColumnData) -> @moonbitlang/core/debug.Repr
Repr

ParquetColumnData::to_string

source

fn ParquetColumnData::to_string(self : ParquetColumnData) -> String

ParquetColumnarFile

source

pub struct ParquetColumnarFile {
  columns : Array[Column]
  column_data : Array[ParquetColumnData]
  row_count : Int
  column_values_cache : Array[Array[Value]]?
  created_by : String?
} derive(@moonbitlang/core/debug.Debug
Debug)

impl Eq for ParquetColumnarFile

impl Show for ParquetColumnarFile

ParquetColumnarFile::column_data

source

fn ParquetColumnarFile::column_data(self : ParquetColumnarFile) -> Array[ParquetColumnData]

ParquetColumnarFile::column_values

source

fn ParquetColumnarFile::column_values(self : ParquetColumnarFile) -> Array[Array[Value]]

ParquetColumnarFile::columns

source

fn ParquetColumnarFile::columns(self : ParquetColumnarFile) -> Array[Column]

ParquetColumnarFile::created_by

source

fn ParquetColumnarFile::created_by(self : ParquetColumnarFile) -> String?

ParquetColumnarFile::equal

source

fn ParquetColumnarFile::equal(self : ParquetColumnarFile, other : ParquetColumnarFile) -> Bool

ParquetColumnarFile::not_equal

source

fn ParquetColumnarFile::not_equal(x : ParquetColumnarFile, y : ParquetColumnarFile) -> Bool

ParquetColumnarFile::output

source

fn ParquetColumnarFile::output(self : ParquetColumnarFile, logger : &Logger) -> Unit

ParquetColumnarFile::row_count

source

fn ParquetColumnarFile::row_count(self : ParquetColumnarFile) -> Int

ParquetColumnarFile::to_repr

source

fn ParquetColumnarFile::to_repr(ParquetColumnarFile) -> @moonbitlang/core/debug.Repr
Repr

ParquetColumnarFile::to_string

source

fn ParquetColumnarFile::to_string(self : ParquetColumnarFile) -> String

ParquetFile

source

pub struct ParquetFile {
  columns : Array[Column]
  row_count : Int
  column_data : Array[ParquetColumnData]
  rows_cache : Array[Array[Value]]?
  created_by : String?
} derive(@moonbitlang/core/debug.Debug
Debug)

impl Eq for ParquetFile

impl Show for ParquetFile

ParquetFile::column_data

source

fn ParquetFile::column_data(self : ParquetFile) -> Array[ParquetColumnData]

ParquetFile::columns

source

fn ParquetFile::columns(self : ParquetFile) -> Array[Column]

ParquetFile::created_by

source

fn ParquetFile::created_by(self : ParquetFile) -> String?

ParquetFile::equal

source

fn ParquetFile::equal(self : ParquetFile, other : ParquetFile) -> Bool

ParquetFile::not_equal

source

fn ParquetFile::not_equal(x : ParquetFile, y : ParquetFile) -> Bool

ParquetFile::output

source

fn ParquetFile::output(self : ParquetFile, logger : &Logger) -> Unit

ParquetFile::row_count

source

fn ParquetFile::row_count(self : ParquetFile) -> Int

ParquetFile::rows

source

fn ParquetFile::rows(self : ParquetFile) -> Array[Array[Value]]

ParquetFile::to_repr

source

fn ParquetFile::to_repr(ParquetFile) -> @moonbitlang/core/debug.Repr
Repr

ParquetFile::to_string

source

fn ParquetFile::to_string(self : ParquetFile) -> String

Repetition

source

pub(all) enum Repetition {
  Required
  Optional
  Repeated
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for Repetition

Repetition::equal

source

fn Repetition::equal(Repetition, Repetition) -> Bool

Repetition::not_equal

source

fn Repetition::not_equal(x : Repetition, y : Repetition) -> Bool

Repetition::output

source

fn Repetition::output(self : Repetition, logger : &Logger) -> Unit

Repetition::to_repr

source

fn Repetition::to_repr(Repetition) -> @moonbitlang/core/debug.Repr
Repr

Repetition::to_string

source

fn Repetition::to_string(self : Repetition) -> String

Value

source

pub(all) enum Value {
  Boolean(Bool)
  Int32(Int)
  Int64(Int64)
  TimestampMicros(Int64)
  Float(Float)
  Double(Double)
  String(String)
  Binary(Bytes)
  Null
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

impl Show for Value

Value::equal

source

fn Value::equal(Value, Value) -> Bool

Value::not_equal

source

fn Value::not_equal(x : Value, y : Value) -> Bool

Value::output

source

fn Value::output(self : Value, logger : &Logger) -> Unit

Value::to_repr

source

fn Value::to_repr(Value) -> @moonbitlang/core/debug.Repr
Repr

Value::to_string

source

fn Value::to_string(self : Value) -> String

new_column

source

fn new_column(name : String, column_type : ColumnType, repetition : Repetition) -> Column

new_parquet_columnar_file

source

fn new_parquet_columnar_file(columns : Array[Column], column_data : Array[ParquetColumnData], row_count : Int, created_by : String?) -> ParquetColumnarFile

new_parquet_file

source

fn new_parquet_file(columns : Array[Column], rows : Array[Array[Value]], created_by : String?) -> ParquetFile

read_bytes

source

fn read_bytes(data : Bytes) -> ParquetFile raise ParquetError

Read a parquet document from bytes.

read_bytes_columnar

source

fn read_bytes_columnar(data : Bytes) -> ParquetColumnarFile raise ParquetError

Read a parquet document from bytes without row materialization.

read_file

source

fn read_file(path : String) -> ParquetFile raise ParquetError

Read a parquet file from disk.

read_file_columnar

source

fn read_file_columnar(path : String) -> ParquetColumnarFile raise ParquetError

Read a parquet file from disk without row materialization.

write_bytes

source

fn write_bytes(file : ParquetFile) -> Bytes raise ParquetError

Encode a parquet document to bytes.

The current writer supports flat schemas of Int32, Int64, String, and Binary, with Required or Optional repetition.

write_file

source

fn write_file(path : String, file : ParquetFile) -> Unit raise ParquetError

Encode a parquet document and write it to disk.

Source Files

Site source Report issue Packages Build queue Skills Statistics

parquet

#mizchi/parquet

#Status

#Benchmark

#Development

#Browser Playground

#License

#ParquetError

#ParquetError::equal

#ParquetError::not_equal

#ParquetError::output

#ParquetError::to_repr

#ParquetError::to_string

#Column

#Column::column_type

#Column::equal

#Column::name

#Column::not_equal

#Column::output

#Column::repetition

#Column::to_repr

#Column::to_string

#ColumnType

#ColumnType::equal

#ColumnType::not_equal

#ColumnType::output

#ColumnType::to_repr

#ColumnType::to_string

#ParquetColumnData

#ParquetColumnData::equal

#ParquetColumnData::length

#ParquetColumnData::not_equal

#ParquetColumnData::output

#ParquetColumnData::to_repr

#ParquetColumnData::to_string

#ParquetColumnarFile

#ParquetColumnarFile::column_data

#ParquetColumnarFile::column_values

#ParquetColumnarFile::columns

#ParquetColumnarFile::created_by

#ParquetColumnarFile::equal

#ParquetColumnarFile::not_equal

#ParquetColumnarFile::output

#ParquetColumnarFile::row_count

#ParquetColumnarFile::to_repr

#ParquetColumnarFile::to_string

#ParquetFile

#ParquetFile::column_data

#ParquetFile::columns

#ParquetFile::created_by

#ParquetFile::equal

#ParquetFile::not_equal

#ParquetFile::output

#ParquetFile::row_count

#ParquetFile::rows

#ParquetFile::to_repr

#ParquetFile::to_string

#Repetition

#Repetition::equal

#Repetition::not_equal

#Repetition::output

#Repetition::to_repr

#Repetition::to_string

#Value

#Value::equal

#Value::not_equal

#Value::output

#Value::to_repr

#Value::to_string

#new_column

#new_parquet_columnar_file

#new_parquet_file

#read_bytes

#read_bytes_columnar

#read_file

#read_file_columnar

#write_bytes

#write_file

Source Files

ParquetError

ParquetError::equal

ParquetError::not_equal

ParquetError::output

ParquetError::to_repr

ParquetError::to_string

Column

Column::column_type

Column::equal

Column::name

Column::not_equal

Column::output

Column::repetition

Column::to_repr

Column::to_string

ColumnType

ColumnType::equal

ColumnType::not_equal

ColumnType::output

ColumnType::to_repr

ColumnType::to_string

ParquetColumnData

ParquetColumnData::equal

ParquetColumnData::length

ParquetColumnData::not_equal

ParquetColumnData::output

ParquetColumnData::to_repr

ParquetColumnData::to_string

ParquetColumnarFile

ParquetColumnarFile::column_data

ParquetColumnarFile::column_values

ParquetColumnarFile::columns

ParquetColumnarFile::created_by

ParquetColumnarFile::equal

ParquetColumnarFile::not_equal

ParquetColumnarFile::output

ParquetColumnarFile::row_count

ParquetColumnarFile::to_repr

ParquetColumnarFile::to_string

ParquetFile

ParquetFile::column_data

ParquetFile::columns

ParquetFile::created_by

ParquetFile::equal

ParquetFile::not_equal

ParquetFile::output

ParquetFile::row_count

ParquetFile::rows

ParquetFile::to_repr

ParquetFile::to_string

Repetition

Repetition::equal

Repetition::not_equal

Repetition::output

Repetition::to_repr

Repetition::to_string

Value

Value::equal

Value::not_equal

Value::output

Value::to_repr

Value::to_string

new_column

new_parquet_columnar_file

new_parquet_file

read_bytes

read_bytes_columnar

read_file

read_file_columnar

write_bytes

write_file