cmd
→
README
T
Column
- ::column_typeColumn::column_type
- ::nameColumn::name
- ::repetitionColumn::repetition
T
ColumnType
T
ParquetColumnData
- ::lengthParquetColumnData::length
T
ParquetColumnarFile
- ::column_dataParquetColumnarFile::column_data
- ::column_valuesParquetColumnarFile::column_values
- ::columnsParquetColumnarFile::columns
- ::created_byParquetColumnarFile::created_by
- ::row_countParquetColumnarFile::row_count
T
ParquetFile
- ::column_dataParquetFile::column_data
- ::columnsParquetFile::columns
- ::created_byParquetFile::created_by
- ::row_countParquetFile::row_count
- ::rowsParquetFile::rows
T
Repetition
T
Value
V
new_column
V
new_parquet_columnar_file
V
new_parquet_file
V
read_bytes
V
read_bytes_columnar
V
read_file
V
read_file_columnar
V
write_bytes
V
write_file

parquet

Parquet reader/writer for MoonBit.

moonbit

parquet

moon add mizchi/parquet@0.2.1

Download zip

Author

mizchi

Version

0.2.1

License

Apache-2.0

Repository

github.com/mizchi/parquet

Last updated

3 months ago

Downloads

Dependencies

moonbitlang/x0.4.40
f4ah6o/duckdb0.6.0

README

#mizchi/parquet

Parquet reader/writer for MoonBit.

#Status

Reader passes the vendored apache-parquet-testing fixtures used in this repo:
- delta_binary_packed
- delta_byte_array
- delta_encoding_optional_column
- delta_encoding_required_column
- int32_with_null_pages
- fixed_length_byte_array
- datapage_v2_empty_datapage.snappy.parquet
- int96_from_spark
- alltypes_plain
- alltypes_dictionary
Writer currently supports flat schemas with Int32, Int64, String, and Binary, with Required / Optional repetition.
DuckDB interoperability is checked both ways:
- DuckDB-written parquet can be read by this implementation.
- MoonBit-written parquet can be read by DuckDB.

#Benchmark

Measured on 2026-03-09 on Apple M5, macOS 26.2, Darwin arm64.

Reproduce:

just bench-compare-all
just bench-compare-rust

Comparison table from just bench-compare-all:

benchmark	moon(js)	moon(wasm-gc)	moon(native)	rust
read delta binary packed benchmark	512.22 us	127.01 us	645.99 us	203.85 us
read int32 null pages benchmark	17.34 us	10.85 us	43.60 us	7.61 us
read fixed length byte array benchmark	87.17 us	10.41 us	54.68 us	unsupported
read alltypes plain benchmark	20.49 us	12.02 us	45.42 us	19.57 us
read alltypes dictionary benchmark	16.83 us	11.04 us	42.53 us	17.31 us
read int96 from spark benchmark	10.90 us	3.52 us	16.19 us	3.28 us
read empty snappy datapage v2 benchmark	3.46 us	1.72 us	6.27 us	2.58 us

Notes:

Rust is the local baseline in tools/rust-bench, built on top of Apache's parquet crate.
fixed_length_byte_array is marked unsupported because the current Rust benchmark tool does not read that fixture.
On this machine, wasm-gc is the fastest MoonBit target for the read-side microbenchmarks above.

#Development

just                  # check + test
just target=js bench
just target=wasm-gc bench
just target=native bench
just bench-compare-all
just e2e-duckdb
moon info

#License

Apache-2.0

#
ParquetError

source

pub(all) suberror ParquetError {
  InvalidData(String)
  Unsupported(String)
  Io(String)
}

impl Eq for ParquetError

impl Show for ParquetError

#
Column

source

pub struct Column {
  name : String
  column_type : ColumnType
  repetition : Repetition
}

impl Eq for Column

impl Show for Column

#
Column::column_type

source

fn Column::column_type(self : Column) -> ColumnType

#
Column::name

source

fn Column::name(self : Column) -> String

#
Column::repetition

source

fn Column::repetition(self : Column) -> Repetition

#
ColumnType

source

pub(all) enum ColumnType {
  Boolean
  Int32
  Int64
  TimestampMicros
  Float
  Double
  String
  Binary
}

impl Eq for ColumnType

impl Show for ColumnType

#
ParquetColumnData

source

pub(all) enum ParquetColumnData {
  Values(Array[Value])
  BooleanValues(Array[Bool])
  Int32Values(Array[Int])
  NullableInt32Values(Array[Int?])
  Int64Values(Array[Int64])
  NullableInt64Values(Array[Int64?])
  TimestampMicrosValues(Array[Int64])
  NullableTimestampMicrosValues(Array[Int64?])
  FloatValues(Array[Float])
  DoubleValues(Array[Double])
  Utf8Values(Array[Bytes])
  NullableUtf8Values(Array[Bytes?])
  BinaryValues(Array[Bytes])
  NullableBinaryValues(Array[Bytes?])
}

impl Eq for ParquetColumnData

impl Show for ParquetColumnData

#
ParquetColumnData::length

source

fn ParquetColumnData::length(self : ParquetColumnData) -> Int

#
ParquetColumnarFile

source

pub struct ParquetColumnarFile {
  columns : Array[Column]
  column_data : Array[ParquetColumnData]
  row_count : Int
  column_values_cache : Array[Array[Value]]?
  created_by : String?
}

impl Eq for ParquetColumnarFile

impl Show for ParquetColumnarFile

#
ParquetColumnarFile::column_data

source

fn ParquetColumnarFile::column_data(self : ParquetColumnarFile) -> Array[ParquetColumnData]

#
ParquetColumnarFile::column_values

source

fn ParquetColumnarFile::column_values(self : ParquetColumnarFile) -> Array[Array[Value]]

#
ParquetColumnarFile::columns

source

fn ParquetColumnarFile::columns(self : ParquetColumnarFile) -> Array[Column]

#
ParquetColumnarFile::created_by

source

fn ParquetColumnarFile::created_by(self : ParquetColumnarFile) -> String?

#
ParquetColumnarFile::row_count

source

fn ParquetColumnarFile::row_count(self : ParquetColumnarFile) -> Int

#
ParquetFile

source

pub struct ParquetFile {
  columns : Array[Column]
  row_count : Int
  column_data : Array[ParquetColumnData]
  rows_cache : Array[Array[Value]]?
  created_by : String?
}

impl Eq for ParquetFile

impl Show for ParquetFile

#
ParquetFile::column_data

source

fn ParquetFile::column_data(self : ParquetFile) -> Array[ParquetColumnData]

#
ParquetFile::columns

source

fn ParquetFile::columns(self : ParquetFile) -> Array[Column]

#
ParquetFile::created_by

source

fn ParquetFile::created_by(self : ParquetFile) -> String?

#
ParquetFile::row_count

source

fn ParquetFile::row_count(self : ParquetFile) -> Int

#
ParquetFile::rows

source

fn ParquetFile::rows(self : ParquetFile) -> Array[Array[Value]]

#
Repetition

source

pub(all) enum Repetition {
  Required
  Optional
  Repeated
}

impl Eq for Repetition

impl Show for Repetition

#
Value

source

pub(all) enum Value {
  Boolean(Bool)
  Int32(Int)
  Int64(Int64)
  TimestampMicros(Int64)
  Float(Float)
  Double(Double)
  String(String)
  Binary(Bytes)
  Null
}

impl Eq for Value

impl Show for Value

#
new_column

source

fn new_column(name : String, column_type : ColumnType, repetition : Repetition) -> Column

#
new_parquet_columnar_file

source

fn new_parquet_columnar_file(columns : Array[Column], column_data : Array[ParquetColumnData], row_count : Int, created_by : String?) -> ParquetColumnarFile

#
new_parquet_file

source

fn new_parquet_file(columns : Array[Column], rows : Array[Array[Value]], created_by : String?) -> ParquetFile

#
read_bytes

source

fn read_bytes(data : Bytes) -> ParquetFile raise ParquetError

Read a parquet document from bytes.

#
read_bytes_columnar

source

fn read_bytes_columnar(data : Bytes) -> ParquetColumnarFile raise ParquetError

Read a parquet document from bytes without row materialization.

#
read_file

source

fn read_file(path : String) -> ParquetFile raise ParquetError

Read a parquet file from disk.

#
read_file_columnar

source

fn read_file_columnar(path : String) -> ParquetColumnarFile raise ParquetError

Read a parquet file from disk without row materialization.

#
write_bytes

source

fn write_bytes(file : ParquetFile) -> Bytes raise ParquetError

Encode a parquet document to bytes.

The current writer supports flat schemas of Int32, Int64, String, and Binary, with Required or Optional repetition.

#
write_file

source

fn write_file(path : String, file : ParquetFile) -> Unit raise ParquetError

Encode a parquet document and write it to disk.

Source Files

Site source Report issue Packages Build queue Skills Statistics

parquet

#mizchi/parquet

#Status

#Benchmark

#Development

#License

#ParquetError

#Column

#Column::column_type

#Column::name

#Column::repetition

#ColumnType

#ParquetColumnData

#ParquetColumnData::length

#ParquetColumnarFile

#ParquetColumnarFile::column_data

#ParquetColumnarFile::column_values

#ParquetColumnarFile::columns

#ParquetColumnarFile::created_by

#ParquetColumnarFile::row_count

#ParquetFile

#ParquetFile::column_data

#ParquetFile::columns

#ParquetFile::created_by

#ParquetFile::row_count

#ParquetFile::rows

#Repetition

#Value

#new_column

#new_parquet_columnar_file

#new_parquet_file

#read_bytes

#read_bytes_columnar

#read_file

#read_file_columnar

#write_bytes

#write_file

Source Files

#
ParquetError

#
Column

#
Column::column_type

#
Column::name

#
Column::repetition

#
ColumnType

#
ParquetColumnData

#
ParquetColumnData::length

#
ParquetColumnarFile

#
ParquetColumnarFile::column_data

#
ParquetColumnarFile::column_values

#
ParquetColumnarFile::columns

#
ParquetColumnarFile::created_by

#
ParquetColumnarFile::row_count

#
ParquetFile

#
ParquetFile::column_data

#
ParquetFile::columns

#
ParquetFile::created_by

#
ParquetFile::row_count

#
ParquetFile::rows

#
Repetition

#
Value

#
new_column

#
new_parquet_columnar_file

#
new_parquet_file

#
read_bytes

#
read_bytes_columnar

#
read_file

#
read_file_columnar

#
write_bytes

#
write_file