#Moonbit/Core Encoding

#Overview

The @encoding package provides an implementation for encoding and decoding strings using various character encodings (e.g., UTF-8).

It supports both streaming and non-streaming (bulk) operations, making it flexible for different use-cases.

#Supported Encoding

UTF8
UTF16 // alias for UTF16LE
UTF16LE
UTF16BE

#Usage

#Decoding

Decode a UTF-8 byte stream:

///|
test {
  // Initialize a streaming UTF-8 decoder
  let decoder = @encoding.decoder(UTF8)

  // Consume byte chunks
  let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
  inspect(decoder.consume(inputs[0]), content="abc")
  inspect(decoder.consume(inputs[1]), content="")
  inspect(decoder.consume(inputs[2]), content="🐰")

  // Finish decoding
  assert_true(decoder.finish().is_empty())
}

#Encoding

Encode a string to UTF-8 bytes:

///|
test {
  // Encode a string to UTF-8
  let src = "你好👀"
  let bytes = @encoding.encode(UTF8, src)
  inspect(
    bytes,
    content=(
      #|b"\xe4\xbd\xa0\xe5\xa5\xbd\xf0\x9f\x91\x80"
    ),
  )
}

Encode a single character to UTF-8 bytes:

///|
test {
  inspect(
    @encoding.to_utf8_bytes('A'),
    content=(
      #|b"A"
    ),
  )
}

MalformedError

source

pub suberror MalformedError {
  MalformedError(Bytes)
} derive(@moonbitlang/core/debug.Debug
Debug)

MalformedError::to_repr

source

#deprecated("`MalformedError::to_repr` is deprecated, use `@moonbitlang/core/debug.Debug::to_repr` instead.")
fn MalformedError::to_repr(MalformedError) -> @moonbitlang/core/debug.Repr
Repr

TruncatedError

source

pub suberror TruncatedError {
  TruncatedError(Bytes)
} derive(@moonbitlang/core/debug.Debug
Debug)

TruncatedError::to_repr

source

#deprecated("`TruncatedError::to_repr` is deprecated, use `@moonbitlang/core/debug.Debug::to_repr` instead.")
fn TruncatedError::to_repr(TruncatedError) -> @moonbitlang/core/debug.Repr
Repr

Decoder

source

type Decoder

Decoder::consume

source

fn Decoder::consume(self : Decoder, input : BytesView) -> String raise

Consume the given byte sequence using the specified Decoder and return the resulting string incrementally.

This function calls decode! with the stream parameter set to true, indicating that more bytes will follow for decoding.

Parameters

self: The Decoder instance used to consume the byte sequence.
input: The byte sequence to be consumed and decoded incrementally.

Returns

A String representing the partially decoded content from the input byte sequence, as more bytes are expected.

Errors

MalformedError: when the byte sequence is not properly formatted according to the specified encoding. TruncatedError: when the byte sequence ends prematurely, implying that more data is expected for complete decoding.

Decoder::decode

source

fn Decoder::decode(self : Decoder, input : BytesView, stream? : Bool) -> String raise

Decode the given byte sequence using the specified Decoder and return the resulting string.

This function can work in streaming mode where bytes are consumed incrementally. When stream is false, it indicates the end of the input and triggers the final decoding step.

Parameters

self: The Decoder instance used to decode the byte sequence.
input: The byte sequence to be decoded.
stream~: A boolean indicating whether more bytes will be supplied for decoding. It defaults to false.

Returns

A String representing the decoded content from the input byte sequence.

Errors

Examples

test {
  let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
  let decoder = @encoding.decoder(UTF8)
  inspect(decoder.decode(inputs[0], stream=true), content="abc")
  inspect(decoder.decode(inputs[1], stream=true), content="")
  inspect(decoder.decode(inputs[2], stream=false), content="🐰")
}

Decoder::decode_lossy

source

fn Decoder::decode_lossy(self : Decoder, input : BytesView, stream? : Bool) -> String

Decode the given byte sequence using the specified Decoder and return the resulting string, replacing any invalid sequences with the Unicode Replacement Character (U+FFFD).

This function can work in streaming mode where bytes are consumed incrementally. When stream is false, it indicates the end of the input and triggers the final decoding step.

Parameters

self: The Decoder instance used to decode the byte sequence.
input: The byte sequence to be decoded.
stream~: A boolean indicating whether more bytes will be supplied for decoding. It defaults to false.

Returns

A String representing the decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (U+FFFD).

Decoder::decode_lossy_to

source

fn Decoder::decode_lossy_to(self : Decoder, input : BytesView, output : StringBuilder, stream? : Bool) -> Unit

Decoder::decode_to

source

fn Decoder::decode_to(self : Decoder, input : BytesView, output : StringBuilder, stream? : Bool) -> Unit raise

Decodes the given byte sequence using the specified decoder and writes the result directly to a StringBuilder. Similar to decode!, but writes the result to an existing StringBuilder instead of creating a new String.

Parameters:

decoder : The decoder instance used to decode the byte sequence.
input : The byte sequence to be decoded.
output : The StringBuilder where the decoded content will be written to.

Throws a MalformedError when the byte sequence is not properly formatted according to the specified encoding.

Example:

test {
  let decoder = @encoding.decoder(UTF8)
  let buf = StringBuilder()
  decoder.decode_to(b"Hello", buf)
  inspect(buf.to_string(), content="Hello")
}

Decoder::finish

source

fn Decoder::finish(self : Decoder) -> String raise

Finalize the decoding process and return the remaining decoded string.

This function calls decode! with the stream parameter set to false, indicating that no more bytes will be supplied and triggering the final decoding step to produce the remaining output.

Parameters

self: The Decoder instance used to finalize the decoding process.

Returns

A String representing the final part of the decoded content, after all byte sequences have been processed.

Errors

MalformedError: This error is raised if the remaining byte sequence is not properly formatted according to the specified encoding. TruncatedError: This error is raised if the remaining byte sequence ends prematurely, implying that more data was expected for complete decoding.

Decoder::lossy_consume

source

fn Decoder::lossy_consume(self : Decoder, input : BytesView) -> String

Consume the given byte sequence using the specified Decoder and return the resulting string incrementally, replacing any invalid sequences with the Unicode Replacement Character (U+FFFD).

This function calls decode_lossy with the stream parameter set to true, indicating that more bytes will follow for decoding.

Parameters

self: The Decoder instance used to consume and decode the byte sequence.
input: The byte sequence to be consumed and decoded incrementally.

Returns

A String representing the partially decoded content from the input byte sequence, with any invalid sequences replaced by the Unicode Replacement Character (U+FFFD), as more bytes are expected.

Decoder::lossy_finish

source

fn Decoder::lossy_finish(self : Decoder) -> String

Finalize the lossy decoding process and return the remaining decoded string, replacing any invalid sequences with the Unicode Replacement Character (U+FFFD).

This function calls decode_lossy with the stream parameter set to false, indicating that no more bytes will be supplied and triggering the final decoding step to produce the remaining output.

Parameters

self: The Decoder instance used to finalize the lossy decoding process.

Returns

A String representing the final part of the decoded content, with any invalid sequences replaced by the Unicode Replacement Character (U+FFFD), after all byte sequences have been processed.

Encoding

source

pub(all) enum Encoding {
  UTF8
  UTF16
  UTF16LE
  UTF16BE
} derive(@moonbitlang/core/debug.Debug
Debug)

Encoding::to_repr

source

#deprecated("`Encoding::to_repr` is deprecated, use `@moonbitlang/core/debug.Debug::to_repr` instead.")
fn Encoding::to_repr(Encoding) -> @moonbitlang/core/debug.Repr
Repr

U_REP

source

let U_REP : Char

The Unicode Replacement Character, which is used to replace invalid or unrecognized sequences during lossy decoding. https://unicode.org/charts/nameslist/n_FFF0.html

decode_to

source

fn decode_to(input : BytesView, output : StringBuilder, encoding~ : Encoding) -> Unit raise

decoder

source

fn decoder(encoding : Encoding) -> Decoder

Create and return a Decoder for the specified character encoding.

The Decoder consumes byte sequences and decodes them into the original string format.

Parameters

encoding: The character encoding format to be used for decoding the input byte sequences.

Returns

A Decoder instance that can be used to decode byte sequences into strings.

Examples

let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
let decoder = decoder(UTF8)
inspect(decoder.consume(inputs[0]), content="abc")
inspect(decoder.consume(inputs[1]), content="")
inspect(decoder.consume(inputs[2]), content="🐰")
assert_true(decoder.finish().is_empty())

encode

source

fn encode(encoding : Encoding, src : String) -> Bytes

Encode a given string to the specified character encoding and returns the resulting bytes.

Parameters

encoding : The target encoding format.
src: The input string to be encoded.

Returns

A bytes representing the encoded string in the selected format.

Examples

test {
  let src = "Hello, World!"
  let encoded_bytes = @encoding.encode(UTF8, src)
  assert_eq(
    encoded_bytes,
    Bytes::from_array([
      72, 101, 108, 108, 111, 44, 32, 87, 111, 114, 108, 100, 33,
    ]),
  )
}

encode_to

source

#deprecated("Use `buffer.write_string_utf8/utf16le/utf16be` instead")
fn encode_to(src : String, buffer : @moonbitlang/core/buffer.Buffer
Buffer, encoding~ : Encoding) -> Unit

Encodes a string into the specified character encoding and writes the result directly into a buffer.

Parameters:

string : The input string to be encoded.
buffer : The buffer where the encoded bytes will be written to.
encoding : The target encoding format. Defaults to UTF8 if not specified.

Example:

test {
  let buf = Buffer()
  let text = "Hello, world"
  buf.write_string_utf16le(text)
  inspect(buf.to_string(), content="Hello, world")
}

to_utf16_bytes

source

#deprecated("Use `to_utf16le_bytes` instead")
fn to_utf16_bytes(value : Char) -> Bytes

Converts a Char to UTF-16LE bytes.

Alias for to_utf16le_bytes.

Note that this function incurs allocation overhead due to the intermediate creation of new buffers.

to_utf16be_bytes

source

fn to_utf16be_bytes(value : Char) -> Bytes

Converts a Char to UTF-16BE bytes.

Note that this function incurs allocation overhead due to the intermediate creation of new buffers.

to_utf16le_bytes

source

fn to_utf16le_bytes(value : Char) -> Bytes

Converts a Char to UTF-16LE bytes.

Note that this function incurs allocation overhead due to the intermediate creation of new buffers.

to_utf8_bytes

source

fn to_utf8_bytes(value : Char) -> Bytes

Converts a Char to UTF-8 bytes.

Note that this function incurs allocation overhead due to the intermediate creation of new buffers.

write_utf16_char

source

#deprecated("Use `buf.write_char_utf16le(value)` instead")
fn write_utf16_char(buf : @moonbitlang/core/buffer.Buffer
Buffer, value : Char) -> Unit

Write a char into buffer as UTF16LE. Alias for write_utf16le_char

write_utf16be_char

source

#deprecated("Use `buf.write_char_utf16be(value)` instead")
fn write_utf16be_char(buf : @moonbitlang/core/buffer.Buffer
Buffer, value : Char) -> Unit

Write a char into buffer as UTF16BE.

write_utf16le_char

source

#deprecated("Use `buf.write_char_utf16le(value)` instead")
fn write_utf16le_char(buf : @moonbitlang/core/buffer.Buffer
Buffer, value : Char) -> Unit

Write a char into buffer as UTF16LE.

write_utf8_char

source

#deprecated("Use `buf.write_char_utf8(value)` instead")
fn write_utf8_char(buf : @moonbitlang/core/buffer.Buffer
Buffer, value : Char) -> Unit

Write a char into buffer as UTF8.

Source Files

Install

Download zip

experimental packages for moonbitlang/core

experimentalmoonbitlangjsontimecrypto

moonbitlang

Apache-2.0

github.com/moonbitlang/x

Updated 6 days ago

774K downloads

Site source Report issue Modules Skills CLI Build queue Statistics

#Moonbit/Core Encoding

#Overview

#Supported Encoding

#Usage

#Decoding

#Encoding

#MalformedError

#MalformedError::to_repr

#TruncatedError

#TruncatedError::to_repr

#Decoder

#Decoder::consume

Parameters

Returns

Errors

#Decoder::decode

Parameters

Returns

Errors

Examples

#Decoder::decode_lossy

Parameters

Returns

#Decoder::decode_lossy_to

#Decoder::decode_to

#Decoder::finish

Parameters

Returns

Errors

#Decoder::lossy_consume

Parameters

Returns

#Decoder::lossy_finish

Parameters

Returns

#Encoding

#Encoding::to_repr

#U_REP

#decode_to

#decoder

Parameters

Returns

Examples

#encode

Parameters

Returns

Examples

#encode_to

#to_utf16_bytes

#to_utf16be_bytes

#to_utf16le_bytes

#to_utf8_bytes

#write_utf16_char

#write_utf16be_char

#write_utf16le_char

#write_utf8_char

Source Files

MalformedError

MalformedError::to_repr

TruncatedError

TruncatedError::to_repr

Decoder

Decoder::consume

Decoder::decode

Decoder::decode_lossy

Decoder::decode_lossy_to

Decoder::decode_to

Decoder::finish

Decoder::lossy_consume

Decoder::lossy_finish

Encoding

Encoding::to_repr

U_REP

decode_to

decoder

encode

encode_to

to_utf16_bytes

to_utf16be_bytes

to_utf16le_bytes

to_utf8_bytes

write_utf16_char

write_utf16be_char

write_utf16le_char

write_utf8_char