Encoding package extracted from moonbitlang/x

moon add tonyfettes/encoding@0.3.9

Download zip

Author

tonyfettes

Version

0.3.9

License

Apache-2.0

Repository

github.com/moonbitlang/x

Last updated

9 months ago

Downloads

README

#tonyfettes/encoding

This package is extracting from moonbitlang/x.

#Moonbit/Core Encoding

#Overview

The @encoding package provides an implementation for encoding and decoding strings using various character encodings (e.g., UTF-8).

It supports both streaming and non-streaming (bulk) operations, making it flexible for different use-cases.

#Supported Encoding

UTF8
UTF16 // alias for UTF16LE
UTF16LE
UTF16BE

#Usage

#Decoding

Decode a UTF-8 byte stream:

// Initialize a streaming UTF-8 decoder
let decoder = @encoding.decoder(UTF8)

// Consume byte chunks
let inputs = [b"abc", b"\xf0", b"\x9f\x90\xb0"] // UTF8(🐰) == <F09F 90B0>
inspect!(decoder.consume!(inputs[0]), content="abc")
inspect!(decoder.consume!(inputs[1]), content="")
inspect!(decoder.consume!(inputs[2]), content="🐰")

// Finish decoding
assert_true!(decoder.finish!().is_empty())

#Encoding

Encode a string to UTF-8 bytes:

// Encode a string to UTF-8
let src = "你好👀"
let bytes = @encoding.encode(UTF8, src)
inspect!(
  bytes,
  content=
    #|b"\xe4\xbd\xa0\xe5\xa5\xbd\xf0\x9f\x91\x80"
  ,
)

Encode a single character to UTF-8 bytes:

inspect!(
  @encoding.to_utf8_bytes('A'),
  content=
    #|b"\x41"
  ,
)

#
Encode

source

pub trait Encode {
  size_hint(Self, encoding~ : Encoding) -> Int
  encode(Self, encoding~ : Encoding) -> Bytes
  encode_to(Self, @moonbitlang/core/buffer.T
T, encoding~ : Encoding) -> Unit
}

impl Encode for Char

impl Encode for String

impl Encode for StringView

fn encode_to(value : @moonbitlang/core/string.StringView
StringView, buffer : @moonbitlang/core/buffer.T
T, encoding~ : Encoding) -> Unit

source

fn size_hint(value : @moonbitlang/core/string.StringView
StringView, encoding~ : Encoding) -> Int

source

#
Decoder

source

type Decoder

#
Decoder::consume

source

fn Decoder::consume(self : Decoder, input : @moonbitlang/core/bytes.View
View) -> String raise DecodingError

Legacy method: same as decode

#
Decoder::decode

source

fn Decoder::decode(self : Decoder, input : @moonbitlang/core/bytes.View
View, stream~ : Bool = ..) -> String raise DecodingError

Decode bytes to string

#
Decoder::decode_lossy

source

fn Decoder::decode_lossy(self : Decoder, input : @moonbitlang/core/bytes.View
View, stream~ : Bool = ..) -> String

Decode bytes to string (lossy)

#
Decoder::decode_lossy_to

source

fn Decoder::decode_lossy_to(self : Decoder, input : @moonbitlang/core/bytes.View
View, output : StringBuilder, stream~ : Bool = ..) -> Unit

Decode bytes and write result to StringBuilder (lossy)

#
Decoder::decode_to

source

fn Decoder::decode_to(self : Decoder, input : @moonbitlang/core/bytes.View
View, output : StringBuilder, stream~ : Bool = ..) -> Unit raise DecodingError

Decode bytes and write result to StringBuilder

#
Decoder::finish

source

fn Decoder::finish(self : Decoder) -> String raise DecodingError

Finish decoding and return any remaining content

#
Decoder::lossy_consume

source

fn Decoder::lossy_consume(self : Decoder, input : @moonbitlang/core/bytes.View
View) -> String

Legacy method: same as decode_lossy

#
Decoder::lossy_finish

source

fn Decoder::lossy_finish(self : Decoder) -> String

Finish decoding in lossy mode

#
DecodingError

source

pub enum DecodingError {
  Malformed(Bytes)
  Truncated(Bytes)
}

impl Show for DecodingError

impl ToJson for DecodingError

#
Encoding

source

pub(all) enum Encoding {
  UTF8
  UTF16LE
  UTF16BE
}

#
U_REP

source

let U_REP : Char

The Unicode Replacement Character, which is used to replace invalid or unrecognized sequences during lossy decoding. https://unicode.org/charts/nameslist/n_FFF0.html

#
decode

source

fn decode(bytes : @moonbitlang/core/bytes.View
View, encoding~ : Encoding = ..) -> String raise DecodingError

Decode bytes to string in one call

#
decode_lossy

source

fn decode_lossy(input : @moonbitlang/core/bytes.View
View, encoding~ : Encoding = ..) -> String

Decode bytes to string in one call (lossy)

#
decode_lossy_to

source

fn decode_lossy_to(input : @moonbitlang/core/bytes.View
View, output : StringBuilder, encoding~ : Encoding = ..) -> Unit

Decode bytes and write to StringBuilder in one call (lossy)

#
decode_to

source

fn decode_to(input : @moonbitlang/core/bytes.View
View, output : StringBuilder, encoding~ : Encoding = ..) -> Unit raise DecodingError

Decode bytes and write to StringBuilder in one call

#
decoder

source

fn decoder(encoding : Encoding) -> Decoder

Create a decoder for the specified encoding

#
encode

source

fn[Encode : Encode] encode(string : Encode, encoding~ : Encoding) -> Bytes

Encode a given string to the specified character encoding and returns the resulting bytes.

Parameters

encoding : The target encoding format.
src: The input string to be encoded.

Returns

A bytes representing the encoded string in the selected format.

Examples

let src = "Hello, World!"
@json.inspect(encode(encoding=UTF8, src).to_array(), content=
  [72,101,108,108,111,44,32,87,111,114,108,100,33]
)

#
encode_to

source

fn[Encode : Encode] encode_to(src : Encode, buffer : @moonbitlang/core/buffer.T
T, encoding~ : Encoding) -> Unit

Encodes a string into the specified character encoding and writes the result directly into a buffer.

Parameters:

string : The input string to be encoded.
buffer : The buffer where the encoded bytes will be written to.
encoding : The target encoding format. Defaults to UTF8 if not specified.

Example:

let buf = @buffer.new()
let text = "Hello, world"
@encoding.encode_to(text, buf, encoding=UTF16LE)
inspect(buf.to_string(), content="Hello, world")

Source Files

Site source Report issue Packages Build queue Skills Statistics

encoding

#tonyfettes/encoding

#Moonbit/Core Encoding

#Overview

#Supported Encoding

#Usage

#Decoding

#Encoding

#Encode

#Decoder

#Decoder::consume

#Decoder::decode

#Decoder::decode_lossy

#Decoder::decode_lossy_to

#Decoder::decode_to

#Decoder::finish

#Decoder::lossy_consume

#Decoder::lossy_finish

#DecodingError

#Encoding

#U_REP

#decode

#decode_lossy

#decode_lossy_to

#decode_to

#decoder

#encode

Parameters

Returns

Examples

#encode_to

Source Files

#
Encode

#
Decoder

#
Decoder::consume

#
Decoder::decode

#
Decoder::decode_lossy

#
Decoder::decode_lossy_to

#
Decoder::decode_to

#
Decoder::finish

#
Decoder::lossy_consume

#
Decoder::lossy_finish

#
DecodingError

#
Encoding

#
U_REP

#
decode

#
decode_lossy

#
decode_lossy_to

#
decode_to

#
decoder

#
encode

#
encode_to