README
- ::equalCoderResult::equal
- ::not_equalCoderResult::not_equal
- ::outputCoderResult::output
- ::to_stringCoderResult::to_string
- ::outputDecodeResult::output
- ::to_stringDecodeResult::to_string
- ::decode_to_stringDecoder::decode_to_string
- ::had_replacementsDecoder::had_replacements
- ::newDecoder::new
- ::outputDecoder::output
- ::resetDecoder::reset
- ::to_stringDecoder::to_string
V
decode
V
decode_half_width_katakana
V
decode_jis_x_0208
V
new_decoder
V
shift_jis_to_utf8

encoding_sjis

encoding_rs-compliant Shift_JIS decoder for MoonBit

Download zip

Author

f4ah6o

Version

0.1.1

License

Apache-2.0

Repository

github.com/horideicom/encoding_sjis.mbt

Last updated

8 hours ago

Downloads

#encoding_sjis

encoding_rs-compliant Shift_JIS decoder for MoonBit.

#Features

Shift_JIS to UTF-8 decoding
Streaming decode API with chunk boundary support
encoding_rs-compliant behavior
Character set support: ASCII, half-width katakana, hiragana, katakana, kanji (JIS X 0208)
Error handling with replacement character (U+FFFD)
Replacement tracking

#Installation

moon add f4ah6o/encoding_sjis@0.1.1

#Usage

#Simple Decode

///|
test "simple decode" {
  let bytes = Bytes::from_array([0x82, 0xA0, 0x82, 0xA2, 0x82, 0xA4]) // "あいう"
  let (result, had_replacements) = @encoding_sjis.decode(src=bytes)
  inspect(result, content="あいう")
  inspect(had_replacements, content="false")
}

#Streaming Decode

///|
test "streaming decode" {
  let decoder = @encoding_sjis.new_decoder()
  let chunk1 = Bytes::from_array([0x82, 0xA0]) // "あ"
  let chunk2 = Bytes::from_array([0x82, 0xA2]) // "い"

  let result1 = decoder.decode_to_string(src=chunk1, false)
  let result2 = decoder.decode_to_string(src=chunk2, true)

  inspect(result1, content="あ")
  inspect(result2, content="い")
}

#Chunk Boundary Handling

The streaming decoder correctly handles multi-byte characters split across chunk boundaries:

///|
test "chunk boundary" {
  let decoder = @encoding_sjis.new_decoder()
  let chunk1 = Bytes::from_array([0x82]) // First byte of "あ"
  let chunk2 = Bytes::from_array([0xA0]) // Second byte of "あ"

  let result1 = decoder.decode_to_string(src=chunk1, false)
  let result2 = decoder.decode_to_string(src=chunk2, true)

  inspect(result1, content="") // Incomplete sequence is buffered
  inspect(result2, content="あ") // Completed after second chunk
}

#Error Handling

Invalid byte sequences are replaced with U+FFFD:

///|
test "error handling" {
  let bytes = Bytes::from_array([0x41, 0x80, 0x42]) // "A" + invalid + "B"
  let (result, had_replacements) = @encoding_sjis.decode(src=bytes)

  inspect(result, content="A�B") // U+FFFD for invalid byte
  inspect(had_replacements, content="true")
}

#Mixed Content

///|
test "mixed content" {
  let bytes = Bytes::from_array([
    0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, // "Hello "
     0x82, 0xA0, 0x82, 0xA2, // "あい"
     0xB1, 0xB2, 0xB3, // "ｱｲｳ" (half-width katakana)
  ])
  let (result, _) = @encoding_sjis.decode(src=bytes)
  inspect(result, content="Hello あいｱｲｳ")
}

#Convenience API

///|
test "shift_jis_to_utf8" {
  let bytes = Bytes::from_array([0x82, 0xA0, 0x82, 0xA2])
  let result = @encoding_sjis.shift_jis_to_utf8(data=bytes)
  inspect(result, content="あい")
}

#API

#Functions

#decode(src: Bytes) -> (String, Bool)

Decodes a Shift_JIS byte sequence to a UTF-8 string (non-streaming).

Returns: (decoded string, whether replacement characters were used)

#new_decoder() -> Decoder

Creates a new decoder for streaming decode operations.

#shift_jis_to_utf8(data: Bytes) -> String

Convenience API that decodes Shift_JIS bytes to UTF-8 string. Does not return replacement information (compatible with jww_parser API).

#Decoder Methods

#Decoder::decode_to_string(src~ : Bytes, Bool) -> String

Decodes a chunk of Shift_JIS bytes to UTF-8 string.

src: Input byte chunk
last: Set true for the final chunk (flushes any pending incomplete sequence)
Returns: Decoded string

#Decoder::reset() -> Unit

Resets the decoder state, clearing any pending bytes and replacement flags.

#Decoder::had_replacements() -> Bool

Returns whether replacement characters (U+FFFD) have been used during decoding.

#Types

#Decoder

Stateful decoder for streaming operations.

pending_first_byte: Stores the first byte of an incomplete 2-byte sequence
had_replacements: Tracks if replacement characters were used

#DecodeResult

Result of a decode operation (for future buffer-based API).

result: CoderResult - operation status
read: Number of bytes read
written: Number of UTF-8 code units written
had_replacements: Whether replacement characters were used

#CoderResult

Result status for streaming operations.

InputEmpty: Input exhausted (normal completion or waiting for more data)
OutputFull: Output buffer full (for future buffer-based API)

#Supported Character Sets

ASCII (0x00-0x7F)
Half-width Katakana (0xA1-0xDF) → U+FF61-U+FF9F
Hiragana (0x82A0-0x82F1)
Full-width Katakana (0x8340-0x8396, 0x83AA-0x83EF, etc.)
Kanji and symbols (JIS X 0208 compliant)

#License

Apache-2.0

CoderResult

source

pub enum CoderResult {
  InputEmpty
  OutputFull
} derive(Eq)

ストリーミングデコード操作の結果 encoding_rsのCoderResultに相当

impl Show for CoderResult

CoderResult::equal

source

fn CoderResult::equal(CoderResult, CoderResult) -> Bool

CoderResult::not_equal

source

fn CoderResult::not_equal(x : CoderResult, y : CoderResult) -> Bool

CoderResult::output

source

fn CoderResult::output(self : CoderResult, logger : &Logger) -> Unit

CoderResult::to_string

source

fn CoderResult::to_string(self : CoderResult) -> String

DecodeResult

source

pub struct DecodeResult {
  result : CoderResult
  read : Int
  written : Int
  had_replacements : Bool
}

デコード操作の結果

impl Show for DecodeResult

DecodeResult::output

source

fn DecodeResult::output(self : DecodeResult, logger : &Logger) -> Unit

DecodeResult::to_string

source

fn DecodeResult::to_string(self : DecodeResult) -> String

Decoder

source

pub struct Decoder {
  pending_first_byte : Int
  had_replacements : Bool
}

Shift_JISデコーダーストリーミング処理のための状態を保持

impl Show for Decoder

Decoder::decode_to_string

source

fn Decoder::decode_to_string(self : Decoder, src~ : Bytes, last : Bool) -> String

Shift_JISバイト列をUTF-8文字列にデコード（ストリーミング対応）

引数:

src: 入力バイト列
last: 最後のチャンクかどうか（falseの場合、不完全なシーケンスを次回に持ち越し）

戻り値: デコードされた文字列

Decoder::had_replacements

source

fn Decoder::had_replacements(self : Decoder) -> Bool

置換文字が使用されたかどうかを返す

Decoder::new

source

fn Decoder::new() -> Decoder

新しいデコーダーを作成

Decoder::output

source

fn Decoder::output(self : Decoder, logger : &Logger) -> Unit

Decoder::reset

source

fn Decoder::reset(self : Decoder) -> Unit

デコーダーの状態をリセット

Decoder::to_string

source

fn Decoder::to_string(self : Decoder) -> String

decode

source

fn decode(src~ : Bytes) -> (String, Bool)

Shift_JISバイト列をUTF-8文字列にデコード（非ストリーミング）

戻り値: (デコードされた文字列, 置換文字が使用されたかどうか)

decode_half_width_katakana

source

fn decode_half_width_katakana(byte~ : Int) -> Char

半角カタカナのShift_JISバイト値(0xA1-0xDF)からUnicode文字へのデコード 0xA1 → U+FF61, 0xA2 → U+FF62, ..., 0xDF → U+FF9F

decode_jis_x_0208

source

fn decode_jis_x_0208(code~ : Int) -> Char?

JIS X 0208 コードポイントからUnicode文字へのデコード不明なコードポイントの場合は None を返す

new_decoder

source

fn new_decoder() -> Decoder

ストリーミングデコード用の新しいデコーダーを作成

shift_jis_to_utf8

source

fn shift_jis_to_utf8(data~ : Bytes) -> String

Shift_JISバイト列をUTF-8文字列に変換 jww_parser互換の簡易API（置換文字の使用状況を返さない）

Source Files

Site source Report issue Packages Build queue Skills Statistics

encoding_sjis

#encoding_sjis

#Features

#Installation

#Usage

#Simple Decode

#Streaming Decode

#Chunk Boundary Handling

#Error Handling

#Mixed Content

#Convenience API

#API

#Functions

#decode(src: Bytes) -> (String, Bool)

#new_decoder() -> Decoder

#shift_jis_to_utf8(data: Bytes) -> String

#Decoder Methods

#Decoder::decode_to_string(src~ : Bytes, Bool) -> String

#Decoder::reset() -> Unit

#Decoder::had_replacements() -> Bool

#Types

#Decoder

#DecodeResult

#CoderResult

#Supported Character Sets

#License

#CoderResult

#CoderResult::equal

#CoderResult::not_equal

#CoderResult::output

#CoderResult::to_string

#DecodeResult

#DecodeResult::output

#DecodeResult::to_string

#Decoder

#Decoder::decode_to_string

#Decoder::had_replacements

#Decoder::new

#Decoder::output

#Decoder::reset

#Decoder::to_string

#decode

#decode_half_width_katakana

#decode_jis_x_0208

#new_decoder

#shift_jis_to_utf8

Source Files

CoderResult

CoderResult::equal

CoderResult::not_equal

CoderResult::output

CoderResult::to_string

DecodeResult

DecodeResult::output

DecodeResult::to_string

Decoder

Decoder::decode_to_string

Decoder::had_replacements

Decoder::new

Decoder::output

Decoder::reset

Decoder::to_string

decode

decode_half_width_katakana

decode_jis_x_0208

new_decoder

shift_jis_to_utf8