moonbitstack/moontoken/piece

README
U
Refused
Tr
Model
- ::byte_bpeTokenizer::byte_bpe
- ::cl100kTokenizer::cl100k
- ::countTokenizer::count
- ::decodeTokenizer::decode
- ::decode_bytesTokenizer::decode_bytes
- ::encodeTokenizer::encode
- ::encode_ordinaryTokenizer::encode_ordinary
- ::lengthTokenizer::length
- ::newTokenizer::new
- ::o200kTokenizer::o200k
- ::ofTokenizer::of
- ::of_protoTokenizer::of_proto
- ::p50k_editTokenizer::p50k_edit
- ::parseTokenizer::parse
- ::piece_bpeTokenizer::piece_bpe
- ::policyTokenizer::policy
- ::r50kTokenizer::r50k
- ::token_bytesTokenizer::token_bytes
- ::unigramTokenizer::unigram
- ::with_policyTokenizer::with_policy

moonbitstack/moontoken/piece does not have a README file

Kind

pub(all) enum Kind {
  Normal
  Unknown
  Control
  UserDefined
  Unused
  RawByte
} derive(Eq)

What a piece is for.

Kind::equal

source

fn Kind::equal(Kind, Kind) -> Bool

Kind::not_equal

source

fn Kind::not_equal(x : Kind, y : Kind) -> Bool

ModelKind

source

pub(all) enum ModelKind {
  Unigram
  Bpe
  Word
  Char
} derive(Eq)

What kind of model a file was trained as.

ModelKind::equal

source

fn ModelKind::equal(ModelKind, ModelKind) -> Bool

ModelKind::not_equal

source

fn ModelKind::not_equal(x : ModelKind, y : ModelKind) -> Bool

Piece

source

pub(all) struct Piece {
  text : Bytes
  score : Float
  kind : Kind
}

One entry of a vocabulary: the text it stands for, the log probability the trainer gave it, and what it is.

PieceBpe

source

pub struct PieceBpe {
  pieces : Pieces
  normalizer : @moonbitstack/moontoken/normalizer.Normalizer
Normalizer
  user_defined : Map[Int, Array[Bytes]]
}

A model that segments text by merging neighbouring symbols, the sentence-piece flavour: symbols are merged in order of the score of the piece they would form, until no merge is left that the vocabulary has.

PieceBpe::decode

source

fn PieceBpe::decode(self : PieceBpe, ids : ArrayView[Int]) -> String raise @moonbitstack/moontoken/refused.Refused
Refused

The text a list of tokens stands for.

PieceBpe::encode

source

fn PieceBpe::encode(self : PieceBpe, text : StringView) -> Array[Int] raise @moonbitstack/moontoken/refused.Refused
Refused

The tokens of text.

PieceBpe::length

source

fn PieceBpe::length(self : PieceBpe) -> Int

How many pieces the vocabulary has.

PieceBpe::new

source

fn PieceBpe::new(pieces~ : Pieces, normalizer? : @moonbitstack/moontoken/normalizer.Normalizer
Normalizer, user_defined? : Array[Bytes]) -> PieceBpe

PieceBpe::pieces

source

fn PieceBpe::pieces(self : PieceBpe) -> Pieces

The vocabulary.

PieceBpe::token_bytes

source

fn PieceBpe::token_bytes(self : PieceBpe, id : Int) -> Bytes?

The token a piece id stands for.

Pieces

source

pub struct Pieces {
  items : Array[Piece]
  exact : Map[Bytes, Int]
  lattice : Map[Bytes, Int]
  longest : Int
  unk_id : Int
  unk_surface : Bytes
  byte_ids : Array[Int]
  byte_fallback : Bool
}

A vocabulary of pieces, with the indexes a model needs: one for the pieces segmentation may use, one for everything.

Pieces::at

source

fn Pieces::at(self : Pieces, id : Int) -> Piece raise @moonbitstack/moontoken/refused.Refused
Refused

The piece with this id.

Pieces::byte_fallback

source

fn Pieces::byte_fallback(self : Pieces) -> Bool

Whether a character with no piece may be spelled out as bytes.

Pieces::byte_id

source

fn Pieces::byte_id(self : Pieces, b : Int) -> Int

The id of the piece for one byte, or -1.

Pieces::id

source

fn Pieces::id(self : Pieces, text : BytesView) -> Int?

The id of a piece, whatever kind it is.

Pieces::lattice_id

source

fn Pieces::lattice_id(self : Pieces, text : BytesView) -> Int?

The id of a piece segmentation may use: an ordinary one or a declared one.

Pieces::length

source

fn Pieces::length(self : Pieces) -> Int

How many pieces there are; ids run from 0 to this less one.

Pieces::longest

source

fn Pieces::longest(self : Pieces) -> Int

The longest piece segmentation may use, in bytes.

Pieces::new

source

fn Pieces::new(items~ : Array[Piece], unk_id? : Int, unk_surface? : String, byte_fallback? : Bool) -> Pieces raise @moonbitstack/moontoken/refused.Refused
Refused

Builds the tables over a list of pieces.

unk_id is the piece that stands in for text nothing covers, and byte_fallback says whether the model can spell a character out as bytes.

Pieces::unk_id

source

fn Pieces::unk_id(self : Pieces) -> Int

The id of the piece that stands in for uncovered text, or -1.

Pieces::unk_surface

source

fn Pieces::unk_surface(self : Pieces) -> Bytes

The text an unknown token stands for when it is decoded.

Proto

source

pub struct Proto {
  pieces : Pieces
  trainer : Trainer
  normalizer : @moonbitstack/moontoken/normalizer.Spec
Spec
}

A model file: its pieces, how it was trained, and how text is normalised before segmentation.

Proto::normalizer

source

fn Proto::normalizer(self : Proto) -> @moonbitstack/moontoken/normalizer.Normalizer
Normalizer raise @moonbitstack/moontoken/refused.Refused
Refused

The normalisation the model expects, ready to apply.

Proto::of

source

fn Proto::of(bytes : Bytes) -> Proto raise @moonbitstack/moontoken/refused.Refused
Refused

Reads a model file.

Proto::pieces

source

fn Proto::pieces(self : Proto) -> Pieces

The vocabulary.

Proto::spec

source

fn Proto::spec(self : Proto) -> @moonbitstack/moontoken/normalizer.Spec
Spec

The normalisation the model expects, as written in the file.

Proto::trainer

source

fn Proto::trainer(self : Proto) -> Trainer

How the model was trained.

Proto::user_defined

source

fn Proto::user_defined(self : Proto) -> Array[Bytes]

The symbols the caller reserved, in the order the file lists them.

Span

source

pub struct Span {
  start : Int
  stop : Int
  id : Int
}

What segmentation produced, before the tokens a model reserves are dealt with: a span of the normalised bytes and the piece that covers it.

Trainer

source

pub(all) struct Trainer {
  kind : ModelKind
  unk_id : Int
  byte_fallback : Bool
  treat_whitespace_as_suffix : Bool
  unk_piece : String
  unk_surface : String
}

The settings a model was trained with that change how it runs.

Unigram

source

pub struct Unigram {
  pieces : Pieces
  normalizer : @moonbitstack/moontoken/normalizer.Normalizer
Normalizer
  min_score : Float
}

A model that segments text by the most likely sequence of pieces.

Each piece carries the log probability the trainer measured for it, and the segmentation is the one whose pieces add up to the largest total. Text that no piece covers becomes the unknown piece, or, when the model can spell it out, one token per byte.

Unigram::decode

source

fn Unigram::decode(self : Unigram, ids : ArrayView[Int]) -> String raise @moonbitstack/moontoken/refused.Refused
Refused

The text a list of tokens stands for.

Unigram::encode

source

fn Unigram::encode(self : Unigram, text : StringView) -> Array[Int] raise @moonbitstack/moontoken/refused.Refused
Refused

The tokens of text.

Unigram::length

source

fn Unigram::length(self : Unigram) -> Int

How many pieces the vocabulary has.

Unigram::new

source

fn Unigram::new(pieces~ : Pieces, normalizer? : @moonbitstack/moontoken/normalizer.Normalizer
Normalizer) -> Unigram

Builds the model over a vocabulary.

Raises when the vocabulary has neither an unknown piece nor byte fallback, because text outside the vocabulary would then have no encoding at all.

Unigram::pieces

source

fn Unigram::pieces(self : Unigram) -> Pieces

The vocabulary.

Unigram::token_bytes

source

fn Unigram::token_bytes(self : Unigram, id : Int) -> Bytes?

The token a piece id stands for, for callers that want the spelling.

byte_of_piece

source

fn byte_of_piece(text : Bytes) -> Int?

The byte a <0xNN> spelling stands for, when it is one.

byte_piece

source

fn byte_piece(b : Int) -> String

The text of the piece that stands for one byte.

Source Files

Install

Download zip

Tokenizers for MoonBit: byte-level BPE and SentencePiece.

tokenizertokenizerstokenizationtokencount-tokensbpebyte-pair-encodingbyte-level-bpesentencepieceunigramtiktokencl100kcl100k_baseo200ko200k_baser50k_basep50k_basetokenizer-jsonvocabularyvocabmergesencodedecodespecial-tokensbyte-fallbackpre-tokenizernormalizationnlpllmpromptcontext-windowmoonbit

moonbitstack

Apache-2.0

github.com/moonbitstack/moontoken

Updated 13 hours ago

10 downloads

Site source Report issue Modules Skills CLI Build queue Statistics

#Kind

#Kind::equal

#Kind::not_equal

#ModelKind

#ModelKind::equal

#ModelKind::not_equal

#Piece

#PieceBpe

#PieceBpe::decode

#PieceBpe::encode

#PieceBpe::length

#PieceBpe::new

#PieceBpe::pieces

#PieceBpe::token_bytes

#Pieces

#Pieces::at

#Pieces::byte_fallback

#Pieces::byte_id

#Pieces::id

#Pieces::lattice_id

#Pieces::length

#Pieces::longest

#Pieces::new

#Pieces::unk_id

#Pieces::unk_surface

#Proto

#Proto::normalizer

#Proto::of

#Proto::pieces

#Proto::spec

#Proto::trainer

#Proto::user_defined

#Span

#Trainer

#Unigram

#Unigram::decode

#Unigram::encode

#Unigram::length

#Unigram::new

#Unigram::pieces

#Unigram::token_bytes

#byte_of_piece

#byte_piece

Source Files

Kind

Kind::equal

Kind::not_equal

ModelKind

ModelKind::equal

ModelKind::not_equal

Piece

PieceBpe

PieceBpe::decode

PieceBpe::encode

PieceBpe::length

PieceBpe::new

PieceBpe::pieces

PieceBpe::token_bytes

Pieces

Pieces::at

Pieces::byte_fallback

Pieces::byte_id

Pieces::id

Pieces::lattice_id

Pieces::length

Pieces::longest

Pieces::new

Pieces::unk_id

Pieces::unk_surface

Proto

Proto::normalizer

Proto::of

Proto::pieces

Proto::spec

Proto::trainer

Proto::user_defined

Span

Trainer

Unigram

Unigram::decode

Unigram::encode

Unigram::length

Unigram::new

Unigram::pieces

Unigram::token_bytes

byte_of_piece

byte_piece