🥮 mooncakes.io
README

Milky2018/minigpt/tokenizer does not have a README file

#
TokenizerError

source
pub(all) suberror TokenizerError {
  UnknownCharacter(Char)
  UnknownToken(String)
  InvalidTokenId(Int)
} derive(Eq, 
Debug)

#
BpeMerge

source
pub struct BpeMerge {
  // private fields
}

#
BpeMerge::BpeMerge

source
fn BpeMerge::BpeMerge(left : String, right : String, merged : String) -> BpeMerge

#
BpeMerge::left

source
fn BpeMerge::left(self : BpeMerge) -> String

#
BpeMerge::merged

source
fn BpeMerge::merged(self : BpeMerge) -> String

#
BpeMerge::right

source
fn BpeMerge::right(self : BpeMerge) -> String

#
BpeTokenizer

source
pub struct BpeTokenizer {
  // private fields
}

#
BpeTokenizer::decode

source
fn BpeTokenizer::decode(self : BpeTokenizer, ids : Array[Int]) -> String raise TokenizerError

#
BpeTokenizer::encode

source
fn BpeTokenizer::encode(self : BpeTokenizer, text : String) -> Array[Int] raise TokenizerError

#
BpeTokenizer::from_vocabulary_and_merges

source
fn BpeTokenizer::from_vocabulary_and_merges(vocabulary : Array[String], merges : Array[BpeMerge]) -> BpeTokenizer

#
BpeTokenizer::kind_name

source
fn BpeTokenizer::kind_name(_self : BpeTokenizer) -> String

#
BpeTokenizer::merges

source
fn BpeTokenizer::merges(self : BpeTokenizer) -> Array[BpeMerge]

#
BpeTokenizer::train

source
fn BpeTokenizer::train(text : String, vocab_size : Int) -> (BpeTokenizer, Array[Int])

#
BpeTokenizer::vocab_size

source
fn BpeTokenizer::vocab_size(self : BpeTokenizer) -> Int

#
BpeTokenizer::vocabulary

source
fn BpeTokenizer::vocabulary(self : BpeTokenizer) -> Array[String]

#
CharTokenizer

source
pub struct CharTokenizer {
  // private fields
}

#
CharTokenizer::decode

source
fn CharTokenizer::decode(self : CharTokenizer, ids : Array[Int]) -> String raise TokenizerError

#
CharTokenizer::encode

source
fn CharTokenizer::encode(self : CharTokenizer, text : String) -> Array[Int] raise TokenizerError

#
CharTokenizer::from_chars

source
fn CharTokenizer::from_chars(chars : Array[Char]) -> CharTokenizer

#
CharTokenizer::kind_name

source
fn CharTokenizer::kind_name(_self : CharTokenizer) -> String

#
CharTokenizer::train

source
fn CharTokenizer::train(text : String) -> (CharTokenizer, Array[Int])

#
CharTokenizer::vocab_size

source
fn CharTokenizer::vocab_size(self : CharTokenizer) -> Int

#
CharTokenizer::vocabulary

source
fn CharTokenizer::vocabulary(self : CharTokenizer) -> Array[Char]

#
TokenDataset

source
pub struct TokenDataset {
  // private fields
}

#
TokenDataset::tokenizer

source
fn TokenDataset::tokenizer(self : TokenDataset) -> Tokenizer

#
TokenDataset::train_ids

source
fn TokenDataset::train_ids(self : TokenDataset) -> Array[Int]

#
TokenDataset::val_ids

source
fn TokenDataset::val_ids(self : TokenDataset) -> Array[Int]

#
Tokenizer

source
pub(all) enum Tokenizer {
  Character(CharTokenizer)
  Word(WordTokenizer)
  Bpe(BpeTokenizer)
}

#
Tokenizer::bpe_merges

source
fn Tokenizer::bpe_merges(self : Tokenizer) -> Array[BpeMerge]

#
Tokenizer::decode

source
fn Tokenizer::decode(self : Tokenizer, ids : Array[Int]) -> String raise TokenizerError

#
Tokenizer::encode

source
fn Tokenizer::encode(self : Tokenizer, text : String) -> Array[Int] raise TokenizerError

#
Tokenizer::from_bpe

source
fn Tokenizer::from_bpe(tokenizer : BpeTokenizer) -> Tokenizer

#
Tokenizer::from_char

source
fn Tokenizer::from_char(tokenizer : CharTokenizer) -> Tokenizer

#
Tokenizer::from_word

source
fn Tokenizer::from_word(tokenizer : WordTokenizer) -> Tokenizer

#
Tokenizer::kind_name

source
fn Tokenizer::kind_name(self : Tokenizer) -> String

#
Tokenizer::train

source
fn Tokenizer::train(text : String, config : TokenizerConfig) -> (Tokenizer, Array[Int])

#
Tokenizer::train_bpe

source
fn Tokenizer::train_bpe(text : String, vocab_size : Int) -> (Tokenizer, Array[Int])

#
Tokenizer::train_char

source
fn Tokenizer::train_char(text : String) -> (Tokenizer, Array[Int])

#
Tokenizer::train_word

source
fn Tokenizer::train_word(text : String) -> (Tokenizer, Array[Int])

#
Tokenizer::vocab_size

source
fn Tokenizer::vocab_size(self : Tokenizer) -> Int

#
Tokenizer::vocabulary

source
fn Tokenizer::vocabulary(self : Tokenizer) -> Array[String]

#
TokenizerConfig

source
pub(all) enum TokenizerConfig {
  CharacterLevel
  WordLevel
  BpeLevel(Int)
} derive(Eq, 
Debug)

#
WordTokenizer

source
pub struct WordTokenizer {
  // private fields
}

#
WordTokenizer::decode

source
fn WordTokenizer::decode(self : WordTokenizer, ids : Array[Int]) -> String raise TokenizerError

#
WordTokenizer::encode

source
fn WordTokenizer::encode(self : WordTokenizer, text : String) -> Array[Int] raise TokenizerError

#
WordTokenizer::from_tokens

source
fn WordTokenizer::from_tokens(tokens : Array[String]) -> WordTokenizer

#
WordTokenizer::kind_name

source
fn WordTokenizer::kind_name(_self : WordTokenizer) -> String

#
WordTokenizer::train

source
fn WordTokenizer::train(text : String) -> (WordTokenizer, Array[Int])

#
WordTokenizer::vocab_size

source
fn WordTokenizer::vocab_size(self : WordTokenizer) -> Int

#
WordTokenizer::vocabulary

source
fn WordTokenizer::vocabulary(self : WordTokenizer) -> Array[String]

#
DEFAULT_BPE_VOCAB_SIZE

source
let DEFAULT_BPE_VOCAB_SIZE : Int

#
prepare_token_dataset

source
fn prepare_token_dataset(text : String, config : TokenizerConfig) -> TokenDataset

Source Files

Powered by MoonBit

Site sourceReport issuePackagesBuild queueSkillsStatistics

© 2026 mooncakes.io