tiktoken

moon add tonyfettes/tiktoken@0.2.1
Download zip
Version
0.2.1
License
Apache-2.0
Last updated
9 months ago
Downloads
43

Dependencies

README

#tonyfettes/tiktoken

#
DecodingError

type DecodingError

#
Encoding

type Encoding

#
Encoding::decode

fn Encoding::decode(self : Encoding, tokens : ArrayView[Int]) -> String raise DecodingError

#
Encoding::encode

fn Encoding::encode(self : Encoding, piece : StringView) -> Array[Int] raise

#
Encoding::new

fn Encoding::new(mergeable_ranks~ : Map[Bytes, Int], special_tokens~ : Map[String, Int], pat_str~ : String) -> Encoding raise

#
Encoding::special_tokens

fn Encoding::special_tokens(self : Encoding, piece : String) -> Int?

Source Files

Powered by MoonBit

Site sourceReport issuePackagesBuild queueSkillsStatistics

© 2026 mooncakes.io