#moonbitlang/lexer

moonbitlang/lexer turns MoonBit source text into token streams used by the parsers in this repository.

The main entry points are tokens_from_string and tokens_from_string_with_utf16_location. Both return a LexResult containing tokens, lexical errors, and docstring/comment metadata.

tokens_from_string uses the lexer default location accounting. tokens_from_string_with_utf16_location tokenizes the same source but reports locations in UTF-16 code units, which is useful for editor and LSP integrations.

#Examples

///|
test "tokenize source text" {
  let result = @lexer.tokens_from_string("let answer = 42", comment=false)
  assert_eq(result.errors.length(), 0)
  debug_inspect(
    result.tokens.map(triple => triple.0),
    content=(
      #|[LET, LIDENT("answer"), EQUAL, INT("42"), EOF]
    ),
  )
}

///|
test "utf16 location mode keeps the same tokens" {
  let source = "let x = \"😀\""
  let normal = @lexer.tokens_from_string(source, comment=false)
  let utf16 = @lexer.tokens_from_string_with_utf16_location(
    source,
    comment=false,
  )
  debug_inspect(
    normal.tokens.map(triple => triple.0),
    content=(
      #|[LET, LIDENT("x"), EQUAL, STRING("😀"), EOF]
    ),
  )
  debug_inspect(
    utf16.tokens.map(triple => triple.0),
    content=(
      #|[LET, LIDENT("x"), EQUAL, STRING("😀"), EOF]
    ),
  )
}

LexResult

source

pub struct LexResult {
  tokens : Array[(@moonbitlang/lexer/tokens.Token
Token, @moonbitlang/lexer/basic.Position
Position, @moonbitlang/lexer/basic.Position
Position)]
  errors : Array[(@moonbitlang/lexer/basic.Position
Position, @moonbitlang/lexer/basic.Position
Position, LexicalError)]
  docstrings : Array[@moonbitlang/core/list.List
List[(@moonbitlang/lexer/basic.Location
Location, @moonbitlang/lexer/tokens.Comment
Comment)]]
}

LexicalError

source

pub enum LexicalError {
  IllegalCharacter(Char)
  UnterminatedString
  UnterminatedStringInVariableInterploation
  InterpInvalidMultilineString
  InterpInvalidComment
  InterpInvalidAttribute
  InterpMissingExpression
  InvalidDotInt(String)
  MissingIdentifierAfterDot
  InvalidByteLiteral(String)
  Reserved_keyword(String)
  InvalidMetavarSyntax(String)
}

impl Show for LexicalError

tokens_from_string

source

fn tokens_from_string(name? : String, start_pos? : @moonbitlang/lexer/basic.Position
Position, is_interpolation? : Bool, enable_metavar? : Bool, comment~ : Bool, str : StringView) -> LexResult

tokens_from_string_with_utf16_location

source

fn tokens_from_string_with_utf16_location(name? : String, start_pos? : @moonbitlang/lexer/basic.Position
Position, is_interpolation? : Bool, enable_metavar? : Bool, comment~ : Bool, str : StringView) -> LexResult