moonrockz/gleam-parser/scanner

README
U
Arg
U
ArgNames
U
Assert
U
AssignName
U
Assignment
U
AssignmentKind
U
BinOp
U
BitArrayOption
U
BitArraySegment
U
BitArraySize
U
CallArg
U
Clause
U
ClauseGuard
U
Comment
U
CommentKind
U
Constant
U
CustomType
U
DecodeError
U
Definition
U
DefinitionCst
U
DefinitionKind
U
Deprecation
U
Diagnostic
U
Dialect
U
Documentation
U
Expr
U
External
U
Function
U
FunctionLiteralKind
U
ImplicitCallArgOrigin
U
Import
U
ImportAlias
U
IncorrectNamePosition
U
IntOperator
U
Keyword
U
Label
U
LexicalErrorType
U
Module
U
ModuleConstant
U
ModuleCst
U
ModuleExtra
U
OracleOutput
U
ParseError
U
ParseErrorDetails
U
ParseErrorType
U
ParseResult
U
Parsed
U
Pattern
U
Position
U
Publicity
U
RecordBeingUpdated
U
RecordConstructor
U
RecordConstructorArg
U
RecordUpdateArg
U
Rule
U
Severity
U
SourceText
U
SpannedString
U
SrcSpan
U
Statement
U
StringPrefixLeftSideAssignment
U
TailPattern
U
Target
U
TargetedDefinition
U
TodoKind
U
Token
U
TokenKind
U
TokenStream
U
Trivia
U
TypeAlias
U
TypeAst
U
TypeAstConstructor
U
TypeAstConstructorName
U
TypeAstFn
U
TypeAstHole
U
TypeAstTuple
U
TypeAstVar
U
UnicodeEscapeError
U
UnqualifiedImport
U
Use
U
UseAssignment
U
Warning
V
canonical
V
decode_expr
V
decode_oracle_output
V
decode_parsed
V
diagnostic_description
V
encode_diagnostics
V
encode_expr
V
encode_oracle_output
V
encode_parsed
V
parse_float_value
V
parse_int_value
V
parse_module
V
parse_text
V
render_plain
V
render_terminal
V
tokenize
V
tokenize_text
V
version

moonrockz/gleam-parser/scanner does not have a README file

Comment

pub(all) struct Comment {
  kind : CommentKind
  content : String
  span : @moonrockz/gleam-parser/span.SrcSpan
SrcSpan
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

A comment. span covers the slashes and the text up to the line break (not included); content is the text after the slashes.

Comment::content_span

source

fn Comment::content_span(self : Comment) -> @moonrockz/gleam-parser/span.SrcSpan
SrcSpan

The compiler's span for the comment: its content, without the slashes.

CommentKind

source

pub(all) enum CommentKind {
  Regular
  Doc
  Module
} derive(Eq, Hash, @moonbitlang/core/debug.Debug
Debug)

The three comment forms: //, /// (documentation) and //// (module documentation).

CommentKind::prefix_length

source

fn CommentKind::prefix_length(self : CommentKind) -> Int

The number of slashes that start the comment.

Keyword

source

pub(all) enum Keyword {
  As
  Assert
  Auto
  Case
  Const
  Delegate
  Derive
  Echo
  Else
  Fn
  If
  Implement
  Import
  Let
  Macro
  Opaque
  Panic
  Pub
  Test
  Todo
  Type
  Use
} derive(Eq, Hash, @moonbitlang/core/debug.Debug
Debug)

The reserved words of Gleam v1.19.2 (string_to_keyword in the compiler's lexer), in alphabetical order.

Keyword::of

source

fn Keyword::of(word : String) -> Keyword?

The keyword for word, or None when word is not reserved.

Keyword::text

source

fn Keyword::text(self : Keyword) -> String

The source text of the keyword.

LexicalErrorType

source

pub(all) enum LexicalErrorType {
  BadStringEscape
  InvalidUnicodeEscape(UnicodeEscapeError)
  DigitOutOfRadix
  NumTrailingUnderscore
  RadixIntNoValue
  MissingExponent
  UnexpectedStringEnd
  UnrecognizedToken(Char)
  InvalidTripleEqual
  MergeConflictIndicator
  UnsupportedProceduralOperator
  VisuallySimilarInvalidCharacter(name~ : String, correct~ : String)
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

The lexical errors of the compiler's LexicalErrorType, one variant each.

LexicalErrorType::hints

source

fn LexicalErrorType::hints(self : LexicalErrorType) -> Array[String]

The compiler's extra lines (to_parse_error_info), in order.

LexicalErrorType::kind

source

fn LexicalErrorType::kind(self : LexicalErrorType) -> String

The variant name, as the rejection check compares it with the oracle.

LexicalErrorType::label

source

fn LexicalErrorType::label(self : LexicalErrorType) -> String

The compiler's label text (LexicalError::to_parse_error_info).

SourceText

source

pub struct SourceText {
  // private fields
}

A source text with lookups between its UTF-16 offsets (MoonBit strings), its UTF-8 byte offsets (the compiler's SrcSpan) and 1-based line and column positions.

Code point boundaries are kept in two parallel sorted arrays: bound_bytes[k] and bound_units[k] are the byte and UTF-16 offsets of the k-th boundary; the last boundary is the end of the text. The low unit of a surrogate pair is not a boundary. line_bounds[l] is the boundary index of line l + 1's first character.

SourceText::byte_length

source

fn SourceText::byte_length(self : SourceText) -> Int

The length of the text in UTF-8 bytes.

SourceText::byte_offset_of

source

fn SourceText::byte_offset_of(self : SourceText, utf16 : Int) -> Int?

The byte offset of the code point boundary at UTF-16 offset utf16, or None when utf16 is out of range or is the low unit of a surrogate pair.

SourceText::line_count

source

fn SourceText::line_count(self : SourceText) -> Int

The number of lines: one more than the number of line breaks.

SourceText::new

source

fn SourceText::new(text : String) -> SourceText

test {
  let st = @scanner.SourceText::new("é\nx")
  debug_inspect(st.byte_length(), content="4")
  debug_inspect(st.position_of(3), content="Some({ line: 2, column: 1 })")
  debug_inspect(st.utf16_offset_of(3), content="Some(2)")
}

SourceText::position_of

source

fn SourceText::position_of(self : SourceText, byte : Int) -> @moonrockz/gleam-parser/span.Position
Position?

The 1-based line and column of the code point boundary at byte offset byte; columns count code points. None when byte is not a boundary.

SourceText::slice

source

fn SourceText::slice(self : SourceText, span : @moonrockz/gleam-parser/span.SrcSpan
SrcSpan) -> String?

The text of a byte span, or None when an end of the span is not a code point boundary of the text.

SourceText::text

source

fn SourceText::text(self : SourceText) -> String

SourceText::utf16_offset_of

source

fn SourceText::utf16_offset_of(self : SourceText, byte : Int) -> Int?

The UTF-16 offset of the code point boundary at byte offset byte, or None when byte is out of range or inside a multi-byte character.

Token

source

pub(all) struct Token {
  kind : TokenKind
  lexeme : String
  span : @moonrockz/gleam-parser/span.SrcSpan
SrcSpan
  trivia_before : ArrayView[Trivia]
  trivia_after : ArrayView[Trivia]
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

A token with the trivia around it. All trivia before a token is its trivia_before; trivia_after is empty except for the last token of a stream, which holds the trivia to the end of the text.

Token::to_source

source

fn Token::to_source(self : Token) -> String

trivia_before, the lexeme and trivia_after, as in the source.

TokenKind

source

pub(all) enum TokenKind {
  Name(String)
  UpName(String)
  DiscardName(String)
  Int(value~ : String, int_value~ : @moonbitlang/core/bigint.BigInt
BigInt)
  Float(value~ : String, float_value~ : Double)
  String(String)
  LeftParen
  RightParen
  LeftSquare
  RightSquare
  LeftBrace
  RightBrace
  Plus
  Minus
  Star
  Slash
  Less
  Greater
  LessEqual
  GreaterEqual
  Percent
  PlusDot
  MinusDot
  StarDot
  SlashDot
  LessDot
  GreaterDot
  LessEqualDot
  GreaterEqualDot
  Concatenate
  Colon
  Comma
  Hash
  Bang
  Equal
  EqualEqual
  NotEqual
  Vbar
  VbarVbar
  AmperAmper
  LtLt
  GtGt
  Pipe
  Dot
  RArrow
  LArrow
  DotDot
  At
  Keyword(Keyword)
  Error
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

The token kinds of the compiler's Token, without the four trivia kinds (CommentNormal, CommentDoc, CommentModule, NewLine), which are Trivia here, and without EndOfFile. Error is the text that a lexical error covers; the compiler stops there, this scanner goes on.

TokenStream

source

pub(all) struct TokenStream {
  tokens : ArrayView[Token]
  trivia : ArrayView[Trivia]
  diagnostics : ArrayView[@moonrockz/gleam-parser/diagnostic.Diagnostic
Diagnostic]
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

The result of a scan. trivia is the trivia of a text with no tokens and empty otherwise. diagnostics are the lexical errors in source order; each has one Error token in tokens.

TokenStream::has_errors

source

fn TokenStream::has_errors(self : TokenStream) -> Bool

Whether any diagnostic has severity Error.

TokenStream::to_source

source

fn TokenStream::to_source(self : TokenStream) -> String

The scanned text, byte for byte.

Trivia

source

pub(all) enum Trivia {
  Whitespace(String, @moonrockz/gleam-parser/span.SrcSpan
SrcSpan)
  Newline(String, @moonrockz/gleam-parser/span.SrcSpan
SrcSpan)
  Comment(Comment)
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Text between tokens. Whitespace is a run of spaces, tabs and form feeds, with the byte order mark when the text starts with one (a byte order mark anywhere else is an unknown character, as in the compiler); Newline is one \n, \r\n or \r; a Comment runs to the end of its line.

Trivia::span

source

fn Trivia::span(self : Trivia) -> @moonrockz/gleam-parser/span.SrcSpan
SrcSpan

Trivia::text

source

fn Trivia::text(self : Trivia) -> String

The source text of the trivia.

UnicodeEscapeError

source

pub(all) enum UnicodeEscapeError {
  MissingOpeningBrace
  ExpectedHexDigitOrCloseBrace
  InvalidNumberOfHexDigits
  InvalidCodepoint
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

Why a \u{...} escape is wrong (InvalidUnicodeEscapeError in the compiler's parse/error.rs).

lexical_diagnostic

source

fn lexical_diagnostic(error : LexicalErrorType, location : @moonrockz/gleam-parser/span.SrcSpan
SrcSpan) -> @moonrockz/gleam-parser/diagnostic.Diagnostic
Diagnostic

The diagnostic for a lexical error at location, in the compiler's shape: title "Syntax error", the label as the message, the compiler's extra lines (to_parse_error_info) as the text. The compiler puts its hints for lexical errors in those lines, so hint is None.

parse_float_value

source

fn parse_float_value(value : String) -> Double?

The number a Float literal denotes (the compiler's LiteralFloatValue::parse, Rust f64::from_str after removing _): 1., 1.e3, 1.0e-3 are numbers; a literal too big is infinity. None when the text is not a number.

parse_int_value

source

fn parse_int_value(value : String) -> @moonbitlang/core/bigint.BigInt
BigInt?

The number an Int literal denotes (the compiler's parse_int_value): an optional -, then 0x, 0o or 0b for radix 16, 8 or 2, then digits with _ separators. None when a character does not fit.

test {
  debug_inspect(@scanner.parse_int_value("-0x1_F"), content="Some(-31)")
  debug_inspect(@scanner.parse_int_value("0x"), content="None")
}

tokenize

source

fn tokenize(source : SourceText) -> TokenStream

Scan Gleam source into tokens with trivia and lexical diagnostics. The scan is lossless: TokenStream::to_source() is the text. A lexical error is a GL-SCAN-001 diagnostic with the compiler's message and one Error token over the text it covers; scanning goes on after it.

test {
  let s = @scanner.tokenize(@scanner.SourceText::new("let x = 1"))
  debug_inspect(
    s.tokens.iter().map(t => t.kind).collect(),
    content="[Keyword(Let), Name(\"x\"), Equal, Int(value=\"1\", int_value=1)]",
  )
  debug_inspect(s.to_source(), content="\"let x = 1\"")
}

tokenize_text

source

fn tokenize_text(text : String) -> TokenStream

tokenize of a SourceText built from text.

wrap

source

fn wrap(text : String) -> String

The compiler's wrap (error.rs): each line longer than 75 bytes is broken at spaces; a word longer than the width is cut into pieces of the width.

Source Files

Site source Report issue Packages Build queue Skills Statistics

#Comment

#Comment::content_span

#CommentKind

#CommentKind::prefix_length

#Keyword

#Keyword::of

#Keyword::text

#LexicalErrorType

#LexicalErrorType::hints

#LexicalErrorType::kind

#LexicalErrorType::label

#SourceText

#SourceText::byte_length

#SourceText::byte_offset_of

#SourceText::line_count

#SourceText::new

#SourceText::position_of

#SourceText::slice

#SourceText::text

#SourceText::utf16_offset_of

#Token

#Token::to_source

#TokenKind

#TokenStream

#TokenStream::has_errors

#TokenStream::to_source

#Trivia

#Trivia::span

#Trivia::text

#UnicodeEscapeError

#lexical_diagnostic

#parse_float_value

#parse_int_value

#tokenize

#tokenize_text

#wrap

Source Files

Comment

Comment::content_span

CommentKind

CommentKind::prefix_length

Keyword

Keyword::of

Keyword::text

LexicalErrorType

LexicalErrorType::hints

LexicalErrorType::kind

LexicalErrorType::label

SourceText

SourceText::byte_length

SourceText::byte_offset_of

SourceText::line_count

SourceText::new

SourceText::position_of

SourceText::slice

SourceText::text

SourceText::utf16_offset_of

Token

Token::to_source

TokenKind

TokenStream

TokenStream::has_errors

TokenStream::to_source

Trivia

Trivia::span

Trivia::text

UnicodeEscapeError

lexical_diagnostic

parse_float_value

parse_int_value

tokenize

tokenize_text

wrap