moonbit-community/treesitter/language does not have a README file

    CompactError

    pub(all) suberror CompactError {
    CompactError(String)
    } derive(
    Debug
    )

    A malformed or unsupported compact language.

    CharacterRange

    pub(all) struct CharacterRange {
    start : Int
    end : Int
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    An inclusive code point range, as in TSCharacterRange.

    ExternalScannerData

    pub(all) struct ExternalScannerData {
    states : Array[Bool]
    symbol_map : Array[Int]
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    FieldMapEntry

    pub(all) struct FieldMapEntry {
    field_id : Int
    child_index : Int
    inherited : Bool
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    Language

    pub(all) struct Language {
    abi_version : Int
    symbol_count : Int
    alias_count : Int
    token_count : Int
    external_token_count : Int
    state_count : Int
    large_state_count : Int
    production_id_count : Int
    field_count : Int
    max_alias_sequence_length : Int
    parse_table : Array[Int]
    small_parse_table : Array[Int]
    small_parse_table_map : Array[Int]
    parse_actions : Array[ParseActionEntry]
    symbol_names : Array[String]
    field_names : Array[String]
    field_map_slices : Array[MapSlice]
    field_map_entries : Array[FieldMapEntry]
    symbol_metadata : Array[SymbolMetadata]
    public_symbol_map : Array[Int]
    alias_map : Array[Int]
    alias_sequences : Array[Int]
    lex_modes : Array[LexMode]
    lex_fn : LexProgram
    keyword_lex_fn : LexProgram?
    keyword_capture_token : Int
    external_scanner : ExternalScannerData
    primary_state_ids : Array[Int]
    name : String
    reserved_words : Array[Int]
    max_reserved_word_set_size : Int
    supertype_count : Int
    supertype_symbols : Array[Int]
    supertype_map_slices : Array[MapSlice]
    supertype_map_entries : Array[Int]
    metadata : LanguageMetadata
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    Field-for-field equivalent of TSLanguage.

    Language::actions

    fn Language::actions(self : Language, state : Int, symbol : Int) -> ArrayView[ParseActionEntry]

    ts_language_actions.

    Language::actions_of

    fn Language::actions_of(self : Language, entry : TableEntry) -> ArrayView[ParseActionEntry]

    Language::alias_at

    fn Language::alias_at(self : Language, production_id : Int, child_index : Int) -> Int

    Language::alias_sequence

    fn Language::alias_sequence(self : Language, production_id : Int) -> Int?

    ts_language_alias_sequence: the start offset into alias_sequences, or None for production id 0.

    Language::alias_sequence_value

    fn Language::alias_sequence_value(self : Language, index : Int) -> Int

    alias_sequences[index] with C's flat-array semantics: indexes past a row read the next row; past the end of the array this returns 0.

    Language::aliases_for_symbol

    fn Language::aliases_for_symbol(self : Language, original_symbol : Int) -> ArrayView[Int]

    ts_language_aliases_for_symbol: the public symbols a symbol may appear as.

    Language::enabled_external_tokens

    fn Language::enabled_external_tokens(self : Language, external_scanner_state : Int) -> ArrayView[Bool]?

    ts_language_enabled_external_tokens: the offset of the enabled-token row in external_scanner.states, or None for external scanner state 0.

    Language::field_id_for_name

    fn Language::field_id_for_name(self : Language, name : StringView) -> Int

    ts_language_field_id_for_name. Field names are sorted, so the search stops at the first name that sorts after name (upstream's strncmp returning -1).

    Language::field_map

    fn Language::field_map(self : Language, production_id : Int) -> (Int, Int)

    ts_language_field_map: [start, end) indices into field_map_entries.

    Language::field_name_for_id

    fn Language::field_name_for_id(self : Language, id : Int) -> String?

    ts_language_field_name_for_id. field_names[0] is NULL in C (stored as "" here), so id 0 has no name.

    Language::from_compact

    fn Language::from_compact(data : BytesView) -> Language raise CompactError

    Decode a language serialized by Language::to_compact.

    Language::from_compact_base64

    fn Language::from_compact_base64(text : StringView) -> Language raise CompactError

    Decode the base64 text written by Language::to_compact_base64; ASCII whitespace (for example the line breaks of a multi-line string literal) is ignored.

    Language::has_actions

    fn Language::has_actions(self : Language, state : Int, symbol : Int) -> Bool

    Language::has_reduce_action

    fn Language::has_reduce_action(self : Language, state : Int, symbol : Int) -> Bool

    Language::is_reserved_word

    fn Language::is_reserved_word(self : Language, state : Int, symbol : Int) -> Bool

    Language::language_metadata

    fn Language::language_metadata(self : Language) -> LanguageMetadata?

    ts_language_metadata.

    Language::language_name

    fn Language::language_name(self : Language) -> String?

    ts_language_name.

    Language::lex_mode_for_state

    fn Language::lex_mode_for_state(self : Language, state : Int) -> LexMode

    ts_language_lex_mode_for_state. ABI 14 tables are converted to the ABI 15 layout when loaded, so this is a direct lookup.

    Language::lookaheads

    fn Language::lookaheads(self : Language, state : Int) -> LookaheadIterator

    ts_language_lookaheads.

    Language::lookup

    fn Language::lookup(self : Language, state : Int, symbol : Int) -> Int

    Lookup the table value for a given symbol and state.

    For non-terminal symbols, the table value represents a successor state. For terminal symbols, it represents an index in the actions table. For 'large' parse states, this is a direct lookup. For 'small' parse states, this requires searching through the symbol groups to find the given symbol.

    Language::next_state

    fn Language::next_state(self : Language, state : Int, symbol : Int) -> Int

    ts_language_next_state.

    Language::public_symbol

    fn Language::public_symbol(self : Language, symbol : Int) -> Int

    ts_language_public_symbol.

    Language::state_is_primary

    fn Language::state_is_primary(self : Language, state : Int) -> Bool

    Whether the state is a "primary state". If this returns false, another state behaves identically to this one with respect to query analysis.

    Language::subtypes

    fn Language::subtypes(self : Language, supertype : Int) -> ArrayView[Int]

    ts_language_subtypes.

    Language::supertypes

    fn Language::supertypes(self : Language) -> ArrayView[Int]

    ts_language_supertypes.

    Language::symbol_count_with_aliases

    fn Language::symbol_count_with_aliases(self : Language) -> Int

    Language::symbol_for_name

    fn Language::symbol_for_name(self : Language, string : StringView, is_named : Bool) -> Int

    ts_language_symbol_for_name.

    Language::symbol_metadata_of

    fn Language::symbol_metadata_of(self : Language, symbol : Int) -> SymbolMetadata

    ts_language_symbol_metadata.

    Language::symbol_name

    fn Language::symbol_name(self : Language, symbol : Int) -> String?

    ts_language_symbol_name.

    Language::symbol_type

    fn Language::symbol_type(self : Language, symbol : Int) -> SymbolType

    ts_language_symbol_type.

    Language::table_entry

    fn Language::table_entry(self : Language, state : Int, symbol : Int) -> TableEntry

    Language::to_compact

    fn Language::to_compact(self : Language) -> Bytes raise CompactError

    Serialize this language in the compact binary format (layer 2: version byte, raw length, LZ77-compressed raw stream). Raises if a field that the format stores unsigned is negative, which a generated language never has.

    Language::to_compact_base64

    fn Language::to_compact_base64(self : Language) -> String raise CompactError

    to_compact as base64 text (layer 1), on a single line.

    LanguageMetadata

    pub(all) struct LanguageMetadata {
    major_version : Int
    minor_version : Int
    patch_version : Int
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    LexCondition

    pub(all) struct LexCondition {
    large_set : (Int, Bool)?
    asserted : RangeCondition?
    negated : RangeCondition?
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    The condition of one transition line: if ((large || asserted) && negated) ADVANCE/SKIP(state). A missing part is omitted from the C expression: the condition is the conjunction of (large || asserted) (if either is present) and negated (if present), so a transition with no part at all is unconditional. The parts have no side effects, so evaluation order does not matter.

    LexMode

    pub(all) struct LexMode {
    lex_state : Int
    external_lex_state : Int
    reserved_word_set_id : Int
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    TSLexerMode (TSLexMode for ABI 14, without reserved_word_set_id). The (TSStateId)(-1) mode of a state that ends a non-terminal extra is { lex_state: 65535, external_lex_state: 0, reserved_word_set_id: 0 }.

    LexProgram

    pub(all) struct LexProgram {
    states : Array[LexStateCode]
    large_character_sets : Array[Array[CharacterRange]]
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    A generated lex function (ts_lex or ts_lex_keywords).

    lex(lexer, state) -> Bool is defined as the emitted C, expanded with the macros of parser.h (START_LEXER, ACCEPT_TOKEN, ADVANCE, ADVANCE_MAP, SKIP, END_STATE):

    result = false // START_LEXER() loop { skip = false // start: lookahead = lexer.lookahead eof = lexer.eof() // eof = lexer->eof(lexer); if state >= states.length() { return false // default: return false; } code = states[state] // case state: if code.accept_token is Some(symbol) { // ACCEPT_TOKEN(symbol) result = true lexer.result_symbol = symbol lexer.mark_end() } if eof && code.eof_advance is Some(s) { // if (eof) ADVANCE(s); next = s } else if code.advance_map has a first (c, s) with c == lookahead { next = s // ADVANCE_MAP(...) } else if code.transitions has a first t whose condition holds { next = t.state // if (...) ADVANCE/SKIP skip = t.skip } else { return result // END_STATE() } state = next // goto next_state; lexer.advance(skip) // next_state: }

    lookahead is the current code point (C int32_t; 0 at end of input).

    LexStateCode

    pub(all) struct LexStateCode {
    accept_token : Int?
    eof_advance : Int?
    advance_map : Array[(Int, Int)]
    transitions : Array[LexTransition]
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    One case of the generated lex function, in emission order: ACCEPT_TOKEN, if (eof) ADVANCE, ADVANCE_MAP, then the transitions.

    LexTransition

    pub(all) struct LexTransition {
    condition : LexCondition
    state : Int
    skip : Bool
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    LookaheadIterator

    pub struct LookaheadIterator {
    // private fields
    }

    Iterate over all of the symbols that are valid in the given state (LookaheadIterator).

    LookaheadIterator::actions

    LookaheadIterator::current_symbol

    fn LookaheadIterator::current_symbol(self : LookaheadIterator) -> Int

    LookaheadIterator::current_symbol_name

    fn LookaheadIterator::current_symbol_name(self : LookaheadIterator) -> String?

    ts_lookahead_iterator_current_symbol_name.

    LookaheadIterator::language

    LookaheadIterator::new

    fn LookaheadIterator::new(language : Language, state : Int) -> LookaheadIterator?

    ts_lookahead_iterator_new.

    LookaheadIterator::next

    fn LookaheadIterator::next(self : LookaheadIterator) -> Bool

    ts_lookahead_iterator__next.

    LookaheadIterator::next_state

    fn LookaheadIterator::next_state(self : LookaheadIterator) -> Int

    LookaheadIterator::reset

    fn LookaheadIterator::reset(self : LookaheadIterator, language : Language, state : Int) -> Bool

    ts_lookahead_iterator_reset.

    LookaheadIterator::reset_state

    fn LookaheadIterator::reset_state(self : LookaheadIterator, state : Int) -> Bool

    ts_lookahead_iterator_reset_state.

    MapSlice

    pub(all) struct MapSlice {
    index : Int
    length : Int
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    TSMapSlice: used to index the field and supertype maps.

    ParseActionEntry

    pub(all) enum ParseActionEntry {
    Entry(count~ : Int, reusable~ : Bool)
    Shift(state~ : Int, extra~ : Bool, repetition~ : Bool)
    Reduce(symbol~ : Int, child_count~ : Int, dynamic_precedence~ : Int, production_id~ : Int)
    Accept
    Recover
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    TSParseAction plus the entry header of TSParseActionEntry.

    ParseActionEntry::normalized

    Truncate an action's fields to the widths of the C TSParseActionEntry union (uint8_t counts and child counts, int16_t dynamic precedence, uint16_t states, symbols and production ids), as compiling parser.c does.

    RangeCondition

    pub(all) struct RangeCondition {
    ranges : Array[CharacterRange]
    is_included : Bool
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    One add_character_range_conditions call in render.rs, with the ranges exactly as passed to it (sorted, disjoint, inclusive).

    Evaluation, with lookahead and eof as in LexProgram:
    • is_included: true iff some range matches, where a range starting at 0 matches iff !eof && lookahead <= end (C: (!eof && lookahead <= end), or (!eof && lookahead == 0) when end is 0), and any other range matches iff start <= lookahead && lookahead <= end.
    • !is_included: true iff no range contains lookahead (C: a conjunction of lookahead != c, (lookahead < start || end < lookahead) and lookahead > end tests). eof plays no part.

    An asserted condition whose characters include U+10FFFF is emitted in the negated form: its ranges are the complement of those characters plus '\0', with is_included false.

    SymbolMetadata

    pub(all) struct SymbolMetadata {
    visible : Bool
    named : Bool
    supertype : Bool
    } derive(Eq, ToJson,
    Debug
    ,
    FromJson
    )

    SymbolType

    pub(all) enum SymbolType {
    Regular
    Anonymous
    Supertype
    Auxiliary
    } derive(Eq,
    Debug
    )

    TableEntry

    pub(all) struct TableEntry {
    actions_start : Int
    action_count : Int
    is_reusable : Bool
    } derive(Eq,
    Debug
    )

    TableEntry: the actions for a state/symbol pair are parse_actions[actions_start:actions_start + action_count].

    compact_format_version

    let compact_format_version : Int

    The version of the compact format written by Language::to_compact.

    language_version

    let language_version : Int

    language_version_with_primary_states

    let language_version_with_primary_states : Int

    language_version_with_reserved_words

    let language_version_with_reserved_words : Int

    tree_sitter_serialization_buffer_size

    let tree_sitter_serialization_buffer_size : Int

    ts_builtin_sym_end

    let ts_builtin_sym_end : Int

    ts_builtin_sym_error

    let ts_builtin_sym_error : Int

    ts_builtin_sym_error_repeat

    let ts_builtin_sym_error_repeat : Int