#syntax

#The line-at-a-time contract

flowchart LR
  S0(["initial_state()"]) --> L1
  L1["line 1 text"] --> T1["tokenize_line"]
  T1 -->|"LineToken[]"| O1["offsets into line 1"]
  T1 -->|"TokenizerState"| T2
  L2["line 2 text"] --> T2["tokenize_line"]
  T2 -->|"LineToken[]"| O2["offsets into line 2"]
  T2 -->|"TokenizerState"| T3["line 3 ..."]

• LineTokenizer::tokenize_line receives one line plus TokenizerState and returns LineToken[] plus the state for the next line. Token offsets are UTF-16 code units; gaps are rendered as plain text.

///|
priv struct KeywordOnly {}

///|
impl @syntax.LineTokenizer for KeywordOnly with fn initial_state(_self) {
  TokenizerState()
}

///|
impl @syntax.LineTokenizer for KeywordOnly with fn tokenize_line(
  _self,
  line_text,
  state,
) {
  let tokens : Array[@syntax.LineToken] = []
  // Only "let" is claimed; every other column is left as a gap.
  if line_text.has_prefix("let") {
    tokens.push({ start: 0, end: 3, tag: Keyword, })
  }
  (tokens, state)
}

///|
test "unclaimed columns stay as gaps rather than forcing a Plain token" {
  let tokenizer : &@syntax.LineTokenizer = KeywordOnly::{ }
  let (tokens, _) = tokenizer.tokenize_line(
    "let x = 1",
    tokenizer.initial_state(),
  )
  debug_inspect(
    tokens,
    content=(
      #|[{ start: 0, end: 3, tag: Keyword }]
    ),
  )
}

#The registry

• TokenizationRegistry registers tokenizers by language id and emits TokenizationChangedEvent { changed_languages, changed_color_map }. Registration, replacement, explicit handle_change, and active removal emit exact language-id arrays with changed_color_map=false; disposing a stale registration is inert. get returns the current synchronous support and is_resolved is always true because this package has no lazy factory state.

///|
test "registration and removal emit exact language-id arrays" {
  let registry = @syntax.TokenizationRegistry()
  let events = []
  registry.on_did_change(event => {
    events.push((event.changed_languages, event.changed_color_map))
  })
  |> ignore
  let registration = registry.register("json", KeywordOnly::{ })
  let found = registry.get("json") is Some(_)
  let missing = registry.get("moonbit") is Some(_)
  registration.dispose()
  let after_removal = registry.get("json") is Some(_)
  // Disposing an already-removed registration is inert.
  registration.dispose()
  debug_inspect(
    (events, found, missing, after_removal),
    content=(
      #|([(["json"], false), (["json"], false)], true, false, false)
    ),
  )
}

///|
test "is_resolved is unconditionally true" {
  let registry = @syntax.TokenizationRegistry()
  registry.register("json", KeywordOnly::{ }) |> ignore
  debug_inspect(
    (registry.is_resolved("json"), registry.is_resolved("never-registered")),
    content=(
      #|(true, true)
    ),
  )
}

///|
test "the color map is retained verbatim and index 2 is the default background" {
  let registry = @syntax.TokenizationRegistry()
  let before = (registry.get_color_map(), registry.get_default_background())
  registry.register("json", KeywordOnly::{ }) |> ignore
  let events = []
  registry.on_did_change(event => {
    events.push((event.changed_languages, event.changed_color_map))
  })
  |> ignore
  registry.set_color_map(["#000000", "#ffffff", "var(--editor-bg)"])
  let short = @syntax.TokenizationRegistry()
  short.set_color_map(["#000000", "#ffffff"])
  debug_inspect(
    (
      before,
      events,
      registry.get_color_map(),
      registry.get_default_background(),
      short.get_default_background(),
    ),
    content=(
      #|(
      #|  (None, None),
      #|  [(["json"], true)],
      #|  Some(["#000000", "#ffffff", "var(--editor-bg)"]),
      #|  Some("var(--editor-bg)"),
      #|  None,
      #|)
    ),
  )
}

///|
test "handle_change forwards the exact array it was given" {
  let registry = @syntax.TokenizationRegistry()
  let events = []
  registry.on_did_change(event => {
    events.push((event.changed_languages, event.changed_color_map))
  })
  |> ignore
  registry.handle_change(["moonbit", "json"])
  debug_inspect(
    events,
    content=(
      #|[(["moonbit", "json"], false)]
    ),
  )
}

#Shared lexer helpers

///|
test "push_token coalesces contiguous same-tag spans" {
  let tokens : Array[@syntax.LineToken] = []
  @syntax.push_token(tokens, 0, 3, Identifier)
  // Contiguous and same tag: merged into the previous token.
  @syntax.push_token(tokens, 3, 7, Identifier)
  // Same tag but a gap at offset 7..8: kept separate.
  @syntax.push_token(tokens, 8, 9, Identifier)
  // Contiguous but a different tag: kept separate.
  @syntax.push_token(tokens, 9, 10, Operator)
  debug_inspect(
    tokens,
    content=(
      #|[
      #|  { start: 0, end: 7, tag: Identifier },
      #|  { start: 8, end: 9, tag: Identifier },
      #|  { start: 9, end: 10, tag: Operator },
      #|]
    ),
  )
}

///|
test "is_capitalized is an ASCII-only first-character heuristic" {
  debug_inspect(
    (
      @syntax.is_capitalized("Array"),
      @syntax.is_capitalized("array"),
      @syntax.is_capitalized("_Array"),
      @syntax.is_capitalized("Ärray"),
    ),
    content=(
      #|(true, false, false, false)
    ),
  )
}

///|
test "panic is_capitalized rejects an empty view" {
  @syntax.is_capitalized("") |> ignore
}

///|
test "the empty state is normal and stack edits preserve snapshots" {
  let normal = @syntax.TokenizerState()
  let template = normal.push_mode(b't')
  let interpolation = template.push_mode(b'i')
  assert_true(normal.last_mode() is None)
  assert_true(template.last_mode() == Some(b't'))
  assert_true(interpolation.last_mode() == Some(b'i'))
  assert_true(interpolation.pop_mode() == template)
  assert_true(template.pop_mode() == normal)
}

#Writing a new lang_*

• Use lexmatch ... with longest; equal-length matches choose the earliest arm. For a nontrivial keyword set, match one complete identifier and classify its spelling afterward. This keeps keyword-prefixed names intact without expanding every keyword into the generated DFA. Keep syntactic roles, such as whether a JSON string is a property name, out of the lexer.
• Carry multiline modes in TokenizerState; use scoped (?i:...) for case-insensitive rules. Dynamic delimiters/backreferences require a small manual scan because a DFA cannot encode them.
• Regex literals use strict single-backslash escapes. Write literal braces as [{]/[}]; escape - and ] inside classes. Single-character bindings are Char, longer bindings are StringView.
• Match re"^$" explicitly to end the loop, then guarantee progress with a final re"^." arm. Its capture is a Char; advance by Char::utf16_len() so an emitted token never splits a surrogate pair. Longer captures are StringViews, whose length() is already measured in UTF-16 code units.

#Boundaries and checks

moon test --target js syntax
moon test --target js syntax/lang_moonbit
moon test --target js syntax/lang_moon_config