#viewer/common/tokens

#Public boundary

classDiagram
  direction TB

  class TokenMetadata {
    <<typed scalar>>
    +TokenMetadata(raw) TokenMetadata
    +to_uint() UInt
    +get_class_name() String
  }

  class LineTokens {
    <<opaque>>
    +LineTokens(words, text, codec) LineTokens
    +convert_to_end_offset(words, text_length)
    +default_token_metadata() TokenMetadata
    +get_count() Int
    +get_line_content() String
    +get_metadata(index) TokenMetadata
    +get_start_offset(index) Int
    +inflate() ViewLineTokens
    +slice_and_inflate(start, end, delta) ViewLineTokens
    +with_inserted(tokens) LineTokens
  }

  class ViewLineTokens {
    <<opaque full-or-sliced view>>
    +get_count() Int
    +get_line_content() String
    +get_end_offset(index) Int
    +get_class_name(index) String
    +get_inline_style(index, color_map) String
    +get_token_text(index) String
  }

  class InsertedToken {
    <<opaque>>
    +InsertedToken(offset, text, metadata) InsertedToken
  }

  class ContiguousMultilineTokensBuilder {
    <<opaque>>
    +ContiguousMultilineTokensBuilder()
    +add(line_number, words)
    +finalize() Array~ContiguousMultilineTokens~
  }

  class ContiguousMultilineTokens {
    <<opaque batch>>
  }

  class ContiguousTokensStore {
    <<opaque>>
    +ContiguousTokensStore(codec)
    +get_tokens(language_id, line_index, text) LineTokens
    +set_multiline_tokens(batches, language_id, get_length) ContiguousTokensStoreUpdate
    +flush()
  }

  class ContiguousTokensStoreUpdate {
    +changes Array~ContiguousTokensStoreChange~
  }

  class ContiguousTokensStoreChange {
    +from_line_number Int
    +to_line_number Int
  }

  LineTokens ..> TokenMetadata : decodes metadata with
  LineTokens ..> InsertedToken : splices
  LineTokens --> ViewLineTokens : creates
  ContiguousMultilineTokensBuilder --> ContiguousMultilineTokens : builds
  ContiguousTokensStore --> ContiguousMultilineTokens : applies
  ContiguousTokensStore --> LineTokens : reads
  ContiguousTokensStore --> ContiguousTokensStoreUpdate : reports
  ContiguousTokensStoreUpdate *-- ContiguousTokensStoreChange

#Packed metadata

bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL

///|
test "a production-shaped metadata word resolves to its token class" {
  let metadata = (2U << @tokens.METADATA_LANGUAGEID_OFFSET) |
    (
      @tokens.STANDARD_TOKEN_TYPE_COMMENT.reinterpret_as_uint() <<
      @tokens.METADATA_TOKEN_TYPE_OFFSET
    ) |
    (
      @tokens.FONT_STYLE_NONE.reinterpret_as_uint() <<
      @tokens.METADATA_FONT_STYLE_OFFSET
    ) |
    (7U << @tokens.METADATA_FOREGROUND_OFFSET) |
    (
      @tokens.COLOR_ID_DEFAULT_BACKGROUND.reinterpret_as_uint() <<
      @tokens.METADATA_BACKGROUND_OFFSET
    )
  inspect(@tokens.TokenMetadata(metadata).get_class_name(), content="mtk7")
}

#Line tokens and views

[end_0, metadata_0, end_1, metadata_1, ...]

///|
test "line tokens expose their consumer-facing full view" {
  let codec = @services.language_id_codec
  let keyword = 3U << @tokens.METADATA_FOREGROUND_OFFSET
  let plain = 1U << @tokens.METADATA_FOREGROUND_OFFSET
  let line = @tokens.LineTokens(
    [3U, keyword, 4U, plain, 5U, keyword],
    "let x",
    codec,
  )
  assert_true(Repr(line).to_string().contains("tokens_count: 3"))
  let view = line.inflate()
  debug_inspect(
    (
      line.get_count(),
      line.get_line_content(),
      [
        for i in 0..<view.get_count() => {
          (
            line.get_start_offset(i),
            view.get_end_offset(i),
            view.get_token_text(i),
            view.get_class_name(i),
          )
        }
      ],
    ),
    content=(
      #|(
      #|  3,
      #|  "let x",
      #|  [(0, 3, "let", "mtk3"), (3, 4, " ", "mtk1"), (4, 5, "x", "mtk3")],
      #|)
    ),
  )
}

///|
test "a sliced view exposes segment-local offsets" {
  let codec = @services.language_id_codec
  let line = @tokens.LineTokens([5U, 1U, 6U, 0U, 11U, 2U], "hello world", codec)
  let view = line.slice_and_inflate(6, 11, 2)
  debug_inspect(
    (view.get_line_content(), view.get_end_offset(0), view.get_token_text(0)),
    content=(
      #|("world", 7, "world")
    ),
  )
}

///|
test "injected text splices a new token" {
  let codec = @services.language_id_codec
  let line = @tokens.LineTokens([5U, 1U], "value", codec)
  let with_hint = line.with_inserted([
    InsertedToken(5, " : Int", TokenMetadata(3U)),
  ])
  let view = with_hint.inflate()
  debug_inspect(
    (
      line.get_line_content(),
      with_hint.get_line_content(),
      [
        for i in 0..<view.get_count() => view.get_token_text(i)
      ],
    ),
    content=(
      #|("value", "value : Int", ["value", " : Int"])
    ),
  )
}

#Per-model store

///|
test "contiguous batches apply and report changed lines" {
  let codec = @services.language_id_codec
  let builder = @tokens.ContiguousMultilineTokensBuilder()
  builder.add(1, [4U, 1U])
  builder.add(2, [4U, 2U])
  let batches = builder.finalize()
  let store = @tokens.ContiguousTokensStore(codec)
  let update = store.set_multiline_tokens(batches, "moonbit", _ => 4)
  debug_inspect(
    (
      batches.length(),
      update.changes.map(change => {
        (change.from_line_number, change.to_line_number)
      }),
      store.get_tokens("moonbit", 0, "abcd").get_metadata(0).to_uint(),
    ),
    content=(
      #|(1, [(1, 2)], 1)
    ),
  )
}

///|
test "flush restores passive fallback reads" {
  let codec = @services.language_id_codec
  let builder = @tokens.ContiguousMultilineTokensBuilder()
  builder.add(1, [4U, 1U])
  let store = @tokens.ContiguousTokensStore(codec)
  store.set_multiline_tokens(builder.finalize(), "moonbit", _ => 4) |> ignore
  let before = store.get_tokens("moonbit", 0, "abcd").get_metadata(0)
  store.flush()
  let after = store.get_tokens("moonbit", 0, "abcd").get_metadata(0)
  assert_true(before != after)
}

#UTF-16 boundary

///|
test "a slice may split a surrogate pair" {
  let codec = @services.language_id_codec
  let line = @tokens.LineTokens([2U, 1U, 3U, 2U], "😀x", codec)
  let split = line.slice_and_inflate(1, 3, 0)
  debug_inspect(
    (line.get_line_content().length(), split.get_line_content().length()),
    content=(
      #|(3, 2)
    ),
  )
}

#Boundaries and checks

moon test --target js viewer/common/tokens