bobzhang/pygments/token

bobzhang/pygments/token does not have a README file

TokenType

pub struct TokenType {
  // private fields
} derive(Compare, Eq, Hash)

A token type: a node in the open hierarchy rooted at token (Token.Name.Builtin, ...). Token types are interned, so equality is identity.

impl Show for TokenType

impl Debug for TokenType

TokenType::compare

source

fn TokenType::compare(TokenType, TokenType) -> Int

TokenType::equal

source

fn TokenType::equal(TokenType, TokenType) -> Bool

TokenType::hash

source

fn TokenType::hash(self : TokenType) -> Int

TokenType::hash_combine

source

fn TokenType::hash_combine(TokenType, Hasher) -> Unit

TokenType::is_subtype_of

source

fn TokenType::is_subtype_of(self : TokenType, other : TokenType) -> Bool

Whether self is other or one of its descendants (Python's self in other).

TokenType::last

source

fn TokenType::last(self : TokenType) -> String

The last path component ("Builtin" for Token.Name.Builtin); empty for the root.

TokenType::not_equal

source

fn TokenType::not_equal(x : TokenType, y : TokenType) -> Bool

TokenType::op_ge

source

fn TokenType::op_ge(x : TokenType, y : TokenType) -> Bool

TokenType::op_gt

source

fn TokenType::op_gt(x : TokenType, y : TokenType) -> Bool

TokenType::op_le

source

fn TokenType::op_le(x : TokenType, y : TokenType) -> Bool

TokenType::op_lt

source

fn TokenType::op_lt(x : TokenType, y : TokenType) -> Bool

TokenType::output

source

#deprecated("`TokenType::output` is deprecated, use `Show::output` instead.")
fn TokenType::output(self : TokenType, logger : &Logger) -> Unit

TokenType::parent

source

fn TokenType::parent(self : TokenType) -> TokenType?

The parent token type, or None for the root.

TokenType::path

source

fn TokenType::path(self : TokenType) -> Array[String]

Path components below the root, e.g. ["Name", "Builtin"].

TokenType::split

source

fn TokenType::split(self : TokenType) -> Array[TokenType]

The ancestors from the root down to self (Python's ttype.split()).

TokenType::sub

source

fn TokenType::sub(self : TokenType, name : String) -> TokenType

The child token type self.<name>, created on first use.

test {
  let t = @token.name.sub("Builtin").sub("Pseudo")
  inspect(t, content="Token.Name.Builtin.Pseudo")
  inspect(t == @token.name_builtin_pseudo, content="true")
}

TokenType::subtypes

source

fn TokenType::subtypes(self : TokenType) -> Array[TokenType]

Direct children created so far.

TokenType::to_repr

source

fn TokenType::to_repr(self : TokenType) -> @moonbitlang/core/debug.Repr
Repr

TokenType::to_string

source

fn TokenType::to_string(self : TokenType) -> String

Canonical name, e.g. Token.Literal.String.Double.

comment

source

let comment : TokenType

Token.Comment

comment_hashbang

source

let comment_hashbang : TokenType

Token.Comment.Hashbang

comment_multiline

source

let comment_multiline : TokenType

Token.Comment.Multiline

comment_preproc

source

let comment_preproc : TokenType

Token.Comment.Preproc

comment_preprocfile

source

let comment_preprocfile : TokenType

Token.Comment.PreprocFile

comment_single

source

let comment_single : TokenType

Token.Comment.Single

comment_special

source

let comment_special : TokenType

Token.Comment.Special

css_class

source

fn css_class(t : TokenType) -> String

Python's _get_ttype_class: the standard short name of the nearest standard ancestor followed by the remaining path components.

test {
  inspect(@token.css_class(@token.name_builtin.sub("Magic")), content="nbMagic")
}

error

source

let error : TokenType

Token.Error

escape

source

let escape : TokenType

Token.Escape

from_string

source

fn from_string(s : StringView) -> TokenType

Python's string_to_tokentype: "String.Double", "Token.Literal.String" and "" (the root) are accepted.

test {
  inspect(
    @token.from_string("String.Double"),
    content="Token.Literal.String.Double",
  )
  inspect(@token.from_string(""), content="Token")
}

generic

source

let generic : TokenType

Token.Generic

generic_deleted

source

let generic_deleted : TokenType

Token.Generic.Deleted

generic_emph

source

let generic_emph : TokenType

Token.Generic.Emph

generic_emphstrong

source

let generic_emphstrong : TokenType

Token.Generic.EmphStrong

generic_error

source

let generic_error : TokenType

Token.Generic.Error

generic_heading

source

let generic_heading : TokenType

Token.Generic.Heading

generic_inserted

source

let generic_inserted : TokenType

Token.Generic.Inserted

generic_output

source

let generic_output : TokenType

Token.Generic.Output

generic_prompt

source

let generic_prompt : TokenType

Token.Generic.Prompt

generic_strong

source

let generic_strong : TokenType

Token.Generic.Strong

generic_subheading

source

let generic_subheading : TokenType

Token.Generic.Subheading

generic_traceback

source

let generic_traceback : TokenType

Token.Generic.Traceback

keyword

source

let keyword : TokenType

Token.Keyword

keyword_constant

source

let keyword_constant : TokenType

Token.Keyword.Constant

keyword_declaration

source

let keyword_declaration : TokenType

Token.Keyword.Declaration

keyword_namespace

source

let keyword_namespace : TokenType

Token.Keyword.Namespace

keyword_pseudo

source

let keyword_pseudo : TokenType

Token.Keyword.Pseudo

keyword_reserved

source

let keyword_reserved : TokenType

Token.Keyword.Reserved

keyword_type

source

let keyword_type : TokenType

Token.Keyword.Type

literal

source

let literal : TokenType

Token.Literal

literal_date

source

let literal_date : TokenType

Token.Literal.Date

name

source

let name : TokenType

Token.Name

name_attribute

source

let name_attribute : TokenType

Token.Name.Attribute

name_builtin

source

let name_builtin : TokenType

Token.Name.Builtin

name_builtin_pseudo

source

let name_builtin_pseudo : TokenType

Token.Name.Builtin.Pseudo

name_class

source

let name_class : TokenType

Token.Name.Class

name_constant

source

let name_constant : TokenType

Token.Name.Constant

name_decorator

source

let name_decorator : TokenType

Token.Name.Decorator

name_entity

source

let name_entity : TokenType

Token.Name.Entity

name_exception

source

let name_exception : TokenType

Token.Name.Exception

name_function

source

let name_function : TokenType

Token.Name.Function

name_function_magic

source

let name_function_magic : TokenType

Token.Name.Function.Magic

name_label

source

let name_label : TokenType

Token.Name.Label

name_namespace

source

let name_namespace : TokenType

Token.Name.Namespace

name_other

source

let name_other : TokenType

Token.Name.Other

name_property

source

let name_property : TokenType

Token.Name.Property

name_tag

source

let name_tag : TokenType

Token.Name.Tag

name_variable

source

let name_variable : TokenType

Token.Name.Variable

name_variable_class

source

let name_variable_class : TokenType

Token.Name.Variable.Class

name_variable_global

source

let name_variable_global : TokenType

Token.Name.Variable.Global

name_variable_instance

source

let name_variable_instance : TokenType

Token.Name.Variable.Instance

name_variable_magic

source

let name_variable_magic : TokenType

Token.Name.Variable.Magic

number

source

let number : TokenType

Token.Literal.Number

number_bin

source

let number_bin : TokenType

Token.Literal.Number.Bin

number_float

source

let number_float : TokenType

Token.Literal.Number.Float

number_hex

source

let number_hex : TokenType

Token.Literal.Number.Hex

number_integer

source

let number_integer : TokenType

Token.Literal.Number.Integer

number_integer_long

source

let number_integer_long : TokenType

Token.Literal.Number.Integer.Long

number_oct

source

let number_oct : TokenType

Token.Literal.Number.Oct

operator

source

let operator : TokenType

Token.Operator

operator_word

source

let operator_word : TokenType

Token.Operator.Word

other

source

let other : TokenType

Token.Other

punctuation

source

let punctuation : TokenType

Token.Punctuation

punctuation_marker

source

let punctuation_marker : TokenType

Token.Punctuation.Marker

standard_names

source

fn standard_names() -> Map[TokenType, String]

Standard token types and their short CSS names (Python's STANDARD_TYPES).

standard_short_name

source

fn standard_short_name(t : TokenType) -> String?

The short CSS class name of a standard token type ("kc" for Keyword.Constant), if t is standard.

string

source

let string : TokenType

Token.Literal.String

string_affix

source

let string_affix : TokenType

Token.Literal.String.Affix

string_backtick

source

let string_backtick : TokenType

Token.Literal.String.Backtick

string_char

source

let string_char : TokenType

Token.Literal.String.Char

string_delimiter

source

let string_delimiter : TokenType

Token.Literal.String.Delimiter

string_doc

source

let string_doc : TokenType

Token.Literal.String.Doc

string_double

source

let string_double : TokenType

Token.Literal.String.Double

string_escape

source

let string_escape : TokenType

Token.Literal.String.Escape

string_heredoc

source

let string_heredoc : TokenType

Token.Literal.String.Heredoc

string_interpol

source

let string_interpol : TokenType

Token.Literal.String.Interpol

string_other

source

let string_other : TokenType

Token.Literal.String.Other

string_regex

source

let string_regex : TokenType

Token.Literal.String.Regex

string_single

source

let string_single : TokenType

Token.Literal.String.Single

string_symbol

source

let string_symbol : TokenType

Token.Literal.String.Symbol

text

source

let text : TokenType

Token.Text

text_whitespace

source

let text_whitespace : TokenType

Token.Text.Whitespace

token

source

let token : TokenType

The root token type Token.

whitespace

source

let whitespace : TokenType

Token.Text.Whitespace (Python's Whitespace)

Source Files

token.mbt

Site source Report issue Packages Build queue Skills Statistics

#TokenType

#TokenType::compare

#TokenType::equal

#TokenType::hash

#TokenType::hash_combine

#TokenType::is_subtype_of

#TokenType::last

#TokenType::not_equal

#TokenType::op_ge

#TokenType::op_gt

#TokenType::op_le

#TokenType::op_lt

#TokenType::output

#TokenType::parent

#TokenType::path

#TokenType::split

#TokenType::sub

#TokenType::subtypes

#TokenType::to_repr

#TokenType::to_string

#comment

#comment_hashbang

#comment_multiline

#comment_preproc

#comment_preprocfile

#comment_single

#comment_special

#css_class

#error

#escape

#from_string

#generic

#generic_deleted

#generic_emph

#generic_emphstrong

#generic_error

#generic_heading

#generic_inserted

#generic_output

#generic_prompt

#generic_strong

#generic_subheading

#generic_traceback

#keyword

#keyword_constant

#keyword_declaration

#keyword_namespace

#keyword_pseudo

#keyword_reserved

#keyword_type

#literal

#literal_date

#name

#name_attribute

#name_builtin

#name_builtin_pseudo

#name_class

#name_constant

#name_decorator

#name_entity

#name_exception

#name_function

#name_function_magic

#name_label

#name_namespace

#name_other

#name_property

#name_tag

#name_variable

#name_variable_class

#name_variable_global

#name_variable_instance

#name_variable_magic

#number

#number_bin

#number_float

#number_hex

#number_integer

#number_integer_long

#number_oct

TokenType

TokenType::compare

TokenType::equal

TokenType::hash

TokenType::hash_combine

TokenType::is_subtype_of

TokenType::last

TokenType::not_equal

TokenType::op_ge

TokenType::op_gt

TokenType::op_le

TokenType::op_lt

TokenType::output

TokenType::parent

TokenType::path

TokenType::split

TokenType::sub

TokenType::subtypes

TokenType::to_repr

TokenType::to_string

comment

comment_hashbang

comment_multiline

comment_preproc

comment_preprocfile

comment_single

comment_special

css_class

error

escape

from_string

generic

generic_deleted

generic_emph

generic_emphstrong

generic_error

generic_heading

generic_inserted

generic_output

generic_prompt

generic_strong

generic_subheading

generic_traceback

keyword

keyword_constant

keyword_declaration

keyword_namespace

keyword_pseudo

keyword_reserved

keyword_type

literal

literal_date

name

name_attribute

name_builtin

name_builtin_pseudo

name_class

name_constant

name_decorator

name_entity

name_exception

name_function

name_function_magic

name_label

name_namespace

name_other

name_property

name_tag

name_variable

name_variable_class

name_variable_global

name_variable_instance

name_variable_magic

number

number_bin

number_float

number_hex

number_integer

number_integer_long

number_oct