bobzhang/pygments/lexer

bobzhang/pygments/lexer does not have a README file

CallbackFn

type CallbackFn = (Ctx, @bobzhang/pygments/regex.Match
Match, Array[CallbackArg]) -> Unit raise

A hand-written callback (Python: a callable token action).

Options

source

type Options = Map[String, String]

Lexer, formatter and filter options. As on the pygmentize command line, every value is a string; typed accessors mirror pygments.util.

Tokenizer

source

type Tokenizer = (Lexer, String, Array[String]) -> Array[Token] raise

Tokenizes preprocessed text starting from a state stack.

LexError

source

pub suberror LexError {
  MissingCallback(String)
  UnknownState(String)
} derive(@moonbitlang/core/debug.Debug
Debug)

Errors raised while lexing.

OptionError

source

pub suberror OptionError {
  OptionError(String)
} derive(@moonbitlang/core/debug.Debug
Debug)

Errors raised for invalid option values.

Action

source

pub(all) enum Action {
  Nop
  Emit(@bobzhang/pygments/token.TokenType
TokenType)
  ByGroups(Array[Action])
  Using(UsingTarget, Array[String]?, Array[(String, String)])
  Call(String, Array[CallbackArg])
}

What to do with a match (Python: the second element of a rule).

CallbackArg

source

pub(all) enum CallbackArg {
  Str(String)
  Tok(@bobzhang/pygments/token.TokenType
TokenType)
  Bool(Bool)
  Int(Int)
} derive(@moonbitlang/core/debug.Debug
Debug)

Arguments captured by a callback factory (e.g. the token type passed to YAML's save_indent(Text, start=True)).

Ctx

source

pub(all) struct Ctx {
  lexer : Lexer
  def : RegexDef
  text : String
  pos : Int
  end : Int
  stack : Array[Int]
  out : Array[Token]
  callbacks : Map[String, (Ctx, @bobzhang/pygments/regex.Match
Match, Array[CallbackArg]) -> Unit raise]
}

The lexing context handed to callbacks (Python's LexerContext plus the output buffer). Callbacks may move pos and end.

Ctx::apply

source

fn Ctx::apply(self : Ctx, action : Action, m : @bobzhang/pygments/regex.Match
Match) -> Unit raise

Applies action to m (Python: calling a token action).

Ctx::emit

source

fn Ctx::emit(self : Ctx, index : Int, ttype : @bobzhang/pygments/token.TokenType
TokenType, value : String) -> Unit

Appends a token to the output.

Ctx::emit_lexed

source

fn Ctx::emit_lexed(self : Ctx, lexer : Lexer, text : String, offset : Int, stack? : Array[String]) -> Unit raise

Lexes text with lexer and appends the tokens shifted by offset (Python: for i, t, v in lx.get_tokens_unprocessed(...): yield i + offset, t, v).

Ctx::is_extended

source

fn Ctx::is_extended(self : Ctx) -> Bool

Whether this context follows ExtendedRegexLexer semantics.

Ctx::new

source

fn Ctx::new(lexer : Lexer, def : RegexDef, text : String, callbacks? : Map[String, (Ctx, @bobzhang/pygments/regex.Match
Match, Array[CallbackArg]) -> Unit raise], pos? : Int, end? : Int, stack? : Array[String]) -> Ctx raise

Creates a context for run_context, e.g. to pre-set the position or stack.

Ctx::pop_state

source

fn Ctx::pop_state(self : Ctx) -> Unit

Pops a state, keeping at least one on the stack.

Ctx::push_state

source

fn Ctx::push_state(self : Ctx, name : String) -> Unit raise

Pushes the state name.

Ctx::set_state_stack

source

fn Ctx::set_state_stack(self : Ctx, names : Array[String]) -> Unit raise

Replaces the state stack.

Ctx::state_stack

source

fn Ctx::state_stack(self : Ctx) -> Array[String]

Names of the states on the stack, bottom first.

Ctx::top_state

source

fn Ctx::top_state(self : Ctx) -> String

Name of the current (top) state.

Filter

source

pub(all) struct Filter {
  name : String
  apply : (Lexer?, Array[(@bobzhang/pygments/token.TokenType
TokenType, String)]) -> Array[(@bobzhang/pygments/token.TokenType
TokenType, String)] raise
}

A token stream filter (see pygments.filters).

Lexer

source

pub struct Lexer {
  info : LexerInfo
  options : Map[String, String]
  stripnl : Bool
  stripall : Bool
  ensurenl : Bool
  tabsize : Int
  filters : Array[Filter]
  // private fields
}

An instance of a lexer class with its options.

Lexer::add_filter

source

fn Lexer::add_filter(self : Lexer, f : Filter) -> Unit

Appends a filter to this lexer.

Lexer::get_tokens

source

fn Lexer::get_tokens(self : Lexer, text : String, unfiltered? : Bool) -> Array[(@bobzhang/pygments/token.TokenType
TokenType, String)] raise

Python's Lexer.get_tokens: preprocesses text, tokenizes it and applies the lexer's filters (unless unfiltered).

Lexer::get_tokens_unprocessed

source

fn Lexer::get_tokens_unprocessed(self : Lexer, text : String, stack? : Array[String]) -> Array[Token] raise

Python's get_tokens_unprocessed: tokenizes text as is, starting with stack (default ["root"]). Offsets are UTF-16 offsets into text.

Lexer::name

source

fn Lexer::name(self : Lexer) -> String

A lexer's display name, e.g. Python.

Lexer::new

source

fn Lexer::new(info : LexerInfo, options : Map[String, String], create : (Map[String, String]) -> Lexer raise, tokenizer : (Lexer, String, Array[String]) -> Array[Token] raise) -> Lexer raise OptionError

Builds a lexer instance. create must construct a new instance of the same lexer class (used by using(this, ...)).

Lexer::preprocess

source

fn Lexer::preprocess(self : Lexer, text : String) -> String

Python's Lexer._preprocess_lexer_input for text input: drops a BOM, normalizes newlines and applies stripall/stripnl/tabsize/ensurenl.

Lexer::recreate

source

fn Lexer::recreate(self : Lexer, options : Map[String, String]) -> Lexer raise

Creates a new instance of the same lexer class with options.

LexerInfo

source

pub(all) struct LexerInfo {
  class_name : String
  name : String
  aliases : Array[String]
  filenames : Array[String]
  alias_filenames : Array[String]
  mimetypes : Array[String]
  priority : Double
  url : String
  version_added : String
  analyse_text : (String) -> Double
  doc : String
}

Static metadata of a lexer class (name, aliases, file patterns, ...).

RegexDef

source

pub struct RegexDef {
  name : String
  state_names : Array[String]
  extended : Bool
  // private fields
}

The processed token definitions of a RegexLexer class: named states, each an ordered list of rules (includes, inheritance, words() and combined() already resolved by the exporter).

RegexDef::new

source

fn RegexDef::new(name : String, state_names : Array[String], rules : Array[Rule], states : Array[Array[Int]], extended? : Bool) -> RegexDef

Builds a definition from rules and, per state, the indices of its rules.

RegexDef::state

source

fn RegexDef::state(self : RegexDef, name : String) -> Int

Index of the state name.

Rule

source

pub struct Rule {
  pattern : String
  flags : Int
  action : Action
  transition : Array[StateOp]?
  // private fields
}

A lexing rule: regex, action and optional transition.

Rule::new

source

fn Rule::new(pattern : String, flags : Int, action : Action, transition : Array[StateOp]?) -> Rule

Creates a rule. transition is None when the rule does not change the state stack.

StateOp

source

pub(all) enum StateOp {
  Push(String)
  Pop
  Dup
} derive(@moonbitlang/core/debug.Debug
Debug)

One state-stack operation of a transition.

Token

source

pub(all) struct Token {
  index : Int
  ttype : @bobzhang/pygments/token.TokenType
TokenType
  value : String
} derive(Eq, @moonbitlang/core/debug.Debug
Debug)

One token produced by a lexer: its UTF-16 start offset, type and text.

UsingTarget

source

pub(all) enum UsingTarget {
  This
  Other((Map[String, String]) -> Lexer raise)
}

Target of a using(...) action.

budget

source

fn budget() -> @bobzhang/pygments/regex.Budget
Budget

The budget of the lexing request in progress.

clamp_score

source

fn clamp_score(x : Double) -> Double

Python's make_analysator wrapper: clamps to [0, 1].

default_budget_steps

source

let default_budget_steps : Int

Default step budget for one lexing request (shared by nested lexers).

delegate

source

fn delegate(root : Lexer, language : Lexer, text : String, needle? : @bobzhang/pygments/token.TokenType
TokenType) -> Array[Token] raise

Python's DelegatingLexer.get_tokens_unprocessed: lexes with language, then lexes the concatenation of all needle tokens with root and re-inserts the other language tokens.

do_insertions

source

fn do_insertions(insertions : Array[(Int, Array[Token])], tokens : Array[Token]) -> Array[Token]

Python's do_insertions: inserts each token list of insertions at its index into the stream tokens, splitting tokens as needed.

doctype_matches

source

fn doctype_matches(text : String, regex : String) -> Bool

Python's doctype_matches.

expand_tabs

source

fn expand_tabs(s : String, tabsize : Int) -> String

Python's str.expandtabs(tabsize).

get_bool_opt

source

fn get_bool_opt(options : Map[String, String], name : String, default : Bool) -> Bool raise OptionError

Python's get_bool_opt.

get_choice_opt

source

fn get_choice_opt(options : Map[String, String], name : String, allowed : Array[String], default : String, normcase? : Bool) -> String raise OptionError

Python's get_choice_opt.

get_int_opt

source

fn get_int_opt(options : Map[String, String], name : String, default : Int) -> Int raise OptionError

Python's get_int_opt.

get_list_opt

source

fn get_list_opt(options : Map[String, String], name : String, default : Array[String]) -> Array[String]

Python's get_list_opt: the value split at whitespace.

html_doctype_matches

source

fn html_doctype_matches(text : String) -> Bool

Python's html_doctype_matches.

is_space

source

fn is_space(c : Char) -> Bool

Python's str.isspace for one character.

looks_like_xml

source

fn looks_like_xml(text : String) -> Bool

Python's looks_like_xml.

matches

source

fn matches(pattern : @bobzhang/pygments/regex.Regex
Regex, text : String) -> Bool

Python's bool(re.match(pattern, text)).

no_analyse

source

fn no_analyse(_text : String) -> Double

analyse_text of lexers that do not define one.

py_slice

source

fn py_slice(s : String, start : Int, end : Int) -> String

Python slicing s[start:end] with clamping (negative end counts from the end, as in Python).

re

source

fn re(pattern : String, flags? : Int) -> @bobzhang/pygments/regex.Regex
Regex

Compiles a pattern known to be valid (aborts otherwise).

run_context

source

fn run_context(ctx : Ctx) -> Unit raise

Runs a RegexLexer or ExtendedRegexLexer over an existing context (Python: get_tokens_unprocessed(text, context)).

run_regex

source

fn run_regex(lexer : Lexer, def : RegexDef, text : String, stack : Array[String], callbacks? : Map[String, (Ctx, @bobzhang/pygments/regex.Match
Match, Array[CallbackArg]) -> Unit raise]) -> Array[Token] raise

Runs a RegexLexer (or ExtendedRegexLexer) over text, starting with the state stack stack. callbacks resolves Action::Call names.

search

source

fn search(pattern : @bobzhang/pygments/regex.Regex
Regex, text : String) -> Bool

Python's bool(re.search(pattern, text, flags)), for analyse_text.

shebang_matches

source

fn shebang_matches(text : String, regex : String) -> Bool

Python's shebang_matches: whether the last component of the shebang line (ignoring -flags) fully matches regex (case-insensitively).

split_whitespace

source

fn split_whitespace(s : String) -> Array[String]

Python's str.split() without arguments.

text_tokenizer

source

fn text_tokenizer(_lexer : Lexer, text : String, _stack : Array[String]) -> Array[Token]

Tokenizer of TextLexer: the whole input as one Text token.

Source Files

Site source Report issue Packages Build queue Skills Statistics

#CallbackFn

#Options

#Tokenizer

#LexError

#OptionError

#Action

#CallbackArg

#Ctx

#Ctx::apply

#Ctx::emit

#Ctx::emit_lexed

#Ctx::is_extended

#Ctx::new

#Ctx::pop_state

#Ctx::push_state

#Ctx::set_state_stack

#Ctx::state_stack

#Ctx::top_state

#Filter

#Lexer

#Lexer::add_filter

#Lexer::get_tokens

#Lexer::get_tokens_unprocessed

#Lexer::name

#Lexer::new

#Lexer::preprocess

#Lexer::recreate

#LexerInfo

#RegexDef

#RegexDef::new

#RegexDef::state

#Rule

#Rule::new

#StateOp

#Token

#UsingTarget

#budget

#clamp_score

#default_budget_steps

#delegate

#do_insertions

#doctype_matches

#expand_tabs

#get_bool_opt

#get_choice_opt

#get_int_opt

#get_list_opt

#html_doctype_matches

#is_space

#looks_like_xml

#matches

#no_analyse

#py_slice

#re

#run_context

#run_regex

#search

#shebang_matches

#split_whitespace

#text_tokenizer

Source Files

CallbackFn

Options

Tokenizer

LexError

OptionError

Action

CallbackArg

Ctx

Ctx::apply

Ctx::emit

Ctx::emit_lexed

Ctx::is_extended

Ctx::new

Ctx::pop_state

Ctx::push_state

Ctx::set_state_stack

Ctx::state_stack

Ctx::top_state

Filter

Lexer

Lexer::add_filter

Lexer::get_tokens

Lexer::get_tokens_unprocessed

Lexer::name

Lexer::new

Lexer::preprocess

Lexer::recreate

LexerInfo

RegexDef

RegexDef::new

RegexDef::state

Rule

Rule::new

StateOp

Token

UsingTarget

budget

clamp_score

default_budget_steps

delegate

do_insertions

doctype_matches

expand_tabs

get_bool_opt

get_choice_opt

get_int_opt

get_list_opt

html_doctype_matches

is_space

looks_like_xml

matches

no_analyse

py_slice

re

run_context

run_regex

search

shebang_matches

split_whitespace

text_tokenizer