README
- ::innerAgentId::inner
T
Block
T
BulletMarker
T
CodeBlockInfo
- ::active_countCrdtDocument::active_count
- ::deleteCrdtDocument::delete
- ::get_textCrdtDocument::get_text
- ::insertCrdtDocument::insert
- ::newCrdtDocument::new
- ::total_countCrdtDocument::total_count
- ::newCrdtSpan::new
T
Document
- ::deleteEditInfo::delete
- ::insertEditInfo::insert
- ::replaceEditInfo::replace
T
EmphasisMarker
T
FenceMarker
T
Frontmatter
T
HardBreakStyle
T
HeadingStyle
- ::newIdGenerator::new
- ::nextIdGenerator::next
T
IncrementalResult
T
Inline
T
LinkDefinition
T
ListItem
- ::compareLogicalId::compare
- ::leLogicalId::le
- ::ltLogicalId::lt
T
OrderedDelimiter
T
ParseResult
- ::defaultRenderOptions::default
- ::with_highlighterRenderOptions::with_highlighter
- ::with_simple_highlighterRenderOptions::with_simple_highlighter
- ::advanceScanner::advance
- ::consumeScanner::consume
- ::consume_strScanner::consume_str
- ::count_charScanner::count_char
- ::count_leading_spacesScanner::count_leading_spaces
- ::is_blank_lineScanner::is_blank_line
- ::is_eofScanner::is_eof
- ::matchesScanner::matches
- ::newScanner::new
- ::peekScanner::peek
- ::peek_atScanner::peek_at
- ::read_lineScanner::read_line
- ::remainingScanner::remaining
- ::restoreScanner::restore
- ::saveScanner::save
- ::skip_lineScanner::skip_line
- ::skip_spacesScanner::skip_spaces
- ::substringScanner::substring
- ::emptySpan::empty
- ::newSpan::new
T
TableAlign
T
TableCell
T
TextRun
- ::emptyTrivia::empty
- ::newTrivia::new
V
char_is
V
char_is_digit
V
is_alphanumeric
V
is_digit
V
is_letter
V
is_punctuation
V
is_whitespace
V
md_free
V
md_parse
V
md_parse_and_render
V
md_parse_incremental
V
md_render_to_html
V
md_render_to_string
V
md_to_html
V
md_to_html_strict
V
parse
V
parse_code_block_info
V
parse_incremental
V
parse_inlines
V
parse_inlines_multipass
V
render_html
V
serialize
V
serialize_definitions

markdown

Incremental Markdown parser and compiler

markdown

parser

cst

incremental

gfm

Download zip

Author

mizchi

Version

0.4.5

License

MIT

Repository

github.com/mizchi/markdown.mbt

Last updated

8 months ago

Downloads

21K

#@mizchi/markdown

CST-based incremental Markdown parser for JavaScript/MoonBit.

A cross-platform (JS/WASM/native) Markdown compiler optimized for real-time editing with incremental parsing.

#Features

Fast: Edit-position based incremental updates inspired by CRDTs Go Brrr. Optimized for speed over edge-case correctness CommonMark 207/542
Lossless CST: Preserves all whitespace, markers, and formatting
Incremental parsing: Re-parses only changed blocks (up to 42x faster)
GFM: GitHub Flavored Markdown support (tables, task lists, strikethrough)
Cross-platform: Works on JS, WASM-GC, and native targets
HTML rendering: Built-in HTML renderer with remark-html compatible output
mdast compatible: AST follows mdast specification

#JavaScript API

npm install @mizchi/markdown

#Usage

import { parse, toHtml, toMarkdown } from "@mizchi/markdown";

// Parse to AST
const ast = parse("# Hello\n\n**Bold** text");
console.log(ast.children[0].type); // "heading"

// Convert to HTML
const html = toHtml("# Hello\n\n**Bold** text");
// => "<h1>Hello</h1>\n<p><strong>Bold</strong> text</p>\n"

// Normalize markdown
const normalized = toMarkdown("# Hello\n\n\n\nWorld");
// => "# Hello\n\nWorld\n"

#Incremental Parsing

For real-time editing scenarios:

import { createDocument, insertEdit } from "@mizchi/markdown";

// Create document handle
const doc = createDocument("# Hello");

// Access AST, HTML, or Markdown
console.log(doc.ast);        // Parsed AST
console.log(doc.toHtml());   // "<h1>Hello</h1>\n"
console.log(doc.toMarkdown()); // "# Hello\n"

// Incremental update (faster than full re-parse)
const edit = insertEdit(7, 6); // Insert 6 chars at position 7
const newDoc = doc.update("# Hello World", edit);

// Free resources when done
doc.dispose();
newDoc.dispose();

#TypeScript Support

Full TypeScript definitions are included:

import { parse, Document, Block, Inline } from "@mizchi/markdown";

const ast: Document = parse("# Hello");
const heading = ast.children[0] as HeadingBlock;
console.log(heading.level); // 1

#MoonBit API

#Installation

moon add mizchi/markdown

#Usage

// Parse markdown
let result = @markdown.parse("# Hello\n\nWorld")
let doc = result.document

// Serialize back (lossless)
let output = @markdown.serialize(doc)

// Render to HTML
let html = @markdown.render_html(doc)

// Or use convenience function
let html = @markdown.md_to_html("# Hello\n\nWorld")

#Incremental Parsing

// Initial parse
let result = @markdown.parse(source)
let doc = result.document

// Create edit info
let edit = @markdown.EditInfo::replace(
  change_start,    // Start position
  old_length,      // Length of replaced text
  new_length       // Length of new text
)

// Incremental update (reuses unchanged blocks)
let inc_result = @markdown.parse_incremental(doc, old_source, new_source, edit)
let new_doc = inc_result.document

#Playground

pnpm install
moon build --target js
pnpm exec vite

#Performance

Document	Full Parse	Incremental	Speedup
10 paragraphs	68.89µs	7.36µs	9.4x
50 paragraphs	327.99µs	8.67µs	37.8x
100 paragraphs	651.14µs	15.25µs	42.7x

#Documentation

See docs/markdown.md for detailed architecture and design.

#CommonMark Compatibility

This parser handles most common Markdown syntax correctly and works well for typical use cases like documentation, blog posts, and notes.

However, some edge cases (deeply nested structures, unusual delimiter combinations) are not fully CommonMark compliant. If you need strict CommonMark compliance, consider using cmark.mbt or other fully compliant parsers.

#License

MIT

AgentId

source

pub(all) type AgentId Int

Agent識別子 (簡易版: Int)

impl Eq for AgentId

impl Show for AgentId

AgentId::inner

source

#deprecated("Use `struct T(A)` to declare a newtype and use `.0` access the underlying type instead.")
fn AgentId::inner(self : AgentId) -> Int

Convert newtype to its underlying type, automatically derived.

Block

source

pub(all) enum Block {
  ThematicBreak(Char, Int, Span, Trivia, Trivia)
  Heading(Int, HeadingStyle, Array[Inline], Int, Span, Trivia, Trivia)
  Paragraph(Array[Inline], Span, Trivia, Trivia)
  FencedCode(FenceMarker, Int, String, String, Int, Span, Trivia, Trivia)
  IndentedCode(String, Span, Trivia, Trivia)
  Blockquote(Array[Block], Span, Trivia, Trivia)
  BulletList(BulletMarker, Bool, Array[ListItem], Span, Trivia, Trivia)
  OrderedList(Int, OrderedDelimiter, Bool, Array[ListItem], Span, Trivia, Trivia)
  HtmlBlock(String, Span, Trivia, Trivia)
  Table(Array[TableCell], Array[TableAlign], Array[Array[TableCell]], Span, Trivia, Trivia)
  BlankLines(Int, Span)
  FootnoteDefinition(String, Array[Block], Span, Trivia, Trivia)
}

Block-level nodes

BulletMarker

source

pub(all) enum BulletMarker {
  Dash
  Asterisk
  Plus
}

List marker for unordered lists: -, *, +

impl Eq for BulletMarker

impl Show for BulletMarker

CodeBlockInfo

source

pub struct CodeBlockInfo {
  lang : String
  filename : String
  meta : String
}

"ts:index.ts {highlight=[1,3]}" → (lang="ts", filename="index.ts", meta="{highlight=[1,3]}")

CrdtDocument

source

pub(all) struct CrdtDocument {
  runs : Array[TextRun]
  gen : IdGenerator
}

実験用ドキュメント構造

CrdtDocument::active_count

source

fn CrdtDocument::active_count(self : CrdtDocument) -> Int

アクティブなRun数

CrdtDocument::delete

source

fn CrdtDocument::delete(self : CrdtDocument, idx : Int) -> Unit

削除（Tombstone化）

CrdtDocument::get_text

source

fn CrdtDocument::get_text(self : CrdtDocument) -> String

アクティブなテキストを取得

CrdtDocument::insert

source

fn CrdtDocument::insert(self : CrdtDocument, text : String) -> LogicalId

テキスト挿入

CrdtDocument::new

source

fn CrdtDocument::new(agent : AgentId) -> CrdtDocument

CrdtDocument::total_count

source

fn CrdtDocument::total_count(self : CrdtDocument) -> Int

全Run数（Tombstone含む）

CrdtSpan

source

pub(all) struct CrdtSpan {
  start_id : LogicalId
  end_id : LogicalId
  cached_from : Int
  cached_to : Int
}

CRDT対応Span (キャッシュ付き)

CrdtSpan::new

source

fn CrdtSpan::new(start_id : LogicalId, end_id : LogicalId, from : Int, to : Int) -> CrdtSpan

Document

source

pub(all) struct Document {
  frontmatter : Frontmatter?
  children : Array[Block]
  span : Span
}

Document root

EditInfo

source

pub(all) struct EditInfo {
  offset : Int
  old_len : Int
  new_len : Int
}

Using #valtype to avoid heap allocation for this small struct

EditInfo::delete

source

fn EditInfo::delete(offset : Int, len : Int) -> EditInfo

Create an edit info for deletion

EditInfo::insert

source

fn EditInfo::insert(offset : Int, len : Int) -> EditInfo

Create an edit info for insertion

EditInfo::replace

source

fn EditInfo::replace(offset : Int, old_len : Int, new_len : Int) -> EditInfo

Create an edit info for replacement

EmphasisMarker

source

pub(all) enum EmphasisMarker {
  Asterisk
  Underscore
}

Emphasis marker: * or _

impl Eq for EmphasisMarker

impl Show for EmphasisMarker

FenceMarker

source

pub(all) enum FenceMarker {
  Backtick
  Tilde
}

Fence marker for code blocks: ``` or ~~~

impl Eq for FenceMarker

impl Show for FenceMarker

Frontmatter

source

pub(all) struct Frontmatter {
  raw : String
  entries : Array[(String, String)]
  span : Span
}

Frontmatter (YAML)

HardBreakStyle

source

pub(all) enum HardBreakStyle {
  TwoSpaces
  Backslash
}

Hard break style

impl Eq for HardBreakStyle

impl Show for HardBreakStyle

HeadingStyle

source

pub(all) enum HeadingStyle {
  Atx
  Setext
}

Heading style

impl Eq for HeadingStyle

impl Show for HeadingStyle

IdGenerator

source

pub(all) struct IdGenerator {
  agent : AgentId
  next_seq : Int
}

LogicalId生成器

IdGenerator::new

source

fn IdGenerator::new(agent : AgentId) -> IdGenerator

IdGenerator::next

source

fn IdGenerator::next(self : IdGenerator) -> LogicalId

IncrementalResult

source

pub(all) struct IncrementalResult {
  document : Document
  reused_before : Int
  reparsed : Int
  reused_after : Int
}

Result of incremental parsing

Inline

source

pub(all) enum Inline {
  Text(String, Span)
  SoftBreak(Span)
  HardBreak(HardBreakStyle, Span)
  Emphasis(EmphasisMarker, Array[Inline], Span)
  Strong(EmphasisMarker, Array[Inline], Span)
  Strikethrough(Array[Inline], Span)
  Code(String, Int, Span)
  Link(Array[Inline], String, String, Span)
  RefLink(Array[Inline], String, Span)
  Autolink(String, Bool, Span)
  Image(String, String, String, Span)
  RefImage(String, String, Span)
  HtmlInline(String, Span)
  FootnoteReference(String, Span)
}

Inline-level nodes

LinkDefinition

source

pub(all) struct LinkDefinition {
  label : String
  url : String
  title : String
  span : Span
}

Link reference definition [label]: url "title"

ListItem

source

pub(all) struct ListItem {
  children : Array[Block]
  checked : Bool?
  marker_offset : Int
  content_offset : Int
  span : Span
}

List item

LogicalId

source

pub(all) struct LogicalId {
  agent : AgentId
  seq : Int
}

論理ID: (agent, seq) ペア

impl Eq for LogicalId

impl Show for LogicalId

LogicalId::compare

source

fn LogicalId::compare(self : LogicalId, other : LogicalId) -> Int

LogicalId の順序比較 (Lamport順序)

LogicalId::le

source

fn LogicalId::le(self : LogicalId, other : LogicalId) -> Bool

LogicalId::lt

source

fn LogicalId::lt(self : LogicalId, other : LogicalId) -> Bool

OrderedDelimiter

source

pub(all) enum OrderedDelimiter {
  Dot
  Paren
}

Ordered list delimiter: . or )

impl Eq for OrderedDelimiter

impl Show for OrderedDelimiter

ParseResult

source

pub(all) struct ParseResult {
  document : Document
  definitions : Array[LinkDefinition]
}

Parse result

RenderOptions

source

pub struct RenderOptions {
  code_highlighter : (CodeBlockInfo, String) -> String?
}

Render options with plugin support

RenderOptions::default

source

fn RenderOptions::default() -> RenderOptions

Create default render options

RenderOptions::with_highlighter

source

fn RenderOptions::with_highlighter(highlighter : (CodeBlockInfo, String) -> String) -> RenderOptions

Create render options with code highlighter

RenderOptions::with_simple_highlighter

source

fn RenderOptions::with_simple_highlighter(highlighter : (String, String) -> String) -> RenderOptions

Create render options with simple highlighter (lang, code) -> String

Scanner

source

pub(all) struct Scanner {
  source : String
  chars : Array[Char]
  pos : Int
  len : Int
  utf16_offsets : Array[Int]?
}

Scanner state - uses Array[Char] for O(1) character access Handles Unicode correctly by tracking UTF-16 offsets for non-BMP characters

Scanner::advance

source

fn Scanner::advance(self : Scanner, n : Int) -> Unit

Advance position by n characters

Scanner::consume

source

fn Scanner::consume(self : Scanner) -> Char?

Consume and return current character (O(1))

Scanner::consume_str

source

fn Scanner::consume_str(self : Scanner, s : String) -> Bool

Match and consume a string

Scanner::count_char

source

fn Scanner::count_char(self : Scanner, c : Char) -> Int

Count consecutive occurrences of a character from current position - O(1) per char

Scanner::count_leading_spaces

source

fn Scanner::count_leading_spaces(self : Scanner) -> Int

Count leading spaces (without advancing) - O(1) per char

Scanner::is_blank_line

source

fn Scanner::is_blank_line(self : Scanner) -> Bool

Check if current line is blank (only whitespace) - O(1) per char

Scanner::is_eof

source

fn Scanner::is_eof(self : Scanner) -> Bool

Check if at end of input

Scanner::matches

source

fn Scanner::matches(self : Scanner, s : String) -> Bool

Match a string at current position - optimized with Array[Char]

Scanner::new

source

fn Scanner::new(source : String) -> Scanner

Create a new scanner

Scanner::peek

source

fn Scanner::peek(self : Scanner) -> Char?

Peek current character (O(1) with Array[Char])

Scanner::peek_at

source

fn Scanner::peek_at(self : Scanner, offset : Int) -> Char?

Peek character at offset from current position (O(1))

Scanner::read_line

source

fn Scanner::read_line(self : Scanner) -> String

Read until end of line (not consuming newline) - O(1) per char

Scanner::remaining

source

fn Scanner::remaining(self : Scanner) -> String

Get remaining substring from current position

Scanner::restore

source

fn Scanner::restore(self : Scanner, pos : Int) -> Unit

Restore to saved position

Scanner::save

source

fn Scanner::save(self : Scanner) -> Int

Save current position

Scanner::skip_line

source

fn Scanner::skip_line(self : Scanner) -> Unit

Skip to next line (consuming newline if present) - O(1) per char

Scanner::skip_spaces

source

fn Scanner::skip_spaces(self : Scanner) -> Int

Skip whitespace (space and tab only) - O(1) per char

Scanner::substring

source

fn Scanner::substring(self : Scanner, start : Int, end : Int) -> String

Get substring from start to end position (code point indices)

Span

source

pub(all) struct Span {
  from : Int
  to : Int
}

Using #valtype to avoid heap allocation for this frequently-used small struct

impl Eq for Span

impl Show for Span

Span::empty

source

fn Span::empty() -> Span

Empty span (for synthetic nodes)

Span::new

source

fn Span::new(from : Int, to : Int) -> Span

Create a span

TableAlign

source

pub(all) enum TableAlign {
  Left
  Center
  Right
  None
}

Table alignment

impl Eq for TableAlign

impl Show for TableAlign

TableCell

source

pub(all) struct TableCell {
  children : Array[Inline]
  span : Span
}

Table cell

TextRun

source

pub(all) struct TextRun {
  id : LogicalId
  content : String
  deleted : Bool
}

TextRun: 連続テキストの圧縮表現

Trivia

source

pub(all) struct Trivia {
  content : String
}

Trivia represents non-semantic characters that should be preserved

impl Eq for Trivia

impl Show for Trivia

Trivia::empty

source

fn Trivia::empty() -> Trivia

Trivia::new

source

fn Trivia::new(content : String) -> Trivia

char_is

source

fn char_is(opt : Char?, c : Char) -> Bool

Check if char option matches specific char

char_is_digit

source

fn char_is_digit(opt : Char?) -> Bool

Check if char option is a digit

is_alphanumeric

source

fn is_alphanumeric(c : Char) -> Bool

Check if character is alphanumeric

is_digit

source

fn is_digit(c : Char) -> Bool

Check if character is a digit

is_letter

source

fn is_letter(c : Char) -> Bool

Check if character is ASCII letter

is_punctuation

source

fn is_punctuation(c : Char) -> Bool

Check if character is a punctuation mark

is_whitespace

source

fn is_whitespace(c : Char) -> Bool

Check if character is ASCII whitespace

md_free

source

fn md_free(handle : Int) -> Unit

Free a document handle to release memory

md_parse

source

fn md_parse(source : String) -> Int

Returns 0 on error

md_parse_and_render

source

fn md_parse_and_render(source : String, strict? : Bool) -> String

When strict=true, uses full CommonMark compliance (slower)

md_parse_incremental

source

fn md_parse_incremental(handle : Int, new_source : String, change_start : Int, old_end : Int, new_end : Int) -> Int

Returns new document handle, or 0 on error

md_render_to_html

source

fn md_render_to_html(handle : Int) -> String

Returns empty string if handle is invalid

md_render_to_string

source

fn md_render_to_string(handle : Int) -> String

Returns empty string if handle is invalid

md_to_html

source

fn md_to_html(source : String) -> String

Parse markdown and render to HTML

md_to_html_strict

source

fn md_to_html_strict(source : String) -> String

Parse markdown and render to HTML (strict mode)

parse

source

fn parse(source : String, strict? : Bool) -> ParseResult

When strict=false (default), uses fast single-pass parser

parse_code_block_info

source

fn parse_code_block_info(info : String) -> CodeBlockInfo

Parse code block info string into components

parse_incremental

source

fn parse_incremental(old_doc : Document, old_source : String, new_source : String, edit : EditInfo) -> IncrementalResult

Parse incrementally using edit hint

parse_inlines

source

fn parse_inlines(text : String, strict? : Bool) -> Array[Inline]

When strict=false (default), uses fast single-pass parser

parse_inlines_multipass

source

fn parse_inlines_multipass(text : String) -> Array[Inline]

Parse all inlines using multi-pass approach

render_html

source

fn render_html(doc : Document) -> String

Render a document to HTML

serialize

source

fn serialize(doc : Document) -> String

Serialize document to markdown string

serialize_definitions

source

fn serialize_definitions(defs : Array[LinkDefinition]) -> String

Serialize link definitions

Source Files

Site source Report issue Packages Build queue Skills Statistics

markdown

#@mizchi/markdown

#Features

#JavaScript API

#Usage

#Incremental Parsing

#TypeScript Support

#MoonBit API

#Installation

#Usage

#Incremental Parsing

#Playground

#Performance

#Documentation

#CommonMark Compatibility

#License

#AgentId

#AgentId::inner

#Block

#BulletMarker

#CodeBlockInfo

#CrdtDocument

#CrdtDocument::active_count

#CrdtDocument::delete

#CrdtDocument::get_text

#CrdtDocument::insert

#CrdtDocument::new

#CrdtDocument::total_count

#CrdtSpan

#CrdtSpan::new

#Document

#EditInfo

#EditInfo::delete

#EditInfo::insert

#EditInfo::replace

#EmphasisMarker

#FenceMarker

#Frontmatter

#HardBreakStyle

#HeadingStyle

#IdGenerator

#IdGenerator::new

#IdGenerator::next

#IncrementalResult

#Inline

#LinkDefinition

#ListItem

#LogicalId

#LogicalId::compare

#LogicalId::le

#LogicalId::lt

#OrderedDelimiter

#ParseResult

#RenderOptions

#RenderOptions::default

#RenderOptions::with_highlighter

#RenderOptions::with_simple_highlighter

#Scanner

#Scanner::advance

#Scanner::consume

#Scanner::consume_str

#Scanner::count_char

#Scanner::count_leading_spaces

#Scanner::is_blank_line

#Scanner::is_eof

#Scanner::matches

#Scanner::new

#Scanner::peek

#Scanner::peek_at

#Scanner::read_line

#Scanner::remaining

#Scanner::restore

#Scanner::save

#Scanner::skip_line

#Scanner::skip_spaces

#Scanner::substring

#Span

#Span::empty

#Span::new

#TableAlign

AgentId

AgentId::inner

Block

BulletMarker

CodeBlockInfo

CrdtDocument

CrdtDocument::active_count

CrdtDocument::delete

CrdtDocument::get_text

CrdtDocument::insert

CrdtDocument::new

CrdtDocument::total_count

CrdtSpan

CrdtSpan::new

Document

EditInfo

EditInfo::delete

EditInfo::insert

EditInfo::replace

EmphasisMarker

FenceMarker

Frontmatter

HardBreakStyle

HeadingStyle

IdGenerator

IdGenerator::new

IdGenerator::next

IncrementalResult

Inline

LinkDefinition

ListItem

LogicalId

LogicalId::compare

LogicalId::le

LogicalId::lt

OrderedDelimiter

ParseResult

RenderOptions

RenderOptions::default

RenderOptions::with_highlighter

RenderOptions::with_simple_highlighter

Scanner

Scanner::advance

Scanner::consume

Scanner::consume_str

Scanner::count_char

Scanner::count_leading_spaces

Scanner::is_blank_line

Scanner::is_eof

Scanner::matches

Scanner::new

Scanner::peek

Scanner::peek_at

Scanner::read_line

Scanner::remaining

Scanner::restore

Scanner::save

Scanner::skip_line

Scanner::skip_spaces

Scanner::substring

Span

Span::empty

Span::new

TableAlign

TableCell

TextRun

Trivia

Trivia::empty

Trivia::new

char_is

char_is_digit

is_alphanumeric

is_digit

is_letter

is_punctuation

is_whitespace

md_free

md_parse

md_parse_and_render

md_parse_incremental