markdown

Incremental Markdown parser and compiler

markdown
parser
cst
incremental
gfm
moon add mizchi/markdown@0.7.4
Download zip
Author
Version
0.7.4
License
MIT
Last updated
18 days ago
Downloads
13K
README

#@mizchi/markdown

CST-based incremental Markdown parser for JavaScript/MoonBit.

A cross-platform (JS/WASM/native) Markdown compiler optimized for real-time editing with incremental parsing.

#Features

  • Fast: Edit-position based incremental updates inspired by CRDTs Go Brrr. Optimized for speed over edge-case correctness CommonMark 207/542
  • Lossless CST: Preserves all whitespace, markers, and formatting
  • Incremental parsing: Re-parses only changed blocks (up to 42x faster)
  • GFM: GitHub Flavored Markdown support (tables, task lists, strikethrough)
  • Cross-platform: Works on JS, WASM-GC, and native targets
  • HTML rendering: Built-in HTML renderer with remark-html compatible output
  • mdast compatible: AST follows mdast specification


#JavaScript API

npm install @mizchi/markdown

#Usage

import { parse, toHtml, toMarkdown } from "@mizchi/markdown"; // Parse to AST const ast = parse("# Hello\n\n**Bold** text"); console.log(ast.children[0].type); // "heading" // Convert to HTML const html = toHtml("See https://example.com/docs\n"); // => '<p>See <a href="https://example.com/docs">https://example.com/docs</a></p>\n' // Disable bare URL links when you need plain text output const plain = toHtml("See https://example.com/docs\n", { autolink: false }); // => "<p>See https://example.com/docs</p>\n" // Normalize markdown const normalized = toMarkdown("# Hello\n\n\n\nWorld"); // => "# Hello\n\nWorld\n"

WikiLinks are disabled by default to keep CommonMark-compatible behavior. Pass { wikilinks: true } to parse [[target]] and [[target|label]].

import { parse, toHtml } from "@mizchi/markdown"; const ast = parse("[[MoonBit#syntax|MoonBit syntax]]", { wikilinks: true }); // ast.children[0].children[0].type === "wikiLink" const html = toHtml("[[MoonBit|MoonBit notes]]", { wikilinks: true }); // => '<p><a href="MoonBit">MoonBit notes</a></p>\n'

#Incremental Parsing

For real-time editing scenarios:

import { createDocument, insertEdit } from "@mizchi/markdown"; // Create document handle const doc = createDocument("# Hello"); // Access AST, HTML, or Markdown console.log(doc.ast); // Parsed AST console.log(doc.toHtml()); // "<h1>Hello</h1>\n" console.log(doc.toMarkdown()); // "# Hello\n" // Incremental update (faster than full re-parse) const edit = insertEdit(7, 6); // Insert 6 chars at position 7 const newDoc = doc.update("# Hello World", edit); // Free resources when done doc.dispose(); newDoc.dispose();

#TypeScript Support

Full TypeScript definitions are included:

import { parse, Document, Block, Inline } from "@mizchi/markdown"; const ast: Document = parse("# Hello"); const heading = ast.children[0] as HeadingBlock; console.log(heading.level); // 1


#MoonBit API

#Installation

moon add mizchi/markdown

#Usage

// Parse markdown
let result = @markdown.parse("# Hello\n\nWorld")
let doc = result.document

// Serialize back (lossless)
let output = @markdown.serialize(doc)

// Render to HTML
let html = @markdown.render_html(doc)

// Or use convenience function
let html = @markdown.md_to_html("# Hello\n\nWorld")
let linked = @markdown.md_to_html("See https://example.com/docs\n")
let plain = @markdown.md_to_html("See https://example.com/docs\n", autolink=false)

// Enable the WikiLink extension explicitly
let wiki_html = @markdown.md_to_html("[[MoonBit|MoonBit notes]]", wikilinks=true)

#Incremental Parsing

// Initial parse
let result = @markdown.parse(source)
let doc = result.document

// Create edit info
let edit = @markdown.EditInfo::replace(
change_start, // Start position
old_length, // Length of replaced text
new_length // Length of new text
)

// Incremental update (reuses unchanged blocks)
let inc_result = @markdown.parse_incremental(doc, old_source, new_source, edit)
let new_doc = inc_result.document

#Typed MDX Declarations

mizchi/markdown/x/mdx extracts MDX JSX components and validates their attributes against a closed, typed schema. It is intended for document metadata and domain declarations, rather than evaluating arbitrary MDX code.

Expressions use a deterministic literal-only subset: strings, booleans, and arrays of strings. For example, requires={["auth.mfa"]} is valid, while requires={loadRequirements()} is rejected.

import {
"mizchi/markdown",
"mizchi/markdown/x/mdx" @mdx,
}

let schema = @mdx.MdxSchema::closed([
@mdx.ComponentSchema::new(
"Fold",
[
@mdx.PropSchema::required("id", @mdx.MdxValueType::Text),
@mdx.PropSchema::required(
"kind",
@mdx.MdxValueType::OneOf(["concept", "procedure"]),
),
@mdx.PropSchema::optional("requires", @mdx.MdxValueType::TextList),
],
),
])

let source = #|<Fold id="auth.mfa" kind="procedure" requires={["auth.login"]} />
let checked = @mdx.type_check_mdx(@markdown.parse(source).document, schema)
let is_valid = checked.is_valid()

#Folddown

mizchi/markdown/x/folddown defines the typed Fold declaration vocabulary for structured, reader-adaptive Markdown. It validates declarations and emits a canonical manifest that retains each Markdown child source; reader selection and rendering consume that manifest. See Folddown for its grammar and boundary. Local external documents use typed <Include> declarations; Folddown drift review defines the provider-neutral LLM review packet and response contract. Its two reader states and two content filters are generated from the entry document's frontmatter DSL. familiarTo marks direct source-language correspondences, so an interest-focused view can omit material already familiar to its reader.


#Playground

pnpm install moon build --target js pnpm exec vite

#Frontend Editor Package

@mizchi/markdown/editor exports the Luna-based markdown editor without bundling syntax highlighters into the initial module. Code block highlighters are loaded on demand through dynamic imports under @mizchi/markdown/highlight.

The editor uses @luna_ui/luna as a JSX runtime and signal library; it is declared as an optional peer dependency — install it alongside @mizchi/markdown only if you use the editor entry. See frontend/editor/README.md for the editor-specific docs.

pnpm add @mizchi/markdown @luna_ui/luna

import { SyntaxHighlightEditor } from "@mizchi/markdown/editor"; import "@mizchi/markdown/editor/style.css"; <SyntaxHighlightEditor value={() => markdown} onChange={(next) => setMarkdown(next)} />;

You can also preload or call a language highlighter explicitly:

import { loadHighlighter } from "@mizchi/markdown/highlight"; const highlightMoonBit = await loadHighlighter("moonbit"); const html = highlightMoonBit?.("fn main { println(\"hi\") }");

The currently split lazy highlighter entries are typescript, moonbit, json, html, css, bash, and rust.

#Performance

DocumentFull ParseIncrementalSpeedup
10 paragraphs68.89µs7.36µs9.4x
50 paragraphs327.99µs8.67µs37.8x
100 paragraphs651.14µs15.25µs42.7x

#Documentation

See docs/markdown.md for detailed architecture and design.

#CommonMark Compatibility

This parser handles most common Markdown syntax correctly and works well for typical use cases like documentation, blog posts, and notes.

However, some edge cases (deeply nested structures, unusual delimiter combinations) are not fully CommonMark compliant. If you need strict CommonMark compliance, consider using cmark.mbt or other fully compliant parsers.

#Credits

  • Fonts: PlemolJP (SIL Open Font License 1.1) — bundled in playground/public/fonts/

#License

MIT

#
Block

pub(all) enum Block {
ThematicBreak(marker~ : Char, count~ : Int, span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
Heading(level~ : Int, style~ : HeadingStyle, children~ : Array[Inline], closing_hashes~ : Int, span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
Paragraph(children~ : Array[Inline], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
FencedCode(fence_marker~ : FenceMarker, fence_length~ : Int, info~ : String, code~ : String, indent~ : Int, span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
IndentedCode(code~ : String, span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
Blockquote(children~ : Array[Block], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
BulletList(marker~ : BulletMarker, tight~ : Bool, items~ : Array[ListItem], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
OrderedList(start~ : Int, delimiter~ : OrderedDelimiter, tight~ : Bool, items~ : Array[ListItem], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
HtmlBlock(html~ : String, span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
Table(header~ : Array[TableCell], alignments~ : Array[TableAlign], rows~ : Array[Array[TableCell]], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
BlankLines(count~ : Int, span~ : Span)
FootnoteDefinition(label~ : String, children~ : Array[Block], span~ : Span, leading_trivia~ : Trivia, trailing_trivia~ : Trivia)
}

Block-level nodes

#
BulletMarker

pub(all) enum BulletMarker {
Dash
Asterisk
Plus
} derive(Eq,
Debug
)

List marker for unordered lists: -, *, +

#
CodeBlockInfo

pub(all) struct CodeBlockInfo {
lang : String
filename : String
meta : String
}

Parsed info string from code block fence.

Format: "lang:filename {meta}"

Examples:
  • "javascript"{ lang: "javascript", filename: "", meta: "" }
  • "js:app.js"{ lang: "js", filename: "app.js", meta: "" }
  • "ts:index.ts {highlight=[1,3]}" { lang: "ts", filename: "index.ts", meta: "{highlight=[1,3]}" }

#
Document

pub(all) struct Document {
frontmatter : Frontmatter?
children : Array[Block]
span : Span
}

Document root

#
EditInfo

pub(all) struct EditInfo {
offset : Int
old_len : Int
new_len : Int
}

Using #valtype to avoid heap allocation for this small struct

#
EditInfo::delete

fn EditInfo::delete(offset : Int, len : Int) -> EditInfo

Create an edit info for deletion

#
EditInfo::insert

fn EditInfo::insert(offset : Int, len : Int) -> EditInfo

Create an edit info for insertion

#
EditInfo::replace

fn EditInfo::replace(offset : Int, old_len : Int, new_len : Int) -> EditInfo

Create an edit info for replacement

#
EmphasisMarker

pub(all) enum EmphasisMarker {
Asterisk
Underscore
} derive(Eq,
Debug
)

Emphasis marker: * or _

#
FenceMarker

pub(all) enum FenceMarker {
Backtick
Tilde
} derive(Eq,
Debug
)

Fence marker for code blocks: ``` or ~~~
impl Show for FenceMarker

#
Frontmatter

pub(all) struct Frontmatter {
raw : String
entries : Array[(String, String)]
span : Span
}

Frontmatter (YAML)

#
HardBreakStyle

pub(all) enum HardBreakStyle {
TwoSpaces
Backslash
} derive(Eq,
Debug
)

Hard break style

#
HeadingStyle

pub(all) enum HeadingStyle {
Atx
Setext
} derive(Eq,
Debug
)

Heading style

#
IncrementalResult

pub(all) struct IncrementalResult {
document : Document
reused_before : Int
reparsed : Int
reused_after : Int
}

Result of incremental parsing

#
Inline

pub(all) enum Inline {
Text(content~ : String, span~ : Span)
SoftBreak(span~ : Span)
HardBreak(style~ : HardBreakStyle, span~ : Span)
Emphasis(marker~ : EmphasisMarker, children~ : Array[Inline], span~ : Span)
Strong(marker~ : EmphasisMarker, children~ : Array[Inline], span~ : Span)
Strikethrough(children~ : Array[Inline], span~ : Span)
Code(content~ : String, backtick_count~ : Int, span~ : Span)
WikiLink(target~ : String, label~ : String, fragment~ : String, span~ : Span)
Link(children~ : Array[Inline], url~ : String, title~ : String, span~ : Span)
RefLink(children~ : Array[Inline], label~ : String, span~ : Span)
Autolink(url~ : String, is_email~ : Bool, span~ : Span)
Image(alt~ : String, url~ : String, title~ : String, span~ : Span)
RefImage(alt~ : String, label~ : String, span~ : Span)
HtmlInline(html~ : String, span~ : Span)
FootnoteReference(label~ : String, span~ : Span)
}

Inline-level nodes

#
LinkDefinition

pub(all) struct LinkDefinition {
label : String
url : String
title : String
span : Span
}

Link reference definition [label]: url "title"

#
ListItem

pub(all) struct ListItem {
children : Array[Block]
checked : Bool?
marker_offset : Int
content_offset : Int
span : Span
}

List item

#
OrderedDelimiter

pub(all) enum OrderedDelimiter {
Dot
Paren
} derive(Eq,
Debug
)

Ordered list delimiter: . or )

#
ParseResult

pub(all) struct ParseResult {
document : Document
definitions : Array[LinkDefinition]
}

Parse result

#
RenderOptions

pub struct RenderOptions {
code_highlighter : (CodeBlockInfo, String) -> String?
}

Render options with plugin support

#
RenderOptions::default

fn RenderOptions::default() -> RenderOptions

Create default render options

#
RenderOptions::with_highlighter

fn RenderOptions::with_highlighter(highlighter : (CodeBlockInfo, String) -> String) -> RenderOptions

Create render options with code highlighter

#
RenderOptions::with_simple_highlighter

fn RenderOptions::with_simple_highlighter(highlighter : (String, String) -> String) -> RenderOptions

Create render options with simple highlighter (lang, code) -> String

#
Scanner

pub(all) struct Scanner {
source : String
chars : Array[Char]
pos : Int
len : Int
utf16_offsets : Array[Int]?
}

Scanner state - uses UTF-16 indexing directly for BMP input and Array[Char] for non-BMP input. Handles Unicode correctly by tracking UTF-16 offsets for non-BMP characters.

#
Scanner::advance

fn Scanner::advance(self : Scanner, n : Int) -> Unit

Advance position by n characters

#
Scanner::consume

fn Scanner::consume(self : Scanner) -> Char?

Consume and return current character (O(1))

#
Scanner::consume_str

fn Scanner::consume_str(self : Scanner, s : String) -> Bool

Match and consume a string

#
Scanner::count_char

fn Scanner::count_char(self : Scanner, c : Char) -> Int

Count consecutive occurrences of a character from current position - O(1) per char

#
Scanner::count_leading_spaces

fn Scanner::count_leading_spaces(self : Scanner) -> Int

Count leading spaces (without advancing) - O(1) per char

#
Scanner::is_blank_line

fn Scanner::is_blank_line(self : Scanner) -> Bool

Check if current line is blank (only whitespace) - O(1) per char

#
Scanner::is_eof

fn Scanner::is_eof(self : Scanner) -> Bool

Check if at end of input

#
Scanner::matches

fn Scanner::matches(self : Scanner, s : String) -> Bool

Match a string at current position - optimized with Array[Char]

#
Scanner::new

fn Scanner::new(source : String) -> Scanner

Create a new scanner

#
Scanner::peek

fn Scanner::peek(self : Scanner) -> Char?

Peek current character (O(1) with Array[Char])

#
Scanner::peek_at

fn Scanner::peek_at(self : Scanner, offset : Int) -> Char?

Peek character at offset from current position (O(1))

#
Scanner::read_line

fn Scanner::read_line(self : Scanner) -> String

Read until end of line (not consuming newline) - O(1) per char

#
Scanner::remaining

fn Scanner::remaining(self : Scanner) -> String

Get remaining substring from current position

#
Scanner::restore

fn Scanner::restore(self : Scanner, pos : Int) -> Unit

Restore to saved position

#
Scanner::save

fn Scanner::save(self : Scanner) -> Int

Save current position

#
Scanner::skip_line

fn Scanner::skip_line(self : Scanner) -> Unit

Skip to next line (consuming newline if present) - O(1) per char

#
Scanner::skip_spaces

fn Scanner::skip_spaces(self : Scanner) -> Int

Skip whitespace (space and tab only) - O(1) per char

#
Scanner::substring

fn Scanner::substring(self : Scanner, start : Int, end : Int) -> String

Get substring from start to end position (code point indices)

#
Span

pub(all) struct Span {
from : Int
to : Int
} derive(Eq,
Debug
)

Using #valtype to avoid heap allocation for this frequently-used small struct
impl Show for Span

#
Span::empty

fn Span::empty() -> Span

Empty span (for synthetic nodes)

#
Span::new

fn Span::new(from : Int, to : Int) -> Span

Create a span

#
TableAlign

pub(all) enum TableAlign {
Left
Center
Right
None
} derive(Eq,
Debug
)

Table alignment
impl Show for TableAlign

#
TableCell

pub(all) struct TableCell {
children : Array[Inline]
span : Span
}

Table cell

#
Trivia

pub(all) struct Trivia {
content : String
} derive(Eq,
Debug
)

Trivia represents non-semantic characters that should be preserved
impl Show for Trivia

#
Trivia::empty

fn Trivia::empty() -> Trivia

#
Trivia::new

fn Trivia::new(content : String) -> Trivia

#
char_is

fn char_is(opt : Char?, c : Char) -> Bool

Check if char option matches specific char

#
char_is_digit

fn char_is_digit(opt : Char?) -> Bool

Check if char option is a digit

#
is_alphanumeric

fn is_alphanumeric(c : Char) -> Bool

Check if character is alphanumeric

#
is_digit

fn is_digit(c : Char) -> Bool

Check if character is a digit

#
is_letter

fn is_letter(c : Char) -> Bool

Check if character is ASCII letter

#
is_punctuation

fn is_punctuation(c : Char) -> Bool

Check if character is a punctuation mark

#
is_unicode_punctuation

fn is_unicode_punctuation(c : Char) -> Bool

Check if character is Unicode punctuation (ASCII subset)

#
is_unicode_whitespace

fn is_unicode_whitespace(c : Char) -> Bool

Check if character is Unicode whitespace

#
is_whitespace

fn is_whitespace(c : Char) -> Bool

Check if character is ASCII whitespace

#
md_parse_and_render

fn md_parse_and_render(source : String, strict? : Bool, wikilinks? : Bool) -> String

Parse markdown source and serialize back to a normalized string. Convenience wrapper around parse + serialize used by compatibility tests.

#
md_to_html

fn md_to_html(source : String, wikilinks? : Bool, autolink? : Bool) -> String

Parse markdown and render to HTML

#
md_to_html_literal

fn md_to_html_literal(source : String, wikilinks? : Bool, positions? : Bool, image_preview? : Bool) -> String

Parse markdown and render with the literal renderer in one step.

#
md_to_html_strict

fn md_to_html_strict(source : String, wikilinks? : Bool, autolink? : Bool) -> String

Parse markdown and render to HTML (strict mode)

#
parse

fn parse(source : String, strict? : Bool, wikilinks? : Bool) -> ParseResult

When strict=false (default), uses fast single-pass parser

#
parse_code_block_info

fn parse_code_block_info(info : String) -> CodeBlockInfo

Parse code block info string into components

#
parse_incremental

fn parse_incremental(old_doc : Document, old_source : String, new_source : String, edit : EditInfo, wikilinks? : Bool) -> IncrementalResult

Parse incrementally using edit hint

#
parse_inlines

fn parse_inlines(text : String, strict? : Bool, wikilinks? : Bool) -> Array[Inline]

When strict=false (default), uses fast single-pass parser

#
render_html

fn render_html(doc : Document, autolink? : Bool) -> String

Render a document to HTML

#
render_html_literal

fn render_html_literal(doc : Document, positions? : Bool, image_preview? : Bool) -> String

Render a document to HTML using the literal (source-preserving) mode.

  • positions: emit data-src-start / data-src-end on top-level block elements (see module docs).
  • image_preview: emit an <img class="md-image-preview"> slot inside each <span class="md-image"> wrapper alongside the ![alt](url) source characters. The <img> carries no visible text (textContent is empty), so the overlay invariant is preserved. If the alt text ends in :wN (for example ![diagram:w500](...)), the preview slot receives width metadata and the real image alt text is diagram. Default display: none ships in @mizchi/markdown/editor/overlay.css; the consumer opts in by adding .with-image-preview to a container above the rendered output (or by overriding the rule themselves).

#
serialize

fn serialize(doc : Document) -> String

Serialize document to markdown string

#
serialize_definitions

fn serialize_definitions(defs : Array[LinkDefinition]) -> String

Serialize link definitions