#moonrag

async fn answer(embedder : &@moonrag.Embedder) -> Array[@moonrag.Scored] raise {
  let store = @moonrag.InMemoryVectorStore::new(embedder)
  let _ = store.add([
    @moonrag.Document::new(
      "the harbour at dawn, fishing boats leaving",
      meta=Map([("topic", @json.Json::string("sea"))]),
    ),
    @moonrag.Document::new("a mountain path in autumn, cold and clear"),
  ])
  let query = embedder.embed(["boats in the harbour"])[0]
  store.search(
    query[:],
    limit=1,
    filter=@moonrag.Filter::Equal("topic", @json.Json::string("sea")),
  )
}

#Install

moon add moonbitstack/moonrag

import {
  "moonbitstack/moonrag",           // documents, contracts, memory store, retrieval
  "moonbitstack/moonrag/split",     // recursive, token, Markdown and HTML splitters
  "moonbitstack/moonrag/load",      // text, CSV, JSON, Markdown, HTML loaders
  "moonbitstack/moonrag/load/file", // the same loaders, reading from disk
  "moonbitstack/moonrag/index",     // incremental indexing
  "moonbitstack/moonrag/sql",       // a store in SQL, over any moondb driver
}

#A document is the unit

let document = @moonrag.Document::new("some text", id="guide.md#2")
  .set_meta("source", @json.Json::string("guide.md"))
  .set_meta("page", @json.Json::number(3.0))

#Filters travel with the query

let recent = @moonrag.Filter::All([
  @moonrag.Filter::Equal("topic", @json.Json::string("sea")),
  @moonrag.Filter::Compare("year", @moonrag.Order::AtLeast, 2025.0),
])

#Embedding is a contract

pub(open) trait Embedder {
  async fn embed(Self, texts : ArrayView[String]) -> Array[Array[Double]] raise
}

#Storing

let embedder : &@moonrag.Embedder = @moonrag.HashingEmbedder::new()
let store = @moonrag.InMemoryVectorStore::new(embedder)
let query_vector = embedder.embed(["boats in the harbour"])[0]
let ids = store.add([document])
let hits = store.search(query_vector, limit=5)
store.delete([ids[0]][:])

#Storing in SQL

pub async fn[T : @moondb.AsyncDriver] search_with(
  driver : T,
  embedder : &@moonrag.Embedder,
) -> Array[@moonrag.Scored] raise {
  let store = @sql.SqlVectorStore::new(driver, embedder, dimension=256)
  store.create()
  let _ = store.add([@moonrag.Document::new("the harbour at dawn")])
  let query = embedder.embed(["the harbour"])[0]
  store.search(query[:], limit=3)
}

• PostgreSQL with pgvector. The vectors live in a vector(n) column. search is one statement: 1 - (embedding <=> $1) is the cosine similarity the rest of the library ranks by, the metadata filter is a WHERE clause over jsonb, and LIMIT cuts in the database. create also adds an HNSW index for cosine distance; above 2000 components the type cannot be indexed and the index is left out.
• SQLite. There is no distance operator, so the rows come back, the filter runs in this process — the same Filter::matches the in-memory store calls — and the cosine is computed by the same function. Vectors are stored as little-endian f64 blobs. A local table read whole per query is the right shape for the sizes SQLite is asked to hold; for a corpus that outgrows it, use PostgreSQL.

let driver = @sql.BlockingDriver::new(@moonsqlite.SqliteDriver::open("app.db"))

#Retrieving

let plain : &@moonrag.Retriever = @moonrag.SimilarityRetriever::new(embedder, store)
let diverse : &@moonrag.Retriever = @moonrag.MmrRetriever::new(embedder, store, lambda=0.3)

#Splitting

let splitter = @split.RecursiveSplitter::new(chunk_size=1000, chunk_overlap=200)
let chunks = splitter.split(document)

fn count_tokens(text : StringView) -> Int32 {
  let mut count = 0
  for word in text.to_owned().split(" ") {
    if word != "" {
      count = count + 1
    }
  }
  count
}

let splitter = @split.TokenSplitter::new(count_tokens, chunk_size=512, chunk_overlap=64)

let markdown = "# Install\n\nmoon add moonbitstack/moonrag.\n\n## Search\n\nBuild a store, then search.\n"
let splitter = @split.MarkdownSplitter::new(chunk_size=1000)
let chunks = splitter.split(@moonrag.Document::new(markdown))
assert_eq(chunks.length(), 2)
assert_eq(chunks[1].str_meta("h1"), Some("Install"))
assert_eq(chunks[1].str_meta("h2"), Some("Search"))

let page_html = "<html><body><h1>Install</h1><p>moon add moonbitstack/moonrag.</p><h2>Search</h2><p>Build a store, then search.</p></body></html>"
let splitter = @split.HtmlSplitter::new(chunk_size=1000)
let sections = splitter.split(@moonrag.Document::new(page_html))
assert_eq(sections.length(), 2)
assert_eq(sections[0].str_meta("h1"), Some("Install"))
assert_eq(sections[1].str_meta("h2"), Some("Search"))

#Loading

let csv_text = "name,city\nalice,London\nbob,Paris\n"
let documents = @moonrag.Loader::load(@load.CsvLoader::new("people.csv", csv_text))

let documents = @file.csv("people.csv")
let text = @file.markdown("notes.md")

#Indexing

let indexer = @index.Indexer::new(store)
let stats = indexer.index(chunks)                        // added / updated / skipped / deleted
let cleaned = indexer.index(chunks, cleanup=@index.Cleanup::Full)

• unchanged — same id, same content hash: skipped, nothing embedded.
• changed — same id, different text or metadata: the old copy is deleted, the new one added.
• gone — Incremental deletes documents whose source appears in this run and whose id does not; Full deletes everything this run did not produce; None deletes nothing.

#Interface

#Conformance

• Splitters. The recursive splitter replays 56 cases and the token splitter 32 — prose, code, CJK, whitespace-heavy and separator-free text, across four sizes, three separator placements and both whitespace settings — produced by a reference Python implementation of the same rules (1.1.3). Chunks match exactly; scripts/gen_split_vectors.py regenerates them.
• Markdown splitting. Eight cases from the header splitter of the same reference release — a preamble, nested levels, adjacent headings, a section that is one code block, and a # inside a fence that must not split — replay chunk text and heading path. Two of the reference's behaviours are deliberately not followed, and are covered by this library's own tests instead: it joins the blocks of a section with a Markdown hard break (" \n"), and it strips every line, which flattens the indentation of code. scripts/gen_langchain_cases.py regenerates the fixtures.
• Ranking and MMR. Five ranking cases and seven MMR cases over fixed vectors: the same documents in the same order, and the same lambda trade-offs, as the reference returns. A score is a cosine summed in a different order, so the fixtures are compared to one part in a trillion; the order is compared exactly.
• Store and retrieval. The behaviours a vector-store standard suite requires — add, search with scores, limit, metadata filtering, delete by id, update by id, empty store, dimension mismatch, order independence — plus a retriever's limit, descending scores, filters and MMR diversity.
• Three backends. The same standard cases run against the in-memory store, against SQLite, and against PostgreSQL with pgvector, which is the only way to know that one contract holds over one table and one process. The pgvector cases read the column type back out of the catalogue and compare the store's score with the one the database computed, so a store that ranked in this process and said otherwise would fail.
• End to end. Load, split, index and retrieve as one test over a page and a note, including a second pass that must embed nothing.
• Against a filesystem. The file front end is tested by writing real files to a temporary directory and reading them back through all five loaders.

cd integration && moon test --target native

#Examples

moon run src/examples/01-ingest --target native

#License