#moonrag

async fn answer(embedder : &@moonrag.Embedder) -> Array[@moonrag.Scored] raise {
  let store = @moonrag.InMemoryVectorStore::new(embedder)
  let _ = store.add([
    @moonrag.Document::new(
      "the harbour at dawn, fishing boats leaving",
      meta=Map([("topic", @json.Json::string("sea"))]),
    ),
    @moonrag.Document::new("a mountain path in autumn, cold and clear"),
  ])
  let query = embedder.embed(["boats in the harbour"])[0]
  store.search(
    query[:],
    limit=1,
    filter=@moonrag.Filter::Equal("topic", @json.Json::string("sea")),
  )
}

#Install

moon add moonbitstack/moonrag

import {
  "moonbitstack/moonrag",           // documents, contracts, memory store, retrieval
  "moonbitstack/moonrag/split",     // recursive, token and HTML splitters
  "moonbitstack/moonrag/load",      // text, CSV, JSON, Markdown, HTML loaders
  "moonbitstack/moonrag/load/file", // the same loaders, reading from disk
  "moonbitstack/moonrag/index",     // incremental indexing
}

#A document is the unit

let document = @moonrag.Document::new("some text", id="guide.md#2")
  .set_meta("source", @json.Json::string("guide.md"))
  .set_meta("page", @json.Json::number(3.0))

#Filters travel with the query

let recent = @moonrag.Filter::All([
  @moonrag.Filter::Equal("topic", @json.Json::string("sea")),
  @moonrag.Filter::Compare("year", @moonrag.Order::AtLeast, 2025.0),
])

#Embedding is a contract

pub(open) trait Embedder {
  async fn embed(Self, texts : ArrayView[String]) -> Array[Array[Double]] raise
}

#Storing

let store = @moonrag.InMemoryVectorStore::new(embedder)
let ids = store.add([document])
let hits = store.search(query_vector, limit=5)
store.delete([ids[0]][:])

#Retrieving

let plain : &@moonrag.Retriever = @moonrag.SimilarityRetriever::new(embedder, store)
let diverse : &@moonrag.Retriever = @moonrag.MmrRetriever::new(embedder, store, lambda=0.3)

#Splitting

let splitter = @split.RecursiveSplitter::new(chunk_size=1000, chunk_overlap=200)
let chunks = splitter.split(document)

let splitter = @split.TokenSplitter::new(count_tokens, chunk_size=512, chunk_overlap=64)

let splitter = @split.HtmlSplitter::new(chunk_size=1000)
let sections = splitter.split(@moonrag.Document::new(page_html))
// sections[1].str_meta("h1") == Some("Install"); .str_meta("h2") == Some("Search")

#Loading

let documents = @moonrag.Loader::load(@load.CsvLoader::new("people.csv", csv_text))

let documents = @file.csv("people.csv")
let text = @file.markdown("notes.md")

#Indexing

let indexer = @index.Indexer::new(store)
let stats = indexer.index(chunks)                        // added / updated / skipped / deleted
let cleaned = indexer.index(chunks, cleanup=@index.Cleanup::Full)

• unchanged — same id, same content hash: skipped, nothing embedded.
• changed — same id, different text or metadata: the old copy is deleted, the new one added.
• gone — Incremental deletes documents whose source appears in this run and whose id does not; Full deletes everything this run did not produce; None deletes nothing.

#Interface

#Conformance

• Splitters. The recursive splitter replays 56 cases and the token splitter 32 — prose, code, CJK, whitespace-heavy and separator-free text, across four sizes, three separator placements and both whitespace settings — produced by a reference Python implementation of the same rules (1.1.3). Chunks match exactly; scripts/gen_split_vectors.py regenerates them.
• Store and retrieval. The behaviours a vector-store standard suite requires — add, search with scores, limit, metadata filtering, delete by id, update by id, empty store, dimension mismatch, order independence — plus a retriever's limit, descending scores, filters and MMR diversity.
• End to end. Load, split, index and retrieve as one test over a page and a note, including a second pass that must embed nothing.
• Against a filesystem. The file front end is tested by writing real files to a temporary directory and reading them back through all five loaders.

#Examples

moon run src/examples/01-ingest --target native

#License