README
- ::addPersistentDB::add
- ::checkpointPersistentDB::checkpoint
- ::compactPersistentDB::compact
- ::count_filteredPersistentDB::count_filtered
- ::deserializePersistentDB::deserialize
- ::dimPersistentDB::dim
- ::findPersistentDB::find
- ::from_snapshotPersistentDB::from_snapshot
- ::getPersistentDB::get
- ::hasPersistentDB::has
- ::in_memoryPersistentDB::in_memory
- ::in_memory_bruteforcePersistentDB::in_memory_bruteforce
- ::in_memory_hnswPersistentDB::in_memory_hnsw
- ::in_memory_ivfPersistentDB::in_memory_ivf
- ::initPersistentDB::init
- ::metricPersistentDB::metric
- ::raw_sizePersistentDB::raw_size
- ::removePersistentDB::remove
- ::scrollPersistentDB::scroll
- ::scroll_filteredPersistentDB::scroll_filtered
- ::searchPersistentDB::search
- ::search_with_exprPersistentDB::search_with_expr
- ::search_with_filterPersistentDB::search_with_filter
- ::serialize_snapshotPersistentDB::serialize_snapshot
- ::sizePersistentDB::size
- ::storePersistentDB::store
- ::strategyPersistentDB::strategy
- ::trainPersistentDB::train
- ::update_attrsPersistentDB::update_attrs
- ::upsertPersistentDB::upsert
- ::wal_byte_sizePersistentDB::wal_byte_size
- ::wal_record_countPersistentDB::wal_record_count
- ::addVectorDB::add
- ::compactVectorDB::compact
- ::count_filteredVectorDB::count_filtered
- ::deserializeVectorDB::deserialize
- ::dimVectorDB::dim
- ::export_jsonlVectorDB::export_jsonl
- ::findVectorDB::find
- ::from_storeVectorDB::from_store
- ::getVectorDB::get
- ::hasVectorDB::has
- ::metricVectorDB::metric
- ::newVectorDB::new
- ::raw_sizeVectorDB::raw_size
- ::removeVectorDB::remove
- ::scrollVectorDB::scroll
- ::scroll_filteredVectorDB::scroll_filtered
- ::searchVectorDB::search
- ::search_with_exprVectorDB::search_with_expr
- ::serializeVectorDB::serialize
- ::sizeVectorDB::size
- ::storeVectorDB::store
- ::strategyVectorDB::strategy
- ::trainVectorDB::train
- ::update_attrsVectorDB::update_attrs
- ::upsertVectorDB::upsert
- ::with_dimVectorDB::with_dim
- ::with_hnswVectorDB::with_hnsw
- ::with_ivfVectorDB::with_ivf
V
empty_attrs

#core/storage

Storage abstraction layer for persistence operations.

#Storage Trait

pub trait Storage {
  read(Self, String, StorageKind) -> Bytes raise IOError
  write(Self, String, Bytes, StorageKind) -> Unit raise IOError
  append(Self, String, Bytes, StorageKind) -> Unit raise IOError
  atomic_write(Self, String, Bytes, StorageKind) -> Unit raise IOError
  del(Self, String, StorageKind) -> Unit raise IOError
  exists(Self, String, StorageKind) -> Bool
  list(Self, StorageKind) -> Array[String]
}

#Implementations

MemoryStorage: In-memory storage for testing
CallbackStorage: Delegates to JS callbacks for custom backends

#StorageKind

Routes data to appropriate backends:

Config: Collection configuration files
Index: Index structures and manifests
Data: Vector data and segments

AsyncStorage

source

pub(open) trait AsyncStorage {
  fn async_read(Self, String, StorageKind, (Bytes) -> Unit, (String) -> Unit) -> Unit
  fn async_write(Self, String, Bytes, StorageKind, () -> Unit, (String) -> Unit) -> Unit
  fn async_atomic_write(Self, String, Bytes, StorageKind, () -> Unit, (String) -> Unit) -> Unit
  fn async_del(Self, String, StorageKind, () -> Unit, (String) -> Unit) -> Unit
  fn async_exists(Self, String, StorageKind, (Bool) -> Unit, (String) -> Unit) -> Unit
  fn async_list(Self, StorageKind, (Array[String]) -> Unit, (String) -> Unit) -> Unit
}

Async storage trait — all targets. All operations return results via MoonBit's async/await mechanism. Implementors must call the provided continuation when I/O completes.

Storage

source

pub trait Storage {
  fn read(Self, String, StorageKind) -> Bytes raise IOError
  fn write(Self, String, Bytes, StorageKind) -> Unit raise IOError
  fn append(Self, String, Bytes, StorageKind) -> Unit raise IOError
  fn atomic_write(Self, String, Bytes, StorageKind) -> Unit raise IOError
  fn del(Self, String, StorageKind) -> Unit raise IOError
  fn exists(Self, String, StorageKind) -> Bool
  fn list(Self, StorageKind) -> Array[String] raise IOError
}

Storage trait - abstraction for file-like storage operations kind parameter allows app to route different data types to different backends

AsyncIOError

source

pub(all) suberror AsyncIOError {
  AsyncIOFailed(String)
}

Error type for async storage operations.

IOError

source

pub(all) suberror IOError {
  NotFound(String)
  PermissionDenied(String)
  IoFailed(String)
}

IO Error types for storage operations

KindAwareStorage

source

pub struct KindAwareStorage {
  config : Map[String, Bytes]
  index : Map[String, Bytes]
  data : Map[String, Bytes]
}

Kind-aware memory storage — TEST UTILITY.

Unlike MemoryStorage (which ignores kind), this stores data in separate maps per StorageKind, enabling tests to verify that operations route to the correct kind.

Not intended for production use. Prefer MemoryStorage for general in-memory usage or CallbackStorage for external backends.

impl AsyncStorage for KindAwareStorage

fn async_atomic_write(self : KindAwareStorage, path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_del(self : KindAwareStorage, path : String, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn async_exists(self : KindAwareStorage, path : String, kind : StorageKind, resolve : (Bool) -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_list(self : KindAwareStorage, kind : StorageKind, resolve : (Array[String]) -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_read(self : KindAwareStorage, path : String, kind : StorageKind, resolve : (Bytes) -> Unit, reject : (String) -> Unit) -> Unit

source

AsyncStorage implementation for KindAwareStorage. Like MemoryStorage, resolve is called synchronously.

fn async_write(self : KindAwareStorage, path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, _reject : (String) -> Unit) -> Unit

source

impl Storage for KindAwareStorage

fn append(self : KindAwareStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

fn atomic_write(self : KindAwareStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

fn del(self : KindAwareStorage, path : String, kind : StorageKind) -> Unit raise IOError

source

fn exists(self : KindAwareStorage, path : String, kind : StorageKind) -> Bool

source

fn list(self : KindAwareStorage, kind : StorageKind) -> Array[String] raise IOError

source

fn read(self : KindAwareStorage, path : String, kind : StorageKind) -> Bytes raise IOError

source

fn write(self : KindAwareStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

KindAwareStorage::list_kind

source

fn KindAwareStorage::list_kind(self : KindAwareStorage, kind : StorageKind) -> Array[String]

List files stored under a specific kind

KindAwareStorage::new

source

fn KindAwareStorage::new() -> KindAwareStorage

MemoryStorage

source

pub struct MemoryStorage {
  data : Map[String, Bytes]
}

Memory-based storage implementation for testing and in-memory use cases

impl AsyncStorage for MemoryStorage

fn async_atomic_write(self : MemoryStorage, path : String, data : Bytes, _kind : StorageKind, resolve : () -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_del(self : MemoryStorage, path : String, _kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn async_exists(self : MemoryStorage, path : String, _kind : StorageKind, resolve : (Bool) -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_list(self : MemoryStorage, _kind : StorageKind, resolve : (Array[String]) -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_read(self : MemoryStorage, path : String, _kind : StorageKind, resolve : (Bytes) -> Unit, reject : (String) -> Unit) -> Unit

source

AsyncStorage trait implementation for MemoryStorage. Since all operations are in-memory, resolve is called synchronously. This enables MemoryStorage to be used with PersistentVectorDB, AsyncWalRuntime, and all async persistence infrastructure — critical for testing.

fn async_write(self : MemoryStorage, path : String, data : Bytes, _kind : StorageKind, resolve : () -> Unit, _reject : (String) -> Unit) -> Unit

source

impl Storage for MemoryStorage

fn append(self : MemoryStorage, path : String, data : Bytes, _kind : StorageKind) -> Unit raise IOError

source

fn atomic_write(self : MemoryStorage, path : String, data : Bytes, _kind : StorageKind) -> Unit raise IOError

source

fn del(self : MemoryStorage, path : String, _kind : StorageKind) -> Unit raise IOError

source

fn exists(self : MemoryStorage, path : String, _kind : StorageKind) -> Bool

source

fn list(self : MemoryStorage, _kind : StorageKind) -> Array[String] raise IOError

source

fn read(self : MemoryStorage, path : String, _kind : StorageKind) -> Bytes raise IOError

source

Storage trait implementation for MemoryStorage MemoryStorage ignores kind - stores all data in single map

fn write(self : MemoryStorage, path : String, data : Bytes, _kind : StorageKind) -> Unit raise IOError

source

MemoryStorage::append

source

fn MemoryStorage::append(self : MemoryStorage, path : String, data : Bytes) -> Unit

Append data to file

MemoryStorage::atomic_write

source

fn MemoryStorage::atomic_write(self : MemoryStorage, path : String, data : Bytes) -> Unit

Atomic write (same as write for in-memory implementation)

MemoryStorage::clear

source

fn MemoryStorage::clear(self : MemoryStorage) -> Unit

Clear all files

MemoryStorage::del

source

fn MemoryStorage::del(self : MemoryStorage, path : String) -> Unit raise IOError

Delete a file

MemoryStorage::exists

source

fn MemoryStorage::exists(self : MemoryStorage, path : String) -> Bool

Check if a file exists

MemoryStorage::list

source

fn MemoryStorage::list(self : MemoryStorage) -> Array[String]

List all files

MemoryStorage::new

source

fn MemoryStorage::new() -> MemoryStorage

Create a new empty MemoryStorage

MemoryStorage::read

source

fn MemoryStorage::read(self : MemoryStorage, path : String) -> Bytes raise IOError

Read file contents

MemoryStorage::write

source

fn MemoryStorage::write(self : MemoryStorage, path : String, data : Bytes) -> Unit

Write file contents (overwrites)

NativeFileStorage

source

pub struct NativeFileStorage {
  root_dir : String
}

impl AsyncStorage for NativeFileStorage

fn async_atomic_write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn async_del(self : NativeFileStorage, path : String, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn async_exists(self : NativeFileStorage, path : String, kind : StorageKind, resolve : (Bool) -> Unit, _reject : (String) -> Unit) -> Unit

source

fn async_list(self : NativeFileStorage, kind : StorageKind, resolve : (Array[String]) -> Unit, reject : (String) -> Unit) -> Unit

source

fn async_read(self : NativeFileStorage, path : String, kind : StorageKind, resolve : (Bytes) -> Unit, reject : (String) -> Unit) -> Unit

source

AsyncStorage implementation for NativeFileStorage. Wraps synchronous filesystem I/O — resolve is called immediately. This enables NativeFileStorage to be used with PersistentVectorDB and all async persistence infrastructure on native targets.

fn async_write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

impl Storage for NativeFileStorage

fn append(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

fn atomic_write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

fn del(self : NativeFileStorage, path : String, kind : StorageKind) -> Unit raise IOError

source

fn exists(self : NativeFileStorage, path : String, kind : StorageKind) -> Bool

source

fn list(self : NativeFileStorage, kind : StorageKind) -> Array[String] raise IOError

source

fn read(self : NativeFileStorage, path : String, kind : StorageKind) -> Bytes raise IOError

source

fn write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

source

NativeFileStorage::append

source

fn NativeFileStorage::append(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

NativeFileStorage::atomic_write

source

fn NativeFileStorage::atomic_write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

NativeFileStorage::del

source

fn NativeFileStorage::del(self : NativeFileStorage, path : String, kind : StorageKind) -> Unit raise IOError

NativeFileStorage::exists

source

fn NativeFileStorage::exists(self : NativeFileStorage, path : String, kind : StorageKind) -> Bool

NativeFileStorage::list

source

fn NativeFileStorage::list(self : NativeFileStorage, kind : StorageKind) -> Array[String] raise IOError

NativeFileStorage::new

source

fn NativeFileStorage::new(root_dir : String) -> NativeFileStorage

NativeFileStorage::read

source

fn NativeFileStorage::read(self : NativeFileStorage, path : String, kind : StorageKind) -> Bytes raise IOError

NativeFileStorage::write

source

fn NativeFileStorage::write(self : NativeFileStorage, path : String, data : Bytes, kind : StorageKind) -> Unit raise IOError

ReplicatedStorage

source

pub struct ReplicatedStorage[T] {
  primary : T
  replicas : Array[T]
  config : ReplicationConfig
}

Replicated storage backend.

T is the underlying AsyncStorage implementation (e.g., JsAsyncCallbackStorage, MemoryStorage, NativeFileStorage).

Invariants:

Primary always receives every operation.
Replicas receive write/del operations only.
Reads go to primary only (read-local semantics).
write_quorum <= 1 + replicas.length()

impl AsyncStorage for ReplicatedStorage[T]

fn[T : AsyncStorage] async_atomic_write(self : ReplicatedStorage[T], path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn[T : AsyncStorage] async_del(self : ReplicatedStorage[T], path : String, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

fn[T : AsyncStorage] async_exists(self : ReplicatedStorage[T], path : String, kind : StorageKind, resolve : (Bool) -> Unit, reject : (String) -> Unit) -> Unit

source

fn[T : AsyncStorage] async_list(self : ReplicatedStorage[T], kind : StorageKind, resolve : (Array[String]) -> Unit, reject : (String) -> Unit) -> Unit

source

fn[T : AsyncStorage] async_read(self : ReplicatedStorage[T], path : String, kind : StorageKind, resolve : (Bytes) -> Unit, reject : (String) -> Unit) -> Unit

source

fn[T : AsyncStorage] async_write(self : ReplicatedStorage[T], path : String, data : Bytes, kind : StorageKind, resolve : () -> Unit, reject : (String) -> Unit) -> Unit

source

ReplicatedStorage::backend_count

source

fn[T] ReplicatedStorage::backend_count(self : ReplicatedStorage[T]) -> Int

Number of total backends (primary + replicas).

ReplicatedStorage::local_only

source

fn[T] ReplicatedStorage::local_only(storage : T) -> ReplicatedStorage[T]

Create a replicated storage with no replicas (passthrough).

ReplicatedStorage::new

source

fn[T] ReplicatedStorage::new(primary : T, replicas : Array[T], config? : ReplicationConfig) -> ReplicatedStorage[T]

Create a replicated storage with replicas and custom config.

ReplicatedStorage::with_all_ack

source

fn[T] ReplicatedStorage::with_all_ack(primary : T, replicas : Array[T]) -> ReplicatedStorage[T]

Create a replicated storage requiring all backends to ack.

ReplicationConfig

source

pub(all) struct ReplicationConfig {
  write_quorum : Int
}

Replication configuration for a single storage backend.

ReplicationConfig::all

source

fn ReplicationConfig::all(replica_count : Int) -> ReplicationConfig

Create a strict replication config requiring all replicas to ack.

ReplicationConfig::default

source

fn ReplicationConfig::default() -> ReplicationConfig

ReplicationConfig::majority

source

fn ReplicationConfig::majority(replica_count : Int) -> ReplicationConfig

Create a majority quorum config.

StorageKind

source

pub(all) enum StorageKind {
  Config
  Index
  Data
}

Storage kind - tells app what type of data is being stored App uses this to route to appropriate backend (e.g., index→DynamoDB, data→S3)

async_atomic_write

source

async fn[S : AsyncStorage] async_atomic_write(storage : S, path : String, data : Bytes, kind : StorageKind) -> Unit

async_del

source

async fn[S : AsyncStorage] async_del(storage : S, path : String, kind : StorageKind) -> Unit

async_exists

source

async fn[S : AsyncStorage] async_exists(storage : S, path : String, kind : StorageKind) -> Bool

async_list

source

async fn[S : AsyncStorage] async_list(storage : S, kind : StorageKind) -> Array[String]

async_read

source

async fn[S : AsyncStorage] async_read(storage : S, path : String, kind : StorageKind) -> Bytes

async_write

source

async fn[S : AsyncStorage] async_write(storage : S, path : String, data : Bytes, kind : StorageKind) -> Unit

collection_config_path

source

fn collection_config_path(collection_name : String) -> String

Path for collection config file

collection_data_path

source

fn collection_data_path(collection_name : String) -> String

Path for collection data (non-distributed)

collection_index_path

source

fn collection_index_path(collection_name : String) -> String

Path for collection index file (distributed)

collection_manifest_path

source

fn collection_manifest_path(collection_name : String) -> String

Path for collection manifest file (distributed)

collection_wal_path

source

fn collection_wal_path(collection_name : String) -> String

Path for collection WAL file

ext_config

source

let ext_config : String

Config file extension (not part of binary format, defined here)

ext_data

source

let ext_data : String

Data file extension (non-distributed, defined here)

is_segment_path

source

fn is_segment_path(path : String, base_name : String) -> Bool

Check if a path is a segment file for a given collection

segment_data_path

source

fn segment_data_path(base : String, pg : Int, part : Int) -> String

Path for data segment file

Source Files

Install

Download zip

High-performance vector database with multiple ANN algorithms

vector-databaseannhnswivfsimilarity-search

trkbt10

Apache-2.0

github.com/trkbt10/vcdb_mbt

Updated 3 months ago

681 downloads

Site source Report issue Modules Skills CLI Build queue Statistics

#core/storage

#Storage Trait

#Implementations

#StorageKind

#AsyncStorage

#Storage

#AsyncIOError

#IOError

#KindAwareStorage

#KindAwareStorage::list_kind

#KindAwareStorage::new

#MemoryStorage

#MemoryStorage::append

#MemoryStorage::atomic_write

#MemoryStorage::clear

#MemoryStorage::del

#MemoryStorage::exists

#MemoryStorage::list

#MemoryStorage::new

#MemoryStorage::read

#MemoryStorage::write

#NativeFileStorage

#NativeFileStorage::append

#NativeFileStorage::atomic_write

#NativeFileStorage::del

#NativeFileStorage::exists

#NativeFileStorage::list

#NativeFileStorage::new

#NativeFileStorage::read

#NativeFileStorage::write

#ReplicatedStorage

#ReplicatedStorage::backend_count

#ReplicatedStorage::local_only

#ReplicatedStorage::new

#ReplicatedStorage::with_all_ack

#ReplicationConfig

#ReplicationConfig::all

#ReplicationConfig::default

#ReplicationConfig::majority

#StorageKind

#async_atomic_write

#async_del

#async_exists

#async_list

#async_read

#async_write

#collection_config_path

#collection_data_path

#collection_index_path

#collection_manifest_path

#collection_wal_path

#ext_config

#ext_data

#is_segment_path

#segment_data_path

Source Files

AsyncStorage

Storage

AsyncIOError

IOError

KindAwareStorage

KindAwareStorage::list_kind

KindAwareStorage::new

MemoryStorage

MemoryStorage::append

MemoryStorage::atomic_write

MemoryStorage::clear

MemoryStorage::del

MemoryStorage::exists

MemoryStorage::list

MemoryStorage::new

MemoryStorage::read

MemoryStorage::write

NativeFileStorage

NativeFileStorage::append

NativeFileStorage::atomic_write

NativeFileStorage::del

NativeFileStorage::exists

NativeFileStorage::list

NativeFileStorage::new

NativeFileStorage::read

NativeFileStorage::write

ReplicatedStorage

ReplicatedStorage::backend_count

ReplicatedStorage::local_only

ReplicatedStorage::new

ReplicatedStorage::with_all_ack

ReplicationConfig

ReplicationConfig::all

ReplicationConfig::default

ReplicationConfig::majority

StorageKind

async_atomic_write

async_del

async_exists

async_list

async_read

async_write

collection_config_path

collection_data_path

collection_index_path

collection_manifest_path

collection_wal_path

ext_config

ext_data

is_segment_path

segment_data_path