From 08fb78721f672e75631e66294ad1ed1ba260e492 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:05:03 +0000 Subject: [PATCH] Simplify: one abstract base class per layer - BaseKVStorage / BaseEventStorage / BaseDocumentStorage define the API once; implementations provide only raw primitives (client and server share an API) - Drop KV indexes, query planner, unique indexes; document stores take index fields - Escape every char <= '/' so child keys stay contiguous; skip-scan is now simple - Event-sourced documents = event storage + any document store for projections - New folder layout (kv-storage, event-storage, document-storage, utils); tests only in test/ Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- CLAUDE.md | 32 +- DESIGN.md | 809 ++++-------------- bench/document-storage.bench.ts | 33 + bench/documents.bench.ts | 34 - bench/event-storage.bench.ts | 22 + bench/events.bench.ts | 27 - bench/keys.bench.ts | 26 - bench/kv-storage.bench.ts | 40 + bench/kv.bench.ts | 29 - package.json | 28 - src/core/encoding.ts | 176 ---- src/core/lru.ts | 30 - src/core/mutex.ts | 15 - src/core/path.ts | 109 --- src/core/trie.ts | 82 -- src/document-storage/base-document-storage.ts | 117 +++ .../document-storage-in-memory.ts | 112 +++ src/document-storage/document-storage-kv.ts | 46 + .../event-sourced-document-storage.ts | 125 +++ src/documents/event-sourced.ts | 606 ------------- src/documents/filter.ts | 155 ---- src/documents/index.ts | 7 - src/documents/indexes.ts | 220 ----- src/documents/types.ts | 90 -- src/documents/ulid.ts | 37 - src/documents/update.ts | 109 --- src/documents/values.ts | 163 ---- src/errors.ts | 50 -- src/event-storage/base-event-storage.ts | 100 +++ src/event-storage/event-storage-kv.ts | 151 ++++ src/events/index.ts | 2 - src/events/kv.ts | 289 ------- src/events/types.ts | 69 -- src/http/client.ts | 115 --- src/http/handler.ts | 115 --- src/http/index.ts | 2 - src/index.ts | 26 +- src/kv-storage/base-kv-storage.ts | 184 ++++ src/kv-storage/kv-storage-http-server.ts | 70 ++ src/kv-storage/kv-storage-http.ts | 79 ++ src/kv-storage/kv-storage-in-memory.ts | 34 + src/kv/children.ts | 236 ----- src/kv/driver.ts | 77 -- src/kv/index.ts | 6 - src/kv/indexes.ts | 157 ---- src/kv/memory.ts | 93 -- src/kv/node.ts | 402 --------- src/kv/sorted.ts | 181 ---- src/testing/index.ts | 1 - src/testing/kv-conformance.ts | 171 ---- src/utils/document-path.ts | 32 + src/utils/document-query.ts | 106 +++ src/utils/document-update.ts | 75 ++ src/utils/errors.ts | 19 + src/utils/json.ts | 24 + src/utils/keys.ts | 43 + src/{types => utils}/schema.ts | 5 +- src/utils/sorted-map.ts | 103 +++ src/utils/ulid.ts | 36 + test/core/encoding.test.ts | 73 -- test/core/path.test.ts | 73 -- .../document-storage.suite.ts | 86 ++ .../document-storage/document-storage.test.ts | 91 ++ test/documents/event-sourced.test.ts | 379 -------- test/documents/units.test.ts | 62 -- test/event-storage/event-storage-kv.test.ts | 99 +++ test/events/kv.test.ts | 200 ----- test/helpers.ts | 94 +- test/http/http.test.ts | 92 -- test/http/node-server.ts | 36 - test/kv-storage/kv-storage-http.test.ts | 54 ++ test/kv-storage/kv-storage-in-memory.test.ts | 78 ++ test/kv-storage/kv-storage.conformance.ts | 168 ++++ test/kv-storage/node-server.ts | 24 + test/kv/indexes.test.ts | 256 ------ test/kv/memory-driver.test.ts | 28 - test/kv/node.test.ts | 354 -------- test/kv/sorted.test.ts | 67 -- test/types.test-d.ts | 112 +-- test/utils/utils.test.ts | 97 +++ 80 files changed, 2518 insertions(+), 6337 deletions(-) create mode 100644 bench/document-storage.bench.ts delete mode 100644 bench/documents.bench.ts create mode 100644 bench/event-storage.bench.ts delete mode 100644 bench/events.bench.ts delete mode 100644 bench/keys.bench.ts create mode 100644 bench/kv-storage.bench.ts delete mode 100644 bench/kv.bench.ts delete mode 100644 src/core/encoding.ts delete mode 100644 src/core/lru.ts delete mode 100644 src/core/mutex.ts delete mode 100644 src/core/path.ts delete mode 100644 src/core/trie.ts create mode 100644 src/document-storage/base-document-storage.ts create mode 100644 src/document-storage/document-storage-in-memory.ts create mode 100644 src/document-storage/document-storage-kv.ts create mode 100644 src/document-storage/event-sourced-document-storage.ts delete mode 100644 src/documents/event-sourced.ts delete mode 100644 src/documents/filter.ts delete mode 100644 src/documents/index.ts delete mode 100644 src/documents/indexes.ts delete mode 100644 src/documents/types.ts delete mode 100644 src/documents/ulid.ts delete mode 100644 src/documents/update.ts delete mode 100644 src/documents/values.ts delete mode 100644 src/errors.ts create mode 100644 src/event-storage/base-event-storage.ts create mode 100644 src/event-storage/event-storage-kv.ts delete mode 100644 src/events/index.ts delete mode 100644 src/events/kv.ts delete mode 100644 src/events/types.ts delete mode 100644 src/http/client.ts delete mode 100644 src/http/handler.ts delete mode 100644 src/http/index.ts create mode 100644 src/kv-storage/base-kv-storage.ts create mode 100644 src/kv-storage/kv-storage-http-server.ts create mode 100644 src/kv-storage/kv-storage-http.ts create mode 100644 src/kv-storage/kv-storage-in-memory.ts delete mode 100644 src/kv/children.ts delete mode 100644 src/kv/driver.ts delete mode 100644 src/kv/index.ts delete mode 100644 src/kv/indexes.ts delete mode 100644 src/kv/memory.ts delete mode 100644 src/kv/node.ts delete mode 100644 src/kv/sorted.ts delete mode 100644 src/testing/index.ts delete mode 100644 src/testing/kv-conformance.ts create mode 100644 src/utils/document-path.ts create mode 100644 src/utils/document-query.ts create mode 100644 src/utils/document-update.ts create mode 100644 src/utils/errors.ts create mode 100644 src/utils/json.ts create mode 100644 src/utils/keys.ts rename src/{types => utils}/schema.ts (87%) create mode 100644 src/utils/sorted-map.ts create mode 100644 src/utils/ulid.ts delete mode 100644 test/core/encoding.test.ts delete mode 100644 test/core/path.test.ts create mode 100644 test/document-storage/document-storage.suite.ts create mode 100644 test/document-storage/document-storage.test.ts delete mode 100644 test/documents/event-sourced.test.ts delete mode 100644 test/documents/units.test.ts create mode 100644 test/event-storage/event-storage-kv.test.ts delete mode 100644 test/events/kv.test.ts delete mode 100644 test/http/http.test.ts delete mode 100644 test/http/node-server.ts create mode 100644 test/kv-storage/kv-storage-http.test.ts create mode 100644 test/kv-storage/kv-storage-in-memory.test.ts create mode 100644 test/kv-storage/kv-storage.conformance.ts create mode 100644 test/kv-storage/node-server.ts delete mode 100644 test/kv/indexes.test.ts delete mode 100644 test/kv/memory-driver.test.ts delete mode 100644 test/kv/node.test.ts delete mode 100644 test/kv/sorted.test.ts create mode 100644 test/utils/utils.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 8553941..9d9b3f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,24 +1,24 @@ # PotentialStorage -Layered TypeScript storage library: KV → Events → Documents. The full design, milestones and -implementation decisions are in [DESIGN.md](./DESIGN.md) — read it before changing anything. -Record new decisions in DESIGN.md section 12. +Layered TypeScript storage library: KV → Events → Documents. The design is in +[DESIGN.md](./DESIGN.md); read it before changing anything and keep it up to date. ## Commands -- `npm test` — `tsc --noEmit` then all Vitest tests (including `*.test-d.ts` type tests) -- `npm run test:only -- ` — Vitest only, e.g. a single file -- `npm run typecheck` — `tsc --noEmit` over src, test and bench -- `npm run bench` — `vitest bench` -- `npm run build` — emit `dist/` -- `npm run format` / `npm run format:check` — Prettier +- `npm test`: `tsc --noEmit`, then all Vitest tests (including `test/types.test-d.ts`) +- `npm run test:only -- `: Vitest only +- `npm run typecheck`, `npm run build`, `npm run bench` +- `npm run format` / `npm run format:check`: Prettier -CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs format check, typecheck, tests and build on Node 20 and 22. +CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, tests and build on Node 20 and 22. -## Rules +## Conventions -- No runtime dependencies in `src/`. Dev dependencies only. -- Drivers stay flat and string-keyed; all path/index/escaping logic lives in the node classes. -- Self vs child dispatch is by `arguments.length`; a passed-but-`undefined`/`null` key throws. -- Multi-result reads are `AsyncIterable`; don't build large arrays internally. -- Keep tests green and commit per milestone. Ask the owner before pushing. +- One abstract base class per layer (`src//base-*.ts`); implementations only provide the raw + primitives, and the user-facing API lives in the base class. +- Classes over loose functions; TS `private`, not `#`; no overloads (use `param?` or tuple-union + rest params); don't override base methods in subclasses. +- No runtime dependencies. Tests live only in `test/`, mirroring `src/`. +- New KV/document implementations must pass `test/kv-storage/kv-storage.conformance.ts` / + `test/document-storage/document-storage.suite.ts`. +- Ask the owner before pushing. diff --git a/DESIGN.md b/DESIGN.md index 93b0e3a..4021168 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,655 +1,176 @@ # PotentialStorage — Design -This document is the design brief agreed with the project owner (sections 0–11), followed by -implementation decisions made where the brief was silent (section 12) and benchmark baselines -(section 13). +Layered TypeScript storage: **KV → Events → Documents**. Each layer is an abstract base class +that implementations extend; each is usable on its own. No runtime dependencies. -## 0. Setup and workflow - -- Repository: `git@git.harvmaster.com:Harvmaster/PotentialStorage.git` (SSH; self-hosted Git, not GitHub). Clone it. If you can't reach it, stop and tell the owner rather than starting a fresh repo elsewhere. -- Inspect what's already in the repo first. If it has tooling (package.json, tsconfig, test runner), keep it. If it's empty, set up: - - TypeScript (`strict: true`, `noUncheckedIndexedAccess: true`), ESM, Node ≥ 20 - - Vitest for tests and `vitest bench` for benchmarks; `expectTypeOf` for type tests - - Package name `potential-storage` unless one already exists - - **No runtime dependencies in the core.** Dev dependencies are fine (e.g. `fast-check` for property tests). -- Copy this brief into the repo as `DESIGN.md` and create a short `CLAUDE.md` pointing at it plus the build/test commands. -- Work milestone by milestone (section 9). Commit after each milestone with a clear message, and keep tests green. Ask the owner before pushing. +This supersedes the original implementation brief (see git history before the `simplify` branch); +section 7 lists what changed and why. ## 1. Principles -1. **Three layers**, each usable on its own: **KV** → **Event** (append-only, built on KV) → **Document** (Mongo-like, built on Events or native). -2. **Drivers vs stores.** Backends implement a tiny, flat, string-keyed _driver_. All path, child, index, escaping and typing logic lives once in the user-facing _node_ classes, so every backend gets identical behaviour. -3. **One navigation primitive: `child(key)`.** There are no collection/document classes. Every node is a store over its direct children, and `child(key)` re-roots to that key. Navigation is synchronous and does no I/O. -4. **No methods on data.** Reads return plain values. -5. **Performance is a requirement.** Batch operations are the primitive; scans are ordered and streamed; nothing replays or scans more than it must (section 8). +1. **One abstract class per layer.** Implementations provide a handful of raw primitives; the + whole user-facing API is written once in the base class. A client (e.g. HTTP) and a server-side + store are the same kind of object, and wrappers (e.g. encryption) are just another subclass + that delegates the primitives. +2. **One navigation primitive: `child(key)`.** Every instance is a node; `child` returns a view of + the same storage rooted deeper (via `Object.create`, so it shares the instance's state). Sync, no I/O. +3. **Self or child by argument.** `get()` reads this node, `get(key)` a child. A key that is passed + but `undefined`/`null` throws (`InvalidKeyError`) instead of silently meaning "self". Methods use + optional parameters or tuple-union rest parameters, not overloads. +4. **Indexes belong to the store.** Document stores receive the fields to index and use them however + suits the backend (in-memory equality maps, native Mongo/SQLite/IndexedDB indexes, or nothing). +5. **Simple over clever.** Code style: classes, TS `private` (not `#`), no method overrides in + subclasses, tests only under `test/`. -## 2. Paths and key encoding (`src/core/path.ts`) - -- A node's path is `readonly string[]` of **unescaped** segments. Number keys are converted with `String()`. -- The separator is a single character, configurable, default `/`. -- **Escaping** (per segment): percent-encode `%`, the separator, and a **leading** `~`. Escaped segments never contain a raw separator. Write `escapeSegment` / `unescapeSegment` with round-trip property tests. -- **Reserved segments** start with `~` (user segments can't, because a leading `~` is escaped): - - `~i` index entries - - `~u` unique-index claims - - `~e` event commits - - `~p` document projections root -- Each node precomputes in its constructor: - - `selfKey` = escaped segments joined by the separator (e.g. `users/123`) - - `prefix` = `selfKey + sep` (`''` for the root) - - No string work per call beyond `prefix + escapeSegment(key)`. -- Keys are ordered **lexicographically** (so `10` sorts before `9`). Document this. An order-preserving numeric key encoding is a possible later option, not part of v1. - -Example layout: +## 2. Layout ``` -users/123 → KV value of users/123 -users/123/posts/p1 → KV value of a nested child -users/~i/byEmail/sada@x.io/123 → index entry (value: empty / true) -orders/o1/~e/000000000007 → event commit #7 of stream orders/o1 -~p/users/123 → document projection { state, version, commit } +src/ + kv-storage/ base-kv-storage.ts, kv-storage-in-memory.ts, kv-storage-http.ts, kv-storage-http-server.ts + event-storage/ base-event-storage.ts, event-storage-kv.ts + document-storage/ base-document-storage.ts, document-storage-in-memory.ts, document-storage-kv.ts, + event-sourced-document-storage.ts + utils/ keys, sorted-map, json, ulid, errors, schema (types), document-path/query/update +test/ mirrors src/; shared suites: kv-storage.conformance.ts, document-storage.suite.ts +bench/ ``` -## 3. Layer 1: KV - -### 3.1 Driver interface (`src/kv/driver.ts`) - -```ts -export interface ScanOptions { - gte?: string - lt?: string - limit?: number - reverse?: boolean - values?: boolean // when false/omitted, yield value as undefined -} - -export interface KVDriver { - getMany(keys: string[]): Promise<(unknown | undefined)[]> // same order as keys - setMany(entries: [key: string, value: unknown][]): Promise - deleteMany(keys: string[]): Promise - scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]> // MUST be key-ordered - - // Optional capabilities. Stores feature-detect them and fall back where possible. - setIfAbsent?(entries: [string, unknown][]): Promise // true = written - batch?(ops: BatchOp[]): Promise // atomic multi-op write - deleteRange?(gte: string, lt: string): Promise -} - -export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string } -``` - -- Values are passed through as JSON-serialisable values. Drivers that need bytes (HTTP, Redis, files) serialise internally via a pluggable `Codec` (default JSON). -- Provide an abstract `BaseKVDriver` with default implementations of `deleteRange` (scan + deleteMany) and helpers, so a minimal driver implements only the four required methods. -- **`InMemoryKVDriver`**: a sorted structure (sorted array with binary search is fine for v1; keep it behind an interface so it can become a B-tree). It implements every optional capability: `setIfAbsent`, atomic `batch` and `deleteRange`. -- Write a reusable **driver conformance suite** `runKVDriverConformance(name, makeDriver)` covering ordering, `gte`/`lt` bounds, reverse, limit, `values` on/off, `getMany` order with missing keys, and each optional capability when present. - -### 3.2 `KVStorage` / `KVNode` (`src/kv/node.ts`) - -```ts -const db = new KVStorage(driver, { separator: '/', indexes: { ... } }) // db is the root KVNode -const users = db.child('users') -const user = users.child('123') -``` - -API of every `KVNode` (types in section 7): - -| Method | Meaning | -| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `child(key)` | Sync. Returns a node at `path + key`. | -| `as()` | Type assertion for untyped use; returns the same node retyped. | -| `get()` / `get(key)` | Own value at `selfKey` / value at child key. | -| `getMany(keys)` | Batch child read, same order, `undefined` for missing. | -| `set(value)` / `set(key, value)` | Write own value / child value. | -| `setMany(entries)` | Batch child write. | -| `delete()` / `delete(key)` | Delete own value / child value only (not descendants). | -| `deleteMany(keys)` | Batch child delete. | -| `deleteTree()` / `deleteTree(key)` | Delete own value **and all descendants** (uses `deleteRange` if available). | -| `keys(opts?)` | `AsyncIterable` of **direct** child keys (unescaped). Options: `limit`, `after`, `reverse`. | -| `entries(opts?)` | `AsyncIterable<[key, value]>` of direct children that have an own value. | -| `tree({ depth })` | Assemble an object from descendant keys up to `depth` (for field-per-key style). Bounded and explicit; never the default. | -| `findBy(index, value \| { gte?, lt? }, opts?)` | Index lookup over direct children (section 3.3). | - -**Arity dispatch (important):** - -- Self vs child is decided by `arguments.length`, never by checking for `undefined`. -- If a key argument is passed but is `undefined` or `null`, throw a `TypeError` (this prevents `users.get(maybeId)` silently reading self). -- The root node (`path = []`) has no own value: `get()`, `set(v)` and `delete()` on root throw. -- Implement one private `#resolve(args)` that every method uses. - -**Direct-children listing (`keys`)** uses a skip-scan: - -1. Scan `gte: prefix`, `lt: prefix + succ(sep)`. -2. For each key, take the first segment after `prefix`. Skip reserved (`~…`) segments. Yield the segment (unescaped) if it differs from the previous one. -3. When the next key is a descendant of the segment just yielded, re-seek the driver to `gte: prefix + escSeg + succ(sep)` (where `succ(c)` is the next char code) instead of iterating through the subtree. Keep using the same iterator while keys are siblings. - -Listing must be O(direct children × seek cost), not O(subtree). A child that has no own value but has descendants **is** listed. - -### 3.3 Indexes (`src/kv/indexes.ts`) - -Indexes are declared on **path patterns** in the `KVStorage` constructor. A pattern names a parent node, and its index applies to that node's direct children's values: - -```ts -new KVStorage(driver, { - indexes: { - users: { byEmail: (u: User) => u.email }, - 'users/*/posts': { byDate: (p: Post) => p.createdAt, byTag: (p: Post) => p.tags }, - }, -}) -``` - -- An index function returns `IndexValue | IndexValue[] | undefined`, where `IndexValue = string | number | boolean | null`. An array means multi-entry; `undefined` means not indexed. -- Compile patterns into a **segment trie** (`*` matches one segment). A node resolves its index set **once, at construction**, in O(depth). -- Entry key: `~i///` with a `true` value. -- **`encodeIndexValue` must be order-preserving and separator-safe:** - - Type tag first, so values order by type: `null` < `false` < `true` < numbers < strings. - - Numbers use an order-preserving float64 encoding (flip the sign bit for positives and all bits for negatives, then fixed-width hex). - - Strings are escaped like segments. - - This enables `findBy(index, { gte, lt })` range scans. -- **Writes to an indexed node:** - 1. Read the old value(s) (`getMany`) and compute the diff of index entries. - 2. If the driver has `batch`, write the value and the index adds/removes atomically. - 3. Otherwise, write index **adds first**, then the value, then index **removes**. Treat index entries as hints: `findBy` re-reads each value, re-checks the index function, and lazily deletes stale entries (read-repair). -- Non-indexed nodes must **not** pay the read-before-write cost. - -## 4. Layer 2: Events (`src/events/`) - -An append-only log per stream. A stream is identified by a node path. Same `child()` navigation as KV. - -```ts -const events = new EventStorage(kv) // takes a KVStorage (or a KVNode as root) -const orders = events.child('orders') - -await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 }) -await orders.child('o1').append([{ type: 'paid' }]) // self form -for await (const e of orders.read('o1', { from: 1 })) { - /* { version, commit, at, data } */ -} -await orders.version('o1') -for await (const id of orders.keys()) { -} // child streams -orders.child('o1').child('shipments') -``` - -- `EventStorage` is the abstract interface. `KVEventStorage` is the KV-backed implementation. Leave room for native backends (EventStoreDB, Kafka) implementing the same interface. -- **Arity:** `append(events, opts?)` vs `append(key, events, opts?)`. Dispatch on `Array.isArray(args[0])`, plus the undefined-key rule from 3.2. `read`, `version` and `keys` follow the 3.2 rules. -- **Versions:** an empty stream has version 0, and the first event has version 1. `expectedVersion: number | 'any'` (default `'any'`). -- **Storage: one key per commit, densely numbered.** - - Key: `/~e/` - - Value: `{ firstVersion, at, events: E[] }` - - The commit sequence (not the event version) is the key so that **any** stale writer collides with the existing key. Keying by version would let a stale writer create overlapping version ranges. -- **Append:** - 1. Get the head `{ commitSeq, version }` (from the cache, or a reverse scan with `limit: 1`). - 2. If `expectedVersion` is a number and doesn't match the head, throw `ConcurrencyError`. - 3. Call `setIfAbsent(nextCommitKey)`. - 4. On `false` (collision): refresh the head. If `expectedVersion` was given, throw `ConcurrencyError`; otherwise retry up to 5 times. - 5. Without `setIfAbsent`, fall back to a per-stream in-process mutex. Document that this mode is single-writer only. -- **Head cache:** a bounded LRU keyed by stream path. Update it on successful append; invalidate it on collision. -- **`read({ from, to, reverse, limit })`** streams events with metadata. To start at version `from`, locate the containing commit by binary search over commit numbers using `getMany` probes (O(log commits) round trips), then forward-scan. -- There is no update or delete. Keys inside `~e` are never rewritten. - -## 5. Layer 3: Documents (`src/documents/`) - -`DocumentStorage` is the interface. There are two implementations: - -- `EventSourcedDocumentStorage(events, { projections?: KVNode })` is the v1 target. Projections default to the `~p` root on the same KV. -- `MongoDocumentStorage(client)` is a later milestone (section 9). - -```ts -const docs = new EventSourcedDocumentStorage(events, { indexes: { users: [{ fields: ['email'], unique: true }, { fields: ['age'] }] } }) -const users = docs.child('users') - -const id = await users.insert({ name: 'Ada', age: 36 }) // generates a time-sortable id -await users.set('123', { name: 'Ada', age: 36 }) -await users.update('123', { $set: { name: 'Ada L.' }, $inc: { logins: 1 } }) -await users.child('123').update({ $unset: { nickname: true } }) // self form -await users.get('123') -await users.delete('123') -for await (const { key, data } of users.find({ age: { $gte: 18 } }, { sort: { age: 1 }, limit: 20 })) { -} -await users.findOne({ email: 'ada@x.io' }) -await users.history('123') // event-sourced only -await users.getAt('123', 17) // state as of version 17 -users.child('123').child('posts') // nested docs, same as KV -``` - -### 5.1 Events and projections - -- The doc at path `P` has its event stream at `P` in `EventStorage`. Its projection is at `~p/P` in the projections KV: `{ state, version }` (and `deleted: true` for tombstones). -- Event types: `{ op: 'set', doc }`, `{ op: 'patch', set?: Record, unset?: path[] }`, `{ op: 'delete' }`. -- **Store resolved deltas, not operators.** For example, `$inc` resolves to `{ set: { logins: 6 } }`, so replay is deterministic. -- Supported update operators in v1: `$set`, `$unset`, `$inc`, `$push`, `$pull`, `$addToSet`, with dot-path support. - -**Write path:** - -1. Read the projection. -2. Compute the new state and patch. -3. `append` with `expectedVersion = projection.version`. -4. Write the projection and index entries (atomic `batch` if available). -5. On `ConcurrencyError`: catch up the projection from the stream, recompute, and retry (bounded). - -**Reads** are a single projection `get`. Never replay on read. - -**Crash between append and projection write:** - -- The projection lags the stream. The next write's `expectedVersion` fails, which triggers catch-up, so the system self-heals. -- Provide `get(key, { consistent: true })` that checks the stream head and catches up first. -- Provide a `catchUp(node?)` maintenance method to repair a subtree. -- `insert` ids are time-sortable (ULID-like). Implement a small generator, with no dependency. - -### 5.2 Queries and document indexes - -- Filter subset: - - `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$exists` - - `$and`, `$or`, `$not` - - implicit equality and dot paths -- Options: `sort`, `limit`, `skip`, `projection` (fields). -- `find` on a node queries that node's **direct children**. -- Indexes are declared per path pattern (same trie mechanism as 3.3). - - Each index has `fields` (compound allowed); entries go under `~p//~i////…/`, using `encodeIndexValue`. - - `unique: true` additionally claims `~u//` via `setIfAbsent`. Throw at construction if the driver lacks `setIfAbsent`. -- **Planner:** - - Pick the index that covers the most leading equality fields, then at most one range field. - - Use the index order for `sort` when it matches. - - Otherwise do a full scan of children projections, with a one-time dev-mode `console.warn` naming the filter. - -## 6. Shared conventions - -- Every layer uses the same navigation (`child`, `as`, `keys`) and the same self/child arity rules. -- All multi-result reads are `AsyncIterable` (streamed, cancellable). Never build large arrays internally. -- Errors are typed classes exported from `src/errors.ts`: `ConcurrencyError`, `UniqueConstraintError`, `CapabilityError`, `InvalidKeyError`. -- Public exports go through `src/index.ts`, with subpath exports per layer if convenient. - -## 7. Types (`src/types/schema.ts`) - -Each node is typed by a single **Spec** that describes its own value and its children by key: - -```ts -export type Spec = { value?: unknown; children?: Record } -export type AnySpec = { value: unknown; children: Record } // untyped default - -export type ValueOf = S extends { value: infer V } ? V : never -export type ChildrenOf = S extends { children: infer C } ? C : {} -export type KeyOf = Extract, string | number> -export type ChildOf> = ChildrenOf[K] - -// Helpers -export type Many = {}> = { children: { [id: string]: { value: V; children: C } } } -export type Fields = { [K in keyof T & string]: { value: T[K] } } -``` - -Node signatures: - -```ts -class KVNode { - child>(key: K): KVNode> - get(): Promise | undefined> - get>(key: K): Promise> | undefined> - // set/delete/getMany/etc. follow the same pattern -} -``` - -Example to cover with `expectTypeOf` tests: - -```ts -type User = { username: string; age: number } -type Post = { title: string; createdAt: string } -type Schema = { - children: { - users: Many }> - profile: { children: { [id: string]: { children: Fields } } } // field-per-key style - } -} - -const db = new KVStorage(driver) -db.child('users').get('123') // User | undefined -db.child('users').child('123').get() // User | undefined (same key) -db.child('users').child('123').child('posts') // KVNode> -db.child('users').child('123').child('postz') // ✗ type error -db.child('profile').child('9').get('age') // number | undefined -db.child('profile').child('9').get('agee') // ✗ type error -``` - -Notes: - -- **Named levels** check keys at compile time; **dynamic `[id: string]` levels** accept any string. -- One node's children can't mix an index signature with named keys (a TS restriction), so schemas alternate between dynamic and named levels. That alternation exists only in types; the runtime API is just `child()`. -- `EventStorage` and `DocumentStorage` nodes use the same Spec machinery (the value type is the event type or the document type respectively). - -## 8. Performance requirements - -- Single-key ops: exactly one driver call (no hidden reads), except indexed writes (one `getMany` before the write) and document writes (projection read plus append plus projection write). -- `*Many` ops: one driver call each (plus index work), never N calls. -- `keys()`: skip-scan, O(direct children) seeks. -- Event head lookup: cached; cold lookup is a single reverse scan with `limit: 1`. -- Document `get`: one driver `get`. -- Add `vitest bench` benchmarks for: - - KV `get`/`set`/`getMany` at 10k/100k keys - - `keys()` on a node with 1k children each having 100 descendants - - event append and read - - document `get`/`update`/indexed `find` vs full-scan `find` -- Record baseline numbers in `DESIGN.md`. - -## 9. Milestones - -1. **Core:** `path.ts` (escape, unescape, encode, `succ`), `encodeIndexValue`, errors. Property tests for round-trips and order preservation. -2. **KV driver:** `KVDriver`, `BaseKVDriver`, `InMemoryKVDriver`, conformance suite. -3. **KV node:** full API from 3.2, including arity rules, skip-scan `keys`, `deleteTree`, `tree`. Add a model-based property test: random ops vs a reference `Map`. -4. **KV indexes:** pattern trie, atomic and non-atomic write paths, read-repair, range `findBy`. -5. **Events:** `KVEventStorage` with commit keys, `setIfAbsent` and mutex modes, head cache, binary-search `read({ from })`, concurrency tests (simulated interleaving with two storage instances on one driver). -6. **Documents:** `EventSourcedDocumentStorage`, update operators, projections, catch-up, queries, planner, unique indexes, `history` and `getAt`. -7. **Types:** Spec machinery wired through all three layers, plus `expectTypeOf` tests from section 7. -8. **HTTP driver:** - - A `createKVHttpHandler(driver)` (framework-agnostic `Request → Response`) plus an `HttpKVDriver` client. - - Endpoints: `POST /getMany`, `POST /setMany`, `POST /deleteMany`, `POST /batch`, `POST /setIfAbsent`, `GET /scan?gte<&limit&reverse&values` (NDJSON stream). - - It must pass the conformance suite end-to-end. -9. **Benchmarks** and a `DESIGN.md` update with results. -10. **(Later, ask first)** `MongoDocumentStorage`: - - Path pattern maps to a collection (e.g. `users/*/posts` → `users.posts`), with a `_parent` field and a compound index `(_parent, _id)`. - - Native indexes map from the same index declarations. - - It should pass the document test suite where semantics allow (no `history`/`getAt`). - -## 10. Decisions already made (don't revisit without asking) - -- `child()`-only navigation; no collection/doc classes. -- `get()` with no argument reads the node's **own key**. It never assembles the subtree; `tree()` does that explicitly. -- A node's own value and its children are independent keys. -- Self/child dispatch by arity; a passed-but-undefined key throws. -- Indexes are declared on path patterns at construction. -- Events use one key per commit, densely numbered. -- Documents store resolved deltas, and reads hit projections only. -- Keys sort lexicographically. -- Values are JSON-serialisable; byte drivers use a codec. - -## 11. Open questions to raise with the owner (don't block on them) - -- Order-preserving numeric **keys** (not just index values)? -- Should multi-writer safety be **required** (a hard error without `setIfAbsent`) rather than a documented single-writer fallback? -- The hard-delete story for immutable event logs (compaction or crypto-shredding) for privacy deletion requests. - -## 12. Implementation decisions - -Decisions taken where the brief was silent. Each is the simplest option consistent with sections 1–10. - -### M1 — Core - -- **Separator** must be a single printable ASCII character other than `%` and `~`. -- **Empty keys are rejected** (`InvalidKeyError`), as are `NaN`/`±Infinity` number keys. An empty segment - would produce `a//b`, which is ambiguous with the subtree layout. -- **Separator escaping** uses `%` + two uppercase hex digits (`/` → `%2F`, `%` → `%25`, leading `~` → `%7E`). -- **`InvalidKeyError` extends `TypeError`**, so "passed-but-undefined key throws a `TypeError`" (3.2) and - the typed error class (6) are the same throw. -- **Index string encoding deviates from "escaped like segments".** Percent-escaping is not - order-preserving (`"a/"` → `"a%2F"` sorts below `"a&"`, and `"a"` + sep sorts above `"a!"`), which - would break `findBy` range scans. Instead every character the index encoder emits sorts strictly - above the separator: code units `<= succ(sep)` are written as `succ(sep)` followed by - `chr(succ(sep) + 1 + c)`. This is monotone and prefix-safe, so `a < b ⇔ enc(a)+sep+… < enc(b)+sep+…` - exactly (property-tested). Encodings are still separator-free. -- **Type tags and number digits**: tags are the first five digits of the digit alphabet; numbers use - 16 digits of `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16 characters - just above the separator. `-0` is stored as `0`; `NaN` cannot be indexed. -- **Ordering** of strings is by UTF-16 code unit (JS `<`). Drivers that compare UTF-8 bytes agree - except for astral-plane characters vs. U+E000–U+FFFF; noted as a driver-author caveat. - -### M2 — KV driver - -- **Conformance suite location**: `src/testing/kv-conformance.ts`, exported as `potential-storage/testing`. - It imports `vitest`, which is an _optional peer dependency_; the core entry point never imports it. -- **`InMemoryKVDriver` clones values** on write and read (`structuredClone`, skipped for primitives) so - stored state can't be mutated through shared references, matching serialising drivers. Opt out - with `{ clone: false }`. -- **Scans tolerate concurrent writes**: the in-memory scan remembers the last key yielded and re-seeks - if the map changed while suspended. Read-repair and `deleteTree` rely on this. The conformance suite - requires it of every driver (deleting an already-yielded key mid-scan must not skip keys). -- **`setMany` bulk path**: ≥32 new keys are sorted and merged (O(n + k log k)); fewer are spliced. -- **`BaseKVDriver.deleteRange`** deletes in chunks of 1000 (scan with `limit`, then `deleteMany`, - repeat), so it never deletes from under its own live iterator. It also provides `get`/`set`/`delete` - single-key helpers. -- **No fake `batch`**: `BaseKVDriver` deliberately does not provide a non-atomic `batch`, since stores - use its presence to mean "atomic". - -### M3 — KV node - -- **Child order is "subtree order", not plain segment order.** The brief's skip-scan assumes a - child's keys are contiguous, but they aren't when a sibling extends a segment with a character - that sorts below the separator (for `/`: space and `!"#$%&'()*+,-.`). With children `a` and `a-1` - the key space is `p/a`, `p/a-1`, `p/a-1/x`, `p/a/x`, so `a`'s descendants sort _after_ `a-1`, and - "yield if different from the previous segment" would emit `a` twice. `keys()`/`entries()` therefore - order children by **escaped segment followed by the separator** (`a-1/` < `a/`), which is the order - of their subtrees in key space. For keys without such characters this is plain lexicographic order. - - Forward listing defers a child seen only through its own key on a small stack (bounded by - segment length) until the scan passes `p//`. It costs no extra driver calls beyond the - brief's one seek per child with descendants. - - Reverse listing needs one batched `getMany` probe per new child whose segment contains a - below-separator character, to find shorter siblings that only have an own key. - - `after` is exclusive in the direction of travel and follows the same order. - - Implemented once in `src/kv/children.ts` (`scanChildren`) and reused by the event and document - layers. A model-based property test checks forward, reverse, `after` and `limit` against - brute force with keys like `a`, `a-1`, `a.b`, `a/x`, `%`, `~t`. - - **Open question for the owner:** keep subtree order, or change segment escaping so the key - layout nests exactly. That would mean escaping every below-separator character with an escape - character above the separator, at the cost of the readable percent-encoding. -- **Errors from async methods are rejections.** Every async method (`get`, `set`, …) is an `async` - function, so a bad key yields a rejected promise rather than a synchronous throw. `child()` is sync - and throws synchronously. -- **`set(undefined)` is rejected** (`TypeError`: use `delete()`); `undefined` means "missing" in reads. -- **`deleteTree()` on the root** is allowed and clears the whole store (chunked scan + delete, since - the root range has no upper bound). Subtree deletion is `delete(selfKey)` plus - `deleteRange(prefix, selfKey + succ(sep))`. It is not `[selfKey, …)`, which would also catch - siblings such as `a!`. -- **`tree({ depth })`** builds a plain object: leaves are values, interior nodes are objects. A node - with both an own value and children exposes its own value under the `''` property (empty keys are - invalid, so it can't collide). Reserved subtrees are skipped. Deeper levels are skipped by seeking - rather than reading. -- **`keys()` and `entries()` yield string keys** (number keys come back as strings). -- **Exposed low-level properties** on every node: `path`, `selfKey`, `prefix`, `end` (subtree upper - bound), `driver`, `separator`, `isRoot`. The event and document layers build on these. - -### M4 — KV indexes - -- **Pattern syntax**: segments separated by the storage separator; `*` matches one segment; `''` - names the root. A literal `*` key can't be targeted by a pattern. -- **Overlapping patterns** that declare the same index name throw at construction. -- **Index trie state is stepped per `child()` call** (O(1) in patterns), so each node resolves its - indexes in O(depth) overall. A node holds two bound index sets: one over its children (for child - writes) and its parent's (for self-form writes such as `users.child('1').set(v)`). -- **Index function results**: `undefined` → not indexed; arrays are multi-entry and de-duplicated - per child; anything that is not `string | number | boolean | null` (or `NaN`) throws a `TypeError` - at write time. Index functions are not called for missing (`undefined`) values. -- **`findBy` results** are `[key, value]` pairs (the value is re-read anyway for verification). - Range queries accept `gt`/`gte`/`lt`/`lte`. Results are in index-value order, then child-key order, - and range queries de-duplicate multi-entry hits. -- **`findBy` batching**: entry keys are read in chunks of 128 (or of the remaining `limit`, if - smaller) and values fetched with one `getMany` per chunk. Stale entries are deleted with one - `deleteMany` per chunk. -- **Read-repair race (non-atomic drivers only)**: a reader can see a concurrent writer's fresh - index add before that writer's value write lands, judge it stale, and delete it. With multiple - writers on a driver without `batch`, this can lose an index entry. Mitigation: `readRepair: false` - on `KVStorage` (stale entries are then only skipped, never deleted). With `batch`, entries never go - stale, so the race doesn't arise. -- **`set` of an indexed child with duplicate keys in one `setMany`**: last write wins, and index - diffs are computed once per key. - -### M5 — Events - -- **`EventStorage` is an interface (type), not a class.** The brief's sample says `new EventStorage(kv)` - but also says it is the abstract interface; the concrete class is `KVEventStorage(kv, opts?)`, which - accepts a `KVStorage` or any `KVNode` as root. Options: `headCacheSize` (default 10 000), - `maxRetries` (default 5) and `now` (clock, default `Date.now`). -- **Commit value**: `{ firstVersion, at, events }`, with `at` in ms since epoch. Read results are - `{ version, commit, at, data }`. -- **The root node is not a stream** (consistent with the root having no own KV value). Non-root - nodes are both a stream and a container of child streams; `keys()` lists child streams, and a - node's `~e` subtree is reserved, so streams and children never mix. -- **`version()` always reads storage** (one reverse scan with `limit: 1`) and refreshes the head cache. - Only `append` trusts the cache. When a cached head disagrees with a numeric `expectedVersion`, - the head is re-read before throwing, so a stale cache never produces a false `ConcurrencyError`. -- **`append([])`** writes nothing; it still checks `expectedVersion` against a fresh head and returns it. -- **Read arity**: `read()` and `read(opts)` are the self form (the first argument is a non-null - object). Any other first argument is a key, so `read(undefined)` throws like every other - passed-but-undefined key. -- **Locating `from`/`to`**: commit `n` has `firstVersion >= n`, so the commit containing version `v` has - `seq <= v`. `read` probes `seq = v` first; with single-event commits that is the answer, in one - `getMany`. Otherwise it runs an 8-ary search, one `getMany` of up to 8 probes per round, which is - O(log₈ commits) round trips. The head is only read when the guess is past the end. -- **Mutex mode** (driver without `setIfAbsent`): a per-stream in-process promise chain serialises - appends. It is safe only with a single writer process. - -### M6 — Documents - -- **Projection value** is `{ state, version, commit }`, or `{ deleted: true, version, commit }` for a - tombstone. Tombstones keep the version, so re-creating a document continues its stream. -- **`expectedCommit` append hint.** Writes pass the projection's `commit` to `append` as - `expectedCommit`, so a write is exactly projection `getMany` + `setIfAbsent` + projection `batch` - (3 driver calls), even with a cold head cache. A hint taken from a projection can only be stale - (which causes a collision, then catch-up and retry), never ahead of the stream, so it cannot - create a gap. Non-KV backends may ignore it. -- **Per-document in-process write lock.** Writes to the same document within one storage instance - are serialised (`KeyedMutex`), so optimistic retries (default 5) are spent only on cross-process - contention. -- **Return values**: `set` → `void`; `insert` → id; `update` → new state, or `undefined` if the document - doesn't exist (pass `{ upsert: true }` to treat missing as `{}`); `delete` → whether a document - was deleted. No-op updates (resulting state equal to current) append nothing. -- **Arity for documents**: in forms with an optional leading key (`get`, `set`, `update`), a non-null - object first argument selects the self form, and any other first argument is a key (so `undefined` - throws). `delete`/`history` use argument count. `getAt(version)` vs `getAt(key, version)` also uses - argument count, because a version and a numeric key are both numbers. -- **The root document node has no own document** (consistent with KV). -- **`keys()` on a document node** lists children holding a live document. Unlike KV `keys()`, it skips - children that only have nested documents or tombstones. -- **`find` results** are `{ key, data }`. `findOne` returns the same shape or `undefined`. -- **Filter semantics vs MongoDB** (see `src/documents/filter.ts`): - - Comparisons are type-bracketed (`$gt: 5` matches only numbers), and cross-type order follows - the index encoding: `null/missing < false < true < numbers < strings < other`. - - On array fields, every positive operator in one condition (`$eq $gt $gte $lt $lte $in`) must be - satisfied by the **same element** (like `$elemMatch`). In MongoDB, `{ s: { $gt: 5, $lt: 10 } }` - matches `[1, 20]`; here it doesn't. This makes index range scans exact. To get MongoDB's - behaviour, split the condition with `$and`. - - `{ f: null }` matches missing fields. `$not` works both at top level (negating a filter) and at - field level. Unknown operators throw. - - `$in` is not index-accelerated in v1. -- **Index entries** for each field value: scalars as-is, arrays as their scalar elements (multikey; - empty arrays → `null`), and missing or non-scalar values as `null`. Every live document therefore - has at least one entry per index, so a full index scan sees all documents. Indexing a document - where more than one field of a compound index has multiple values throws (as MongoDB does). -- **Index names** default to the fields joined by `,`. -- **Unique indexes**: claims are scoped **per parent node** (`/~u//`), like one - MongoDB collection per parent. Value tuples where every field is missing/null are not claimed - (sparse). Claims are taken before appending, then released if the append fails. A claim value is - `[ownerKey, claimedAtMs]`. A claim held by another document is taken over only if that - document, after catching up from its stream, no longer holds the value, **and** the claim is older - than `claimGraceMs` (default 30 s). The owner might otherwise be between claiming and appending. - The residual risk is a writer stalling for longer than the grace period between claiming and - appending. -- **Planner**: per index, count leading equality fields (k), then whether field k has a range; - score = 2k + (range ? 1 : 0). Ties prefer an index whose order satisfies `sort` after dropping - equality-constrained sort fields. With no usable predicate, an index matching `sort` is still used - as an ordered full index scan, which streams and stops early with `limit`. Range bounds are clamped - to the bound's type class. Index-backed results are re-read from projections, re-checked against the - full filter, de-duplicated, and stale entries are read-repaired. -- **Result order** when `sort` is not given is the plan's order (index order, or child-key order for - a full scan). Ties in sorted results are broken by child key in the direction of the first sort - field, both for index-ordered and in-memory sorts. -- **In-memory sort** keeps a bounded top `skip + limit` buffer when a limit is given. Without a limit, - it buffers all matches; this is the one place results are materialised. -- **Full-scan warning**: `console.warn` once per filter _shape_ (values replaced by `?`) per storage - instance, only for non-empty filters. On by default unless `NODE_ENV === 'production'`; set it with - `warnOnFullScan`. -- **Catch-up** re-asserts unique claims for the caught-up state with plain writes, since the log is - the source of truth. A slow catch-up can briefly overwrite a newer projection with an older one; - the next write's `expectedVersion` check detects this and heals it. `catchUp()` walks the subtree - via event-stream `keys()` and returns the number of documents repaired. -- **Projections for non-KV event stores** must be passed explicitly (`projections: KVNode`); the - projection key of a document is then `projections.prefix + docKey`. - -### M7 — Types - -- **`AnySpec` uses an index signature** (`{ [key: string]: AnySpec }`), not `Record`, so - `KeyOf` is `string | number` and number keys type-check on untyped stores. -- **`ChildOf` guards with `infer C extends Spec`** so that `KVNode>` satisfies the `Spec` - constraint generically. -- **`setMany` entries are a union of precise `[key, value]` pairs** (`EntryOf`), so on named levels a - value must match its own key (`['age', 'x']` is rejected). -- **Listings**: `keys()` yields `` `${KeyOf}` `` (string literals on named levels, `string` on dynamic - ones). `entries()` and `findBy()` yield `[string, V]` on dynamic levels. -- **"Accept any node" parameters** are typed `KVNode` / `EventNode`. A typed node is - assignable to these, but not to `KVNode`, because `KeyOf` collapses to `never`. -- **Type tests** live in `test/types.test-d.ts` and are enforced by `tsc --noEmit` (part of `npm test`), - including the `@ts-expect-error` negative cases from section 7. - -### M8 — HTTP driver - -- **Extra endpoints** beyond the brief's list: `POST /deleteRange {gte, lt}` and - `GET /capabilities`. Unsupported optional operations return `501`, driver errors `500`, and unknown - routes `404`. -- **Missing vs `null`**: `/getMany` returns `{ values, missing }`, where `missing` lists the indices of - absent keys, because `null` is a legitimate stored value. -- **Codec scope**: the codec serialises whole request/response bodies and each NDJSON scan line - (`[key]` or `[key, value]`), so it must not emit newlines. The default is JSON. -- **Capabilities on the client**: `HttpKVDriver` defines `setIfAbsent`/`batch` only when the server - supports them, so stores' feature detection (e.g. unique indexes requiring `setIfAbsent`, atomic - index writes using `batch`) behaves as it would locally. `HttpKVDriver.connect()` discovers them via - `/capabilities`. The constructor takes them explicitly (default: all). Without native - `deleteRange` the client falls back to `BaseKVDriver`'s chunked delete. -- **Scan streaming**: the server pulls the driver iterator in chunks that grow from 16 to 256 - entries, so short reads (skip-scan re-seeks) don't make it run far ahead. Cancelling the client - iterator cancels the response body, which calls `return()` on the server-side iterator. -- **Optional `headers`** (e.g. auth) and an injectable `fetch` on the client; an optional `basePath` on - the handler. -- The conformance suite runs end-to-end in-process (fetch → handler), over a real `node:http` - socket, and against a minimal server driver with discovered capabilities. A smoke test runs KV - indexes, events and unique-indexed documents over HTTP. - -### M9 — Benchmarks and in-memory driver performance - -- **The in-memory driver's sorted array was replaced by `ChunkedSortedMap`** (sorted chunks of ≤1024 - keys, split in half on overflow; values in a `Map`). Profiling showed 59% of document-update time - in the sorted array's O(n) `splice` once the store held ~70k keys (each write adds commit, index - and claim keys). Inserts and deletes are now O(log n + 1024). Iteration is O(1) per step and - re-seeks after a mutation, which replaces the old version-counter revalidation in the driver. - The structure stays behind the `SortedMap` interface; a B-tree remains possible. -- **Value cloning uses a hand-written JSON deep copy** instead of `structuredClone` (about 10× cheaper - per value), in the in-memory driver and the document layer. Values are JSON by contract (§10). - - - -## 13. Benchmark baselines - -`npm run bench` (Vitest bench), in-memory driver with default cloning, Node 22.22, Linux x64 cloud -container (shared CPU, so expect ±10–20% run-to-run). Mean time per operation. - -**KV** (`bench/kv.bench.ts`): values are small objects. - -| Operation | 10k keys | 100k keys | -| -------------------- | -------- | --------- | -| `get` | 1.2 µs | 1.6 µs | -| `set` (overwrite) | 1.7 µs | 2.0 µs | -| `getMany` × 100 keys | 21 µs | 22 µs | - -**`keys()`** (`bench/keys.bench.ts`): node with 1 000 children, each with 100 descendants (101k keys). - -| Operation | Mean | -| -------------------------------- | ------ | -| `keys()` skip-scan (1 000 seeks) | 2.9 ms | -| Reference: full subtree scan | 36 ms | - -**Events** (`bench/events.bench.ts`): single-event commits. - -| Operation | Mean | -| ---------------------------------------------- | ------ | -| `append` 1 event (warm head cache) | 6.3 µs | -| `read` 100 events from version 5 000 of 10 000 | 174 µs | -| `read` full 10 000-event stream | 21 ms | -| `version()` (reverse scan, `limit: 1`) | 3.9 µs | - -**Documents** (`bench/documents.bench.ts`): 10 000 docs, indexes on `age` and unique `email`. - -| Operation | Mean | -| ----------------------------------------------------------------- | ------- | -| `get` | 2.6 µs | -| `update` (`$inc`, indexed node: projection read + append + batch) | 22 µs | -| `findOne` by unique index (`email`) | 16 µs | -| `find` indexed (`age = k`, ~100 hits) | 0.57 ms | -| `find` full scan (`city = k`, ~200 hits of 10k) | 28 ms | - -For comparison, the first run with the sorted-array map and `structuredClone` measured: KV `get` -3.7 µs, `getMany`×100 186 µs, `append` 34 µs, document `update` 110 µs, full scan 57 ms, and -full stream read 62 ms. +## 3. Keys (`utils/keys.ts`) + +- A node's key is its escaped path segments joined by `/` (`users/123`). The separator is fixed. +- **Escaping:** every character at or below `/` (space, `!"#$%&'()*+,-.`, `/`, control characters) + and `~` itself becomes `~XX` (uppercase hex). Escaped segments contain only characters above `/`, + so all keys of one child sort together (`p/a`, `p/a/…`, then `p/a~2D1` for `a-1`). That keeps + child listing a plain skip-scan. +- **Reserved segments** are `~` + a lowercase letter (`~e` = event commits). Escaping only produces + `~` + uppercase hex, so user keys can't collide. +- Keys sort lexicographically by escaped form (`10` before `9`; `a-1` → `a~2D1` after `a0`). +- Empty keys, `NaN`/`Infinity` and non-string/number keys are rejected. + +## 4. KV storage + +`BaseKVStorage` primitives (absolute keys): + +| Primitive | | +| ------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `readMany(keys)` | values in order, `undefined` for missing | +| `writeMany(entries)` | | +| `removeMany(keys)` | | +| `writeIfAbsent(entries)` | `true` = written; atomic per key (needed by events) | +| `scan({ gte, lt, limit, reverse, values })` | key-ordered; must keep working if keys change mid-scan (keys present throughout appear exactly once) | + +Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / `set(key, value)`, +`setMany`, `delete(key?)`, `deleteMany`, `deleteTree(key?)`, `keys({ limit, after, reverse })`, +`entries(...)`. The root has no own value. A node's own value and its children are independent keys. + +- **`keys()`** is a skip-scan: one seek per child that has descendants; siblings without + descendants share one iterator. It lists children that only have descendants, and skips + reserved segments. +- **The only KV "index" is the ordered key space.** There is no `find`: with KV you know where to look, + or you list a node's children. +- **Implementations:** `KVStorageInMemory` (`SortedMap`: sorted chunks of ≤1024 keys; values deep-copied); + `KVStorageHttp` (client) + `KVStorageHttpServer` (wraps any KV storage; `handle(Request) → Response`, + `POST /` with JSON bodies, `scan` streamed as NDJSON). The server exposes the whole + underlying store (primitives use absolute keys). +- **Wrappers** (e.g. a future encryption layer) extend `BaseKVStorage` and delegate the primitives. + Keys must stay plaintext (or be deterministically encrypted) for prefix scans to work; values can be + encrypted freely. + +## 5. Event storage + +`BaseEventStorage` primitives over stream paths: `appendTo`, `readFrom`, `versionOf`, `streamsOf`. +Node API: `append(events, opts?)` / `append(key, events, opts?)`, `read(opts?)` / `read(key, opts?)`, +`version(key?)`, `keys()`. The root is not a stream. An empty stream is version 0; the first event is 1. +`expectedVersion: number | 'any'` (default `'any'`). + +**`EventStorageKV`** (on any KV storage): + +- One key per append, densely numbered: `/~e/` → `{ firstVersion, at, events }`. + Keying by commit number (not version) means any writer with a stale view collides with the + existing key. +- Append: head from a bounded cache (else one reverse scan, `limit: 1`) → check `expectedVersion` + (re-reading the head before rejecting, in case the cache is stale) → `writeIfAbsent`. On a lost race: + `ConcurrencyError` if a version was expected, otherwise retry (default 5). +- `read({ from })`: commit `n` starts at a version ≥ n, so try commit `from` first (exact for + single-event commits), else binary-search; then forward-scan. +- Multi-writer safe on any store, because `writeIfAbsent` is a required primitive. + +## 6. Document storage + +`BaseDocumentStorage` primitives over document paths: + +| Primitive | | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| `readMany(paths)` | | +| `mutate(path, change)` | maps current state → `DocChange` (or `null` = no-op), applies it, returns the new state; the implementation decides how it's made safe | +| `query(parent, DocumentQuery)` | children of `parent`, filtered, sorted and paged | + +Node API: `get(key?)`, `getMany`, `set(doc)` / `set(key, doc)`, `insert(doc)` → ULID-like id, +`update(spec, opts?)` / `update(key, spec, opts?)` → new state or `undefined` (`{ upsert }`), +`delete(key?)` → existed, `find(filter, { sort, limit, skip })`, `findOne`, `keys()`. + +- **Queries** (`utils/document-query.ts`): `{ field: value }` (dot paths; `null` also matches missing), + `$eq $gt $gte $lt $lte $in`. Comparisons are type-bracketed. On arrays a condition matches the array + or any one element. Sorting order: missing/null < booleans < numbers < strings < other. +- **Updates** (`utils/document-update.ts`): `$set $unset $inc $push $pull $addToSet` (`$each` for + push/addToSet). They resolve to a `DocChange` with final values (`$inc` → `set: { n: 6 }`), so + changes replay deterministically. +- **Indexes:** `{ indexes: { users: ['email', 'age'], 'users/*/posts': ['createdAt'] } }`, passed to + the implementation. + +**Implementations** + +- `DocumentStorageInMemory`: maps per collection; equality index per declared field (used for + `{ f: v }` / `{ f: { $eq: v } }`), otherwise scans. Writes are atomic in-process. +- `DocumentStorageKV`: each document is the KV value at its path; queries scan the collection's + children; no indexes; concurrent writes to one document are last-writer-wins. +- `EventSourcedDocumentStorage(events, projections)`: each document's `DocChange`s are appended to + its own stream. Current state lives in **any** `BaseDocumentStorage` used as the projection store, + which serves all reads, queries and indexes. + - Projections carry `_version` (reserved; stripped from results), used as `expectedVersion` on the + next append. + - If a write crashes between append and projection update, the next write conflicts, rebuilds the + projection from the stream, and retries. `catchUp(key?)` rebuilds on demand. + - Missing projections with stream history (deleted documents, crashed first writes) are rebuilt + before writing. + - Writes to one document are serialised in-process, so retries only happen across processes. + - Extras: `history(key?)`, `getAt(version)` / `getAt(key, version)`. Children keep these methods + in their types through interface declaration merging (types only). + +## 7. Changes from the original brief + +- Drivers + node classes → one abstract class per layer (client and server share an API). +- KV indexes (`findBy`, pattern trie, order-preserving value encoding, read-repair) removed. + Document indexes are declared as fields and handled by each store; unique indexes and the query + planner were removed. +- Queries: `$ne $nin $exists $and $or $not` and projections dropped. +- Separator fixed to `/`, and every character ≤ `/` is escaped so child keys stay contiguous. This + removed the subtree-order workaround in child listing. +- Event-sourced documents are events + a projection store instead of a bespoke projection format. +- `writeIfAbsent` is a required primitive (no single-writer mutex fallback; no optional `batch` / + `deleteRange`). +- `tree()` removed (use `entries()` for field-per-key reads). + +## 8. Open questions + +- Hard deletes from immutable event logs (privacy requests): compaction vs crypto-shredding. The + encryption wrapper could support per-subject keys. +- Order-preserving numeric keys (`9` before `10`) as an opt-in key encoding. + +## 9. Benchmark baselines + +`npm run bench`, in-memory stores, Node 22, Linux x64 cloud container (shared CPU, ±10–20%). Mean per operation. + +| Benchmark | Mean | +| -------------------------------------------------------------------- | --------------- | +| KV `get` / `set` @ 10k keys | 1.5 / 1.7 µs | +| KV `get` / `set` @ 100k keys | 2.0 / 2.3 µs | +| KV `getMany` × 100 | 25–35 µs | +| `keys()` on 1k children × 100 descendants (skip-scan vs full scan) | 2.8 ms vs 34 ms | +| Event `append` (warm cache) | 7.8 µs | +| Event `read` 100 from version 5 000 of 10k | 131 µs | +| Event `read` full 10k stream | 18 ms | +| Documents in-memory @ 10k: `get` / `update` | 1.8 / 6.6 µs | +| Documents in-memory: `find` indexed (~100 hits) / scan (~200 of 10k) | 0.20 / 11 ms | +| Event-sourced @ 10k: `get` / `update` | 1.6 / 17 µs | +| Event-sourced: `find` indexed / scan | 0.38 / 12 ms | diff --git a/bench/document-storage.bench.ts b/bench/document-storage.bench.ts new file mode 100644 index 0000000..064a92f --- /dev/null +++ b/bench/document-storage.bench.ts @@ -0,0 +1,33 @@ +import { bench, describe } from 'vitest' +import { DocumentStorageInMemory } from '../src/document-storage/document-storage-in-memory.js' +import { EventSourcedDocumentStorage } from '../src/document-storage/event-sourced-document-storage.js' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' +import type { BaseDocumentStorage } from '../src/document-storage/base-document-storage.js' + +const N = 10_000 +const stores: [string, BaseDocumentStorage][] = [ + ['in-memory', new DocumentStorageInMemory({ indexes: { users: ['age'] } })], + ['event-sourced', new EventSourcedDocumentStorage(new EventStorageKV(new KVStorageInMemory()), new DocumentStorageInMemory({ indexes: { users: ['age'] } }))], +] +for (const [name, docs] of stores) { + const users = docs.child('users') + for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, age: i % 100, city: `c${i % 50}` }) + let i = 0 + describe(`documents (${name}) @ 10k`, () => { + bench('get', async () => { + await users.get(`u${i++ % N}`) + }) + bench('update ($inc)', async () => { + await users.update(`u${i++ % N}`, { $inc: { logins: 1 } }) + }) + bench('find by indexed field (~100 hits)', async () => { + for await (const _ of users.find({ age: i++ % 100 })) { + } + }) + bench('find by unindexed field (scan, ~200 hits)', async () => { + for await (const _ of users.find({ city: `c${i++ % 50}` })) { + } + }) + }) +} diff --git a/bench/documents.bench.ts b/bench/documents.bench.ts deleted file mode 100644 index 29aefdb..0000000 --- a/bench/documents.bench.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' -import { KVEventStorage } from '../src/events/kv.js' -import { EventSourcedDocumentStorage } from '../src/documents/event-sourced.js' - -const N = 10_000 -const docs = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { - indexes: { users: [{ fields: ['age'] }, { fields: ['email'], unique: true }] }, - warnOnFullScan: false, -}) -const users = docs.child('users') -for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, email: `u${i}@x.io`, age: i % 100, city: `c${i % 50}` }) -let i = 0 - -describe(`documents @ ${N / 1000}k docs`, () => { - bench('get', async () => { - await users.get(`u${i++ % N}`) - }) - bench('update ($inc, indexed node)', async () => { - await users.update(`u${i++ % N}`, { $inc: { logins: 1 } }) - }) - bench('findOne by unique index (email)', async () => { - await users.findOne({ email: `u${i++ % N}@x.io` }) - }) - bench('find indexed (age = k, ~100 hits)', async () => { - for await (const _ of users.find({ age: i++ % 100 })) { - } - }) - bench('find full scan (city = k, ~200 hits)', async () => { - for await (const _ of users.find({ city: `c${i++ % 50}` })) { - } - }) -}) diff --git a/bench/event-storage.bench.ts b/bench/event-storage.bench.ts new file mode 100644 index 0000000..0c1a695 --- /dev/null +++ b/bench/event-storage.bench.ts @@ -0,0 +1,22 @@ +import { bench, describe } from 'vitest' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' + +const events = new EventStorageKV(new KVStorageInMemory()) +const streams = events.child('streams') +for (let i = 0; i < 10_000; i++) await streams.append('long', [{ i }]) +let s = 0 + +describe('events', () => { + bench('append 1 event (warm cache)', async () => { + await streams.append(`s${s++ % 1000}`, [{ type: 'e' }]) + }) + bench('read 100 events from version 5000 of 10k', async () => { + for await (const _ of streams.read('long', { from: 5000, limit: 100 })) { + } + }) + bench('read full 10k-event stream', async () => { + for await (const _ of streams.read('long')) { + } + }) +}) diff --git a/bench/events.bench.ts b/bench/events.bench.ts deleted file mode 100644 index 22200cc..0000000 --- a/bench/events.bench.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' -import { KVEventStorage } from '../src/events/kv.js' - -const events = new KVEventStorage(new KVStorage(new InMemoryKVDriver())) -const streams = events.child('streams') -const long = streams.child('long') -for (let i = 0; i < 10_000; i++) await long.append([{ type: 'tick', i }]) -let s = 0 - -describe('events', () => { - bench('append 1 event (warm head cache)', async () => { - await streams.append(`s${s++ % 1000}`, [{ type: 'e' }]) - }) - bench('read 100 events from version 5000 of 10k', async () => { - for await (const _ of long.read({ from: 5000, limit: 100 })) { - } - }) - bench('read full 10k-event stream', async () => { - for await (const _ of long.read()) { - } - }) - bench('version() (cold head lookup)', async () => { - await long.version() - }) -}) diff --git a/bench/keys.bench.ts b/bench/keys.bench.ts deleted file mode 100644 index dac2c87..0000000 --- a/bench/keys.bench.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' - -// 1k children, each with 100 descendants (100k keys under the node). -const db = new KVStorage(new InMemoryKVDriver({ clone: false })) -const node = db.child('n') -const writes: [string, number][] = [] -for (let c = 0; c < 1000; c++) { - writes.push([`n/c${c}`, c]) - for (let d = 0; d < 100; d++) writes.push([`n/c${c}/d${d}`, d]) -} -await db.driver.setMany(writes) - -describe('keys() on 1k children x 100 descendants', () => { - bench('keys() skip-scan', async () => { - let n = 0 - for await (const _ of node.keys()) n++ - if (n !== 1000) throw new Error(`expected 1000, got ${n}`) - }) - bench('naive full subtree scan (reference)', async () => { - let n = 0 - for await (const _ of db.driver.scan({ gte: 'n/', lt: 'n0' })) n++ - if (n !== 101_000) throw new Error(`expected 101000, got ${n}`) - }) -}) diff --git a/bench/kv-storage.bench.ts b/bench/kv-storage.bench.ts new file mode 100644 index 0000000..8db217c --- /dev/null +++ b/bench/kv-storage.bench.ts @@ -0,0 +1,40 @@ +import { bench, describe } from 'vitest' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' + +for (const n of [10_000, 100_000]) { + const db = new KVStorageInMemory() + await db.writeMany(Array.from({ length: n }, (_, i) => [`users/u${i}`, { name: `user ${i}`, n: i }])) + const users = db.child('users') + const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`) + let i = 0 + describe(`KV @ ${n / 1000}k keys`, () => { + bench('get', async () => { + await users.get(`u${i++ % n}`) + }) + bench('set', async () => { + await users.set(`u${i++ % n}`, { name: 'x', n: i }) + }) + bench('getMany x100', async () => { + await users.getMany(keys) + }) + }) +} + +const tree = new KVStorageInMemory() +const writes: [string, number][] = [] +for (let c = 0; c < 1000; c++) { + writes.push([`n/c${c}`, c]) + for (let d = 0; d < 100; d++) writes.push([`n/c${c}/d${d}`, d]) +} +await tree.writeMany(writes) + +describe('keys() on 1k children x 100 descendants', () => { + bench('keys() skip-scan', async () => { + for await (const _ of tree.child('n').keys()) { + } + }) + bench('full subtree scan (reference)', async () => { + for await (const _ of tree.scan({ gte: 'n/', lt: 'n0' })) { + } + }) +}) diff --git a/bench/kv.bench.ts b/bench/kv.bench.ts deleted file mode 100644 index f293231..0000000 --- a/bench/kv.bench.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' - -async function populated(n: number) { - const db = new KVStorage(new InMemoryKVDriver()) - const users = db.child('users') - const batch: [string, { name: string; n: number }][] = [] - for (let i = 0; i < n; i++) batch.push([`u${i}`, { name: `user ${i}`, n: i }]) - await users.setMany(batch) - return users -} - -for (const n of [10_000, 100_000]) { - const users = await populated(n) - const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`) - let i = 0 - describe(`KV @ ${n / 1000}k keys`, () => { - bench('get', async () => { - await users.get(`u${i++ % n}`) - }) - bench('set (overwrite)', async () => { - await users.set(`u${i++ % n}`, { name: 'x', n: i }) - }) - bench('getMany x100', async () => { - await users.getMany(keys) - }) - }) -} diff --git a/package.json b/package.json index 5046331..1f1e75a 100644 --- a/package.json +++ b/package.json @@ -12,26 +12,6 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" - }, - "./kv": { - "types": "./dist/kv/index.d.ts", - "import": "./dist/kv/index.js" - }, - "./events": { - "types": "./dist/events/index.d.ts", - "import": "./dist/events/index.js" - }, - "./documents": { - "types": "./dist/documents/index.d.ts", - "import": "./dist/documents/index.js" - }, - "./http": { - "types": "./dist/http/index.d.ts", - "import": "./dist/http/index.js" - }, - "./testing": { - "types": "./dist/testing/index.d.ts", - "import": "./dist/testing/index.js" } }, "files": [ @@ -53,13 +33,5 @@ "prettier": "^3.9.10", "typescript": "^5.9.3", "vitest": "^3.2.7" - }, - "peerDependencies": { - "vitest": ">=3" - }, - "peerDependenciesMeta": { - "vitest": { - "optional": true - } } } diff --git a/src/core/encoding.ts b/src/core/encoding.ts deleted file mode 100644 index b7642f7..0000000 --- a/src/core/encoding.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { DEFAULT_SEPARATOR, succ } from './path.js' - -export type IndexValue = string | number | boolean | null - -/** - * Order-preserving, separator-safe encoding of index values. - * - * Every character the encoder emits sorts strictly above the separator. That makes encodings - * prefix-safe inside keys: `enc(a) + sep + …` sorts below `enc(b)` whenever `enc(a)` is a proper - * prefix of `enc(b)`, so `a < b ⇔ key(a, …) < key(b, …)` and range scans over index entries are exact. - * - * Layout: one type-tag character, then a body. - * - Type tags order `null < false < true < number < string`. - * - Numbers: IEEE-754 float64, big-endian; positive values have the sign bit flipped, negative - * values have all bits flipped; written as 16 fixed-width "hex" digits. `-0` is normalised to `0`; - * `NaN` is rejected. - * - Strings: each code unit `c` with `c <= E` (where `E = succ(sep)`) is written as `E` followed by - * `chr(E + 1 + c)`; every other code unit is written as-is. This is monotone, so order is kept. - * - * The 16 digits are `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16 - * characters immediately after the separator. - */ -export interface IndexValueCodec { - readonly sep: string - encode(value: IndexValue): string - decode(encoded: string): IndexValue -} - -const codecs = new Map() - -export function indexValueCodec(sep: string = DEFAULT_SEPARATOR): IndexValueCodec { - let codec = codecs.get(sep) - if (!codec) { - codec = makeCodec(sep) - codecs.set(sep, codec) - } - return codec -} - -export function encodeIndexValue(value: IndexValue, sep: string = DEFAULT_SEPARATOR): string { - return indexValueCodec(sep).encode(value) -} - -export function decodeIndexValue(encoded: string, sep: string = DEFAULT_SEPARATOR): IndexValue { - return indexValueCodec(sep).decode(encoded) -} - -/** Encodes a compound value as `enc(v1) + sep + enc(v2) + …`. */ -export function encodeIndexValues(values: readonly IndexValue[], sep: string = DEFAULT_SEPARATOR): string { - const codec = indexValueCodec(sep) - let out = '' - for (let i = 0; i < values.length; i++) { - if (i > 0) out += sep - out += codec.encode(values[i]!) - } - return out -} - -export function isIndexValue(v: unknown): v is IndexValue { - return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v)) -} - -function makeCodec(sep: string): IndexValueCodec { - const sepCode = sep.charCodeAt(0) - const digits = sepCode < 0x30 ? '0123456789abcdef' : Array.from({ length: 16 }, (_, i) => String.fromCharCode(sepCode + 1 + i)).join('') - const digitValue = new Map([...digits].map((d, i) => [d, i])) - const TAG_NULL = digits[0]! - const TAG_FALSE = digits[1]! - const TAG_TRUE = digits[2]! - const TAG_NUMBER = digits[3]! - const TAG_STRING = digits[4]! - const esc = succ(sep) - const escCode = esc.charCodeAt(0) - - const view = new DataView(new ArrayBuffer(8)) - - function hex32(n: number): string { - let out = '' - for (let shift = 28; shift >= 0; shift -= 4) out += digits[(n >>> shift) & 0xf] - return out - } - - function encodeNumber(n: number): string { - if (Number.isNaN(n)) throw new TypeError('NaN cannot be indexed') - view.setFloat64(0, n === 0 ? 0 : n) // normalise -0 - let hi = view.getUint32(0) - let lo = view.getUint32(4) - if (hi & 0x80000000) { - hi = ~hi >>> 0 - lo = ~lo >>> 0 - } else { - hi = (hi ^ 0x80000000) >>> 0 - } - return hex32(hi) + hex32(lo) - } - - function decodeNumber(body: string): number { - if (body.length !== 16) throw new TypeError(`Bad encoded number ${JSON.stringify(body)}`) - let hi = 0 - let lo = 0 - for (let i = 0; i < 8; i++) hi = (hi << 4) | digitValue.get(body[i]!)! - for (let i = 8; i < 16; i++) lo = (lo << 4) | digitValue.get(body[i]!)! - hi >>>= 0 - lo >>>= 0 - if (hi & 0x80000000) { - hi = (hi ^ 0x80000000) >>> 0 - } else { - hi = ~hi >>> 0 - lo = ~lo >>> 0 - } - view.setUint32(0, hi) - view.setUint32(4, lo) - return view.getFloat64(0) - } - - function encodeString(s: string): string { - let out = '' - let start = 0 - for (let i = 0; i < s.length; i++) { - const c = s.charCodeAt(i) - if (c <= escCode) { - out += s.slice(start, i) + esc + String.fromCharCode(escCode + 1 + c) - start = i + 1 - } - } - return start === 0 ? s : out + s.slice(start) - } - - function decodeString(body: string): string { - if (body.indexOf(esc) === -1) return body - let out = '' - for (let i = 0; i < body.length; i++) { - const c = body[i]! - if (c === esc) { - const next = body.charCodeAt(++i) - out += String.fromCharCode(next - escCode - 1) - } else out += c - } - return out - } - - return { - sep, - encode(value) { - if (value === null) return TAG_NULL - switch (typeof value) { - case 'boolean': - return value ? TAG_TRUE : TAG_FALSE - case 'number': - return TAG_NUMBER + encodeNumber(value) - case 'string': - return TAG_STRING + encodeString(value) - default: - throw new TypeError(`Cannot index value of type ${typeof value}`) - } - }, - decode(encoded) { - const tag = encoded[0] - const body = encoded.slice(1) - switch (tag) { - case TAG_NULL: - return null - case TAG_FALSE: - return false - case TAG_TRUE: - return true - case TAG_NUMBER: - return decodeNumber(body) - case TAG_STRING: - return decodeString(body) - default: - throw new TypeError(`Bad encoded index value ${JSON.stringify(encoded)}`) - } - }, - } -} diff --git a/src/core/lru.ts b/src/core/lru.ts deleted file mode 100644 index c66c07a..0000000 --- a/src/core/lru.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** Minimal bounded LRU over `Map` insertion order. */ -export class LRU { - readonly #map = new Map() - - constructor(readonly capacity: number) {} - - get(key: K): V | undefined { - const v = this.#map.get(key) - if (v !== undefined) { - this.#map.delete(key) - this.#map.set(key, v) - } - return v - } - - set(key: K, value: V): void { - if (this.capacity <= 0) return - this.#map.delete(key) - this.#map.set(key, value) - if (this.#map.size > this.capacity) this.#map.delete(this.#map.keys().next().value as K) - } - - delete(key: K): void { - this.#map.delete(key) - } - - get size(): number { - return this.#map.size - } -} diff --git a/src/core/mutex.ts b/src/core/mutex.ts deleted file mode 100644 index 3df05ab..0000000 --- a/src/core/mutex.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** Per-key in-process mutex: calls with the same key run one at a time, in order. */ -export class KeyedMutex { - readonly #tails = new Map>() - - run(key: string, fn: () => Promise): Promise { - const prev = this.#tails.get(key) ?? Promise.resolve() - const result = prev.then(fn, fn) - const tail = result.catch(() => {}) - this.#tails.set(key, tail) - void tail.then(() => { - if (this.#tails.get(key) === tail) this.#tails.delete(key) - }) - return result - } -} diff --git a/src/core/path.ts b/src/core/path.ts deleted file mode 100644 index bceeef9..0000000 --- a/src/core/path.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { InvalidKeyError } from '../errors.js' - -export const DEFAULT_SEPARATOR = '/' - -/** Reserved segments. User segments can never start with `~` (a leading `~` is escaped). */ -export const RESERVED = { - index: '~i', - unique: '~u', - events: '~e', - projections: '~p', -} as const - -export type Key = string | number - -/** The character immediately after `c` (by UTF-16 code unit). */ -export function succ(c: string): string { - return String.fromCharCode(c.charCodeAt(0) + 1) -} - -export function validateSeparator(sep: string): void { - if (sep.length !== 1) throw new TypeError(`Separator must be a single character, got ${JSON.stringify(sep)}`) - const code = sep.charCodeAt(0) - if (code <= 0x20 || code >= 0x7f || sep === '%' || sep === '~') { - throw new TypeError(`Separator must be printable ASCII other than "%" and "~", got ${JSON.stringify(sep)}`) - } -} - -function hex2(code: number): string { - return '%' + code.toString(16).toUpperCase().padStart(2, '0') -} - -/** Converts a user key to a segment string, rejecting `undefined`/`null`/empty. */ -export function toSegment(key: unknown): string { - if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`) - if (typeof key === 'number') { - if (!Number.isFinite(key)) throw new InvalidKeyError(`Numeric key must be finite, got ${key}`) - return String(key) - } - if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or number, got ${typeof key}`) - if (key === '') throw new InvalidKeyError('Key must not be empty') - return key -} - -/** - * Escapes one path segment: percent-encodes `%`, the separator, and a leading `~`. - * The result never contains a raw separator and never starts with `~`. - */ -export function escapeSegment(segment: string, sep: string = DEFAULT_SEPARATOR): string { - // Fast path: nothing to escape (the overwhelmingly common case). - if (segment.indexOf('%') === -1 && segment.indexOf(sep) === -1 && segment.charCodeAt(0) !== 0x7e) return segment - let out = '' - for (let i = 0; i < segment.length; i++) { - const c = segment[i]! - if (c === '%' || c === sep || (i === 0 && c === '~')) out += hex2(c.charCodeAt(0)) - else out += c - } - return out -} - -export function unescapeSegment(escaped: string): string { - if (escaped.indexOf('%') === -1) return escaped - return escaped.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16))) -} - -export function isReservedSegment(escaped: string): boolean { - return escaped.charCodeAt(0) === 0x7e // '~' -} - -/** Joins unescaped segments into an escaped key. */ -export function joinPath(path: readonly string[], sep: string = DEFAULT_SEPARATOR): string { - let out = '' - for (let i = 0; i < path.length; i++) { - if (i > 0) out += sep - out += escapeSegment(path[i]!, sep) - } - return out -} - -/** Splits an escaped key into unescaped segments. */ -export function splitPath(key: string, sep: string = DEFAULT_SEPARATOR): string[] { - if (key === '') return [] - return key.split(sep).map(unescapeSegment) -} - -/** - * Precomputed key strings for a node path. - * - `selfKey`: escaped segments joined by the separator (`''` for the root) - * - `prefix`: `selfKey + sep` (`''` for the root) — every descendant key starts with it - * - `end`: exclusive upper bound of the subtree range (`selfKey + succ(sep)`; `undefined` for the root) - */ -export interface PathKeys { - readonly path: readonly string[] - readonly sep: string - readonly selfKey: string - readonly prefix: string - readonly end: string | undefined -} - -export function pathKeys(path: readonly string[], sep: string): PathKeys { - const selfKey = joinPath(path, sep) - const root = path.length === 0 - return { - path, - sep, - selfKey, - prefix: root ? '' : selfKey + sep, - end: root ? undefined : selfKey + succ(sep), - } -} diff --git a/src/core/trie.ts b/src/core/trie.ts deleted file mode 100644 index f377f18..0000000 --- a/src/core/trie.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Segment trie over path patterns. `*` matches exactly one segment. - * A node's matching state is computed incrementally: `step(parentState, segment)`, - * so resolving a path is O(depth) and `child()` is O(1) in the number of patterns. - */ -export interface TrieNode { - readonly literal: Map> - star: TrieNode | undefined - values: { pattern: string; value: T }[] -} - -export type TrieState = readonly TrieNode[] - -export class PatternTrie { - readonly root: TrieNode = newNode() - - constructor(readonly sep: string) {} - - /** `''` names the root node. */ - add(pattern: string, value: T): void { - let node = this.root - for (const seg of splitPattern(pattern, this.sep)) { - if (seg === '*') node = node.star ??= newNode() - else { - let next = node.literal.get(seg) - if (!next) node.literal.set(seg, (next = newNode())) - node = next - } - } - node.values.push({ pattern, value }) - } - - get rootState(): TrieState { - return [this.root] - } - - step(state: TrieState, segment: string): TrieState { - if (state.length === 0) return state - if (state.length === 1) { - const n = state[0]! - const lit = n.literal.get(segment) - if (lit && n.star) return [lit, n.star] - return lit ? [lit] : n.star ? [n.star] : EMPTY - } - const out: TrieNode[] = [] - for (const n of state) { - const lit = n.literal.get(segment) - if (lit) out.push(lit) - if (n.star) out.push(n.star) - } - return out - } - - resolve(path: readonly string[]): TrieState { - let state = this.rootState - for (const seg of path) state = this.step(state, seg) - return state - } - - /** Values attached to every pattern matching the state's path. */ - static values(state: TrieState): { pattern: string; value: T }[] { - if (state.length === 1) return state[0]!.values - return state.flatMap((n) => n.values) - } -} - -const EMPTY: TrieState = [] - -function newNode(): TrieNode { - return { literal: new Map(), star: undefined, values: [] } -} - -export function splitPattern(pattern: string, sep: string): string[] { - return pattern === '' ? [] : pattern.split(sep) -} - -/** Whether two patterns can match the same path. */ -export function patternsOverlap(a: string, b: string, sep: string): boolean { - const sa = splitPattern(a, sep) - const sb = splitPattern(b, sep) - return sa.length === sb.length && sa.every((s, i) => s === '*' || sb[i] === '*' || s === sb[i]) -} diff --git a/src/document-storage/base-document-storage.ts b/src/document-storage/base-document-storage.ts new file mode 100644 index 0000000..7730ebf --- /dev/null +++ b/src/document-storage/base-document-storage.ts @@ -0,0 +1,117 @@ +import type { Doc } from '../utils/document-path.js' +import { DocumentQuery, type Filter, type FindOptions, type FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange, type UpdateSpec } from '../utils/document-update.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt, ValueOf } from '../utils/schema.js' +import { Ulid } from '../utils/ulid.js' + +/** + * Fields to index, per collection path pattern (`*` matches one segment), e.g. + * `{ users: ['email', 'age'], 'users/*\/posts': ['createdAt'] }`. Each implementation decides + * how (or whether) to use them; a native database maps them onto its own indexes. + */ +export type IndexDeclarations = Record + +export interface UpdateOptions { + /** Treat a missing document as `{}`. */ + upsert?: boolean +} + +/** Document type of a node's children. */ +export type ChildDoc = ValueOf>> + +/** + * Document storage. Every node is a document (its own) and a collection of child documents. + * Implementations provide three primitives over document paths; the user-facing API is + * defined here once. + */ +export abstract class BaseDocumentStorage { + /** Unescaped path of this node (`[]` for the root, which has no own document). */ + path: readonly string[] = [] + protected readonly ids = new Ulid() + + abstract readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> + /** + * Maps the current state of one document to a change (`null` = no change), applies it and + * returns the new state. Implementations decide how this is made safe against concurrent writers. + */ + abstract mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise + /** Children of `parent` matching the query, filtered, sorted and paged. */ + abstract query(parent: readonly string[], query: DocumentQuery): AsyncIterable + + child>(key: K): BaseDocumentStorage> { + const node = Object.create(this) as BaseDocumentStorage> + node.path = [...this.path, Keys.segment(key)] + return node + } + + as(): BaseDocumentStorage { + return this as unknown as BaseDocumentStorage + } + + async get | undefined = undefined>(key?: K): Promise | undefined> { + const [doc] = await this.readMany([this.pathOf(arguments.length > 0, key)]) + return doc as ValueAt | undefined + } + + async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { + return (await this.readMany(keys.map((k) => this.pathOf(true, k)))) as (ValueOf> | undefined)[] + } + + /** `set(doc)` replaces this document; `set(key, doc)` a child document. */ + async set>(...args: [doc: ValueOf] | [key: K, doc: ValueOf>]): Promise { + const doc = args[args.length - 1] + if (typeof doc !== 'object' || doc === null || Array.isArray(doc)) throw new TypeError('A document must be a plain object') + await this.mutate(this.pathOf(args.length > 1, args[0]), () => ({ op: 'set', doc: doc as Doc })) + } + + /** Adds a child document under a new time-sortable id and returns the id. */ + async insert(doc: ChildDoc): Promise { + const id = this.ids.generate() + await this.set(id as KeyOf, doc as never) + return id + } + + /** + * `update(spec, options?)` this document; `update(key, spec, options?)` a child. + * Returns the new state, or `undefined` if the document doesn't exist (unless `upsert`). + */ + async update>( + ...args: [spec: UpdateSpec, options?: UpdateOptions] | [key: K, spec: UpdateSpec, options?: UpdateOptions] + ): Promise | undefined> { + const hasKey = typeof args[0] !== 'object' || args[0] === null + const spec = args[hasKey ? 1 : 0] as UpdateSpec + const options = (args[hasKey ? 2 : 1] ?? {}) as UpdateOptions + const next = await this.mutate(this.pathOf(hasKey, args[0]), (current) => (current || options.upsert ? DocumentUpdate.resolve(current ?? {}, spec) : null)) + return next as ValueAt | undefined + } + + /** Deletes this document (or a child). Returns whether one existed. */ + async delete(key?: KeyOf): Promise { + let existed = false + await this.mutate(this.pathOf(arguments.length > 0, key), (current) => ((existed = current !== undefined) ? { op: 'delete' } : null)) + return existed + } + + /** Child documents matching `filter`. */ + find(filter: Filter = {}, options: FindOptions = {}): AsyncIterable>> { + return this.query(this.path, new DocumentQuery(filter, options)) as AsyncIterable>> + } + + async findOne(filter: Filter = {}, options: FindOptions = {}): Promise> | undefined> { + for await (const result of this.find(filter, { ...options, limit: 1 })) return result + return undefined + } + + /** Keys of child documents. */ + async *keys(): AsyncGenerator { + for await (const { key } of this.find()) yield key + } + + /** Path of a child (`hasKey`) or this node; validates keys. */ + protected pathOf(hasKey: boolean, key: unknown): readonly string[] { + if (hasKey) return [...this.path, Keys.segment(key)] + if (this.path.length === 0) throw new TypeError('The root node has no own document') + return this.path + } +} diff --git a/src/document-storage/document-storage-in-memory.ts b/src/document-storage/document-storage-in-memory.ts new file mode 100644 index 0000000..8cff273 --- /dev/null +++ b/src/document-storage/document-storage-in-memory.ts @@ -0,0 +1,112 @@ +import type { Doc } from '../utils/document-path.js' +import { DocumentPath } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { Json } from '../utils/json.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseDocumentStorage, type IndexDeclarations } from './base-document-storage.js' + +export interface DocumentStorageInMemoryOptions { + /** Fields with an equality index, per collection pattern. */ + indexes?: IndexDeclarations +} + +/** Field → JSON-encoded value → child keys. */ +type CollectionIndex = Map>> + +/** + * In-memory document storage. Declared index fields get an equality index per collection, used + * when a filter has `{ field: value }` or `{ field: { $eq: value } }`; everything else is a scan. + */ +export class DocumentStorageInMemory extends BaseDocumentStorage { + private readonly collections = new Map>() + private readonly indexes = new Map() + private readonly declarations: [pattern: string[], fields: string[]][] + + constructor(options: DocumentStorageInMemoryOptions = {}) { + super() + this.declarations = Object.entries(options.indexes ?? {}).map(([pattern, fields]) => [pattern === '' ? [] : pattern.split('/'), fields]) + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return paths.map((path) => Json.clone(this.collection(path.slice(0, -1)).get(path.at(-1)!))) + } + + async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const parent = path.slice(0, -1) + const key = path.at(-1)! + const docs = this.collection(parent) + const current = docs.get(key) + const resolved = change(Json.clone(current)) + if (!resolved) return Json.clone(current) + const next = DocumentUpdate.apply(current, resolved) + if (next) docs.set(key, next) + else docs.delete(key) + this.reindex(parent, key, current, next) + return Json.clone(next) + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + const docs = this.collection(parent) + const keys = [...(this.lookup(parent, query) ?? docs.keys())].sort() + yield* query.run( + (async function* () { + for (const key of keys) { + const doc = docs.get(key) + if (doc) yield { key, data: Json.clone(doc) } + } + })(), + ) + } + + private collection(parent: readonly string[]): Map { + const id = Keys.join(parent) + let docs = this.collections.get(id) + if (!docs) this.collections.set(id, (docs = new Map())) + return docs + } + + /** Candidate keys from an equality index, if the filter allows one. */ + private lookup(parent: readonly string[], query: DocumentQuery): Iterable | undefined { + const index = this.indexes.get(Keys.join(parent)) + for (const field of this.fieldsFor(parent)) { + let value = query.filter[field] + if (Json.isObject(value) && Object.keys(value).length === 1 && '$eq' in value) value = value.$eq + if (value === undefined || typeof value === 'object') continue + return index?.get(field)?.get(JSON.stringify(value)) ?? [] + } + return undefined + } + + private reindex(parent: readonly string[], key: string, before: Doc | undefined, after: Doc | undefined): void { + const fields = this.fieldsFor(parent) + if (fields.length === 0) return + const id = Keys.join(parent) + let index = this.indexes.get(id) + if (!index) this.indexes.set(id, (index = new Map())) + for (const field of fields) { + let byValue = index.get(field) + if (!byValue) index.set(field, (byValue = new Map())) + for (const value of DocumentStorageInMemory.indexValues(before, field)) byValue.get(value)?.delete(key) + for (const value of DocumentStorageInMemory.indexValues(after, field)) { + let keys = byValue.get(value) + if (!keys) byValue.set(value, (keys = new Set())) + keys.add(key) + } + } + } + + private fieldsFor(parent: readonly string[]): string[] { + return this.declarations + .filter(([pattern]) => pattern.length === parent.length && pattern.every((s, i) => s === '*' || s === parent[i])) + .flatMap(([, fields]) => fields) + } + + /** Index keys for a field: its value, plus each element of an array (missing indexes as `null`). */ + private static indexValues(doc: Doc | undefined, field: string): string[] { + if (!doc) return [] + const value = DocumentPath.get(doc, field) ?? null + return Array.isArray(value) ? value.map((v) => JSON.stringify(v)) : [JSON.stringify(value)] + } +} diff --git a/src/document-storage/document-storage-kv.ts b/src/document-storage/document-storage-kv.ts new file mode 100644 index 0000000..7a915b4 --- /dev/null +++ b/src/document-storage/document-storage-kv.ts @@ -0,0 +1,46 @@ +import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js' +import type { Doc } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseDocumentStorage } from './base-document-storage.js' + +/** + * Document storage on any KV storage: each document is the value at its path. Queries scan the + * collection's direct children (no indexes). Writes are read-modify-write, so concurrent writers + * to the same document are last-writer-wins; use `EventSourcedDocumentStorage` if that matters. + */ +export class DocumentStorageKV extends BaseDocumentStorage { + constructor(private readonly kv: BaseKVStorage) { + super() + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return (await this.kv.readMany(paths.map((p) => this.keyOf(p)))) as (Doc | undefined)[] + } + + async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const key = this.keyOf(path) + const [current] = (await this.kv.readMany([key])) as (Doc | undefined)[] + const resolved = change(current) + if (!resolved) return current + const next = DocumentUpdate.apply(current, resolved) + if (next) await this.kv.writeMany([[key, next]]) + else await this.kv.removeMany([key]) + return next + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + const collection = parent.reduce((node, segment) => node.child(segment), this.kv) + yield* query.run( + (async function* () { + for await (const [key, data] of collection.entries()) yield { key, data: data as Doc } + })(), + ) + } + + private keyOf(path: readonly string[]): string { + return this.kv.prefix + Keys.join(path) + } +} diff --git a/src/document-storage/event-sourced-document-storage.ts b/src/document-storage/event-sourced-document-storage.ts new file mode 100644 index 0000000..d9f27c8 --- /dev/null +++ b/src/document-storage/event-sourced-document-storage.ts @@ -0,0 +1,125 @@ +import type { BaseEventStorage, StoredEvent } from '../event-storage/base-event-storage.js' +import type { Doc } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { ConcurrencyError } from '../utils/errors.js' +import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt } from '../utils/schema.js' +import { BaseDocumentStorage } from './base-document-storage.js' + +export interface EventSourcedDocumentStorageOptions { + /** Retries after a write conflict, each preceded by a catch-up. Default 5. */ + maxRetries?: number +} + +// Types only: children of an event-sourced store are event-sourced stores too (child() creates +// them with Object.create, so they share this class's prototype). +export interface EventSourcedDocumentStorage { + child>(key: K): EventSourcedDocumentStorage> +} + +/** + * Documents as event streams: each document's changes are appended to its own stream, and the + * current state is kept in a projection store (any `BaseDocumentStorage`), which serves all + * reads and queries. + * + * Projections carry a `_version` field (stripped from results) used as `expectedVersion` on the + * next append. If a write crashes between appending and updating the projection, the next + * write's append conflicts, the projection is rebuilt from the stream, and the write retries. + * `_version` is therefore reserved in documents. + */ +export class EventSourcedDocumentStorage extends BaseDocumentStorage { + private readonly maxRetries: number + /** Tail of the pending writes per document, so writes in one process don't conflict with each other. */ + private readonly locks = new Map>() + + constructor( + private readonly events: BaseEventStorage, + private readonly projections: BaseDocumentStorage, + options: EventSourcedDocumentStorageOptions = {}, + ) { + super() + this.maxRetries = options.maxRetries ?? 5 + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return (await this.projections.readMany(paths)).map((doc) => EventSourcedDocumentStorage.strip(doc)) + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + for await (const { key, data } of this.projections.query(parent, query)) yield { key, data: EventSourcedDocumentStorage.strip(data)! } + } + + mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const id = JSON.stringify(path) + const result = (this.locks.get(id) ?? Promise.resolve()).then(() => this.write(path, change)) + const tail = result.catch(() => {}) + this.locks.set(id, tail) + void tail.then(() => this.locks.get(id) === tail && this.locks.delete(id)) + return result + } + + private async write(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + for (let attempt = 0; ; attempt++) { + const [stored] = await this.projections.readMany([path]) + const version = (stored?._version as number | undefined) ?? (await this.events.versionOf(path)) + if (!stored && version > 0) { + // No projection but the stream has history (deleted, or a crashed first write): rebuild. + if ((await this.rebuild(path)) !== undefined) continue + } + const current = EventSourcedDocumentStorage.strip(stored) + const resolved = change(current) + if (!resolved) return current + const next = DocumentUpdate.apply(current, resolved) + try { + await this.events.appendTo(path, [resolved], version) + } catch (error) { + if (!(error instanceof ConcurrencyError) || attempt >= this.maxRetries) throw error + await this.rebuild(path) + continue + } + await this.project(path, next, version + 1) + return next + } + } + + /** Events of this document (or a child). */ + history(key?: KeyOf): AsyncIterable> { + return this.events.readFrom(this.pathOf(arguments.length > 0, key), {}) as AsyncIterable> + } + + /** State as of `version`: `getAt(version)` for this document, `getAt(key, version)` for a child. */ + async getAt>(...args: [version: number] | [key: K, version: number]): Promise | undefined> { + const path = this.pathOf(args.length > 1, args[0]) + let state: Doc | undefined + for await (const event of this.events.readFrom(path, { to: args[args.length - 1] as number })) { + state = DocumentUpdate.apply(state, event.data as DocChange) + } + return state as ValueAt | undefined + } + + /** Rebuilds the projection of this document (or a child) from its stream. Returns the state. */ + catchUp(key?: KeyOf): Promise { + return this.rebuild(this.pathOf(arguments.length > 0, key)) + } + + private async rebuild(path: readonly string[]): Promise { + let state: Doc | undefined + let version = 0 + for await (const event of this.events.readFrom(path, {})) { + state = DocumentUpdate.apply(state, event.data as DocChange) + version = event.version + } + await this.project(path, state, version) + return state + } + + private async project(path: readonly string[], state: Doc | undefined, version: number): Promise { + await this.projections.mutate(path, () => (state ? { op: 'set', doc: { ...state, _version: version } } : { op: 'delete' })) + } + + private static strip(doc: Doc | undefined): Doc | undefined { + if (!doc) return undefined + const { _version, ...state } = doc + return state + } +} diff --git a/src/documents/event-sourced.ts b/src/documents/event-sourced.ts deleted file mode 100644 index a79f3a8..0000000 --- a/src/documents/event-sourced.ts +++ /dev/null @@ -1,606 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ, toSegment, unescapeSegment } from '../core/path.js' -import type { TrieState } from '../core/trie.js' -import { CapabilityError, ConcurrencyError, UniqueConstraintError } from '../errors.js' -import { KeyedMutex } from '../core/mutex.js' -import { KVEventNode } from '../events/kv.js' -import type { EventNode, ReadOptions, StoredEvent } from '../events/types.js' -import { scanChildren } from '../kv/children.js' -import type { BatchOp, KVDriver } from '../kv/driver.js' -import type { KVNode } from '../kv/node.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import { analyze, matches, type Filter } from './filter.js' -import { DocIndexes, DocIndexRegistry, type DocIndexDeclarations, type Plan } from './indexes.js' -import type { ChildDoc, DocKeysOptions, EventSourcedDocumentNode, FindOptions, FindResult, GetOptions, UpdateOptions } from './types.js' -import { ulid } from './ulid.js' -import { applyEvent, applyPatch, resolveUpdate, toPatchEvent, type DocEvent, type UpdateSpec } from './update.js' -import { clone, compareValues, deepEqual, getPath, isPlainObject, setPath, sortKey, unsetPath, type Doc } from './values.js' - -export interface EventSourcedDocumentStorageOptions { - /** Where projections live. Default: the `~p` root of the event store's KV. Required for non-KV event stores. */ - projections?: KVNode - /** Path pattern → indexes over that node's direct children. */ - indexes?: DocIndexDeclarations - /** Delete stale index entries found by `find`. Default `true`. */ - readRepair?: boolean - /** Retries after a write conflict (each preceded by a catch-up). Default 5. */ - maxRetries?: number - /** Warn once per filter shape when `find` falls back to a full scan. Default: `NODE_ENV !== 'production'`. */ - warnOnFullScan?: boolean - /** Id generator for `insert`. Default: ULID-like. */ - newId?: () => string - /** - * A unique claim whose owner no longer holds the value is only taken over once it is older - * than this, since its owner may be mid-write. Default 30 000 ms. - */ - claimGraceMs?: number - /** Clock for claim timestamps. Default `Date.now`. */ - now?: () => number -} - -/** Stored projection of one document. */ -export type Projection = - { state: Doc; version: number; commit: number; deleted?: undefined } | { deleted: true; version: number; commit: number; state?: undefined } - -interface DocContext { - readonly driver: KVDriver - readonly sep: string - /** Projection-space prefix: projection key of a doc = `projBase + docSelfKey`. */ - readonly projBase: string - readonly registry: DocIndexRegistry - readonly readRepair: boolean - readonly maxRetries: number - readonly warnOnFullScan: boolean - readonly warned: Set - readonly newId: () => string - readonly claimGraceMs: number - readonly now: () => number - /** Serialises writes to one document within this process, so retries only arise across processes. */ - readonly writeLock: KeyedMutex -} - -/** Stored value of a unique claim: owner child segment (escaped) and claim time. */ -type Claim = [owner: string, at: number] - -interface Target { - ev: EventNode - parentEv: EventNode - esc: string - /** Projection key of the document. */ - projKey: string - /** Projection-space prefix of its parent (sibling projection key = `parentProj + esc`). */ - parentProj: string - idx: DocIndexes | undefined -} - -type Hit = { esc: string; state: Doc } - -const FIND_CHUNK = 128 - -/** - * Document node backed by an event stream per document and a projection per document. - * Writes: read projection → resolve delta → append (expectedVersion = projection.version) → - * write projection + index entries. Reads hit the projection only. - */ -export class EventSourcedDocumentNodeImpl implements EventSourcedDocumentNode { - readonly path: readonly string[] - /** Document-space key (escaped), relative to the store root. */ - readonly selfKey: string - readonly #ctx: DocContext - readonly #ev: EventNode - readonly #parentEv: EventNode | undefined - readonly #esc: string - readonly #prefix: string - readonly #state: TrieState - readonly #selfIdx: DocIndexes | undefined - readonly #childIdx: DocIndexes | undefined - - /** @internal Use `EventSourcedDocumentStorage` and `child()`. */ - constructor( - ctx: DocContext, - ev: EventNode, - parentEv: EventNode | undefined, - path: readonly string[], - selfKey: string, - esc: string, - state: TrieState, - selfIdx: DocIndexes | undefined, - ) { - this.#ctx = ctx - this.#ev = ev - this.#parentEv = parentEv - this.path = path - this.selfKey = selfKey - this.#esc = esc - this.#prefix = path.length === 0 ? '' : selfKey + ctx.sep - this.#state = state - this.#selfIdx = selfIdx - this.#childIdx = ctx.registry.bind(state, ctx.projBase + this.#prefix) - } - - child>(key: K): EventSourcedDocumentNodeImpl> { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - return new EventSourcedDocumentNodeImpl( - this.#ctx, - this.#ev.child(seg), - this.#ev, - [...this.path, seg], - this.#prefix + esc, - esc, - this.#ctx.registry.trie.step(this.#state, seg), - this.#childIdx, - ) - } - - as(): EventSourcedDocumentNodeImpl { - return this as unknown as EventSourcedDocumentNodeImpl - } - - // ---------------------------------------------------------------- reads - - get(opts?: GetOptions): Promise | undefined> - get>(key: K, opts?: GetOptions): Promise> | undefined> - async get(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const opts = (rest[0] ?? {}) as GetOptions - let proj = await this.#projection(t) - if (opts.consistent && (await t.ev.version()) > (proj?.version ?? 0)) { - await this.#catchUpDoc(t) - proj = await this.#projection(t) - } - return proj?.state - } - - history(): AsyncIterable> - history>(key: K): AsyncIterable> - history(...args: unknown[]): AsyncIterable> { - const t = args.length === 0 ? this.#self() : this.#child(args[0]) - return t.ev.read() as AsyncIterable> - } - - getAt(version: number): Promise | undefined> - getAt>(key: K, version: number): Promise> | undefined> - async getAt(...args: unknown[]): Promise { - const t = args.length <= 1 ? this.#self() : this.#child(args[0]) - const version = args[args.length - 1] as number - if (!Number.isInteger(version) || version < 0) throw new TypeError(`Invalid version ${String(version)}`) - let state: Doc | undefined - for await (const e of t.ev.read({ to: version } satisfies ReadOptions)) state = applyEvent(state, e.data as DocEvent) - return state - } - - async *find(filter: Filter = {}, opts: FindOptions = {}): AsyncGenerator>> { - const skip = opts.skip ?? 0 - const limit = opts.limit ?? Infinity - if (limit <= 0) return - const sort = Object.entries(opts.sort ?? {}) as [string, 1 | -1][] - const plan = this.#childIdx?.plan(analyze(filter), sort) - if (!plan && Object.keys(filter).length > 0) this.#warnFullScan(filter) - const source = plan ? this.#indexScan(plan, skip + limit) : this.#projectionScan() - const project = opts.projection ? projector(opts.projection) : (d: Doc) => d - - if (sort.length && !plan?.sorted) { - // In-memory sort, bounded to skip + limit when a limit is given. - const cap = skip + limit - const cmp = comparator(sort) - const top: Hit[] = [] - for await (const hit of source) { - if (!matches(hit.state, filter)) continue - if (top.length >= cap && cmp(hit, top[top.length - 1]!) >= 0) continue - let lo = 0 - let hi = top.length - while (lo < hi) { - const mid = (lo + hi) >>> 1 - if (cmp(top[mid]!, hit) <= 0) lo = mid + 1 - else hi = mid - } - top.splice(lo, 0, hit) - if (top.length > cap) top.pop() - } - for (let i = skip; i < top.length; i++) yield { key: unescapeSegment(top[i]!.esc), data: project(top[i]!.state) as ChildDoc } - return - } - - let skipped = 0 - let emitted = 0 - for await (const hit of source) { - if (!matches(hit.state, filter)) continue - if (skipped < skip) { - skipped++ - continue - } - yield { key: unescapeSegment(hit.esc), data: project(hit.state) as ChildDoc } - if (++emitted >= limit) return - } - } - - async findOne(filter: Filter = {}, opts: FindOptions = {}): Promise> | undefined> { - for await (const r of this.find(filter, { ...opts, limit: 1 })) return r - return undefined - } - - async *keys(opts: DocKeysOptions = {}): AsyncGenerator> { - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - const after = opts.after === undefined ? undefined : toSegment(opts.after) - const { prefix, end } = this.#childRange() - for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { after, reverse: opts.reverse, values: true })) { - if (!hit.hasOwn || (hit.value as Projection).deleted) continue - yield unescapeSegment(hit.esc) as KeyString - if (--remaining <= 0) return - } - } - - // ---------------------------------------------------------------- writes - - set(doc: ValueOf): Promise - set>(key: K, doc: ValueOf>): Promise - async set(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const doc = rest[0] - if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object') - const snapshot = clone(doc) - await this.#write(t, () => ({ op: 'set', doc: snapshot })) - } - - async insert(doc: ChildDoc): Promise { - if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object') - const id = this.#ctx.newId() - const t = this.#child(id) - const snapshot = clone(doc) - await this.#write(t, (cur) => { - if (cur) throw new Error(`Document "${id}" already exists`) - return { op: 'set', doc: snapshot } - }) - return id - } - - update(spec: UpdateSpec, opts?: UpdateOptions): Promise | undefined> - update>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise> | undefined> - async update(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const spec = rest[0] as UpdateSpec - const opts = (rest[1] ?? {}) as UpdateOptions - if (!isPlainObject(spec)) throw new TypeError('update() expects an update spec') - const result = await this.#write(t, (cur) => { - if (!cur && !opts.upsert) return null - const base = cur ?? {} - const patch = resolveUpdate(base, spec) - if (cur && deepEqual(applyPatch(base, patch), cur)) return null // no-op: no event - return toPatchEvent(patch) - }) - return result - } - - delete(): Promise - delete>(key: K): Promise - async delete(...args: unknown[]): Promise { - const t = args.length === 0 ? this.#self() : this.#child(args[0]) - let deleted = false - await this.#write(t, (cur) => { - deleted = cur !== undefined - return cur ? { op: 'delete' } : null - }) - return deleted - } - - // ---------------------------------------------------------------- maintenance - - async catchUp(): Promise { - let repaired = 0 - if (this.path.length > 0 && (await this.#catchUpDoc(this.#self()))) repaired++ - for await (const key of this.#ev.keys()) repaired += await this.child(key as KeyOf).catchUp() - return repaired - } - - // ---------------------------------------------------------------- internals - - #self(): Target { - if (this.path.length === 0) throw new TypeError('The root node has no own document') - const parentProj = this.#ctx.projBase + this.selfKey.slice(0, this.selfKey.length - this.#esc.length) - return { ev: this.#ev, parentEv: this.#parentEv!, esc: this.#esc, projKey: this.#ctx.projBase + this.selfKey, parentProj, idx: this.#selfIdx } - } - - #child(key: unknown): Target { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - const parentProj = this.#ctx.projBase + this.#prefix - return { ev: this.#ev.child(seg), parentEv: this.#ev, esc, projKey: parentProj + esc, parentProj, idx: this.#childIdx } - } - - /** Target for a sibling of `t` given its escaped segment. */ - #sibling(t: Target, esc: string): Target { - return { ev: t.parentEv.child(unescapeSegment(esc)), parentEv: t.parentEv, esc, projKey: t.parentProj + esc, parentProj: t.parentProj, idx: t.idx } - } - - /** Self form when there are no args or the first is a non-null object; otherwise key form. */ - #split(args: unknown[]): [Target, unknown[]] { - if (args.length === 0 || (typeof args[0] === 'object' && args[0] !== null)) return [this.#self(), args] - return [this.#child(args[0]), args.slice(1)] - } - - #childRange(): { prefix: string; end: string | undefined } { - const prefix = this.#ctx.projBase + this.#prefix - return { prefix, end: prefix === '' ? undefined : prefix.slice(0, -1) + succ(this.#ctx.sep) } - } - - async #projection(t: Target): Promise { - return (await this.#ctx.driver.getMany([t.projKey]))[0] as Projection | undefined - } - - /** - * The write path. `compute` maps the current state to an event (or `null` for a no-op). - * Returns the resulting state. - */ - #write(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise { - return this.#ctx.writeLock.run(t.projKey, () => this.#writeUnlocked(t, compute)) - } - - async #writeUnlocked(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise { - const { maxRetries } = this.#ctx - for (let attempt = 0; ; attempt++) { - const proj = await this.#projection(t) - const cur = proj?.state - const event = compute(cur) - if (!event) return cur - const next = applyEvent(cur, event) - - const oldClaims = t.idx?.claimsFor(cur) - const newClaims = t.idx?.claimsFor(next) - const acquired: string[] = [] - if (newClaims?.size) { - const wanted = new Map([...newClaims].filter(([k]) => !oldClaims!.has(k))) - if (wanted.size) await this.#claim(t, wanted, acquired) - } - - let result - try { - result = await t.ev.append([event], { expectedVersion: proj?.version ?? 0, expectedCommit: proj?.commit ?? 0 }) - } catch (err) { - if (acquired.length) await this.#ctx.driver.deleteMany(acquired) - if (err instanceof ConcurrencyError && attempt < maxRetries) { - await this.#catchUpDoc(t) - continue - } - throw err - } - const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : [] - const projection: Projection = next - ? { state: next, version: result.version, commit: result.commit } - : { deleted: true, version: result.version, commit: result.commit } - await this.#writeProjection(t, cur, next, projection, releases) - return next - } - } - - /** - * Claims unique values for `t`; records successful claims in `acquired`. Throws - * `UniqueConstraintError` on conflict. A claim held by another document is taken over only if - * that document, caught up to its stream head, no longer holds the value and the claim is older - * than `claimGraceMs` (its owner might otherwise be between claiming and appending). - */ - async #claim(t: Target, claims: Map, acquired: string[]): Promise { - const { driver, now, claimGraceMs } = this.#ctx - const keys = [...claims.keys()] - const written = await driver.setIfAbsent!(keys.map((k) => [k, [t.esc, now()] satisfies Claim])) - const lost = keys.filter((k, i) => { - if (written[i]) acquired.push(k) - return !written[i] - }) - if (!lost.length) return - const holders = (await driver.getMany(lost)) as (Claim | undefined)[] - const fail = async (k: string, owner?: string): Promise => { - if (acquired.length) await driver.deleteMany(acquired.splice(0)) - const c = claims.get(k)! - throw new UniqueConstraintError(c.index, c.values, owner === undefined ? undefined : unescapeSegment(owner)) - } - for (let i = 0; i < lost.length; i++) { - const k = lost[i]! - const holder = holders[i] - if (holder?.[0] === t.esc) continue // already ours (e.g. left by an interrupted write) - if (holder) { - const owner = this.#sibling(t, holder[0]) - await this.#catchUpDoc(owner) - const ownerProj = await this.#projection(owner) - if (t.idx!.claimsFor(ownerProj?.state).has(k)) await fail(k, holder[0]) - if (now() - holder[1] < claimGraceMs) await fail(k, holder[0]) - await driver.deleteMany([k]) - } - const [ok] = await driver.setIfAbsent!([[k, [t.esc, now()] satisfies Claim]]) - if (!ok) await fail(k) - acquired.push(k) - } - } - - async #writeProjection(t: Target, cur: Doc | undefined, next: Doc | undefined, projection: Projection, releases: string[]): Promise { - const { driver } = this.#ctx - if (!t.idx) { - await driver.setMany([[t.projKey, projection]]) - return - } - const before = t.idx.entriesFor(t.esc, cur) - const after = t.idx.entriesFor(t.esc, next) - const adds = [...after].filter((e) => !before.has(e)) - const removes = [...before].filter((e) => !after.has(e)).concat(releases) - if (driver.batch) { - const ops: BatchOp[] = adds.map((key) => ({ type: 'set', key, value: true })) - ops.push({ type: 'set', key: t.projKey, value: projection }) - for (const key of removes) ops.push({ type: 'delete', key }) - await driver.batch(ops) - return - } - if (adds.length) await driver.setMany(adds.map((k) => [k, true])) - await driver.setMany([[t.projKey, projection]]) - if (removes.length) await driver.deleteMany(removes) - } - - /** Replays events newer than the projection into it. Returns whether anything changed. */ - async #catchUpDoc(t: Target): Promise { - const proj = await this.#projection(t) - const cur = proj?.state - let state = cur - let version = proj?.version ?? 0 - let commit = proj?.commit ?? 0 - for await (const e of t.ev.read({ from: version + 1 })) { - state = applyEvent(state, e.data as DocEvent) - version = e.version - commit = e.commit - } - if (version === (proj?.version ?? 0)) return false - const oldClaims = t.idx?.claimsFor(cur) - const newClaims = t.idx?.claimsFor(state) - if (newClaims?.size) { - // The log is the source of truth: (re)assert claims for the caught-up state. - const missing = [...newClaims.keys()].filter((k) => !oldClaims!.has(k)) - if (missing.length) await this.#ctx.driver.setMany(missing.map((k) => [k, [t.esc, this.#ctx.now()] satisfies Claim])) - } - const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : [] - const projection: Projection = state ? { state, version, commit } : { deleted: true, version, commit } - await this.#writeProjection(t, cur, state, projection, releases) - return true - } - - async *#projectionScan(): AsyncGenerator { - const { prefix, end } = this.#childRange() - for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { values: true })) { - const p = hit.value as Projection | undefined - if (hit.hasOwn && p?.state) yield { esc: hit.esc, state: p.state } - } - } - - async *#indexScan(plan: Plan, want: number): AsyncGenerator { - const idx = this.#childIdx! - const { driver, readRepair } = this.#ctx - const parentProj = this.#ctx.projBase + this.#prefix - const seen = new Set() - let chunk: string[] = [] - let chunkSize = Math.min(FIND_CHUNK, Math.max(1, want)) - const it = driver.scan({ gte: plan.gte, lt: plan.lt, reverse: plan.reverse })[Symbol.asyncIterator]() - - async function* flush(): AsyncGenerator { - const entries = chunk - chunk = [] - chunkSize = FIND_CHUNK - const escs = entries.map((e) => idx.childOf(plan.index, e)) - const projs = (await driver.getMany(escs.map((e) => parentProj + e))) as (Projection | undefined)[] - const stale: string[] = [] - try { - for (let i = 0; i < entries.length; i++) { - const esc = escs[i]! - const state = projs[i]?.state - if (!idx.entriesFor(esc, state).has(entries[i]!)) { - stale.push(entries[i]!) - continue - } - if (seen.has(esc)) continue - seen.add(esc) - yield { esc, state: state! } - } - } finally { - if (readRepair && stale.length) await driver.deleteMany(stale) - } - } - - try { - for (;;) { - const r = await it.next() - if (r.done) break - chunk.push(r.value[0]) - if (chunk.length >= chunkSize) yield* flush() - } - if (chunk.length) yield* flush() - } finally { - await it.return?.() - } - } - - #warnFullScan(filter: Filter): void { - if (!this.#ctx.warnOnFullScan) return - const shape = JSON.stringify(filterShape(filter)) - if (this.#ctx.warned.has(shape)) return - this.#ctx.warned.add(shape) - console.warn(`[potential-storage] find() on "${this.selfKey || '(root)'}" has no usable index; full scan for filter ${shape}`) - } -} - -/** Filter with values replaced by `?`, for de-duplicating warnings. */ -function filterShape(f: unknown): unknown { - if (Array.isArray(f)) return f.map(filterShape) - if (isPlainObject(f)) - return Object.fromEntries( - Object.entries(f).map(([k, v]) => [ - k, - k.startsWith('$') && (k === '$and' || k === '$or' || k === '$not') ? filterShape(v) : isPlainObject(v) ? filterShape(v) : '?', - ]), - ) - return '?' -} - -function comparator(sort: [string, 1 | -1][]): (a: Hit, b: Hit) => number { - const tieDir = sort[0]?.[1] ?? 1 - return (a, b) => { - for (const [field, dir] of sort) { - const d = compareValues(sortKey(a.state, field, dir), sortKey(b.state, field, dir)) - if (d !== 0) return d * dir - } - return a.esc === b.esc ? 0 : (a.esc < b.esc ? -1 : 1) * tieDir - } -} - -function projector(spec: Record): (d: Doc) => Doc { - const fields = Object.entries(spec) - const include = fields.some(([, v]) => v === 1 || v === true) - if (include && fields.some(([, v]) => v === 0 || v === false)) throw new TypeError('Projection cannot mix inclusion and exclusion') - return (d) => { - if (include) { - const out: Doc = {} - for (const [f] of fields) { - const v = getPath(d, f) - if (v !== undefined) setPath(out, f, clone(v)) - } - return out - } - const out = clone(d) - for (const [f] of fields) unsetPath(out, f) - return out - } -} - -/** Event-sourced document storage: the root document node. */ -export class EventSourcedDocumentStorage extends EventSourcedDocumentNodeImpl { - constructor(events: EventNode, opts: EventSourcedDocumentStorageOptions = {}) { - let driver: KVDriver - let sep: string - let projBase: string - if (opts.projections) { - driver = opts.projections.driver - sep = opts.projections.separator - projBase = opts.projections.prefix - } else if (events instanceof KVEventNode) { - const kv = events.kv - driver = kv.driver - sep = kv.separator - projBase = kv.prefix + RESERVED.projections + sep - } else { - throw new TypeError('EventSourcedDocumentStorage needs `projections` when the event store is not KV-backed') - } - const registry = new DocIndexRegistry(opts.indexes ?? {}, sep) - if (registry.hasUnique && !driver.setIfAbsent) throw new CapabilityError('setIfAbsent', 'unique document indexes require it') - const ctx: DocContext = { - driver, - sep, - projBase, - registry, - readRepair: opts.readRepair ?? true, - maxRetries: opts.maxRetries ?? 5, - warnOnFullScan: opts.warnOnFullScan ?? (typeof process === 'undefined' || process.env?.NODE_ENV !== 'production'), - warned: new Set(), - newId: opts.newId ?? ulid, - claimGraceMs: opts.claimGraceMs ?? 30_000, - now: opts.now ?? Date.now, - writeLock: new KeyedMutex(), - } - super(ctx, events, undefined, [], '', '', registry.trie.rootState, undefined) - } -} diff --git a/src/documents/filter.ts b/src/documents/filter.ts deleted file mode 100644 index f2a4f9e..0000000 --- a/src/documents/filter.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' -import { compareValues, deepEqual, isPlainObject, isScalar, resolvePath, typeRank, type Doc } from './values.js' - -/** - * Query filter (Mongo subset): field conditions with implicit equality and dot paths, - * `$eq $ne $gt $gte $lt $lte $in $nin $exists`, and `$and $or $not`. - * - * Semantics differences from MongoDB (see DESIGN.md §12, M6): - * - Comparisons are type-bracketed: `$gt: 5` only matches numbers. - * - On array fields, all positive operators in one condition (`$eq $gt $gte $lt $lte $in`) must be - * satisfied by the same element (like `$elemMatch`), so index range scans are exact. - */ -export type Filter = { [field: string]: unknown; $and?: Filter[]; $or?: Filter[]; $not?: Filter } - -const POSITIVE = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in']) -const FIELD_LEVEL = new Set(['$ne', '$nin', '$exists', '$not']) - -export function isOperatorObject(v: unknown): v is Record { - if (!isPlainObject(v)) return false - const keys = Object.keys(v) - return keys.length > 0 && keys.every((k) => k.startsWith('$')) -} - -export function matches(doc: unknown, filter: Filter): boolean { - for (const key of Object.keys(filter)) { - const cond = filter[key] - switch (key) { - case '$and': - if (!(cond as Filter[]).every((f) => matches(doc, f))) return false - break - case '$or': - if (!(cond as Filter[]).some((f) => matches(doc, f))) return false - break - case '$not': - if (matches(doc, cond as Filter)) return false - break - default: - if (key.startsWith('$')) throw new TypeError(`Unknown top-level operator ${key}`) - if (!matchesField(resolvePath(doc, key), cond)) return false - } - } - return true -} - -/** Whether the values found at a path satisfy a field condition. */ -export function matchesField(values: unknown[], cond: unknown): boolean { - if (!isOperatorObject(cond)) return someCandidate(values, (c) => eq(c, cond)) - const positive: [string, unknown][] = [] - for (const [op, arg] of Object.entries(cond)) { - if (POSITIVE.has(op)) positive.push([op, arg]) - else if (!FIELD_LEVEL.has(op)) throw new TypeError(`Unknown operator ${op}`) - } - for (const [op, arg] of Object.entries(cond)) { - switch (op) { - case '$ne': - if (someCandidate(values, (c) => eq(c, arg))) return false - break - case '$nin': - if (someCandidate(values, (c) => (arg as unknown[]).some((x) => eq(c, x)))) return false - break - case '$exists': - if (values.length > 0 !== Boolean(arg)) return false - break - case '$not': - if (matchesField(values, arg)) return false - break - } - } - if (positive.length === 0) return true - return someCandidate(values, (c) => positive.every(([op, arg]) => applyPositive(op, c, arg))) -} - -/** Tests each value and, for arrays, each element. Missing paths test `undefined` once. */ -function someCandidate(values: unknown[], test: (c: unknown) => boolean): boolean { - if (values.length === 0) return test(undefined) - for (const v of values) { - if (test(v)) return true - if (Array.isArray(v) && v.some(test)) return true - } - return false -} - -function eq(c: unknown, x: unknown): boolean { - if (x === null) return c === null || c === undefined - return deepEqual(c, x) -} - -function applyPositive(op: string, c: unknown, arg: unknown): boolean { - switch (op) { - case '$eq': - return eq(c, arg) - case '$in': - if (!Array.isArray(arg)) throw new TypeError('$in expects an array') - return arg.some((x) => eq(c, x)) - default: { - const rc = typeRank(c) - if (rc !== typeRank(arg) || rc === 4) return false - const d = compareValues(c, arg) - return op === '$gt' ? d > 0 : op === '$gte' ? d >= 0 : op === '$lt' ? d < 0 : d <= 0 - } - } -} - -// ------------------------------------------------------------------ planner analysis - -export interface RangePred { - gt?: IndexValue - gte?: IndexValue - lt?: IndexValue - lte?: IndexValue -} - -export interface FieldPred { - eq?: IndexValue - range?: RangePred -} - -/** - * Index-usable predicates per field, from the top level and top-level `$and`. - * Only scalar `$eq`/implicit equality and same-typed scalar range bounds qualify. - */ -export function analyze(filter: Filter, out = new Map()): Map { - for (const key of Object.keys(filter)) { - const cond = filter[key] - if (key === '$and') { - for (const f of cond as Filter[]) analyze(f, out) - continue - } - if (key.startsWith('$')) continue - const pred = out.get(key) ?? {} - if (!isOperatorObject(cond)) { - if (isScalar(cond)) pred.eq ??= cond - } else { - if ('$eq' in cond && isScalar(cond.$eq)) pred.eq ??= cond.$eq - const range: RangePred = {} - let rank = -1 - let ok = false - for (const op of ['$gt', '$gte', '$lt', '$lte'] as const) { - if (!(op in cond)) continue - const v = cond[op] - if (!isScalar(v)) continue - const r = typeRank(v) - if (rank !== -1 && r !== rank) continue - rank = r - range[op.slice(1) as keyof RangePred] = v - ok = true - } - if (ok && !pred.range) pred.range = range - } - if (pred.eq !== undefined || pred.range) out.set(key, pred) - } - return out -} - -export type { Doc } diff --git a/src/documents/index.ts b/src/documents/index.ts deleted file mode 100644 index 8a2ad21..0000000 --- a/src/documents/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './types.js' -export * from './event-sourced.js' -export * from './filter.js' -export * from './update.js' -export * from './indexes.js' -export * from './ulid.js' -export { compareValues, deepEqual, type Doc } from './values.js' diff --git a/src/documents/indexes.ts b/src/documents/indexes.ts deleted file mode 100644 index 8d9d77f..0000000 --- a/src/documents/indexes.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { encodeIndexValues, indexValueCodec, type IndexValue } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ } from '../core/path.js' -import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js' -import type { FieldPred, RangePred } from './filter.js' -import { indexScalars, resolvePath, typeRank, type Doc } from './values.js' - -export interface DocIndexSpec { - /** Default: the fields joined with `,`. */ - name?: string - fields: string[] - /** Claims `~u//` per value tuple. Documents where every field is missing/null are not claimed. */ - unique?: boolean -} - -/** Path pattern (`*` = one segment, `''` = root) → indexes over that node's direct children. */ -export type DocIndexDeclarations = Record - -export interface SortSpec { - [field: string]: 1 | -1 -} - -interface CompiledDocIndex { - name: string - fields: string[] - segs: string[][] - unique: boolean - /** `escapeSegment(name) + sep` */ - head: string -} - -export interface Plan { - index: string - gte: string - lt: string - reverse: boolean - /** Index order satisfies the requested sort. */ - sorted: boolean -} - -/** Document indexes over the children of one node. */ -export class DocIndexes { - readonly #list: CompiledDocIndex[] - readonly #byName: Map - readonly #enc: (v: IndexValue) => string - - constructor( - list: CompiledDocIndex[], - /** `~i` */ - readonly base: string, - /** `~u` */ - readonly uniqueBase: string, - readonly sep: string, - ) { - this.#list = list - this.#byName = new Map(list.map((i) => [i.name, i])) - const codec = indexValueCodec(sep) - this.#enc = (v) => codec.encode(v) - } - - get hasUnique(): boolean { - return this.#list.some((i) => i.unique) - } - - /** Value tuples a state produces for one index (cartesian over multikey fields). */ - #tuples(idx: CompiledDocIndex, state: Doc): IndexValue[][] { - let tuples: IndexValue[][] = [[]] - let multi = 0 - for (const segs of idx.segs) { - const found = resolvePath(state, segs) - const scalars = - found.length === 0 ? [null] : found.length === 1 && !Array.isArray(found[0]) ? indexScalars(found[0]) : [...new Set(found.flatMap(indexScalars))] - if (tuples.length === 1 && scalars.length === 1) { - tuples[0]!.push(scalars[0]!) - continue - } - if (scalars.length > 1 && ++multi > 1) throw new TypeError(`Index "${idx.name}": cannot index more than one array field per document`) - tuples = tuples.flatMap((t) => scalars.map((s) => [...t, s])) - } - return tuples - } - - entriesFor(escChild: string, state: Doc | undefined, out: Set = new Set()): Set { - if (!state) return out - for (const idx of this.#list) { - const stem = this.base + idx.head - for (const t of this.#tuples(idx, state)) out.add(stem + encodeIndexValues(t, this.sep) + this.sep + escChild) - } - return out - } - - /** Unique claim key → index name and values. */ - claimsFor(state: Doc | undefined): Map { - const out = new Map() - if (!state) return out - for (const idx of this.#list) { - if (!idx.unique) continue - for (const t of this.#tuples(idx, state)) { - if (t.every((v) => v === null)) continue - out.set(this.uniqueBase + idx.head + encodeIndexValues(t, this.sep), { index: idx.name, values: t }) - } - } - return out - } - - /** Escaped child segment of an entry key of index `name`. */ - childOf(name: string, entryKey: string): string { - const idx = this.#byName.get(name)! - let pos = this.base.length + idx.head.length - for (let i = 0; i < idx.fields.length; i++) pos = entryKey.indexOf(this.sep, pos) + 1 - return entryKey.slice(pos) - } - - /** - * Chooses the index covering the most leading equality fields, then at most one range field. - * Ties prefer an index whose order satisfies `sort`. With no usable predicate, an index whose - * order satisfies `sort` is still used (a full, ordered index scan). Returns `undefined` for a - * full projection scan. - */ - plan(preds: Map, sort: [string, 1 | -1][]): Plan | undefined { - let best: { idx: CompiledDocIndex; k: number; range: RangePred | undefined; sorted: boolean; dir: 1 | -1; score: number } | undefined - for (const idx of this.#list) { - let k = 0 - while (k < idx.fields.length && preds.get(idx.fields[k]!)?.eq !== undefined) k++ - const range = k < idx.fields.length ? preds.get(idx.fields[k]!)?.range : undefined - // Sort fields constrained by equality are constant; drop them before matching. - const eqFields = new Set(idx.fields.slice(0, k)) - const rest = sort.filter(([f]) => !eqFields.has(f)) - // Ties are broken by child key in the direction of the first sort field. - const dir = rest[0]?.[1] ?? sort[0]?.[1] ?? 1 - const sorted = rest.length === 0 || (rest.every(([f, d], i) => idx.fields[k + i] === f && d === dir) && rest.length <= idx.fields.length - k) - const score = k * 2 + (range ? 1 : 0) - if (score === 0 && !(sorted && rest.length > 0)) continue - if (!best || score > best.score || (score === best.score && sorted && !best.sorted)) best = { idx, k, range, sorted, dir, score } - } - if (!best) return undefined - const { idx, k, range, sorted, dir } = best - const succSep = succ(this.sep) - let prefix = this.base + idx.head - for (let i = 0; i < k; i++) prefix += this.#enc(preds.get(idx.fields[i]!)!.eq!) + this.sep - // prefix ends with sep: the range of everything under it is [prefix, prefix-without-sep + succ(sep)). - const all = { gte: prefix, lt: prefix.slice(0, -1) + succSep } - let { gte, lt } = all - if (range) { - const sample = (range.gt ?? range.gte ?? range.lt ?? range.lte) as IndexValue - const [lo, hi] = typeBounds(typeRank(sample), this.#enc) - gte = prefix + lo - lt = hi === undefined ? all.lt : prefix + hi - if (range.gte !== undefined) gte = max(gte, prefix + this.#enc(range.gte)) - if (range.gt !== undefined) gte = max(gte, prefix + this.#enc(range.gt) + succSep) - if (range.lt !== undefined) lt = min(lt, prefix + this.#enc(range.lt)) - if (range.lte !== undefined) lt = min(lt, prefix + this.#enc(range.lte) + succSep) - } - return { index: idx.name, gte, lt, reverse: sorted && dir === -1, sorted } - } -} - -/** Encoded `[lower, upper)` of one type class; `undefined` upper = end of index. */ -function typeBounds(rank: number, enc: (v: IndexValue) => string): [string, string | undefined] { - switch (rank) { - case 0: - return [enc(null), enc(false)] - case 1: - return [enc(false), enc(-Infinity)] - case 2: - return [enc(-Infinity), enc('')] - default: - return [enc(''), undefined] - } -} - -const max = (a: string, b: string) => (a > b ? a : b) -const min = (a: string, b: string) => (a < b ? a : b) - -export class DocIndexRegistry { - readonly trie: PatternTrie - readonly empty: boolean - readonly hasUnique: boolean - - constructor( - decls: DocIndexDeclarations, - readonly sep: string, - ) { - this.trie = new PatternTrie(sep) - const patterns = Object.keys(decls) - const compiled = new Map() - for (const p of patterns) { - const list = decls[p]!.map((spec): CompiledDocIndex => { - if (!spec.fields.length) throw new TypeError(`Index on "${p}" has no fields`) - const name = spec.name ?? spec.fields.join(',') - return { name, fields: spec.fields, segs: spec.fields.map((f) => f.split('.')), unique: spec.unique === true, head: escapeSegment(name, sep) + sep } - }) - const names = list.map((i) => i.name) - const dup = names.find((n, i) => names.indexOf(n) !== i) - if (dup) throw new TypeError(`Duplicate index name "${dup}" on "${p}"`) - compiled.set(p, list) - } - for (let i = 0; i < patterns.length; i++) { - for (let j = i + 1; j < patterns.length; j++) { - const a = patterns[i]! - const b = patterns[j]! - if (!patternsOverlap(a, b, sep)) continue - const bn = new Set(compiled.get(b)!.map((x) => x.name)) - const clash = compiled.get(a)!.filter((x) => bn.has(x.name)) - if (clash.length) throw new TypeError(`Index name(s) ${clash.map((c) => c.name).join(', ')} declared on overlapping patterns "${a}" and "${b}"`) - } - } - for (const p of patterns) this.trie.add(p, compiled.get(p)!) - this.empty = patterns.length === 0 - this.hasUnique = [...compiled.values()].some((l) => l.some((i) => i.unique)) - } - - /** `projPrefix` is the projection-space prefix of the node (``). */ - bind(state: TrieState, projPrefix: string): DocIndexes | undefined { - if (this.empty) return undefined - const found = PatternTrie.values(state) - if (found.length === 0) return undefined - const list = found.flatMap((f) => f.value) - return new DocIndexes(list, projPrefix + RESERVED.index + this.sep, projPrefix + RESERVED.unique + this.sep, this.sep) - } -} diff --git a/src/documents/types.ts b/src/documents/types.ts deleted file mode 100644 index 2435656..0000000 --- a/src/documents/types.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { StoredEvent } from '../events/types.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import type { Filter } from './filter.js' -import type { SortSpec } from './indexes.js' -import type { DocEvent, UpdateSpec } from './update.js' - -export interface FindOptions { - sort?: SortSpec - limit?: number - skip?: number - /** Inclusion (`{ a: 1 }`) or exclusion (`{ a: 0 }`) of dot-path fields. */ - projection?: Record -} - -export interface FindResult { - key: string - data: T -} - -export interface GetOptions { - /** Check the stream head and catch the projection up first (one extra scan). */ - consistent?: boolean -} - -export interface UpdateOptions { - /** Treat a missing document as `{}`. */ - upsert?: boolean -} - -export interface DocKeysOptions { - limit?: number - after?: string | number - reverse?: boolean -} - -/** Document type of a node's children. */ -export type ChildDoc = ValueOf>> - -/** - * One node of a document store: a document (its own) and a collection of child documents. - * Same navigation and self/child arity rules as `KVNode`; in forms with an optional leading key, - * a non-null object first argument selects the self form. - */ -export interface DocumentNode { - readonly path: readonly string[] - child>(key: K): DocumentNode> - as(): DocumentNode - - get(opts?: GetOptions): Promise | undefined> - get>(key: K, opts?: GetOptions): Promise> | undefined> - - set(doc: ValueOf): Promise - set>(key: K, doc: ValueOf>): Promise - - /** Inserts a child with a generated time-sortable id; returns the id. */ - insert(doc: ChildDoc): Promise - - /** Returns the new state, or `undefined` if the document doesn't exist (and no `upsert`). */ - update(spec: UpdateSpec, opts?: UpdateOptions): Promise | undefined> - update>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise> | undefined> - - /** Returns whether a document was deleted. */ - delete(): Promise - delete>(key: K): Promise - - /** Queries direct children. */ - find(filter?: Filter, opts?: FindOptions): AsyncIterable>> - findOne(filter?: Filter, opts?: FindOptions): Promise> | undefined> - - /** Keys of direct children that hold a (non-deleted) document. */ - keys(opts?: DocKeysOptions): AsyncIterable> -} - -/** Document store backed by an event log: adds history and point-in-time reads. */ -export interface EventSourcedDocumentNode extends DocumentNode { - child>(key: K): EventSourcedDocumentNode> - as(): EventSourcedDocumentNode - - history(): AsyncIterable> - history>(key: K): AsyncIterable> - - /** State as of `version` (`undefined` if missing or deleted then). */ - getAt(version: number): Promise | undefined> - getAt>(key: K, version: number): Promise> | undefined> - - /** Rebuilds lagging projections for this node and all descendants; returns how many were repaired. */ - catchUp(): Promise -} - -export type DocumentStorage = DocumentNode diff --git a/src/documents/ulid.ts b/src/documents/ulid.ts deleted file mode 100644 index ecf27ac..0000000 --- a/src/documents/ulid.ts +++ /dev/null @@ -1,37 +0,0 @@ -const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32 - -/** - * ULID-like id generator: 10 chars of millisecond timestamp + 16 chars of randomness, - * lexicographically sortable by creation time. Ids generated within the same millisecond - * increment the random part, so they stay strictly increasing per generator. - */ -export function createIdGenerator(now: () => number = Date.now): () => string { - let lastTime = -1 - const rand = new Uint8Array(16) // one base32 digit (0–31) per byte - return () => { - let t = now() - if (t <= lastTime) { - t = lastTime - // Increment the random part (base 32, right to left). - let i = 15 - while (i >= 0 && rand[i] === 31) rand[i--] = 0 - if (i < 0) throw new Error('ULID random overflow within one millisecond') - rand[i]!++ - } else { - crypto.getRandomValues(rand) - for (let i = 0; i < 16; i++) rand[i]! &= 31 - // Leave headroom so increments rarely overflow. - rand[0]! &= 15 - lastTime = t - } - let out = '' - for (let i = 9; i >= 0; i--) { - out = ALPHABET[t % 32]! + out - t = Math.floor(t / 32) - } - for (let i = 0; i < 16; i++) out += ALPHABET[rand[i]!] - return out - } -} - -export const ulid = createIdGenerator() diff --git a/src/documents/update.ts b/src/documents/update.ts deleted file mode 100644 index 79ea9ab..0000000 --- a/src/documents/update.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { isOperatorObject, matches, matchesField, type Filter } from './filter.js' -import { clone, deepEqual, getPath, isPlainObject, setPath, unsetPath, type Doc } from './values.js' - -/** Update operators supported in v1. Paths may use dots. */ -export interface UpdateSpec { - $set?: Record - $unset?: Record - $inc?: Record - $push?: Record - $pull?: Record - $addToSet?: Record -} - -/** Stored document events: resolved deltas, never operators, so replay is deterministic. */ -export type DocEvent = { op: 'set'; doc: Doc } | { op: 'patch'; set?: Record; unset?: string[] } | { op: 'delete' } - -export interface Patch { - set: Record - unset: string[] -} - -const OPS = new Set(['$set', '$unset', '$inc', '$push', '$pull', '$addToSet']) - -/** Resolves operators against `state` into a patch of final values. */ -export function resolveUpdate(state: Doc, spec: UpdateSpec): Patch { - const keys = Object.keys(spec) - if (keys.length === 0) throw new TypeError('Empty update') - for (const k of keys) if (!OPS.has(k)) throw new TypeError(`Unsupported update operator ${k}`) - - const patch: Patch = { set: {}, unset: [] } - const touched: string[] = [] - const touch = (path: string) => { - for (const t of touched) { - if (t === path || t.startsWith(path + '.') || path.startsWith(t + '.')) throw new TypeError(`Update paths conflict: "${t}" and "${path}"`) - } - touched.push(path) - } - const arrayAt = (path: string, op: string): unknown[] => { - const cur = getPath(state, path) - if (cur === undefined || cur === null) return [] - if (!Array.isArray(cur)) throw new TypeError(`${op} on non-array field "${path}"`) - return cur - } - const eachOf = (v: unknown): unknown[] => (isPlainObject(v) && Object.keys(v).length === 1 && Array.isArray(v.$each) ? (v.$each as unknown[]) : [v]) - - for (const [path, v] of Object.entries(spec.$set ?? {})) { - touch(path) - patch.set[path] = clone(v) - } - for (const path of Object.keys(spec.$unset ?? {})) { - touch(path) - patch.unset.push(path) - } - for (const [path, by] of Object.entries(spec.$inc ?? {})) { - touch(path) - if (typeof by !== 'number') throw new TypeError(`$inc by non-number at "${path}"`) - const cur = getPath(state, path) - if (cur !== undefined && cur !== null && typeof cur !== 'number') throw new TypeError(`$inc on non-number field "${path}"`) - patch.set[path] = ((cur as number | undefined) ?? 0) + by - } - for (const [path, v] of Object.entries(spec.$push ?? {})) { - touch(path) - patch.set[path] = [...clone(arrayAt(path, '$push')), ...clone(eachOf(v))] - } - for (const [path, v] of Object.entries(spec.$addToSet ?? {})) { - touch(path) - const next = clone(arrayAt(path, '$addToSet')) - for (const item of eachOf(v)) if (!next.some((x) => deepEqual(x, item))) next.push(clone(item)) - patch.set[path] = next - } - for (const [path, cond] of Object.entries(spec.$pull ?? {})) { - touch(path) - patch.set[path] = clone(arrayAt(path, '$pull')).filter((el) => !pullMatches(el, cond)) - } - return patch -} - -function pullMatches(el: unknown, cond: unknown): boolean { - if (isOperatorObject(cond)) return matchesField([el], cond) - if (isPlainObject(cond) && isPlainObject(el)) return matches(el, cond as Filter) - return deepEqual(el, cond) -} - -/** Applies a resolved patch to a copy of `state`. */ -export function applyPatch(state: Doc, patch: { set?: Record; unset?: string[] }): Doc { - const next = clone(state) - for (const [path, v] of Object.entries(patch.set ?? {})) setPath(next, path, clone(v)) - for (const path of patch.unset ?? []) unsetPath(next, path) - return next -} - -/** Folds one event into a state (`undefined` = no document). */ -export function applyEvent(state: Doc | undefined, e: DocEvent): Doc | undefined { - switch (e.op) { - case 'set': - return clone(e.doc) - case 'patch': - return applyPatch(state ?? {}, e) - case 'delete': - return undefined - } -} - -export function toPatchEvent(p: Patch): DocEvent { - const e: { op: 'patch'; set?: Record; unset?: string[] } = { op: 'patch' } - if (Object.keys(p.set).length) e.set = p.set - if (p.unset.length) e.unset = p.unset - return e -} diff --git a/src/documents/values.ts b/src/documents/values.ts deleted file mode 100644 index 4c416b3..0000000 --- a/src/documents/values.ts +++ /dev/null @@ -1,163 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' - -/** Plain JSON object. */ -export type Doc = Record - -export function isPlainObject(v: unknown): v is Doc { - return typeof v === 'object' && v !== null && !Array.isArray(v) -} - -export function isScalar(v: unknown): v is IndexValue { - return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v)) -} - -export function splitPath(path: string): string[] { - return path.split('.') -} - -/** - * Values reachable at a dot path. Arrays are traversed: a non-numeric segment applied to an - * array maps over its elements. Missing paths yield no values. - */ -export function resolvePath(doc: unknown, path: string | readonly string[]): unknown[] { - const segs = typeof path === 'string' ? splitPath(path) : path - const out: unknown[] = [] - walk(doc, segs, 0, out) - return out -} - -function walk(v: unknown, segs: readonly string[], i: number, out: unknown[]): void { - if (i === segs.length) { - if (v !== undefined) out.push(v) - return - } - const seg = segs[i]! - if (Array.isArray(v)) { - if (/^\d+$/.test(seg)) walk(v[Number(seg)], segs, i + 1, out) - else for (const el of v) if (isPlainObject(el) || Array.isArray(el)) walk(el, segs, i, out) - return - } - if (isPlainObject(v) && Object.hasOwn(v, seg)) walk(v[seg], segs, i + 1, out) -} - -/** Value at a dot path without array traversal (numeric segments index arrays). */ -export function getPath(doc: unknown, path: string): unknown { - let v = doc - for (const seg of splitPath(path)) { - if (Array.isArray(v) && /^\d+$/.test(seg)) v = v[Number(seg)] - else if (isPlainObject(v) && Object.hasOwn(v, seg)) v = v[seg] - else return undefined - } - return v -} - -/** Sets a dot path in place, creating intermediate objects. */ -export function setPath(doc: Doc, path: string, value: unknown): void { - const segs = splitPath(path) - let v: unknown = doc - for (let i = 0; i < segs.length - 1; i++) { - const seg = segs[i]! - const container = v as Record - let next = Array.isArray(container) && /^\d+$/.test(seg) ? container[Number(seg)] : container[seg] - if (next === undefined || next === null) { - next = {} - if (Array.isArray(container)) container[Number(seg)] = next - else container[seg] = next - } else if (typeof next !== 'object') { - throw new TypeError(`Cannot set "${path}": "${segs.slice(0, i + 1).join('.')}" is not an object`) - } - v = next - } - const last = segs.at(-1)! - if (Array.isArray(v) && /^\d+$/.test(last)) v[Number(last)] = value - else (v as Doc)[last] = value -} - -export function unsetPath(doc: Doc, path: string): void { - const segs = splitPath(path) - let v: unknown = doc - for (let i = 0; i < segs.length - 1; i++) { - v = isPlainObject(v) ? v[segs[i]!] : Array.isArray(v) && /^\d+$/.test(segs[i]!) ? v[Number(segs[i])] : undefined - if (typeof v !== 'object' || v === null) return - } - const last = segs.at(-1)! - if (Array.isArray(v) && /^\d+$/.test(last)) - v[Number(last)] = null // Mongo semantics: keep positions - else if (isPlainObject(v)) delete v[last] -} - -export function deepEqual(a: unknown, b: unknown): boolean { - if (a === b) return true - if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false - if (Array.isArray(a) !== Array.isArray(b)) return false - if (Array.isArray(a)) { - const bb = b as unknown[] - return a.length === bb.length && a.every((x, i) => deepEqual(x, bb[i])) - } - const ka = Object.keys(a) - const kb = Object.keys(b) - return ka.length === kb.length && ka.every((k) => Object.hasOwn(b, k) && deepEqual((a as Doc)[k], (b as Doc)[k])) -} - -/** Deep copy of a JSON-compatible value (documents are JSON). */ -export function clone(v: T): T { - if (v === null || typeof v !== 'object') return v - if (Array.isArray(v)) return v.map(clone) as T - const out: Doc = {} - for (const k in v) out[k] = clone((v as Doc)[k]) - return out as T -} - -/** Type class used for bracketing comparisons, in index-encoding order. */ -export function typeRank(v: unknown): number { - if (v === null || v === undefined) return 0 - if (typeof v === 'boolean') return 1 - if (typeof v === 'number') return 2 - if (typeof v === 'string') return 3 - return 4 -} - -/** - * Total order matching the index encoding for scalars: - * `null/missing < false < true < numbers < strings < anything else`. - */ -export function compareValues(a: unknown, b: unknown): number { - const ra = typeRank(a) - const rb = typeRank(b) - if (ra !== rb) return ra < rb ? -1 : 1 - if (ra === 1 || ra === 2 || ra === 3) return a === b ? 0 : (a as number) < (b as number) ? -1 : 1 - if (ra === 4) { - const sa = JSON.stringify(a) - const sb = JSON.stringify(b) - return sa === sb ? 0 : sa < sb ? -1 : 1 - } - return 0 -} - -/** - * The scalar(s) an index stores for a field value: scalars as-is, arrays as their scalar - * elements (multikey; empty → null), and missing or non-scalar values as `null`. - */ -export function indexScalars(v: unknown): IndexValue[] { - if (Array.isArray(v)) { - const out: IndexValue[] = [] - for (const el of v) out.push(isScalar(el) ? el : null) - return out.length ? out : [null] - } - return [isScalar(v) ? v : null] -} - -/** Sort key for a field under a direction; arrays sort by their min (asc) or max (desc) element. */ -export function sortKey(doc: unknown, path: string, dir: 1 | -1): unknown { - const vals = resolvePath(doc, path) - if (vals.length === 0) return null - let best: unknown - let first = true - for (const v of vals) { - for (const s of indexScalars(v)) { - if (first || compareValues(s, best) * dir < 0) best = s - first = false - } - } - return best -} diff --git a/src/errors.ts b/src/errors.ts deleted file mode 100644 index 8b3528a..0000000 --- a/src/errors.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** Base class for all PotentialStorage errors. */ -export class PotentialStorageError extends Error { - constructor(message: string) { - super(message) - this.name = new.target.name - } -} - -/** An optimistic-concurrency check failed (stale `expectedVersion` or a lost commit race). */ -export class ConcurrencyError extends PotentialStorageError { - constructor( - readonly stream: string, - readonly expectedVersion: number | 'any', - readonly actualVersion: number, - ) { - super(`Concurrency conflict on "${stream}": expected version ${expectedVersion}, actual ${actualVersion}`) - } -} - -/** A unique index already holds the given value for another key. */ -export class UniqueConstraintError extends PotentialStorageError { - constructor( - readonly index: string, - readonly values: readonly unknown[], - readonly existingKey?: string, - ) { - super(`Unique index "${index}" already contains ${JSON.stringify(values)}${existingKey === undefined ? '' : ` (key "${existingKey}")`}`) - } -} - -/** The driver lacks an optional capability required by the requested feature. */ -export class CapabilityError extends PotentialStorageError { - constructor( - readonly capability: string, - detail?: string, - ) { - super(`Driver lacks capability "${capability}"${detail ? `: ${detail}` : ''}`) - } -} - -/** - * A key argument is invalid: passed but `undefined`/`null`, empty, or otherwise unusable. - * Extends `TypeError` so callers can catch either. - */ -export class InvalidKeyError extends TypeError { - constructor(message: string) { - super(message) - this.name = 'InvalidKeyError' - } -} diff --git a/src/event-storage/base-event-storage.ts b/src/event-storage/base-event-storage.ts new file mode 100644 index 0000000..173e361 --- /dev/null +++ b/src/event-storage/base-event-storage.ts @@ -0,0 +1,100 @@ +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueAt, ValueOf } from '../utils/schema.js' + +export interface AppendOptions { + /** Version the stream must be at (0 = empty). Default `'any'`. */ + expectedVersion?: number | 'any' +} + +export interface AppendResult { + /** Stream version after the append. */ + version: number +} + +export interface ReadOptions { + /** First version to read (inclusive). Default 1. */ + from?: number + /** Last version to read (inclusive). Default: the head. */ + to?: number + limit?: number +} + +export interface StoredEvent { + version: number + /** Append timestamp (ms since epoch). */ + at: number + data: E +} + +export interface StreamKeysOptions { + limit?: number + after?: string | number + reverse?: boolean +} + +/** + * Append-only event streams. Every node is a stream (its own events) and a parent of child + * streams. Implementations provide four primitives over stream paths; the user-facing, + * self-or-child API is defined here once. + */ +export abstract class BaseEventStorage { + /** Unescaped path of this node (`[]` for the root, which is not a stream). */ + path: readonly string[] = [] + + abstract appendTo(stream: readonly string[], events: unknown[], expectedVersion: number | 'any'): Promise + abstract readFrom(stream: readonly string[], options: ReadOptions): AsyncIterable> + abstract versionOf(stream: readonly string[]): Promise + abstract streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncIterable + + child>(key: K): BaseEventStorage> { + const node = Object.create(this) as BaseEventStorage> + node.path = [...this.path, Keys.segment(key)] + return node + } + + as(): BaseEventStorage { + return this as unknown as BaseEventStorage + } + + /** `append(events, options?)` on this stream; `append(key, events, options?)` on a child stream. */ + append>( + ...args: [events: ValueOf[], options?: AppendOptions] | [key: K, events: ValueOf>[], options?: AppendOptions] + ): Promise { + const hasKey = !Array.isArray(args[0]) + const events = args[hasKey ? 1 : 0] + const options = (args[hasKey ? 2 : 1] ?? {}) as AppendOptions + if (!Array.isArray(events)) return Promise.reject(new TypeError('append() expects an array of events')) + return this.withStream(hasKey, args[0], (stream) => this.appendTo(stream, events, options.expectedVersion ?? 'any')) + } + + /** `read(options?)` this stream; `read(key, options?)` a child stream. */ + async *read>(...args: [options?: ReadOptions] | [key: K, options?: ReadOptions]): AsyncGenerator>> { + const hasKey = typeof args[0] !== 'object' && args.length > 0 + const stream = this.streamPath(hasKey, args[0]) + yield* this.readFrom(stream, ((hasKey ? args[1] : args[0]) ?? {}) as ReadOptions) as AsyncIterable>> + } + + /** Current version of this stream (or a child's); 0 when empty. */ + version(key?: KeyOf): Promise { + return this.withStream(arguments.length > 0, key, (stream) => this.versionOf(stream)) + } + + /** Keys of child streams. */ + keys(options: StreamKeysOptions = {}): AsyncIterable> { + return this.streamsOf(this.path, options) as AsyncIterable> + } + + private withStream(hasKey: boolean, key: unknown, fn: (stream: readonly string[]) => Promise): Promise { + try { + return fn(this.streamPath(hasKey, key)) + } catch (error) { + return Promise.reject(error) + } + } + + private streamPath(hasKey: boolean, key: unknown): readonly string[] { + if (hasKey) return [...this.path, Keys.segment(key)] + if (this.path.length === 0) throw new TypeError('The root node is not a stream') + return this.path + } +} diff --git a/src/event-storage/event-storage-kv.ts b/src/event-storage/event-storage-kv.ts new file mode 100644 index 0000000..aa5e00a --- /dev/null +++ b/src/event-storage/event-storage-kv.ts @@ -0,0 +1,151 @@ +import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js' +import { ConcurrencyError } from '../utils/errors.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseEventStorage, type AppendResult, type ReadOptions, type StoredEvent, type StreamKeysOptions } from './base-event-storage.js' + +/** Stored value of one commit key. */ +interface Commit { + firstVersion: number + at: number + events: unknown[] +} + +interface Head { + commit: number + version: number +} + +export interface EventStorageKVOptions { + /** Retries after losing a commit race when `expectedVersion` is `'any'`. Default 5. */ + maxRetries?: number + /** Max streams whose head is cached. Default 10 000. */ + cacheSize?: number + now?: () => number +} + +/** + * Event storage on any KV storage. Each append is one commit key, densely numbered: + * `/~e/` → `{ firstVersion, at, events }`. Commits are claimed with + * `writeIfAbsent`, so a writer with a stale view always collides with the existing key. + */ +export class EventStorageKV extends BaseEventStorage { + private readonly heads = new Map() + private readonly maxRetries: number + private readonly cacheSize: number + private readonly now: () => number + + constructor( + private readonly kv: BaseKVStorage, + options: EventStorageKVOptions = {}, + ) { + super() + this.maxRetries = options.maxRetries ?? 5 + this.cacheSize = options.cacheSize ?? 10_000 + this.now = options.now ?? Date.now + } + + async appendTo(stream: readonly string[], events: unknown[], expectedVersion: number | 'any'): Promise { + const base = this.commitPrefix(stream) + for (let attempt = 0; ; attempt++) { + let head = this.heads.get(base) ?? (await this.loadHead(base)) + // The cache may be stale; only trust a mismatch after re-reading. + if (expectedVersion !== 'any' && head.version !== expectedVersion) head = await this.loadHead(base) + if (expectedVersion !== 'any' && head.version !== expectedVersion) throw new ConcurrencyError(base, expectedVersion, head.version) + if (events.length === 0) return { version: head.version } + + const commit: Commit = { firstVersion: head.version + 1, at: this.now(), events } + const [written] = await this.kv.writeIfAbsent([[base + EventStorageKV.pad(head.commit + 1), commit]]) + if (written) { + const next = { commit: head.commit + 1, version: head.version + events.length } + this.cacheHead(base, next) + return { version: next.version } + } + this.heads.delete(base) + if (expectedVersion !== 'any' || attempt >= this.maxRetries) { + throw new ConcurrencyError(base, expectedVersion, (await this.loadHead(base)).version) + } + } + } + + async *readFrom(stream: readonly string[], options: ReadOptions): AsyncGenerator> { + const from = Math.max(1, options.from ?? 1) + const to = options.to ?? Infinity + let remaining = options.limit ?? Infinity + if (to < from || remaining <= 0) return + const base = this.commitPrefix(stream) + const start = from === 1 ? 1 : await this.findCommit(base, from) + if (start === 0) return + for await (const [, value] of this.kv.scan({ gte: base + EventStorageKV.pad(start), lt: this.commitEnd(base), values: true })) { + const commit = value as Commit + for (let i = 0; i < commit.events.length; i++) { + const version = commit.firstVersion + i + if (version < from) continue + if (version > to || remaining-- <= 0) return + yield { version, at: commit.at, data: commit.events[i] } + } + } + } + + async versionOf(stream: readonly string[]): Promise { + return (await this.loadHead(this.commitPrefix(stream))).version + } + + async *streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncGenerator { + yield* parent.reduce((node, segment) => node.child(segment), this.kv).keys(options) + } + + /** Last commit of a stream: one reverse scan with limit 1. Refreshes the cache. */ + private async loadHead(base: string): Promise { + for await (const [key, value] of this.kv.scan({ gte: base, lt: this.commitEnd(base), reverse: true, limit: 1, values: true })) { + const commit = value as Commit + const head = { commit: Number(key.slice(base.length)), version: commit.firstVersion + commit.events.length - 1 } + this.cacheHead(base, head) + return head + } + return { commit: 0, version: 0 } + } + + /** + * Commit containing `version`, or 0 if the stream is shorter. Commit `n` starts at a version + * >= n, so it's at most `version`: try that first (exact when commits hold one event each), + * then binary-search the commits below it. + */ + private async findCommit(base: string, version: number): Promise { + const firstVersion = async (commit: number) => ((await this.kv.readMany([base + EventStorageKV.pad(commit)]))[0] as Commit | undefined)?.firstVersion + const guess = await firstVersion(version) + if (guess !== undefined && guess <= version) return version + let hi = version - 1 + if (guess === undefined) { + const head = await this.loadHead(base) + if (head.version < version) return 0 + hi = head.commit + } + let lo = 1 + while (lo < hi) { + const mid = (lo + hi + 1) >>> 1 + if ((await firstVersion(mid))! <= version) lo = mid + else hi = mid - 1 + } + return lo + } + + private cacheHead(base: string, head: Head): void { + this.heads.delete(base) + this.heads.set(base, head) + if (this.heads.size > this.cacheSize) this.heads.delete(this.heads.keys().next().value!) + } + + private commitPrefix(stream: readonly string[]): string { + const prefix = this.kv.prefix + Keys.join(stream) + return `${prefix}/~e/` + } + + private commitEnd(base: string): string { + return base.slice(0, -1) + Keys.afterSeparator + } + + private static pad(commit: number): string { + return String(commit).padStart(12, '0') + } +} diff --git a/src/events/index.ts b/src/events/index.ts deleted file mode 100644 index 057ee0f..0000000 --- a/src/events/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './types.js' -export * from './kv.js' diff --git a/src/events/kv.ts b/src/events/kv.ts deleted file mode 100644 index 2ccf9fa..0000000 --- a/src/events/kv.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { LRU } from '../core/lru.js' -import { KeyedMutex } from '../core/mutex.js' -import { escapeSegment, RESERVED, succ, toSegment } from '../core/path.js' -import { ConcurrencyError } from '../errors.js' -import type { KVDriver } from '../kv/driver.js' -import type { KVNode } from '../kv/node.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import type { AppendOptions, AppendResult, EventNode, ReadOptions, StoredEvent, StreamKeysOptions } from './types.js' - -export interface KVEventStorageOptions { - /** Max streams whose head is cached. Default 10 000. */ - headCacheSize?: number - /** Retries after a commit collision when `expectedVersion` is `'any'`. Default 5. */ - maxRetries?: number - /** Clock for commit timestamps. Default `Date.now`. */ - now?: () => number -} - -/** Stored value of one commit key. */ -export interface CommitRecord { - firstVersion: number - at: number - events: E[] -} - -interface Head { - commit: number - version: number -} - -interface Stream { - /** Stream's own key (escaped); identifies it in the cache and in errors. */ - key: string - /** `/~e/` — commit keys are `base + pad12(seq)`. */ - base: string - /** Exclusive upper bound of commit keys. */ - end: string -} - -/** @internal */ -interface EventContext { - readonly driver: KVDriver - readonly sep: string - readonly heads: LRU - readonly mutex: KeyedMutex - readonly maxRetries: number - readonly now: () => number -} - -const SEQ_WIDTH = 12 -const SEARCH_FANOUT = 8 -const EMPTY_HEAD: Head = { commit: 0, version: 0 } - -const pad = (seq: number) => String(seq).padStart(SEQ_WIDTH, '0') - -/** - * Event node over a `KVNode`. Each append is one commit key, densely numbered: - * `/~e/` → `{ firstVersion, at, events }`. Commits are claimed with - * `setIfAbsent`, so any stale writer collides with the existing key. Without `setIfAbsent`, - * appends are serialised by an in-process per-stream mutex (single-writer only). - */ -export class KVEventNode implements EventNode { - readonly #ctx: EventContext - readonly #kv: KVNode - - /** @internal Use `KVEventStorage` and `child()`. */ - constructor(ctx: EventContext, kv: KVNode) { - this.#ctx = ctx - this.#kv = kv - } - - get path(): readonly string[] { - return this.#kv.path - } - - /** The underlying KV node. */ - get kv(): KVNode { - return this.#kv - } - - child>(key: K): KVEventNode> { - return new KVEventNode(this.#ctx, this.#kv.child(key as string).as()) - } - - as(): KVEventNode { - return this as unknown as KVEventNode - } - - append(events: readonly ValueOf[], opts?: AppendOptions): Promise - append>(key: K, events: readonly ValueOf>[], opts?: AppendOptions): Promise - async append(...args: unknown[]): Promise { - const self = Array.isArray(args[0]) - const stream = self ? this.#self() : this.#child(args[0]) - const events = args[self ? 0 : 1] - const opts = (args[self ? 1 : 2] ?? {}) as AppendOptions - if (!Array.isArray(events)) throw new TypeError('append() expects an array of events') - const expected = opts.expectedVersion ?? 'any' - if (expected !== 'any' && !(Number.isInteger(expected) && expected >= 0)) throw new TypeError(`Invalid expectedVersion ${String(expected)}`) - const hint = expected !== 'any' && Number.isInteger(opts.expectedCommit) ? { commit: opts.expectedCommit!, version: expected } : undefined - return this.#append(stream, events, expected, hint) - } - - read(opts?: ReadOptions): AsyncIterable>> - read>(key: K, opts?: ReadOptions): AsyncIterable>>> - async *read(...args: unknown[]): AsyncGenerator> { - const self = args.length === 0 || (typeof args[0] === 'object' && args[0] !== null) - const stream = self ? this.#self() : this.#child(args[0]) - yield* this.#read(stream, ((self ? args[0] : args[1]) ?? {}) as ReadOptions) - } - - version(): Promise - version>(key: K): Promise - async version(...args: unknown[]): Promise { - const stream = args.length === 0 ? this.#self() : this.#child(args[0]) - return (await this.#loadHead(stream)).version - } - - keys(opts: StreamKeysOptions = {}): AsyncIterable> { - return this.#kv.keys(opts) as AsyncIterable> - } - - // ---------------------------------------------------------------- internals - - #self(): Stream { - if (this.#kv.isRoot) throw new TypeError('The root node is not a stream') - return this.#streamAt(this.#kv.selfKey) - } - - #child(key: unknown): Stream { - return this.#streamAt(this.#kv.prefix + escapeSegment(toSegment(key), this.#ctx.sep)) - } - - #streamAt(key: string): Stream { - const sep = this.#ctx.sep - const head = key + sep + RESERVED.events - return { key, base: head + sep, end: head + succ(sep) } - } - - async #loadHead(s: Stream): Promise { - for await (const [key, value] of this.#ctx.driver.scan({ gte: s.base, lt: s.end, reverse: true, limit: 1, values: true })) { - const c = value as CommitRecord - const head = { commit: Number(key.slice(s.base.length)), version: c.firstVersion + c.events.length - 1 } - this.#ctx.heads.set(s.key, head) - return head - } - return EMPTY_HEAD - } - - async #append(s: Stream, events: readonly unknown[], expected: number | 'any', hint?: Head): Promise { - const { driver, heads, maxRetries } = this.#ctx - if (events.length === 0) { - const head = await this.#loadHead(s) - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - return { version: head.version, commit: head.commit } - } - if (!driver.setIfAbsent) return this.#ctx.mutex.run(s.key, () => this.#appendSingleWriter(s, events, expected)) - - for (let attempt = 0; ; attempt++) { - let head = heads.get(s.key) ?? (attempt === 0 ? hint : undefined) - if (!head) head = await this.#loadHead(s) - else if (expected !== 'any' && head.version !== expected) head = await this.#loadHead(s) // cache may be stale - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - - const seq = head.commit + 1 - const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] } - const [written] = await driver.setIfAbsent([[s.base + pad(seq), record]]) - if (written) { - const next = { commit: seq, version: head.version + events.length } - heads.set(s.key, next) - return next - } - // Lost the race for this commit number. - heads.delete(s.key) - if (expected !== 'any' || attempt >= maxRetries) { - throw new ConcurrencyError(s.key, expected, (await this.#loadHead(s)).version) - } - } - } - - async #appendSingleWriter(s: Stream, events: readonly unknown[], expected: number | 'any'): Promise { - const head = this.#ctx.heads.get(s.key) ?? (await this.#loadHead(s)) - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - const seq = head.commit + 1 - const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] } - await this.#ctx.driver.setMany([[s.base + pad(seq), record]]) - const next = { commit: seq, version: head.version + events.length } - this.#ctx.heads.set(s.key, next) - return next - } - - /** - * Sequence number of the commit containing `version`, or 0 if the stream hasn't reached it. - * Commit `n` has `firstVersion >= n`, so the answer is at most `version`: probe that first - * (one round trip when commits hold single events), then k-ary search with `getMany` probes. - */ - async #locate(s: Stream, version: number): Promise { - const { driver } = this.#ctx - const [guess] = (await driver.getMany([s.base + pad(version)])) as (CommitRecord | undefined)[] - if (guess && guess.firstVersion <= version) return version - let hi: number - if (guess) hi = version - 1 - else { - const head = await this.#loadHead(s) - if (head.version < version) return 0 - hi = head.commit - } - let lo = 1 // commit 1 starts at version 1 - while (lo < hi) { - const probes: number[] = [] - for (let i = 1; i <= SEARCH_FANOUT; i++) { - const p = lo + Math.ceil(((hi - lo) * i) / SEARCH_FANOUT) - if (p !== probes.at(-1)) probes.push(p) - } - const found = (await driver.getMany(probes.map((p) => s.base + pad(p)))) as (CommitRecord | undefined)[] - for (let i = 0; i < probes.length; i++) { - const c = found[i] - if (c && c.firstVersion <= version) lo = probes[i]! - else { - hi = probes[i]! - 1 - break - } - } - } - return lo - } - - async *#read(s: Stream, opts: ReadOptions): AsyncGenerator> { - const from = Math.max(1, opts.from ?? 1) - const to = opts.to ?? Infinity - let remaining = opts.limit ?? Infinity - if (to < from || remaining <= 0) return - const { driver } = this.#ctx - - if (!opts.reverse) { - let gte = s.base - if (from > 1) { - const seq = await this.#locate(s, from) - if (seq === 0) return - gte = s.base + pad(seq) - } - for await (const [key, value] of driver.scan({ gte, lt: s.end, values: true })) { - const c = value as CommitRecord - const commit = Number(key.slice(s.base.length)) - for (let i = 0; i < c.events.length; i++) { - const version = c.firstVersion + i - if (version < from) continue - if (version > to) return - yield { version, commit, at: c.at, data: c.events[i] } - if (--remaining <= 0) return - } - } - return - } - - let lt = s.end - if (to !== Infinity) { - const seq = await this.#locate(s, to) - if (seq !== 0) lt = s.base + pad(seq + 1) - } - for await (const [key, value] of driver.scan({ gte: s.base, lt, reverse: true, values: true })) { - const c = value as CommitRecord - const commit = Number(key.slice(s.base.length)) - for (let i = c.events.length - 1; i >= 0; i--) { - const version = c.firstVersion + i - if (version > to) continue - if (version < from) return - yield { version, commit, at: c.at, data: c.events[i] } - if (--remaining <= 0) return - } - } - } -} - -/** Event storage over a `KVStorage` (or any `KVNode` as root). */ -export class KVEventStorage extends KVEventNode { - constructor(kv: KVNode, opts: KVEventStorageOptions = {}) { - super( - { - driver: kv.driver, - sep: kv.separator, - heads: new LRU(opts.headCacheSize ?? 10_000), - mutex: new KeyedMutex(), - maxRetries: opts.maxRetries ?? 5, - now: opts.now ?? Date.now, - }, - kv, - ) - } -} diff --git a/src/events/types.ts b/src/events/types.ts deleted file mode 100644 index db798d8..0000000 --- a/src/events/types.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' - -export interface AppendOptions { - /** Version the stream must be at (0 = empty). Default `'any'`. */ - expectedVersion?: number | 'any' - /** - * Commit sequence of the head at `expectedVersion`, if the caller knows it (e.g. from a - * projection). Lets a cold append skip the head lookup. Must come from the stream itself: a - * stale hint only causes a collision, but an invented one could leave a gap. Ignored unless - * `expectedVersion` is a number; backends may ignore it. - */ - expectedCommit?: number -} - -export interface AppendResult { - /** Stream version after the append (the last event's version). */ - version: number - /** Commit sequence number of this append (the current head commit when no events were given). */ - commit: number -} - -export interface ReadOptions { - /** First version to read (inclusive). Default 1. */ - from?: number - /** Last version to read (inclusive). Default: the head. */ - to?: number - reverse?: boolean - limit?: number -} - -export interface StoredEvent { - version: number - commit: number - /** Commit timestamp (ms since epoch). */ - at: number - data: E -} - -export interface StreamKeysOptions { - limit?: number - after?: string | number - reverse?: boolean -} - -/** - * One node of an event store: a stream (its own events) and a container of child streams. - * Same navigation and self/child arity rules as `KVNode`. - */ -export interface EventNode { - readonly path: readonly string[] - child>(key: K): EventNode> - as(): EventNode - - append(events: readonly ValueOf[], opts?: AppendOptions): Promise - append>(key: K, events: readonly ValueOf>[], opts?: AppendOptions): Promise - - read(opts?: ReadOptions): AsyncIterable>> - read>(key: K, opts?: ReadOptions): AsyncIterable>>> - - /** Current version (0 = empty). Always reads storage. */ - version(): Promise - version>(key: K): Promise - - /** Child stream keys. */ - keys(opts?: StreamKeysOptions): AsyncIterable> -} - -/** The root of an event store. Native backends (EventStoreDB, Kafka, …) implement this too. */ -export type EventStorage = EventNode diff --git a/src/http/client.ts b/src/http/client.ts deleted file mode 100644 index 86e4916..0000000 --- a/src/http/client.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { BaseKVDriver, JsonCodec, type BatchOp, type Codec, type ScanOptions } from '../kv/driver.js' -import type { HttpCapabilities } from './handler.js' - -export interface HttpKVDriverOptions { - /** Base URL of a `createKVHttpHandler` endpoint, e.g. `https://host/kv`. */ - url: string - /** Optional capabilities the server's driver supports. Default: all. `connect()` discovers them. */ - capabilities?: Partial - codec?: Codec - headers?: Record - fetch?: typeof fetch -} - -/** - * Driver client for `createKVHttpHandler`. Optional methods exist only when the server - * supports them, so stores' feature detection works across HTTP. - */ -export class HttpKVDriver extends BaseKVDriver { - readonly #url: string - readonly #codec: Codec - readonly #headers: Record - readonly #fetch: typeof fetch - readonly #nativeDeleteRange: boolean - - setIfAbsent?: (entries: [string, unknown][]) => Promise - batch?: (ops: BatchOp[]) => Promise - - /** Creates a client after asking the server which optional capabilities it has. */ - static async connect(opts: Omit): Promise { - const probe = new HttpKVDriver({ ...opts, capabilities: {} }) - const capabilities = (await probe.#request('GET', '/capabilities')) as HttpCapabilities - return new HttpKVDriver({ ...opts, capabilities }) - } - - constructor(opts: HttpKVDriverOptions) { - super() - this.#url = opts.url.replace(/\/$/, '') - this.#codec = opts.codec ?? JsonCodec - this.#headers = opts.headers ?? {} - this.#fetch = opts.fetch ?? globalThis.fetch.bind(globalThis) - const caps = { setIfAbsent: true, batch: true, deleteRange: true, ...opts.capabilities } - this.#nativeDeleteRange = caps.deleteRange - if (caps.setIfAbsent) this.setIfAbsent = async (entries) => ((await this.#request('POST', '/setIfAbsent', { entries })) as { written: boolean[] }).written - if (caps.batch) this.batch = async (ops) => void (await this.#request('POST', '/batch', { ops })) - } - - async getMany(keys: string[]): Promise { - if (keys.length === 0) return [] - const { values, missing } = (await this.#request('POST', '/getMany', { keys })) as { values: unknown[]; missing: number[] } - for (const i of missing) values[i] = undefined - return values - } - - async setMany(entries: [string, unknown][]): Promise { - if (entries.length) await this.#request('POST', '/setMany', { entries }) - } - - async deleteMany(keys: string[]): Promise { - if (keys.length) await this.#request('POST', '/deleteMany', { keys }) - } - - override async deleteRange(gte: string, lt: string): Promise { - if (!this.#nativeDeleteRange) return super.deleteRange(gte, lt) - await this.#request('POST', '/deleteRange', { gte, lt }) - } - - async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> { - if (opts.limit !== undefined && opts.limit <= 0) return - const q = new URLSearchParams() - if (opts.gte !== undefined) q.set('gte', opts.gte) - if (opts.lt !== undefined) q.set('lt', opts.lt) - if (opts.limit !== undefined && opts.limit !== Infinity) q.set('limit', String(opts.limit)) - if (opts.reverse) q.set('reverse', 'true') - if (opts.values) q.set('values', 'true') - const res = await this.#fetch(`${this.#url}/scan?${q}`, { headers: this.#headers }) - if (!res.ok || !res.body) throw await httpError(res) - const reader = res.body.getReader() - const decoder = new TextDecoder() - let buf = '' - try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - buf += decoder.decode(value, { stream: true }) - let nl: number - while ((nl = buf.indexOf('\n')) !== -1) { - const line = buf.slice(0, nl) - buf = buf.slice(nl + 1) - const [key, v] = this.#codec.decode(line) as [string, unknown?] - yield [key, v] - } - } - } finally { - await reader.cancel().catch(() => {}) - } - } - - async #request(method: 'GET' | 'POST', path: string, body?: unknown): Promise { - const res = await this.#fetch(this.#url + path, { - method, - headers: { ...this.#headers, ...(body === undefined ? {} : { 'content-type': 'application/json' }) }, - body: body === undefined ? undefined : this.#codec.encode(body), - }) - if (!res.ok) throw await httpError(res) - return res.status === 204 ? undefined : this.#codec.decode(await res.text()) - } -} - -async function httpError(res: Response): Promise { - let detail = '' - try { - detail = await res.text() - } catch {} - return new Error(`HTTP ${res.status}: ${detail}`) -} diff --git a/src/http/handler.ts b/src/http/handler.ts deleted file mode 100644 index c557ecd..0000000 --- a/src/http/handler.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { JsonCodec, type BatchOp, type Codec, type KVDriver, type ScanOptions } from '../kv/driver.js' - -export interface KVHttpHandlerOptions { - /** Serialises request/response bodies and NDJSON scan lines. Must not emit newlines. Default JSON. */ - codec?: Codec - /** Path prefix to strip, e.g. `/kv`. Default `''`. */ - basePath?: string -} - -export interface HttpCapabilities { - setIfAbsent: boolean - batch: boolean - deleteRange: boolean -} - -/** - * Framework-agnostic HTTP front for a driver (`Request → Response`). - * - * - `POST /getMany {keys}` → `{values, missing}` (`missing` = indices of absent keys) - * - `POST /setMany {entries}`, `POST /deleteMany {keys}`, `POST /batch {ops}` → 204 - * - `POST /setIfAbsent {entries}` → `{written}` - * - `POST /deleteRange {gte, lt}` → 204 - * - `GET /scan?gte<&limit&reverse&values` → NDJSON, one `[key]` or `[key, value]` per line - * - `GET /capabilities` → `{setIfAbsent, batch, deleteRange}` - */ -export function createKVHttpHandler(driver: KVDriver, opts: KVHttpHandlerOptions = {}): (req: Request) => Promise { - const codec = opts.codec ?? JsonCodec - const base = opts.basePath ?? '' - const reply = (body: unknown, status = 200) => new Response(codec.encode(body), { status, headers: { 'content-type': 'application/json' } }) - const noContent = () => new Response(null, { status: 204 }) - const fail = (status: number, message: string) => reply({ error: message }, status) - - return async (req) => { - const url = new URL(req.url) - if (!url.pathname.startsWith(base)) return fail(404, 'Not found') - const route = `${req.method} ${url.pathname.slice(base.length)}` - try { - const body = async () => codec.decode(await req.text()) as T - switch (route) { - case 'POST /getMany': { - const { keys } = await body<{ keys: string[] }>() - const found = await driver.getMany(keys) - const missing: number[] = [] - const values = found.map((v, i) => (v === undefined ? (missing.push(i), null) : v)) - return reply({ values, missing }) - } - case 'POST /setMany': - await driver.setMany((await body<{ entries: [string, unknown][] }>()).entries) - return noContent() - case 'POST /deleteMany': - await driver.deleteMany((await body<{ keys: string[] }>()).keys) - return noContent() - case 'POST /setIfAbsent': - if (!driver.setIfAbsent) return fail(501, 'setIfAbsent not supported') - return reply({ written: await driver.setIfAbsent((await body<{ entries: [string, unknown][] }>()).entries) }) - case 'POST /batch': - if (!driver.batch) return fail(501, 'batch not supported') - await driver.batch((await body<{ ops: BatchOp[] }>()).ops) - return noContent() - case 'POST /deleteRange': { - if (!driver.deleteRange) return fail(501, 'deleteRange not supported') - const { gte, lt } = await body<{ gte: string; lt: string }>() - await driver.deleteRange(gte, lt) - return noContent() - } - case 'GET /capabilities': - return reply({ setIfAbsent: !!driver.setIfAbsent, batch: !!driver.batch, deleteRange: !!driver.deleteRange } satisfies HttpCapabilities) - case 'GET /scan': - return scanResponse(driver, parseScan(url.searchParams), codec) - default: - return fail(404, `No route ${route}`) - } - } catch (err) { - return fail(500, err instanceof Error ? err.message : String(err)) - } - } -} - -function parseScan(q: URLSearchParams): ScanOptions { - const opts: ScanOptions = {} - if (q.has('gte')) opts.gte = q.get('gte')! - if (q.has('lt')) opts.lt = q.get('lt')! - if (q.has('limit')) opts.limit = Number(q.get('limit')) - if (q.get('reverse') === 'true') opts.reverse = true - if (q.get('values') === 'true') opts.values = true - return opts -} - -function scanResponse(driver: KVDriver, opts: ScanOptions, codec: Codec): Response { - const it = driver.scan(opts)[Symbol.asyncIterator]() - const encoder = new TextEncoder() - // Chunks grow so short (skip-scan) reads don't make the server run far ahead. - let chunk = 16 - const stream = new ReadableStream({ - async pull(controller) { - let buf = '' - for (let i = 0; i < chunk; i++) { - const r = await it.next() - if (r.done) { - if (buf) controller.enqueue(encoder.encode(buf)) - controller.close() - return - } - const [key, value] = r.value - buf += codec.encode(opts.values ? [key, value] : [key]) + '\n' - } - chunk = Math.min(chunk * 2, 256) - controller.enqueue(encoder.encode(buf)) - }, - async cancel() { - await it.return?.() - }, - }) - return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } }) -} diff --git a/src/http/index.ts b/src/http/index.ts deleted file mode 100644 index 13ba7ec..0000000 --- a/src/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './handler.js' -export * from './client.js' diff --git a/src/index.ts b/src/index.ts index 9400bff..78841b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,18 @@ -export * from './errors.js' -export * from './core/path.js' -export * from './core/encoding.js' -export * from './kv/index.js' -export * from './types/schema.js' -export * from './events/index.js' -export * from './documents/index.js' -export * from './http/index.js' +export * from './kv-storage/base-kv-storage.js' +export * from './kv-storage/kv-storage-in-memory.js' +export * from './kv-storage/kv-storage-http.js' +export * from './kv-storage/kv-storage-http-server.js' +export * from './event-storage/base-event-storage.js' +export * from './event-storage/event-storage-kv.js' +export * from './document-storage/base-document-storage.js' +export * from './document-storage/document-storage-in-memory.js' +export * from './document-storage/document-storage-kv.js' +export * from './document-storage/event-sourced-document-storage.js' +export * from './utils/document-path.js' +export * from './utils/document-query.js' +export * from './utils/document-update.js' +export * from './utils/errors.js' +export * from './utils/json.js' +export * from './utils/keys.js' +export * from './utils/schema.js' +export * from './utils/ulid.js' diff --git a/src/kv-storage/base-kv-storage.ts b/src/kv-storage/base-kv-storage.ts new file mode 100644 index 0000000..e13a874 --- /dev/null +++ b/src/kv-storage/base-kv-storage.ts @@ -0,0 +1,184 @@ +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueAt, ValueOf } from '../utils/schema.js' + +export interface ScanOptions { + /** Inclusive lower bound. */ + gte?: string + /** Exclusive upper bound. */ + lt?: string + limit?: number + reverse?: boolean + /** Include values; otherwise they are `undefined`. */ + values?: boolean +} + +export interface KeysOptions { + limit?: number + /** Start after this child key (exclusive), in the direction of travel. */ + after?: string | number + reverse?: boolean +} + +/** + * Ordered key-value storage. Implementations provide five raw primitives over absolute keys; + * everything else (navigation, self/child reads and writes, child listing) is defined here once. + * + * Every instance is a node: `child(key)` returns a view of the same storage rooted at that key. + * Methods taking an optional `key` act on that child, or on this node's own value when omitted. + * A key that is passed but `undefined`/`null` throws rather than silently meaning "self". + */ +export abstract class BaseKVStorage { + /** Unescaped path of this node (`[]` for the root). */ + path: readonly string[] = [] + /** This node's own key (`''` for the root). */ + key = '' + /** Prefix of every descendant key (`''` for the root). */ + prefix = '' + + /** Values for absolute keys, in order; `undefined` for missing. */ + abstract readMany(keys: string[]): Promise + abstract writeMany(entries: [key: string, value: unknown][]): Promise + abstract removeMany(keys: string[]): Promise + /** Writes each entry only if its key is absent; `true` = written. Must be atomic per key. */ + abstract writeIfAbsent(entries: [key: string, value: unknown][]): Promise + /** Entries in key order (descending when `reverse`). */ + abstract scan(options?: ScanOptions): AsyncIterable<[key: string, value: unknown]> + + // ------------------------------------------------------------------ navigation + + /** A view of this storage rooted at `key`. Synchronous; no I/O. */ + child>(key: K): BaseKVStorage> { + const node = Object.create(this) as BaseKVStorage> + node.path = [...this.path, Keys.segment(key)] + node.key = this.prefix + Keys.escape(Keys.segment(key)) + node.prefix = node.key + Keys.separator + return node + } + + /** Re-types this node. No runtime effect. */ + as(): BaseKVStorage { + return this as unknown as BaseKVStorage + } + + // ------------------------------------------------------------------ reads and writes + + async get | undefined = undefined>(key?: K): Promise | undefined> { + const [value] = await this.readMany([this.keyOf(arguments.length > 0, key)]) + return value as ValueAt | undefined + } + + async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { + return (await this.readMany(keys.map((k) => this.keyOf(true, k)))) as (ValueOf> | undefined)[] + } + + /** `set(value)` writes this node's own value; `set(key, value)` a child's. */ + async set>(...args: [value: ValueOf] | [key: K, value: ValueOf>]): Promise { + const value = args[args.length - 1] + if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') + await this.writeMany([[this.keyOf(args.length > 1, args[0]), value]]) + } + + async setMany(entries: readonly EntryOf[]): Promise { + if (entries.some(([, v]) => v === undefined)) throw new TypeError('Cannot set undefined; use delete()') + await this.writeMany(entries.map(([k, v]) => [this.keyOf(true, k), v])) + } + + /** Deletes this node's own value (or a child's), not descendants. */ + async delete(key?: KeyOf): Promise { + await this.removeMany([this.keyOf(arguments.length > 0, key)]) + } + + async deleteMany(keys: readonly KeyOf[]): Promise { + await this.removeMany(keys.map((k) => this.keyOf(true, k))) + } + + /** Deletes this node's own value (or a child's) and all descendants. */ + async deleteTree(key?: KeyOf): Promise { + const own = this.keyOf(arguments.length > 0, key, true) + if (own !== '') await this.removeMany([own]) + const prefix = own === '' ? '' : own + Keys.separator + for (;;) { + const batch: string[] = [] + for await (const [k] of this.scan({ gte: prefix, lt: own === '' ? undefined : own + Keys.afterSeparator, limit: 1000 })) batch.push(k) + if (batch.length === 0) return + await this.removeMany(batch) + } + } + + // ------------------------------------------------------------------ listing + + /** + * Direct child keys in key order, including children that only have descendants. + * Skip-scan: one seek per child that has descendants, not one read per descendant. + */ + async *keys(options: KeysOptions = {}): AsyncGenerator> { + for await (const [key] of this.children(options, false)) yield key as KeyString + } + + /** Direct children that have their own value, as `[key, value]`. */ + async *entries(options: KeysOptions = {}): AsyncGenerator> { + let remaining = options.limit ?? Infinity + for await (const [key, value, hasValue] of this.children({ ...options, limit: undefined }, true)) { + if (remaining <= 0) return + if (!hasValue) continue + remaining-- + yield [key, value] as StringEntryOf + } + } + + private async *children(options: KeysOptions, values: boolean): AsyncGenerator<[key: string, value: unknown, hasValue: boolean]> { + const { reverse = false } = options + let remaining = options.limit ?? Infinity + let gte = this.prefix + let lt = this.key === '' ? undefined : this.key + Keys.afterSeparator + if (options.after !== undefined) { + const after = this.prefix + Keys.escape(Keys.segment(options.after)) + if (reverse) lt = after + else gte = after + Keys.afterSeparator + } + let last: string | undefined + let restart = true + let iterator: AsyncIterator<[string, unknown]> | undefined + try { + while (remaining > 0) { + if (restart) { + await iterator?.return?.() + iterator = this.scan({ gte, lt, reverse, values })[Symbol.asyncIterator]() + restart = false + } + const next = await iterator!.next() + if (next.done) return + const [key, value] = next.value + const end = key.indexOf(Keys.separator, this.prefix.length) + const segment = key.slice(this.prefix.length, end === -1 ? undefined : end) + const isOwn = end === -1 + if (segment !== last && !Keys.isReserved(segment)) { + // Forward, a child's own key comes before its descendants; in reverse it comes after + // them and is skipped by the jump below, so read it directly when values are wanted. + let own = isOwn ? value : undefined + if (!isOwn && reverse && values) [own] = await this.readMany([this.prefix + segment]) + yield [Keys.unescape(segment), own, isOwn || own !== undefined] + remaining-- + } + last = segment + // Jump over the rest of this child's subtree instead of reading it. + if (!isOwn || Keys.isReserved(segment)) { + if (reverse) lt = this.prefix + segment + else gte = this.prefix + segment + Keys.afterSeparator + restart = true + } + } + } finally { + await iterator?.return?.() + } + } + + // ------------------------------------------------------------------ internals + + /** Absolute key for a child (`hasKey`) or this node; validates keys. */ + protected keyOf(hasKey: boolean, key: unknown, allowRoot = false): string { + if (hasKey) return this.prefix + Keys.escape(Keys.segment(key)) + if (this.key === '' && !allowRoot) throw new TypeError('The root node has no own value') + return this.key + } +} diff --git a/src/kv-storage/kv-storage-http-server.ts b/src/kv-storage/kv-storage-http-server.ts new file mode 100644 index 0000000..54be60d --- /dev/null +++ b/src/kv-storage/kv-storage-http-server.ts @@ -0,0 +1,70 @@ +import type { BaseKVStorage, ScanOptions } from './base-kv-storage.js' + +/** + * Serves any KV storage over HTTP for `KVStorageHttp`. Framework-agnostic: `handle(Request)` + * returns a `Response`. All routes are `POST /` with a JSON body; `scan` streams NDJSON. + * The raw primitives use absolute keys, so this exposes the whole store behind `storage`. + */ +export class KVStorageHttpServer { + constructor( + private readonly storage: BaseKVStorage, + private readonly basePath = '', + ) {} + + async handle(request: Request): Promise { + const path = new URL(request.url).pathname + if (request.method !== 'POST' || !path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') + try { + const body = (await request.json()) as { keys: string[]; entries: [string, unknown][] } & ScanOptions + switch (path.slice(this.basePath.length)) { + case '/readMany': { + const values = await this.storage.readMany(body.keys) + const missing = values.flatMap((v, i) => (v === undefined ? [i] : [])) + return Response.json({ values: values.map((v) => v ?? null), missing }) + } + case '/writeMany': + await this.storage.writeMany(body.entries) + return new Response(null, { status: 204 }) + case '/removeMany': + await this.storage.removeMany(body.keys) + return new Response(null, { status: 204 }) + case '/writeIfAbsent': + return Response.json(await this.storage.writeIfAbsent(body.entries)) + case '/scan': + return this.scan(body) + default: + return KVStorageHttpServer.error(404, 'Not found') + } + } catch (error) { + return KVStorageHttpServer.error(500, error instanceof Error ? error.message : String(error)) + } + } + + private scan(options: ScanOptions): Response { + const iterator = this.storage.scan(options)[Symbol.asyncIterator]() + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async pull(controller) { + // Send in small batches so short reads (skip-scan seeks) don't make the server run far ahead. + let chunk = '' + for (let i = 0; i < 32; i++) { + const next = await iterator.next() + if (next.done) { + if (chunk) controller.enqueue(encoder.encode(chunk)) + return controller.close() + } + chunk += JSON.stringify(options.values ? next.value : [next.value[0]]) + '\n' + } + controller.enqueue(encoder.encode(chunk)) + }, + async cancel() { + await iterator.return?.() + }, + }) + return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } }) + } + + private static error(status: number, message: string): Response { + return Response.json({ error: message }, { status }) + } +} diff --git a/src/kv-storage/kv-storage-http.ts b/src/kv-storage/kv-storage-http.ts new file mode 100644 index 0000000..ced0b4e --- /dev/null +++ b/src/kv-storage/kv-storage-http.ts @@ -0,0 +1,79 @@ +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' + +export interface KVStorageHttpOptions { + /** Base URL of a `KVStorageHttpServer`, e.g. `https://host/kv`. */ + url: string + headers?: Record + fetch?: typeof fetch +} + +/** KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage. */ +export class KVStorageHttp extends BaseKVStorage { + private readonly url: string + private readonly headers: Record + private readonly fetcher: typeof fetch + + constructor(options: KVStorageHttpOptions) { + super() + this.url = options.url.replace(/\/$/, '') + this.headers = options.headers ?? {} + this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis) + } + + async readMany(keys: string[]): Promise { + if (keys.length === 0) return [] + const { values, missing } = (await this.post('readMany', { keys })) as { values: unknown[]; missing: number[] } + for (const i of missing) values[i] = undefined + return values + } + + async writeMany(entries: [string, unknown][]): Promise { + if (entries.length) await this.post('writeMany', { entries }) + } + + async removeMany(keys: string[]): Promise { + if (keys.length) await this.post('removeMany', { keys }) + } + + async writeIfAbsent(entries: [string, unknown][]): Promise { + return entries.length ? ((await this.post('writeIfAbsent', { entries })) as boolean[]) : [] + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + if (options.limit !== undefined && options.limit <= 0) return + const response = await this.request('scan', options) + const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader() + let buffer = '' + try { + for (;;) { + const { done, value } = await reader.read() + if (done) return + buffer += value + const lines = buffer.split('\n') + buffer = lines.pop()! + for (const line of lines) { + const [key, value] = JSON.parse(line) as [string, unknown?] + yield [key, value] + } + } + } finally { + await reader.cancel().catch(() => {}) + } + } + + private async post(path: string, body: unknown): Promise { + const response = await this.request(path, body) + return response.status === 204 ? undefined : response.json() + } + + private async request(path: string, body: unknown): Promise { + const response = await this.fetcher(`${this.url}/${path}`, { + method: 'POST', + headers: { ...this.headers, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`) + return response + } +} diff --git a/src/kv-storage/kv-storage-in-memory.ts b/src/kv-storage/kv-storage-in-memory.ts new file mode 100644 index 0000000..6883e1a --- /dev/null +++ b/src/kv-storage/kv-storage-in-memory.ts @@ -0,0 +1,34 @@ +import { Json } from '../utils/json.js' +import { SortedMap } from '../utils/sorted-map.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' + +/** In-memory KV storage. Values are deep-copied on write and read. */ +export class KVStorageInMemory extends BaseKVStorage { + private readonly data = new SortedMap() + + async readMany(keys: string[]): Promise { + return keys.map((k) => Json.clone(this.data.get(k))) + } + + async writeMany(entries: [string, unknown][]): Promise { + for (const [k, v] of entries) this.data.set(k, Json.clone(v)) + } + + async removeMany(keys: string[]): Promise { + for (const k of keys) this.data.delete(k) + } + + async writeIfAbsent(entries: [string, unknown][]): Promise { + return entries.map(([k, v]) => !this.data.has(k) && (this.data.set(k, Json.clone(v)), true)) + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + let remaining = options.limit ?? Infinity + if (remaining <= 0) return + for (const key of this.data.keys(options.gte, options.lt, options.reverse)) { + yield [key, options.values ? Json.clone(this.data.get(key)) : undefined] + if (--remaining <= 0) return + } + } +} diff --git a/src/kv/children.ts b/src/kv/children.ts deleted file mode 100644 index b295233..0000000 --- a/src/kv/children.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { escapeSegment, succ } from '../core/path.js' -import type { KVDriver } from './driver.js' - -/** - * Direct-children skip-scan, shared by every layer. - * - * ## Child order - * - * Children are ordered by **escaped segment followed by the separator** ("subtree order": the - * order in which their descendants appear in key space). For segments without characters that - * sort below the separator this is plain lexicographic order. When one segment extends another - * with such a character (`a` vs `a-1` with `/`), the longer one comes first (`a-1/` < `a/`). - * - * This is the one order that can be produced without extra reads: a child's own key - * (`p/a`) sorts *before* siblings that extend it (`p/a-1`), while its descendants (`p/a/…`) sort - * after them. The forward scan therefore defers yielding a child seen only through its own key - * until the scan passes `p/a/`, using a stack bounded by segment length. - * - * ## Cost - * - * - Forward: one seek per child that has descendants (to jump over its subtree), one at the start, - * and one per reserved (`~…`) region. Siblings without descendants share an iterator. - * `after` adds one `getMany` when the `after` segment contains a below-separator character. - * - Reverse: as forward, plus one `getMany` per newly found child whose segment contains a - * below-separator character (to find shorter siblings that only have an own key), or that was - * found through a descendant when `own` is requested. - */ -export interface ChildHit { - /** Escaped segment. */ - esc: string - /** Whether the child has an own value. Only accurate when `own` was requested. */ - hasOwn: boolean - /** The own value, when `values` was requested and `hasOwn`. */ - value: unknown -} - -export interface ChildScanOptions { - /** Unescaped segment to start after (exclusive), in the direction of travel. */ - after?: string - reverse?: boolean - limit?: number - /** Fetch own values. Implies `own`. */ - values?: boolean - /** Require accurate `hasOwn`. */ - own?: boolean -} - -const RESERVED_START = '~' -const RESERVED_END = succ('~') - -export async function* scanChildren( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions = {}, -): AsyncGenerator { - const limit = opts.limit ?? Infinity - if (limit <= 0) return - if (opts.reverse) yield* reverseScan(driver, prefix, end, sep, opts, limit) - else yield* forwardScan(driver, prefix, end, sep, opts, limit) -} - -/** Prefixes `E[0..j)` (j ≥ 1) where `E[j]` sorts below the separator. */ -function lowPrefixes(esc: string, sepCode: number): string[] { - let out: string[] | undefined - for (let j = 1; j < esc.length; j++) { - if (esc.charCodeAt(j) < sepCode) (out ??= []).push(esc.slice(0, j)) - } - return out ?? NONE -} - -const NONE: string[] = [] - -async function* forwardScan( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions, - limit: number, -): AsyncGenerator { - const sepCode = sep.charCodeAt(0) - const succSep = succ(sep) - const values = opts.values === true - const pending: ChildHit[] = [] - let emitted = 0 - let gte = prefix - - if (opts.after !== undefined) { - const escA = escapeSegment(opts.after, sep) - gte = prefix + escA + succSep - // Shorter siblings that extend into `after` with a low char come after it, but their own - // keys sort before `gte`; probe for them. - const cands = lowPrefixes(escA, sepCode) - if (cands.length) { - const found = await driver.getMany(cands.map((g) => prefix + g)) - for (let i = 0; i < cands.length; i++) { - if (found[i] !== undefined) pending.push({ esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined }) - } - } - } - - const prefixLen = prefix.length - let it = driver.scan({ gte, lt: end, values })[Symbol.asyncIterator]() - const reseek = async (from: string) => { - await it.return?.() - it = driver.scan({ gte: from, lt: end, values })[Symbol.asyncIterator]() - } - - try { - outer: for (;;) { - const r = await it.next() - if (r.done) { - while (pending.length) { - yield pending.pop()! - if (++emitted >= limit) return - } - return - } - const key = r.value[0] - const cut = key.indexOf(sep, prefixLen) - const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut) - - while (pending.length) { - const top = pending[pending.length - 1]! - if (top.esc === esc) { - // First descendant of a deferred child: emit it and jump over its subtree. - pending.pop() - yield top - if (++emitted >= limit) return - await reseek(prefix + esc + succSep) - continue outer - } - if (top.esc + sep < esc + sep) { - pending.pop() - yield top - if (++emitted >= limit) return - } else break - } - - if (esc.startsWith(RESERVED_START)) { - await reseek(prefix + RESERVED_END) - continue - } - if (cut === -1) { - // Own key: defer until we know no sibling extending it with a low char comes next. - pending.push({ esc, hasOwn: true, value: r.value[1] }) - continue - } - // Descendant of a child with no own value (an own key would have sorted first). - yield { esc, hasOwn: false, value: undefined } - if (++emitted >= limit) return - await reseek(prefix + esc + succSep) - } - } finally { - await it.return?.() - } -} - -async function* reverseScan( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions, - limit: number, -): AsyncGenerator { - const sepCode = sep.charCodeAt(0) - const values = opts.values === true - const needOwn = values || opts.own === true - // Segments already emitted whose own key may still appear (it sorts after everything else of theirs). - const done: string[] = [] - let emitted = 0 - let lt = end - - if (opts.after !== undefined) { - const escA = escapeSegment(opts.after, sep) - lt = prefix + escA + sep - // `after` itself and shorter siblings extending into it with a low char sort after it in - // child order but their own keys fall below `lt`: mark them done. - done.push(...lowPrefixes(escA, sepCode), escA) - } - - const prefixLen = prefix.length - let it = driver.scan({ gte: prefix, lt, reverse: true, values })[Symbol.asyncIterator]() - const reseek = async (below: string) => { - await it.return?.() - it = driver.scan({ gte: prefix, lt: below, reverse: true, values })[Symbol.asyncIterator]() - } - - try { - for (;;) { - const r = await it.next() - if (r.done) return - const key = r.value[0] - const cut = key.indexOf(sep, prefixLen) - const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut) - - if (esc.startsWith(RESERVED_START)) { - await reseek(prefix + RESERVED_START) - continue - } - while (done.length && !esc.startsWith(done[done.length - 1]!)) done.pop() - if (done.length && done[done.length - 1] === esc) { - done.pop() - if (cut !== -1) await reseek(prefix + esc + sep) - continue - } - - // New child. Shorter siblings that it extends with a low char come before it in reverse - // order; any with descendants were already emitted, so only own-key-only ones are unknown. - const cands = lowPrefixes(esc, sepCode).filter((g) => !done.includes(g)) - const probeOwn = cut !== -1 && needOwn - let found: unknown[] = NONE - if (cands.length || probeOwn) { - const probe = cands.map((g) => prefix + g) - if (probeOwn) probe.push(prefix + esc) - found = await driver.getMany(probe) - } - for (let i = 0; i < cands.length; i++) { - if (found[i] === undefined) continue - yield { esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined } - done.push(cands[i]!) - if (++emitted >= limit) return - } - const own = cut === -1 ? r.value[1] : probeOwn ? found[cands.length] : undefined - yield { esc, hasOwn: cut === -1 || own !== undefined, value: values ? own : undefined } - done.push(esc) - if (++emitted >= limit) return - if (cut !== -1) await reseek(prefix + esc + sep) - } - } finally { - await it.return?.() - } -} diff --git a/src/kv/driver.ts b/src/kv/driver.ts deleted file mode 100644 index f7f3b23..0000000 --- a/src/kv/driver.ts +++ /dev/null @@ -1,77 +0,0 @@ -export interface ScanOptions { - gte?: string - lt?: string - limit?: number - reverse?: boolean - /** When false/omitted, yielded values are `undefined`. */ - values?: boolean -} - -export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string } - -/** - * A flat, string-keyed backend. Values are JSON-serialisable; byte-oriented backends serialise - * internally with a {@link Codec}. All path, index and typing logic lives above this interface. - */ -export interface KVDriver { - /** Same order as `keys`; `undefined` for missing keys. */ - getMany(keys: string[]): Promise<(unknown | undefined)[]> - setMany(entries: [key: string, value: unknown][]): Promise - deleteMany(keys: string[]): Promise - /** MUST yield in key order (descending when `reverse`). `gte` is inclusive, `lt` exclusive. */ - scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]> - - // Optional capabilities. Stores feature-detect them and fall back where possible. - /** Writes each entry only if its key is absent. `true` = written. */ - setIfAbsent?(entries: [string, unknown][]): Promise - /** Applies all ops atomically. */ - batch?(ops: BatchOp[]): Promise - deleteRange?(gte: string, lt: string): Promise -} - -/** Serialises values for byte/string-oriented backends. */ -export interface Codec { - encode(value: unknown): string - decode(encoded: string): unknown -} - -export const JsonCodec: Codec = { - encode: (value) => JSON.stringify(value), - decode: (encoded) => JSON.parse(encoded), -} - -/** - * Base class for drivers: implement the four required methods and get `deleteRange` - * (scan + deleteMany in chunks) and single-key helpers for free. - */ -export abstract class BaseKVDriver implements KVDriver { - abstract getMany(keys: string[]): Promise<(unknown | undefined)[]> - abstract setMany(entries: [string, unknown][]): Promise - abstract deleteMany(keys: string[]): Promise - abstract scan(opts: ScanOptions): AsyncIterable<[string, unknown]> - - /** Chunk size used by the default `deleteRange`. */ - protected deleteRangeChunk = 1000 - - async deleteRange(gte: string, lt: string): Promise { - for (;;) { - const keys: string[] = [] - for await (const [key] of this.scan({ gte, lt, limit: this.deleteRangeChunk })) keys.push(key) - if (keys.length === 0) return - await this.deleteMany(keys) - if (keys.length < this.deleteRangeChunk) return - } - } - - async get(key: string): Promise { - return (await this.getMany([key]))[0] - } - - set(key: string, value: unknown): Promise { - return this.setMany([[key, value]]) - } - - delete(key: string): Promise { - return this.deleteMany([key]) - } -} diff --git a/src/kv/index.ts b/src/kv/index.ts deleted file mode 100644 index cfab7c0..0000000 --- a/src/kv/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './driver.js' -export * from './memory.js' -export * from './sorted.js' -export * from './node.js' -export * from './indexes.js' -export * from './children.js' diff --git a/src/kv/indexes.ts b/src/kv/indexes.ts deleted file mode 100644 index 2138b10..0000000 --- a/src/kv/indexes.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { indexValueCodec, isIndexValue, type IndexValue, type IndexValueCodec } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ } from '../core/path.js' -import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js' - -export type { IndexValue } - -/** Maps a child's value to index value(s). An array means multi-entry; `undefined` means not indexed. */ -export type IndexFn = (value: V) => IndexValue | readonly IndexValue[] | undefined - -/** Index name → function, for the direct children of nodes matching one pattern. */ -export type IndexDefs = Record - -/** Path pattern (`*` = one segment, `''` = root) → index definitions. */ -export type IndexDeclarations = Record - -/** A value or a range over index values. Bounds combine (`{ gte, lt }`). */ -export type IndexQuery = IndexValue | IndexRange - -export interface IndexRange { - gt?: IndexValue - gte?: IndexValue - lt?: IndexValue - lte?: IndexValue -} - -export function isIndexRange(q: unknown): q is IndexRange { - return typeof q === 'object' && q !== null -} - -interface CompiledIndex { - name: string - /** `escapeSegment(name) + sep` */ - head: string - fn: IndexFn -} - -/** Indexes over the direct children of one concrete node. Entry keys are rooted at `base`. */ -export class BoundIndexes { - readonly #byName: Map - readonly #list: CompiledIndex[] - readonly #codec: IndexValueCodec - - constructor( - defs: CompiledIndex[], - /** `~i` */ - readonly base: string, - readonly sep: string, - ) { - this.#list = defs - this.#byName = new Map(defs.map((d) => [d.name, d])) - this.#codec = indexValueCodec(sep) - } - - has(name: string): boolean { - return this.#byName.has(name) - } - - /** All index entry keys for a child (escaped segment) holding `value`. */ - entriesFor(escChild: string, value: unknown, out: Set = new Set()): Set { - if (value === undefined) return out - for (const idx of this.#list) this.#entriesOf(idx, escChild, value, out) - return out - } - - #entriesOf(idx: CompiledIndex, escChild: string, value: unknown, out: Set): void { - const r = idx.fn(value) - if (r === undefined) return - const stem = this.base + idx.head - const tail = this.sep + escChild - if (Array.isArray(r)) { - for (const v of r as IndexValue[]) out.add(stem + this.#encode(idx.name, v) + tail) - } else out.add(stem + this.#encode(idx.name, r as IndexValue) + tail) - } - - #encode(name: string, v: unknown): string { - if (!isIndexValue(v)) throw new TypeError(`Index "${name}" returned a non-indexable value: ${String(v)}`) - return this.#codec.encode(v) - } - - /** Whether `entryKey` is among the entries `value` currently produces for index `name`. */ - matches(name: string, escChild: string, value: unknown, entryKey: string): boolean { - if (value === undefined) return false - const out = new Set() - this.#entriesOf(this.#get(name), escChild, value, out) - return out.has(entryKey) - } - - /** Scan bounds over the entries of index `name` for a query. */ - range(name: string, q: IndexQuery): { gte: string; lt: string } { - const stem = this.base + this.#get(name).head - const succSep = succ(this.sep) - const enc = (v: IndexValue) => stem + this.#encode(name, v) - if (!isIndexRange(q)) { - const k = enc(q) - return { gte: k + this.sep, lt: k + succSep } - } - let gte = stem - let lt = stem.slice(0, -1) + succSep - if (q.gte !== undefined) gte = maxStr(gte, enc(q.gte)) - if (q.gt !== undefined) gte = maxStr(gte, enc(q.gt) + succSep) - if (q.lt !== undefined) lt = minStr(lt, enc(q.lt)) - if (q.lte !== undefined) lt = minStr(lt, enc(q.lte) + succSep) - return { gte, lt } - } - - /** Splits an entry key under index `name` into the escaped child segment. */ - childOf(name: string, entryKey: string): string { - const start = this.base.length + this.#get(name).head.length - return entryKey.slice(entryKey.indexOf(this.sep, start) + 1) - } - - #get(name: string): CompiledIndex { - const idx = this.#byName.get(name) - if (!idx) throw new TypeError(`Unknown index "${name}"`) - return idx - } -} - -const maxStr = (a: string, b: string) => (a > b ? a : b) -const minStr = (a: string, b: string) => (a < b ? a : b) - -/** Compiles index declarations into a pattern trie and binds them to nodes. */ -export class IndexRegistry { - readonly trie: PatternTrie - readonly empty: boolean - - constructor( - decls: IndexDeclarations, - readonly sep: string, - ) { - this.trie = new PatternTrie(sep) - const patterns = Object.keys(decls) - for (let i = 0; i < patterns.length; i++) { - for (let j = i + 1; j < patterns.length; j++) { - const a = patterns[i]! - const b = patterns[j]! - if (!patternsOverlap(a, b, sep)) continue - const clash = Object.keys(decls[a]!).filter((n) => n in decls[b]!) - if (clash.length) throw new TypeError(`Index name(s) ${clash.join(', ')} declared on overlapping patterns "${a}" and "${b}"`) - } - } - for (const p of patterns) this.trie.add(p, decls[p]!) - this.empty = patterns.length === 0 - } - - /** Indexes over the children of the node with trie state `state` and key prefix `prefix`. */ - bind(state: TrieState, prefix: string): BoundIndexes | undefined { - if (this.empty) return undefined - const found = PatternTrie.values(state) - if (found.length === 0) return undefined - const defs: CompiledIndex[] = [] - for (const { value } of found) { - for (const [name, fn] of Object.entries(value)) defs.push({ name, head: escapeSegment(name, this.sep) + this.sep, fn }) - } - return new BoundIndexes(defs, prefix + RESERVED.index + this.sep, this.sep) - } -} diff --git a/src/kv/memory.ts b/src/kv/memory.ts deleted file mode 100644 index dabbc92..0000000 --- a/src/kv/memory.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { BaseKVDriver, type BatchOp, type ScanOptions } from './driver.js' -import { ChunkedSortedMap, type SortedMap } from './sorted.js' - -export interface InMemoryKVDriverOptions { - /** - * Deep-copy values on write and read so callers can't mutate stored state through a shared - * reference (matching serialising drivers). Values must be JSON-compatible. Default `true`. - * Disable for speed when values are treated as immutable. - */ - clone?: boolean -} - -/** - * In-memory driver over a sorted map. Implements every optional capability: - * `setIfAbsent`, atomic `batch` and `deleteRange`. - */ -export class InMemoryKVDriver extends BaseKVDriver { - readonly #data: SortedMap = new ChunkedSortedMap() - readonly #clone: (v: unknown) => unknown - - constructor(opts: InMemoryKVDriverOptions = {}) { - super() - this.#clone = opts.clone === false ? identity : cloneValue - } - - get size(): number { - return this.#data.size - } - - async getMany(keys: string[]): Promise { - const out = new Array(keys.length) - for (let i = 0; i < keys.length; i++) out[i] = this.#clone(this.#data.get(keys[i]!)) - return out - } - - async setMany(entries: [string, unknown][]): Promise { - const clone = this.#clone - this.#data.setMany(entries.map(([k, v]) => [k, clone(v)])) - } - - async deleteMany(keys: string[]): Promise { - for (const key of keys) this.#data.delete(key) - } - - async setIfAbsent(entries: [string, unknown][]): Promise { - return entries.map(([key, value]) => { - if (this.#data.has(key)) return false - this.#data.set(key, this.#clone(value)) - return true - }) - } - - async batch(ops: BatchOp[]): Promise { - // Applied synchronously within one microtask, so no other operation can observe a partial batch. - for (const op of ops) { - if (op.type === 'set') this.#data.set(op.key, this.#clone(op.value)) - else this.#data.delete(op.key) - } - } - - override async deleteRange(gte: string, lt: string): Promise { - const doomed = [...this.#data.keys({ gte, lt })] - for (const key of doomed) this.#data.delete(key) - } - - async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> { - const { gte, lt, reverse = false, values = false } = opts - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - // The key iterator re-seeks by itself if the map changes while we're suspended. - for (const key of this.#data.keys({ gte, lt, reverse })) { - yield [key, values ? this.#clone(this.#data.get(key)) : undefined] - if (--remaining <= 0) return - } - } -} - -function identity(v: unknown): unknown { - return v -} - -/** Deep copy of a JSON-compatible value (much faster than `structuredClone`). */ -function cloneValue(v: unknown): unknown { - if (v === null || typeof v !== 'object') return v - if (Array.isArray(v)) { - const out = new Array(v.length) - for (let i = 0; i < v.length; i++) out[i] = cloneValue(v[i]) - return out - } - const out: Record = {} - for (const k in v) out[k] = cloneValue((v as Record)[k]) - return out -} diff --git a/src/kv/node.ts b/src/kv/node.ts deleted file mode 100644 index dea3b6d..0000000 --- a/src/kv/node.ts +++ /dev/null @@ -1,402 +0,0 @@ -import { DEFAULT_SEPARATOR, escapeSegment, succ, toSegment, unescapeSegment, validateSeparator } from '../core/path.js' -import type { TrieState } from '../core/trie.js' -import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueOf } from '../types/schema.js' -import { scanChildren } from './children.js' -import type { BatchOp, KVDriver } from './driver.js' -import { BoundIndexes, IndexRegistry, isIndexRange, type IndexDeclarations, type IndexDefs, type IndexQuery } from './indexes.js' - -export interface KVStorageOptions { - /** Single printable ASCII character other than `%` and `~`. Default `/`. */ - separator?: string - /** Path pattern → index definitions over that node's direct children. */ - indexes?: IndexDeclarations - /** - * Delete stale index entries found by `findBy` (read-repair). Default `true`. - * See DESIGN.md §12 (M4) for the race with concurrent non-atomic writers. - */ - readRepair?: boolean -} - -export interface KeysOptions { - limit?: number - /** Start after this child key (exclusive), in the direction of travel. */ - after?: string | number - reverse?: boolean -} - -export interface TreeOptions { - /** Levels below this node to include; `1` = direct children only. */ - depth: number -} - -export interface FindByOptions { - limit?: number - reverse?: boolean -} - -/** @internal */ -export interface KVContext { - readonly driver: KVDriver - readonly sep: string - readonly registry: IndexRegistry - readonly readRepair: boolean -} - -interface Slot { - key: string - esc: string - value: unknown -} - -const FIND_CHUNK = 128 -const DELETE_CHUNK = 1000 - -/** - * A store over its direct children. `child(key)` re-roots to that key (sync, no I/O). - * Methods take an optional leading key: without it they act on this node's own value, - * with it on that child's. The choice is made by argument count; a passed-but-`undefined` - * key throws. - */ -export class KVNode { - /** Unescaped path segments. */ - readonly path: readonly string[] - /** This node's own key (escaped). `''` for the root. */ - readonly selfKey: string - /** Prefix of every descendant key. `''` for the root. */ - readonly prefix: string - /** Exclusive upper bound of the subtree range; `undefined` for the root. */ - readonly end: string | undefined - - readonly #ctx: KVContext - readonly #esc: string - readonly #state: TrieState - readonly #selfIdx: BoundIndexes | undefined - readonly #childIdx: BoundIndexes | undefined - - /** @internal Use `KVStorage` and `child()`. */ - constructor(ctx: KVContext, path: readonly string[], selfKey: string, esc: string, state: TrieState, selfIdx: BoundIndexes | undefined) { - this.#ctx = ctx - this.path = path - this.selfKey = selfKey - this.#esc = esc - const root = path.length === 0 - this.prefix = root ? '' : selfKey + ctx.sep - this.end = root ? undefined : selfKey + succ(ctx.sep) - this.#state = state - this.#selfIdx = selfIdx - this.#childIdx = ctx.registry.bind(state, this.prefix) - } - - get driver(): KVDriver { - return this.#ctx.driver - } - - get separator(): string { - return this.#ctx.sep - } - - get isRoot(): boolean { - return this.path.length === 0 - } - - child>(key: K): KVNode> { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - return new KVNode(this.#ctx, [...this.path, seg], this.prefix + esc, esc, this.#ctx.registry.trie.step(this.#state, seg), this.#childIdx) - } - - /** Re-types this node. No runtime effect. */ - as(): KVNode { - return this as unknown as KVNode - } - - get(): Promise | undefined> - get>(key: K): Promise> | undefined> - async get(...args: unknown[]): Promise { - const t = this.#resolve(args, 0) - return (await this.#ctx.driver.getMany([t.key]))[0] - } - - async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { - if (keys.length === 0) return [] - return (await this.#ctx.driver.getMany(keys.map((k) => this.#childKey(k)))) as (ValueOf> | undefined)[] - } - - set(value: ValueOf): Promise - set>(key: K, value: ValueOf>): Promise - async set(...args: unknown[]): Promise { - const t = this.#resolve(args, 1) - const value = args[args.length - 1] - if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') - return this.#write(t.idx, [{ key: t.key, esc: t.esc, value }]) - } - - async setMany(entries: readonly EntryOf[]): Promise { - if (entries.length === 0) return - const slots = entries.map(([k, value]): Slot => { - if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') - const esc = escapeSegment(toSegment(k), this.#ctx.sep) - return { key: this.prefix + esc, esc, value } - }) - return this.#write(this.#childIdx, slots) - } - - /** Deletes the own value (or a child's own value) only, not descendants. */ - delete(): Promise - delete>(key: K): Promise - async delete(...args: unknown[]): Promise { - const t = this.#resolve(args, 0) - return this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }]) - } - - async deleteMany(keys: readonly KeyOf[]): Promise { - if (keys.length === 0) return - const slots = keys.map((k): Slot => { - const esc = escapeSegment(toSegment(k), this.#ctx.sep) - return { key: this.prefix + esc, esc, value: undefined } - }) - return this.#write(this.#childIdx, slots) - } - - /** Deletes the own value and all descendants (or a child's). On the root, deletes everything. */ - deleteTree(): Promise - deleteTree>(key: K): Promise - async deleteTree(...args: unknown[]): Promise { - if (args.length === 0 && this.isRoot) return this.#deleteRange('', undefined) - const t = this.#resolve(args, 0) - await this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }]) - await this.#deleteRange(t.key + this.#ctx.sep, t.key + succ(this.#ctx.sep)) - } - - /** - * Direct child keys (unescaped), including children that only have descendants. - * Order: see `scanChildren` — lexicographic by escaped segment followed by the separator. - */ - async *keys(opts: KeysOptions = {}): AsyncGenerator> { - const after = opts.after === undefined ? undefined : toSegment(opts.after) - for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, { - after, - reverse: opts.reverse, - limit: opts.limit, - })) { - yield unescapeSegment(hit.esc) as KeyString - } - } - - /** Direct children that have an own value, as `[key, value]`. */ - async *entries(opts: KeysOptions = {}): AsyncGenerator> { - const after = opts.after === undefined ? undefined : toSegment(opts.after) - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, { - after, - reverse: opts.reverse, - values: true, - })) { - if (!hit.hasOwn) continue - yield [unescapeSegment(hit.esc), hit.value] as StringEntryOf - if (--remaining <= 0) return - } - } - - /** - * Assembles descendants up to `depth` levels into a plain object: leaves become values, - * nodes with children become objects. A node with both an own value and children exposes the - * own value under the `''` property (empty keys are invalid, so it can't collide). - * Reserved (`~…`) subtrees are skipped. This node's own value is not included. - */ - async tree>(opts: TreeOptions): Promise { - const { depth } = opts - if (!(depth >= 1)) throw new RangeError('tree() depth must be >= 1') - const { driver, sep } = this.#ctx - type TNode = { own?: unknown; has: boolean; kids?: Map } - const root: TNode = { has: false } - const prefix = this.prefix - let it = driver.scan({ gte: prefix, lt: this.end, values: true })[Symbol.asyncIterator]() - const reseek = async (gte: string) => { - await it.return?.() - it = driver.scan({ gte, lt: this.end, values: true })[Symbol.asyncIterator]() - } - try { - outer: for (;;) { - const r = await it.next() - if (r.done) break - const [key, value] = r.value - const segs = key.slice(prefix.length).split(sep) - let base = prefix - for (let i = 0; i < segs.length && i < depth; i++) { - if (segs[i]!.startsWith('~')) { - await reseek(base + succ('~')) - continue outer - } - base += segs[i]! + sep - } - if (segs.length > depth) { - await reseek(prefix + segs.slice(0, depth).join(sep) + succ(sep)) - continue - } - let node = root - for (const s of segs) { - const kids = (node.kids ??= new Map()) - const seg = unescapeSegment(s) - let next = kids.get(seg) - if (!next) kids.set(seg, (next = { has: false })) - node = next - } - node.own = value - node.has = true - } - } finally { - await it.return?.() - } - const build = (n: TNode): unknown => { - if (!n.kids) return n.own - const obj: Record = {} - if (n.has) obj[''] = n.own - for (const [k, v] of n.kids) obj[k] = build(v) - return obj - } - return (root.kids ? build(root) : {}) as T - } - - /** - * Looks up direct children by an index declared for this node, by exact value or range. - * Each hit's value is re-read and re-checked; stale entries are skipped (and deleted when - * `readRepair` is on). Multi-entry hits are de-duplicated. - */ - async *findBy(index: string, query: IndexQuery, opts: FindByOptions = {}): AsyncGenerator> { - const idx = this.#childIdx - if (!idx || !idx.has(index)) throw new TypeError(`No index "${index}" on node "${this.selfKey}"`) - const { driver, readRepair } = this.#ctx - const { gte, lt } = idx.range(index, query) - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - const seen = isIndexRange(query) ? new Set() : undefined - let chunk: string[] = [] - const it = driver.scan({ gte, lt, reverse: opts.reverse })[Symbol.asyncIterator]() - - // Processes one chunk of entry keys; returns false when the limit is reached. - const flush = async function* (this: KVNode): AsyncGenerator, boolean> { - const entries = chunk - chunk = [] - const escs = entries.map((e) => idx.childOf(index, e)) - const values = await driver.getMany(escs.map((e) => this.prefix + e)) - const stale: string[] = [] - try { - for (let i = 0; i < entries.length; i++) { - const esc = escs[i]! - if (!idx.matches(index, esc, values[i], entries[i]!)) { - stale.push(entries[i]!) - continue - } - if (seen) { - if (seen.has(esc)) continue - seen.add(esc) - } - yield [unescapeSegment(esc), values[i]] as StringEntryOf - if (--remaining <= 0) return false - } - return true - } finally { - if (readRepair && stale.length) await driver.deleteMany(stale) - } - } - - try { - for (;;) { - const r = await it.next() - if (r.done) break - chunk.push(r.value[0]) - // Small first chunk so `findOne`-style lookups cost one probe. - if (chunk.length >= (remaining < FIND_CHUNK ? remaining : FIND_CHUNK)) { - if (!(yield* flush.call(this))) return - } - } - if (chunk.length) yield* flush.call(this) - } finally { - await it.return?.() - } - } - - // ---------------------------------------------------------------- internals - - #childKey(key: unknown): string { - return this.prefix + escapeSegment(toSegment(key), this.#ctx.sep) - } - - /** Self vs child by argument count; `selfArity` is the self form's argument count. */ - #resolve(args: readonly unknown[], selfArity: number): { key: string; esc: string; idx: BoundIndexes | undefined } { - if (args.length <= selfArity) { - if (this.isRoot) throw new TypeError('The root node has no own value') - return { key: this.selfKey, esc: this.#esc, idx: this.#selfIdx } - } - const esc = escapeSegment(toSegment(args[0]), this.#ctx.sep) - return { key: this.prefix + esc, esc, idx: this.#childIdx } - } - - /** Writes slots (`value: undefined` = delete), maintaining indexes when present. */ - async #write(idx: BoundIndexes | undefined, slots: Slot[]): Promise { - const { driver } = this.#ctx - if (!idx) { - const sets: [string, unknown][] = [] - const dels: string[] = [] - for (const s of slots) { - if (s.value === undefined) dels.push(s.key) - else sets.push([s.key, s.value]) - } - if (sets.length) await driver.setMany(sets) - if (dels.length) await driver.deleteMany(dels) - return - } - // Last write per key wins. - const byKey = new Map() - for (const s of slots) byKey.set(s.key, s) - const unique = [...byKey.values()] - const old = await driver.getMany(unique.map((s) => s.key)) - const adds: string[] = [] - const removes: string[] = [] - for (let i = 0; i < unique.length; i++) { - const s = unique[i]! - const before = idx.entriesFor(s.esc, old[i]) - const after = idx.entriesFor(s.esc, s.value) - for (const e of after) if (!before.has(e)) adds.push(e) - for (const e of before) if (!after.has(e)) removes.push(e) - } - if (driver.batch) { - const ops: BatchOp[] = [] - for (const k of adds) ops.push({ type: 'set', key: k, value: true }) - for (const s of unique) ops.push(s.value === undefined ? { type: 'delete', key: s.key } : { type: 'set', key: s.key, value: s.value }) - for (const k of removes) ops.push({ type: 'delete', key: k }) - return driver.batch(ops) - } - // Non-atomic: adds first, then values, then removes. Entries are hints; findBy re-checks. - if (adds.length) await driver.setMany(adds.map((k) => [k, true])) - const sets = unique.filter((s) => s.value !== undefined) - const dels = unique.filter((s) => s.value === undefined) - if (sets.length) await driver.setMany(sets.map((s) => [s.key, s.value])) - if (dels.length) await driver.deleteMany(dels.map((s) => s.key)) - if (removes.length) await driver.deleteMany(removes) - } - - async #deleteRange(gte: string, lt: string | undefined): Promise { - const { driver } = this.#ctx - if (lt !== undefined && driver.deleteRange) return driver.deleteRange(gte, lt) - for (;;) { - const keys: string[] = [] - for await (const [k] of driver.scan({ gte, lt, limit: DELETE_CHUNK })) keys.push(k) - if (keys.length === 0) return - await driver.deleteMany(keys) - if (keys.length < DELETE_CHUNK) return - } - } -} - -/** The root `KVNode` over a driver. */ -export class KVStorage extends KVNode { - constructor(driver: KVDriver, opts: KVStorageOptions = {}) { - const sep = opts.separator ?? DEFAULT_SEPARATOR - validateSeparator(sep) - const registry = new IndexRegistry(opts.indexes ?? {}, sep) - super({ driver, sep, registry, readRepair: opts.readRepair ?? true }, [], '', '', registry.trie.rootState, undefined) - } -} diff --git a/src/kv/sorted.ts b/src/kv/sorted.ts deleted file mode 100644 index 53d72d7..0000000 --- a/src/kv/sorted.ts +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Ordered string-keyed map used by the in-memory driver. Kept behind an interface so the - * implementation can be swapped (e.g. for a B-tree) without touching the driver. - */ -export interface SortedMap { - readonly size: number - get(key: string): V | undefined - has(key: string): boolean - set(key: string, value: V): void - /** Inserts many entries; may be faster than repeated `set`. */ - setMany(entries: Iterable<[string, V]>): void - delete(key: string): boolean - /** - * Keys in `[gte, lt)` in order (descending when `reverse`). The iterator stays valid across - * mutations between steps: it resumes after (before) the last key it returned. - */ - keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator -} - -/** - * Sorted chunks of at most `maxChunk` keys (split in half on overflow) plus a `Map` for values. - * Inserts and deletes cost O(log n + maxChunk); iteration is O(1) per step while the map - * is unchanged and re-seeks in O(log n) after a mutation. - */ -export class ChunkedSortedMap implements SortedMap { - constructor(readonly maxChunk = 1024) {} - - #chunks: string[][] = [[]] - readonly #values = new Map() - #version = 0 - - get size(): number { - return this.#values.size - } - - get(key: string): V | undefined { - return this.#values.get(key) - } - - has(key: string): boolean { - return this.#values.has(key) - } - - set(key: string, value: V): void { - if (!this.#values.has(key)) this.#insert(key) - this.#values.set(key, value) - } - - setMany(entries: Iterable<[string, V]>): void { - const fresh: string[] = [] - for (const [key, value] of entries) { - if (!this.#values.has(key)) fresh.push(key) - this.#values.set(key, value) - } - if (fresh.length === 0) return - if (fresh.length < 64 || fresh.length * 8 < this.#values.size) { - for (const key of fresh) this.#insert(key) - return - } - // Bulk path: merge and re-chunk, O(n + k log k). - fresh.sort(cmp) - const old = this.#chunks.flat() - const merged = new Array(old.length + fresh.length) - let i = 0 - let j = 0 - let k = 0 - while (i < old.length && j < fresh.length) merged[k++] = old[i]! < fresh[j]! ? old[i++]! : fresh[j++]! - while (i < old.length) merged[k++] = old[i++]! - while (j < fresh.length) merged[k++] = fresh[j++]! - const half = Math.max(1, this.maxChunk >>> 1) - this.#chunks = [] - for (let s = 0; s < merged.length; s += half) this.#chunks.push(merged.slice(s, s + half)) - this.#version++ - } - - delete(key: string): boolean { - if (!this.#values.delete(key)) return false - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - chunk.splice(lowerBound(chunk, key), 1) - if (chunk.length === 0 && this.#chunks.length > 1) this.#chunks.splice(c, 1) - this.#version++ - return true - } - - *keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator { - const { gte, lt, reverse = false } = range - let version = this.#version - let c: number - let i: number - // Position of the first candidate. - if (reverse) { - ;[c, i] = lt === undefined ? this.#end() : this.#seek(lt, false) - ;[c, i] = this.#prev(c, i) - } else { - ;[c, i] = gte === undefined ? [0, 0] : this.#seek(gte, false) - ;[c, i] = this.#normalize(c, i) - } - for (;;) { - const key = this.#chunks[c]?.[i] - if (key === undefined || c < 0) return - if (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt) return - yield key - if (this.#version !== version) { - version = this.#version - ;[c, i] = this.#seek(key, !reverse) - if (reverse) [c, i] = this.#prev(c, i) - else [c, i] = this.#normalize(c, i) - } else if (reverse) [c, i] = this.#prev(c, i) - else [c, i] = this.#normalize(c, i + 1) - } - } - - #insert(key: string): void { - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - chunk.splice(lowerBound(chunk, key), 0, key) - if (chunk.length > this.maxChunk) this.#chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1)) - this.#version++ - } - - /** Index of the last chunk whose first key is <= key (0 if none). */ - #chunkFor(key: string): number { - const chunks = this.#chunks - let lo = 0 - let hi = chunks.length - 1 - while (lo < hi) { - const mid = (lo + hi + 1) >>> 1 - if (chunks[mid]![0]! <= key) lo = mid - else hi = mid - 1 - } - return lo - } - - /** Position of the first key `>= key` (`> key` when exclusive); may point one past a chunk's end. */ - #seek(key: string, exclusive: boolean): [number, number] { - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - let i = lowerBound(chunk, key) - if (exclusive && chunk[i] === key) i++ - return [c, i] - } - - #end(): [number, number] { - const c = this.#chunks.length - 1 - return [c, this.#chunks[c]!.length] - } - - /** Moves a position that is past its chunk's end to the start of the next chunk. */ - #normalize(c: number, i: number): [number, number] { - while (c < this.#chunks.length && i >= this.#chunks[c]!.length) { - c++ - i = 0 - } - return [c, i] - } - - #prev(c: number, i: number): [number, number] { - i-- - while (i < 0 && c > 0) { - c-- - i = this.#chunks[c]!.length - 1 - } - return i < 0 ? [-1, -1] : [c, i] - } -} - -function lowerBound(arr: string[], key: string): number { - let lo = 0 - let hi = arr.length - while (lo < hi) { - const mid = (lo + hi) >>> 1 - if (arr[mid]! < key) lo = mid + 1 - else hi = mid - } - return lo -} - -function cmp(a: string, b: string): number { - return a < b ? -1 : a > b ? 1 : 0 -} diff --git a/src/testing/index.ts b/src/testing/index.ts deleted file mode 100644 index 9629877..0000000 --- a/src/testing/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './kv-conformance.js' diff --git a/src/testing/kv-conformance.ts b/src/testing/kv-conformance.ts deleted file mode 100644 index 956c77a..0000000 --- a/src/testing/kv-conformance.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { beforeEach, describe, expect, it } from 'vitest' -import type { KVDriver, ScanOptions } from '../kv/driver.js' - -/** - * Reusable conformance suite for {@link KVDriver} implementations. Call it from a Vitest test file: - * - * ```ts - * runKVDriverConformance('MyDriver', () => new MyDriver()) - * ``` - * - * `makeDriver` must return an empty driver each time. - */ -export function runKVDriverConformance(name: string, makeDriver: () => KVDriver | Promise): void { - describe(`KVDriver conformance: ${name}`, () => { - let d: KVDriver - - beforeEach(async () => { - d = await makeDriver() - }) - - async function collect(opts: ScanOptions): Promise<[string, unknown][]> { - const out: [string, unknown][] = [] - for await (const e of d.scan(opts)) out.push(e) - return out - } - - const keysOf = async (opts: ScanOptions) => (await collect(opts)).map(([k]) => k) - - async function seed(): Promise { - // Deliberately inserted out of order. - await d.setMany([ - ['b', 2], - ['a/2', { n: 'a2' }], - ['c', [3]], - ['a', 1], - ['a/1', 'a1'], - ['a0', null], - ['10', 10], - ['9', 9], - ]) - } - - it('round-trips JSON values with getMany in key order, undefined for missing', async () => { - await seed() - expect(await d.getMany(['c', 'missing', 'a/2', 'a0', 'a'])).toEqual([[3], undefined, { n: 'a2' }, null, 1]) - expect(await d.getMany([])).toEqual([]) - }) - - it('overwrites on setMany and deletes with deleteMany', async () => { - await seed() - await d.setMany([['a', 'one']]) - await d.deleteMany(['b', 'missing']) - expect(await d.getMany(['a', 'b'])).toEqual(['one', undefined]) - }) - - it('scans all keys in lexicographic order', async () => { - await seed() - expect(await keysOf({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c']) - }) - - it('honours gte (inclusive) and lt (exclusive)', async () => { - await seed() - expect(await keysOf({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2']) - expect(await keysOf({ gte: 'a', lt: 'b' })).toEqual(['a', 'a/1', 'a/2', 'a0']) - expect(await keysOf({ gte: 'a0' })).toEqual(['a0', 'b', 'c']) - expect(await keysOf({ lt: 'a' })).toEqual(['10', '9']) - expect(await keysOf({ gte: 'zz' })).toEqual([]) - expect(await keysOf({ gte: 'b', lt: 'b' })).toEqual([]) - }) - - it('scans in reverse within bounds', async () => { - await seed() - expect(await keysOf({ reverse: true })).toEqual(['c', 'b', 'a0', 'a/2', 'a/1', 'a', '9', '10']) - expect(await keysOf({ reverse: true, gte: 'a', lt: 'a0' })).toEqual(['a/2', 'a/1', 'a']) - }) - - it('honours limit, forwards and in reverse', async () => { - await seed() - expect(await keysOf({ limit: 2 })).toEqual(['10', '9']) - expect(await keysOf({ limit: 2, reverse: true, lt: 'b' })).toEqual(['a0', 'a/2']) - expect(await keysOf({ limit: 0 })).toEqual([]) - }) - - it('yields values only when values: true', async () => { - await seed() - expect(await collect({ gte: 'a/', lt: 'a0' })).toEqual([ - ['a/1', undefined], - ['a/2', undefined], - ]) - expect(await collect({ gte: 'a/', lt: 'a0', values: true })).toEqual([ - ['a/1', 'a1'], - ['a/2', { n: 'a2' }], - ]) - }) - - it('tolerates writes while a scan is suspended', async () => { - await d.setMany([ - ['k1', 1], - ['k2', 2], - ['k3', 3], - ]) - const seen: string[] = [] - for await (const [key] of d.scan({})) { - seen.push(key) - if (key === 'k1') await d.deleteMany(['k1']) - } - expect(seen).toEqual(['k1', 'k2', 'k3']) - }) - - it('does not let callers mutate stored values through returned references', async () => { - await d.setMany([['o', { n: 1 }]]) - const [v] = (await d.getMany(['o'])) as [{ n: number }] - v.n = 2 - expect(await d.getMany(['o'])).toEqual([{ n: 1 }]) - }) - - it('stores unicode keys and orders them by code unit', async () => { - await d.setMany([ - ['é', 1], - ['z', 2], - ['日本', 3], - ]) - expect(await keysOf({})).toEqual(['z', 'é', '日本']) - }) - - describe('optional: setIfAbsent', () => { - it('writes only absent keys and reports which were written', async (ctx) => { - if (!d.setIfAbsent) return ctx.skip() - await d.setMany([['x', 1]]) - expect( - await d.setIfAbsent([ - ['x', 2], - ['y', 3], - ]), - ).toEqual([false, true]) - expect(await d.getMany(['x', 'y'])).toEqual([1, 3]) - }) - - it('only the first of concurrent claims wins', async (ctx) => { - if (!d.setIfAbsent) return ctx.skip() - const results = await Promise.all([1, 2, 3, 4].map((n) => d.setIfAbsent!([['claim', n]]))) - expect(results.flat().filter(Boolean)).toHaveLength(1) - }) - }) - - describe('optional: batch', () => { - it('applies sets and deletes in order', async (ctx) => { - if (!d.batch) return ctx.skip() - await d.setMany([['gone', 1]]) - await d.batch([ - { type: 'set', key: 'p', value: 1 }, - { type: 'delete', key: 'gone' }, - { type: 'set', key: 'q', value: { a: [1] } }, - { type: 'set', key: 'p', value: 2 }, - ]) - expect(await d.getMany(['p', 'q', 'gone'])).toEqual([2, { a: [1] }, undefined]) - }) - }) - - describe('optional: deleteRange', () => { - it('deletes exactly [gte, lt)', async (ctx) => { - if (!d.deleteRange) return ctx.skip() - await seed() - await d.deleteRange('a/', 'a0') - expect(await keysOf({})).toEqual(['10', '9', 'a', 'a0', 'b', 'c']) - await d.deleteRange('a', 'b') - expect(await keysOf({})).toEqual(['10', '9', 'b', 'c']) - }) - }) - }) -} diff --git a/src/utils/document-path.ts b/src/utils/document-path.ts new file mode 100644 index 0000000..5d2fb2a --- /dev/null +++ b/src/utils/document-path.ts @@ -0,0 +1,32 @@ +import { Json } from './json.js' + +export type Doc = Record + +/** Dot-path access into plain JSON documents (`'address.city'`, `'tags.0'`). */ +export class DocumentPath { + static get(doc: unknown, path: string): unknown { + let value = doc + for (const segment of path.split('.')) { + if (Array.isArray(value) || Json.isObject(value)) value = (value as Record)[segment] + else return undefined + } + return value + } + + /** Sets a path in place, creating intermediate objects. */ + static set(doc: Doc, path: string, value: unknown): void { + const segments = path.split('.') + let target = doc as Record + for (const segment of segments.slice(0, -1)) { + if (typeof target[segment] !== 'object' || target[segment] === null) target[segment] = {} + target = target[segment] as Record + } + target[segments.at(-1)!] = value + } + + static unset(doc: Doc, path: string): void { + const segments = path.split('.') + const parent = segments.length === 1 ? doc : DocumentPath.get(doc, segments.slice(0, -1).join('.')) + if (Json.isObject(parent)) delete parent[segments.at(-1)!] + } +} diff --git a/src/utils/document-query.ts b/src/utils/document-query.ts new file mode 100644 index 0000000..f030539 --- /dev/null +++ b/src/utils/document-query.ts @@ -0,0 +1,106 @@ +import { DocumentPath, type Doc } from './document-path.js' +import { Json } from './json.js' + +export type Comparison = { $eq?: unknown; $gt?: unknown; $gte?: unknown; $lt?: unknown; $lte?: unknown; $in?: unknown[] } + +/** Field (dot path) → value (equality) or comparison operators. All fields must match. */ +export type Filter = Record + +export interface FindOptions { + /** Field → 1 (ascending) or -1 (descending). */ + sort?: Record + limit?: number + skip?: number +} + +export interface FindResult { + key: string + data: T +} + +const OPERATORS = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in']) + +/** + * Filter evaluation, sorting and paging for document stores that don't query natively. + * + * - `{ field: value }` matches equal values; `null` also matches a missing field. + * - Comparisons only match values of the same type (`$gt: 5` never matches a string). + * - On an array field, a condition matches if the array itself or any one element satisfies it. + */ +export class DocumentQuery { + constructor( + readonly filter: Filter = {}, + readonly options: FindOptions = {}, + ) {} + + matches(doc: Doc): boolean { + return Object.entries(this.filter).every(([path, condition]) => DocumentQuery.matchesValue(DocumentPath.get(doc, path), condition)) + } + + /** Whether one field value satisfies a condition. */ + static matchesValue(value: unknown, condition: unknown): boolean { + const candidates = Array.isArray(value) ? [value, ...value] : [value] + if (!DocumentQuery.isComparison(condition)) return candidates.some((c) => DocumentQuery.equal(c, condition)) + const ops = Object.entries(condition) + return candidates.some((c) => ops.every(([op, arg]) => DocumentQuery.compare(op, c, arg))) + } + + /** Applies filter, sort, skip and limit to a stream of candidates. */ + async *run(source: AsyncIterable): AsyncGenerator { + const { sort, skip = 0, limit = Infinity } = this.options + if (limit <= 0) return + if (sort && Object.keys(sort).length) { + const all: FindResult[] = [] + for await (const result of source) if (this.matches(result.data)) all.push(result) + all.sort((a, b) => this.compareResults(a, b)) + yield* all.slice(skip, skip + limit) + return + } + let skipped = 0 + let emitted = 0 + for await (const result of source) { + if (!this.matches(result.data)) continue + if (skipped++ < skip) continue + yield result + if (++emitted >= limit) return + } + } + + private compareResults(a: FindResult, b: FindResult): number { + for (const [path, direction] of Object.entries(this.options.sort!)) { + const order = DocumentQuery.order(DocumentPath.get(a.data, path), DocumentPath.get(b.data, path)) + if (order !== 0) return order * direction + } + return a.key < b.key ? -1 : a.key > b.key ? 1 : 0 + } + + private static isComparison(condition: unknown): condition is Comparison { + if (!Json.isObject(condition)) return false + const keys = Object.keys(condition) + if (keys.length === 0 || !keys.every((k) => k.startsWith('$'))) return false + for (const k of keys) if (!OPERATORS.has(k)) throw new TypeError(`Unsupported operator ${k}`) + return true + } + + private static equal(value: unknown, expected: unknown): boolean { + return expected === null ? value === null || value === undefined : Json.equal(value, expected) + } + + private static compare(op: string, value: unknown, arg: unknown): boolean { + if (op === '$eq') return DocumentQuery.equal(value, arg) + if (op === '$in') return (arg as unknown[]).some((x) => DocumentQuery.equal(value, x)) + if (typeof value !== typeof arg || !['number', 'string', 'boolean'].includes(typeof value)) return false + const order = DocumentQuery.order(value, arg) + return op === '$gt' ? order > 0 : op === '$gte' ? order >= 0 : op === '$lt' ? order < 0 : order <= 0 + } + + /** Total order: missing/null < booleans < numbers < strings < anything else. */ + static order(a: unknown, b: unknown): number { + const rank = (v: unknown) => (v === undefined || v === null ? 0 : (({ boolean: 1, number: 2, string: 3 } as Record)[typeof v] ?? 4)) + const ra = rank(a) + const rb = rank(b) + if (ra !== rb) return ra - rb + if (ra === 4) [a, b] = [JSON.stringify(a), JSON.stringify(b)] + return a === b ? 0 : (a as number) < (b as number) ? -1 : 1 + } +} diff --git a/src/utils/document-update.ts b/src/utils/document-update.ts new file mode 100644 index 0000000..590fbc0 --- /dev/null +++ b/src/utils/document-update.ts @@ -0,0 +1,75 @@ +import { DocumentPath, type Doc } from './document-path.js' +import { DocumentQuery } from './document-query.js' +import { Json } from './json.js' + +/** Update operators. Paths may use dots. `$push`/`$addToSet` accept `{ $each: [...] }`. */ +export interface UpdateSpec { + $set?: Record + $unset?: Record + $inc?: Record + $push?: Record + $pull?: Record + $addToSet?: Record +} + +/** + * A resolved change to one document. Operators are resolved to final values (`$inc` becomes + * `set: { n: 6 }`), so replaying changes is deterministic. + */ +export type DocChange = { op: 'set'; doc: Doc } | { op: 'patch'; set: Record; unset: string[] } | { op: 'delete' } + +export class DocumentUpdate { + /** Resolves update operators against the current state into a patch. */ + static resolve(state: Doc, spec: UpdateSpec): DocChange { + const set: Record = {} + const unset: string[] = [] + const array = (path: string) => { + const current = DocumentPath.get(state, path) ?? [] + if (!Array.isArray(current)) throw new TypeError(`"${path}" is not an array`) + return Json.clone(current) + } + const items = (value: unknown) => (Json.isObject(value) && Array.isArray(value.$each) ? value.$each : [value]) + + for (const [op, fields] of Object.entries(spec)) { + for (const [path, value] of Object.entries(fields as Record)) { + if (path in set || unset.includes(path)) throw new TypeError(`"${path}" is updated twice`) + switch (op) { + case '$set': + set[path] = Json.clone(value) + break + case '$unset': + unset.push(path) + break + case '$inc': { + const current = DocumentPath.get(state, path) ?? 0 + if (typeof current !== 'number' || typeof value !== 'number') throw new TypeError(`$inc needs numbers at "${path}"`) + set[path] = current + value + break + } + case '$push': + set[path] = [...array(path), ...Json.clone(items(value))] + break + case '$addToSet': + set[path] = items(value).reduce((acc, item) => (acc.some((x) => Json.equal(x, item)) ? acc : [...acc, Json.clone(item)]), array(path)) + break + case '$pull': + set[path] = array(path).filter((item) => !DocumentQuery.matchesValue(item, value)) + break + default: + throw new TypeError(`Unsupported update operator ${op}`) + } + } + } + return { op: 'patch', set, unset } + } + + /** Applies a change to a state (`undefined` = no document). */ + static apply(state: Doc | undefined, change: DocChange): Doc | undefined { + if (change.op === 'delete') return undefined + if (change.op === 'set') return Json.clone(change.doc) + const next = Json.clone(state ?? {}) + for (const [path, value] of Object.entries(change.set)) DocumentPath.set(next, path, Json.clone(value)) + for (const path of change.unset) DocumentPath.unset(next, path) + return next + } +} diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 0000000..8b95d83 --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,19 @@ +/** An optimistic-concurrency check failed (stale `expectedVersion` or a lost commit race). */ +export class ConcurrencyError extends Error { + constructor( + readonly stream: string, + readonly expectedVersion: number | 'any', + readonly actualVersion: number, + ) { + super(`Concurrency conflict on "${stream}": expected version ${expectedVersion}, actual ${actualVersion}`) + this.name = 'ConcurrencyError' + } +} + +/** A key argument was passed but is `undefined`/`null`, empty, or not a string/number. */ +export class InvalidKeyError extends TypeError { + constructor(message: string) { + super(message) + this.name = 'InvalidKeyError' + } +} diff --git a/src/utils/json.ts b/src/utils/json.ts new file mode 100644 index 0000000..8c666b7 --- /dev/null +++ b/src/utils/json.ts @@ -0,0 +1,24 @@ +/** Helpers for JSON-compatible values (all stored values are JSON by contract). */ +export class Json { + /** Deep copy; much cheaper than `structuredClone` for plain JSON. */ + static clone(value: T): T { + if (value === null || typeof value !== 'object') return value + if (Array.isArray(value)) return value.map((v) => Json.clone(v)) as T + const out: Record = {} + for (const k in value) out[k] = Json.clone((value as Record)[k]) + return out as T + } + + static equal(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false + if (Array.isArray(a) !== Array.isArray(b)) return false + const ka = Object.keys(a) + const kb = Object.keys(b) + return ka.length === kb.length && ka.every((k) => Json.equal((a as Record)[k], (b as Record)[k])) + } + + static isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) + } +} diff --git a/src/utils/keys.ts b/src/utils/keys.ts new file mode 100644 index 0000000..e7a9149 --- /dev/null +++ b/src/utils/keys.ts @@ -0,0 +1,43 @@ +import { InvalidKeyError } from './errors.js' + +/** + * Key encoding. A node's key is its escaped path segments joined by `/`. + * + * Every character that sorts at or below `/` (and `~` itself) is escaped as `~XX` (hex). Escaped + * segments therefore only contain characters above `/`, so all keys of one child sort together: + * `p/a`, `p/a/...`, then siblings like `p/a~2D1` (`a-1`). That keeps child listing a simple skip-scan. + * + * Segments starting with `~` + a lowercase letter are reserved for internal use (e.g. `~e` for + * event commits); escaping only ever produces `~` + uppercase hex, so user keys can't collide. + */ +export class Keys { + static readonly separator = '/' + /** The character right after `/`: `prefix + segment + '0'` is the first key past a subtree. */ + static readonly afterSeparator = '0' + + /** Validates a user key and converts it to a segment. */ + static segment(key: unknown): string { + if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`) + if (typeof key === 'number' && Number.isFinite(key)) return String(key) + if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or finite number, got ${String(key)}`) + if (key === '') throw new InvalidKeyError('Key must not be empty') + return key + } + + static escape(segment: string): string { + if (!/[\x00-\x2f~]/.test(segment)) return segment + return segment.replace(/[\x00-\x2f~]/g, (c) => '~' + c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')) + } + + static unescape(segment: string): string { + return segment.includes('~') ? segment.replace(/~([0-9A-F]{2})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16))) : segment + } + + static join(path: readonly string[]): string { + return path.map((s) => Keys.escape(s)).join(Keys.separator) + } + + static isReserved(escapedSegment: string): boolean { + return /^~[a-z]/.test(escapedSegment) + } +} diff --git a/src/types/schema.ts b/src/utils/schema.ts similarity index 87% rename from src/types/schema.ts rename to src/utils/schema.ts index 80abbd2..71f340f 100644 --- a/src/types/schema.ts +++ b/src/utils/schema.ts @@ -15,13 +15,16 @@ export type ChildrenOf = S extends { children: infer C } ? C : {} export type KeyOf = Extract, string | number> export type ChildOf> = ChildrenOf[K] extends infer C extends Spec ? C : never +/** Value of this node (`K = undefined`) or of child `K`. */ +export type ValueAt = [K] extends [KeyOf] ? ValueOf> : ValueOf + /** Child keys as yielded at runtime (always strings). */ export type KeyString = `${KeyOf}` /** `[key, value]` for each child key, as a union of precise pairs. */ export type EntryOf = { [K in KeyOf]: [K, ValueOf>] }[KeyOf] -/** `[key, value]` as yielded at runtime (string keys). Dynamic levels collapse to `[string, V]`. */ +/** `[key, value]` as yielded at runtime (string keys). */ export type StringEntryOf = string extends KeyOf ? [string, ValueOf & string>>] : { [K in KeyOf]: [`${K}`, ValueOf>] }[KeyOf] diff --git a/src/utils/sorted-map.ts b/src/utils/sorted-map.ts new file mode 100644 index 0000000..aeee7e3 --- /dev/null +++ b/src/utils/sorted-map.ts @@ -0,0 +1,103 @@ +/** + * Ordered string-keyed map: sorted chunks of at most `maxChunk` keys (split in half when full) + * plus a `Map` for values. Inserts and deletes cost O(log n + maxChunk). + */ +export class SortedMap { + private chunks: string[][] = [[]] + private readonly values = new Map() + /** Bumped on every insert/delete so iterators know to re-seek. */ + private version = 0 + + constructor(private readonly maxChunk = 1024) {} + + get size(): number { + return this.values.size + } + + get(key: string): V | undefined { + return this.values.get(key) + } + + has(key: string): boolean { + return this.values.has(key) + } + + set(key: string, value: V): void { + if (!this.values.has(key)) { + const c = this.chunkFor(key) + const chunk = this.chunks[c]! + chunk.splice(SortedMap.lowerBound(chunk, key), 0, key) + if (chunk.length > this.maxChunk) this.chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1)) + this.version++ + } + this.values.set(key, value) + } + + delete(key: string): boolean { + if (!this.values.delete(key)) return false + const c = this.chunkFor(key) + const chunk = this.chunks[c]! + chunk.splice(SortedMap.lowerBound(chunk, key), 1) + if (chunk.length === 0 && this.chunks.length > 1) this.chunks.splice(c, 1) + this.version++ + return true + } + + /** + * Keys in `[gte, lt)`, descending when `reverse`. Safe across mutations between steps: + * after a change it re-seeks from the last key returned. + */ + *keys(gte?: string, lt?: string, reverse = false): IterableIterator { + let last: string | undefined + let version = -1 + let c = 0 + let i = 0 + for (;;) { + if (version !== this.version) { + version = this.version + const from = last ?? (reverse ? lt : gte) + if (from === undefined) { + c = reverse ? this.chunks.length - 1 : 0 + i = reverse ? this.chunks[c]!.length - 1 : 0 + } else { + c = this.chunkFor(from) + i = SortedMap.lowerBound(this.chunks[c]!, from) + // Forward: skip `last` itself. Reverse: step to the key before `from`. + if (reverse) i-- + else if (last !== undefined && this.chunks[c]![i] === last) i++ + } + } + // Move across chunk boundaries. + while (!reverse && c < this.chunks.length - 1 && i >= this.chunks[c]!.length) (c++, (i = 0)) + while (reverse && c > 0 && i < 0) i = this.chunks[--c]!.length - 1 + const key = this.chunks[c]?.[i] + if (key === undefined || (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt)) return + last = key + yield key + i += reverse ? -1 : 1 + } + } + + /** Index of the last chunk whose first key is <= key (0 if none). */ + private chunkFor(key: string): number { + let lo = 0 + let hi = this.chunks.length - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >>> 1 + if (this.chunks[mid]![0]! <= key) lo = mid + else hi = mid - 1 + } + return lo + } + + private static lowerBound(arr: string[], key: string): number { + let lo = 0 + let hi = arr.length + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (arr[mid]! < key) lo = mid + 1 + else hi = mid + } + return lo + } +} diff --git a/src/utils/ulid.ts b/src/utils/ulid.ts new file mode 100644 index 0000000..fb42411 --- /dev/null +++ b/src/utils/ulid.ts @@ -0,0 +1,36 @@ +const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32 + +/** + * ULID-like ids: 10 chars of millisecond time + 16 random chars, sortable by creation time. + * Ids from one generator within the same millisecond increment the random part, so they stay + * strictly increasing. + */ +export class Ulid { + private lastTime = -1 + private readonly random = new Uint8Array(16) // one base32 digit (0–31) per byte + + constructor(private readonly now: () => number = Date.now) {} + + generate(): string { + let time = this.now() + if (time <= this.lastTime) { + time = this.lastTime + let i = 15 + while (i >= 0 && this.random[i] === 31) this.random[i--] = 0 + if (i < 0) throw new Error('ULID overflow within one millisecond') + this.random[i]!++ + } else { + crypto.getRandomValues(this.random) + for (let i = 0; i < 16; i++) this.random[i]! &= 31 + this.random[0]! &= 15 // headroom for increments + this.lastTime = time + } + let out = '' + for (let i = 0; i < 10; i++) { + out = ALPHABET[time % 32]! + out + time = Math.floor(time / 32) + } + for (const digit of this.random) out += ALPHABET[digit] + return out + } +} diff --git a/test/core/encoding.test.ts b/test/core/encoding.test.ts deleted file mode 100644 index 409f07f..0000000 --- a/test/core/encoding.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { decodeIndexValue, encodeIndexValue, type IndexValue } from '../../src/core/encoding.js' - -const separators = fc.constantFrom('/', ':', '.', '|', '#', '!') - -const indexValue: fc.Arbitrary = fc.oneof( - fc.constant(null), - fc.boolean(), - fc.double({ noNaN: true }), - fc.integer(), - fc.string({ unit: 'binary' }), - fc.string({ unit: fc.constantFrom('/', ':', '0', '1', 'a', '%', '~', '\0', '!') }), -) - -const rank = (v: IndexValue) => (v === null ? 0 : v === false ? 1 : v === true ? 2 : typeof v === 'number' ? 3 : 4) - -function compare(a: IndexValue, b: IndexValue): number { - const ra = rank(a) - const rb = rank(b) - if (ra !== rb) return ra - rb - if (typeof a === 'number' && typeof b === 'number') return a === b ? 0 : a < b ? -1 : 1 - if (typeof a === 'string' && typeof b === 'string') return a === b ? 0 : a < b ? -1 : 1 - return 0 -} - -const sign = (n: number) => (n < 0 ? -1 : n > 0 ? 1 : 0) -const cmpStr = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0) - -describe('encodeIndexValue', () => { - it('round-trips (property)', () => { - fc.assert( - fc.property(indexValue, separators, (v, sep) => { - const decoded = decodeIndexValue(encodeIndexValue(v, sep), sep) - if (typeof v === 'number' && v === 0) - expect(decoded).toBe(0) // -0 normalised - else expect(decoded).toEqual(v) - }), - ) - }) - - it('emits only characters above the separator (property)', () => { - fc.assert( - fc.property(indexValue, separators, (v, sep) => { - for (const c of encodeIndexValue(v, sep)) expect(c > sep).toBe(true) - }), - ) - }) - - it('preserves order, including when followed by a separator and suffix (property)', () => { - fc.assert( - fc.property(indexValue, indexValue, fc.string(), fc.string(), separators, (a, b, sa, sb, sep) => { - const ea = encodeIndexValue(a, sep) - const eb = encodeIndexValue(b, sep) - const expected = sign(compare(a, b)) - expect(cmpStr(ea, eb)).toBe(expected) - if (expected !== 0) expect(cmpStr(ea + sep + sa, eb + sep + sb)).toBe(expected) - }), - { numRuns: 2000 }, - ) - }) - - it('orders types null < false < true < numbers < strings', () => { - const values: IndexValue[] = ['', 'a', -Infinity, -1, 0, 1e300, Infinity, true, false, null] - const sorted = [...values].sort((a, b) => cmpStr(encodeIndexValue(a), encodeIndexValue(b))) - expect(sorted).toEqual([null, false, true, -Infinity, -1, 0, 1e300, Infinity, '', 'a']) - }) - - it('rejects NaN and non-index values', () => { - expect(() => encodeIndexValue(NaN)).toThrow() - expect(() => encodeIndexValue({} as never)).toThrow() - }) -}) diff --git a/test/core/path.test.ts b/test/core/path.test.ts deleted file mode 100644 index 39279d8..0000000 --- a/test/core/path.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { escapeSegment, joinPath, pathKeys, splitPath, succ, toSegment, unescapeSegment, validateSeparator } from '../../src/core/path.js' -import { InvalidKeyError } from '../../src/errors.js' - -const separators = fc.constantFrom('/', ':', '.', '|', '#', '\\', '!') - -describe('escapeSegment', () => { - it('escapes %, the separator and a leading ~', () => { - expect(escapeSegment('a/b')).toBe('a%2Fb') - expect(escapeSegment('50%')).toBe('50%25') - expect(escapeSegment('~x')).toBe('%7Ex') - expect(escapeSegment('x~')).toBe('x~') - expect(escapeSegment('a:b', ':')).toBe('a%3Ab') - expect(escapeSegment('a/b', ':')).toBe('a/b') - expect(escapeSegment('plain')).toBe('plain') - }) - - it('round-trips (property)', () => { - fc.assert( - fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => { - expect(unescapeSegment(escapeSegment(s, sep))).toBe(s) - }), - ) - }) - - it('never emits a raw separator or a leading ~ (property)', () => { - fc.assert( - fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => { - const e = escapeSegment(s, sep) - expect(e.includes(sep)).toBe(false) - expect(e.startsWith('~')).toBe(false) - }), - ) - }) - - it('joins and splits paths (property)', () => { - fc.assert( - fc.property(fc.array(fc.string({ minLength: 1 }), { maxLength: 6 }), separators, (path, sep) => { - expect(splitPath(joinPath(path, sep), sep)).toEqual(path) - }), - ) - }) -}) - -describe('pathKeys', () => { - it('precomputes selfKey, prefix and subtree end', () => { - expect(pathKeys(['users', '123'], '/')).toMatchObject({ selfKey: 'users/123', prefix: 'users/123/', end: 'users/1230' }) - expect(pathKeys([], '/')).toMatchObject({ selfKey: '', prefix: '', end: undefined }) - expect(pathKeys(['a/b'], '/').selfKey).toBe('a%2Fb') - }) -}) - -describe('misc', () => { - it('succ returns the next char', () => { - expect(succ('/')).toBe('0') - }) - - it('toSegment rejects undefined, null, empty and non-finite', () => { - expect(() => toSegment(undefined)).toThrow(InvalidKeyError) - expect(() => toSegment(null)).toThrow(TypeError) - expect(() => toSegment('')).toThrow(InvalidKeyError) - expect(() => toSegment(NaN)).toThrow(InvalidKeyError) - expect(toSegment(12)).toBe('12') - }) - - it('validates separators', () => { - expect(() => validateSeparator('//')).toThrow() - expect(() => validateSeparator('%')).toThrow() - expect(() => validateSeparator('~')).toThrow() - expect(() => validateSeparator(':')).not.toThrow() - }) -}) diff --git a/test/document-storage/document-storage.suite.ts b/test/document-storage/document-storage.suite.ts new file mode 100644 index 0000000..dc489c4 --- /dev/null +++ b/test/document-storage/document-storage.suite.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest' +import type { BaseDocumentStorage } from '../../src/document-storage/base-document-storage.js' +import { collect } from '../helpers.js' + +type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } } + +/** Behaviour every document storage must have. `make` returns an empty storage. */ +export function runDocumentStorageSuite(name: string, make: () => BaseDocumentStorage): void { + describe(`document storage: ${name}`, () => { + const users = () => make().child('users') + + it('inserts with time-sortable ids, gets and lists', async () => { + const u = users() + const a = await u.insert({ name: 'Ada' }) + const b = await u.insert({ name: 'Bob' }) + expect(a < b).toBe(true) + expect(await u.get(a)).toEqual({ name: 'Ada' }) + expect(await collect(u.keys())).toEqual([a, b]) + }) + + it('set / get / delete in self and child forms', async () => { + const u = users() + await u.set('1', { name: 'Ada' }) + expect(await u.child('1').get()).toEqual({ name: 'Ada' }) + await u.child('1').set({ name: 'Ada L.' }) + expect(await u.getMany(['1', '2'])).toEqual([{ name: 'Ada L.' }, undefined]) + expect(await u.delete('1')).toBe(true) + expect(await u.child('1').delete()).toBe(false) + expect(await u.get('1')).toBeUndefined() + await u.set('1', { name: 'Again' }) + expect(await u.get('1')).toEqual({ name: 'Again' }) + }) + + it('updates with operators, returns the new state, and supports upsert', async () => { + const u = users() + await u.set('1', { name: 'Ada', tags: ['a'], logins: 1 }) + expect(await u.update('1', { $inc: { logins: 2 }, $push: { tags: 'b' }, $set: { 'address.city': 'London' } })).toEqual({ + name: 'Ada', + tags: ['a', 'b'], + logins: 3, + address: { city: 'London' }, + }) + expect(await u.child('1').update({ $unset: { address: 1 } })).toEqual({ name: 'Ada', tags: ['a', 'b'], logins: 3 }) + expect(await u.update('missing', { $set: { a: 1 } })).toBeUndefined() + expect(await u.update('missing', { $inc: { n: 1 } }, { upsert: true })).toEqual({ n: 1 }) + }) + + it('rejects bad keys and documents', async () => { + const u = users() + await expect(u.get(undefined as never)).rejects.toThrow(TypeError) + await expect(u.update(undefined as never, { $set: {} })).rejects.toThrow(TypeError) + await expect(make().get()).rejects.toThrow(/root/) + await expect(u.set('x', [] as never)).rejects.toThrow(/plain object/) + }) + + it('nests documents under documents', async () => { + const u = users() + await u.set('1', { name: 'Ada' }) + await u.child('1').child('posts').set('p1', { title: 'Hi' }) + expect(await u.child('1').child('posts').get('p1')).toEqual({ title: 'Hi' }) + expect((await collect(u.find())).map((r) => r.key)).toEqual(['1']) + }) + + it('finds with equality, comparisons, $in, dot paths, sort, skip and limit', async () => { + const u = users() + const rows: [string, User][] = [ + ['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }], + ['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'] }], + ['c', { name: 'Cy', age: 51, address: { city: 'London' } }], + ['d', { name: 'Di', age: 20 }], + ] + for (const [k, v] of rows) await u.set(k, v) + const keys = async (filter: Record, options = {}) => (await collect(u.find(filter, options))).map((r) => r.key) + expect(await keys({ age: 20 })).toEqual(['b', 'd']) + expect(await keys({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd']) + expect(await keys({ age: { $in: [36, 51] } })).toEqual(['a', 'c']) + expect(await keys({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c']) + expect(await keys({ tags: 'y' })).toEqual(['a', 'b']) + expect(await keys({ email: null })).toEqual(['c', 'd']) + expect(await keys({}, { sort: { age: -1 } })).toEqual(['c', 'a', 'b', 'd']) + expect(await keys({}, { sort: { age: 1 }, skip: 1, limit: 2 })).toEqual(['d', 'a']) + expect(await u.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: rows[1]![1] }) + expect(await u.findOne({ email: 'nobody' })).toBeUndefined() + }) + }) +} diff --git a/test/document-storage/document-storage.test.ts b/test/document-storage/document-storage.test.ts new file mode 100644 index 0000000..d4add11 --- /dev/null +++ b/test/document-storage/document-storage.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' +import { DocumentStorageInMemory } from '../../src/document-storage/document-storage-in-memory.js' +import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' +import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { collect, KVStorageCounting } from '../helpers.js' +import { runDocumentStorageSuite } from './document-storage.suite.js' + +const indexes = { users: ['email', 'age'] } + +runDocumentStorageSuite('DocumentStorageInMemory', () => new DocumentStorageInMemory()) +runDocumentStorageSuite('DocumentStorageInMemory (indexed)', () => new DocumentStorageInMemory({ indexes })) +runDocumentStorageSuite('DocumentStorageKV', () => new DocumentStorageKV(new KVStorageInMemory())) +runDocumentStorageSuite('EventSourced (in-memory projections)', () => { + const kv = new KVStorageInMemory() + return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageInMemory({ indexes })) +}) +runDocumentStorageSuite('EventSourced (KV projections)', () => { + const kv = new KVStorageInMemory() + return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))) +}) + +describe('DocumentStorageInMemory indexes', () => { + it('keeps the equality index in sync with updates and deletes', async () => { + const users = new DocumentStorageInMemory({ indexes }).child('users') + await users.set('1', { email: 'a', age: 1 }) + await users.set('2', { email: 'b', age: 1 }) + await users.update('1', { $set: { email: 'c' } }) + await users.delete('2') + expect(await collect(users.find({ email: 'a' }))).toEqual([]) + expect(await collect(users.find({ email: 'c' }))).toEqual([{ key: '1', data: { email: 'c', age: 1 } }]) + expect(await collect(users.find({ age: { $eq: 1 } }))).toHaveLength(1) + }) +}) + +describe('EventSourcedDocumentStorage', () => { + function setup() { + const kv = new KVStorageCounting() + const events = new EventStorageKV(kv) + const projections = new DocumentStorageKV(kv.child('projections')) + return { kv, events, projections, docs: new EventSourcedDocumentStorage(events, projections) } + } + + it('records resolved changes as history and replays any version', async () => { + const users = setup().docs.child('users') + await users.set('1', { n: 5 }) + await users.update('1', { $inc: { n: 1 } }) + await users.delete('1') + await users.set('1', { n: 10 }) + expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([ + { op: 'set', doc: { n: 5 } }, + { op: 'patch', set: { n: 6 }, unset: [] }, + { op: 'delete' }, + { op: 'set', doc: { n: 10 } }, + ]) + expect(await users.getAt('1', 2)).toEqual({ n: 6 }) + expect(await users.child('1').getAt(3)).toBeUndefined() + expect(await users.get('1')).toEqual({ n: 10 }) + }) + + it('heals a projection left behind by a crash between append and projection write', async () => { + const { docs, events } = setup() + const users = docs.child('users') + await users.set('1', { n: 1 }) + await events.child('users').append('1', [{ op: 'patch', set: { n: 2 }, unset: [] }]) // projection never written + expect(await users.get('1')).toEqual({ n: 1 }) + expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3 }) + await events.child('users').append('2', [{ op: 'set', doc: { n: 7 } }]) // first write crashed + expect(await users.catchUp('2')).toEqual({ n: 7 }) + expect(await users.get('2')).toEqual({ n: 7 }) + }) + + it('two instances incrementing the same document converge', async () => { + const kv = new KVStorageInMemory() + const make = () => new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))).child('users') + const a = make() + const b = make() + await a.set('1', { n: 0 }) + await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })])) + expect(await a.get('1')).toEqual({ n: 10 }) + }) + + it('a write is a projection read, an append and a projection write', async () => { + const { docs, kv } = setup() + await docs.child('users').set('1', { n: 1 }) + kv.reset() + await docs.child('users').update('1', { $inc: { n: 1 } }) + expect(kv.calls).toEqual({ readMany: 2, writeIfAbsent: 1, writeMany: 1 }) + }) +}) diff --git a/test/documents/event-sourced.test.ts b/test/documents/event-sourced.test.ts deleted file mode 100644 index 293bc1c..0000000 --- a/test/documents/event-sourced.test.ts +++ /dev/null @@ -1,379 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { EventSourcedDocumentStorage, type EventSourcedDocumentStorageOptions } from '../../src/documents/event-sourced.js' -import type { DocIndexDeclarations } from '../../src/documents/indexes.js' -import type { Filter } from '../../src/documents/filter.js' -import { CapabilityError, UniqueConstraintError } from '../../src/errors.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } } - -const userIndexes: DocIndexDeclarations = { - users: [{ fields: ['email'], unique: true }, { fields: ['age'] }, { name: 'cityAge', fields: ['address.city', 'age'] }, { fields: ['tags'] }], - 'users/*/posts': [{ fields: ['createdAt'] }], -} - -function setup(opts: { hide?: Capability[]; inner?: InMemoryKVDriver } & EventSourcedDocumentStorageOptions = {}) { - const { hide = [], inner = new InMemoryKVDriver(), ...docOpts } = opts - const driver = new CountingDriver(inner, hide) - const kv = new KVStorage(driver) - const events = new KVEventStorage(kv) - const docs = new EventSourcedDocumentStorage(events, { indexes: userIndexes, warnOnFullScan: false, ...docOpts }) - return { driver, kv, events, docs, users: docs.child('users') } -} - -afterEach(() => vi.restoreAllMocks()) - -describe.each([ - ['atomic batch', [] as Capability[]], - ['non-atomic', ['batch'] as Capability[]], -])('EventSourcedDocumentStorage (%s)', (_, hide) => { - describe('CRUD', () => { - it('inserts with time-sortable ids and reads back', async () => { - const { users } = setup({ hide }) - const a = await users.insert({ name: 'Ada', age: 36 }) - const b = await users.insert({ name: 'Bob', age: 20 }) - expect(a).toMatch(/^[0-9A-HJKMNP-TV-Z]{26}$/) - expect(a < b).toBe(true) - expect(await users.get(a)).toEqual({ name: 'Ada', age: 36 }) - expect(await collect(users.keys())).toEqual([a, b]) - }) - - it('set / get / delete in self and child forms', async () => { - const { users } = setup({ hide }) - await users.set('123', { name: 'Ada', age: 36 }) - expect(await users.child('123').get()).toEqual({ name: 'Ada', age: 36 }) - await users.child('123').set({ name: 'Ada L.' }) - expect(await users.get('123')).toEqual({ name: 'Ada L.' }) - expect(await users.delete('123')).toBe(true) - expect(await users.delete('123')).toBe(false) - expect(await users.get('123')).toBeUndefined() - expect(await collect(users.keys())).toEqual([]) - await users.set('123', { name: 'Again' }) - expect(await users.get('123')).toEqual({ name: 'Again' }) - }) - - it('applies arity rules', async () => { - const { users, docs } = setup({ hide }) - const id: string | undefined = undefined - await expect(users.get(id as never)).rejects.toThrow(TypeError) - await expect(users.update(id as never, { $set: { a: 1 } })).rejects.toThrow(TypeError) - await expect(users.delete(id as never)).rejects.toThrow(TypeError) - await expect(docs.get()).rejects.toThrow(/root/) - await expect(users.set('x', [] as never)).rejects.toThrow(/plain object/) - }) - - it('nests documents under documents', async () => { - const { users } = setup({ hide }) - await users.set('u1', { name: 'Ada' }) - const posts = users.child('u1').child('posts') - await posts.set('p1', { title: 'Hello', createdAt: '2024-01' }) - expect(await posts.get('p1')).toEqual({ title: 'Hello', createdAt: '2024-01' }) - expect((await collect(users.find())).map((r) => r.key)).toEqual(['u1']) - expect((await collect(posts.find({ createdAt: { $gte: '2024' } }))).map((r) => r.key)).toEqual(['p1']) - }) - }) - - describe('update', () => { - it('supports $set $unset $inc $push $pull $addToSet with dot paths', async () => { - const { users } = setup({ hide }) - await users.set('1', { name: 'Ada', tags: ['a'], address: { city: 'X' }, nickname: 'A' } as User) - const r = await users.update('1', { - $set: { name: 'Ada L.', 'address.zip': '123' }, - $inc: { logins: 1 }, - $push: { tags: { $each: ['b', 'c'] } }, - $unset: { nickname: true }, - }) - expect(r).toEqual({ name: 'Ada L.', tags: ['a', 'b', 'c'], address: { city: 'X', zip: '123' }, logins: 1 }) - await users.update('1', { $pull: { tags: { $in: ['a', 'c'] } }, $addToSet: { 'address.list': { $each: [1, 1, 2] } } }) - await users.child('1').update({ $inc: { logins: 5 }, $addToSet: { tags: 'b' } }) - expect(await users.get('1')).toEqual({ name: 'Ada L.', tags: ['b'], address: { city: 'X', zip: '123', list: [1, 2] }, logins: 6 }) - }) - - it('returns undefined for missing docs unless upsert', async () => { - const { users } = setup({ hide }) - expect(await users.update('nope', { $set: { a: 1 } })).toBeUndefined() - expect(await users.get('nope')).toBeUndefined() - expect(await users.update('nope', { $inc: { n: 2 } }, { upsert: true })).toEqual({ n: 2 }) - }) - - it('rejects conflicting paths and bad operators', async () => { - const { users } = setup({ hide }) - await users.set('1', { age: 1 }) - await expect(users.update('1', { $set: { a: 1 }, $unset: { 'a.b': 1 } })).rejects.toThrow(/conflict/) - await expect(users.update('1', { $rename: {} } as never)).rejects.toThrow(/Unsupported/) - await expect(users.update('1', { $inc: { name: 1 } }).then(() => users.update('1', { $push: { age: 1 } }))).rejects.toThrow(/non-array/) - }) - - it('stores resolved deltas, not operators, and skips no-op updates', async () => { - const { users } = setup({ hide }) - await users.set('1', { logins: 5 }) - await users.update('1', { $inc: { logins: 1 } }) - await users.update('1', { $set: { logins: 6 } }) // no-op - expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([ - { op: 'set', doc: { logins: 5 } }, - { op: 'patch', set: { logins: 6 } }, - ]) - }) - }) - - describe('history and getAt', () => { - it('replays to any version', async () => { - const { users } = setup({ hide }) - await users.set('1', { n: 1 }) - await users.update('1', { $inc: { n: 1 } }) - await users.delete('1') - await users.set('1', { n: 10 }) - expect(await users.getAt('1', 0)).toBeUndefined() - expect(await users.getAt('1', 1)).toEqual({ n: 1 }) - expect(await users.getAt('1', 2)).toEqual({ n: 2 }) - expect(await users.getAt('1', 3)).toBeUndefined() - expect(await users.child('1').getAt(4)).toEqual({ n: 10 }) - expect(await users.getAt('1', 99)).toEqual({ n: 10 }) - expect((await collect(users.child('1').history())).map((e) => e.version)).toEqual([1, 2, 3, 4]) - }) - }) - - describe('find', () => { - async function seed() { - const ctx = setup({ hide }) - const rows: [string, User][] = [ - ['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }], - ['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'], address: { city: 'Paris' } }], - ['c', { name: 'Cy', age: 51, tags: [], address: { city: 'London' } }], - ['d', { name: 'Di', age: 20, email: 'di@x' }], - ['e', { name: 'Ed', email: 'ed@x', address: { city: 'London' } }], - ] - for (const [k, v] of rows) await ctx.users.set(k, v) - await ctx.users.set('gone', { name: 'Gone', age: 20 }) - await ctx.users.delete('gone') - return ctx - } - const keysOf = async (it: AsyncIterable<{ key: string }>) => (await collect(it)).map((r) => r.key) - - it('evaluates comparison, logical and element operators', async () => { - const { users } = await seed() - // Unsorted results come in index or key order depending on the plan; compare as sets. - const q = async (f: Filter) => (await keysOf(users.find(f))).sort() - expect(await q({})).toEqual(['a', 'b', 'c', 'd', 'e']) - expect(await q({ age: 20 })).toEqual(['b', 'd']) - expect(await q({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd']) - expect(await q({ age: { $gt: 20 } })).toEqual(['a', 'c']) - expect(await q({ age: { $ne: 20 } })).toEqual(['a', 'c', 'e']) - expect(await q({ age: { $in: [36, 51] } })).toEqual(['a', 'c']) - expect(await q({ age: { $nin: [36, 51] } })).toEqual(['b', 'd', 'e']) - expect(await q({ age: { $exists: false } })).toEqual(['e']) - expect(await q({ age: null })).toEqual(['e']) - expect(await q({ email: { $exists: true }, age: { $lte: 20 } })).toEqual(['b', 'd']) - expect(await q({ 'address.city': 'London' })).toEqual(['a', 'c', 'e']) - expect(await q({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c']) - expect(await q({ tags: 'y' })).toEqual(['a', 'b']) - expect(await q({ tags: ['y'] })).toEqual(['b']) - expect(await q({ $or: [{ age: 51 }, { name: 'Bob' }] })).toEqual(['b', 'c']) - expect(await q({ $and: [{ age: { $gte: 20 } }, { age: { $lte: 20 } }] })).toEqual(['b', 'd']) - expect(await q({ $not: { age: 20 } })).toEqual(['a', 'c', 'e']) - expect(await q({ age: { $not: { $gte: 30 } } })).toEqual(['b', 'd', 'e']) - expect(await q({ name: { $gt: 5 } })).toEqual([]) // type-bracketed - }) - - it('sorts (index-backed and in-memory), skips, limits and projects', async () => { - const { users } = await seed() - expect(await keysOf(users.find({}, { sort: { age: 1 } }))).toEqual(['e', 'b', 'd', 'a', 'c']) - expect(await keysOf(users.find({}, { sort: { age: -1 } }))).toEqual(['c', 'a', 'd', 'b', 'e']) - expect(await keysOf(users.find({}, { sort: { name: -1 }, skip: 1, limit: 2 }))).toEqual(['d', 'c']) - expect(await keysOf(users.find({ age: { $gte: 20 } }, { sort: { age: -1 }, limit: 2 }))).toEqual(['c', 'a']) - expect(await keysOf(users.find({ 'address.city': 'London' }, { sort: { age: 1 } }))).toEqual(['e', 'a', 'c']) - expect(await collect(users.find({ age: 36 }, { projection: { name: 1, 'address.city': 1 } }))).toEqual([ - { key: 'a', data: { name: 'Ada', address: { city: 'London' } } }, - ]) - expect(await collect(users.find({ age: 36 }, { projection: { tags: 0, address: 0, email: 0 } }))).toEqual([{ key: 'a', data: { name: 'Ada', age: 36 } }]) - expect(await users.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: expect.objectContaining({ name: 'Bob' }) }) - expect(await users.findOne({ email: 'nobody' })).toBeUndefined() - }) - - it('uses an index: findOne costs one entry scan and one projection read', async () => { - const { users, driver } = await seed() - driver.reset() - await users.findOne({ email: 'di@x' }) - expect(driver.calls).toEqual({ scan: 1, getMany: 1 }) - }) - - it('warns once per filter shape on full scans', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const { users } = setup({ hide, warnOnFullScan: true }) - await collect(users.find({ name: 'x' })) - await collect(users.find({ name: 'y' })) - await collect(users.find({ age: 1 })) - expect(warn).toHaveBeenCalledTimes(1) - expect(warn.mock.calls[0]![0]).toMatch(/full scan for filter \{"name":"\?"\}/) - }) - }) - - describe('unique indexes', () => { - it('rejects duplicates and frees values on change and delete', async () => { - const { users } = setup({ hide }) - await users.set('1', { email: 'a@x' }) - await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError) - await expect(users.insert({ email: 'a@x' })).rejects.toMatchObject({ index: 'email', existingKey: '1' }) - expect(await collect(users.keys())).toEqual(['1']) - await users.update('1', { $set: { email: 'b@x' } }) - await users.set('2', { email: 'a@x' }) - await users.delete('2') - await users.set('3', { email: 'a@x' }) - await users.set('4', {}) // missing values are not claimed - await users.set('5', {}) - await users.set('3', { email: 'a@x', name: 'same value, same doc' }) - }) - - it('exactly one of concurrent writers claiming the same value wins', async () => { - const { users } = setup({ hide }) - const results = await Promise.allSettled(['1', '2', '3'].map((k) => users.set(k, { email: 'same' }))) - expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) - expect((await collect(users.find({ email: 'same' }))).length).toBe(1) - }) - - it('takes over a stale claim only after the grace period', async () => { - let t = 0 - const { users, driver } = setup({ hide, now: () => t, claimGraceMs: 1000 }) - // Leave an orphaned claim: doc 1's projection and events vanish but its claim remains. - await users.set('1', { email: 'a@x' }) - await driver.inner.deleteMany(['~p/users/1']) - await driver.inner.deleteMany((await collect(driver.inner.scan({ gte: 'users/1/', lt: 'users/10' }))).map(([k]) => k)) - await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError) - t = 5000 - await users.set('2', { email: 'a@x' }) - expect(await users.get('2')).toEqual({ email: 'a@x' }) - }) - - it('requires setIfAbsent', () => { - const kv = new KVStorage(new CountingDriver(new InMemoryKVDriver(), ['setIfAbsent'])) - expect(() => new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: userIndexes })).toThrow(CapabilityError) - }) - }) - - describe('projection lag (crash between append and projection write)', () => { - async function lagged() { - const ctx = setup({ hide }) - await ctx.users.set('1', { n: 1, email: 'a@x' }) - // Simulate a crash: the event is appended but the projection is never written. - await ctx.events.child('users').append('1', [{ op: 'patch', set: { n: 2, email: 'b@x' } }]) - return ctx - } - - it('reads see the stale projection; consistent reads catch up', async () => { - const { users } = await lagged() - expect(await users.get('1')).toEqual({ n: 1, email: 'a@x' }) - expect(await users.get('1', { consistent: true })).toEqual({ n: 2, email: 'b@x' }) - expect(await users.findOne({ email: 'b@x' })).toMatchObject({ key: '1' }) - expect(await users.findOne({ email: 'a@x' })).toBeUndefined() - }) - - it('the next write self-heals', async () => { - const { users } = await lagged() - expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3, email: 'b@x' }) - expect((await collect(users.history('1'))).map((e) => e.version)).toEqual([1, 2, 3]) - await users.set('2', { email: 'a@x' }) // old claim was released by the catch-up - }) - - it('catchUp() repairs a subtree', async () => { - const { users, docs, events } = await lagged() - await events - .child('users') - .child('9') - .child('posts') - .append('p', [{ op: 'set', doc: { createdAt: 'x' } }]) - expect(await docs.catchUp()).toBe(2) - expect(await users.get('1')).toEqual({ n: 2, email: 'b@x' }) - expect(await users.child('9').child('posts').get('p')).toEqual({ createdAt: 'x' }) - expect(await docs.catchUp()).toBe(0) - }) - }) - - describe('concurrency', () => { - it('two storage instances incrementing the same doc converge', async () => { - const inner = new InMemoryKVDriver() - const a = setup({ hide, inner }).users - const b = setup({ hide, inner }).users - await a.set('1', { n: 0 }) - await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })])) - expect(await a.get('1')).toEqual({ n: 10 }) - expect(await b.get('1', { consistent: true })).toEqual({ n: 10 }) - }) - }) -}) - -describe('costs', () => { - it('get is one driver call; update is projection read + append + projection write, even cold', async () => { - const inner = new InMemoryKVDriver() - await setup({ inner }).users.set('1', { name: 'a', age: 1 }) - const { users, driver } = setup({ inner }) // fresh instance: cold head cache - driver.reset() - await users.get('1') - expect(driver.calls).toEqual({ getMany: 1 }) - driver.reset() - await users.update('1', { $set: { age: 2 } }) - expect(driver.calls).toEqual({ getMany: 1, setIfAbsent: 1, batch: 1 }) - }) -}) - -describe('property: indexed find equals unindexed find', () => { - const doc = fc.record( - { - age: fc.option(fc.integer({ min: 0, max: 5 }), { nil: undefined }), - email: fc.constantFrom('a', 'b', 'c', 'd', 'e', 'f'), - tags: fc.array(fc.constantFrom('x', 'y', 'z'), { maxLength: 3 }), - address: fc.option(fc.record({ city: fc.constantFrom('L', 'P') }), { nil: undefined }), - }, - { requiredKeys: [] }, - ) - const bound = fc.integer({ min: -1, max: 6 }) - const filter: fc.Arbitrary = fc.oneof( - fc.record({ age: bound }), - fc.record({ age: fc.record({ $gte: bound, $lt: bound }, { requiredKeys: [] }) }), - fc.record({ age: fc.record({ $gt: bound, $lte: bound }, { requiredKeys: [] }) }), - fc.record({ tags: fc.constantFrom('x', 'y', 'z') }), - fc.record({ tags: fc.record({ $gte: fc.constantFrom('x', 'y'), $lt: fc.constantFrom('y', 'zz') }) }), - fc.record({ 'address.city': fc.constantFrom('L', 'P'), age: fc.record({ $gt: bound }) }), - fc.record({ 'address.city': fc.constantFrom('L', 'P') }), - fc.constant({ age: null }), - fc.constant({}), - ) - const sort = fc.option(fc.record({ age: fc.constantFrom(1 as const, -1 as const) }), { nil: undefined }) - - it('returns the same results in the same order', async () => { - await fc.assert( - fc.asyncProperty( - fc.array(fc.tuple(fc.constantFrom('1', '2', '3', '4', '5'), doc), { maxLength: 12 }), - filter, - sort, - fc.option(fc.integer({ min: 1, max: 4 }), { nil: undefined }), - async (writes, f, s, limit) => { - const plain = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { warnOnFullScan: false }).child('users') - const indexed = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { - warnOnFullScan: false, - indexes: { users: userIndexes.users!.map((i) => ({ ...i, unique: false })) }, - }).child('users') - for (const [k, v] of writes) { - await plain.set(k, v) - await indexed.set(k, v) - } - const opts = { ...(s ? { sort: s } : {}), ...(limit ? { limit } : {}) } - const a = await collect(plain.find(f, opts)) - const b = await collect(indexed.find(f, opts)) - if (s) { - expect(b).toEqual(a) - return - } - // Unsorted order depends on the plan: compare as sets (a limited result is any subset of that size). - const all = new Set((await collect(plain.find(f))).map((r) => r.key)) - expect(b).toHaveLength(a.length) - for (const r of b) expect(all.has(r.key)).toBe(true) - }, - ), - { numRuns: 300 }, - ) - }) -}) diff --git a/test/documents/units.test.ts b/test/documents/units.test.ts deleted file mode 100644 index 16f5018..0000000 --- a/test/documents/units.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createIdGenerator } from '../../src/documents/ulid.js' -import { matches } from '../../src/documents/filter.js' -import { applyPatch, resolveUpdate } from '../../src/documents/update.js' - -describe('ulid', () => { - it('is strictly increasing within one millisecond and sorts by time', () => { - let t = 1_700_000_000_000 - const gen = createIdGenerator(() => t) - const ids = Array.from({ length: 1000 }, () => gen()) - expect([...ids].sort()).toEqual(ids) - expect(new Set(ids).size).toBe(1000) - t += 1 - const later = gen() - expect(later > ids.at(-1)!).toBe(true) - expect(later.slice(0, 10) > ids[0]!.slice(0, 10)).toBe(true) - }) -}) - -describe('filter semantics', () => { - it('requires one array element to satisfy all positive operators of a condition', () => { - expect(matches({ s: [1, 20] }, { s: { $gt: 5, $lt: 10 } })).toBe(false) // MongoDB would match - expect(matches({ s: [1, 7] }, { s: { $gt: 5, $lt: 10 } })).toBe(true) - expect(matches({ s: [1, 20] }, { $and: [{ s: { $gt: 5 } }, { s: { $lt: 10 } }] })).toBe(true) - }) - - it('traverses arrays of sub-documents on dot paths', () => { - expect(matches({ a: [{ b: 1 }, { b: 2 }] }, { 'a.b': 2 })).toBe(true) - expect(matches({ a: [{ b: 1 }] }, { 'a.0.b': 1 })).toBe(true) - expect(matches({ a: [{ b: 1 }] }, { 'a.b': { $exists: true } })).toBe(true) - expect(matches({ a: [] }, { 'a.b': { $exists: true } })).toBe(false) - }) - - it('treats null as matching missing', () => { - expect(matches({}, { x: null })).toBe(true) - expect(matches({ x: null }, { x: { $ne: null } })).toBe(false) - expect(matches({ x: 0 }, { x: { $ne: null } })).toBe(true) - }) - - it('rejects unknown operators', () => { - expect(() => matches({}, { x: { $regex: 'a' } })).toThrow(/Unknown operator/) - expect(() => matches({}, { $nor: [] } as never)).toThrow(/Unknown top-level/) - }) -}) - -describe('update resolution', () => { - it('$pull by sub-document filter and by condition', () => { - const state = { - items: [ - { k: 1, v: 'a' }, - { k: 2, v: 'b' }, - ], - n: [1, 5, 9], - } - const patch = resolveUpdate(state, { $pull: { items: { k: 2 }, n: { $gte: 5 } } }) - expect(applyPatch(state, patch)).toEqual({ items: [{ k: 1, v: 'a' }], n: [1] }) - }) - - it('$unset of an array position nulls it', () => { - expect(applyPatch({ a: [1, 2] }, resolveUpdate({ a: [1, 2] }, { $unset: { 'a.0': true } }))).toEqual({ a: [null, 2] }) - }) -}) diff --git a/test/event-storage/event-storage-kv.test.ts b/test/event-storage/event-storage-kv.test.ts new file mode 100644 index 0000000..4b0af71 --- /dev/null +++ b/test/event-storage/event-storage-kv.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { ConcurrencyError } from '../../src/utils/errors.js' +import { collect, KVStorageCounting } from '../helpers.js' + +function setup(kv = new KVStorageCounting()) { + let t = 1000 + return { kv, events: new EventStorageKV(kv, { now: () => t++ }) } +} + +describe('EventStorageKV', () => { + it('appends and reads in self and child forms', async () => { + const orders = setup().events.child('orders') + expect(await orders.append('o1', [{ type: 'created' }], { expectedVersion: 0 })).toEqual({ version: 1 }) + expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3 }) + expect(await collect(orders.read('o1'))).toEqual([ + { version: 1, at: 1000, data: { type: 'created' } }, + { version: 2, at: 1001, data: { type: 'paid' } }, + { version: 3, at: 1001, data: { type: 'shipped' } }, + ]) + expect(await collect(orders.child('o1').read({ from: 3 }))).toHaveLength(1) + expect(await orders.version('o1')).toBe(3) + expect(await orders.version('none')).toBe(0) + }) + + it('stores one key per commit', async () => { + const { kv, events } = setup() + await events.child('s').append(['a']) + await events.child('s').append(['b', 'c']) + expect((await collect(kv.scan())).map(([k]) => k)).toEqual(['s/~e/000000000001', 's/~e/000000000002']) + }) + + it('enforces expectedVersion and the arity rules', async () => { + const { events } = setup() + const s = events.child('s') + await s.append(['a']) + await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError) + await expect(events.append(['x'])).rejects.toThrow(/root/) + await expect(events.child('o').append(undefined as never, ['x'] as never)).rejects.toThrow(TypeError) + await expect(events.child('o').version(undefined as never)).rejects.toThrow(TypeError) + }) + + it('lists child streams', async () => { + const orders = setup().events.child('orders') + await orders.append('o1', ['a']) + await orders.append('o2', ['b']) + await orders.child('o1').append('notes', ['n']) + expect(await collect(orders.keys())).toEqual(['o1', 'o2']) + expect(await collect(orders.child('o1').keys())).toEqual(['notes']) + }) + + it('a warm append is one writeIfAbsent; a cold one adds one reverse scan', async () => { + const { kv, events } = setup() + await events.child('s').append(['a']) + kv.reset() + await events.child('s').append(['b']) + expect(kv.calls).toEqual({ writeIfAbsent: 1 }) + const cold = setup(kv).events + kv.reset() + await cold.child('s').append(['c']) + expect(kv.calls).toEqual({ scan: 1, writeIfAbsent: 1 }) + }) + + it('a writer with a stale cache collides; with "any" it retries', async () => { + const kv = new KVStorageInMemory() + const a = setup(new KVStorageCounting(kv)).events.child('s') + const b = setup(new KVStorageCounting(kv)).events.child('s') + await a.append(['a1']) + await b.append(['b1']) + await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 }) + expect(await a.append(['a3'])).toEqual({ version: 3 }) + const results = await Promise.allSettled([a.append(['x'], { expectedVersion: 3 }), b.append(['y'], { expectedVersion: 3 })]) + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) + }) + + it('reads any window correctly (property)', async () => { + await fc.assert( + fc.asyncProperty( + fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 20 }), + fc.record({ + from: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }), + to: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }), + limit: fc.option(fc.nat(10), { nil: undefined }), + }), + async (sizes, options) => { + const s = setup().events.child('s') + let n = 0 + for (const size of sizes) await s.append(Array.from({ length: size }, () => ++n)) + let expected = Array.from({ length: n }, (_, i) => i + 1).filter((v) => v >= (options.from ?? 1) && v <= (options.to ?? Infinity)) + if (options.limit !== undefined) expected = expected.slice(0, options.limit) + expect((await collect(s.read(options))).map((e) => e.data)).toEqual(expected) + }, + ), + { numRuns: 200 }, + ) + }) +}) diff --git a/test/events/kv.test.ts b/test/events/kv.test.ts deleted file mode 100644 index a1a393c..0000000 --- a/test/events/kv.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { ConcurrencyError } from '../../src/errors.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -function setup(hide: Capability[] = [], inner = new InMemoryKVDriver()) { - const driver = new CountingDriver(inner, hide) - let t = 1000 - const events = new KVEventStorage(new KVStorage(driver), { now: () => t++ }) - return { driver, events } -} - -describe('KVEventStorage', () => { - it('appends and reads with versions and metadata', async () => { - const { events } = setup() - const orders = events.child('orders') - expect(await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 })).toEqual({ version: 1, commit: 1 }) - expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3, commit: 2 }) - expect(await collect(orders.read('o1'))).toEqual([ - { version: 1, commit: 1, at: 1000, data: { type: 'created', total: 40 } }, - { version: 2, commit: 2, at: 1001, data: { type: 'paid' } }, - { version: 3, commit: 2, at: 1001, data: { type: 'shipped' } }, - ]) - expect(await orders.version('o1')).toBe(3) - expect(await orders.child('o1').version()).toBe(3) - expect(await orders.version('nope')).toBe(0) - }) - - it('stores one key per commit under /~e/', async () => { - const { events, driver } = setup() - await events.child('orders').append('o1', ['a']) - await events.child('orders').append('o1', ['b', 'c']) - expect(await collect(driver.inner.scan({ values: true }))).toEqual([ - ['orders/o1/~e/000000000001', { firstVersion: 1, at: 1000, events: ['a'] }], - ['orders/o1/~e/000000000002', { firstVersion: 2, at: 1001, events: ['b', 'c'] }], - ]) - }) - - it('enforces expectedVersion', async () => { - const { events } = setup() - const s = events.child('s') - await s.append(['a'], { expectedVersion: 0 }) - await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError) - await expect(s.append(['b'], { expectedVersion: 5 })).rejects.toMatchObject({ expectedVersion: 5, actualVersion: 1 }) - expect(await s.append(['b'], { expectedVersion: 1 })).toEqual({ version: 2, commit: 2 }) - await expect(s.append([], { expectedVersion: 1 })).rejects.toBeInstanceOf(ConcurrencyError) - expect(await s.append([])).toEqual({ version: 2, commit: 2 }) - }) - - it('applies the arity rules', async () => { - const { events } = setup() - const orders = events.child('orders') - const id: string | undefined = undefined - await expect(orders.append(id as never, ['x'] as never)).rejects.toThrow(TypeError) - await expect(orders.version(id as never)).rejects.toThrow(TypeError) - await expect(collect(orders.read(id as never))).rejects.toThrow(TypeError) - await expect(events.append(['x'])).rejects.toThrow(/root/) - await expect(orders.append('o1', 'x' as never)).rejects.toThrow(/array/) - }) - - it('lists child streams and keeps nested streams separate', async () => { - const { events } = setup() - const orders = events.child('orders') - await orders.append('o1', ['a']) - await orders.append('o2', ['b']) - await orders.child('o1').child('shipments').append('s1', ['ship']) - await orders.child('o1').append('notes', ['n']) - expect(await collect(orders.keys())).toEqual(['o1', 'o2']) - expect(await collect(orders.child('o1').keys())).toEqual(['notes', 'shipments']) - expect((await collect(orders.read('o1'))).map((e) => e.data)).toEqual(['a']) - }) -}) - -describe('read windows (property)', () => { - it('matches brute force for any commit layout', async () => { - await fc.assert( - fc.asyncProperty( - fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 25 }), - fc.record({ - from: fc.option(fc.integer({ min: -1, max: 70 }), { nil: undefined }), - to: fc.option(fc.integer({ min: 0, max: 70 }), { nil: undefined }), - reverse: fc.boolean(), - limit: fc.option(fc.integer({ min: 0, max: 10 }), { nil: undefined }), - }), - async (sizes, opts) => { - const { events } = setup() - const s = events.child('s') - const all: number[] = [] - for (const n of sizes) { - const batch = Array.from({ length: n }, (_, i) => all.length + i + 1) - await s.append(batch) - all.push(...batch) - } - let exp = all.filter((v) => v >= Math.max(1, opts.from ?? 1) && v <= (opts.to ?? Infinity)) - if (opts.reverse) exp.reverse() - if (opts.limit !== undefined) exp = exp.slice(0, opts.limit) - const got = await collect(s.read(opts)) - expect(got.map((e) => e.data)).toEqual(exp) - expect(got.map((e) => e.version)).toEqual(exp) - }, - ), - { numRuns: 300 }, - ) - }) -}) - -describe('costs', () => { - it('cached head: a warm append is a single setIfAbsent', async () => { - const { events, driver } = setup() - const s = events.child('s') - await s.append(['a']) - driver.reset() - await s.append(['b']) - await s.append(['c'], { expectedVersion: 2 }) - expect(driver.calls).toEqual({ setIfAbsent: 2 }) - }) - - it('cold head lookup is one reverse scan with limit 1', async () => { - const inner = new InMemoryKVDriver() - await setup([], inner).events.child('s').append(['a', 'b']) - const { events, driver } = setup([], inner) - expect(await events.child('s').append(['c'])).toEqual({ version: 3, commit: 2 }) - expect(driver.calls).toEqual({ scan: 1, setIfAbsent: 1 }) - }) - - it('read({ from }) finds single-event commits in one probe', async () => { - const { events, driver } = setup() - const s = events.child('s') - for (let i = 0; i < 200; i++) await s.append([i]) - driver.reset() - expect((await collect(s.read({ from: 150, limit: 1 }))).map((e) => e.version)).toEqual([150]) - expect(driver.calls).toEqual({ getMany: 1, scan: 1 }) - }) - - it('read({ from }) uses O(log commits) probes for multi-event commits', async () => { - const { events, driver } = setup() - const s = events.child('s') - for (let i = 0; i < 1000; i++) await s.append(i % 3 === 0 ? [i, i] : [i]) - driver.reset() - const [first] = await collect(s.read({ from: 777, limit: 1 })) - expect(first?.version).toBe(777) - // 1 guess probe + ceil(log8(1000)) search rounds, then the read scan. The guess hit an - // existing commit, so no head scan was needed. - expect(driver.calls.getMany).toBeLessThanOrEqual(1 + 4) - expect(driver.calls.scan).toBe(1) - }) -}) - -describe('concurrency', () => { - it('a stale writer collides with the existing commit key', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await a.append(['a1']) - await b.append(['b1']) // a's cached head is now stale - await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 }) - // 'any' retries after refreshing the head - expect(await a.append(['a3'])).toEqual({ version: 3, commit: 3 }) - expect((await collect(b.read())).map((e) => e.data)).toEqual(['a1', 'b1', 'a3']) - }) - - it('a stale cached head is refreshed before rejecting expectedVersion', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await a.append(['a1']) - await b.append(['b1']) - expect(await a.append(['a2'], { expectedVersion: 2 })).toEqual({ version: 3, commit: 3 }) - }) - - it('interleaved writers from two instances produce a dense, non-overlapping log', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await Promise.all([a.append(['a1']), b.append(['b1']), a.append(['a2', 'a3']), b.append(['b2'])]) - const log = await collect(a.read()) - expect(log.map((e) => e.version)).toEqual([1, 2, 3, 4, 5]) - expect(new Set(log.map((e) => e.data))).toEqual(new Set(['a1', 'a2', 'a3', 'b1', 'b2'])) - }) - - it('exactly one of several optimistic writers at the same version wins', async () => { - const inner = new InMemoryKVDriver() - const writers = [0, 1, 2, 3].map(() => setup([], inner).events.child('s')) - const results = await Promise.allSettled(writers.map((w, i) => w.append([i], { expectedVersion: 0 }))) - expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) - expect(results.filter((r) => r.status === 'rejected').every((r) => (r as PromiseRejectedResult).reason instanceof ConcurrencyError)).toBe(true) - }) - - it('without setIfAbsent, an in-process mutex serialises appends (single-writer mode)', async () => { - const { events, driver } = setup(['setIfAbsent']) - const s = events.child('s') - await Promise.all(Array.from({ length: 20 }, (_, i) => s.append([i]))) - expect((await collect(s.read())).map((e) => e.version)).toEqual(Array.from({ length: 20 }, (_, i) => i + 1)) - expect(driver.calls.setIfAbsent).toBeUndefined() - await expect(Promise.all([s.append(['x'], { expectedVersion: 20 }), s.append(['y'], { expectedVersion: 20 })])).rejects.toBeInstanceOf(ConcurrencyError) - }) -}) diff --git a/test/helpers.ts b/test/helpers.ts index fbb656f..0c00824 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,83 +1,45 @@ -import { BaseKVDriver, type BatchOp, type KVDriver, type ScanOptions } from '../src/kv/driver.js' -import { InMemoryKVDriver } from '../src/kv/memory.js' +import { BaseKVStorage, type ScanOptions } from '../src/kv-storage/base-kv-storage.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' -/** A minimal driver implementing only the four required methods, over a plain Map. */ -export class MinimalDriver extends BaseKVDriver { - readonly map = new Map() - async getMany(keys: string[]) { - return keys.map((k) => (this.map.has(k) ? JSON.parse(this.map.get(k)!) : undefined)) - } - async setMany(entries: [string, unknown][]) { - for (const [k, v] of entries) this.map.set(k, JSON.stringify(v)) - } - async deleteMany(keys: string[]) { - for (const k of keys) this.map.delete(k) - } - async *scan({ gte, lt, limit = Infinity, reverse, values }: ScanOptions): AsyncGenerator<[string, unknown]> { - let keys = [...this.map.keys()].filter((k) => (gte === undefined || k >= gte) && (lt === undefined || k < lt)).sort() - if (reverse) keys.reverse() - keys = keys.slice(0, limit) - for (const k of keys) { - if (!this.map.has(k)) continue - yield [k, values ? JSON.parse(this.map.get(k)!) : undefined] - } - } +export async function collect(iterable: AsyncIterable): Promise { + const out: T[] = [] + for await (const item of iterable) out.push(item) + return out } -export type Capability = 'setIfAbsent' | 'batch' | 'deleteRange' - /** - * Wraps a driver, counting calls per method and optionally hiding optional capabilities. - * `scan` counts each scan started (i.e. each seek). + * Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern + * (e.g. an encryption layer): extend BaseKVStorage and delegate the five primitives. */ -export class CountingDriver implements KVDriver { +export class KVStorageCounting extends BaseKVStorage { calls: Record = {} - setIfAbsent?: KVDriver['setIfAbsent'] - batch?: KVDriver['batch'] - deleteRange?: KVDriver['deleteRange'] - constructor( - readonly inner: KVDriver = new InMemoryKVDriver(), - hide: Capability[] = [], - ) { - if (inner.setIfAbsent && !hide.includes('setIfAbsent')) this.setIfAbsent = (e: [string, unknown][]) => (this.#count('setIfAbsent'), inner.setIfAbsent!(e)) - if (inner.batch && !hide.includes('batch')) this.batch = (ops: BatchOp[]) => (this.#count('batch'), inner.batch!(ops)) - if (inner.deleteRange && !hide.includes('deleteRange')) - this.deleteRange = (gte: string, lt: string) => (this.#count('deleteRange'), inner.deleteRange!(gte, lt)) - } - - #count(m: string): void { - this.calls[m] = (this.calls[m] ?? 0) + 1 + constructor(readonly inner: BaseKVStorage = new KVStorageInMemory()) { + super() } reset(): void { this.calls = {} } - get total(): number { - return Object.values(this.calls).reduce((a, b) => a + b, 0) + readMany(keys: string[]) { + return this.count('readMany', () => this.inner.readMany(keys)) + } + writeMany(entries: [string, unknown][]) { + return this.count('writeMany', () => this.inner.writeMany(entries)) + } + removeMany(keys: string[]) { + return this.count('removeMany', () => this.inner.removeMany(keys)) + } + writeIfAbsent(entries: [string, unknown][]) { + return this.count('writeIfAbsent', () => this.inner.writeIfAbsent(entries)) + } + scan(options?: ScanOptions) { + return this.count('scan', () => this.inner.scan(options)) } - getMany(keys: string[]) { - this.#count('getMany') - return this.inner.getMany(keys) - } - setMany(entries: [string, unknown][]) { - this.#count('setMany') - return this.inner.setMany(entries) - } - deleteMany(keys: string[]) { - this.#count('deleteMany') - return this.inner.deleteMany(keys) - } - scan(opts: ScanOptions) { - this.#count('scan') - return this.inner.scan(opts) + private count(name: string, fn: () => T): T { + this.calls[name] = (this.calls[name] ?? 0) + 1 + return fn() } } - -export async function collect(it: AsyncIterable): Promise { - const out: T[] = [] - for await (const x of it) out.push(x) - return out -} diff --git a/test/http/http.test.ts b/test/http/http.test.ts deleted file mode 100644 index 274efaf..0000000 --- a/test/http/http.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import type { KVDriver } from '../../src/kv/driver.js' -import { KVStorage } from '../../src/kv/node.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { EventSourcedDocumentStorage } from '../../src/documents/event-sourced.js' -import { createKVHttpHandler } from '../../src/http/handler.js' -import { HttpKVDriver } from '../../src/http/client.js' -import { runKVDriverConformance } from '../../src/testing/kv-conformance.js' -import { collect, CountingDriver, MinimalDriver } from '../helpers.js' -import { serve } from './node-server.js' - -/** A client wired straight to a handler, no socket. */ -function inProcess(server: KVDriver = new InMemoryKVDriver(), caps?: ConstructorParameters[0]['capabilities']) { - const handler = createKVHttpHandler(server, { basePath: '/kv' }) - return new HttpKVDriver({ url: 'http://test/kv', capabilities: caps, fetch: async (input, init) => handler(new Request(input, init)) }) -} - -runKVDriverConformance('HttpKVDriver (in-process)', () => inProcess()) -runKVDriverConformance('HttpKVDriver (minimal server driver, discovered capabilities)', () => { - const handler = createKVHttpHandler(new MinimalDriver()) - return HttpKVDriver.connect({ url: 'http://test', fetch: async (input, init) => handler(new Request(input, init)) }) -}) - -describe('over a real socket', () => { - let server: Awaited> - const backends: InMemoryKVDriver[] = [] - beforeAll(async () => { - // One server; each driver gets a fresh backend selected by path. - server = await serve(async (req) => { - const m = /^\/db(\d+)/.exec(new URL(req.url).pathname) - const backend = backends[Number(m?.[1])] - if (!backend) return new Response('no db', { status: 404 }) - return createKVHttpHandler(backend, { basePath: `/db${m![1]}` })(req) - }) - }) - afterAll(() => server.close()) - - const fresh = () => { - backends.push(new InMemoryKVDriver()) - return HttpKVDriver.connect({ url: `${server.url}/db${backends.length - 1}` }) - } - - runKVDriverConformance('HttpKVDriver (node:http)', fresh) - - it('cancels a server-side scan when the client stops early', async () => { - const d = await fresh() - await d.setMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number])) - const first: string[] = [] - for await (const [k] of d.scan({})) { - first.push(k) - if (first.length === 3) break - } - expect(first).toEqual(['k00000', 'k00001', 'k00002']) - }) - - it('runs the whole stack (KV indexes, events, documents) over HTTP', async () => { - const kv = new KVStorage(await fresh(), { indexes: { users: { byAge: (u: { age: number }) => u.age } } }) - await kv.child('users').set('1', { age: 30 }) - expect(await collect(kv.child('users').findBy('byAge', { gte: 18 }))).toEqual([['1', { age: 30 }]]) - const docs = new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: { people: [{ fields: ['email'], unique: true }] } }) - const id = await docs.child('people').insert({ email: 'a@x', n: 1 }) - await docs.child('people').update(id, { $inc: { n: 1 } }) - await expect(docs.child('people').insert({ email: 'a@x' })).rejects.toThrow(/Unique/) - expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } }) - }) -}) - -describe('handler', () => { - it('reports capabilities and rejects unsupported operations with 501', async () => { - const handler = createKVHttpHandler(new MinimalDriver()) - const caps = await (await handler(new Request('http://x/capabilities'))).json() - expect(caps).toEqual({ setIfAbsent: false, batch: false, deleteRange: true }) - const res = await handler(new Request('http://x/batch', { method: 'POST', body: JSON.stringify({ ops: [] }) })) - expect(res.status).toBe(501) - }) - - it('maps driver errors to 500 and unknown routes to 404', async () => { - const broken = new CountingDriver(new InMemoryKVDriver()) - broken.getMany = () => Promise.reject(new Error('boom')) - const handler = createKVHttpHandler(broken) - expect((await handler(new Request('http://x/getMany', { method: 'POST', body: '{"keys":["a"]}' }))).status).toBe(500) - expect((await handler(new Request('http://x/nope'))).status).toBe(404) - await expect(inProcess(new MinimalDriver(), { batch: true }).batch!([])).rejects.toThrow(/HTTP 501/) - }) - - it('distinguishes null values from missing keys', async () => { - const d = inProcess() - await d.setMany([['n', null]]) - expect(await d.getMany(['n', 'missing'])).toEqual([null, undefined]) - }) -}) diff --git a/test/http/node-server.ts b/test/http/node-server.ts deleted file mode 100644 index 05b1551..0000000 --- a/test/http/node-server.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { createServer, type IncomingHttpHeaders } from 'node:http' -import type { AddressInfo } from 'node:net' - -/** Serves a `Request → Response` handler over a real socket (test helper). */ -export async function serve(handler: (req: Request) => Promise): Promise<{ url: string; close: () => Promise }> { - const server = createServer(async (req, res) => { - const chunks: Buffer[] = [] - for await (const c of req) chunks.push(c as Buffer) - const headers = Object.entries(req.headers as IncomingHttpHeaders).flatMap(([k, v]) => (v === undefined ? [] : [[k, String(v)] as [string, string]])) - const request = new Request(`http://localhost${req.url}`, { - method: req.method, - headers, - body: req.method === 'GET' || req.method === 'HEAD' ? undefined : Buffer.concat(chunks), - }) - const response = await handler(request) - res.writeHead(response.status, Object.fromEntries(response.headers)) - if (!response.body) return void res.end() - const reader = response.body.getReader() - res.on('close', () => void reader.cancel().catch(() => {})) - try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - res.write(value) - } - } finally { - res.end() - } - }) - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - const { port } = server.address() as AddressInfo - return { - url: `http://127.0.0.1:${port}`, - close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))), - } -} diff --git a/test/kv-storage/kv-storage-http.test.ts b/test/kv-storage/kv-storage-http.test.ts new file mode 100644 index 0000000..8a5a3b9 --- /dev/null +++ b/test/kv-storage/kv-storage-http.test.ts @@ -0,0 +1,54 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js' +import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' +import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' +import { serve } from './node-server.js' + +runKVStorageConformance('KVStorageHttp (in-process)', () => { + const server = new KVStorageHttpServer(new KVStorageInMemory(), '/kv') + return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) }) +}) + +describe('over a real socket', () => { + let server: Awaited> + const stores: KVStorageHttpServer[] = [] + beforeAll(async () => { + // One socket; each test storage gets its own backend under /. + server = await serve((request) => stores[Number(new URL(request.url).pathname.split('/')[1])]!.handle(request)) + }) + afterAll(() => server.close()) + const fresh = () => { + stores.push(new KVStorageHttpServer(new KVStorageInMemory(), `/${stores.length}`)) + return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` }) + } + + runKVStorageConformance('KVStorageHttp (node:http)', fresh) + + it('stops a long scan early', async () => { + const db = fresh() + await db.writeMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number])) + const seen: string[] = [] + for await (const [key] of db.scan()) if (seen.push(key) === 3) break + expect(seen).toEqual(['k00000', 'k00001', 'k00002']) + }) + + it('runs events and event-sourced documents over HTTP', async () => { + const kv = fresh() + const docs = new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))) + const id = await docs.child('people').insert({ email: 'a@x', n: 1 }) + await docs.child('people').update(id, { $inc: { n: 1 } }) + expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } }) + }) +}) + +describe('server', () => { + it('maps errors to 404 / 500', async () => { + const server = new KVStorageHttpServer(new KVStorageInMemory()) + expect((await server.handle(new Request('http://x/nope', { method: 'POST', body: '{}' }))).status).toBe(404) + expect((await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys": 5}' }))).status).toBe(500) + }) +}) diff --git a/test/kv-storage/kv-storage-in-memory.test.ts b/test/kv-storage/kv-storage-in-memory.test.ts new file mode 100644 index 0000000..68c09a5 --- /dev/null +++ b/test/kv-storage/kv-storage-in-memory.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js' +import { collect, KVStorageCounting } from '../helpers.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' + +runKVStorageConformance('KVStorageInMemory', () => new KVStorageInMemory()) +runKVStorageConformance('wrapper (KVStorageCounting)', () => new KVStorageCounting()) + +describe('costs', () => { + it('single-key operations are one primitive call', async () => { + const db = new KVStorageCounting() + const users = db.child('users') + await users.set('1', 'x') + await users.get('1') + await users.child('1').get() + await users.delete('1') + expect(db.calls).toEqual({ writeMany: 1, readMany: 2, removeMany: 1 }) + }) + + it('keys() seeks once per child with descendants, not once per key', async () => { + const db = new KVStorageCounting() + const writes: [string, number][] = [] + for (let c = 0; c < 50; c++) for (let d = 0; d < 20; d++) writes.push([`n/c${c}/d${d}`, d]) + for (let c = 0; c < 50; c++) writes.push([`n/leaf${c}`, c]) + await db.writeMany(writes) + db.reset() + expect(await collect(db.child('n').keys())).toHaveLength(100) + expect(db.calls).toEqual({ scan: 51 }) + }) +}) + +describe('model-based: random writes vs a reference map', () => { + const segment = fc.constantFrom('a', 'a-1', 'a.b', 'b', 'x/y', '%', '~e', '1', '10', '9', 'ab', 'a!', 'é') + const path = fc.array(segment, { minLength: 1, maxLength: 3 }) + const op = fc.oneof( + fc.record({ t: fc.constant('set' as const), path, value: fc.integer() }), + fc.record({ t: fc.constant('delete' as const), path }), + fc.record({ t: fc.constant('deleteTree' as const), path }), + ) + const at = (db: BaseKVStorage, p: string[]) => p.reduce((n, s) => n.child(s), db) + + it('get, keys and entries agree with the model', async () => { + await fc.assert( + fc.asyncProperty(fc.array(op, { maxLength: 30 }), fc.array(segment, { maxLength: 2 }), async (ops, parent) => { + const db = new KVStorageInMemory() + const model = new Map() + const isUnder = (p: string[], q: string[]) => q.length >= p.length && p.every((s, i) => q[i] === s) + for (const o of ops) { + if (o.t === 'set') (await at(db, o.path).set(o.value), model.set(JSON.stringify(o.path), o.value)) + else if (o.t === 'delete') (await at(db, o.path).delete(), model.delete(JSON.stringify(o.path))) + else { + await at(db, o.path).deleteTree() + for (const k of [...model.keys()]) if (isUnder(o.path, JSON.parse(k))) model.delete(k) + } + } + for (const [k, v] of model) expect(await at(db, JSON.parse(k)).get()).toBe(v) + const children = new Map() + for (const [k, v] of model) { + const p = JSON.parse(k) as string[] + if (p.length <= parent.length || !isUnder(parent, p)) continue + const own = p.length === parent.length + 1 + if (own || !children.has(p[parent.length]!)) children.set(p[parent.length]!, own ? v : children.get(p[parent.length]!)) + } + const node = at(db, parent) + const listed = await collect(node.keys()) + expect([...listed].sort()).toEqual([...children.keys()].sort()) + expect(await collect(node.keys({ reverse: true }))).toEqual([...listed].reverse()) + const withValues = listed.filter((k) => children.get(k) !== undefined) + expect(await collect(node.entries())).toEqual(withValues.map((k) => [k, children.get(k)])) + expect(await collect(node.entries({ reverse: true }))).toEqual(withValues.reverse().map((k) => [k, children.get(k)])) + for (let i = 0; i < listed.length; i++) expect(await collect(node.keys({ after: listed[i]! }))).toEqual(listed.slice(i + 1)) + }), + { numRuns: 300 }, + ) + }) +}) diff --git a/test/kv-storage/kv-storage.conformance.ts b/test/kv-storage/kv-storage.conformance.ts new file mode 100644 index 0000000..939afe4 --- /dev/null +++ b/test/kv-storage/kv-storage.conformance.ts @@ -0,0 +1,168 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import type { BaseKVStorage, ScanOptions } from '../../src/kv-storage/base-kv-storage.js' +import { InvalidKeyError } from '../../src/utils/errors.js' +import { collect } from '../helpers.js' + +/** Behaviour every KV storage must have. `make` returns an empty storage. */ +export function runKVStorageConformance(name: string, make: () => BaseKVStorage | Promise): void { + describe(`KV storage conformance: ${name}`, () => { + let db: BaseKVStorage + beforeEach(async () => { + db = await make() + }) + const scanKeys = async (options: ScanOptions) => (await collect(db.scan(options))).map(([k]) => k) + + describe('primitives', () => { + beforeEach(async () => { + await db.writeMany([ + ['b', 2], + ['a/2', { n: 'a2' }], + ['c', [3]], + ['a', 1], + ['a/1', 'a1'], + ['a0', null], + ['10', 10], + ['9', 9], + ]) + }) + + it('reads in request order with undefined for missing keys and null preserved', async () => { + expect(await db.readMany(['c', 'missing', 'a/2', 'a0'])).toEqual([[3], undefined, { n: 'a2' }, null]) + }) + + it('scans in key order with bounds, reverse, limit and optional values', async () => { + expect(await scanKeys({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c']) + expect(await scanKeys({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2']) + expect(await scanKeys({ reverse: true, gte: 'a', lt: 'b' })).toEqual(['a0', 'a/2', 'a/1', 'a']) + expect(await scanKeys({ limit: 2, reverse: true })).toEqual(['c', 'b']) + expect(await scanKeys({ limit: 0 })).toEqual([]) + expect(await collect(db.scan({ gte: 'a/', lt: 'a0', values: true }))).toEqual([ + ['a/1', 'a1'], + ['a/2', { n: 'a2' }], + ]) + expect((await collect(db.scan({ gte: 'b' })))[0]).toEqual(['b', undefined]) + }) + + it('removes keys', async () => { + await db.removeMany(['b', 'missing']) + expect(await db.readMany(['b', 'c'])).toEqual([undefined, [3]]) + }) + + it('writeIfAbsent writes only absent keys, and only one concurrent claim wins', async () => { + expect( + await db.writeIfAbsent([ + ['a', 'x'], + ['new', 'y'], + ]), + ).toEqual([false, true]) + expect(await db.readMany(['a', 'new'])).toEqual([1, 'y']) + const results = await Promise.all([1, 2, 3].map((n) => db.writeIfAbsent([['claim', n]]))) + expect(results.flat().filter(Boolean)).toHaveLength(1) + }) + + it('keeps scanning correctly when keys change mid-scan', async () => { + // Keys removed mid-scan may or may not still appear (stores may read ahead), but every key + // present throughout must appear exactly once, in order. + const seen: string[] = [] + for await (const [key] of db.scan({})) { + seen.push(key) + if (key === 'a') await db.removeMany(['a', 'a/1']) + } + expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c']) + }) + + it('does not share references with callers', async () => { + const [value] = (await db.readMany(['a/2'])) as [{ n: string }] + value.n = 'changed' + expect(await db.readMany(['a/2'])).toEqual([{ n: 'a2' }]) + }) + }) + + describe('nodes', () => { + it('reads and writes own and child values at the same key', async () => { + const users = db.child('users') + await users.set('123', { name: 'Ada' }) + expect(await users.child('123').get()).toEqual({ name: 'Ada' }) + await users.child('123').set({ name: 'Bob' }) + expect(await users.get('123')).toEqual({ name: 'Bob' }) + expect(await db.readMany(['users/123'])).toEqual([{ name: 'Bob' }]) + await db.child('n').set(5, 'five') + expect(await db.child('n').get('5')).toBe('five') + }) + + it('batch reads, writes and deletes children', async () => { + const n = db.child('n') + await n.setMany([ + ['a', 1], + ['b', 2], + ]) + expect(await n.getMany(['b', 'x', 'a'])).toEqual([2, undefined, 1]) + await n.deleteMany(['a']) + expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2]) + }) + + it('keeps own values and children independent', async () => { + const user = db.child('users').child('1') + await user.set('own') + await user.set('name', 'Ada') + await user.delete() + expect(await user.get()).toBeUndefined() + expect(await user.get('name')).toBe('Ada') + }) + + it('rejects passed-but-undefined keys, root values and undefined values', async () => { + const users = db.child('users') + const id: string | undefined = undefined + await expect(users.get(id as never)).rejects.toThrow(InvalidKeyError) + await expect(users.delete(null as never)).rejects.toThrow(TypeError) + await expect(users.set(id as never, 1 as never)).rejects.toThrow(TypeError) + await expect(users.set('x', undefined)).rejects.toThrow(/undefined/) + await expect(db.get()).rejects.toThrow(/root/) + expect(() => db.child('')).toThrow(InvalidKeyError) + }) + + it('escapes keys so special characters round-trip', async () => { + const n = db.child('a/b') + for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) await n.set(key, key) + for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) expect(await n.get(key)).toBe(key) + expect((await collect(n.keys())).sort()).toEqual(['50%', 'a b', 'a-1', 'a.b', 'x/y', '~e', 'é'].sort()) + }) + + it('lists direct children (including ones with only descendants), skipping reserved keys', async () => { + const n = db.child('n') + await n.set('a', 1) + await n.child('b').set('deep', 1) + await n.set('c', 3) + await n.child('c').child('x').set('y', 1) + await n.set('a-1', 4) + await db.writeMany([['n/~e/000000000001', {}]]) + expect(await collect(n.keys())).toEqual(['a', 'a-1', 'b', 'c']) + expect(await collect(n.keys({ reverse: true }))).toEqual(['c', 'b', 'a-1', 'a']) + expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['a-1', 'b']) + expect(await collect(n.keys({ after: 'c', reverse: true }))).toEqual(['b', 'a-1', 'a']) + expect(await collect(n.entries())).toEqual([ + ['a', 1], + ['a-1', 4], + ['c', 3], + ]) + expect(await collect(n.entries({ reverse: true, limit: 2 }))).toEqual([ + ['c', 3], + ['a-1', 4], + ]) + expect(await collect(db.keys())).toEqual(['n']) + }) + + it('deleteTree removes a subtree but not lookalike siblings', async () => { + const n = db.child('n') + await n.set('a', 1) + await n.child('a').child('b').set('c', 3) + await n.set('a!', 2) + await n.set('a0', 3) + await n.deleteTree('a') + expect(await collect(n.keys())).toEqual(['a0', 'a!']) // 'a!' is stored as 'a~21' + await db.deleteTree() + expect(await collect(db.scan())).toEqual([]) + }) + }) + }) +} diff --git a/test/kv-storage/node-server.ts b/test/kv-storage/node-server.ts new file mode 100644 index 0000000..8660188 --- /dev/null +++ b/test/kv-storage/node-server.ts @@ -0,0 +1,24 @@ +import { createServer } from 'node:http' +import type { AddressInfo } from 'node:net' + +/** Serves a `Request → Response` handler over a real socket (test helper). */ +export async function serve(handle: (request: Request) => Promise): Promise<{ url: string; close: () => Promise }> { + const server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(chunk as Buffer) + const response = await handle( + new Request(`http://localhost${req.url}`, { method: req.method, body: req.method === 'POST' ? Buffer.concat(chunks) : undefined }), + ) + res.writeHead(response.status, Object.fromEntries(response.headers)) + if (!response.body) return void res.end() + const reader = response.body.getReader() + res.on('close', () => void reader.cancel().catch(() => {})) + for (let next = await reader.read(); !next.done; next = await reader.read()) res.write(next.value) + res.end() + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))), + } +} diff --git a/test/kv/indexes.test.ts b/test/kv/indexes.test.ts deleted file mode 100644 index ee57e37..0000000 --- a/test/kv/indexes.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage, type KVStorageOptions } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import type { KVDriver } from '../../src/kv/driver.js' -import { encodeIndexValue } from '../../src/core/encoding.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -type User = { email?: string; age?: number; tags?: string[] } -type Post = { createdAt: string; tags: string[] } - -const indexes: KVStorageOptions['indexes'] = { - users: { - byEmail: (u: User) => u.email, - byAge: (u: User) => u.age, - byTag: (u: User) => u.tags, - }, - 'users/*/posts': { byDate: (p: Post) => p.createdAt, byTag: (p: Post) => p.tags }, -} - -function setup(hide: Capability[] = []) { - const driver = new CountingDriver(new InMemoryKVDriver(), hide) - return { driver, db: new KVStorage(driver, { indexes }) } -} - -const modes: [string, Capability[]][] = [ - ['atomic batch', []], - ['non-atomic', ['batch']], -] - -describe.each(modes)('indexes (%s)', (_, hide) => { - it('writes entries under /~i///', async () => { - const { db, driver } = setup(hide) - await db.child('users').set('123', { email: 'sada@x.io', age: 30 }) - const keys = (await collect(driver.inner.scan({}))).map(([k]) => k) - expect(keys).toEqual([`users/123`, `users/~i/byAge/${encodeIndexValue(30)}/123`, `users/~i/byEmail/${encodeIndexValue('sada@x.io')}/123`]) - }) - - it('finds by exact value and keeps entries in sync on update and delete', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('1', { email: 'a@x', age: 30 }) - await users.set('2', { email: 'b@x', age: 30 }) - expect(await collect(users.findBy('byEmail', 'a@x'))).toEqual([['1', { email: 'a@x', age: 30 }]]) - expect((await collect(users.findBy('byAge', 30))).map(([k]) => k)).toEqual(['1', '2']) - await users.set('1', { email: 'c@x', age: 31 }) - expect(await collect(users.findBy('byEmail', 'a@x'))).toEqual([]) - expect((await collect(users.findBy('byEmail', 'c@x'))).map(([k]) => k)).toEqual(['1']) - await users.delete('2') - expect(await collect(users.findBy('byAge', 30))).toEqual([]) - }) - - it('self-form writes on a child update the parent index', async () => { - const { db } = setup(hide) - const u = db.child('users').child('7') - await u.set({ email: 'self@x' }) - expect((await collect(db.child('users').findBy('byEmail', 'self@x'))).map(([k]) => k)).toEqual(['7']) - await u.delete() - expect(await collect(db.child('users').findBy('byEmail', 'self@x'))).toEqual([]) - }) - - it('deleteTree removes the child entry', async () => { - const { db, driver } = setup(hide) - await db.child('users').set('7', { email: 'e' }) - await db.child('users').child('7').child('posts').set('p', { createdAt: '2024', tags: [] }) - await db.child('users').deleteTree('7') - expect(await collect(driver.inner.scan({}))).toEqual([]) - }) - - it('supports multi-entry values, de-duplicated in range queries', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('1', { tags: ['a', 'b', 'a'] }) - await users.set('2', { tags: ['b'] }) - expect((await collect(users.findBy('byTag', 'b'))).map(([k]) => k)).toEqual(['1', '2']) - expect((await collect(users.findBy('byTag', { gte: 'a', lt: 'c' }))).map(([k]) => k)).toEqual(['1', '2']) - await users.set('1', { tags: ['c'] }) - expect((await collect(users.findBy('byTag', 'b'))).map(([k]) => k)).toEqual(['2']) - }) - - it('range queries are ordered by value and honour bounds, limit and reverse', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.setMany([ - ['a', { age: 10 }], - ['b', { age: 9 }], - ['c', { age: -1.5 }], - ['d', { age: 100 }], - ['e', { age: 20 }], - ]) - const ages = async (q: object, o = {}) => (await collect(users.findBy('byAge', q as never, o))).map(([, v]) => (v as User).age) - expect(await ages({})).toEqual([-1.5, 9, 10, 20, 100]) - expect(await ages({ gte: 9, lt: 20 })).toEqual([9, 10]) - expect(await ages({ gt: 9, lte: 20 })).toEqual([10, 20]) - expect(await ages({ gte: 0 }, { limit: 2 })).toEqual([9, 10]) - expect(await ages({ lt: 100 }, { reverse: true })).toEqual([20, 10, 9, -1.5]) - }) - - it('applies pattern indexes with * to nested nodes only', async () => { - const { db } = setup(hide) - const posts = db.child('users').child('u1').child('posts') - await posts.set('p1', { createdAt: '2024-02', tags: ['x'] }) - await posts.set('p2', { createdAt: '2024-01', tags: ['y'] }) - expect((await collect(posts.findBy('byDate', { gte: '2024' }))).map(([k]) => k)).toEqual(['p2', 'p1']) - await expect(db.child('other').findBy('byDate', 'x').next()).rejects.toThrow(/No index/) - }) - - it('handles child keys and values needing escaping', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('a/b%~', { email: 'we/ird%~@x' }) - expect(await collect(users.findBy('byEmail', 'we/ird%~@x'))).toEqual([['a/b%~', { email: 'we/ird%~@x' }]]) - }) -}) - -describe('index costs', () => { - it('non-indexed nodes do not read before writing', async () => { - const { db, driver } = setup() - driver.reset() - await db.child('other').set('1', { email: 'x' }) - await db.child('other').setMany([['2', {}]]) - expect(driver.calls).toEqual({ setMany: 2 }) - }) - - it('indexed writes: one getMany then one atomic batch', async () => { - const { db, driver } = setup() - driver.reset() - await db.child('users').setMany([ - ['1', { email: 'a' }], - ['2', { email: 'b' }], - ]) - expect(driver.calls).toEqual({ getMany: 1, batch: 1 }) - }) - - it('findBy with a small limit makes one value read', async () => { - const { db, driver } = setup() - await db.child('users').setMany(Array.from({ length: 300 }, (_, i) => [String(i), { age: 1 }] as [string, User])) - driver.reset() - expect(await collect(db.child('users').findBy('byAge', 1, { limit: 1 }))).toHaveLength(1) - expect(driver.calls).toEqual({ scan: 1, getMany: 1 }) - driver.reset() - expect(await collect(db.child('users').findBy('byAge', 1))).toHaveLength(300) - expect(driver.calls).toEqual({ scan: 1, getMany: 3 }) - }) -}) - -describe('non-atomic crash recovery (read-repair)', () => { - /** Fails the n-th mutating call. */ - class CrashingDriver extends CountingDriver { - failAt = Infinity - #n = 0 - override setMany(entries: [string, unknown][]) { - if (++this.#n === this.failAt) return Promise.reject(new Error('crash')) - return super.setMany(entries) - } - override deleteMany(keys: string[]) { - if (++this.#n === this.failAt) return Promise.reject(new Error('crash')) - return super.deleteMany(keys) - } - } - - it('skips and deletes stale entries left by a crash after the index add', async () => { - const driver = new CrashingDriver(new InMemoryKVDriver(), ['batch']) - const users = new KVStorage(driver, { indexes }).child('users') - await users.set('1', { email: 'old' }) - driver.failAt = 2 + 2 // adds succeed, value write crashes - await expect(users.set('1', { email: 'new' })).rejects.toThrow('crash') - // The 'new' entry is a dangling hint; the value still says 'old'. - expect(await collect(users.findBy('byEmail', 'new'))).toEqual([]) - expect(await collect(users.findBy('byEmail', 'old'))).toEqual([['1', { email: 'old' }]]) - const entries = (await collect(driver.inner.scan({ gte: 'users/~i/' }))).map(([k]) => k) - expect(entries).toEqual([`users/~i/byEmail/${encodeIndexValue('old')}/1`]) - }) - - it('skips stale entries left by a crash before the index remove', async () => { - const driver = new CrashingDriver(new InMemoryKVDriver(), ['batch']) - const users = new KVStorage(driver, { indexes }).child('users') - await users.set('1', { email: 'old' }) - driver.failAt = 2 + 3 // adds + value succeed, remove crashes - await expect(users.set('1', { email: 'new' })).rejects.toThrow('crash') - expect(await collect(users.findBy('byEmail', 'old'))).toEqual([]) - expect(await collect(users.findBy('byEmail', 'new'))).toEqual([['1', { email: 'new' }]]) - }) - - it('can leave stale entries in place with readRepair: false', async () => { - const driver = new InMemoryKVDriver() - await driver.setMany([[`users/~i/byEmail/${encodeIndexValue('x')}/1`, true]]) - const users = new KVStorage(driver, { indexes, readRepair: false }).child('users') - expect(await collect(users.findBy('byEmail', 'x'))).toEqual([]) - expect(await collect(driver.scan({}))).toHaveLength(1) - }) -}) - -describe('index declarations', () => { - it('rejects the same index name on overlapping patterns', () => { - expect(() => new KVStorage(new InMemoryKVDriver(), { indexes: { 'a/*': { i: () => 1 }, '*/b': { i: () => 2 } } })).toThrow(/overlapping/) - expect(() => new KVStorage(new InMemoryKVDriver(), { indexes: { 'a/*': { i: () => 1 }, '*/b': { j: () => 2 } } })).not.toThrow() - }) - - it('rejects non-indexable index function results', async () => { - const db = new KVStorage(new InMemoryKVDriver(), { indexes: { '': { bad: () => ({}) as never } } }) - await expect(db.set('x', 1)).rejects.toThrow(/non-indexable/) - }) -}) - -describe('property: findBy equals brute-force filter', () => { - const val = fc.record({ - age: fc.option(fc.integer({ min: -3, max: 3 }), { nil: undefined }), - tags: fc.array(fc.constantFrom('a', 'b', 'c/d'), { maxLength: 3 }), - }) - const op = fc.oneof( - fc.record({ t: fc.constant('set' as const), k: fc.constantFrom('1', '2', '3', 'x/y'), v: val }), - fc.record({ t: fc.constant('del' as const), k: fc.constantFrom('1', '2', '3', 'x/y') }), - ) - for (const [mode, hide] of modes) { - it(mode, async () => { - await fc.assert( - fc.asyncProperty(fc.array(op, { maxLength: 20 }), fc.integer({ min: -4, max: 4 }), fc.integer({ min: -4, max: 4 }), async (ops, lo, hi) => { - const driver: KVDriver = new CountingDriver(new InMemoryKVDriver(), hide) - const users = new KVStorage(driver, { indexes }).child('users') - const model = new Map() - for (const o of ops) { - if (o.t === 'set') { - await users.set(o.k, o.v) - model.set(o.k, o.v) - } else { - await users.delete(o.k) - model.delete(o.k) - } - } - const got = (await collect(users.findBy('byAge', { gte: lo, lt: hi }))).map(([k]) => k).sort() - const exp = [...model] - .filter(([, v]) => v.age !== undefined && v.age >= lo && v.age < hi) - .map(([k]) => k) - .sort() - expect(got).toEqual(exp) - for (const tag of ['a', 'b', 'c/d']) { - const g = (await collect(users.findBy('byTag', tag))).map(([k]) => k).sort() - expect(g).toEqual( - [...model] - .filter(([, v]) => v.tags?.includes(tag)) - .map(([k]) => k) - .sort(), - ) - } - // No stale entries are ever left behind by clean writes. - let entryCount = 0 - for await (const _ of driver.scan({ gte: 'users/~i/', lt: 'users/~i0' })) entryCount++ - const expectedEntries = [...model.values()].reduce((n, v) => n + (v.age !== undefined ? 1 : 0) + new Set(v.tags).size, 0) - expect(entryCount).toBe(expectedEntries) - }), - { numRuns: 200 }, - ) - }) - } -}) diff --git a/test/kv/memory-driver.test.ts b/test/kv/memory-driver.test.ts deleted file mode 100644 index df7fe71..0000000 --- a/test/kv/memory-driver.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { runKVDriverConformance } from '../../src/testing/kv-conformance.js' -import { CountingDriver, MinimalDriver } from '../helpers.js' - -runKVDriverConformance('InMemoryKVDriver', () => new InMemoryKVDriver()) -runKVDriverConformance('BaseKVDriver (minimal)', () => new MinimalDriver()) -runKVDriverConformance('CountingDriver(InMemory)', () => new CountingDriver()) - -describe('InMemoryKVDriver', () => { - it('keeps order after large bulk merges', async () => { - const d = new InMemoryKVDriver() - const keys = Array.from({ length: 5000 }, (_, i) => `k${(i * 7919) % 5000}`) - await d.setMany(keys.slice(0, 2500).map((k) => [k, 1])) - await d.setMany(keys.slice(2500).map((k) => [k, 2])) - const seen: string[] = [] - for await (const [k] of d.scan({})) seen.push(k) - expect(seen).toEqual([...new Set(keys)].sort()) - expect(d.size).toBe(5000) - }) - - it('can skip cloning', async () => { - const d = new InMemoryKVDriver({ clone: false }) - const v = { n: 1 } - await d.setMany([['k', v]]) - expect((await d.getMany(['k']))[0]).toBe(v) - }) -}) diff --git a/test/kv/node.test.ts b/test/kv/node.test.ts deleted file mode 100644 index 52af85f..0000000 --- a/test/kv/node.test.ts +++ /dev/null @@ -1,354 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { escapeSegment } from '../../src/core/path.js' -import { InvalidKeyError } from '../../src/errors.js' -import { collect, CountingDriver, MinimalDriver } from '../helpers.js' - -function setup() { - const driver = new CountingDriver() - const db = new KVStorage(driver) - return { driver, db } -} - -describe('KVNode basics', () => { - it('reads and writes own and child values at the same key', async () => { - const { db } = setup() - const users = db.child('users') - await users.set('123', { name: 'Ada' }) - expect(await users.get('123')).toEqual({ name: 'Ada' }) - expect(await users.child('123').get()).toEqual({ name: 'Ada' }) - await users.child('123').set({ name: 'Bob' }) - expect(await users.get('123')).toEqual({ name: 'Bob' }) - expect(users.child('123').selfKey).toBe('users/123') - }) - - it('treats number keys as strings', async () => { - const { db } = setup() - await db.child('n').set(5, 'five') - expect(await db.child('n').get('5')).toBe('five') - }) - - it('escapes keys containing the separator, % and leading ~', async () => { - const { db, driver } = setup() - const n = db.child('a/b') - await n.set('~x', 1) - await n.set('50%', 2) - expect(await collect(driver.inner.scan({}))).toEqual([ - ['a%2Fb/%7Ex', undefined], - ['a%2Fb/50%25', undefined], - ]) - // Ordered by escaped segment: '%7Ex' < '50%25'. - expect(await collect(n.keys())).toEqual(['~x', '50%']) - }) - - it('own value and children are independent keys', async () => { - const { db } = setup() - const u = db.child('users').child('1') - await u.set('own') - await u.set('name', 'Ada') - await u.delete() - expect(await u.get()).toBeUndefined() - expect(await u.get('name')).toBe('Ada') - }) - - it('getMany / setMany / deleteMany', async () => { - const { db, driver } = setup() - const n = db.child('n') - driver.reset() - await n.setMany([ - ['a', 1], - ['b', 2], - ]) - expect(await n.getMany(['b', 'x', 'a'])).toEqual([2, undefined, 1]) - await n.deleteMany(['a', 'x']) - expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2]) - expect(driver.calls).toEqual({ setMany: 1, getMany: 2, deleteMany: 1 }) - }) - - it('single-key ops make exactly one driver call', async () => { - const { db, driver } = setup() - const n = db.child('users') - driver.reset() - await n.set('1', 'x') - await n.get('1') - await n.child('1').get() - await n.delete('1') - expect(driver.calls).toEqual({ setMany: 1, getMany: 2, deleteMany: 1 }) - }) - - it('rejects setting undefined', async () => { - const { db } = setup() - await expect(db.child('a').set('b', undefined)).rejects.toThrow(TypeError) - }) -}) - -describe('arity dispatch', () => { - it('throws on passed-but-undefined or null keys instead of reading self', async () => { - const { db } = setup() - const users = db.child('users') - const maybe: string | undefined = undefined - await expect(users.get(maybe as never)).rejects.toThrow(InvalidKeyError) - await expect(users.get(null as never)).rejects.toThrow(TypeError) - await expect(users.set(maybe as never, 1 as never)).rejects.toThrow(TypeError) - await expect(users.delete(maybe as never)).rejects.toThrow(TypeError) - await expect(users.getMany([maybe as never])).rejects.toThrow(TypeError) - await expect(users.deleteTree(maybe as never)).rejects.toThrow(TypeError) - expect(() => users.child(maybe as never)).toThrow(TypeError) - }) - - it('root has no own value', async () => { - const { db } = setup() - await expect(db.get()).rejects.toThrow(TypeError) - await expect(db.set(1)).rejects.toThrow(TypeError) - await expect(db.delete()).rejects.toThrow(TypeError) - }) - - it('rejects empty keys', () => { - const { db } = setup() - expect(() => db.child('')).toThrow(InvalidKeyError) - }) -}) - -describe('keys / entries', () => { - it('lists direct children, including ones with only descendants, skipping reserved', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.child('b').set('deep', 1) - await n.set('c', 3) - await n.child('c').child('x').set('y', 1) - await db.driver.setMany([['n/~i/idx/v/a', true]]) - await n.set('\u{1F600}', 'emoji') - expect(await collect(n.keys())).toEqual(['a', 'b', 'c', '\u{1F600}']) - expect(await collect(n.keys({ reverse: true }))).toEqual(['\u{1F600}', 'c', 'b', 'a']) - expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['b', 'c']) - expect(await collect(n.keys({ after: 'c', reverse: true }))).toEqual(['b', 'a']) - expect(await collect(n.entries())).toEqual([ - ['a', 1], - ['c', 3], - ['\u{1F600}', 'emoji'], - ]) - expect(await collect(n.entries({ reverse: true, limit: 2 }))).toEqual([ - ['\u{1F600}', 'emoji'], - ['c', 3], - ]) - }) - - it('root keys skip the reserved ~p/~e roots', async () => { - const { db } = setup() - await db.driver.setMany([ - ['~p/users/1', {}], - ['users/1', {}], - ]) - expect(await collect(db.keys())).toEqual(['users']) - }) - - it('orders siblings that extend another with a below-separator char first', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.set('a-1', 2) - await n.child('a').set('x', 3) - expect(await collect(n.keys())).toEqual(['a-1', 'a']) - expect(await collect(n.keys({ reverse: true }))).toEqual(['a', 'a-1']) - expect(await collect(n.entries())).toEqual([ - ['a-1', 2], - ['a', 1], - ]) - }) - - it('is a skip-scan: seeks are O(children), not O(subtree)', async () => { - const { db, driver } = setup() - const n = db.child('n') - const writes: [string, unknown][] = [] - for (let c = 0; c < 50; c++) { - writes.push([`n/c${String(c).padStart(3, '0')}`, c]) - for (let d = 0; d < 100; d++) writes.push([`n/c${String(c).padStart(3, '0')}/d${d}`, d]) - } - await driver.inner.setMany(writes) - driver.reset() - const keys = await collect(n.keys()) - expect(keys).toHaveLength(50) - // One initial scan plus one seek per child with descendants. - expect(driver.calls.scan).toBe(51) - expect(driver.calls.getMany).toBeUndefined() - }) - - it('siblings without descendants share one iterator', async () => { - const { db, driver } = setup() - const n = db.child('n') - await n.setMany(Array.from({ length: 100 }, (_, i) => [`k${i}`, i] as [string, number])) - driver.reset() - expect(await collect(n.keys())).toHaveLength(100) - expect(driver.calls.scan).toBe(1) - }) -}) - -describe('deleteTree', () => { - it('deletes own value and all descendants but not lookalike siblings', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.child('a').set('b', 2) - await n.child('a').child('b').set('c', 3) - await n.set('a!', 'sibling') - await n.set('a0', 'sibling') - await n.deleteTree('a') - expect(await collect(n.keys())).toEqual(['a!', 'a0']) - await n.child('a0').deleteTree() - expect(await collect(n.keys())).toEqual(['a!']) - }) - - it('works without deleteRange, and on root', async () => { - const driver = new CountingDriver(new InMemoryKVDriver(), ['deleteRange']) - const db = new KVStorage(driver) - for (let i = 0; i < 2500; i++) await db.child('x').child(String(i)).set('v', i) - await db.child('y').set(1, 1) - await db.child('x').deleteTree() - expect(await collect(db.keys())).toEqual(['y']) - await db.deleteTree() - expect(await collect(driver.scan({}))).toEqual([]) - }) -}) - -describe('tree', () => { - it('assembles descendants up to depth, field-per-key style', async () => { - const { db } = setup() - const p = db.child('profile').child('9') - await p.set('username', 'ada') - await p.set('age', 36) - await p.child('address').set('city', 'London') - await p.child('address').child('geo').set('lat', 51) - await p.child('address').set('own') - await db.driver.setMany([['profile/9/~i/x/y', true]]) - expect(await p.tree({ depth: 1 })).toEqual({ username: 'ada', age: 36, address: 'own' }) - expect(await p.tree({ depth: 2 })).toEqual({ username: 'ada', age: 36, address: { '': 'own', city: 'London', geo: undefined } }) - expect(await p.tree({ depth: 3 })).toEqual({ username: 'ada', age: 36, address: { '': 'own', city: 'London', geo: { lat: 51 } } }) - }) -}) - -// ------------------------------------------------------------------ model-based - -type Path = string[] -const SEP = '/' -const segArb = fc.constantFrom('a', 'a-1', 'a.b', 'b', 'a/x', '%', '~t', '1', '10', '9', 'ab', 'a!', 'é') -const pathArb = fc.array(segArb, { minLength: 1, maxLength: 3 }) -const valArb = fc.oneof(fc.integer(), fc.string(), fc.constant(null), fc.record({ n: fc.integer() })) - -type Op = - | { t: 'set'; path: Path; value: unknown; selfForm: boolean } - | { t: 'delete'; path: Path; selfForm: boolean } - | { t: 'deleteTree'; path: Path } - | { t: 'setMany'; parent: Path; entries: [string, unknown][] } - -const opArb: fc.Arbitrary = fc.oneof( - { weight: 5, arbitrary: fc.record({ t: fc.constant('set' as const), path: pathArb, value: valArb, selfForm: fc.boolean() }) }, - { weight: 2, arbitrary: fc.record({ t: fc.constant('delete' as const), path: pathArb, selfForm: fc.boolean() }) }, - { weight: 1, arbitrary: fc.record({ t: fc.constant('deleteTree' as const), path: pathArb }) }, - { - weight: 1, - arbitrary: fc.record({ - t: fc.constant('setMany' as const), - parent: fc.array(segArb, { maxLength: 2 }), - entries: fc.array(fc.tuple(segArb, valArb), { maxLength: 4 }), - }), - }, -) - -const enc = (p: Path) => JSON.stringify(p) -const isPrefix = (p: Path, q: Path) => p.length <= q.length && p.every((s, i) => q[i] === s) - -/** Expected child order: escaped segment + separator, lexicographic. */ -function expectedChildren(model: Map, parent: Path): { key: string; own: unknown; hasOwn: boolean }[] { - const kids = new Map() - for (const [k, v] of model) { - const p = JSON.parse(k) as Path - if (p.length <= parent.length || !isPrefix(parent, p)) continue - const seg = p[parent.length]! - const e = kids.get(seg) ?? { own: undefined, hasOwn: false } - if (p.length === parent.length + 1) Object.assign(e, { own: v, hasOwn: true }) - kids.set(seg, e) - } - return [...kids] - .map(([key, e]) => ({ key, ...e })) - .sort((x, y) => { - const a = escapeSegment(x.key, SEP) + SEP - const b = escapeSegment(y.key, SEP) + SEP - return a < b ? -1 : a > b ? 1 : 0 - }) -} - -function nodeAt(db: KVStorage, path: Path) { - let n = db as ReturnType - for (const s of path) n = n.child(s) - return n -} - -describe('model-based (random ops vs reference Map)', () => { - for (const [name, make] of [ - ['InMemory', () => new InMemoryKVDriver()], - ['Minimal', () => new MinimalDriver()], - ] as const) { - it(`matches the model (${name})`, async () => { - await fc.assert( - fc.asyncProperty( - fc.array(opArb, { maxLength: 30 }), - fc.array(fc.array(segArb, { maxLength: 2 }), { minLength: 1, maxLength: 3 }), - async (ops, probes) => { - const db = new KVStorage(make()) - const model = new Map() - for (const op of ops) { - switch (op.t) { - case 'set': { - if (op.selfForm) await nodeAt(db, op.path).set(op.value) - else await nodeAt(db, op.path.slice(0, -1)).set(op.path.at(-1)!, op.value) - model.set(enc(op.path), op.value) - break - } - case 'delete': { - if (op.selfForm) await nodeAt(db, op.path).delete() - else await nodeAt(db, op.path.slice(0, -1)).delete(op.path.at(-1)!) - model.delete(enc(op.path)) - break - } - case 'deleteTree': { - await nodeAt(db, op.path).deleteTree() - for (const k of [...model.keys()]) if (isPrefix(op.path, JSON.parse(k))) model.delete(k) - break - } - case 'setMany': { - await nodeAt(db, op.parent).setMany(op.entries) - for (const [k, v] of op.entries) model.set(enc([...op.parent, k]), v) - break - } - } - } - for (const [k, v] of model) expect(await nodeAt(db, JSON.parse(k)).get()).toEqual(v) - for (const parent of probes) { - const node = nodeAt(db, parent) - const exp = expectedChildren(model, parent) - const keys = exp.map((e) => e.key) - expect(await collect(node.keys())).toEqual(keys) - expect(await collect(node.keys({ reverse: true }))).toEqual([...keys].reverse()) - expect(await collect(node.entries())).toEqual(exp.filter((e) => e.hasOwn).map((e) => [e.key, e.own])) - expect(await collect(node.entries({ reverse: true }))).toEqual( - exp - .filter((e) => e.hasOwn) - .map((e) => [e.key, e.own]) - .reverse(), - ) - for (let i = 0; i < keys.length; i++) { - expect(await collect(node.keys({ after: keys[i]! }))).toEqual(keys.slice(i + 1)) - expect(await collect(node.keys({ after: keys[i]!, reverse: true }))).toEqual(keys.slice(0, i).reverse()) - expect(await collect(node.keys({ limit: i + 1 }))).toEqual(keys.slice(0, i + 1)) - } - } - }, - ), - { numRuns: 300 }, - ) - }, 60_000) - } -}) diff --git a/test/kv/sorted.test.ts b/test/kv/sorted.test.ts deleted file mode 100644 index 7ded610..0000000 --- a/test/kv/sorted.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { ChunkedSortedMap } from '../../src/kv/sorted.js' - -const key = fc.string({ unit: fc.constantFrom('a', 'b', 'c', '/', '0'), minLength: 1, maxLength: 4 }) -type Op = { t: 'set'; k: string } | { t: 'del'; k: string } | { t: 'bulk'; ks: string[] } -const op: fc.Arbitrary = fc.oneof( - { weight: 5, arbitrary: fc.record({ t: fc.constant('set' as const), k: key }) }, - { weight: 3, arbitrary: fc.record({ t: fc.constant('del' as const), k: key }) }, - { weight: 1, arbitrary: fc.record({ t: fc.constant('bulk' as const), ks: fc.array(key, { minLength: 60, maxLength: 120 }) }) }, -) - -describe('ChunkedSortedMap (model-based, tiny chunks)', () => { - it('matches a sorted reference for ranges in both directions', () => { - fc.assert( - fc.property(fc.array(op, { maxLength: 80 }), fc.integer({ min: 2, max: 6 }), key, key, (ops, chunk, a, b) => { - const m = new ChunkedSortedMap(chunk) - const ref = new Map() - ops.forEach((o, i) => { - if (o.t === 'set') (m.set(o.k, i), ref.set(o.k, i)) - else if (o.t === 'del') expect(m.delete(o.k)).toBe(ref.delete(o.k)) - else (m.setMany(o.ks.map((k) => [k, i])), o.ks.forEach((k) => ref.set(k, i))) - }) - const sorted = [...ref.keys()].sort() - const [gte, lt] = a < b ? [a, b] : [b, a] - expect(m.size).toBe(ref.size) - expect([...m.keys({})]).toEqual(sorted) - expect([...m.keys({ reverse: true })]).toEqual([...sorted].reverse()) - expect([...m.keys({ gte, lt })]).toEqual(sorted.filter((k) => k >= gte && k < lt)) - expect([...m.keys({ gte, lt, reverse: true })]).toEqual(sorted.filter((k) => k >= gte && k < lt).reverse()) - for (const k of sorted) expect(m.get(k)).toBe(ref.get(k)) - }), - { numRuns: 500 }, - ) - }) - - it('iterators resume correctly after mutations between steps', () => { - fc.assert( - fc.property( - fc.uniqueArray(key, { minLength: 1, maxLength: 60 }), - fc.array(fc.tuple(fc.boolean(), key), { maxLength: 40 }), - fc.boolean(), - (initial, muts, reverse) => { - const m = new ChunkedSortedMap(3) - initial.forEach((k) => m.set(k, 0)) - const live = new Set(initial) - const seen: string[] = [] - let step = 0 - for (const k of m.keys({ reverse })) { - seen.push(k) - const mut = muts[step++] - if (mut) { - const [add, mk] = mut - if (add) (m.set(mk, 1), live.add(mk)) - else (m.delete(mk), live.delete(mk)) - } - } - // Strictly monotone, and every key present throughout the iteration was visited. - for (let i = 1; i < seen.length; i++) expect(reverse ? seen[i]! < seen[i - 1]! : seen[i]! > seen[i - 1]!).toBe(true) - const stable = initial.filter((k) => !muts.some(([, mk]) => mk === k)) - for (const k of stable) expect(seen).toContain(k) - }, - ), - { numRuns: 500 }, - ) - }) -}) diff --git a/test/types.test-d.ts b/test/types.test-d.ts index 35fcfaa..7f589f6 100644 --- a/test/types.test-d.ts +++ b/test/types.test-d.ts @@ -1,123 +1,89 @@ import { describe, expectTypeOf, it } from 'vitest' -import { KVStorage, type KVNode } from '../src/kv/node.js' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVEventStorage } from '../src/events/kv.js' -import type { StoredEvent } from '../src/events/types.js' -import { EventSourcedDocumentStorage } from '../src/documents/event-sourced.js' -import type { FindResult } from '../src/documents/types.js' -import type { AnySpec, Fields, Many } from '../src/types/schema.js' +import type { BaseKVStorage } from '../src/kv-storage/base-kv-storage.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import type { StoredEvent } from '../src/event-storage/base-event-storage.js' +import { DocumentStorageInMemory } from '../src/document-storage/document-storage-in-memory.js' +import { EventSourcedDocumentStorage } from '../src/document-storage/event-sourced-document-storage.js' +import type { DocChange } from '../src/utils/document-update.js' +import type { FindResult } from '../src/utils/document-query.js' +import type { Fields, Many } from '../src/utils/schema.js' type User = { username: string; age: number } type Post = { title: string; createdAt: string } type Schema = { children: { users: Many }> - profile: { children: { [id: string]: { children: Fields } } } // field-per-key style + profile: { children: { [id: string]: { children: Fields } } } } } -const driver = new InMemoryKVDriver() +describe('KV types', () => { + const db = new KVStorageInMemory() -describe('KV types (section 7)', () => { - const db = new KVStorage(driver) - - it('reads child values at dynamic levels', () => { + it('types reads at dynamic and named levels', () => { expectTypeOf(db.child('users').get('123')).toEqualTypeOf>() expectTypeOf(db.child('users').child('123').get()).toEqualTypeOf>() - expectTypeOf(db.child('users').getMany(['1', '2'])).toEqualTypeOf>() - }) - - it('navigates named levels', () => { - expectTypeOf(db.child('users').child('123').child('posts')).toEqualTypeOf>>() - // @ts-expect-error unknown named child - db.child('users').child('123').child('postz') - // @ts-expect-error unknown top-level child - db.child('userz') - }) - - it('reads field-per-key values', () => { expectTypeOf(db.child('profile').child('9').get('age')).toEqualTypeOf>() - expectTypeOf(db.child('profile').child('9').get('username')).toEqualTypeOf>() + expectTypeOf(db.child('users').child('123').child('posts')).toEqualTypeOf>>() + // @ts-expect-error unknown child + db.child('users').child('123').child('postz') // @ts-expect-error unknown field db.child('profile').child('9').get('agee') }) - it('checks written values', () => { + it('types writes', () => { db.child('users').set('1', { username: 'a', age: 1 }) db.child('users').child('1').set({ username: 'a', age: 1 }) - // @ts-expect-error wrong value type + // @ts-expect-error missing field db.child('users').set('1', { username: 'a' }) db.child('profile') .child('9') .setMany([ ['age', 3], - ['username', 'x'], - ]) - db.child('profile') - .child('9') - .setMany([ // @ts-expect-error value doesn't match its key - ['age', 'x'], + ['username', 3], ]) }) it('types listings', () => { - expectTypeOf(db.child('users').keys()).toEqualTypeOf>() expectTypeOf(db.child('profile').child('9').keys()).toEqualTypeOf>() expectTypeOf(db.child('users').entries()).toEqualTypeOf>() }) - it('defaults to untyped and supports as()', () => { - const raw = new KVStorage(driver) - expectTypeOf(raw.child('anything').child(5).get('x')).toEqualTypeOf>() - const users = raw.child('users').as>() - expectTypeOf(users.get('1')).toEqualTypeOf>() - expectTypeOf(raw).toMatchTypeOf>() + it('is untyped by default', () => { + expectTypeOf(new KVStorageInMemory().child('a').child(1).get('b')).toEqualTypeOf>() }) }) -type OrderEvent = { type: 'created'; total: number } | { type: 'paid' } -type EventSchema = { children: { orders: Many }> } } +describe('event and document types', () => { + type OrderEvent = { type: 'created'; total: number } | { type: 'paid' } + const orders = new EventStorageKV<{ children: { orders: Many } }>(new KVStorageInMemory()).child('orders') + const users = new DocumentStorageInMemory<{ children: { users: Many }> } }>().child('users') -describe('event types', () => { - const events = new KVEventStorage(new KVStorage(driver)) - const orders = events.child('orders') - - it('checks appended events', () => { - orders.append('o1', [{ type: 'created', total: 40 }]) + it('types events', () => { + orders.append('o1', [{ type: 'created', total: 1 }]) orders.child('o1').append([{ type: 'paid' }]) // @ts-expect-error not an OrderEvent orders.append('o1', [{ type: 'refunded' }]) - orders - .child('o1') - .child('shipments') - .append('s1', [{ carrier: 'x' }]) + expectTypeOf(orders.read('o1')).toEqualTypeOf>>() }) - it('types read results', () => { - expectTypeOf(orders.read('o1')).toEqualTypeOf>>() - expectTypeOf(orders.version('o1')).toEqualTypeOf>() - }) -}) - -type DocSchema = { children: { users: Many }> } } - -describe('document types', () => { - const docs = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(driver))) - const users = docs.child('users') - - it('types reads and writes', () => { + it('types documents', () => { expectTypeOf(users.get('1')).toEqualTypeOf>() - expectTypeOf(users.child('1').get()).toEqualTypeOf>() - expectTypeOf(users.insert({ username: 'a', age: 1 })).toEqualTypeOf>() - // @ts-expect-error missing field - users.insert({ username: 'a' }) expectTypeOf(users.update('1', { $inc: { age: 1 } })).toEqualTypeOf>() expectTypeOf(users.child('1').child('posts').get('p')).toEqualTypeOf>() + expectTypeOf(users.find()).toEqualTypeOf>>() + // @ts-expect-error missing field + users.insert({ username: 'a' }) }) - it('types queries', () => { - expectTypeOf(users.find({ age: { $gte: 18 } })).toEqualTypeOf>>() - expectTypeOf(users.findOne()).toEqualTypeOf | undefined>>() + it('keeps event-sourced methods on children', () => { + const docs = new EventSourcedDocumentStorage<{ children: { users: Many } }>( + new EventStorageKV(new KVStorageInMemory()), + new DocumentStorageInMemory(), + ) + expectTypeOf(docs.child('users').history('1')).toEqualTypeOf>>() + expectTypeOf(docs.child('users').getAt('1', 3)).toEqualTypeOf>() }) }) diff --git a/test/utils/utils.test.ts b/test/utils/utils.test.ts new file mode 100644 index 0000000..85ac5db --- /dev/null +++ b/test/utils/utils.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { Keys } from '../../src/utils/keys.js' +import { SortedMap } from '../../src/utils/sorted-map.js' +import { Ulid } from '../../src/utils/ulid.js' +import { DocumentQuery } from '../../src/utils/document-query.js' +import { DocumentUpdate } from '../../src/utils/document-update.js' + +describe('Keys', () => { + it('round-trips any segment and never emits a character at or below the separator', () => { + fc.assert( + fc.property(fc.string({ unit: 'binary', minLength: 1 }), (s) => { + const escaped = Keys.escape(s) + expect(Keys.unescape(escaped)).toBe(s) + expect([...escaped].every((c) => c > '/')).toBe(true) + expect(Keys.isReserved(escaped)).toBe(false) + }), + ) + }) + + it('keeps every child subtree contiguous in key order', () => { + fc.assert( + fc.property( + fc.uniqueArray(fc.string({ unit: fc.constantFrom('a', '-', '.', '/', '~', ' ', '0'), minLength: 1, maxLength: 3 }), { minLength: 2 }), + (segments) => { + const keys = segments.flatMap((s) => [Keys.escape(s), Keys.escape(s) + '/x']).sort() + const order = keys.map((k) => k.split('/')[0]) + // Each segment's keys appear as one consecutive run. + expect(new Set(order).size).toBe(order.filter((s, i) => s !== order[i - 1]).length) + }, + ), + ) + }) +}) + +describe('SortedMap', () => { + it('matches a sorted reference, including iteration across mutations', () => { + fc.assert( + fc.property( + fc.array(fc.tuple(fc.boolean(), fc.string({ unit: fc.constantFrom('a', 'b', 'c'), maxLength: 4 })), { maxLength: 200 }), + fc.boolean(), + (ops, reverse) => { + const map = new SortedMap(4) + const ref = new Set() + for (const [add, key] of ops) add ? (map.set(key, 1), ref.add(key)) : (map.delete(key), ref.delete(key)) + const sorted = [...ref].sort() + expect([...map.keys(undefined, undefined, reverse)]).toEqual(reverse ? sorted.reverse() : sorted) + // Deleting each key as we pass it must not skip the rest. + const seen = [...map.keys()].length + let visited = 0 + for (const key of map.keys()) (visited++, map.delete(key)) + expect(visited).toBe(seen) + }, + ), + ) + }) +}) + +describe('Ulid', () => { + it('is strictly increasing within a millisecond and sorts by time', () => { + let t = 1_700_000_000_000 + const ulid = new Ulid(() => t) + const ids = Array.from({ length: 500 }, () => ulid.generate()) + expect([...ids].sort()).toEqual(ids) + t++ + expect(ulid.generate() > ids.at(-1)!).toBe(true) + }) +}) + +describe('DocumentQuery', () => { + const q = (filter: object, doc: object) => new DocumentQuery(filter as never).matches(doc as never) + it('matches equality, comparisons, $in, dot paths, arrays and null-as-missing', () => { + expect(q({ a: 1 }, { a: 1 })).toBe(true) + expect(q({ 'a.b': 'x' }, { a: { b: 'x' } })).toBe(true) + expect(q({ n: { $gte: 2, $lt: 5 } }, { n: 4 })).toBe(true) + expect(q({ n: { $gt: 2 } }, { n: '9' })).toBe(false) + expect(q({ n: { $in: [1, 2] } }, { n: 2 })).toBe(true) + expect(q({ tags: 'x' }, { tags: ['y', 'x'] })).toBe(true) + expect(q({ tags: ['x'] }, { tags: ['x'] })).toBe(true) + expect(q({ a: null }, {})).toBe(true) + expect(() => q({ a: { $regex: 'x' } }, {})).toThrow(/Unsupported/) + }) +}) + +describe('DocumentUpdate', () => { + it('resolves operators to final values', () => { + const state = { n: 5, tags: ['a', 'b'], nested: { x: 1 } } + const change = DocumentUpdate.resolve(state, { $inc: { n: 1 }, $push: { tags: { $each: ['c'] } }, $unset: { 'nested.x': 1 }, $set: { 'nested.y': 2 } }) + expect(change).toEqual({ op: 'patch', set: { n: 6, tags: ['a', 'b', 'c'], 'nested.y': 2 }, unset: ['nested.x'] }) + expect(DocumentUpdate.apply(state, change)).toEqual({ n: 6, tags: ['a', 'b', 'c'], nested: { y: 2 } }) + expect(DocumentUpdate.apply(state, DocumentUpdate.resolve(state, { $pull: { tags: 'a' }, $addToSet: { more: { $each: [1, 1] } } }))).toMatchObject({ + tags: ['b'], + more: [1], + }) + expect(() => DocumentUpdate.resolve(state, { $set: { n: 1 }, $inc: { n: 1 } })).toThrow(/twice/) + }) +})