From 08fb78721f672e75631e66294ad1ed1ba260e492 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:05:03 +0000 Subject: [PATCH 01/11] Simplify: one abstract base class per layer - BaseKVStorage / BaseEventStorage / BaseDocumentStorage define the API once; implementations provide only raw primitives (client and server share an API) - Drop KV indexes, query planner, unique indexes; document stores take index fields - Escape every char <= '/' so child keys stay contiguous; skip-scan is now simple - Event-sourced documents = event storage + any document store for projections - New folder layout (kv-storage, event-storage, document-storage, utils); tests only in test/ Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- CLAUDE.md | 32 +- DESIGN.md | 809 ++++-------------- bench/document-storage.bench.ts | 33 + bench/documents.bench.ts | 34 - bench/event-storage.bench.ts | 22 + bench/events.bench.ts | 27 - bench/keys.bench.ts | 26 - bench/kv-storage.bench.ts | 40 + bench/kv.bench.ts | 29 - package.json | 28 - src/core/encoding.ts | 176 ---- src/core/lru.ts | 30 - src/core/mutex.ts | 15 - src/core/path.ts | 109 --- src/core/trie.ts | 82 -- src/document-storage/base-document-storage.ts | 117 +++ .../document-storage-in-memory.ts | 112 +++ src/document-storage/document-storage-kv.ts | 46 + .../event-sourced-document-storage.ts | 125 +++ src/documents/event-sourced.ts | 606 ------------- src/documents/filter.ts | 155 ---- src/documents/index.ts | 7 - src/documents/indexes.ts | 220 ----- src/documents/types.ts | 90 -- src/documents/ulid.ts | 37 - src/documents/update.ts | 109 --- src/documents/values.ts | 163 ---- src/errors.ts | 50 -- src/event-storage/base-event-storage.ts | 100 +++ src/event-storage/event-storage-kv.ts | 151 ++++ src/events/index.ts | 2 - src/events/kv.ts | 289 ------- src/events/types.ts | 69 -- src/http/client.ts | 115 --- src/http/handler.ts | 115 --- src/http/index.ts | 2 - src/index.ts | 26 +- src/kv-storage/base-kv-storage.ts | 184 ++++ src/kv-storage/kv-storage-http-server.ts | 70 ++ src/kv-storage/kv-storage-http.ts | 79 ++ src/kv-storage/kv-storage-in-memory.ts | 34 + src/kv/children.ts | 236 ----- src/kv/driver.ts | 77 -- src/kv/index.ts | 6 - src/kv/indexes.ts | 157 ---- src/kv/memory.ts | 93 -- src/kv/node.ts | 402 --------- src/kv/sorted.ts | 181 ---- src/testing/index.ts | 1 - src/testing/kv-conformance.ts | 171 ---- src/utils/document-path.ts | 32 + src/utils/document-query.ts | 106 +++ src/utils/document-update.ts | 75 ++ src/utils/errors.ts | 19 + src/utils/json.ts | 24 + src/utils/keys.ts | 43 + src/{types => utils}/schema.ts | 5 +- src/utils/sorted-map.ts | 103 +++ src/utils/ulid.ts | 36 + test/core/encoding.test.ts | 73 -- test/core/path.test.ts | 73 -- .../document-storage.suite.ts | 86 ++ .../document-storage/document-storage.test.ts | 91 ++ test/documents/event-sourced.test.ts | 379 -------- test/documents/units.test.ts | 62 -- test/event-storage/event-storage-kv.test.ts | 99 +++ test/events/kv.test.ts | 200 ----- test/helpers.ts | 94 +- test/http/http.test.ts | 92 -- test/http/node-server.ts | 36 - test/kv-storage/kv-storage-http.test.ts | 54 ++ test/kv-storage/kv-storage-in-memory.test.ts | 78 ++ test/kv-storage/kv-storage.conformance.ts | 168 ++++ test/kv-storage/node-server.ts | 24 + test/kv/indexes.test.ts | 256 ------ test/kv/memory-driver.test.ts | 28 - test/kv/node.test.ts | 354 -------- test/kv/sorted.test.ts | 67 -- test/types.test-d.ts | 112 +-- test/utils/utils.test.ts | 97 +++ 80 files changed, 2518 insertions(+), 6337 deletions(-) create mode 100644 bench/document-storage.bench.ts delete mode 100644 bench/documents.bench.ts create mode 100644 bench/event-storage.bench.ts delete mode 100644 bench/events.bench.ts delete mode 100644 bench/keys.bench.ts create mode 100644 bench/kv-storage.bench.ts delete mode 100644 bench/kv.bench.ts delete mode 100644 src/core/encoding.ts delete mode 100644 src/core/lru.ts delete mode 100644 src/core/mutex.ts delete mode 100644 src/core/path.ts delete mode 100644 src/core/trie.ts create mode 100644 src/document-storage/base-document-storage.ts create mode 100644 src/document-storage/document-storage-in-memory.ts create mode 100644 src/document-storage/document-storage-kv.ts create mode 100644 src/document-storage/event-sourced-document-storage.ts delete mode 100644 src/documents/event-sourced.ts delete mode 100644 src/documents/filter.ts delete mode 100644 src/documents/index.ts delete mode 100644 src/documents/indexes.ts delete mode 100644 src/documents/types.ts delete mode 100644 src/documents/ulid.ts delete mode 100644 src/documents/update.ts delete mode 100644 src/documents/values.ts delete mode 100644 src/errors.ts create mode 100644 src/event-storage/base-event-storage.ts create mode 100644 src/event-storage/event-storage-kv.ts delete mode 100644 src/events/index.ts delete mode 100644 src/events/kv.ts delete mode 100644 src/events/types.ts delete mode 100644 src/http/client.ts delete mode 100644 src/http/handler.ts delete mode 100644 src/http/index.ts create mode 100644 src/kv-storage/base-kv-storage.ts create mode 100644 src/kv-storage/kv-storage-http-server.ts create mode 100644 src/kv-storage/kv-storage-http.ts create mode 100644 src/kv-storage/kv-storage-in-memory.ts delete mode 100644 src/kv/children.ts delete mode 100644 src/kv/driver.ts delete mode 100644 src/kv/index.ts delete mode 100644 src/kv/indexes.ts delete mode 100644 src/kv/memory.ts delete mode 100644 src/kv/node.ts delete mode 100644 src/kv/sorted.ts delete mode 100644 src/testing/index.ts delete mode 100644 src/testing/kv-conformance.ts create mode 100644 src/utils/document-path.ts create mode 100644 src/utils/document-query.ts create mode 100644 src/utils/document-update.ts create mode 100644 src/utils/errors.ts create mode 100644 src/utils/json.ts create mode 100644 src/utils/keys.ts rename src/{types => utils}/schema.ts (87%) create mode 100644 src/utils/sorted-map.ts create mode 100644 src/utils/ulid.ts delete mode 100644 test/core/encoding.test.ts delete mode 100644 test/core/path.test.ts create mode 100644 test/document-storage/document-storage.suite.ts create mode 100644 test/document-storage/document-storage.test.ts delete mode 100644 test/documents/event-sourced.test.ts delete mode 100644 test/documents/units.test.ts create mode 100644 test/event-storage/event-storage-kv.test.ts delete mode 100644 test/events/kv.test.ts delete mode 100644 test/http/http.test.ts delete mode 100644 test/http/node-server.ts create mode 100644 test/kv-storage/kv-storage-http.test.ts create mode 100644 test/kv-storage/kv-storage-in-memory.test.ts create mode 100644 test/kv-storage/kv-storage.conformance.ts create mode 100644 test/kv-storage/node-server.ts delete mode 100644 test/kv/indexes.test.ts delete mode 100644 test/kv/memory-driver.test.ts delete mode 100644 test/kv/node.test.ts delete mode 100644 test/kv/sorted.test.ts create mode 100644 test/utils/utils.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 8553941..9d9b3f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,24 +1,24 @@ # PotentialStorage -Layered TypeScript storage library: KV → Events → Documents. The full design, milestones and -implementation decisions are in [DESIGN.md](./DESIGN.md) — read it before changing anything. -Record new decisions in DESIGN.md section 12. +Layered TypeScript storage library: KV → Events → Documents. The design is in +[DESIGN.md](./DESIGN.md); read it before changing anything and keep it up to date. ## Commands -- `npm test` — `tsc --noEmit` then all Vitest tests (including `*.test-d.ts` type tests) -- `npm run test:only -- ` — Vitest only, e.g. a single file -- `npm run typecheck` — `tsc --noEmit` over src, test and bench -- `npm run bench` — `vitest bench` -- `npm run build` — emit `dist/` -- `npm run format` / `npm run format:check` — Prettier +- `npm test`: `tsc --noEmit`, then all Vitest tests (including `test/types.test-d.ts`) +- `npm run test:only -- `: Vitest only +- `npm run typecheck`, `npm run build`, `npm run bench` +- `npm run format` / `npm run format:check`: Prettier -CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs format check, typecheck, tests and build on Node 20 and 22. +CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, tests and build on Node 20 and 22. -## Rules +## Conventions -- No runtime dependencies in `src/`. Dev dependencies only. -- Drivers stay flat and string-keyed; all path/index/escaping logic lives in the node classes. -- Self vs child dispatch is by `arguments.length`; a passed-but-`undefined`/`null` key throws. -- Multi-result reads are `AsyncIterable`; don't build large arrays internally. -- Keep tests green and commit per milestone. Ask the owner before pushing. +- One abstract base class per layer (`src//base-*.ts`); implementations only provide the raw + primitives, and the user-facing API lives in the base class. +- Classes over loose functions; TS `private`, not `#`; no overloads (use `param?` or tuple-union + rest params); don't override base methods in subclasses. +- No runtime dependencies. Tests live only in `test/`, mirroring `src/`. +- New KV/document implementations must pass `test/kv-storage/kv-storage.conformance.ts` / + `test/document-storage/document-storage.suite.ts`. +- Ask the owner before pushing. diff --git a/DESIGN.md b/DESIGN.md index 93b0e3a..4021168 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,655 +1,176 @@ # PotentialStorage — Design -This document is the design brief agreed with the project owner (sections 0–11), followed by -implementation decisions made where the brief was silent (section 12) and benchmark baselines -(section 13). +Layered TypeScript storage: **KV → Events → Documents**. Each layer is an abstract base class +that implementations extend; each is usable on its own. No runtime dependencies. -## 0. Setup and workflow - -- Repository: `git@git.harvmaster.com:Harvmaster/PotentialStorage.git` (SSH; self-hosted Git, not GitHub). Clone it. If you can't reach it, stop and tell the owner rather than starting a fresh repo elsewhere. -- Inspect what's already in the repo first. If it has tooling (package.json, tsconfig, test runner), keep it. If it's empty, set up: - - TypeScript (`strict: true`, `noUncheckedIndexedAccess: true`), ESM, Node ≥ 20 - - Vitest for tests and `vitest bench` for benchmarks; `expectTypeOf` for type tests - - Package name `potential-storage` unless one already exists - - **No runtime dependencies in the core.** Dev dependencies are fine (e.g. `fast-check` for property tests). -- Copy this brief into the repo as `DESIGN.md` and create a short `CLAUDE.md` pointing at it plus the build/test commands. -- Work milestone by milestone (section 9). Commit after each milestone with a clear message, and keep tests green. Ask the owner before pushing. +This supersedes the original implementation brief (see git history before the `simplify` branch); +section 7 lists what changed and why. ## 1. Principles -1. **Three layers**, each usable on its own: **KV** → **Event** (append-only, built on KV) → **Document** (Mongo-like, built on Events or native). -2. **Drivers vs stores.** Backends implement a tiny, flat, string-keyed _driver_. All path, child, index, escaping and typing logic lives once in the user-facing _node_ classes, so every backend gets identical behaviour. -3. **One navigation primitive: `child(key)`.** There are no collection/document classes. Every node is a store over its direct children, and `child(key)` re-roots to that key. Navigation is synchronous and does no I/O. -4. **No methods on data.** Reads return plain values. -5. **Performance is a requirement.** Batch operations are the primitive; scans are ordered and streamed; nothing replays or scans more than it must (section 8). +1. **One abstract class per layer.** Implementations provide a handful of raw primitives; the + whole user-facing API is written once in the base class. A client (e.g. HTTP) and a server-side + store are the same kind of object, and wrappers (e.g. encryption) are just another subclass + that delegates the primitives. +2. **One navigation primitive: `child(key)`.** Every instance is a node; `child` returns a view of + the same storage rooted deeper (via `Object.create`, so it shares the instance's state). Sync, no I/O. +3. **Self or child by argument.** `get()` reads this node, `get(key)` a child. A key that is passed + but `undefined`/`null` throws (`InvalidKeyError`) instead of silently meaning "self". Methods use + optional parameters or tuple-union rest parameters, not overloads. +4. **Indexes belong to the store.** Document stores receive the fields to index and use them however + suits the backend (in-memory equality maps, native Mongo/SQLite/IndexedDB indexes, or nothing). +5. **Simple over clever.** Code style: classes, TS `private` (not `#`), no method overrides in + subclasses, tests only under `test/`. -## 2. Paths and key encoding (`src/core/path.ts`) - -- A node's path is `readonly string[]` of **unescaped** segments. Number keys are converted with `String()`. -- The separator is a single character, configurable, default `/`. -- **Escaping** (per segment): percent-encode `%`, the separator, and a **leading** `~`. Escaped segments never contain a raw separator. Write `escapeSegment` / `unescapeSegment` with round-trip property tests. -- **Reserved segments** start with `~` (user segments can't, because a leading `~` is escaped): - - `~i` index entries - - `~u` unique-index claims - - `~e` event commits - - `~p` document projections root -- Each node precomputes in its constructor: - - `selfKey` = escaped segments joined by the separator (e.g. `users/123`) - - `prefix` = `selfKey + sep` (`''` for the root) - - No string work per call beyond `prefix + escapeSegment(key)`. -- Keys are ordered **lexicographically** (so `10` sorts before `9`). Document this. An order-preserving numeric key encoding is a possible later option, not part of v1. - -Example layout: +## 2. Layout ``` -users/123 → KV value of users/123 -users/123/posts/p1 → KV value of a nested child -users/~i/byEmail/sada@x.io/123 → index entry (value: empty / true) -orders/o1/~e/000000000007 → event commit #7 of stream orders/o1 -~p/users/123 → document projection { state, version, commit } +src/ + kv-storage/ base-kv-storage.ts, kv-storage-in-memory.ts, kv-storage-http.ts, kv-storage-http-server.ts + event-storage/ base-event-storage.ts, event-storage-kv.ts + document-storage/ base-document-storage.ts, document-storage-in-memory.ts, document-storage-kv.ts, + event-sourced-document-storage.ts + utils/ keys, sorted-map, json, ulid, errors, schema (types), document-path/query/update +test/ mirrors src/; shared suites: kv-storage.conformance.ts, document-storage.suite.ts +bench/ ``` -## 3. Layer 1: KV - -### 3.1 Driver interface (`src/kv/driver.ts`) - -```ts -export interface ScanOptions { - gte?: string - lt?: string - limit?: number - reverse?: boolean - values?: boolean // when false/omitted, yield value as undefined -} - -export interface KVDriver { - getMany(keys: string[]): Promise<(unknown | undefined)[]> // same order as keys - setMany(entries: [key: string, value: unknown][]): Promise - deleteMany(keys: string[]): Promise - scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]> // MUST be key-ordered - - // Optional capabilities. Stores feature-detect them and fall back where possible. - setIfAbsent?(entries: [string, unknown][]): Promise // true = written - batch?(ops: BatchOp[]): Promise // atomic multi-op write - deleteRange?(gte: string, lt: string): Promise -} - -export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string } -``` - -- Values are passed through as JSON-serialisable values. Drivers that need bytes (HTTP, Redis, files) serialise internally via a pluggable `Codec` (default JSON). -- Provide an abstract `BaseKVDriver` with default implementations of `deleteRange` (scan + deleteMany) and helpers, so a minimal driver implements only the four required methods. -- **`InMemoryKVDriver`**: a sorted structure (sorted array with binary search is fine for v1; keep it behind an interface so it can become a B-tree). It implements every optional capability: `setIfAbsent`, atomic `batch` and `deleteRange`. -- Write a reusable **driver conformance suite** `runKVDriverConformance(name, makeDriver)` covering ordering, `gte`/`lt` bounds, reverse, limit, `values` on/off, `getMany` order with missing keys, and each optional capability when present. - -### 3.2 `KVStorage` / `KVNode` (`src/kv/node.ts`) - -```ts -const db = new KVStorage(driver, { separator: '/', indexes: { ... } }) // db is the root KVNode -const users = db.child('users') -const user = users.child('123') -``` - -API of every `KVNode` (types in section 7): - -| Method | Meaning | -| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `child(key)` | Sync. Returns a node at `path + key`. | -| `as()` | Type assertion for untyped use; returns the same node retyped. | -| `get()` / `get(key)` | Own value at `selfKey` / value at child key. | -| `getMany(keys)` | Batch child read, same order, `undefined` for missing. | -| `set(value)` / `set(key, value)` | Write own value / child value. | -| `setMany(entries)` | Batch child write. | -| `delete()` / `delete(key)` | Delete own value / child value only (not descendants). | -| `deleteMany(keys)` | Batch child delete. | -| `deleteTree()` / `deleteTree(key)` | Delete own value **and all descendants** (uses `deleteRange` if available). | -| `keys(opts?)` | `AsyncIterable` of **direct** child keys (unescaped). Options: `limit`, `after`, `reverse`. | -| `entries(opts?)` | `AsyncIterable<[key, value]>` of direct children that have an own value. | -| `tree({ depth })` | Assemble an object from descendant keys up to `depth` (for field-per-key style). Bounded and explicit; never the default. | -| `findBy(index, value \| { gte?, lt? }, opts?)` | Index lookup over direct children (section 3.3). | - -**Arity dispatch (important):** - -- Self vs child is decided by `arguments.length`, never by checking for `undefined`. -- If a key argument is passed but is `undefined` or `null`, throw a `TypeError` (this prevents `users.get(maybeId)` silently reading self). -- The root node (`path = []`) has no own value: `get()`, `set(v)` and `delete()` on root throw. -- Implement one private `#resolve(args)` that every method uses. - -**Direct-children listing (`keys`)** uses a skip-scan: - -1. Scan `gte: prefix`, `lt: prefix + succ(sep)`. -2. For each key, take the first segment after `prefix`. Skip reserved (`~…`) segments. Yield the segment (unescaped) if it differs from the previous one. -3. When the next key is a descendant of the segment just yielded, re-seek the driver to `gte: prefix + escSeg + succ(sep)` (where `succ(c)` is the next char code) instead of iterating through the subtree. Keep using the same iterator while keys are siblings. - -Listing must be O(direct children × seek cost), not O(subtree). A child that has no own value but has descendants **is** listed. - -### 3.3 Indexes (`src/kv/indexes.ts`) - -Indexes are declared on **path patterns** in the `KVStorage` constructor. A pattern names a parent node, and its index applies to that node's direct children's values: - -```ts -new KVStorage(driver, { - indexes: { - users: { byEmail: (u: User) => u.email }, - 'users/*/posts': { byDate: (p: Post) => p.createdAt, byTag: (p: Post) => p.tags }, - }, -}) -``` - -- An index function returns `IndexValue | IndexValue[] | undefined`, where `IndexValue = string | number | boolean | null`. An array means multi-entry; `undefined` means not indexed. -- Compile patterns into a **segment trie** (`*` matches one segment). A node resolves its index set **once, at construction**, in O(depth). -- Entry key: `~i///` with a `true` value. -- **`encodeIndexValue` must be order-preserving and separator-safe:** - - Type tag first, so values order by type: `null` < `false` < `true` < numbers < strings. - - Numbers use an order-preserving float64 encoding (flip the sign bit for positives and all bits for negatives, then fixed-width hex). - - Strings are escaped like segments. - - This enables `findBy(index, { gte, lt })` range scans. -- **Writes to an indexed node:** - 1. Read the old value(s) (`getMany`) and compute the diff of index entries. - 2. If the driver has `batch`, write the value and the index adds/removes atomically. - 3. Otherwise, write index **adds first**, then the value, then index **removes**. Treat index entries as hints: `findBy` re-reads each value, re-checks the index function, and lazily deletes stale entries (read-repair). -- Non-indexed nodes must **not** pay the read-before-write cost. - -## 4. Layer 2: Events (`src/events/`) - -An append-only log per stream. A stream is identified by a node path. Same `child()` navigation as KV. - -```ts -const events = new EventStorage(kv) // takes a KVStorage (or a KVNode as root) -const orders = events.child('orders') - -await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 }) -await orders.child('o1').append([{ type: 'paid' }]) // self form -for await (const e of orders.read('o1', { from: 1 })) { - /* { version, commit, at, data } */ -} -await orders.version('o1') -for await (const id of orders.keys()) { -} // child streams -orders.child('o1').child('shipments') -``` - -- `EventStorage` is the abstract interface. `KVEventStorage` is the KV-backed implementation. Leave room for native backends (EventStoreDB, Kafka) implementing the same interface. -- **Arity:** `append(events, opts?)` vs `append(key, events, opts?)`. Dispatch on `Array.isArray(args[0])`, plus the undefined-key rule from 3.2. `read`, `version` and `keys` follow the 3.2 rules. -- **Versions:** an empty stream has version 0, and the first event has version 1. `expectedVersion: number | 'any'` (default `'any'`). -- **Storage: one key per commit, densely numbered.** - - Key: `/~e/` - - Value: `{ firstVersion, at, events: E[] }` - - The commit sequence (not the event version) is the key so that **any** stale writer collides with the existing key. Keying by version would let a stale writer create overlapping version ranges. -- **Append:** - 1. Get the head `{ commitSeq, version }` (from the cache, or a reverse scan with `limit: 1`). - 2. If `expectedVersion` is a number and doesn't match the head, throw `ConcurrencyError`. - 3. Call `setIfAbsent(nextCommitKey)`. - 4. On `false` (collision): refresh the head. If `expectedVersion` was given, throw `ConcurrencyError`; otherwise retry up to 5 times. - 5. Without `setIfAbsent`, fall back to a per-stream in-process mutex. Document that this mode is single-writer only. -- **Head cache:** a bounded LRU keyed by stream path. Update it on successful append; invalidate it on collision. -- **`read({ from, to, reverse, limit })`** streams events with metadata. To start at version `from`, locate the containing commit by binary search over commit numbers using `getMany` probes (O(log commits) round trips), then forward-scan. -- There is no update or delete. Keys inside `~e` are never rewritten. - -## 5. Layer 3: Documents (`src/documents/`) - -`DocumentStorage` is the interface. There are two implementations: - -- `EventSourcedDocumentStorage(events, { projections?: KVNode })` is the v1 target. Projections default to the `~p` root on the same KV. -- `MongoDocumentStorage(client)` is a later milestone (section 9). - -```ts -const docs = new EventSourcedDocumentStorage(events, { indexes: { users: [{ fields: ['email'], unique: true }, { fields: ['age'] }] } }) -const users = docs.child('users') - -const id = await users.insert({ name: 'Ada', age: 36 }) // generates a time-sortable id -await users.set('123', { name: 'Ada', age: 36 }) -await users.update('123', { $set: { name: 'Ada L.' }, $inc: { logins: 1 } }) -await users.child('123').update({ $unset: { nickname: true } }) // self form -await users.get('123') -await users.delete('123') -for await (const { key, data } of users.find({ age: { $gte: 18 } }, { sort: { age: 1 }, limit: 20 })) { -} -await users.findOne({ email: 'ada@x.io' }) -await users.history('123') // event-sourced only -await users.getAt('123', 17) // state as of version 17 -users.child('123').child('posts') // nested docs, same as KV -``` - -### 5.1 Events and projections - -- The doc at path `P` has its event stream at `P` in `EventStorage`. Its projection is at `~p/P` in the projections KV: `{ state, version }` (and `deleted: true` for tombstones). -- Event types: `{ op: 'set', doc }`, `{ op: 'patch', set?: Record, unset?: path[] }`, `{ op: 'delete' }`. -- **Store resolved deltas, not operators.** For example, `$inc` resolves to `{ set: { logins: 6 } }`, so replay is deterministic. -- Supported update operators in v1: `$set`, `$unset`, `$inc`, `$push`, `$pull`, `$addToSet`, with dot-path support. - -**Write path:** - -1. Read the projection. -2. Compute the new state and patch. -3. `append` with `expectedVersion = projection.version`. -4. Write the projection and index entries (atomic `batch` if available). -5. On `ConcurrencyError`: catch up the projection from the stream, recompute, and retry (bounded). - -**Reads** are a single projection `get`. Never replay on read. - -**Crash between append and projection write:** - -- The projection lags the stream. The next write's `expectedVersion` fails, which triggers catch-up, so the system self-heals. -- Provide `get(key, { consistent: true })` that checks the stream head and catches up first. -- Provide a `catchUp(node?)` maintenance method to repair a subtree. -- `insert` ids are time-sortable (ULID-like). Implement a small generator, with no dependency. - -### 5.2 Queries and document indexes - -- Filter subset: - - `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$exists` - - `$and`, `$or`, `$not` - - implicit equality and dot paths -- Options: `sort`, `limit`, `skip`, `projection` (fields). -- `find` on a node queries that node's **direct children**. -- Indexes are declared per path pattern (same trie mechanism as 3.3). - - Each index has `fields` (compound allowed); entries go under `~p//~i////…/`, using `encodeIndexValue`. - - `unique: true` additionally claims `~u//` via `setIfAbsent`. Throw at construction if the driver lacks `setIfAbsent`. -- **Planner:** - - Pick the index that covers the most leading equality fields, then at most one range field. - - Use the index order for `sort` when it matches. - - Otherwise do a full scan of children projections, with a one-time dev-mode `console.warn` naming the filter. - -## 6. Shared conventions - -- Every layer uses the same navigation (`child`, `as`, `keys`) and the same self/child arity rules. -- All multi-result reads are `AsyncIterable` (streamed, cancellable). Never build large arrays internally. -- Errors are typed classes exported from `src/errors.ts`: `ConcurrencyError`, `UniqueConstraintError`, `CapabilityError`, `InvalidKeyError`. -- Public exports go through `src/index.ts`, with subpath exports per layer if convenient. - -## 7. Types (`src/types/schema.ts`) - -Each node is typed by a single **Spec** that describes its own value and its children by key: - -```ts -export type Spec = { value?: unknown; children?: Record } -export type AnySpec = { value: unknown; children: Record } // untyped default - -export type ValueOf = S extends { value: infer V } ? V : never -export type ChildrenOf = S extends { children: infer C } ? C : {} -export type KeyOf = Extract, string | number> -export type ChildOf> = ChildrenOf[K] - -// Helpers -export type Many = {}> = { children: { [id: string]: { value: V; children: C } } } -export type Fields = { [K in keyof T & string]: { value: T[K] } } -``` - -Node signatures: - -```ts -class KVNode { - child>(key: K): KVNode> - get(): Promise | undefined> - get>(key: K): Promise> | undefined> - // set/delete/getMany/etc. follow the same pattern -} -``` - -Example to cover with `expectTypeOf` tests: - -```ts -type User = { username: string; age: number } -type Post = { title: string; createdAt: string } -type Schema = { - children: { - users: Many }> - profile: { children: { [id: string]: { children: Fields } } } // field-per-key style - } -} - -const db = new KVStorage(driver) -db.child('users').get('123') // User | undefined -db.child('users').child('123').get() // User | undefined (same key) -db.child('users').child('123').child('posts') // KVNode> -db.child('users').child('123').child('postz') // ✗ type error -db.child('profile').child('9').get('age') // number | undefined -db.child('profile').child('9').get('agee') // ✗ type error -``` - -Notes: - -- **Named levels** check keys at compile time; **dynamic `[id: string]` levels** accept any string. -- One node's children can't mix an index signature with named keys (a TS restriction), so schemas alternate between dynamic and named levels. That alternation exists only in types; the runtime API is just `child()`. -- `EventStorage` and `DocumentStorage` nodes use the same Spec machinery (the value type is the event type or the document type respectively). - -## 8. Performance requirements - -- Single-key ops: exactly one driver call (no hidden reads), except indexed writes (one `getMany` before the write) and document writes (projection read plus append plus projection write). -- `*Many` ops: one driver call each (plus index work), never N calls. -- `keys()`: skip-scan, O(direct children) seeks. -- Event head lookup: cached; cold lookup is a single reverse scan with `limit: 1`. -- Document `get`: one driver `get`. -- Add `vitest bench` benchmarks for: - - KV `get`/`set`/`getMany` at 10k/100k keys - - `keys()` on a node with 1k children each having 100 descendants - - event append and read - - document `get`/`update`/indexed `find` vs full-scan `find` -- Record baseline numbers in `DESIGN.md`. - -## 9. Milestones - -1. **Core:** `path.ts` (escape, unescape, encode, `succ`), `encodeIndexValue`, errors. Property tests for round-trips and order preservation. -2. **KV driver:** `KVDriver`, `BaseKVDriver`, `InMemoryKVDriver`, conformance suite. -3. **KV node:** full API from 3.2, including arity rules, skip-scan `keys`, `deleteTree`, `tree`. Add a model-based property test: random ops vs a reference `Map`. -4. **KV indexes:** pattern trie, atomic and non-atomic write paths, read-repair, range `findBy`. -5. **Events:** `KVEventStorage` with commit keys, `setIfAbsent` and mutex modes, head cache, binary-search `read({ from })`, concurrency tests (simulated interleaving with two storage instances on one driver). -6. **Documents:** `EventSourcedDocumentStorage`, update operators, projections, catch-up, queries, planner, unique indexes, `history` and `getAt`. -7. **Types:** Spec machinery wired through all three layers, plus `expectTypeOf` tests from section 7. -8. **HTTP driver:** - - A `createKVHttpHandler(driver)` (framework-agnostic `Request → Response`) plus an `HttpKVDriver` client. - - Endpoints: `POST /getMany`, `POST /setMany`, `POST /deleteMany`, `POST /batch`, `POST /setIfAbsent`, `GET /scan?gte<&limit&reverse&values` (NDJSON stream). - - It must pass the conformance suite end-to-end. -9. **Benchmarks** and a `DESIGN.md` update with results. -10. **(Later, ask first)** `MongoDocumentStorage`: - - Path pattern maps to a collection (e.g. `users/*/posts` → `users.posts`), with a `_parent` field and a compound index `(_parent, _id)`. - - Native indexes map from the same index declarations. - - It should pass the document test suite where semantics allow (no `history`/`getAt`). - -## 10. Decisions already made (don't revisit without asking) - -- `child()`-only navigation; no collection/doc classes. -- `get()` with no argument reads the node's **own key**. It never assembles the subtree; `tree()` does that explicitly. -- A node's own value and its children are independent keys. -- Self/child dispatch by arity; a passed-but-undefined key throws. -- Indexes are declared on path patterns at construction. -- Events use one key per commit, densely numbered. -- Documents store resolved deltas, and reads hit projections only. -- Keys sort lexicographically. -- Values are JSON-serialisable; byte drivers use a codec. - -## 11. Open questions to raise with the owner (don't block on them) - -- Order-preserving numeric **keys** (not just index values)? -- Should multi-writer safety be **required** (a hard error without `setIfAbsent`) rather than a documented single-writer fallback? -- The hard-delete story for immutable event logs (compaction or crypto-shredding) for privacy deletion requests. - -## 12. Implementation decisions - -Decisions taken where the brief was silent. Each is the simplest option consistent with sections 1–10. - -### M1 — Core - -- **Separator** must be a single printable ASCII character other than `%` and `~`. -- **Empty keys are rejected** (`InvalidKeyError`), as are `NaN`/`±Infinity` number keys. An empty segment - would produce `a//b`, which is ambiguous with the subtree layout. -- **Separator escaping** uses `%` + two uppercase hex digits (`/` → `%2F`, `%` → `%25`, leading `~` → `%7E`). -- **`InvalidKeyError` extends `TypeError`**, so "passed-but-undefined key throws a `TypeError`" (3.2) and - the typed error class (6) are the same throw. -- **Index string encoding deviates from "escaped like segments".** Percent-escaping is not - order-preserving (`"a/"` → `"a%2F"` sorts below `"a&"`, and `"a"` + sep sorts above `"a!"`), which - would break `findBy` range scans. Instead every character the index encoder emits sorts strictly - above the separator: code units `<= succ(sep)` are written as `succ(sep)` followed by - `chr(succ(sep) + 1 + c)`. This is monotone and prefix-safe, so `a < b ⇔ enc(a)+sep+… < enc(b)+sep+…` - exactly (property-tested). Encodings are still separator-free. -- **Type tags and number digits**: tags are the first five digits of the digit alphabet; numbers use - 16 digits of `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16 characters - just above the separator. `-0` is stored as `0`; `NaN` cannot be indexed. -- **Ordering** of strings is by UTF-16 code unit (JS `<`). Drivers that compare UTF-8 bytes agree - except for astral-plane characters vs. U+E000–U+FFFF; noted as a driver-author caveat. - -### M2 — KV driver - -- **Conformance suite location**: `src/testing/kv-conformance.ts`, exported as `potential-storage/testing`. - It imports `vitest`, which is an _optional peer dependency_; the core entry point never imports it. -- **`InMemoryKVDriver` clones values** on write and read (`structuredClone`, skipped for primitives) so - stored state can't be mutated through shared references, matching serialising drivers. Opt out - with `{ clone: false }`. -- **Scans tolerate concurrent writes**: the in-memory scan remembers the last key yielded and re-seeks - if the map changed while suspended. Read-repair and `deleteTree` rely on this. The conformance suite - requires it of every driver (deleting an already-yielded key mid-scan must not skip keys). -- **`setMany` bulk path**: ≥32 new keys are sorted and merged (O(n + k log k)); fewer are spliced. -- **`BaseKVDriver.deleteRange`** deletes in chunks of 1000 (scan with `limit`, then `deleteMany`, - repeat), so it never deletes from under its own live iterator. It also provides `get`/`set`/`delete` - single-key helpers. -- **No fake `batch`**: `BaseKVDriver` deliberately does not provide a non-atomic `batch`, since stores - use its presence to mean "atomic". - -### M3 — KV node - -- **Child order is "subtree order", not plain segment order.** The brief's skip-scan assumes a - child's keys are contiguous, but they aren't when a sibling extends a segment with a character - that sorts below the separator (for `/`: space and `!"#$%&'()*+,-.`). With children `a` and `a-1` - the key space is `p/a`, `p/a-1`, `p/a-1/x`, `p/a/x`, so `a`'s descendants sort _after_ `a-1`, and - "yield if different from the previous segment" would emit `a` twice. `keys()`/`entries()` therefore - order children by **escaped segment followed by the separator** (`a-1/` < `a/`), which is the order - of their subtrees in key space. For keys without such characters this is plain lexicographic order. - - Forward listing defers a child seen only through its own key on a small stack (bounded by - segment length) until the scan passes `p//`. It costs no extra driver calls beyond the - brief's one seek per child with descendants. - - Reverse listing needs one batched `getMany` probe per new child whose segment contains a - below-separator character, to find shorter siblings that only have an own key. - - `after` is exclusive in the direction of travel and follows the same order. - - Implemented once in `src/kv/children.ts` (`scanChildren`) and reused by the event and document - layers. A model-based property test checks forward, reverse, `after` and `limit` against - brute force with keys like `a`, `a-1`, `a.b`, `a/x`, `%`, `~t`. - - **Open question for the owner:** keep subtree order, or change segment escaping so the key - layout nests exactly. That would mean escaping every below-separator character with an escape - character above the separator, at the cost of the readable percent-encoding. -- **Errors from async methods are rejections.** Every async method (`get`, `set`, …) is an `async` - function, so a bad key yields a rejected promise rather than a synchronous throw. `child()` is sync - and throws synchronously. -- **`set(undefined)` is rejected** (`TypeError`: use `delete()`); `undefined` means "missing" in reads. -- **`deleteTree()` on the root** is allowed and clears the whole store (chunked scan + delete, since - the root range has no upper bound). Subtree deletion is `delete(selfKey)` plus - `deleteRange(prefix, selfKey + succ(sep))`. It is not `[selfKey, …)`, which would also catch - siblings such as `a!`. -- **`tree({ depth })`** builds a plain object: leaves are values, interior nodes are objects. A node - with both an own value and children exposes its own value under the `''` property (empty keys are - invalid, so it can't collide). Reserved subtrees are skipped. Deeper levels are skipped by seeking - rather than reading. -- **`keys()` and `entries()` yield string keys** (number keys come back as strings). -- **Exposed low-level properties** on every node: `path`, `selfKey`, `prefix`, `end` (subtree upper - bound), `driver`, `separator`, `isRoot`. The event and document layers build on these. - -### M4 — KV indexes - -- **Pattern syntax**: segments separated by the storage separator; `*` matches one segment; `''` - names the root. A literal `*` key can't be targeted by a pattern. -- **Overlapping patterns** that declare the same index name throw at construction. -- **Index trie state is stepped per `child()` call** (O(1) in patterns), so each node resolves its - indexes in O(depth) overall. A node holds two bound index sets: one over its children (for child - writes) and its parent's (for self-form writes such as `users.child('1').set(v)`). -- **Index function results**: `undefined` → not indexed; arrays are multi-entry and de-duplicated - per child; anything that is not `string | number | boolean | null` (or `NaN`) throws a `TypeError` - at write time. Index functions are not called for missing (`undefined`) values. -- **`findBy` results** are `[key, value]` pairs (the value is re-read anyway for verification). - Range queries accept `gt`/`gte`/`lt`/`lte`. Results are in index-value order, then child-key order, - and range queries de-duplicate multi-entry hits. -- **`findBy` batching**: entry keys are read in chunks of 128 (or of the remaining `limit`, if - smaller) and values fetched with one `getMany` per chunk. Stale entries are deleted with one - `deleteMany` per chunk. -- **Read-repair race (non-atomic drivers only)**: a reader can see a concurrent writer's fresh - index add before that writer's value write lands, judge it stale, and delete it. With multiple - writers on a driver without `batch`, this can lose an index entry. Mitigation: `readRepair: false` - on `KVStorage` (stale entries are then only skipped, never deleted). With `batch`, entries never go - stale, so the race doesn't arise. -- **`set` of an indexed child with duplicate keys in one `setMany`**: last write wins, and index - diffs are computed once per key. - -### M5 — Events - -- **`EventStorage` is an interface (type), not a class.** The brief's sample says `new EventStorage(kv)` - but also says it is the abstract interface; the concrete class is `KVEventStorage(kv, opts?)`, which - accepts a `KVStorage` or any `KVNode` as root. Options: `headCacheSize` (default 10 000), - `maxRetries` (default 5) and `now` (clock, default `Date.now`). -- **Commit value**: `{ firstVersion, at, events }`, with `at` in ms since epoch. Read results are - `{ version, commit, at, data }`. -- **The root node is not a stream** (consistent with the root having no own KV value). Non-root - nodes are both a stream and a container of child streams; `keys()` lists child streams, and a - node's `~e` subtree is reserved, so streams and children never mix. -- **`version()` always reads storage** (one reverse scan with `limit: 1`) and refreshes the head cache. - Only `append` trusts the cache. When a cached head disagrees with a numeric `expectedVersion`, - the head is re-read before throwing, so a stale cache never produces a false `ConcurrencyError`. -- **`append([])`** writes nothing; it still checks `expectedVersion` against a fresh head and returns it. -- **Read arity**: `read()` and `read(opts)` are the self form (the first argument is a non-null - object). Any other first argument is a key, so `read(undefined)` throws like every other - passed-but-undefined key. -- **Locating `from`/`to`**: commit `n` has `firstVersion >= n`, so the commit containing version `v` has - `seq <= v`. `read` probes `seq = v` first; with single-event commits that is the answer, in one - `getMany`. Otherwise it runs an 8-ary search, one `getMany` of up to 8 probes per round, which is - O(log₈ commits) round trips. The head is only read when the guess is past the end. -- **Mutex mode** (driver without `setIfAbsent`): a per-stream in-process promise chain serialises - appends. It is safe only with a single writer process. - -### M6 — Documents - -- **Projection value** is `{ state, version, commit }`, or `{ deleted: true, version, commit }` for a - tombstone. Tombstones keep the version, so re-creating a document continues its stream. -- **`expectedCommit` append hint.** Writes pass the projection's `commit` to `append` as - `expectedCommit`, so a write is exactly projection `getMany` + `setIfAbsent` + projection `batch` - (3 driver calls), even with a cold head cache. A hint taken from a projection can only be stale - (which causes a collision, then catch-up and retry), never ahead of the stream, so it cannot - create a gap. Non-KV backends may ignore it. -- **Per-document in-process write lock.** Writes to the same document within one storage instance - are serialised (`KeyedMutex`), so optimistic retries (default 5) are spent only on cross-process - contention. -- **Return values**: `set` → `void`; `insert` → id; `update` → new state, or `undefined` if the document - doesn't exist (pass `{ upsert: true }` to treat missing as `{}`); `delete` → whether a document - was deleted. No-op updates (resulting state equal to current) append nothing. -- **Arity for documents**: in forms with an optional leading key (`get`, `set`, `update`), a non-null - object first argument selects the self form, and any other first argument is a key (so `undefined` - throws). `delete`/`history` use argument count. `getAt(version)` vs `getAt(key, version)` also uses - argument count, because a version and a numeric key are both numbers. -- **The root document node has no own document** (consistent with KV). -- **`keys()` on a document node** lists children holding a live document. Unlike KV `keys()`, it skips - children that only have nested documents or tombstones. -- **`find` results** are `{ key, data }`. `findOne` returns the same shape or `undefined`. -- **Filter semantics vs MongoDB** (see `src/documents/filter.ts`): - - Comparisons are type-bracketed (`$gt: 5` matches only numbers), and cross-type order follows - the index encoding: `null/missing < false < true < numbers < strings < other`. - - On array fields, every positive operator in one condition (`$eq $gt $gte $lt $lte $in`) must be - satisfied by the **same element** (like `$elemMatch`). In MongoDB, `{ s: { $gt: 5, $lt: 10 } }` - matches `[1, 20]`; here it doesn't. This makes index range scans exact. To get MongoDB's - behaviour, split the condition with `$and`. - - `{ f: null }` matches missing fields. `$not` works both at top level (negating a filter) and at - field level. Unknown operators throw. - - `$in` is not index-accelerated in v1. -- **Index entries** for each field value: scalars as-is, arrays as their scalar elements (multikey; - empty arrays → `null`), and missing or non-scalar values as `null`. Every live document therefore - has at least one entry per index, so a full index scan sees all documents. Indexing a document - where more than one field of a compound index has multiple values throws (as MongoDB does). -- **Index names** default to the fields joined by `,`. -- **Unique indexes**: claims are scoped **per parent node** (`/~u//`), like one - MongoDB collection per parent. Value tuples where every field is missing/null are not claimed - (sparse). Claims are taken before appending, then released if the append fails. A claim value is - `[ownerKey, claimedAtMs]`. A claim held by another document is taken over only if that - document, after catching up from its stream, no longer holds the value, **and** the claim is older - than `claimGraceMs` (default 30 s). The owner might otherwise be between claiming and appending. - The residual risk is a writer stalling for longer than the grace period between claiming and - appending. -- **Planner**: per index, count leading equality fields (k), then whether field k has a range; - score = 2k + (range ? 1 : 0). Ties prefer an index whose order satisfies `sort` after dropping - equality-constrained sort fields. With no usable predicate, an index matching `sort` is still used - as an ordered full index scan, which streams and stops early with `limit`. Range bounds are clamped - to the bound's type class. Index-backed results are re-read from projections, re-checked against the - full filter, de-duplicated, and stale entries are read-repaired. -- **Result order** when `sort` is not given is the plan's order (index order, or child-key order for - a full scan). Ties in sorted results are broken by child key in the direction of the first sort - field, both for index-ordered and in-memory sorts. -- **In-memory sort** keeps a bounded top `skip + limit` buffer when a limit is given. Without a limit, - it buffers all matches; this is the one place results are materialised. -- **Full-scan warning**: `console.warn` once per filter _shape_ (values replaced by `?`) per storage - instance, only for non-empty filters. On by default unless `NODE_ENV === 'production'`; set it with - `warnOnFullScan`. -- **Catch-up** re-asserts unique claims for the caught-up state with plain writes, since the log is - the source of truth. A slow catch-up can briefly overwrite a newer projection with an older one; - the next write's `expectedVersion` check detects this and heals it. `catchUp()` walks the subtree - via event-stream `keys()` and returns the number of documents repaired. -- **Projections for non-KV event stores** must be passed explicitly (`projections: KVNode`); the - projection key of a document is then `projections.prefix + docKey`. - -### M7 — Types - -- **`AnySpec` uses an index signature** (`{ [key: string]: AnySpec }`), not `Record`, so - `KeyOf` is `string | number` and number keys type-check on untyped stores. -- **`ChildOf` guards with `infer C extends Spec`** so that `KVNode>` satisfies the `Spec` - constraint generically. -- **`setMany` entries are a union of precise `[key, value]` pairs** (`EntryOf`), so on named levels a - value must match its own key (`['age', 'x']` is rejected). -- **Listings**: `keys()` yields `` `${KeyOf}` `` (string literals on named levels, `string` on dynamic - ones). `entries()` and `findBy()` yield `[string, V]` on dynamic levels. -- **"Accept any node" parameters** are typed `KVNode` / `EventNode`. A typed node is - assignable to these, but not to `KVNode`, because `KeyOf` collapses to `never`. -- **Type tests** live in `test/types.test-d.ts` and are enforced by `tsc --noEmit` (part of `npm test`), - including the `@ts-expect-error` negative cases from section 7. - -### M8 — HTTP driver - -- **Extra endpoints** beyond the brief's list: `POST /deleteRange {gte, lt}` and - `GET /capabilities`. Unsupported optional operations return `501`, driver errors `500`, and unknown - routes `404`. -- **Missing vs `null`**: `/getMany` returns `{ values, missing }`, where `missing` lists the indices of - absent keys, because `null` is a legitimate stored value. -- **Codec scope**: the codec serialises whole request/response bodies and each NDJSON scan line - (`[key]` or `[key, value]`), so it must not emit newlines. The default is JSON. -- **Capabilities on the client**: `HttpKVDriver` defines `setIfAbsent`/`batch` only when the server - supports them, so stores' feature detection (e.g. unique indexes requiring `setIfAbsent`, atomic - index writes using `batch`) behaves as it would locally. `HttpKVDriver.connect()` discovers them via - `/capabilities`. The constructor takes them explicitly (default: all). Without native - `deleteRange` the client falls back to `BaseKVDriver`'s chunked delete. -- **Scan streaming**: the server pulls the driver iterator in chunks that grow from 16 to 256 - entries, so short reads (skip-scan re-seeks) don't make it run far ahead. Cancelling the client - iterator cancels the response body, which calls `return()` on the server-side iterator. -- **Optional `headers`** (e.g. auth) and an injectable `fetch` on the client; an optional `basePath` on - the handler. -- The conformance suite runs end-to-end in-process (fetch → handler), over a real `node:http` - socket, and against a minimal server driver with discovered capabilities. A smoke test runs KV - indexes, events and unique-indexed documents over HTTP. - -### M9 — Benchmarks and in-memory driver performance - -- **The in-memory driver's sorted array was replaced by `ChunkedSortedMap`** (sorted chunks of ≤1024 - keys, split in half on overflow; values in a `Map`). Profiling showed 59% of document-update time - in the sorted array's O(n) `splice` once the store held ~70k keys (each write adds commit, index - and claim keys). Inserts and deletes are now O(log n + 1024). Iteration is O(1) per step and - re-seeks after a mutation, which replaces the old version-counter revalidation in the driver. - The structure stays behind the `SortedMap` interface; a B-tree remains possible. -- **Value cloning uses a hand-written JSON deep copy** instead of `structuredClone` (about 10× cheaper - per value), in the in-memory driver and the document layer. Values are JSON by contract (§10). - - - -## 13. Benchmark baselines - -`npm run bench` (Vitest bench), in-memory driver with default cloning, Node 22.22, Linux x64 cloud -container (shared CPU, so expect ±10–20% run-to-run). Mean time per operation. - -**KV** (`bench/kv.bench.ts`): values are small objects. - -| Operation | 10k keys | 100k keys | -| -------------------- | -------- | --------- | -| `get` | 1.2 µs | 1.6 µs | -| `set` (overwrite) | 1.7 µs | 2.0 µs | -| `getMany` × 100 keys | 21 µs | 22 µs | - -**`keys()`** (`bench/keys.bench.ts`): node with 1 000 children, each with 100 descendants (101k keys). - -| Operation | Mean | -| -------------------------------- | ------ | -| `keys()` skip-scan (1 000 seeks) | 2.9 ms | -| Reference: full subtree scan | 36 ms | - -**Events** (`bench/events.bench.ts`): single-event commits. - -| Operation | Mean | -| ---------------------------------------------- | ------ | -| `append` 1 event (warm head cache) | 6.3 µs | -| `read` 100 events from version 5 000 of 10 000 | 174 µs | -| `read` full 10 000-event stream | 21 ms | -| `version()` (reverse scan, `limit: 1`) | 3.9 µs | - -**Documents** (`bench/documents.bench.ts`): 10 000 docs, indexes on `age` and unique `email`. - -| Operation | Mean | -| ----------------------------------------------------------------- | ------- | -| `get` | 2.6 µs | -| `update` (`$inc`, indexed node: projection read + append + batch) | 22 µs | -| `findOne` by unique index (`email`) | 16 µs | -| `find` indexed (`age = k`, ~100 hits) | 0.57 ms | -| `find` full scan (`city = k`, ~200 hits of 10k) | 28 ms | - -For comparison, the first run with the sorted-array map and `structuredClone` measured: KV `get` -3.7 µs, `getMany`×100 186 µs, `append` 34 µs, document `update` 110 µs, full scan 57 ms, and -full stream read 62 ms. +## 3. Keys (`utils/keys.ts`) + +- A node's key is its escaped path segments joined by `/` (`users/123`). The separator is fixed. +- **Escaping:** every character at or below `/` (space, `!"#$%&'()*+,-.`, `/`, control characters) + and `~` itself becomes `~XX` (uppercase hex). Escaped segments contain only characters above `/`, + so all keys of one child sort together (`p/a`, `p/a/…`, then `p/a~2D1` for `a-1`). That keeps + child listing a plain skip-scan. +- **Reserved segments** are `~` + a lowercase letter (`~e` = event commits). Escaping only produces + `~` + uppercase hex, so user keys can't collide. +- Keys sort lexicographically by escaped form (`10` before `9`; `a-1` → `a~2D1` after `a0`). +- Empty keys, `NaN`/`Infinity` and non-string/number keys are rejected. + +## 4. KV storage + +`BaseKVStorage` primitives (absolute keys): + +| Primitive | | +| ------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `readMany(keys)` | values in order, `undefined` for missing | +| `writeMany(entries)` | | +| `removeMany(keys)` | | +| `writeIfAbsent(entries)` | `true` = written; atomic per key (needed by events) | +| `scan({ gte, lt, limit, reverse, values })` | key-ordered; must keep working if keys change mid-scan (keys present throughout appear exactly once) | + +Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / `set(key, value)`, +`setMany`, `delete(key?)`, `deleteMany`, `deleteTree(key?)`, `keys({ limit, after, reverse })`, +`entries(...)`. The root has no own value. A node's own value and its children are independent keys. + +- **`keys()`** is a skip-scan: one seek per child that has descendants; siblings without + descendants share one iterator. It lists children that only have descendants, and skips + reserved segments. +- **The only KV "index" is the ordered key space.** There is no `find`: with KV you know where to look, + or you list a node's children. +- **Implementations:** `KVStorageInMemory` (`SortedMap`: sorted chunks of ≤1024 keys; values deep-copied); + `KVStorageHttp` (client) + `KVStorageHttpServer` (wraps any KV storage; `handle(Request) → Response`, + `POST /` with JSON bodies, `scan` streamed as NDJSON). The server exposes the whole + underlying store (primitives use absolute keys). +- **Wrappers** (e.g. a future encryption layer) extend `BaseKVStorage` and delegate the primitives. + Keys must stay plaintext (or be deterministically encrypted) for prefix scans to work; values can be + encrypted freely. + +## 5. Event storage + +`BaseEventStorage` primitives over stream paths: `appendTo`, `readFrom`, `versionOf`, `streamsOf`. +Node API: `append(events, opts?)` / `append(key, events, opts?)`, `read(opts?)` / `read(key, opts?)`, +`version(key?)`, `keys()`. The root is not a stream. An empty stream is version 0; the first event is 1. +`expectedVersion: number | 'any'` (default `'any'`). + +**`EventStorageKV`** (on any KV storage): + +- One key per append, densely numbered: `/~e/` → `{ firstVersion, at, events }`. + Keying by commit number (not version) means any writer with a stale view collides with the + existing key. +- Append: head from a bounded cache (else one reverse scan, `limit: 1`) → check `expectedVersion` + (re-reading the head before rejecting, in case the cache is stale) → `writeIfAbsent`. On a lost race: + `ConcurrencyError` if a version was expected, otherwise retry (default 5). +- `read({ from })`: commit `n` starts at a version ≥ n, so try commit `from` first (exact for + single-event commits), else binary-search; then forward-scan. +- Multi-writer safe on any store, because `writeIfAbsent` is a required primitive. + +## 6. Document storage + +`BaseDocumentStorage` primitives over document paths: + +| Primitive | | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| `readMany(paths)` | | +| `mutate(path, change)` | maps current state → `DocChange` (or `null` = no-op), applies it, returns the new state; the implementation decides how it's made safe | +| `query(parent, DocumentQuery)` | children of `parent`, filtered, sorted and paged | + +Node API: `get(key?)`, `getMany`, `set(doc)` / `set(key, doc)`, `insert(doc)` → ULID-like id, +`update(spec, opts?)` / `update(key, spec, opts?)` → new state or `undefined` (`{ upsert }`), +`delete(key?)` → existed, `find(filter, { sort, limit, skip })`, `findOne`, `keys()`. + +- **Queries** (`utils/document-query.ts`): `{ field: value }` (dot paths; `null` also matches missing), + `$eq $gt $gte $lt $lte $in`. Comparisons are type-bracketed. On arrays a condition matches the array + or any one element. Sorting order: missing/null < booleans < numbers < strings < other. +- **Updates** (`utils/document-update.ts`): `$set $unset $inc $push $pull $addToSet` (`$each` for + push/addToSet). They resolve to a `DocChange` with final values (`$inc` → `set: { n: 6 }`), so + changes replay deterministically. +- **Indexes:** `{ indexes: { users: ['email', 'age'], 'users/*/posts': ['createdAt'] } }`, passed to + the implementation. + +**Implementations** + +- `DocumentStorageInMemory`: maps per collection; equality index per declared field (used for + `{ f: v }` / `{ f: { $eq: v } }`), otherwise scans. Writes are atomic in-process. +- `DocumentStorageKV`: each document is the KV value at its path; queries scan the collection's + children; no indexes; concurrent writes to one document are last-writer-wins. +- `EventSourcedDocumentStorage(events, projections)`: each document's `DocChange`s are appended to + its own stream. Current state lives in **any** `BaseDocumentStorage` used as the projection store, + which serves all reads, queries and indexes. + - Projections carry `_version` (reserved; stripped from results), used as `expectedVersion` on the + next append. + - If a write crashes between append and projection update, the next write conflicts, rebuilds the + projection from the stream, and retries. `catchUp(key?)` rebuilds on demand. + - Missing projections with stream history (deleted documents, crashed first writes) are rebuilt + before writing. + - Writes to one document are serialised in-process, so retries only happen across processes. + - Extras: `history(key?)`, `getAt(version)` / `getAt(key, version)`. Children keep these methods + in their types through interface declaration merging (types only). + +## 7. Changes from the original brief + +- Drivers + node classes → one abstract class per layer (client and server share an API). +- KV indexes (`findBy`, pattern trie, order-preserving value encoding, read-repair) removed. + Document indexes are declared as fields and handled by each store; unique indexes and the query + planner were removed. +- Queries: `$ne $nin $exists $and $or $not` and projections dropped. +- Separator fixed to `/`, and every character ≤ `/` is escaped so child keys stay contiguous. This + removed the subtree-order workaround in child listing. +- Event-sourced documents are events + a projection store instead of a bespoke projection format. +- `writeIfAbsent` is a required primitive (no single-writer mutex fallback; no optional `batch` / + `deleteRange`). +- `tree()` removed (use `entries()` for field-per-key reads). + +## 8. Open questions + +- Hard deletes from immutable event logs (privacy requests): compaction vs crypto-shredding. The + encryption wrapper could support per-subject keys. +- Order-preserving numeric keys (`9` before `10`) as an opt-in key encoding. + +## 9. Benchmark baselines + +`npm run bench`, in-memory stores, Node 22, Linux x64 cloud container (shared CPU, ±10–20%). Mean per operation. + +| Benchmark | Mean | +| -------------------------------------------------------------------- | --------------- | +| KV `get` / `set` @ 10k keys | 1.5 / 1.7 µs | +| KV `get` / `set` @ 100k keys | 2.0 / 2.3 µs | +| KV `getMany` × 100 | 25–35 µs | +| `keys()` on 1k children × 100 descendants (skip-scan vs full scan) | 2.8 ms vs 34 ms | +| Event `append` (warm cache) | 7.8 µs | +| Event `read` 100 from version 5 000 of 10k | 131 µs | +| Event `read` full 10k stream | 18 ms | +| Documents in-memory @ 10k: `get` / `update` | 1.8 / 6.6 µs | +| Documents in-memory: `find` indexed (~100 hits) / scan (~200 of 10k) | 0.20 / 11 ms | +| Event-sourced @ 10k: `get` / `update` | 1.6 / 17 µs | +| Event-sourced: `find` indexed / scan | 0.38 / 12 ms | diff --git a/bench/document-storage.bench.ts b/bench/document-storage.bench.ts new file mode 100644 index 0000000..064a92f --- /dev/null +++ b/bench/document-storage.bench.ts @@ -0,0 +1,33 @@ +import { bench, describe } from 'vitest' +import { DocumentStorageInMemory } from '../src/document-storage/document-storage-in-memory.js' +import { EventSourcedDocumentStorage } from '../src/document-storage/event-sourced-document-storage.js' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' +import type { BaseDocumentStorage } from '../src/document-storage/base-document-storage.js' + +const N = 10_000 +const stores: [string, BaseDocumentStorage][] = [ + ['in-memory', new DocumentStorageInMemory({ indexes: { users: ['age'] } })], + ['event-sourced', new EventSourcedDocumentStorage(new EventStorageKV(new KVStorageInMemory()), new DocumentStorageInMemory({ indexes: { users: ['age'] } }))], +] +for (const [name, docs] of stores) { + const users = docs.child('users') + for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, age: i % 100, city: `c${i % 50}` }) + let i = 0 + describe(`documents (${name}) @ 10k`, () => { + bench('get', async () => { + await users.get(`u${i++ % N}`) + }) + bench('update ($inc)', async () => { + await users.update(`u${i++ % N}`, { $inc: { logins: 1 } }) + }) + bench('find by indexed field (~100 hits)', async () => { + for await (const _ of users.find({ age: i++ % 100 })) { + } + }) + bench('find by unindexed field (scan, ~200 hits)', async () => { + for await (const _ of users.find({ city: `c${i++ % 50}` })) { + } + }) + }) +} diff --git a/bench/documents.bench.ts b/bench/documents.bench.ts deleted file mode 100644 index 29aefdb..0000000 --- a/bench/documents.bench.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' -import { KVEventStorage } from '../src/events/kv.js' -import { EventSourcedDocumentStorage } from '../src/documents/event-sourced.js' - -const N = 10_000 -const docs = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { - indexes: { users: [{ fields: ['age'] }, { fields: ['email'], unique: true }] }, - warnOnFullScan: false, -}) -const users = docs.child('users') -for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, email: `u${i}@x.io`, age: i % 100, city: `c${i % 50}` }) -let i = 0 - -describe(`documents @ ${N / 1000}k docs`, () => { - bench('get', async () => { - await users.get(`u${i++ % N}`) - }) - bench('update ($inc, indexed node)', async () => { - await users.update(`u${i++ % N}`, { $inc: { logins: 1 } }) - }) - bench('findOne by unique index (email)', async () => { - await users.findOne({ email: `u${i++ % N}@x.io` }) - }) - bench('find indexed (age = k, ~100 hits)', async () => { - for await (const _ of users.find({ age: i++ % 100 })) { - } - }) - bench('find full scan (city = k, ~200 hits)', async () => { - for await (const _ of users.find({ city: `c${i++ % 50}` })) { - } - }) -}) diff --git a/bench/event-storage.bench.ts b/bench/event-storage.bench.ts new file mode 100644 index 0000000..0c1a695 --- /dev/null +++ b/bench/event-storage.bench.ts @@ -0,0 +1,22 @@ +import { bench, describe } from 'vitest' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' + +const events = new EventStorageKV(new KVStorageInMemory()) +const streams = events.child('streams') +for (let i = 0; i < 10_000; i++) await streams.append('long', [{ i }]) +let s = 0 + +describe('events', () => { + bench('append 1 event (warm cache)', async () => { + await streams.append(`s${s++ % 1000}`, [{ type: 'e' }]) + }) + bench('read 100 events from version 5000 of 10k', async () => { + for await (const _ of streams.read('long', { from: 5000, limit: 100 })) { + } + }) + bench('read full 10k-event stream', async () => { + for await (const _ of streams.read('long')) { + } + }) +}) diff --git a/bench/events.bench.ts b/bench/events.bench.ts deleted file mode 100644 index 22200cc..0000000 --- a/bench/events.bench.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' -import { KVEventStorage } from '../src/events/kv.js' - -const events = new KVEventStorage(new KVStorage(new InMemoryKVDriver())) -const streams = events.child('streams') -const long = streams.child('long') -for (let i = 0; i < 10_000; i++) await long.append([{ type: 'tick', i }]) -let s = 0 - -describe('events', () => { - bench('append 1 event (warm head cache)', async () => { - await streams.append(`s${s++ % 1000}`, [{ type: 'e' }]) - }) - bench('read 100 events from version 5000 of 10k', async () => { - for await (const _ of long.read({ from: 5000, limit: 100 })) { - } - }) - bench('read full 10k-event stream', async () => { - for await (const _ of long.read()) { - } - }) - bench('version() (cold head lookup)', async () => { - await long.version() - }) -}) diff --git a/bench/keys.bench.ts b/bench/keys.bench.ts deleted file mode 100644 index dac2c87..0000000 --- a/bench/keys.bench.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' - -// 1k children, each with 100 descendants (100k keys under the node). -const db = new KVStorage(new InMemoryKVDriver({ clone: false })) -const node = db.child('n') -const writes: [string, number][] = [] -for (let c = 0; c < 1000; c++) { - writes.push([`n/c${c}`, c]) - for (let d = 0; d < 100; d++) writes.push([`n/c${c}/d${d}`, d]) -} -await db.driver.setMany(writes) - -describe('keys() on 1k children x 100 descendants', () => { - bench('keys() skip-scan', async () => { - let n = 0 - for await (const _ of node.keys()) n++ - if (n !== 1000) throw new Error(`expected 1000, got ${n}`) - }) - bench('naive full subtree scan (reference)', async () => { - let n = 0 - for await (const _ of db.driver.scan({ gte: 'n/', lt: 'n0' })) n++ - if (n !== 101_000) throw new Error(`expected 101000, got ${n}`) - }) -}) diff --git a/bench/kv-storage.bench.ts b/bench/kv-storage.bench.ts new file mode 100644 index 0000000..8db217c --- /dev/null +++ b/bench/kv-storage.bench.ts @@ -0,0 +1,40 @@ +import { bench, describe } from 'vitest' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' + +for (const n of [10_000, 100_000]) { + const db = new KVStorageInMemory() + await db.writeMany(Array.from({ length: n }, (_, i) => [`users/u${i}`, { name: `user ${i}`, n: i }])) + const users = db.child('users') + const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`) + let i = 0 + describe(`KV @ ${n / 1000}k keys`, () => { + bench('get', async () => { + await users.get(`u${i++ % n}`) + }) + bench('set', async () => { + await users.set(`u${i++ % n}`, { name: 'x', n: i }) + }) + bench('getMany x100', async () => { + await users.getMany(keys) + }) + }) +} + +const tree = new KVStorageInMemory() +const writes: [string, number][] = [] +for (let c = 0; c < 1000; c++) { + writes.push([`n/c${c}`, c]) + for (let d = 0; d < 100; d++) writes.push([`n/c${c}/d${d}`, d]) +} +await tree.writeMany(writes) + +describe('keys() on 1k children x 100 descendants', () => { + bench('keys() skip-scan', async () => { + for await (const _ of tree.child('n').keys()) { + } + }) + bench('full subtree scan (reference)', async () => { + for await (const _ of tree.scan({ gte: 'n/', lt: 'n0' })) { + } + }) +}) diff --git a/bench/kv.bench.ts b/bench/kv.bench.ts deleted file mode 100644 index f293231..0000000 --- a/bench/kv.bench.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { bench, describe } from 'vitest' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVStorage } from '../src/kv/node.js' - -async function populated(n: number) { - const db = new KVStorage(new InMemoryKVDriver()) - const users = db.child('users') - const batch: [string, { name: string; n: number }][] = [] - for (let i = 0; i < n; i++) batch.push([`u${i}`, { name: `user ${i}`, n: i }]) - await users.setMany(batch) - return users -} - -for (const n of [10_000, 100_000]) { - const users = await populated(n) - const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`) - let i = 0 - describe(`KV @ ${n / 1000}k keys`, () => { - bench('get', async () => { - await users.get(`u${i++ % n}`) - }) - bench('set (overwrite)', async () => { - await users.set(`u${i++ % n}`, { name: 'x', n: i }) - }) - bench('getMany x100', async () => { - await users.getMany(keys) - }) - }) -} diff --git a/package.json b/package.json index 5046331..1f1e75a 100644 --- a/package.json +++ b/package.json @@ -12,26 +12,6 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" - }, - "./kv": { - "types": "./dist/kv/index.d.ts", - "import": "./dist/kv/index.js" - }, - "./events": { - "types": "./dist/events/index.d.ts", - "import": "./dist/events/index.js" - }, - "./documents": { - "types": "./dist/documents/index.d.ts", - "import": "./dist/documents/index.js" - }, - "./http": { - "types": "./dist/http/index.d.ts", - "import": "./dist/http/index.js" - }, - "./testing": { - "types": "./dist/testing/index.d.ts", - "import": "./dist/testing/index.js" } }, "files": [ @@ -53,13 +33,5 @@ "prettier": "^3.9.10", "typescript": "^5.9.3", "vitest": "^3.2.7" - }, - "peerDependencies": { - "vitest": ">=3" - }, - "peerDependenciesMeta": { - "vitest": { - "optional": true - } } } diff --git a/src/core/encoding.ts b/src/core/encoding.ts deleted file mode 100644 index b7642f7..0000000 --- a/src/core/encoding.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { DEFAULT_SEPARATOR, succ } from './path.js' - -export type IndexValue = string | number | boolean | null - -/** - * Order-preserving, separator-safe encoding of index values. - * - * Every character the encoder emits sorts strictly above the separator. That makes encodings - * prefix-safe inside keys: `enc(a) + sep + …` sorts below `enc(b)` whenever `enc(a)` is a proper - * prefix of `enc(b)`, so `a < b ⇔ key(a, …) < key(b, …)` and range scans over index entries are exact. - * - * Layout: one type-tag character, then a body. - * - Type tags order `null < false < true < number < string`. - * - Numbers: IEEE-754 float64, big-endian; positive values have the sign bit flipped, negative - * values have all bits flipped; written as 16 fixed-width "hex" digits. `-0` is normalised to `0`; - * `NaN` is rejected. - * - Strings: each code unit `c` with `c <= E` (where `E = succ(sep)`) is written as `E` followed by - * `chr(E + 1 + c)`; every other code unit is written as-is. This is monotone, so order is kept. - * - * The 16 digits are `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16 - * characters immediately after the separator. - */ -export interface IndexValueCodec { - readonly sep: string - encode(value: IndexValue): string - decode(encoded: string): IndexValue -} - -const codecs = new Map() - -export function indexValueCodec(sep: string = DEFAULT_SEPARATOR): IndexValueCodec { - let codec = codecs.get(sep) - if (!codec) { - codec = makeCodec(sep) - codecs.set(sep, codec) - } - return codec -} - -export function encodeIndexValue(value: IndexValue, sep: string = DEFAULT_SEPARATOR): string { - return indexValueCodec(sep).encode(value) -} - -export function decodeIndexValue(encoded: string, sep: string = DEFAULT_SEPARATOR): IndexValue { - return indexValueCodec(sep).decode(encoded) -} - -/** Encodes a compound value as `enc(v1) + sep + enc(v2) + …`. */ -export function encodeIndexValues(values: readonly IndexValue[], sep: string = DEFAULT_SEPARATOR): string { - const codec = indexValueCodec(sep) - let out = '' - for (let i = 0; i < values.length; i++) { - if (i > 0) out += sep - out += codec.encode(values[i]!) - } - return out -} - -export function isIndexValue(v: unknown): v is IndexValue { - return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v)) -} - -function makeCodec(sep: string): IndexValueCodec { - const sepCode = sep.charCodeAt(0) - const digits = sepCode < 0x30 ? '0123456789abcdef' : Array.from({ length: 16 }, (_, i) => String.fromCharCode(sepCode + 1 + i)).join('') - const digitValue = new Map([...digits].map((d, i) => [d, i])) - const TAG_NULL = digits[0]! - const TAG_FALSE = digits[1]! - const TAG_TRUE = digits[2]! - const TAG_NUMBER = digits[3]! - const TAG_STRING = digits[4]! - const esc = succ(sep) - const escCode = esc.charCodeAt(0) - - const view = new DataView(new ArrayBuffer(8)) - - function hex32(n: number): string { - let out = '' - for (let shift = 28; shift >= 0; shift -= 4) out += digits[(n >>> shift) & 0xf] - return out - } - - function encodeNumber(n: number): string { - if (Number.isNaN(n)) throw new TypeError('NaN cannot be indexed') - view.setFloat64(0, n === 0 ? 0 : n) // normalise -0 - let hi = view.getUint32(0) - let lo = view.getUint32(4) - if (hi & 0x80000000) { - hi = ~hi >>> 0 - lo = ~lo >>> 0 - } else { - hi = (hi ^ 0x80000000) >>> 0 - } - return hex32(hi) + hex32(lo) - } - - function decodeNumber(body: string): number { - if (body.length !== 16) throw new TypeError(`Bad encoded number ${JSON.stringify(body)}`) - let hi = 0 - let lo = 0 - for (let i = 0; i < 8; i++) hi = (hi << 4) | digitValue.get(body[i]!)! - for (let i = 8; i < 16; i++) lo = (lo << 4) | digitValue.get(body[i]!)! - hi >>>= 0 - lo >>>= 0 - if (hi & 0x80000000) { - hi = (hi ^ 0x80000000) >>> 0 - } else { - hi = ~hi >>> 0 - lo = ~lo >>> 0 - } - view.setUint32(0, hi) - view.setUint32(4, lo) - return view.getFloat64(0) - } - - function encodeString(s: string): string { - let out = '' - let start = 0 - for (let i = 0; i < s.length; i++) { - const c = s.charCodeAt(i) - if (c <= escCode) { - out += s.slice(start, i) + esc + String.fromCharCode(escCode + 1 + c) - start = i + 1 - } - } - return start === 0 ? s : out + s.slice(start) - } - - function decodeString(body: string): string { - if (body.indexOf(esc) === -1) return body - let out = '' - for (let i = 0; i < body.length; i++) { - const c = body[i]! - if (c === esc) { - const next = body.charCodeAt(++i) - out += String.fromCharCode(next - escCode - 1) - } else out += c - } - return out - } - - return { - sep, - encode(value) { - if (value === null) return TAG_NULL - switch (typeof value) { - case 'boolean': - return value ? TAG_TRUE : TAG_FALSE - case 'number': - return TAG_NUMBER + encodeNumber(value) - case 'string': - return TAG_STRING + encodeString(value) - default: - throw new TypeError(`Cannot index value of type ${typeof value}`) - } - }, - decode(encoded) { - const tag = encoded[0] - const body = encoded.slice(1) - switch (tag) { - case TAG_NULL: - return null - case TAG_FALSE: - return false - case TAG_TRUE: - return true - case TAG_NUMBER: - return decodeNumber(body) - case TAG_STRING: - return decodeString(body) - default: - throw new TypeError(`Bad encoded index value ${JSON.stringify(encoded)}`) - } - }, - } -} diff --git a/src/core/lru.ts b/src/core/lru.ts deleted file mode 100644 index c66c07a..0000000 --- a/src/core/lru.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** Minimal bounded LRU over `Map` insertion order. */ -export class LRU { - readonly #map = new Map() - - constructor(readonly capacity: number) {} - - get(key: K): V | undefined { - const v = this.#map.get(key) - if (v !== undefined) { - this.#map.delete(key) - this.#map.set(key, v) - } - return v - } - - set(key: K, value: V): void { - if (this.capacity <= 0) return - this.#map.delete(key) - this.#map.set(key, value) - if (this.#map.size > this.capacity) this.#map.delete(this.#map.keys().next().value as K) - } - - delete(key: K): void { - this.#map.delete(key) - } - - get size(): number { - return this.#map.size - } -} diff --git a/src/core/mutex.ts b/src/core/mutex.ts deleted file mode 100644 index 3df05ab..0000000 --- a/src/core/mutex.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** Per-key in-process mutex: calls with the same key run one at a time, in order. */ -export class KeyedMutex { - readonly #tails = new Map>() - - run(key: string, fn: () => Promise): Promise { - const prev = this.#tails.get(key) ?? Promise.resolve() - const result = prev.then(fn, fn) - const tail = result.catch(() => {}) - this.#tails.set(key, tail) - void tail.then(() => { - if (this.#tails.get(key) === tail) this.#tails.delete(key) - }) - return result - } -} diff --git a/src/core/path.ts b/src/core/path.ts deleted file mode 100644 index bceeef9..0000000 --- a/src/core/path.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { InvalidKeyError } from '../errors.js' - -export const DEFAULT_SEPARATOR = '/' - -/** Reserved segments. User segments can never start with `~` (a leading `~` is escaped). */ -export const RESERVED = { - index: '~i', - unique: '~u', - events: '~e', - projections: '~p', -} as const - -export type Key = string | number - -/** The character immediately after `c` (by UTF-16 code unit). */ -export function succ(c: string): string { - return String.fromCharCode(c.charCodeAt(0) + 1) -} - -export function validateSeparator(sep: string): void { - if (sep.length !== 1) throw new TypeError(`Separator must be a single character, got ${JSON.stringify(sep)}`) - const code = sep.charCodeAt(0) - if (code <= 0x20 || code >= 0x7f || sep === '%' || sep === '~') { - throw new TypeError(`Separator must be printable ASCII other than "%" and "~", got ${JSON.stringify(sep)}`) - } -} - -function hex2(code: number): string { - return '%' + code.toString(16).toUpperCase().padStart(2, '0') -} - -/** Converts a user key to a segment string, rejecting `undefined`/`null`/empty. */ -export function toSegment(key: unknown): string { - if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`) - if (typeof key === 'number') { - if (!Number.isFinite(key)) throw new InvalidKeyError(`Numeric key must be finite, got ${key}`) - return String(key) - } - if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or number, got ${typeof key}`) - if (key === '') throw new InvalidKeyError('Key must not be empty') - return key -} - -/** - * Escapes one path segment: percent-encodes `%`, the separator, and a leading `~`. - * The result never contains a raw separator and never starts with `~`. - */ -export function escapeSegment(segment: string, sep: string = DEFAULT_SEPARATOR): string { - // Fast path: nothing to escape (the overwhelmingly common case). - if (segment.indexOf('%') === -1 && segment.indexOf(sep) === -1 && segment.charCodeAt(0) !== 0x7e) return segment - let out = '' - for (let i = 0; i < segment.length; i++) { - const c = segment[i]! - if (c === '%' || c === sep || (i === 0 && c === '~')) out += hex2(c.charCodeAt(0)) - else out += c - } - return out -} - -export function unescapeSegment(escaped: string): string { - if (escaped.indexOf('%') === -1) return escaped - return escaped.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16))) -} - -export function isReservedSegment(escaped: string): boolean { - return escaped.charCodeAt(0) === 0x7e // '~' -} - -/** Joins unescaped segments into an escaped key. */ -export function joinPath(path: readonly string[], sep: string = DEFAULT_SEPARATOR): string { - let out = '' - for (let i = 0; i < path.length; i++) { - if (i > 0) out += sep - out += escapeSegment(path[i]!, sep) - } - return out -} - -/** Splits an escaped key into unescaped segments. */ -export function splitPath(key: string, sep: string = DEFAULT_SEPARATOR): string[] { - if (key === '') return [] - return key.split(sep).map(unescapeSegment) -} - -/** - * Precomputed key strings for a node path. - * - `selfKey`: escaped segments joined by the separator (`''` for the root) - * - `prefix`: `selfKey + sep` (`''` for the root) — every descendant key starts with it - * - `end`: exclusive upper bound of the subtree range (`selfKey + succ(sep)`; `undefined` for the root) - */ -export interface PathKeys { - readonly path: readonly string[] - readonly sep: string - readonly selfKey: string - readonly prefix: string - readonly end: string | undefined -} - -export function pathKeys(path: readonly string[], sep: string): PathKeys { - const selfKey = joinPath(path, sep) - const root = path.length === 0 - return { - path, - sep, - selfKey, - prefix: root ? '' : selfKey + sep, - end: root ? undefined : selfKey + succ(sep), - } -} diff --git a/src/core/trie.ts b/src/core/trie.ts deleted file mode 100644 index f377f18..0000000 --- a/src/core/trie.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Segment trie over path patterns. `*` matches exactly one segment. - * A node's matching state is computed incrementally: `step(parentState, segment)`, - * so resolving a path is O(depth) and `child()` is O(1) in the number of patterns. - */ -export interface TrieNode { - readonly literal: Map> - star: TrieNode | undefined - values: { pattern: string; value: T }[] -} - -export type TrieState = readonly TrieNode[] - -export class PatternTrie { - readonly root: TrieNode = newNode() - - constructor(readonly sep: string) {} - - /** `''` names the root node. */ - add(pattern: string, value: T): void { - let node = this.root - for (const seg of splitPattern(pattern, this.sep)) { - if (seg === '*') node = node.star ??= newNode() - else { - let next = node.literal.get(seg) - if (!next) node.literal.set(seg, (next = newNode())) - node = next - } - } - node.values.push({ pattern, value }) - } - - get rootState(): TrieState { - return [this.root] - } - - step(state: TrieState, segment: string): TrieState { - if (state.length === 0) return state - if (state.length === 1) { - const n = state[0]! - const lit = n.literal.get(segment) - if (lit && n.star) return [lit, n.star] - return lit ? [lit] : n.star ? [n.star] : EMPTY - } - const out: TrieNode[] = [] - for (const n of state) { - const lit = n.literal.get(segment) - if (lit) out.push(lit) - if (n.star) out.push(n.star) - } - return out - } - - resolve(path: readonly string[]): TrieState { - let state = this.rootState - for (const seg of path) state = this.step(state, seg) - return state - } - - /** Values attached to every pattern matching the state's path. */ - static values(state: TrieState): { pattern: string; value: T }[] { - if (state.length === 1) return state[0]!.values - return state.flatMap((n) => n.values) - } -} - -const EMPTY: TrieState = [] - -function newNode(): TrieNode { - return { literal: new Map(), star: undefined, values: [] } -} - -export function splitPattern(pattern: string, sep: string): string[] { - return pattern === '' ? [] : pattern.split(sep) -} - -/** Whether two patterns can match the same path. */ -export function patternsOverlap(a: string, b: string, sep: string): boolean { - const sa = splitPattern(a, sep) - const sb = splitPattern(b, sep) - return sa.length === sb.length && sa.every((s, i) => s === '*' || sb[i] === '*' || s === sb[i]) -} diff --git a/src/document-storage/base-document-storage.ts b/src/document-storage/base-document-storage.ts new file mode 100644 index 0000000..7730ebf --- /dev/null +++ b/src/document-storage/base-document-storage.ts @@ -0,0 +1,117 @@ +import type { Doc } from '../utils/document-path.js' +import { DocumentQuery, type Filter, type FindOptions, type FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange, type UpdateSpec } from '../utils/document-update.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt, ValueOf } from '../utils/schema.js' +import { Ulid } from '../utils/ulid.js' + +/** + * Fields to index, per collection path pattern (`*` matches one segment), e.g. + * `{ users: ['email', 'age'], 'users/*\/posts': ['createdAt'] }`. Each implementation decides + * how (or whether) to use them; a native database maps them onto its own indexes. + */ +export type IndexDeclarations = Record + +export interface UpdateOptions { + /** Treat a missing document as `{}`. */ + upsert?: boolean +} + +/** Document type of a node's children. */ +export type ChildDoc = ValueOf>> + +/** + * Document storage. Every node is a document (its own) and a collection of child documents. + * Implementations provide three primitives over document paths; the user-facing API is + * defined here once. + */ +export abstract class BaseDocumentStorage { + /** Unescaped path of this node (`[]` for the root, which has no own document). */ + path: readonly string[] = [] + protected readonly ids = new Ulid() + + abstract readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> + /** + * Maps the current state of one document to a change (`null` = no change), applies it and + * returns the new state. Implementations decide how this is made safe against concurrent writers. + */ + abstract mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise + /** Children of `parent` matching the query, filtered, sorted and paged. */ + abstract query(parent: readonly string[], query: DocumentQuery): AsyncIterable + + child>(key: K): BaseDocumentStorage> { + const node = Object.create(this) as BaseDocumentStorage> + node.path = [...this.path, Keys.segment(key)] + return node + } + + as(): BaseDocumentStorage { + return this as unknown as BaseDocumentStorage + } + + async get | undefined = undefined>(key?: K): Promise | undefined> { + const [doc] = await this.readMany([this.pathOf(arguments.length > 0, key)]) + return doc as ValueAt | undefined + } + + async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { + return (await this.readMany(keys.map((k) => this.pathOf(true, k)))) as (ValueOf> | undefined)[] + } + + /** `set(doc)` replaces this document; `set(key, doc)` a child document. */ + async set>(...args: [doc: ValueOf] | [key: K, doc: ValueOf>]): Promise { + const doc = args[args.length - 1] + if (typeof doc !== 'object' || doc === null || Array.isArray(doc)) throw new TypeError('A document must be a plain object') + await this.mutate(this.pathOf(args.length > 1, args[0]), () => ({ op: 'set', doc: doc as Doc })) + } + + /** Adds a child document under a new time-sortable id and returns the id. */ + async insert(doc: ChildDoc): Promise { + const id = this.ids.generate() + await this.set(id as KeyOf, doc as never) + return id + } + + /** + * `update(spec, options?)` this document; `update(key, spec, options?)` a child. + * Returns the new state, or `undefined` if the document doesn't exist (unless `upsert`). + */ + async update>( + ...args: [spec: UpdateSpec, options?: UpdateOptions] | [key: K, spec: UpdateSpec, options?: UpdateOptions] + ): Promise | undefined> { + const hasKey = typeof args[0] !== 'object' || args[0] === null + const spec = args[hasKey ? 1 : 0] as UpdateSpec + const options = (args[hasKey ? 2 : 1] ?? {}) as UpdateOptions + const next = await this.mutate(this.pathOf(hasKey, args[0]), (current) => (current || options.upsert ? DocumentUpdate.resolve(current ?? {}, spec) : null)) + return next as ValueAt | undefined + } + + /** Deletes this document (or a child). Returns whether one existed. */ + async delete(key?: KeyOf): Promise { + let existed = false + await this.mutate(this.pathOf(arguments.length > 0, key), (current) => ((existed = current !== undefined) ? { op: 'delete' } : null)) + return existed + } + + /** Child documents matching `filter`. */ + find(filter: Filter = {}, options: FindOptions = {}): AsyncIterable>> { + return this.query(this.path, new DocumentQuery(filter, options)) as AsyncIterable>> + } + + async findOne(filter: Filter = {}, options: FindOptions = {}): Promise> | undefined> { + for await (const result of this.find(filter, { ...options, limit: 1 })) return result + return undefined + } + + /** Keys of child documents. */ + async *keys(): AsyncGenerator { + for await (const { key } of this.find()) yield key + } + + /** Path of a child (`hasKey`) or this node; validates keys. */ + protected pathOf(hasKey: boolean, key: unknown): readonly string[] { + if (hasKey) return [...this.path, Keys.segment(key)] + if (this.path.length === 0) throw new TypeError('The root node has no own document') + return this.path + } +} diff --git a/src/document-storage/document-storage-in-memory.ts b/src/document-storage/document-storage-in-memory.ts new file mode 100644 index 0000000..8cff273 --- /dev/null +++ b/src/document-storage/document-storage-in-memory.ts @@ -0,0 +1,112 @@ +import type { Doc } from '../utils/document-path.js' +import { DocumentPath } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { Json } from '../utils/json.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseDocumentStorage, type IndexDeclarations } from './base-document-storage.js' + +export interface DocumentStorageInMemoryOptions { + /** Fields with an equality index, per collection pattern. */ + indexes?: IndexDeclarations +} + +/** Field → JSON-encoded value → child keys. */ +type CollectionIndex = Map>> + +/** + * In-memory document storage. Declared index fields get an equality index per collection, used + * when a filter has `{ field: value }` or `{ field: { $eq: value } }`; everything else is a scan. + */ +export class DocumentStorageInMemory extends BaseDocumentStorage { + private readonly collections = new Map>() + private readonly indexes = new Map() + private readonly declarations: [pattern: string[], fields: string[]][] + + constructor(options: DocumentStorageInMemoryOptions = {}) { + super() + this.declarations = Object.entries(options.indexes ?? {}).map(([pattern, fields]) => [pattern === '' ? [] : pattern.split('/'), fields]) + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return paths.map((path) => Json.clone(this.collection(path.slice(0, -1)).get(path.at(-1)!))) + } + + async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const parent = path.slice(0, -1) + const key = path.at(-1)! + const docs = this.collection(parent) + const current = docs.get(key) + const resolved = change(Json.clone(current)) + if (!resolved) return Json.clone(current) + const next = DocumentUpdate.apply(current, resolved) + if (next) docs.set(key, next) + else docs.delete(key) + this.reindex(parent, key, current, next) + return Json.clone(next) + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + const docs = this.collection(parent) + const keys = [...(this.lookup(parent, query) ?? docs.keys())].sort() + yield* query.run( + (async function* () { + for (const key of keys) { + const doc = docs.get(key) + if (doc) yield { key, data: Json.clone(doc) } + } + })(), + ) + } + + private collection(parent: readonly string[]): Map { + const id = Keys.join(parent) + let docs = this.collections.get(id) + if (!docs) this.collections.set(id, (docs = new Map())) + return docs + } + + /** Candidate keys from an equality index, if the filter allows one. */ + private lookup(parent: readonly string[], query: DocumentQuery): Iterable | undefined { + const index = this.indexes.get(Keys.join(parent)) + for (const field of this.fieldsFor(parent)) { + let value = query.filter[field] + if (Json.isObject(value) && Object.keys(value).length === 1 && '$eq' in value) value = value.$eq + if (value === undefined || typeof value === 'object') continue + return index?.get(field)?.get(JSON.stringify(value)) ?? [] + } + return undefined + } + + private reindex(parent: readonly string[], key: string, before: Doc | undefined, after: Doc | undefined): void { + const fields = this.fieldsFor(parent) + if (fields.length === 0) return + const id = Keys.join(parent) + let index = this.indexes.get(id) + if (!index) this.indexes.set(id, (index = new Map())) + for (const field of fields) { + let byValue = index.get(field) + if (!byValue) index.set(field, (byValue = new Map())) + for (const value of DocumentStorageInMemory.indexValues(before, field)) byValue.get(value)?.delete(key) + for (const value of DocumentStorageInMemory.indexValues(after, field)) { + let keys = byValue.get(value) + if (!keys) byValue.set(value, (keys = new Set())) + keys.add(key) + } + } + } + + private fieldsFor(parent: readonly string[]): string[] { + return this.declarations + .filter(([pattern]) => pattern.length === parent.length && pattern.every((s, i) => s === '*' || s === parent[i])) + .flatMap(([, fields]) => fields) + } + + /** Index keys for a field: its value, plus each element of an array (missing indexes as `null`). */ + private static indexValues(doc: Doc | undefined, field: string): string[] { + if (!doc) return [] + const value = DocumentPath.get(doc, field) ?? null + return Array.isArray(value) ? value.map((v) => JSON.stringify(v)) : [JSON.stringify(value)] + } +} diff --git a/src/document-storage/document-storage-kv.ts b/src/document-storage/document-storage-kv.ts new file mode 100644 index 0000000..7a915b4 --- /dev/null +++ b/src/document-storage/document-storage-kv.ts @@ -0,0 +1,46 @@ +import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js' +import type { Doc } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseDocumentStorage } from './base-document-storage.js' + +/** + * Document storage on any KV storage: each document is the value at its path. Queries scan the + * collection's direct children (no indexes). Writes are read-modify-write, so concurrent writers + * to the same document are last-writer-wins; use `EventSourcedDocumentStorage` if that matters. + */ +export class DocumentStorageKV extends BaseDocumentStorage { + constructor(private readonly kv: BaseKVStorage) { + super() + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return (await this.kv.readMany(paths.map((p) => this.keyOf(p)))) as (Doc | undefined)[] + } + + async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const key = this.keyOf(path) + const [current] = (await this.kv.readMany([key])) as (Doc | undefined)[] + const resolved = change(current) + if (!resolved) return current + const next = DocumentUpdate.apply(current, resolved) + if (next) await this.kv.writeMany([[key, next]]) + else await this.kv.removeMany([key]) + return next + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + const collection = parent.reduce((node, segment) => node.child(segment), this.kv) + yield* query.run( + (async function* () { + for await (const [key, data] of collection.entries()) yield { key, data: data as Doc } + })(), + ) + } + + private keyOf(path: readonly string[]): string { + return this.kv.prefix + Keys.join(path) + } +} diff --git a/src/document-storage/event-sourced-document-storage.ts b/src/document-storage/event-sourced-document-storage.ts new file mode 100644 index 0000000..d9f27c8 --- /dev/null +++ b/src/document-storage/event-sourced-document-storage.ts @@ -0,0 +1,125 @@ +import type { BaseEventStorage, StoredEvent } from '../event-storage/base-event-storage.js' +import type { Doc } from '../utils/document-path.js' +import type { DocumentQuery, FindResult } from '../utils/document-query.js' +import { DocumentUpdate, type DocChange } from '../utils/document-update.js' +import { ConcurrencyError } from '../utils/errors.js' +import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt } from '../utils/schema.js' +import { BaseDocumentStorage } from './base-document-storage.js' + +export interface EventSourcedDocumentStorageOptions { + /** Retries after a write conflict, each preceded by a catch-up. Default 5. */ + maxRetries?: number +} + +// Types only: children of an event-sourced store are event-sourced stores too (child() creates +// them with Object.create, so they share this class's prototype). +export interface EventSourcedDocumentStorage { + child>(key: K): EventSourcedDocumentStorage> +} + +/** + * Documents as event streams: each document's changes are appended to its own stream, and the + * current state is kept in a projection store (any `BaseDocumentStorage`), which serves all + * reads and queries. + * + * Projections carry a `_version` field (stripped from results) used as `expectedVersion` on the + * next append. If a write crashes between appending and updating the projection, the next + * write's append conflicts, the projection is rebuilt from the stream, and the write retries. + * `_version` is therefore reserved in documents. + */ +export class EventSourcedDocumentStorage extends BaseDocumentStorage { + private readonly maxRetries: number + /** Tail of the pending writes per document, so writes in one process don't conflict with each other. */ + private readonly locks = new Map>() + + constructor( + private readonly events: BaseEventStorage, + private readonly projections: BaseDocumentStorage, + options: EventSourcedDocumentStorageOptions = {}, + ) { + super() + this.maxRetries = options.maxRetries ?? 5 + } + + async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> { + return (await this.projections.readMany(paths)).map((doc) => EventSourcedDocumentStorage.strip(doc)) + } + + async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator { + for await (const { key, data } of this.projections.query(parent, query)) yield { key, data: EventSourcedDocumentStorage.strip(data)! } + } + + mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + const id = JSON.stringify(path) + const result = (this.locks.get(id) ?? Promise.resolve()).then(() => this.write(path, change)) + const tail = result.catch(() => {}) + this.locks.set(id, tail) + void tail.then(() => this.locks.get(id) === tail && this.locks.delete(id)) + return result + } + + private async write(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { + for (let attempt = 0; ; attempt++) { + const [stored] = await this.projections.readMany([path]) + const version = (stored?._version as number | undefined) ?? (await this.events.versionOf(path)) + if (!stored && version > 0) { + // No projection but the stream has history (deleted, or a crashed first write): rebuild. + if ((await this.rebuild(path)) !== undefined) continue + } + const current = EventSourcedDocumentStorage.strip(stored) + const resolved = change(current) + if (!resolved) return current + const next = DocumentUpdate.apply(current, resolved) + try { + await this.events.appendTo(path, [resolved], version) + } catch (error) { + if (!(error instanceof ConcurrencyError) || attempt >= this.maxRetries) throw error + await this.rebuild(path) + continue + } + await this.project(path, next, version + 1) + return next + } + } + + /** Events of this document (or a child). */ + history(key?: KeyOf): AsyncIterable> { + return this.events.readFrom(this.pathOf(arguments.length > 0, key), {}) as AsyncIterable> + } + + /** State as of `version`: `getAt(version)` for this document, `getAt(key, version)` for a child. */ + async getAt>(...args: [version: number] | [key: K, version: number]): Promise | undefined> { + const path = this.pathOf(args.length > 1, args[0]) + let state: Doc | undefined + for await (const event of this.events.readFrom(path, { to: args[args.length - 1] as number })) { + state = DocumentUpdate.apply(state, event.data as DocChange) + } + return state as ValueAt | undefined + } + + /** Rebuilds the projection of this document (or a child) from its stream. Returns the state. */ + catchUp(key?: KeyOf): Promise { + return this.rebuild(this.pathOf(arguments.length > 0, key)) + } + + private async rebuild(path: readonly string[]): Promise { + let state: Doc | undefined + let version = 0 + for await (const event of this.events.readFrom(path, {})) { + state = DocumentUpdate.apply(state, event.data as DocChange) + version = event.version + } + await this.project(path, state, version) + return state + } + + private async project(path: readonly string[], state: Doc | undefined, version: number): Promise { + await this.projections.mutate(path, () => (state ? { op: 'set', doc: { ...state, _version: version } } : { op: 'delete' })) + } + + private static strip(doc: Doc | undefined): Doc | undefined { + if (!doc) return undefined + const { _version, ...state } = doc + return state + } +} diff --git a/src/documents/event-sourced.ts b/src/documents/event-sourced.ts deleted file mode 100644 index a79f3a8..0000000 --- a/src/documents/event-sourced.ts +++ /dev/null @@ -1,606 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ, toSegment, unescapeSegment } from '../core/path.js' -import type { TrieState } from '../core/trie.js' -import { CapabilityError, ConcurrencyError, UniqueConstraintError } from '../errors.js' -import { KeyedMutex } from '../core/mutex.js' -import { KVEventNode } from '../events/kv.js' -import type { EventNode, ReadOptions, StoredEvent } from '../events/types.js' -import { scanChildren } from '../kv/children.js' -import type { BatchOp, KVDriver } from '../kv/driver.js' -import type { KVNode } from '../kv/node.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import { analyze, matches, type Filter } from './filter.js' -import { DocIndexes, DocIndexRegistry, type DocIndexDeclarations, type Plan } from './indexes.js' -import type { ChildDoc, DocKeysOptions, EventSourcedDocumentNode, FindOptions, FindResult, GetOptions, UpdateOptions } from './types.js' -import { ulid } from './ulid.js' -import { applyEvent, applyPatch, resolveUpdate, toPatchEvent, type DocEvent, type UpdateSpec } from './update.js' -import { clone, compareValues, deepEqual, getPath, isPlainObject, setPath, sortKey, unsetPath, type Doc } from './values.js' - -export interface EventSourcedDocumentStorageOptions { - /** Where projections live. Default: the `~p` root of the event store's KV. Required for non-KV event stores. */ - projections?: KVNode - /** Path pattern → indexes over that node's direct children. */ - indexes?: DocIndexDeclarations - /** Delete stale index entries found by `find`. Default `true`. */ - readRepair?: boolean - /** Retries after a write conflict (each preceded by a catch-up). Default 5. */ - maxRetries?: number - /** Warn once per filter shape when `find` falls back to a full scan. Default: `NODE_ENV !== 'production'`. */ - warnOnFullScan?: boolean - /** Id generator for `insert`. Default: ULID-like. */ - newId?: () => string - /** - * A unique claim whose owner no longer holds the value is only taken over once it is older - * than this, since its owner may be mid-write. Default 30 000 ms. - */ - claimGraceMs?: number - /** Clock for claim timestamps. Default `Date.now`. */ - now?: () => number -} - -/** Stored projection of one document. */ -export type Projection = - { state: Doc; version: number; commit: number; deleted?: undefined } | { deleted: true; version: number; commit: number; state?: undefined } - -interface DocContext { - readonly driver: KVDriver - readonly sep: string - /** Projection-space prefix: projection key of a doc = `projBase + docSelfKey`. */ - readonly projBase: string - readonly registry: DocIndexRegistry - readonly readRepair: boolean - readonly maxRetries: number - readonly warnOnFullScan: boolean - readonly warned: Set - readonly newId: () => string - readonly claimGraceMs: number - readonly now: () => number - /** Serialises writes to one document within this process, so retries only arise across processes. */ - readonly writeLock: KeyedMutex -} - -/** Stored value of a unique claim: owner child segment (escaped) and claim time. */ -type Claim = [owner: string, at: number] - -interface Target { - ev: EventNode - parentEv: EventNode - esc: string - /** Projection key of the document. */ - projKey: string - /** Projection-space prefix of its parent (sibling projection key = `parentProj + esc`). */ - parentProj: string - idx: DocIndexes | undefined -} - -type Hit = { esc: string; state: Doc } - -const FIND_CHUNK = 128 - -/** - * Document node backed by an event stream per document and a projection per document. - * Writes: read projection → resolve delta → append (expectedVersion = projection.version) → - * write projection + index entries. Reads hit the projection only. - */ -export class EventSourcedDocumentNodeImpl implements EventSourcedDocumentNode { - readonly path: readonly string[] - /** Document-space key (escaped), relative to the store root. */ - readonly selfKey: string - readonly #ctx: DocContext - readonly #ev: EventNode - readonly #parentEv: EventNode | undefined - readonly #esc: string - readonly #prefix: string - readonly #state: TrieState - readonly #selfIdx: DocIndexes | undefined - readonly #childIdx: DocIndexes | undefined - - /** @internal Use `EventSourcedDocumentStorage` and `child()`. */ - constructor( - ctx: DocContext, - ev: EventNode, - parentEv: EventNode | undefined, - path: readonly string[], - selfKey: string, - esc: string, - state: TrieState, - selfIdx: DocIndexes | undefined, - ) { - this.#ctx = ctx - this.#ev = ev - this.#parentEv = parentEv - this.path = path - this.selfKey = selfKey - this.#esc = esc - this.#prefix = path.length === 0 ? '' : selfKey + ctx.sep - this.#state = state - this.#selfIdx = selfIdx - this.#childIdx = ctx.registry.bind(state, ctx.projBase + this.#prefix) - } - - child>(key: K): EventSourcedDocumentNodeImpl> { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - return new EventSourcedDocumentNodeImpl( - this.#ctx, - this.#ev.child(seg), - this.#ev, - [...this.path, seg], - this.#prefix + esc, - esc, - this.#ctx.registry.trie.step(this.#state, seg), - this.#childIdx, - ) - } - - as(): EventSourcedDocumentNodeImpl { - return this as unknown as EventSourcedDocumentNodeImpl - } - - // ---------------------------------------------------------------- reads - - get(opts?: GetOptions): Promise | undefined> - get>(key: K, opts?: GetOptions): Promise> | undefined> - async get(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const opts = (rest[0] ?? {}) as GetOptions - let proj = await this.#projection(t) - if (opts.consistent && (await t.ev.version()) > (proj?.version ?? 0)) { - await this.#catchUpDoc(t) - proj = await this.#projection(t) - } - return proj?.state - } - - history(): AsyncIterable> - history>(key: K): AsyncIterable> - history(...args: unknown[]): AsyncIterable> { - const t = args.length === 0 ? this.#self() : this.#child(args[0]) - return t.ev.read() as AsyncIterable> - } - - getAt(version: number): Promise | undefined> - getAt>(key: K, version: number): Promise> | undefined> - async getAt(...args: unknown[]): Promise { - const t = args.length <= 1 ? this.#self() : this.#child(args[0]) - const version = args[args.length - 1] as number - if (!Number.isInteger(version) || version < 0) throw new TypeError(`Invalid version ${String(version)}`) - let state: Doc | undefined - for await (const e of t.ev.read({ to: version } satisfies ReadOptions)) state = applyEvent(state, e.data as DocEvent) - return state - } - - async *find(filter: Filter = {}, opts: FindOptions = {}): AsyncGenerator>> { - const skip = opts.skip ?? 0 - const limit = opts.limit ?? Infinity - if (limit <= 0) return - const sort = Object.entries(opts.sort ?? {}) as [string, 1 | -1][] - const plan = this.#childIdx?.plan(analyze(filter), sort) - if (!plan && Object.keys(filter).length > 0) this.#warnFullScan(filter) - const source = plan ? this.#indexScan(plan, skip + limit) : this.#projectionScan() - const project = opts.projection ? projector(opts.projection) : (d: Doc) => d - - if (sort.length && !plan?.sorted) { - // In-memory sort, bounded to skip + limit when a limit is given. - const cap = skip + limit - const cmp = comparator(sort) - const top: Hit[] = [] - for await (const hit of source) { - if (!matches(hit.state, filter)) continue - if (top.length >= cap && cmp(hit, top[top.length - 1]!) >= 0) continue - let lo = 0 - let hi = top.length - while (lo < hi) { - const mid = (lo + hi) >>> 1 - if (cmp(top[mid]!, hit) <= 0) lo = mid + 1 - else hi = mid - } - top.splice(lo, 0, hit) - if (top.length > cap) top.pop() - } - for (let i = skip; i < top.length; i++) yield { key: unescapeSegment(top[i]!.esc), data: project(top[i]!.state) as ChildDoc } - return - } - - let skipped = 0 - let emitted = 0 - for await (const hit of source) { - if (!matches(hit.state, filter)) continue - if (skipped < skip) { - skipped++ - continue - } - yield { key: unescapeSegment(hit.esc), data: project(hit.state) as ChildDoc } - if (++emitted >= limit) return - } - } - - async findOne(filter: Filter = {}, opts: FindOptions = {}): Promise> | undefined> { - for await (const r of this.find(filter, { ...opts, limit: 1 })) return r - return undefined - } - - async *keys(opts: DocKeysOptions = {}): AsyncGenerator> { - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - const after = opts.after === undefined ? undefined : toSegment(opts.after) - const { prefix, end } = this.#childRange() - for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { after, reverse: opts.reverse, values: true })) { - if (!hit.hasOwn || (hit.value as Projection).deleted) continue - yield unescapeSegment(hit.esc) as KeyString - if (--remaining <= 0) return - } - } - - // ---------------------------------------------------------------- writes - - set(doc: ValueOf): Promise - set>(key: K, doc: ValueOf>): Promise - async set(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const doc = rest[0] - if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object') - const snapshot = clone(doc) - await this.#write(t, () => ({ op: 'set', doc: snapshot })) - } - - async insert(doc: ChildDoc): Promise { - if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object') - const id = this.#ctx.newId() - const t = this.#child(id) - const snapshot = clone(doc) - await this.#write(t, (cur) => { - if (cur) throw new Error(`Document "${id}" already exists`) - return { op: 'set', doc: snapshot } - }) - return id - } - - update(spec: UpdateSpec, opts?: UpdateOptions): Promise | undefined> - update>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise> | undefined> - async update(...args: unknown[]): Promise { - const [t, rest] = this.#split(args) - const spec = rest[0] as UpdateSpec - const opts = (rest[1] ?? {}) as UpdateOptions - if (!isPlainObject(spec)) throw new TypeError('update() expects an update spec') - const result = await this.#write(t, (cur) => { - if (!cur && !opts.upsert) return null - const base = cur ?? {} - const patch = resolveUpdate(base, spec) - if (cur && deepEqual(applyPatch(base, patch), cur)) return null // no-op: no event - return toPatchEvent(patch) - }) - return result - } - - delete(): Promise - delete>(key: K): Promise - async delete(...args: unknown[]): Promise { - const t = args.length === 0 ? this.#self() : this.#child(args[0]) - let deleted = false - await this.#write(t, (cur) => { - deleted = cur !== undefined - return cur ? { op: 'delete' } : null - }) - return deleted - } - - // ---------------------------------------------------------------- maintenance - - async catchUp(): Promise { - let repaired = 0 - if (this.path.length > 0 && (await this.#catchUpDoc(this.#self()))) repaired++ - for await (const key of this.#ev.keys()) repaired += await this.child(key as KeyOf).catchUp() - return repaired - } - - // ---------------------------------------------------------------- internals - - #self(): Target { - if (this.path.length === 0) throw new TypeError('The root node has no own document') - const parentProj = this.#ctx.projBase + this.selfKey.slice(0, this.selfKey.length - this.#esc.length) - return { ev: this.#ev, parentEv: this.#parentEv!, esc: this.#esc, projKey: this.#ctx.projBase + this.selfKey, parentProj, idx: this.#selfIdx } - } - - #child(key: unknown): Target { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - const parentProj = this.#ctx.projBase + this.#prefix - return { ev: this.#ev.child(seg), parentEv: this.#ev, esc, projKey: parentProj + esc, parentProj, idx: this.#childIdx } - } - - /** Target for a sibling of `t` given its escaped segment. */ - #sibling(t: Target, esc: string): Target { - return { ev: t.parentEv.child(unescapeSegment(esc)), parentEv: t.parentEv, esc, projKey: t.parentProj + esc, parentProj: t.parentProj, idx: t.idx } - } - - /** Self form when there are no args or the first is a non-null object; otherwise key form. */ - #split(args: unknown[]): [Target, unknown[]] { - if (args.length === 0 || (typeof args[0] === 'object' && args[0] !== null)) return [this.#self(), args] - return [this.#child(args[0]), args.slice(1)] - } - - #childRange(): { prefix: string; end: string | undefined } { - const prefix = this.#ctx.projBase + this.#prefix - return { prefix, end: prefix === '' ? undefined : prefix.slice(0, -1) + succ(this.#ctx.sep) } - } - - async #projection(t: Target): Promise { - return (await this.#ctx.driver.getMany([t.projKey]))[0] as Projection | undefined - } - - /** - * The write path. `compute` maps the current state to an event (or `null` for a no-op). - * Returns the resulting state. - */ - #write(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise { - return this.#ctx.writeLock.run(t.projKey, () => this.#writeUnlocked(t, compute)) - } - - async #writeUnlocked(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise { - const { maxRetries } = this.#ctx - for (let attempt = 0; ; attempt++) { - const proj = await this.#projection(t) - const cur = proj?.state - const event = compute(cur) - if (!event) return cur - const next = applyEvent(cur, event) - - const oldClaims = t.idx?.claimsFor(cur) - const newClaims = t.idx?.claimsFor(next) - const acquired: string[] = [] - if (newClaims?.size) { - const wanted = new Map([...newClaims].filter(([k]) => !oldClaims!.has(k))) - if (wanted.size) await this.#claim(t, wanted, acquired) - } - - let result - try { - result = await t.ev.append([event], { expectedVersion: proj?.version ?? 0, expectedCommit: proj?.commit ?? 0 }) - } catch (err) { - if (acquired.length) await this.#ctx.driver.deleteMany(acquired) - if (err instanceof ConcurrencyError && attempt < maxRetries) { - await this.#catchUpDoc(t) - continue - } - throw err - } - const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : [] - const projection: Projection = next - ? { state: next, version: result.version, commit: result.commit } - : { deleted: true, version: result.version, commit: result.commit } - await this.#writeProjection(t, cur, next, projection, releases) - return next - } - } - - /** - * Claims unique values for `t`; records successful claims in `acquired`. Throws - * `UniqueConstraintError` on conflict. A claim held by another document is taken over only if - * that document, caught up to its stream head, no longer holds the value and the claim is older - * than `claimGraceMs` (its owner might otherwise be between claiming and appending). - */ - async #claim(t: Target, claims: Map, acquired: string[]): Promise { - const { driver, now, claimGraceMs } = this.#ctx - const keys = [...claims.keys()] - const written = await driver.setIfAbsent!(keys.map((k) => [k, [t.esc, now()] satisfies Claim])) - const lost = keys.filter((k, i) => { - if (written[i]) acquired.push(k) - return !written[i] - }) - if (!lost.length) return - const holders = (await driver.getMany(lost)) as (Claim | undefined)[] - const fail = async (k: string, owner?: string): Promise => { - if (acquired.length) await driver.deleteMany(acquired.splice(0)) - const c = claims.get(k)! - throw new UniqueConstraintError(c.index, c.values, owner === undefined ? undefined : unescapeSegment(owner)) - } - for (let i = 0; i < lost.length; i++) { - const k = lost[i]! - const holder = holders[i] - if (holder?.[0] === t.esc) continue // already ours (e.g. left by an interrupted write) - if (holder) { - const owner = this.#sibling(t, holder[0]) - await this.#catchUpDoc(owner) - const ownerProj = await this.#projection(owner) - if (t.idx!.claimsFor(ownerProj?.state).has(k)) await fail(k, holder[0]) - if (now() - holder[1] < claimGraceMs) await fail(k, holder[0]) - await driver.deleteMany([k]) - } - const [ok] = await driver.setIfAbsent!([[k, [t.esc, now()] satisfies Claim]]) - if (!ok) await fail(k) - acquired.push(k) - } - } - - async #writeProjection(t: Target, cur: Doc | undefined, next: Doc | undefined, projection: Projection, releases: string[]): Promise { - const { driver } = this.#ctx - if (!t.idx) { - await driver.setMany([[t.projKey, projection]]) - return - } - const before = t.idx.entriesFor(t.esc, cur) - const after = t.idx.entriesFor(t.esc, next) - const adds = [...after].filter((e) => !before.has(e)) - const removes = [...before].filter((e) => !after.has(e)).concat(releases) - if (driver.batch) { - const ops: BatchOp[] = adds.map((key) => ({ type: 'set', key, value: true })) - ops.push({ type: 'set', key: t.projKey, value: projection }) - for (const key of removes) ops.push({ type: 'delete', key }) - await driver.batch(ops) - return - } - if (adds.length) await driver.setMany(adds.map((k) => [k, true])) - await driver.setMany([[t.projKey, projection]]) - if (removes.length) await driver.deleteMany(removes) - } - - /** Replays events newer than the projection into it. Returns whether anything changed. */ - async #catchUpDoc(t: Target): Promise { - const proj = await this.#projection(t) - const cur = proj?.state - let state = cur - let version = proj?.version ?? 0 - let commit = proj?.commit ?? 0 - for await (const e of t.ev.read({ from: version + 1 })) { - state = applyEvent(state, e.data as DocEvent) - version = e.version - commit = e.commit - } - if (version === (proj?.version ?? 0)) return false - const oldClaims = t.idx?.claimsFor(cur) - const newClaims = t.idx?.claimsFor(state) - if (newClaims?.size) { - // The log is the source of truth: (re)assert claims for the caught-up state. - const missing = [...newClaims.keys()].filter((k) => !oldClaims!.has(k)) - if (missing.length) await this.#ctx.driver.setMany(missing.map((k) => [k, [t.esc, this.#ctx.now()] satisfies Claim])) - } - const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : [] - const projection: Projection = state ? { state, version, commit } : { deleted: true, version, commit } - await this.#writeProjection(t, cur, state, projection, releases) - return true - } - - async *#projectionScan(): AsyncGenerator { - const { prefix, end } = this.#childRange() - for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { values: true })) { - const p = hit.value as Projection | undefined - if (hit.hasOwn && p?.state) yield { esc: hit.esc, state: p.state } - } - } - - async *#indexScan(plan: Plan, want: number): AsyncGenerator { - const idx = this.#childIdx! - const { driver, readRepair } = this.#ctx - const parentProj = this.#ctx.projBase + this.#prefix - const seen = new Set() - let chunk: string[] = [] - let chunkSize = Math.min(FIND_CHUNK, Math.max(1, want)) - const it = driver.scan({ gte: plan.gte, lt: plan.lt, reverse: plan.reverse })[Symbol.asyncIterator]() - - async function* flush(): AsyncGenerator { - const entries = chunk - chunk = [] - chunkSize = FIND_CHUNK - const escs = entries.map((e) => idx.childOf(plan.index, e)) - const projs = (await driver.getMany(escs.map((e) => parentProj + e))) as (Projection | undefined)[] - const stale: string[] = [] - try { - for (let i = 0; i < entries.length; i++) { - const esc = escs[i]! - const state = projs[i]?.state - if (!idx.entriesFor(esc, state).has(entries[i]!)) { - stale.push(entries[i]!) - continue - } - if (seen.has(esc)) continue - seen.add(esc) - yield { esc, state: state! } - } - } finally { - if (readRepair && stale.length) await driver.deleteMany(stale) - } - } - - try { - for (;;) { - const r = await it.next() - if (r.done) break - chunk.push(r.value[0]) - if (chunk.length >= chunkSize) yield* flush() - } - if (chunk.length) yield* flush() - } finally { - await it.return?.() - } - } - - #warnFullScan(filter: Filter): void { - if (!this.#ctx.warnOnFullScan) return - const shape = JSON.stringify(filterShape(filter)) - if (this.#ctx.warned.has(shape)) return - this.#ctx.warned.add(shape) - console.warn(`[potential-storage] find() on "${this.selfKey || '(root)'}" has no usable index; full scan for filter ${shape}`) - } -} - -/** Filter with values replaced by `?`, for de-duplicating warnings. */ -function filterShape(f: unknown): unknown { - if (Array.isArray(f)) return f.map(filterShape) - if (isPlainObject(f)) - return Object.fromEntries( - Object.entries(f).map(([k, v]) => [ - k, - k.startsWith('$') && (k === '$and' || k === '$or' || k === '$not') ? filterShape(v) : isPlainObject(v) ? filterShape(v) : '?', - ]), - ) - return '?' -} - -function comparator(sort: [string, 1 | -1][]): (a: Hit, b: Hit) => number { - const tieDir = sort[0]?.[1] ?? 1 - return (a, b) => { - for (const [field, dir] of sort) { - const d = compareValues(sortKey(a.state, field, dir), sortKey(b.state, field, dir)) - if (d !== 0) return d * dir - } - return a.esc === b.esc ? 0 : (a.esc < b.esc ? -1 : 1) * tieDir - } -} - -function projector(spec: Record): (d: Doc) => Doc { - const fields = Object.entries(spec) - const include = fields.some(([, v]) => v === 1 || v === true) - if (include && fields.some(([, v]) => v === 0 || v === false)) throw new TypeError('Projection cannot mix inclusion and exclusion') - return (d) => { - if (include) { - const out: Doc = {} - for (const [f] of fields) { - const v = getPath(d, f) - if (v !== undefined) setPath(out, f, clone(v)) - } - return out - } - const out = clone(d) - for (const [f] of fields) unsetPath(out, f) - return out - } -} - -/** Event-sourced document storage: the root document node. */ -export class EventSourcedDocumentStorage extends EventSourcedDocumentNodeImpl { - constructor(events: EventNode, opts: EventSourcedDocumentStorageOptions = {}) { - let driver: KVDriver - let sep: string - let projBase: string - if (opts.projections) { - driver = opts.projections.driver - sep = opts.projections.separator - projBase = opts.projections.prefix - } else if (events instanceof KVEventNode) { - const kv = events.kv - driver = kv.driver - sep = kv.separator - projBase = kv.prefix + RESERVED.projections + sep - } else { - throw new TypeError('EventSourcedDocumentStorage needs `projections` when the event store is not KV-backed') - } - const registry = new DocIndexRegistry(opts.indexes ?? {}, sep) - if (registry.hasUnique && !driver.setIfAbsent) throw new CapabilityError('setIfAbsent', 'unique document indexes require it') - const ctx: DocContext = { - driver, - sep, - projBase, - registry, - readRepair: opts.readRepair ?? true, - maxRetries: opts.maxRetries ?? 5, - warnOnFullScan: opts.warnOnFullScan ?? (typeof process === 'undefined' || process.env?.NODE_ENV !== 'production'), - warned: new Set(), - newId: opts.newId ?? ulid, - claimGraceMs: opts.claimGraceMs ?? 30_000, - now: opts.now ?? Date.now, - writeLock: new KeyedMutex(), - } - super(ctx, events, undefined, [], '', '', registry.trie.rootState, undefined) - } -} diff --git a/src/documents/filter.ts b/src/documents/filter.ts deleted file mode 100644 index f2a4f9e..0000000 --- a/src/documents/filter.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' -import { compareValues, deepEqual, isPlainObject, isScalar, resolvePath, typeRank, type Doc } from './values.js' - -/** - * Query filter (Mongo subset): field conditions with implicit equality and dot paths, - * `$eq $ne $gt $gte $lt $lte $in $nin $exists`, and `$and $or $not`. - * - * Semantics differences from MongoDB (see DESIGN.md §12, M6): - * - Comparisons are type-bracketed: `$gt: 5` only matches numbers. - * - On array fields, all positive operators in one condition (`$eq $gt $gte $lt $lte $in`) must be - * satisfied by the same element (like `$elemMatch`), so index range scans are exact. - */ -export type Filter = { [field: string]: unknown; $and?: Filter[]; $or?: Filter[]; $not?: Filter } - -const POSITIVE = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in']) -const FIELD_LEVEL = new Set(['$ne', '$nin', '$exists', '$not']) - -export function isOperatorObject(v: unknown): v is Record { - if (!isPlainObject(v)) return false - const keys = Object.keys(v) - return keys.length > 0 && keys.every((k) => k.startsWith('$')) -} - -export function matches(doc: unknown, filter: Filter): boolean { - for (const key of Object.keys(filter)) { - const cond = filter[key] - switch (key) { - case '$and': - if (!(cond as Filter[]).every((f) => matches(doc, f))) return false - break - case '$or': - if (!(cond as Filter[]).some((f) => matches(doc, f))) return false - break - case '$not': - if (matches(doc, cond as Filter)) return false - break - default: - if (key.startsWith('$')) throw new TypeError(`Unknown top-level operator ${key}`) - if (!matchesField(resolvePath(doc, key), cond)) return false - } - } - return true -} - -/** Whether the values found at a path satisfy a field condition. */ -export function matchesField(values: unknown[], cond: unknown): boolean { - if (!isOperatorObject(cond)) return someCandidate(values, (c) => eq(c, cond)) - const positive: [string, unknown][] = [] - for (const [op, arg] of Object.entries(cond)) { - if (POSITIVE.has(op)) positive.push([op, arg]) - else if (!FIELD_LEVEL.has(op)) throw new TypeError(`Unknown operator ${op}`) - } - for (const [op, arg] of Object.entries(cond)) { - switch (op) { - case '$ne': - if (someCandidate(values, (c) => eq(c, arg))) return false - break - case '$nin': - if (someCandidate(values, (c) => (arg as unknown[]).some((x) => eq(c, x)))) return false - break - case '$exists': - if (values.length > 0 !== Boolean(arg)) return false - break - case '$not': - if (matchesField(values, arg)) return false - break - } - } - if (positive.length === 0) return true - return someCandidate(values, (c) => positive.every(([op, arg]) => applyPositive(op, c, arg))) -} - -/** Tests each value and, for arrays, each element. Missing paths test `undefined` once. */ -function someCandidate(values: unknown[], test: (c: unknown) => boolean): boolean { - if (values.length === 0) return test(undefined) - for (const v of values) { - if (test(v)) return true - if (Array.isArray(v) && v.some(test)) return true - } - return false -} - -function eq(c: unknown, x: unknown): boolean { - if (x === null) return c === null || c === undefined - return deepEqual(c, x) -} - -function applyPositive(op: string, c: unknown, arg: unknown): boolean { - switch (op) { - case '$eq': - return eq(c, arg) - case '$in': - if (!Array.isArray(arg)) throw new TypeError('$in expects an array') - return arg.some((x) => eq(c, x)) - default: { - const rc = typeRank(c) - if (rc !== typeRank(arg) || rc === 4) return false - const d = compareValues(c, arg) - return op === '$gt' ? d > 0 : op === '$gte' ? d >= 0 : op === '$lt' ? d < 0 : d <= 0 - } - } -} - -// ------------------------------------------------------------------ planner analysis - -export interface RangePred { - gt?: IndexValue - gte?: IndexValue - lt?: IndexValue - lte?: IndexValue -} - -export interface FieldPred { - eq?: IndexValue - range?: RangePred -} - -/** - * Index-usable predicates per field, from the top level and top-level `$and`. - * Only scalar `$eq`/implicit equality and same-typed scalar range bounds qualify. - */ -export function analyze(filter: Filter, out = new Map()): Map { - for (const key of Object.keys(filter)) { - const cond = filter[key] - if (key === '$and') { - for (const f of cond as Filter[]) analyze(f, out) - continue - } - if (key.startsWith('$')) continue - const pred = out.get(key) ?? {} - if (!isOperatorObject(cond)) { - if (isScalar(cond)) pred.eq ??= cond - } else { - if ('$eq' in cond && isScalar(cond.$eq)) pred.eq ??= cond.$eq - const range: RangePred = {} - let rank = -1 - let ok = false - for (const op of ['$gt', '$gte', '$lt', '$lte'] as const) { - if (!(op in cond)) continue - const v = cond[op] - if (!isScalar(v)) continue - const r = typeRank(v) - if (rank !== -1 && r !== rank) continue - rank = r - range[op.slice(1) as keyof RangePred] = v - ok = true - } - if (ok && !pred.range) pred.range = range - } - if (pred.eq !== undefined || pred.range) out.set(key, pred) - } - return out -} - -export type { Doc } diff --git a/src/documents/index.ts b/src/documents/index.ts deleted file mode 100644 index 8a2ad21..0000000 --- a/src/documents/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './types.js' -export * from './event-sourced.js' -export * from './filter.js' -export * from './update.js' -export * from './indexes.js' -export * from './ulid.js' -export { compareValues, deepEqual, type Doc } from './values.js' diff --git a/src/documents/indexes.ts b/src/documents/indexes.ts deleted file mode 100644 index 8d9d77f..0000000 --- a/src/documents/indexes.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { encodeIndexValues, indexValueCodec, type IndexValue } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ } from '../core/path.js' -import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js' -import type { FieldPred, RangePred } from './filter.js' -import { indexScalars, resolvePath, typeRank, type Doc } from './values.js' - -export interface DocIndexSpec { - /** Default: the fields joined with `,`. */ - name?: string - fields: string[] - /** Claims `~u//` per value tuple. Documents where every field is missing/null are not claimed. */ - unique?: boolean -} - -/** Path pattern (`*` = one segment, `''` = root) → indexes over that node's direct children. */ -export type DocIndexDeclarations = Record - -export interface SortSpec { - [field: string]: 1 | -1 -} - -interface CompiledDocIndex { - name: string - fields: string[] - segs: string[][] - unique: boolean - /** `escapeSegment(name) + sep` */ - head: string -} - -export interface Plan { - index: string - gte: string - lt: string - reverse: boolean - /** Index order satisfies the requested sort. */ - sorted: boolean -} - -/** Document indexes over the children of one node. */ -export class DocIndexes { - readonly #list: CompiledDocIndex[] - readonly #byName: Map - readonly #enc: (v: IndexValue) => string - - constructor( - list: CompiledDocIndex[], - /** `~i` */ - readonly base: string, - /** `~u` */ - readonly uniqueBase: string, - readonly sep: string, - ) { - this.#list = list - this.#byName = new Map(list.map((i) => [i.name, i])) - const codec = indexValueCodec(sep) - this.#enc = (v) => codec.encode(v) - } - - get hasUnique(): boolean { - return this.#list.some((i) => i.unique) - } - - /** Value tuples a state produces for one index (cartesian over multikey fields). */ - #tuples(idx: CompiledDocIndex, state: Doc): IndexValue[][] { - let tuples: IndexValue[][] = [[]] - let multi = 0 - for (const segs of idx.segs) { - const found = resolvePath(state, segs) - const scalars = - found.length === 0 ? [null] : found.length === 1 && !Array.isArray(found[0]) ? indexScalars(found[0]) : [...new Set(found.flatMap(indexScalars))] - if (tuples.length === 1 && scalars.length === 1) { - tuples[0]!.push(scalars[0]!) - continue - } - if (scalars.length > 1 && ++multi > 1) throw new TypeError(`Index "${idx.name}": cannot index more than one array field per document`) - tuples = tuples.flatMap((t) => scalars.map((s) => [...t, s])) - } - return tuples - } - - entriesFor(escChild: string, state: Doc | undefined, out: Set = new Set()): Set { - if (!state) return out - for (const idx of this.#list) { - const stem = this.base + idx.head - for (const t of this.#tuples(idx, state)) out.add(stem + encodeIndexValues(t, this.sep) + this.sep + escChild) - } - return out - } - - /** Unique claim key → index name and values. */ - claimsFor(state: Doc | undefined): Map { - const out = new Map() - if (!state) return out - for (const idx of this.#list) { - if (!idx.unique) continue - for (const t of this.#tuples(idx, state)) { - if (t.every((v) => v === null)) continue - out.set(this.uniqueBase + idx.head + encodeIndexValues(t, this.sep), { index: idx.name, values: t }) - } - } - return out - } - - /** Escaped child segment of an entry key of index `name`. */ - childOf(name: string, entryKey: string): string { - const idx = this.#byName.get(name)! - let pos = this.base.length + idx.head.length - for (let i = 0; i < idx.fields.length; i++) pos = entryKey.indexOf(this.sep, pos) + 1 - return entryKey.slice(pos) - } - - /** - * Chooses the index covering the most leading equality fields, then at most one range field. - * Ties prefer an index whose order satisfies `sort`. With no usable predicate, an index whose - * order satisfies `sort` is still used (a full, ordered index scan). Returns `undefined` for a - * full projection scan. - */ - plan(preds: Map, sort: [string, 1 | -1][]): Plan | undefined { - let best: { idx: CompiledDocIndex; k: number; range: RangePred | undefined; sorted: boolean; dir: 1 | -1; score: number } | undefined - for (const idx of this.#list) { - let k = 0 - while (k < idx.fields.length && preds.get(idx.fields[k]!)?.eq !== undefined) k++ - const range = k < idx.fields.length ? preds.get(idx.fields[k]!)?.range : undefined - // Sort fields constrained by equality are constant; drop them before matching. - const eqFields = new Set(idx.fields.slice(0, k)) - const rest = sort.filter(([f]) => !eqFields.has(f)) - // Ties are broken by child key in the direction of the first sort field. - const dir = rest[0]?.[1] ?? sort[0]?.[1] ?? 1 - const sorted = rest.length === 0 || (rest.every(([f, d], i) => idx.fields[k + i] === f && d === dir) && rest.length <= idx.fields.length - k) - const score = k * 2 + (range ? 1 : 0) - if (score === 0 && !(sorted && rest.length > 0)) continue - if (!best || score > best.score || (score === best.score && sorted && !best.sorted)) best = { idx, k, range, sorted, dir, score } - } - if (!best) return undefined - const { idx, k, range, sorted, dir } = best - const succSep = succ(this.sep) - let prefix = this.base + idx.head - for (let i = 0; i < k; i++) prefix += this.#enc(preds.get(idx.fields[i]!)!.eq!) + this.sep - // prefix ends with sep: the range of everything under it is [prefix, prefix-without-sep + succ(sep)). - const all = { gte: prefix, lt: prefix.slice(0, -1) + succSep } - let { gte, lt } = all - if (range) { - const sample = (range.gt ?? range.gte ?? range.lt ?? range.lte) as IndexValue - const [lo, hi] = typeBounds(typeRank(sample), this.#enc) - gte = prefix + lo - lt = hi === undefined ? all.lt : prefix + hi - if (range.gte !== undefined) gte = max(gte, prefix + this.#enc(range.gte)) - if (range.gt !== undefined) gte = max(gte, prefix + this.#enc(range.gt) + succSep) - if (range.lt !== undefined) lt = min(lt, prefix + this.#enc(range.lt)) - if (range.lte !== undefined) lt = min(lt, prefix + this.#enc(range.lte) + succSep) - } - return { index: idx.name, gte, lt, reverse: sorted && dir === -1, sorted } - } -} - -/** Encoded `[lower, upper)` of one type class; `undefined` upper = end of index. */ -function typeBounds(rank: number, enc: (v: IndexValue) => string): [string, string | undefined] { - switch (rank) { - case 0: - return [enc(null), enc(false)] - case 1: - return [enc(false), enc(-Infinity)] - case 2: - return [enc(-Infinity), enc('')] - default: - return [enc(''), undefined] - } -} - -const max = (a: string, b: string) => (a > b ? a : b) -const min = (a: string, b: string) => (a < b ? a : b) - -export class DocIndexRegistry { - readonly trie: PatternTrie - readonly empty: boolean - readonly hasUnique: boolean - - constructor( - decls: DocIndexDeclarations, - readonly sep: string, - ) { - this.trie = new PatternTrie(sep) - const patterns = Object.keys(decls) - const compiled = new Map() - for (const p of patterns) { - const list = decls[p]!.map((spec): CompiledDocIndex => { - if (!spec.fields.length) throw new TypeError(`Index on "${p}" has no fields`) - const name = spec.name ?? spec.fields.join(',') - return { name, fields: spec.fields, segs: spec.fields.map((f) => f.split('.')), unique: spec.unique === true, head: escapeSegment(name, sep) + sep } - }) - const names = list.map((i) => i.name) - const dup = names.find((n, i) => names.indexOf(n) !== i) - if (dup) throw new TypeError(`Duplicate index name "${dup}" on "${p}"`) - compiled.set(p, list) - } - for (let i = 0; i < patterns.length; i++) { - for (let j = i + 1; j < patterns.length; j++) { - const a = patterns[i]! - const b = patterns[j]! - if (!patternsOverlap(a, b, sep)) continue - const bn = new Set(compiled.get(b)!.map((x) => x.name)) - const clash = compiled.get(a)!.filter((x) => bn.has(x.name)) - if (clash.length) throw new TypeError(`Index name(s) ${clash.map((c) => c.name).join(', ')} declared on overlapping patterns "${a}" and "${b}"`) - } - } - for (const p of patterns) this.trie.add(p, compiled.get(p)!) - this.empty = patterns.length === 0 - this.hasUnique = [...compiled.values()].some((l) => l.some((i) => i.unique)) - } - - /** `projPrefix` is the projection-space prefix of the node (``). */ - bind(state: TrieState, projPrefix: string): DocIndexes | undefined { - if (this.empty) return undefined - const found = PatternTrie.values(state) - if (found.length === 0) return undefined - const list = found.flatMap((f) => f.value) - return new DocIndexes(list, projPrefix + RESERVED.index + this.sep, projPrefix + RESERVED.unique + this.sep, this.sep) - } -} diff --git a/src/documents/types.ts b/src/documents/types.ts deleted file mode 100644 index 2435656..0000000 --- a/src/documents/types.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { StoredEvent } from '../events/types.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import type { Filter } from './filter.js' -import type { SortSpec } from './indexes.js' -import type { DocEvent, UpdateSpec } from './update.js' - -export interface FindOptions { - sort?: SortSpec - limit?: number - skip?: number - /** Inclusion (`{ a: 1 }`) or exclusion (`{ a: 0 }`) of dot-path fields. */ - projection?: Record -} - -export interface FindResult { - key: string - data: T -} - -export interface GetOptions { - /** Check the stream head and catch the projection up first (one extra scan). */ - consistent?: boolean -} - -export interface UpdateOptions { - /** Treat a missing document as `{}`. */ - upsert?: boolean -} - -export interface DocKeysOptions { - limit?: number - after?: string | number - reverse?: boolean -} - -/** Document type of a node's children. */ -export type ChildDoc = ValueOf>> - -/** - * One node of a document store: a document (its own) and a collection of child documents. - * Same navigation and self/child arity rules as `KVNode`; in forms with an optional leading key, - * a non-null object first argument selects the self form. - */ -export interface DocumentNode { - readonly path: readonly string[] - child>(key: K): DocumentNode> - as(): DocumentNode - - get(opts?: GetOptions): Promise | undefined> - get>(key: K, opts?: GetOptions): Promise> | undefined> - - set(doc: ValueOf): Promise - set>(key: K, doc: ValueOf>): Promise - - /** Inserts a child with a generated time-sortable id; returns the id. */ - insert(doc: ChildDoc): Promise - - /** Returns the new state, or `undefined` if the document doesn't exist (and no `upsert`). */ - update(spec: UpdateSpec, opts?: UpdateOptions): Promise | undefined> - update>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise> | undefined> - - /** Returns whether a document was deleted. */ - delete(): Promise - delete>(key: K): Promise - - /** Queries direct children. */ - find(filter?: Filter, opts?: FindOptions): AsyncIterable>> - findOne(filter?: Filter, opts?: FindOptions): Promise> | undefined> - - /** Keys of direct children that hold a (non-deleted) document. */ - keys(opts?: DocKeysOptions): AsyncIterable> -} - -/** Document store backed by an event log: adds history and point-in-time reads. */ -export interface EventSourcedDocumentNode extends DocumentNode { - child>(key: K): EventSourcedDocumentNode> - as(): EventSourcedDocumentNode - - history(): AsyncIterable> - history>(key: K): AsyncIterable> - - /** State as of `version` (`undefined` if missing or deleted then). */ - getAt(version: number): Promise | undefined> - getAt>(key: K, version: number): Promise> | undefined> - - /** Rebuilds lagging projections for this node and all descendants; returns how many were repaired. */ - catchUp(): Promise -} - -export type DocumentStorage = DocumentNode diff --git a/src/documents/ulid.ts b/src/documents/ulid.ts deleted file mode 100644 index ecf27ac..0000000 --- a/src/documents/ulid.ts +++ /dev/null @@ -1,37 +0,0 @@ -const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32 - -/** - * ULID-like id generator: 10 chars of millisecond timestamp + 16 chars of randomness, - * lexicographically sortable by creation time. Ids generated within the same millisecond - * increment the random part, so they stay strictly increasing per generator. - */ -export function createIdGenerator(now: () => number = Date.now): () => string { - let lastTime = -1 - const rand = new Uint8Array(16) // one base32 digit (0–31) per byte - return () => { - let t = now() - if (t <= lastTime) { - t = lastTime - // Increment the random part (base 32, right to left). - let i = 15 - while (i >= 0 && rand[i] === 31) rand[i--] = 0 - if (i < 0) throw new Error('ULID random overflow within one millisecond') - rand[i]!++ - } else { - crypto.getRandomValues(rand) - for (let i = 0; i < 16; i++) rand[i]! &= 31 - // Leave headroom so increments rarely overflow. - rand[0]! &= 15 - lastTime = t - } - let out = '' - for (let i = 9; i >= 0; i--) { - out = ALPHABET[t % 32]! + out - t = Math.floor(t / 32) - } - for (let i = 0; i < 16; i++) out += ALPHABET[rand[i]!] - return out - } -} - -export const ulid = createIdGenerator() diff --git a/src/documents/update.ts b/src/documents/update.ts deleted file mode 100644 index 79ea9ab..0000000 --- a/src/documents/update.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { isOperatorObject, matches, matchesField, type Filter } from './filter.js' -import { clone, deepEqual, getPath, isPlainObject, setPath, unsetPath, type Doc } from './values.js' - -/** Update operators supported in v1. Paths may use dots. */ -export interface UpdateSpec { - $set?: Record - $unset?: Record - $inc?: Record - $push?: Record - $pull?: Record - $addToSet?: Record -} - -/** Stored document events: resolved deltas, never operators, so replay is deterministic. */ -export type DocEvent = { op: 'set'; doc: Doc } | { op: 'patch'; set?: Record; unset?: string[] } | { op: 'delete' } - -export interface Patch { - set: Record - unset: string[] -} - -const OPS = new Set(['$set', '$unset', '$inc', '$push', '$pull', '$addToSet']) - -/** Resolves operators against `state` into a patch of final values. */ -export function resolveUpdate(state: Doc, spec: UpdateSpec): Patch { - const keys = Object.keys(spec) - if (keys.length === 0) throw new TypeError('Empty update') - for (const k of keys) if (!OPS.has(k)) throw new TypeError(`Unsupported update operator ${k}`) - - const patch: Patch = { set: {}, unset: [] } - const touched: string[] = [] - const touch = (path: string) => { - for (const t of touched) { - if (t === path || t.startsWith(path + '.') || path.startsWith(t + '.')) throw new TypeError(`Update paths conflict: "${t}" and "${path}"`) - } - touched.push(path) - } - const arrayAt = (path: string, op: string): unknown[] => { - const cur = getPath(state, path) - if (cur === undefined || cur === null) return [] - if (!Array.isArray(cur)) throw new TypeError(`${op} on non-array field "${path}"`) - return cur - } - const eachOf = (v: unknown): unknown[] => (isPlainObject(v) && Object.keys(v).length === 1 && Array.isArray(v.$each) ? (v.$each as unknown[]) : [v]) - - for (const [path, v] of Object.entries(spec.$set ?? {})) { - touch(path) - patch.set[path] = clone(v) - } - for (const path of Object.keys(spec.$unset ?? {})) { - touch(path) - patch.unset.push(path) - } - for (const [path, by] of Object.entries(spec.$inc ?? {})) { - touch(path) - if (typeof by !== 'number') throw new TypeError(`$inc by non-number at "${path}"`) - const cur = getPath(state, path) - if (cur !== undefined && cur !== null && typeof cur !== 'number') throw new TypeError(`$inc on non-number field "${path}"`) - patch.set[path] = ((cur as number | undefined) ?? 0) + by - } - for (const [path, v] of Object.entries(spec.$push ?? {})) { - touch(path) - patch.set[path] = [...clone(arrayAt(path, '$push')), ...clone(eachOf(v))] - } - for (const [path, v] of Object.entries(spec.$addToSet ?? {})) { - touch(path) - const next = clone(arrayAt(path, '$addToSet')) - for (const item of eachOf(v)) if (!next.some((x) => deepEqual(x, item))) next.push(clone(item)) - patch.set[path] = next - } - for (const [path, cond] of Object.entries(spec.$pull ?? {})) { - touch(path) - patch.set[path] = clone(arrayAt(path, '$pull')).filter((el) => !pullMatches(el, cond)) - } - return patch -} - -function pullMatches(el: unknown, cond: unknown): boolean { - if (isOperatorObject(cond)) return matchesField([el], cond) - if (isPlainObject(cond) && isPlainObject(el)) return matches(el, cond as Filter) - return deepEqual(el, cond) -} - -/** Applies a resolved patch to a copy of `state`. */ -export function applyPatch(state: Doc, patch: { set?: Record; unset?: string[] }): Doc { - const next = clone(state) - for (const [path, v] of Object.entries(patch.set ?? {})) setPath(next, path, clone(v)) - for (const path of patch.unset ?? []) unsetPath(next, path) - return next -} - -/** Folds one event into a state (`undefined` = no document). */ -export function applyEvent(state: Doc | undefined, e: DocEvent): Doc | undefined { - switch (e.op) { - case 'set': - return clone(e.doc) - case 'patch': - return applyPatch(state ?? {}, e) - case 'delete': - return undefined - } -} - -export function toPatchEvent(p: Patch): DocEvent { - const e: { op: 'patch'; set?: Record; unset?: string[] } = { op: 'patch' } - if (Object.keys(p.set).length) e.set = p.set - if (p.unset.length) e.unset = p.unset - return e -} diff --git a/src/documents/values.ts b/src/documents/values.ts deleted file mode 100644 index 4c416b3..0000000 --- a/src/documents/values.ts +++ /dev/null @@ -1,163 +0,0 @@ -import type { IndexValue } from '../core/encoding.js' - -/** Plain JSON object. */ -export type Doc = Record - -export function isPlainObject(v: unknown): v is Doc { - return typeof v === 'object' && v !== null && !Array.isArray(v) -} - -export function isScalar(v: unknown): v is IndexValue { - return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v)) -} - -export function splitPath(path: string): string[] { - return path.split('.') -} - -/** - * Values reachable at a dot path. Arrays are traversed: a non-numeric segment applied to an - * array maps over its elements. Missing paths yield no values. - */ -export function resolvePath(doc: unknown, path: string | readonly string[]): unknown[] { - const segs = typeof path === 'string' ? splitPath(path) : path - const out: unknown[] = [] - walk(doc, segs, 0, out) - return out -} - -function walk(v: unknown, segs: readonly string[], i: number, out: unknown[]): void { - if (i === segs.length) { - if (v !== undefined) out.push(v) - return - } - const seg = segs[i]! - if (Array.isArray(v)) { - if (/^\d+$/.test(seg)) walk(v[Number(seg)], segs, i + 1, out) - else for (const el of v) if (isPlainObject(el) || Array.isArray(el)) walk(el, segs, i, out) - return - } - if (isPlainObject(v) && Object.hasOwn(v, seg)) walk(v[seg], segs, i + 1, out) -} - -/** Value at a dot path without array traversal (numeric segments index arrays). */ -export function getPath(doc: unknown, path: string): unknown { - let v = doc - for (const seg of splitPath(path)) { - if (Array.isArray(v) && /^\d+$/.test(seg)) v = v[Number(seg)] - else if (isPlainObject(v) && Object.hasOwn(v, seg)) v = v[seg] - else return undefined - } - return v -} - -/** Sets a dot path in place, creating intermediate objects. */ -export function setPath(doc: Doc, path: string, value: unknown): void { - const segs = splitPath(path) - let v: unknown = doc - for (let i = 0; i < segs.length - 1; i++) { - const seg = segs[i]! - const container = v as Record - let next = Array.isArray(container) && /^\d+$/.test(seg) ? container[Number(seg)] : container[seg] - if (next === undefined || next === null) { - next = {} - if (Array.isArray(container)) container[Number(seg)] = next - else container[seg] = next - } else if (typeof next !== 'object') { - throw new TypeError(`Cannot set "${path}": "${segs.slice(0, i + 1).join('.')}" is not an object`) - } - v = next - } - const last = segs.at(-1)! - if (Array.isArray(v) && /^\d+$/.test(last)) v[Number(last)] = value - else (v as Doc)[last] = value -} - -export function unsetPath(doc: Doc, path: string): void { - const segs = splitPath(path) - let v: unknown = doc - for (let i = 0; i < segs.length - 1; i++) { - v = isPlainObject(v) ? v[segs[i]!] : Array.isArray(v) && /^\d+$/.test(segs[i]!) ? v[Number(segs[i])] : undefined - if (typeof v !== 'object' || v === null) return - } - const last = segs.at(-1)! - if (Array.isArray(v) && /^\d+$/.test(last)) - v[Number(last)] = null // Mongo semantics: keep positions - else if (isPlainObject(v)) delete v[last] -} - -export function deepEqual(a: unknown, b: unknown): boolean { - if (a === b) return true - if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false - if (Array.isArray(a) !== Array.isArray(b)) return false - if (Array.isArray(a)) { - const bb = b as unknown[] - return a.length === bb.length && a.every((x, i) => deepEqual(x, bb[i])) - } - const ka = Object.keys(a) - const kb = Object.keys(b) - return ka.length === kb.length && ka.every((k) => Object.hasOwn(b, k) && deepEqual((a as Doc)[k], (b as Doc)[k])) -} - -/** Deep copy of a JSON-compatible value (documents are JSON). */ -export function clone(v: T): T { - if (v === null || typeof v !== 'object') return v - if (Array.isArray(v)) return v.map(clone) as T - const out: Doc = {} - for (const k in v) out[k] = clone((v as Doc)[k]) - return out as T -} - -/** Type class used for bracketing comparisons, in index-encoding order. */ -export function typeRank(v: unknown): number { - if (v === null || v === undefined) return 0 - if (typeof v === 'boolean') return 1 - if (typeof v === 'number') return 2 - if (typeof v === 'string') return 3 - return 4 -} - -/** - * Total order matching the index encoding for scalars: - * `null/missing < false < true < numbers < strings < anything else`. - */ -export function compareValues(a: unknown, b: unknown): number { - const ra = typeRank(a) - const rb = typeRank(b) - if (ra !== rb) return ra < rb ? -1 : 1 - if (ra === 1 || ra === 2 || ra === 3) return a === b ? 0 : (a as number) < (b as number) ? -1 : 1 - if (ra === 4) { - const sa = JSON.stringify(a) - const sb = JSON.stringify(b) - return sa === sb ? 0 : sa < sb ? -1 : 1 - } - return 0 -} - -/** - * The scalar(s) an index stores for a field value: scalars as-is, arrays as their scalar - * elements (multikey; empty → null), and missing or non-scalar values as `null`. - */ -export function indexScalars(v: unknown): IndexValue[] { - if (Array.isArray(v)) { - const out: IndexValue[] = [] - for (const el of v) out.push(isScalar(el) ? el : null) - return out.length ? out : [null] - } - return [isScalar(v) ? v : null] -} - -/** Sort key for a field under a direction; arrays sort by their min (asc) or max (desc) element. */ -export function sortKey(doc: unknown, path: string, dir: 1 | -1): unknown { - const vals = resolvePath(doc, path) - if (vals.length === 0) return null - let best: unknown - let first = true - for (const v of vals) { - for (const s of indexScalars(v)) { - if (first || compareValues(s, best) * dir < 0) best = s - first = false - } - } - return best -} diff --git a/src/errors.ts b/src/errors.ts deleted file mode 100644 index 8b3528a..0000000 --- a/src/errors.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** Base class for all PotentialStorage errors. */ -export class PotentialStorageError extends Error { - constructor(message: string) { - super(message) - this.name = new.target.name - } -} - -/** An optimistic-concurrency check failed (stale `expectedVersion` or a lost commit race). */ -export class ConcurrencyError extends PotentialStorageError { - constructor( - readonly stream: string, - readonly expectedVersion: number | 'any', - readonly actualVersion: number, - ) { - super(`Concurrency conflict on "${stream}": expected version ${expectedVersion}, actual ${actualVersion}`) - } -} - -/** A unique index already holds the given value for another key. */ -export class UniqueConstraintError extends PotentialStorageError { - constructor( - readonly index: string, - readonly values: readonly unknown[], - readonly existingKey?: string, - ) { - super(`Unique index "${index}" already contains ${JSON.stringify(values)}${existingKey === undefined ? '' : ` (key "${existingKey}")`}`) - } -} - -/** The driver lacks an optional capability required by the requested feature. */ -export class CapabilityError extends PotentialStorageError { - constructor( - readonly capability: string, - detail?: string, - ) { - super(`Driver lacks capability "${capability}"${detail ? `: ${detail}` : ''}`) - } -} - -/** - * A key argument is invalid: passed but `undefined`/`null`, empty, or otherwise unusable. - * Extends `TypeError` so callers can catch either. - */ -export class InvalidKeyError extends TypeError { - constructor(message: string) { - super(message) - this.name = 'InvalidKeyError' - } -} diff --git a/src/event-storage/base-event-storage.ts b/src/event-storage/base-event-storage.ts new file mode 100644 index 0000000..173e361 --- /dev/null +++ b/src/event-storage/base-event-storage.ts @@ -0,0 +1,100 @@ +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueAt, ValueOf } from '../utils/schema.js' + +export interface AppendOptions { + /** Version the stream must be at (0 = empty). Default `'any'`. */ + expectedVersion?: number | 'any' +} + +export interface AppendResult { + /** Stream version after the append. */ + version: number +} + +export interface ReadOptions { + /** First version to read (inclusive). Default 1. */ + from?: number + /** Last version to read (inclusive). Default: the head. */ + to?: number + limit?: number +} + +export interface StoredEvent { + version: number + /** Append timestamp (ms since epoch). */ + at: number + data: E +} + +export interface StreamKeysOptions { + limit?: number + after?: string | number + reverse?: boolean +} + +/** + * Append-only event streams. Every node is a stream (its own events) and a parent of child + * streams. Implementations provide four primitives over stream paths; the user-facing, + * self-or-child API is defined here once. + */ +export abstract class BaseEventStorage { + /** Unescaped path of this node (`[]` for the root, which is not a stream). */ + path: readonly string[] = [] + + abstract appendTo(stream: readonly string[], events: unknown[], expectedVersion: number | 'any'): Promise + abstract readFrom(stream: readonly string[], options: ReadOptions): AsyncIterable> + abstract versionOf(stream: readonly string[]): Promise + abstract streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncIterable + + child>(key: K): BaseEventStorage> { + const node = Object.create(this) as BaseEventStorage> + node.path = [...this.path, Keys.segment(key)] + return node + } + + as(): BaseEventStorage { + return this as unknown as BaseEventStorage + } + + /** `append(events, options?)` on this stream; `append(key, events, options?)` on a child stream. */ + append>( + ...args: [events: ValueOf[], options?: AppendOptions] | [key: K, events: ValueOf>[], options?: AppendOptions] + ): Promise { + const hasKey = !Array.isArray(args[0]) + const events = args[hasKey ? 1 : 0] + const options = (args[hasKey ? 2 : 1] ?? {}) as AppendOptions + if (!Array.isArray(events)) return Promise.reject(new TypeError('append() expects an array of events')) + return this.withStream(hasKey, args[0], (stream) => this.appendTo(stream, events, options.expectedVersion ?? 'any')) + } + + /** `read(options?)` this stream; `read(key, options?)` a child stream. */ + async *read>(...args: [options?: ReadOptions] | [key: K, options?: ReadOptions]): AsyncGenerator>> { + const hasKey = typeof args[0] !== 'object' && args.length > 0 + const stream = this.streamPath(hasKey, args[0]) + yield* this.readFrom(stream, ((hasKey ? args[1] : args[0]) ?? {}) as ReadOptions) as AsyncIterable>> + } + + /** Current version of this stream (or a child's); 0 when empty. */ + version(key?: KeyOf): Promise { + return this.withStream(arguments.length > 0, key, (stream) => this.versionOf(stream)) + } + + /** Keys of child streams. */ + keys(options: StreamKeysOptions = {}): AsyncIterable> { + return this.streamsOf(this.path, options) as AsyncIterable> + } + + private withStream(hasKey: boolean, key: unknown, fn: (stream: readonly string[]) => Promise): Promise { + try { + return fn(this.streamPath(hasKey, key)) + } catch (error) { + return Promise.reject(error) + } + } + + private streamPath(hasKey: boolean, key: unknown): readonly string[] { + if (hasKey) return [...this.path, Keys.segment(key)] + if (this.path.length === 0) throw new TypeError('The root node is not a stream') + return this.path + } +} diff --git a/src/event-storage/event-storage-kv.ts b/src/event-storage/event-storage-kv.ts new file mode 100644 index 0000000..aa5e00a --- /dev/null +++ b/src/event-storage/event-storage-kv.ts @@ -0,0 +1,151 @@ +import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js' +import { ConcurrencyError } from '../utils/errors.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseEventStorage, type AppendResult, type ReadOptions, type StoredEvent, type StreamKeysOptions } from './base-event-storage.js' + +/** Stored value of one commit key. */ +interface Commit { + firstVersion: number + at: number + events: unknown[] +} + +interface Head { + commit: number + version: number +} + +export interface EventStorageKVOptions { + /** Retries after losing a commit race when `expectedVersion` is `'any'`. Default 5. */ + maxRetries?: number + /** Max streams whose head is cached. Default 10 000. */ + cacheSize?: number + now?: () => number +} + +/** + * Event storage on any KV storage. Each append is one commit key, densely numbered: + * `/~e/` → `{ firstVersion, at, events }`. Commits are claimed with + * `writeIfAbsent`, so a writer with a stale view always collides with the existing key. + */ +export class EventStorageKV extends BaseEventStorage { + private readonly heads = new Map() + private readonly maxRetries: number + private readonly cacheSize: number + private readonly now: () => number + + constructor( + private readonly kv: BaseKVStorage, + options: EventStorageKVOptions = {}, + ) { + super() + this.maxRetries = options.maxRetries ?? 5 + this.cacheSize = options.cacheSize ?? 10_000 + this.now = options.now ?? Date.now + } + + async appendTo(stream: readonly string[], events: unknown[], expectedVersion: number | 'any'): Promise { + const base = this.commitPrefix(stream) + for (let attempt = 0; ; attempt++) { + let head = this.heads.get(base) ?? (await this.loadHead(base)) + // The cache may be stale; only trust a mismatch after re-reading. + if (expectedVersion !== 'any' && head.version !== expectedVersion) head = await this.loadHead(base) + if (expectedVersion !== 'any' && head.version !== expectedVersion) throw new ConcurrencyError(base, expectedVersion, head.version) + if (events.length === 0) return { version: head.version } + + const commit: Commit = { firstVersion: head.version + 1, at: this.now(), events } + const [written] = await this.kv.writeIfAbsent([[base + EventStorageKV.pad(head.commit + 1), commit]]) + if (written) { + const next = { commit: head.commit + 1, version: head.version + events.length } + this.cacheHead(base, next) + return { version: next.version } + } + this.heads.delete(base) + if (expectedVersion !== 'any' || attempt >= this.maxRetries) { + throw new ConcurrencyError(base, expectedVersion, (await this.loadHead(base)).version) + } + } + } + + async *readFrom(stream: readonly string[], options: ReadOptions): AsyncGenerator> { + const from = Math.max(1, options.from ?? 1) + const to = options.to ?? Infinity + let remaining = options.limit ?? Infinity + if (to < from || remaining <= 0) return + const base = this.commitPrefix(stream) + const start = from === 1 ? 1 : await this.findCommit(base, from) + if (start === 0) return + for await (const [, value] of this.kv.scan({ gte: base + EventStorageKV.pad(start), lt: this.commitEnd(base), values: true })) { + const commit = value as Commit + for (let i = 0; i < commit.events.length; i++) { + const version = commit.firstVersion + i + if (version < from) continue + if (version > to || remaining-- <= 0) return + yield { version, at: commit.at, data: commit.events[i] } + } + } + } + + async versionOf(stream: readonly string[]): Promise { + return (await this.loadHead(this.commitPrefix(stream))).version + } + + async *streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncGenerator { + yield* parent.reduce((node, segment) => node.child(segment), this.kv).keys(options) + } + + /** Last commit of a stream: one reverse scan with limit 1. Refreshes the cache. */ + private async loadHead(base: string): Promise { + for await (const [key, value] of this.kv.scan({ gte: base, lt: this.commitEnd(base), reverse: true, limit: 1, values: true })) { + const commit = value as Commit + const head = { commit: Number(key.slice(base.length)), version: commit.firstVersion + commit.events.length - 1 } + this.cacheHead(base, head) + return head + } + return { commit: 0, version: 0 } + } + + /** + * Commit containing `version`, or 0 if the stream is shorter. Commit `n` starts at a version + * >= n, so it's at most `version`: try that first (exact when commits hold one event each), + * then binary-search the commits below it. + */ + private async findCommit(base: string, version: number): Promise { + const firstVersion = async (commit: number) => ((await this.kv.readMany([base + EventStorageKV.pad(commit)]))[0] as Commit | undefined)?.firstVersion + const guess = await firstVersion(version) + if (guess !== undefined && guess <= version) return version + let hi = version - 1 + if (guess === undefined) { + const head = await this.loadHead(base) + if (head.version < version) return 0 + hi = head.commit + } + let lo = 1 + while (lo < hi) { + const mid = (lo + hi + 1) >>> 1 + if ((await firstVersion(mid))! <= version) lo = mid + else hi = mid - 1 + } + return lo + } + + private cacheHead(base: string, head: Head): void { + this.heads.delete(base) + this.heads.set(base, head) + if (this.heads.size > this.cacheSize) this.heads.delete(this.heads.keys().next().value!) + } + + private commitPrefix(stream: readonly string[]): string { + const prefix = this.kv.prefix + Keys.join(stream) + return `${prefix}/~e/` + } + + private commitEnd(base: string): string { + return base.slice(0, -1) + Keys.afterSeparator + } + + private static pad(commit: number): string { + return String(commit).padStart(12, '0') + } +} diff --git a/src/events/index.ts b/src/events/index.ts deleted file mode 100644 index 057ee0f..0000000 --- a/src/events/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './types.js' -export * from './kv.js' diff --git a/src/events/kv.ts b/src/events/kv.ts deleted file mode 100644 index 2ccf9fa..0000000 --- a/src/events/kv.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { LRU } from '../core/lru.js' -import { KeyedMutex } from '../core/mutex.js' -import { escapeSegment, RESERVED, succ, toSegment } from '../core/path.js' -import { ConcurrencyError } from '../errors.js' -import type { KVDriver } from '../kv/driver.js' -import type { KVNode } from '../kv/node.js' -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' -import type { AppendOptions, AppendResult, EventNode, ReadOptions, StoredEvent, StreamKeysOptions } from './types.js' - -export interface KVEventStorageOptions { - /** Max streams whose head is cached. Default 10 000. */ - headCacheSize?: number - /** Retries after a commit collision when `expectedVersion` is `'any'`. Default 5. */ - maxRetries?: number - /** Clock for commit timestamps. Default `Date.now`. */ - now?: () => number -} - -/** Stored value of one commit key. */ -export interface CommitRecord { - firstVersion: number - at: number - events: E[] -} - -interface Head { - commit: number - version: number -} - -interface Stream { - /** Stream's own key (escaped); identifies it in the cache and in errors. */ - key: string - /** `/~e/` — commit keys are `base + pad12(seq)`. */ - base: string - /** Exclusive upper bound of commit keys. */ - end: string -} - -/** @internal */ -interface EventContext { - readonly driver: KVDriver - readonly sep: string - readonly heads: LRU - readonly mutex: KeyedMutex - readonly maxRetries: number - readonly now: () => number -} - -const SEQ_WIDTH = 12 -const SEARCH_FANOUT = 8 -const EMPTY_HEAD: Head = { commit: 0, version: 0 } - -const pad = (seq: number) => String(seq).padStart(SEQ_WIDTH, '0') - -/** - * Event node over a `KVNode`. Each append is one commit key, densely numbered: - * `/~e/` → `{ firstVersion, at, events }`. Commits are claimed with - * `setIfAbsent`, so any stale writer collides with the existing key. Without `setIfAbsent`, - * appends are serialised by an in-process per-stream mutex (single-writer only). - */ -export class KVEventNode implements EventNode { - readonly #ctx: EventContext - readonly #kv: KVNode - - /** @internal Use `KVEventStorage` and `child()`. */ - constructor(ctx: EventContext, kv: KVNode) { - this.#ctx = ctx - this.#kv = kv - } - - get path(): readonly string[] { - return this.#kv.path - } - - /** The underlying KV node. */ - get kv(): KVNode { - return this.#kv - } - - child>(key: K): KVEventNode> { - return new KVEventNode(this.#ctx, this.#kv.child(key as string).as()) - } - - as(): KVEventNode { - return this as unknown as KVEventNode - } - - append(events: readonly ValueOf[], opts?: AppendOptions): Promise - append>(key: K, events: readonly ValueOf>[], opts?: AppendOptions): Promise - async append(...args: unknown[]): Promise { - const self = Array.isArray(args[0]) - const stream = self ? this.#self() : this.#child(args[0]) - const events = args[self ? 0 : 1] - const opts = (args[self ? 1 : 2] ?? {}) as AppendOptions - if (!Array.isArray(events)) throw new TypeError('append() expects an array of events') - const expected = opts.expectedVersion ?? 'any' - if (expected !== 'any' && !(Number.isInteger(expected) && expected >= 0)) throw new TypeError(`Invalid expectedVersion ${String(expected)}`) - const hint = expected !== 'any' && Number.isInteger(opts.expectedCommit) ? { commit: opts.expectedCommit!, version: expected } : undefined - return this.#append(stream, events, expected, hint) - } - - read(opts?: ReadOptions): AsyncIterable>> - read>(key: K, opts?: ReadOptions): AsyncIterable>>> - async *read(...args: unknown[]): AsyncGenerator> { - const self = args.length === 0 || (typeof args[0] === 'object' && args[0] !== null) - const stream = self ? this.#self() : this.#child(args[0]) - yield* this.#read(stream, ((self ? args[0] : args[1]) ?? {}) as ReadOptions) - } - - version(): Promise - version>(key: K): Promise - async version(...args: unknown[]): Promise { - const stream = args.length === 0 ? this.#self() : this.#child(args[0]) - return (await this.#loadHead(stream)).version - } - - keys(opts: StreamKeysOptions = {}): AsyncIterable> { - return this.#kv.keys(opts) as AsyncIterable> - } - - // ---------------------------------------------------------------- internals - - #self(): Stream { - if (this.#kv.isRoot) throw new TypeError('The root node is not a stream') - return this.#streamAt(this.#kv.selfKey) - } - - #child(key: unknown): Stream { - return this.#streamAt(this.#kv.prefix + escapeSegment(toSegment(key), this.#ctx.sep)) - } - - #streamAt(key: string): Stream { - const sep = this.#ctx.sep - const head = key + sep + RESERVED.events - return { key, base: head + sep, end: head + succ(sep) } - } - - async #loadHead(s: Stream): Promise { - for await (const [key, value] of this.#ctx.driver.scan({ gte: s.base, lt: s.end, reverse: true, limit: 1, values: true })) { - const c = value as CommitRecord - const head = { commit: Number(key.slice(s.base.length)), version: c.firstVersion + c.events.length - 1 } - this.#ctx.heads.set(s.key, head) - return head - } - return EMPTY_HEAD - } - - async #append(s: Stream, events: readonly unknown[], expected: number | 'any', hint?: Head): Promise { - const { driver, heads, maxRetries } = this.#ctx - if (events.length === 0) { - const head = await this.#loadHead(s) - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - return { version: head.version, commit: head.commit } - } - if (!driver.setIfAbsent) return this.#ctx.mutex.run(s.key, () => this.#appendSingleWriter(s, events, expected)) - - for (let attempt = 0; ; attempt++) { - let head = heads.get(s.key) ?? (attempt === 0 ? hint : undefined) - if (!head) head = await this.#loadHead(s) - else if (expected !== 'any' && head.version !== expected) head = await this.#loadHead(s) // cache may be stale - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - - const seq = head.commit + 1 - const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] } - const [written] = await driver.setIfAbsent([[s.base + pad(seq), record]]) - if (written) { - const next = { commit: seq, version: head.version + events.length } - heads.set(s.key, next) - return next - } - // Lost the race for this commit number. - heads.delete(s.key) - if (expected !== 'any' || attempt >= maxRetries) { - throw new ConcurrencyError(s.key, expected, (await this.#loadHead(s)).version) - } - } - } - - async #appendSingleWriter(s: Stream, events: readonly unknown[], expected: number | 'any'): Promise { - const head = this.#ctx.heads.get(s.key) ?? (await this.#loadHead(s)) - if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version) - const seq = head.commit + 1 - const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] } - await this.#ctx.driver.setMany([[s.base + pad(seq), record]]) - const next = { commit: seq, version: head.version + events.length } - this.#ctx.heads.set(s.key, next) - return next - } - - /** - * Sequence number of the commit containing `version`, or 0 if the stream hasn't reached it. - * Commit `n` has `firstVersion >= n`, so the answer is at most `version`: probe that first - * (one round trip when commits hold single events), then k-ary search with `getMany` probes. - */ - async #locate(s: Stream, version: number): Promise { - const { driver } = this.#ctx - const [guess] = (await driver.getMany([s.base + pad(version)])) as (CommitRecord | undefined)[] - if (guess && guess.firstVersion <= version) return version - let hi: number - if (guess) hi = version - 1 - else { - const head = await this.#loadHead(s) - if (head.version < version) return 0 - hi = head.commit - } - let lo = 1 // commit 1 starts at version 1 - while (lo < hi) { - const probes: number[] = [] - for (let i = 1; i <= SEARCH_FANOUT; i++) { - const p = lo + Math.ceil(((hi - lo) * i) / SEARCH_FANOUT) - if (p !== probes.at(-1)) probes.push(p) - } - const found = (await driver.getMany(probes.map((p) => s.base + pad(p)))) as (CommitRecord | undefined)[] - for (let i = 0; i < probes.length; i++) { - const c = found[i] - if (c && c.firstVersion <= version) lo = probes[i]! - else { - hi = probes[i]! - 1 - break - } - } - } - return lo - } - - async *#read(s: Stream, opts: ReadOptions): AsyncGenerator> { - const from = Math.max(1, opts.from ?? 1) - const to = opts.to ?? Infinity - let remaining = opts.limit ?? Infinity - if (to < from || remaining <= 0) return - const { driver } = this.#ctx - - if (!opts.reverse) { - let gte = s.base - if (from > 1) { - const seq = await this.#locate(s, from) - if (seq === 0) return - gte = s.base + pad(seq) - } - for await (const [key, value] of driver.scan({ gte, lt: s.end, values: true })) { - const c = value as CommitRecord - const commit = Number(key.slice(s.base.length)) - for (let i = 0; i < c.events.length; i++) { - const version = c.firstVersion + i - if (version < from) continue - if (version > to) return - yield { version, commit, at: c.at, data: c.events[i] } - if (--remaining <= 0) return - } - } - return - } - - let lt = s.end - if (to !== Infinity) { - const seq = await this.#locate(s, to) - if (seq !== 0) lt = s.base + pad(seq + 1) - } - for await (const [key, value] of driver.scan({ gte: s.base, lt, reverse: true, values: true })) { - const c = value as CommitRecord - const commit = Number(key.slice(s.base.length)) - for (let i = c.events.length - 1; i >= 0; i--) { - const version = c.firstVersion + i - if (version > to) continue - if (version < from) return - yield { version, commit, at: c.at, data: c.events[i] } - if (--remaining <= 0) return - } - } - } -} - -/** Event storage over a `KVStorage` (or any `KVNode` as root). */ -export class KVEventStorage extends KVEventNode { - constructor(kv: KVNode, opts: KVEventStorageOptions = {}) { - super( - { - driver: kv.driver, - sep: kv.separator, - heads: new LRU(opts.headCacheSize ?? 10_000), - mutex: new KeyedMutex(), - maxRetries: opts.maxRetries ?? 5, - now: opts.now ?? Date.now, - }, - kv, - ) - } -} diff --git a/src/events/types.ts b/src/events/types.ts deleted file mode 100644 index db798d8..0000000 --- a/src/events/types.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js' - -export interface AppendOptions { - /** Version the stream must be at (0 = empty). Default `'any'`. */ - expectedVersion?: number | 'any' - /** - * Commit sequence of the head at `expectedVersion`, if the caller knows it (e.g. from a - * projection). Lets a cold append skip the head lookup. Must come from the stream itself: a - * stale hint only causes a collision, but an invented one could leave a gap. Ignored unless - * `expectedVersion` is a number; backends may ignore it. - */ - expectedCommit?: number -} - -export interface AppendResult { - /** Stream version after the append (the last event's version). */ - version: number - /** Commit sequence number of this append (the current head commit when no events were given). */ - commit: number -} - -export interface ReadOptions { - /** First version to read (inclusive). Default 1. */ - from?: number - /** Last version to read (inclusive). Default: the head. */ - to?: number - reverse?: boolean - limit?: number -} - -export interface StoredEvent { - version: number - commit: number - /** Commit timestamp (ms since epoch). */ - at: number - data: E -} - -export interface StreamKeysOptions { - limit?: number - after?: string | number - reverse?: boolean -} - -/** - * One node of an event store: a stream (its own events) and a container of child streams. - * Same navigation and self/child arity rules as `KVNode`. - */ -export interface EventNode { - readonly path: readonly string[] - child>(key: K): EventNode> - as(): EventNode - - append(events: readonly ValueOf[], opts?: AppendOptions): Promise - append>(key: K, events: readonly ValueOf>[], opts?: AppendOptions): Promise - - read(opts?: ReadOptions): AsyncIterable>> - read>(key: K, opts?: ReadOptions): AsyncIterable>>> - - /** Current version (0 = empty). Always reads storage. */ - version(): Promise - version>(key: K): Promise - - /** Child stream keys. */ - keys(opts?: StreamKeysOptions): AsyncIterable> -} - -/** The root of an event store. Native backends (EventStoreDB, Kafka, …) implement this too. */ -export type EventStorage = EventNode diff --git a/src/http/client.ts b/src/http/client.ts deleted file mode 100644 index 86e4916..0000000 --- a/src/http/client.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { BaseKVDriver, JsonCodec, type BatchOp, type Codec, type ScanOptions } from '../kv/driver.js' -import type { HttpCapabilities } from './handler.js' - -export interface HttpKVDriverOptions { - /** Base URL of a `createKVHttpHandler` endpoint, e.g. `https://host/kv`. */ - url: string - /** Optional capabilities the server's driver supports. Default: all. `connect()` discovers them. */ - capabilities?: Partial - codec?: Codec - headers?: Record - fetch?: typeof fetch -} - -/** - * Driver client for `createKVHttpHandler`. Optional methods exist only when the server - * supports them, so stores' feature detection works across HTTP. - */ -export class HttpKVDriver extends BaseKVDriver { - readonly #url: string - readonly #codec: Codec - readonly #headers: Record - readonly #fetch: typeof fetch - readonly #nativeDeleteRange: boolean - - setIfAbsent?: (entries: [string, unknown][]) => Promise - batch?: (ops: BatchOp[]) => Promise - - /** Creates a client after asking the server which optional capabilities it has. */ - static async connect(opts: Omit): Promise { - const probe = new HttpKVDriver({ ...opts, capabilities: {} }) - const capabilities = (await probe.#request('GET', '/capabilities')) as HttpCapabilities - return new HttpKVDriver({ ...opts, capabilities }) - } - - constructor(opts: HttpKVDriverOptions) { - super() - this.#url = opts.url.replace(/\/$/, '') - this.#codec = opts.codec ?? JsonCodec - this.#headers = opts.headers ?? {} - this.#fetch = opts.fetch ?? globalThis.fetch.bind(globalThis) - const caps = { setIfAbsent: true, batch: true, deleteRange: true, ...opts.capabilities } - this.#nativeDeleteRange = caps.deleteRange - if (caps.setIfAbsent) this.setIfAbsent = async (entries) => ((await this.#request('POST', '/setIfAbsent', { entries })) as { written: boolean[] }).written - if (caps.batch) this.batch = async (ops) => void (await this.#request('POST', '/batch', { ops })) - } - - async getMany(keys: string[]): Promise { - if (keys.length === 0) return [] - const { values, missing } = (await this.#request('POST', '/getMany', { keys })) as { values: unknown[]; missing: number[] } - for (const i of missing) values[i] = undefined - return values - } - - async setMany(entries: [string, unknown][]): Promise { - if (entries.length) await this.#request('POST', '/setMany', { entries }) - } - - async deleteMany(keys: string[]): Promise { - if (keys.length) await this.#request('POST', '/deleteMany', { keys }) - } - - override async deleteRange(gte: string, lt: string): Promise { - if (!this.#nativeDeleteRange) return super.deleteRange(gte, lt) - await this.#request('POST', '/deleteRange', { gte, lt }) - } - - async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> { - if (opts.limit !== undefined && opts.limit <= 0) return - const q = new URLSearchParams() - if (opts.gte !== undefined) q.set('gte', opts.gte) - if (opts.lt !== undefined) q.set('lt', opts.lt) - if (opts.limit !== undefined && opts.limit !== Infinity) q.set('limit', String(opts.limit)) - if (opts.reverse) q.set('reverse', 'true') - if (opts.values) q.set('values', 'true') - const res = await this.#fetch(`${this.#url}/scan?${q}`, { headers: this.#headers }) - if (!res.ok || !res.body) throw await httpError(res) - const reader = res.body.getReader() - const decoder = new TextDecoder() - let buf = '' - try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - buf += decoder.decode(value, { stream: true }) - let nl: number - while ((nl = buf.indexOf('\n')) !== -1) { - const line = buf.slice(0, nl) - buf = buf.slice(nl + 1) - const [key, v] = this.#codec.decode(line) as [string, unknown?] - yield [key, v] - } - } - } finally { - await reader.cancel().catch(() => {}) - } - } - - async #request(method: 'GET' | 'POST', path: string, body?: unknown): Promise { - const res = await this.#fetch(this.#url + path, { - method, - headers: { ...this.#headers, ...(body === undefined ? {} : { 'content-type': 'application/json' }) }, - body: body === undefined ? undefined : this.#codec.encode(body), - }) - if (!res.ok) throw await httpError(res) - return res.status === 204 ? undefined : this.#codec.decode(await res.text()) - } -} - -async function httpError(res: Response): Promise { - let detail = '' - try { - detail = await res.text() - } catch {} - return new Error(`HTTP ${res.status}: ${detail}`) -} diff --git a/src/http/handler.ts b/src/http/handler.ts deleted file mode 100644 index c557ecd..0000000 --- a/src/http/handler.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { JsonCodec, type BatchOp, type Codec, type KVDriver, type ScanOptions } from '../kv/driver.js' - -export interface KVHttpHandlerOptions { - /** Serialises request/response bodies and NDJSON scan lines. Must not emit newlines. Default JSON. */ - codec?: Codec - /** Path prefix to strip, e.g. `/kv`. Default `''`. */ - basePath?: string -} - -export interface HttpCapabilities { - setIfAbsent: boolean - batch: boolean - deleteRange: boolean -} - -/** - * Framework-agnostic HTTP front for a driver (`Request → Response`). - * - * - `POST /getMany {keys}` → `{values, missing}` (`missing` = indices of absent keys) - * - `POST /setMany {entries}`, `POST /deleteMany {keys}`, `POST /batch {ops}` → 204 - * - `POST /setIfAbsent {entries}` → `{written}` - * - `POST /deleteRange {gte, lt}` → 204 - * - `GET /scan?gte<&limit&reverse&values` → NDJSON, one `[key]` or `[key, value]` per line - * - `GET /capabilities` → `{setIfAbsent, batch, deleteRange}` - */ -export function createKVHttpHandler(driver: KVDriver, opts: KVHttpHandlerOptions = {}): (req: Request) => Promise { - const codec = opts.codec ?? JsonCodec - const base = opts.basePath ?? '' - const reply = (body: unknown, status = 200) => new Response(codec.encode(body), { status, headers: { 'content-type': 'application/json' } }) - const noContent = () => new Response(null, { status: 204 }) - const fail = (status: number, message: string) => reply({ error: message }, status) - - return async (req) => { - const url = new URL(req.url) - if (!url.pathname.startsWith(base)) return fail(404, 'Not found') - const route = `${req.method} ${url.pathname.slice(base.length)}` - try { - const body = async () => codec.decode(await req.text()) as T - switch (route) { - case 'POST /getMany': { - const { keys } = await body<{ keys: string[] }>() - const found = await driver.getMany(keys) - const missing: number[] = [] - const values = found.map((v, i) => (v === undefined ? (missing.push(i), null) : v)) - return reply({ values, missing }) - } - case 'POST /setMany': - await driver.setMany((await body<{ entries: [string, unknown][] }>()).entries) - return noContent() - case 'POST /deleteMany': - await driver.deleteMany((await body<{ keys: string[] }>()).keys) - return noContent() - case 'POST /setIfAbsent': - if (!driver.setIfAbsent) return fail(501, 'setIfAbsent not supported') - return reply({ written: await driver.setIfAbsent((await body<{ entries: [string, unknown][] }>()).entries) }) - case 'POST /batch': - if (!driver.batch) return fail(501, 'batch not supported') - await driver.batch((await body<{ ops: BatchOp[] }>()).ops) - return noContent() - case 'POST /deleteRange': { - if (!driver.deleteRange) return fail(501, 'deleteRange not supported') - const { gte, lt } = await body<{ gte: string; lt: string }>() - await driver.deleteRange(gte, lt) - return noContent() - } - case 'GET /capabilities': - return reply({ setIfAbsent: !!driver.setIfAbsent, batch: !!driver.batch, deleteRange: !!driver.deleteRange } satisfies HttpCapabilities) - case 'GET /scan': - return scanResponse(driver, parseScan(url.searchParams), codec) - default: - return fail(404, `No route ${route}`) - } - } catch (err) { - return fail(500, err instanceof Error ? err.message : String(err)) - } - } -} - -function parseScan(q: URLSearchParams): ScanOptions { - const opts: ScanOptions = {} - if (q.has('gte')) opts.gte = q.get('gte')! - if (q.has('lt')) opts.lt = q.get('lt')! - if (q.has('limit')) opts.limit = Number(q.get('limit')) - if (q.get('reverse') === 'true') opts.reverse = true - if (q.get('values') === 'true') opts.values = true - return opts -} - -function scanResponse(driver: KVDriver, opts: ScanOptions, codec: Codec): Response { - const it = driver.scan(opts)[Symbol.asyncIterator]() - const encoder = new TextEncoder() - // Chunks grow so short (skip-scan) reads don't make the server run far ahead. - let chunk = 16 - const stream = new ReadableStream({ - async pull(controller) { - let buf = '' - for (let i = 0; i < chunk; i++) { - const r = await it.next() - if (r.done) { - if (buf) controller.enqueue(encoder.encode(buf)) - controller.close() - return - } - const [key, value] = r.value - buf += codec.encode(opts.values ? [key, value] : [key]) + '\n' - } - chunk = Math.min(chunk * 2, 256) - controller.enqueue(encoder.encode(buf)) - }, - async cancel() { - await it.return?.() - }, - }) - return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } }) -} diff --git a/src/http/index.ts b/src/http/index.ts deleted file mode 100644 index 13ba7ec..0000000 --- a/src/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './handler.js' -export * from './client.js' diff --git a/src/index.ts b/src/index.ts index 9400bff..78841b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,18 @@ -export * from './errors.js' -export * from './core/path.js' -export * from './core/encoding.js' -export * from './kv/index.js' -export * from './types/schema.js' -export * from './events/index.js' -export * from './documents/index.js' -export * from './http/index.js' +export * from './kv-storage/base-kv-storage.js' +export * from './kv-storage/kv-storage-in-memory.js' +export * from './kv-storage/kv-storage-http.js' +export * from './kv-storage/kv-storage-http-server.js' +export * from './event-storage/base-event-storage.js' +export * from './event-storage/event-storage-kv.js' +export * from './document-storage/base-document-storage.js' +export * from './document-storage/document-storage-in-memory.js' +export * from './document-storage/document-storage-kv.js' +export * from './document-storage/event-sourced-document-storage.js' +export * from './utils/document-path.js' +export * from './utils/document-query.js' +export * from './utils/document-update.js' +export * from './utils/errors.js' +export * from './utils/json.js' +export * from './utils/keys.js' +export * from './utils/schema.js' +export * from './utils/ulid.js' diff --git a/src/kv-storage/base-kv-storage.ts b/src/kv-storage/base-kv-storage.ts new file mode 100644 index 0000000..e13a874 --- /dev/null +++ b/src/kv-storage/base-kv-storage.ts @@ -0,0 +1,184 @@ +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueAt, ValueOf } from '../utils/schema.js' + +export interface ScanOptions { + /** Inclusive lower bound. */ + gte?: string + /** Exclusive upper bound. */ + lt?: string + limit?: number + reverse?: boolean + /** Include values; otherwise they are `undefined`. */ + values?: boolean +} + +export interface KeysOptions { + limit?: number + /** Start after this child key (exclusive), in the direction of travel. */ + after?: string | number + reverse?: boolean +} + +/** + * Ordered key-value storage. Implementations provide five raw primitives over absolute keys; + * everything else (navigation, self/child reads and writes, child listing) is defined here once. + * + * Every instance is a node: `child(key)` returns a view of the same storage rooted at that key. + * Methods taking an optional `key` act on that child, or on this node's own value when omitted. + * A key that is passed but `undefined`/`null` throws rather than silently meaning "self". + */ +export abstract class BaseKVStorage { + /** Unescaped path of this node (`[]` for the root). */ + path: readonly string[] = [] + /** This node's own key (`''` for the root). */ + key = '' + /** Prefix of every descendant key (`''` for the root). */ + prefix = '' + + /** Values for absolute keys, in order; `undefined` for missing. */ + abstract readMany(keys: string[]): Promise + abstract writeMany(entries: [key: string, value: unknown][]): Promise + abstract removeMany(keys: string[]): Promise + /** Writes each entry only if its key is absent; `true` = written. Must be atomic per key. */ + abstract writeIfAbsent(entries: [key: string, value: unknown][]): Promise + /** Entries in key order (descending when `reverse`). */ + abstract scan(options?: ScanOptions): AsyncIterable<[key: string, value: unknown]> + + // ------------------------------------------------------------------ navigation + + /** A view of this storage rooted at `key`. Synchronous; no I/O. */ + child>(key: K): BaseKVStorage> { + const node = Object.create(this) as BaseKVStorage> + node.path = [...this.path, Keys.segment(key)] + node.key = this.prefix + Keys.escape(Keys.segment(key)) + node.prefix = node.key + Keys.separator + return node + } + + /** Re-types this node. No runtime effect. */ + as(): BaseKVStorage { + return this as unknown as BaseKVStorage + } + + // ------------------------------------------------------------------ reads and writes + + async get | undefined = undefined>(key?: K): Promise | undefined> { + const [value] = await this.readMany([this.keyOf(arguments.length > 0, key)]) + return value as ValueAt | undefined + } + + async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { + return (await this.readMany(keys.map((k) => this.keyOf(true, k)))) as (ValueOf> | undefined)[] + } + + /** `set(value)` writes this node's own value; `set(key, value)` a child's. */ + async set>(...args: [value: ValueOf] | [key: K, value: ValueOf>]): Promise { + const value = args[args.length - 1] + if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') + await this.writeMany([[this.keyOf(args.length > 1, args[0]), value]]) + } + + async setMany(entries: readonly EntryOf[]): Promise { + if (entries.some(([, v]) => v === undefined)) throw new TypeError('Cannot set undefined; use delete()') + await this.writeMany(entries.map(([k, v]) => [this.keyOf(true, k), v])) + } + + /** Deletes this node's own value (or a child's), not descendants. */ + async delete(key?: KeyOf): Promise { + await this.removeMany([this.keyOf(arguments.length > 0, key)]) + } + + async deleteMany(keys: readonly KeyOf[]): Promise { + await this.removeMany(keys.map((k) => this.keyOf(true, k))) + } + + /** Deletes this node's own value (or a child's) and all descendants. */ + async deleteTree(key?: KeyOf): Promise { + const own = this.keyOf(arguments.length > 0, key, true) + if (own !== '') await this.removeMany([own]) + const prefix = own === '' ? '' : own + Keys.separator + for (;;) { + const batch: string[] = [] + for await (const [k] of this.scan({ gte: prefix, lt: own === '' ? undefined : own + Keys.afterSeparator, limit: 1000 })) batch.push(k) + if (batch.length === 0) return + await this.removeMany(batch) + } + } + + // ------------------------------------------------------------------ listing + + /** + * Direct child keys in key order, including children that only have descendants. + * Skip-scan: one seek per child that has descendants, not one read per descendant. + */ + async *keys(options: KeysOptions = {}): AsyncGenerator> { + for await (const [key] of this.children(options, false)) yield key as KeyString + } + + /** Direct children that have their own value, as `[key, value]`. */ + async *entries(options: KeysOptions = {}): AsyncGenerator> { + let remaining = options.limit ?? Infinity + for await (const [key, value, hasValue] of this.children({ ...options, limit: undefined }, true)) { + if (remaining <= 0) return + if (!hasValue) continue + remaining-- + yield [key, value] as StringEntryOf + } + } + + private async *children(options: KeysOptions, values: boolean): AsyncGenerator<[key: string, value: unknown, hasValue: boolean]> { + const { reverse = false } = options + let remaining = options.limit ?? Infinity + let gte = this.prefix + let lt = this.key === '' ? undefined : this.key + Keys.afterSeparator + if (options.after !== undefined) { + const after = this.prefix + Keys.escape(Keys.segment(options.after)) + if (reverse) lt = after + else gte = after + Keys.afterSeparator + } + let last: string | undefined + let restart = true + let iterator: AsyncIterator<[string, unknown]> | undefined + try { + while (remaining > 0) { + if (restart) { + await iterator?.return?.() + iterator = this.scan({ gte, lt, reverse, values })[Symbol.asyncIterator]() + restart = false + } + const next = await iterator!.next() + if (next.done) return + const [key, value] = next.value + const end = key.indexOf(Keys.separator, this.prefix.length) + const segment = key.slice(this.prefix.length, end === -1 ? undefined : end) + const isOwn = end === -1 + if (segment !== last && !Keys.isReserved(segment)) { + // Forward, a child's own key comes before its descendants; in reverse it comes after + // them and is skipped by the jump below, so read it directly when values are wanted. + let own = isOwn ? value : undefined + if (!isOwn && reverse && values) [own] = await this.readMany([this.prefix + segment]) + yield [Keys.unescape(segment), own, isOwn || own !== undefined] + remaining-- + } + last = segment + // Jump over the rest of this child's subtree instead of reading it. + if (!isOwn || Keys.isReserved(segment)) { + if (reverse) lt = this.prefix + segment + else gte = this.prefix + segment + Keys.afterSeparator + restart = true + } + } + } finally { + await iterator?.return?.() + } + } + + // ------------------------------------------------------------------ internals + + /** Absolute key for a child (`hasKey`) or this node; validates keys. */ + protected keyOf(hasKey: boolean, key: unknown, allowRoot = false): string { + if (hasKey) return this.prefix + Keys.escape(Keys.segment(key)) + if (this.key === '' && !allowRoot) throw new TypeError('The root node has no own value') + return this.key + } +} diff --git a/src/kv-storage/kv-storage-http-server.ts b/src/kv-storage/kv-storage-http-server.ts new file mode 100644 index 0000000..54be60d --- /dev/null +++ b/src/kv-storage/kv-storage-http-server.ts @@ -0,0 +1,70 @@ +import type { BaseKVStorage, ScanOptions } from './base-kv-storage.js' + +/** + * Serves any KV storage over HTTP for `KVStorageHttp`. Framework-agnostic: `handle(Request)` + * returns a `Response`. All routes are `POST /` with a JSON body; `scan` streams NDJSON. + * The raw primitives use absolute keys, so this exposes the whole store behind `storage`. + */ +export class KVStorageHttpServer { + constructor( + private readonly storage: BaseKVStorage, + private readonly basePath = '', + ) {} + + async handle(request: Request): Promise { + const path = new URL(request.url).pathname + if (request.method !== 'POST' || !path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') + try { + const body = (await request.json()) as { keys: string[]; entries: [string, unknown][] } & ScanOptions + switch (path.slice(this.basePath.length)) { + case '/readMany': { + const values = await this.storage.readMany(body.keys) + const missing = values.flatMap((v, i) => (v === undefined ? [i] : [])) + return Response.json({ values: values.map((v) => v ?? null), missing }) + } + case '/writeMany': + await this.storage.writeMany(body.entries) + return new Response(null, { status: 204 }) + case '/removeMany': + await this.storage.removeMany(body.keys) + return new Response(null, { status: 204 }) + case '/writeIfAbsent': + return Response.json(await this.storage.writeIfAbsent(body.entries)) + case '/scan': + return this.scan(body) + default: + return KVStorageHttpServer.error(404, 'Not found') + } + } catch (error) { + return KVStorageHttpServer.error(500, error instanceof Error ? error.message : String(error)) + } + } + + private scan(options: ScanOptions): Response { + const iterator = this.storage.scan(options)[Symbol.asyncIterator]() + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async pull(controller) { + // Send in small batches so short reads (skip-scan seeks) don't make the server run far ahead. + let chunk = '' + for (let i = 0; i < 32; i++) { + const next = await iterator.next() + if (next.done) { + if (chunk) controller.enqueue(encoder.encode(chunk)) + return controller.close() + } + chunk += JSON.stringify(options.values ? next.value : [next.value[0]]) + '\n' + } + controller.enqueue(encoder.encode(chunk)) + }, + async cancel() { + await iterator.return?.() + }, + }) + return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } }) + } + + private static error(status: number, message: string): Response { + return Response.json({ error: message }, { status }) + } +} diff --git a/src/kv-storage/kv-storage-http.ts b/src/kv-storage/kv-storage-http.ts new file mode 100644 index 0000000..ced0b4e --- /dev/null +++ b/src/kv-storage/kv-storage-http.ts @@ -0,0 +1,79 @@ +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' + +export interface KVStorageHttpOptions { + /** Base URL of a `KVStorageHttpServer`, e.g. `https://host/kv`. */ + url: string + headers?: Record + fetch?: typeof fetch +} + +/** KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage. */ +export class KVStorageHttp extends BaseKVStorage { + private readonly url: string + private readonly headers: Record + private readonly fetcher: typeof fetch + + constructor(options: KVStorageHttpOptions) { + super() + this.url = options.url.replace(/\/$/, '') + this.headers = options.headers ?? {} + this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis) + } + + async readMany(keys: string[]): Promise { + if (keys.length === 0) return [] + const { values, missing } = (await this.post('readMany', { keys })) as { values: unknown[]; missing: number[] } + for (const i of missing) values[i] = undefined + return values + } + + async writeMany(entries: [string, unknown][]): Promise { + if (entries.length) await this.post('writeMany', { entries }) + } + + async removeMany(keys: string[]): Promise { + if (keys.length) await this.post('removeMany', { keys }) + } + + async writeIfAbsent(entries: [string, unknown][]): Promise { + return entries.length ? ((await this.post('writeIfAbsent', { entries })) as boolean[]) : [] + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + if (options.limit !== undefined && options.limit <= 0) return + const response = await this.request('scan', options) + const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader() + let buffer = '' + try { + for (;;) { + const { done, value } = await reader.read() + if (done) return + buffer += value + const lines = buffer.split('\n') + buffer = lines.pop()! + for (const line of lines) { + const [key, value] = JSON.parse(line) as [string, unknown?] + yield [key, value] + } + } + } finally { + await reader.cancel().catch(() => {}) + } + } + + private async post(path: string, body: unknown): Promise { + const response = await this.request(path, body) + return response.status === 204 ? undefined : response.json() + } + + private async request(path: string, body: unknown): Promise { + const response = await this.fetcher(`${this.url}/${path}`, { + method: 'POST', + headers: { ...this.headers, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`) + return response + } +} diff --git a/src/kv-storage/kv-storage-in-memory.ts b/src/kv-storage/kv-storage-in-memory.ts new file mode 100644 index 0000000..6883e1a --- /dev/null +++ b/src/kv-storage/kv-storage-in-memory.ts @@ -0,0 +1,34 @@ +import { Json } from '../utils/json.js' +import { SortedMap } from '../utils/sorted-map.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' + +/** In-memory KV storage. Values are deep-copied on write and read. */ +export class KVStorageInMemory extends BaseKVStorage { + private readonly data = new SortedMap() + + async readMany(keys: string[]): Promise { + return keys.map((k) => Json.clone(this.data.get(k))) + } + + async writeMany(entries: [string, unknown][]): Promise { + for (const [k, v] of entries) this.data.set(k, Json.clone(v)) + } + + async removeMany(keys: string[]): Promise { + for (const k of keys) this.data.delete(k) + } + + async writeIfAbsent(entries: [string, unknown][]): Promise { + return entries.map(([k, v]) => !this.data.has(k) && (this.data.set(k, Json.clone(v)), true)) + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + let remaining = options.limit ?? Infinity + if (remaining <= 0) return + for (const key of this.data.keys(options.gte, options.lt, options.reverse)) { + yield [key, options.values ? Json.clone(this.data.get(key)) : undefined] + if (--remaining <= 0) return + } + } +} diff --git a/src/kv/children.ts b/src/kv/children.ts deleted file mode 100644 index b295233..0000000 --- a/src/kv/children.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { escapeSegment, succ } from '../core/path.js' -import type { KVDriver } from './driver.js' - -/** - * Direct-children skip-scan, shared by every layer. - * - * ## Child order - * - * Children are ordered by **escaped segment followed by the separator** ("subtree order": the - * order in which their descendants appear in key space). For segments without characters that - * sort below the separator this is plain lexicographic order. When one segment extends another - * with such a character (`a` vs `a-1` with `/`), the longer one comes first (`a-1/` < `a/`). - * - * This is the one order that can be produced without extra reads: a child's own key - * (`p/a`) sorts *before* siblings that extend it (`p/a-1`), while its descendants (`p/a/…`) sort - * after them. The forward scan therefore defers yielding a child seen only through its own key - * until the scan passes `p/a/`, using a stack bounded by segment length. - * - * ## Cost - * - * - Forward: one seek per child that has descendants (to jump over its subtree), one at the start, - * and one per reserved (`~…`) region. Siblings without descendants share an iterator. - * `after` adds one `getMany` when the `after` segment contains a below-separator character. - * - Reverse: as forward, plus one `getMany` per newly found child whose segment contains a - * below-separator character (to find shorter siblings that only have an own key), or that was - * found through a descendant when `own` is requested. - */ -export interface ChildHit { - /** Escaped segment. */ - esc: string - /** Whether the child has an own value. Only accurate when `own` was requested. */ - hasOwn: boolean - /** The own value, when `values` was requested and `hasOwn`. */ - value: unknown -} - -export interface ChildScanOptions { - /** Unescaped segment to start after (exclusive), in the direction of travel. */ - after?: string - reverse?: boolean - limit?: number - /** Fetch own values. Implies `own`. */ - values?: boolean - /** Require accurate `hasOwn`. */ - own?: boolean -} - -const RESERVED_START = '~' -const RESERVED_END = succ('~') - -export async function* scanChildren( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions = {}, -): AsyncGenerator { - const limit = opts.limit ?? Infinity - if (limit <= 0) return - if (opts.reverse) yield* reverseScan(driver, prefix, end, sep, opts, limit) - else yield* forwardScan(driver, prefix, end, sep, opts, limit) -} - -/** Prefixes `E[0..j)` (j ≥ 1) where `E[j]` sorts below the separator. */ -function lowPrefixes(esc: string, sepCode: number): string[] { - let out: string[] | undefined - for (let j = 1; j < esc.length; j++) { - if (esc.charCodeAt(j) < sepCode) (out ??= []).push(esc.slice(0, j)) - } - return out ?? NONE -} - -const NONE: string[] = [] - -async function* forwardScan( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions, - limit: number, -): AsyncGenerator { - const sepCode = sep.charCodeAt(0) - const succSep = succ(sep) - const values = opts.values === true - const pending: ChildHit[] = [] - let emitted = 0 - let gte = prefix - - if (opts.after !== undefined) { - const escA = escapeSegment(opts.after, sep) - gte = prefix + escA + succSep - // Shorter siblings that extend into `after` with a low char come after it, but their own - // keys sort before `gte`; probe for them. - const cands = lowPrefixes(escA, sepCode) - if (cands.length) { - const found = await driver.getMany(cands.map((g) => prefix + g)) - for (let i = 0; i < cands.length; i++) { - if (found[i] !== undefined) pending.push({ esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined }) - } - } - } - - const prefixLen = prefix.length - let it = driver.scan({ gte, lt: end, values })[Symbol.asyncIterator]() - const reseek = async (from: string) => { - await it.return?.() - it = driver.scan({ gte: from, lt: end, values })[Symbol.asyncIterator]() - } - - try { - outer: for (;;) { - const r = await it.next() - if (r.done) { - while (pending.length) { - yield pending.pop()! - if (++emitted >= limit) return - } - return - } - const key = r.value[0] - const cut = key.indexOf(sep, prefixLen) - const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut) - - while (pending.length) { - const top = pending[pending.length - 1]! - if (top.esc === esc) { - // First descendant of a deferred child: emit it and jump over its subtree. - pending.pop() - yield top - if (++emitted >= limit) return - await reseek(prefix + esc + succSep) - continue outer - } - if (top.esc + sep < esc + sep) { - pending.pop() - yield top - if (++emitted >= limit) return - } else break - } - - if (esc.startsWith(RESERVED_START)) { - await reseek(prefix + RESERVED_END) - continue - } - if (cut === -1) { - // Own key: defer until we know no sibling extending it with a low char comes next. - pending.push({ esc, hasOwn: true, value: r.value[1] }) - continue - } - // Descendant of a child with no own value (an own key would have sorted first). - yield { esc, hasOwn: false, value: undefined } - if (++emitted >= limit) return - await reseek(prefix + esc + succSep) - } - } finally { - await it.return?.() - } -} - -async function* reverseScan( - driver: KVDriver, - prefix: string, - end: string | undefined, - sep: string, - opts: ChildScanOptions, - limit: number, -): AsyncGenerator { - const sepCode = sep.charCodeAt(0) - const values = opts.values === true - const needOwn = values || opts.own === true - // Segments already emitted whose own key may still appear (it sorts after everything else of theirs). - const done: string[] = [] - let emitted = 0 - let lt = end - - if (opts.after !== undefined) { - const escA = escapeSegment(opts.after, sep) - lt = prefix + escA + sep - // `after` itself and shorter siblings extending into it with a low char sort after it in - // child order but their own keys fall below `lt`: mark them done. - done.push(...lowPrefixes(escA, sepCode), escA) - } - - const prefixLen = prefix.length - let it = driver.scan({ gte: prefix, lt, reverse: true, values })[Symbol.asyncIterator]() - const reseek = async (below: string) => { - await it.return?.() - it = driver.scan({ gte: prefix, lt: below, reverse: true, values })[Symbol.asyncIterator]() - } - - try { - for (;;) { - const r = await it.next() - if (r.done) return - const key = r.value[0] - const cut = key.indexOf(sep, prefixLen) - const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut) - - if (esc.startsWith(RESERVED_START)) { - await reseek(prefix + RESERVED_START) - continue - } - while (done.length && !esc.startsWith(done[done.length - 1]!)) done.pop() - if (done.length && done[done.length - 1] === esc) { - done.pop() - if (cut !== -1) await reseek(prefix + esc + sep) - continue - } - - // New child. Shorter siblings that it extends with a low char come before it in reverse - // order; any with descendants were already emitted, so only own-key-only ones are unknown. - const cands = lowPrefixes(esc, sepCode).filter((g) => !done.includes(g)) - const probeOwn = cut !== -1 && needOwn - let found: unknown[] = NONE - if (cands.length || probeOwn) { - const probe = cands.map((g) => prefix + g) - if (probeOwn) probe.push(prefix + esc) - found = await driver.getMany(probe) - } - for (let i = 0; i < cands.length; i++) { - if (found[i] === undefined) continue - yield { esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined } - done.push(cands[i]!) - if (++emitted >= limit) return - } - const own = cut === -1 ? r.value[1] : probeOwn ? found[cands.length] : undefined - yield { esc, hasOwn: cut === -1 || own !== undefined, value: values ? own : undefined } - done.push(esc) - if (++emitted >= limit) return - if (cut !== -1) await reseek(prefix + esc + sep) - } - } finally { - await it.return?.() - } -} diff --git a/src/kv/driver.ts b/src/kv/driver.ts deleted file mode 100644 index f7f3b23..0000000 --- a/src/kv/driver.ts +++ /dev/null @@ -1,77 +0,0 @@ -export interface ScanOptions { - gte?: string - lt?: string - limit?: number - reverse?: boolean - /** When false/omitted, yielded values are `undefined`. */ - values?: boolean -} - -export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string } - -/** - * A flat, string-keyed backend. Values are JSON-serialisable; byte-oriented backends serialise - * internally with a {@link Codec}. All path, index and typing logic lives above this interface. - */ -export interface KVDriver { - /** Same order as `keys`; `undefined` for missing keys. */ - getMany(keys: string[]): Promise<(unknown | undefined)[]> - setMany(entries: [key: string, value: unknown][]): Promise - deleteMany(keys: string[]): Promise - /** MUST yield in key order (descending when `reverse`). `gte` is inclusive, `lt` exclusive. */ - scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]> - - // Optional capabilities. Stores feature-detect them and fall back where possible. - /** Writes each entry only if its key is absent. `true` = written. */ - setIfAbsent?(entries: [string, unknown][]): Promise - /** Applies all ops atomically. */ - batch?(ops: BatchOp[]): Promise - deleteRange?(gte: string, lt: string): Promise -} - -/** Serialises values for byte/string-oriented backends. */ -export interface Codec { - encode(value: unknown): string - decode(encoded: string): unknown -} - -export const JsonCodec: Codec = { - encode: (value) => JSON.stringify(value), - decode: (encoded) => JSON.parse(encoded), -} - -/** - * Base class for drivers: implement the four required methods and get `deleteRange` - * (scan + deleteMany in chunks) and single-key helpers for free. - */ -export abstract class BaseKVDriver implements KVDriver { - abstract getMany(keys: string[]): Promise<(unknown | undefined)[]> - abstract setMany(entries: [string, unknown][]): Promise - abstract deleteMany(keys: string[]): Promise - abstract scan(opts: ScanOptions): AsyncIterable<[string, unknown]> - - /** Chunk size used by the default `deleteRange`. */ - protected deleteRangeChunk = 1000 - - async deleteRange(gte: string, lt: string): Promise { - for (;;) { - const keys: string[] = [] - for await (const [key] of this.scan({ gte, lt, limit: this.deleteRangeChunk })) keys.push(key) - if (keys.length === 0) return - await this.deleteMany(keys) - if (keys.length < this.deleteRangeChunk) return - } - } - - async get(key: string): Promise { - return (await this.getMany([key]))[0] - } - - set(key: string, value: unknown): Promise { - return this.setMany([[key, value]]) - } - - delete(key: string): Promise { - return this.deleteMany([key]) - } -} diff --git a/src/kv/index.ts b/src/kv/index.ts deleted file mode 100644 index cfab7c0..0000000 --- a/src/kv/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './driver.js' -export * from './memory.js' -export * from './sorted.js' -export * from './node.js' -export * from './indexes.js' -export * from './children.js' diff --git a/src/kv/indexes.ts b/src/kv/indexes.ts deleted file mode 100644 index 2138b10..0000000 --- a/src/kv/indexes.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { indexValueCodec, isIndexValue, type IndexValue, type IndexValueCodec } from '../core/encoding.js' -import { escapeSegment, RESERVED, succ } from '../core/path.js' -import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js' - -export type { IndexValue } - -/** Maps a child's value to index value(s). An array means multi-entry; `undefined` means not indexed. */ -export type IndexFn = (value: V) => IndexValue | readonly IndexValue[] | undefined - -/** Index name → function, for the direct children of nodes matching one pattern. */ -export type IndexDefs = Record - -/** Path pattern (`*` = one segment, `''` = root) → index definitions. */ -export type IndexDeclarations = Record - -/** A value or a range over index values. Bounds combine (`{ gte, lt }`). */ -export type IndexQuery = IndexValue | IndexRange - -export interface IndexRange { - gt?: IndexValue - gte?: IndexValue - lt?: IndexValue - lte?: IndexValue -} - -export function isIndexRange(q: unknown): q is IndexRange { - return typeof q === 'object' && q !== null -} - -interface CompiledIndex { - name: string - /** `escapeSegment(name) + sep` */ - head: string - fn: IndexFn -} - -/** Indexes over the direct children of one concrete node. Entry keys are rooted at `base`. */ -export class BoundIndexes { - readonly #byName: Map - readonly #list: CompiledIndex[] - readonly #codec: IndexValueCodec - - constructor( - defs: CompiledIndex[], - /** `~i` */ - readonly base: string, - readonly sep: string, - ) { - this.#list = defs - this.#byName = new Map(defs.map((d) => [d.name, d])) - this.#codec = indexValueCodec(sep) - } - - has(name: string): boolean { - return this.#byName.has(name) - } - - /** All index entry keys for a child (escaped segment) holding `value`. */ - entriesFor(escChild: string, value: unknown, out: Set = new Set()): Set { - if (value === undefined) return out - for (const idx of this.#list) this.#entriesOf(idx, escChild, value, out) - return out - } - - #entriesOf(idx: CompiledIndex, escChild: string, value: unknown, out: Set): void { - const r = idx.fn(value) - if (r === undefined) return - const stem = this.base + idx.head - const tail = this.sep + escChild - if (Array.isArray(r)) { - for (const v of r as IndexValue[]) out.add(stem + this.#encode(idx.name, v) + tail) - } else out.add(stem + this.#encode(idx.name, r as IndexValue) + tail) - } - - #encode(name: string, v: unknown): string { - if (!isIndexValue(v)) throw new TypeError(`Index "${name}" returned a non-indexable value: ${String(v)}`) - return this.#codec.encode(v) - } - - /** Whether `entryKey` is among the entries `value` currently produces for index `name`. */ - matches(name: string, escChild: string, value: unknown, entryKey: string): boolean { - if (value === undefined) return false - const out = new Set() - this.#entriesOf(this.#get(name), escChild, value, out) - return out.has(entryKey) - } - - /** Scan bounds over the entries of index `name` for a query. */ - range(name: string, q: IndexQuery): { gte: string; lt: string } { - const stem = this.base + this.#get(name).head - const succSep = succ(this.sep) - const enc = (v: IndexValue) => stem + this.#encode(name, v) - if (!isIndexRange(q)) { - const k = enc(q) - return { gte: k + this.sep, lt: k + succSep } - } - let gte = stem - let lt = stem.slice(0, -1) + succSep - if (q.gte !== undefined) gte = maxStr(gte, enc(q.gte)) - if (q.gt !== undefined) gte = maxStr(gte, enc(q.gt) + succSep) - if (q.lt !== undefined) lt = minStr(lt, enc(q.lt)) - if (q.lte !== undefined) lt = minStr(lt, enc(q.lte) + succSep) - return { gte, lt } - } - - /** Splits an entry key under index `name` into the escaped child segment. */ - childOf(name: string, entryKey: string): string { - const start = this.base.length + this.#get(name).head.length - return entryKey.slice(entryKey.indexOf(this.sep, start) + 1) - } - - #get(name: string): CompiledIndex { - const idx = this.#byName.get(name) - if (!idx) throw new TypeError(`Unknown index "${name}"`) - return idx - } -} - -const maxStr = (a: string, b: string) => (a > b ? a : b) -const minStr = (a: string, b: string) => (a < b ? a : b) - -/** Compiles index declarations into a pattern trie and binds them to nodes. */ -export class IndexRegistry { - readonly trie: PatternTrie - readonly empty: boolean - - constructor( - decls: IndexDeclarations, - readonly sep: string, - ) { - this.trie = new PatternTrie(sep) - const patterns = Object.keys(decls) - for (let i = 0; i < patterns.length; i++) { - for (let j = i + 1; j < patterns.length; j++) { - const a = patterns[i]! - const b = patterns[j]! - if (!patternsOverlap(a, b, sep)) continue - const clash = Object.keys(decls[a]!).filter((n) => n in decls[b]!) - if (clash.length) throw new TypeError(`Index name(s) ${clash.join(', ')} declared on overlapping patterns "${a}" and "${b}"`) - } - } - for (const p of patterns) this.trie.add(p, decls[p]!) - this.empty = patterns.length === 0 - } - - /** Indexes over the children of the node with trie state `state` and key prefix `prefix`. */ - bind(state: TrieState, prefix: string): BoundIndexes | undefined { - if (this.empty) return undefined - const found = PatternTrie.values(state) - if (found.length === 0) return undefined - const defs: CompiledIndex[] = [] - for (const { value } of found) { - for (const [name, fn] of Object.entries(value)) defs.push({ name, head: escapeSegment(name, this.sep) + this.sep, fn }) - } - return new BoundIndexes(defs, prefix + RESERVED.index + this.sep, this.sep) - } -} diff --git a/src/kv/memory.ts b/src/kv/memory.ts deleted file mode 100644 index dabbc92..0000000 --- a/src/kv/memory.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { BaseKVDriver, type BatchOp, type ScanOptions } from './driver.js' -import { ChunkedSortedMap, type SortedMap } from './sorted.js' - -export interface InMemoryKVDriverOptions { - /** - * Deep-copy values on write and read so callers can't mutate stored state through a shared - * reference (matching serialising drivers). Values must be JSON-compatible. Default `true`. - * Disable for speed when values are treated as immutable. - */ - clone?: boolean -} - -/** - * In-memory driver over a sorted map. Implements every optional capability: - * `setIfAbsent`, atomic `batch` and `deleteRange`. - */ -export class InMemoryKVDriver extends BaseKVDriver { - readonly #data: SortedMap = new ChunkedSortedMap() - readonly #clone: (v: unknown) => unknown - - constructor(opts: InMemoryKVDriverOptions = {}) { - super() - this.#clone = opts.clone === false ? identity : cloneValue - } - - get size(): number { - return this.#data.size - } - - async getMany(keys: string[]): Promise { - const out = new Array(keys.length) - for (let i = 0; i < keys.length; i++) out[i] = this.#clone(this.#data.get(keys[i]!)) - return out - } - - async setMany(entries: [string, unknown][]): Promise { - const clone = this.#clone - this.#data.setMany(entries.map(([k, v]) => [k, clone(v)])) - } - - async deleteMany(keys: string[]): Promise { - for (const key of keys) this.#data.delete(key) - } - - async setIfAbsent(entries: [string, unknown][]): Promise { - return entries.map(([key, value]) => { - if (this.#data.has(key)) return false - this.#data.set(key, this.#clone(value)) - return true - }) - } - - async batch(ops: BatchOp[]): Promise { - // Applied synchronously within one microtask, so no other operation can observe a partial batch. - for (const op of ops) { - if (op.type === 'set') this.#data.set(op.key, this.#clone(op.value)) - else this.#data.delete(op.key) - } - } - - override async deleteRange(gte: string, lt: string): Promise { - const doomed = [...this.#data.keys({ gte, lt })] - for (const key of doomed) this.#data.delete(key) - } - - async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> { - const { gte, lt, reverse = false, values = false } = opts - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - // The key iterator re-seeks by itself if the map changes while we're suspended. - for (const key of this.#data.keys({ gte, lt, reverse })) { - yield [key, values ? this.#clone(this.#data.get(key)) : undefined] - if (--remaining <= 0) return - } - } -} - -function identity(v: unknown): unknown { - return v -} - -/** Deep copy of a JSON-compatible value (much faster than `structuredClone`). */ -function cloneValue(v: unknown): unknown { - if (v === null || typeof v !== 'object') return v - if (Array.isArray(v)) { - const out = new Array(v.length) - for (let i = 0; i < v.length; i++) out[i] = cloneValue(v[i]) - return out - } - const out: Record = {} - for (const k in v) out[k] = cloneValue((v as Record)[k]) - return out -} diff --git a/src/kv/node.ts b/src/kv/node.ts deleted file mode 100644 index dea3b6d..0000000 --- a/src/kv/node.ts +++ /dev/null @@ -1,402 +0,0 @@ -import { DEFAULT_SEPARATOR, escapeSegment, succ, toSegment, unescapeSegment, validateSeparator } from '../core/path.js' -import type { TrieState } from '../core/trie.js' -import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueOf } from '../types/schema.js' -import { scanChildren } from './children.js' -import type { BatchOp, KVDriver } from './driver.js' -import { BoundIndexes, IndexRegistry, isIndexRange, type IndexDeclarations, type IndexDefs, type IndexQuery } from './indexes.js' - -export interface KVStorageOptions { - /** Single printable ASCII character other than `%` and `~`. Default `/`. */ - separator?: string - /** Path pattern → index definitions over that node's direct children. */ - indexes?: IndexDeclarations - /** - * Delete stale index entries found by `findBy` (read-repair). Default `true`. - * See DESIGN.md §12 (M4) for the race with concurrent non-atomic writers. - */ - readRepair?: boolean -} - -export interface KeysOptions { - limit?: number - /** Start after this child key (exclusive), in the direction of travel. */ - after?: string | number - reverse?: boolean -} - -export interface TreeOptions { - /** Levels below this node to include; `1` = direct children only. */ - depth: number -} - -export interface FindByOptions { - limit?: number - reverse?: boolean -} - -/** @internal */ -export interface KVContext { - readonly driver: KVDriver - readonly sep: string - readonly registry: IndexRegistry - readonly readRepair: boolean -} - -interface Slot { - key: string - esc: string - value: unknown -} - -const FIND_CHUNK = 128 -const DELETE_CHUNK = 1000 - -/** - * A store over its direct children. `child(key)` re-roots to that key (sync, no I/O). - * Methods take an optional leading key: without it they act on this node's own value, - * with it on that child's. The choice is made by argument count; a passed-but-`undefined` - * key throws. - */ -export class KVNode { - /** Unescaped path segments. */ - readonly path: readonly string[] - /** This node's own key (escaped). `''` for the root. */ - readonly selfKey: string - /** Prefix of every descendant key. `''` for the root. */ - readonly prefix: string - /** Exclusive upper bound of the subtree range; `undefined` for the root. */ - readonly end: string | undefined - - readonly #ctx: KVContext - readonly #esc: string - readonly #state: TrieState - readonly #selfIdx: BoundIndexes | undefined - readonly #childIdx: BoundIndexes | undefined - - /** @internal Use `KVStorage` and `child()`. */ - constructor(ctx: KVContext, path: readonly string[], selfKey: string, esc: string, state: TrieState, selfIdx: BoundIndexes | undefined) { - this.#ctx = ctx - this.path = path - this.selfKey = selfKey - this.#esc = esc - const root = path.length === 0 - this.prefix = root ? '' : selfKey + ctx.sep - this.end = root ? undefined : selfKey + succ(ctx.sep) - this.#state = state - this.#selfIdx = selfIdx - this.#childIdx = ctx.registry.bind(state, this.prefix) - } - - get driver(): KVDriver { - return this.#ctx.driver - } - - get separator(): string { - return this.#ctx.sep - } - - get isRoot(): boolean { - return this.path.length === 0 - } - - child>(key: K): KVNode> { - const seg = toSegment(key) - const esc = escapeSegment(seg, this.#ctx.sep) - return new KVNode(this.#ctx, [...this.path, seg], this.prefix + esc, esc, this.#ctx.registry.trie.step(this.#state, seg), this.#childIdx) - } - - /** Re-types this node. No runtime effect. */ - as(): KVNode { - return this as unknown as KVNode - } - - get(): Promise | undefined> - get>(key: K): Promise> | undefined> - async get(...args: unknown[]): Promise { - const t = this.#resolve(args, 0) - return (await this.#ctx.driver.getMany([t.key]))[0] - } - - async getMany>(keys: readonly K[]): Promise<(ValueOf> | undefined)[]> { - if (keys.length === 0) return [] - return (await this.#ctx.driver.getMany(keys.map((k) => this.#childKey(k)))) as (ValueOf> | undefined)[] - } - - set(value: ValueOf): Promise - set>(key: K, value: ValueOf>): Promise - async set(...args: unknown[]): Promise { - const t = this.#resolve(args, 1) - const value = args[args.length - 1] - if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') - return this.#write(t.idx, [{ key: t.key, esc: t.esc, value }]) - } - - async setMany(entries: readonly EntryOf[]): Promise { - if (entries.length === 0) return - const slots = entries.map(([k, value]): Slot => { - if (value === undefined) throw new TypeError('Cannot set undefined; use delete()') - const esc = escapeSegment(toSegment(k), this.#ctx.sep) - return { key: this.prefix + esc, esc, value } - }) - return this.#write(this.#childIdx, slots) - } - - /** Deletes the own value (or a child's own value) only, not descendants. */ - delete(): Promise - delete>(key: K): Promise - async delete(...args: unknown[]): Promise { - const t = this.#resolve(args, 0) - return this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }]) - } - - async deleteMany(keys: readonly KeyOf[]): Promise { - if (keys.length === 0) return - const slots = keys.map((k): Slot => { - const esc = escapeSegment(toSegment(k), this.#ctx.sep) - return { key: this.prefix + esc, esc, value: undefined } - }) - return this.#write(this.#childIdx, slots) - } - - /** Deletes the own value and all descendants (or a child's). On the root, deletes everything. */ - deleteTree(): Promise - deleteTree>(key: K): Promise - async deleteTree(...args: unknown[]): Promise { - if (args.length === 0 && this.isRoot) return this.#deleteRange('', undefined) - const t = this.#resolve(args, 0) - await this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }]) - await this.#deleteRange(t.key + this.#ctx.sep, t.key + succ(this.#ctx.sep)) - } - - /** - * Direct child keys (unescaped), including children that only have descendants. - * Order: see `scanChildren` — lexicographic by escaped segment followed by the separator. - */ - async *keys(opts: KeysOptions = {}): AsyncGenerator> { - const after = opts.after === undefined ? undefined : toSegment(opts.after) - for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, { - after, - reverse: opts.reverse, - limit: opts.limit, - })) { - yield unescapeSegment(hit.esc) as KeyString - } - } - - /** Direct children that have an own value, as `[key, value]`. */ - async *entries(opts: KeysOptions = {}): AsyncGenerator> { - const after = opts.after === undefined ? undefined : toSegment(opts.after) - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, { - after, - reverse: opts.reverse, - values: true, - })) { - if (!hit.hasOwn) continue - yield [unescapeSegment(hit.esc), hit.value] as StringEntryOf - if (--remaining <= 0) return - } - } - - /** - * Assembles descendants up to `depth` levels into a plain object: leaves become values, - * nodes with children become objects. A node with both an own value and children exposes the - * own value under the `''` property (empty keys are invalid, so it can't collide). - * Reserved (`~…`) subtrees are skipped. This node's own value is not included. - */ - async tree>(opts: TreeOptions): Promise { - const { depth } = opts - if (!(depth >= 1)) throw new RangeError('tree() depth must be >= 1') - const { driver, sep } = this.#ctx - type TNode = { own?: unknown; has: boolean; kids?: Map } - const root: TNode = { has: false } - const prefix = this.prefix - let it = driver.scan({ gte: prefix, lt: this.end, values: true })[Symbol.asyncIterator]() - const reseek = async (gte: string) => { - await it.return?.() - it = driver.scan({ gte, lt: this.end, values: true })[Symbol.asyncIterator]() - } - try { - outer: for (;;) { - const r = await it.next() - if (r.done) break - const [key, value] = r.value - const segs = key.slice(prefix.length).split(sep) - let base = prefix - for (let i = 0; i < segs.length && i < depth; i++) { - if (segs[i]!.startsWith('~')) { - await reseek(base + succ('~')) - continue outer - } - base += segs[i]! + sep - } - if (segs.length > depth) { - await reseek(prefix + segs.slice(0, depth).join(sep) + succ(sep)) - continue - } - let node = root - for (const s of segs) { - const kids = (node.kids ??= new Map()) - const seg = unescapeSegment(s) - let next = kids.get(seg) - if (!next) kids.set(seg, (next = { has: false })) - node = next - } - node.own = value - node.has = true - } - } finally { - await it.return?.() - } - const build = (n: TNode): unknown => { - if (!n.kids) return n.own - const obj: Record = {} - if (n.has) obj[''] = n.own - for (const [k, v] of n.kids) obj[k] = build(v) - return obj - } - return (root.kids ? build(root) : {}) as T - } - - /** - * Looks up direct children by an index declared for this node, by exact value or range. - * Each hit's value is re-read and re-checked; stale entries are skipped (and deleted when - * `readRepair` is on). Multi-entry hits are de-duplicated. - */ - async *findBy(index: string, query: IndexQuery, opts: FindByOptions = {}): AsyncGenerator> { - const idx = this.#childIdx - if (!idx || !idx.has(index)) throw new TypeError(`No index "${index}" on node "${this.selfKey}"`) - const { driver, readRepair } = this.#ctx - const { gte, lt } = idx.range(index, query) - let remaining = opts.limit ?? Infinity - if (remaining <= 0) return - const seen = isIndexRange(query) ? new Set() : undefined - let chunk: string[] = [] - const it = driver.scan({ gte, lt, reverse: opts.reverse })[Symbol.asyncIterator]() - - // Processes one chunk of entry keys; returns false when the limit is reached. - const flush = async function* (this: KVNode): AsyncGenerator, boolean> { - const entries = chunk - chunk = [] - const escs = entries.map((e) => idx.childOf(index, e)) - const values = await driver.getMany(escs.map((e) => this.prefix + e)) - const stale: string[] = [] - try { - for (let i = 0; i < entries.length; i++) { - const esc = escs[i]! - if (!idx.matches(index, esc, values[i], entries[i]!)) { - stale.push(entries[i]!) - continue - } - if (seen) { - if (seen.has(esc)) continue - seen.add(esc) - } - yield [unescapeSegment(esc), values[i]] as StringEntryOf - if (--remaining <= 0) return false - } - return true - } finally { - if (readRepair && stale.length) await driver.deleteMany(stale) - } - } - - try { - for (;;) { - const r = await it.next() - if (r.done) break - chunk.push(r.value[0]) - // Small first chunk so `findOne`-style lookups cost one probe. - if (chunk.length >= (remaining < FIND_CHUNK ? remaining : FIND_CHUNK)) { - if (!(yield* flush.call(this))) return - } - } - if (chunk.length) yield* flush.call(this) - } finally { - await it.return?.() - } - } - - // ---------------------------------------------------------------- internals - - #childKey(key: unknown): string { - return this.prefix + escapeSegment(toSegment(key), this.#ctx.sep) - } - - /** Self vs child by argument count; `selfArity` is the self form's argument count. */ - #resolve(args: readonly unknown[], selfArity: number): { key: string; esc: string; idx: BoundIndexes | undefined } { - if (args.length <= selfArity) { - if (this.isRoot) throw new TypeError('The root node has no own value') - return { key: this.selfKey, esc: this.#esc, idx: this.#selfIdx } - } - const esc = escapeSegment(toSegment(args[0]), this.#ctx.sep) - return { key: this.prefix + esc, esc, idx: this.#childIdx } - } - - /** Writes slots (`value: undefined` = delete), maintaining indexes when present. */ - async #write(idx: BoundIndexes | undefined, slots: Slot[]): Promise { - const { driver } = this.#ctx - if (!idx) { - const sets: [string, unknown][] = [] - const dels: string[] = [] - for (const s of slots) { - if (s.value === undefined) dels.push(s.key) - else sets.push([s.key, s.value]) - } - if (sets.length) await driver.setMany(sets) - if (dels.length) await driver.deleteMany(dels) - return - } - // Last write per key wins. - const byKey = new Map() - for (const s of slots) byKey.set(s.key, s) - const unique = [...byKey.values()] - const old = await driver.getMany(unique.map((s) => s.key)) - const adds: string[] = [] - const removes: string[] = [] - for (let i = 0; i < unique.length; i++) { - const s = unique[i]! - const before = idx.entriesFor(s.esc, old[i]) - const after = idx.entriesFor(s.esc, s.value) - for (const e of after) if (!before.has(e)) adds.push(e) - for (const e of before) if (!after.has(e)) removes.push(e) - } - if (driver.batch) { - const ops: BatchOp[] = [] - for (const k of adds) ops.push({ type: 'set', key: k, value: true }) - for (const s of unique) ops.push(s.value === undefined ? { type: 'delete', key: s.key } : { type: 'set', key: s.key, value: s.value }) - for (const k of removes) ops.push({ type: 'delete', key: k }) - return driver.batch(ops) - } - // Non-atomic: adds first, then values, then removes. Entries are hints; findBy re-checks. - if (adds.length) await driver.setMany(adds.map((k) => [k, true])) - const sets = unique.filter((s) => s.value !== undefined) - const dels = unique.filter((s) => s.value === undefined) - if (sets.length) await driver.setMany(sets.map((s) => [s.key, s.value])) - if (dels.length) await driver.deleteMany(dels.map((s) => s.key)) - if (removes.length) await driver.deleteMany(removes) - } - - async #deleteRange(gte: string, lt: string | undefined): Promise { - const { driver } = this.#ctx - if (lt !== undefined && driver.deleteRange) return driver.deleteRange(gte, lt) - for (;;) { - const keys: string[] = [] - for await (const [k] of driver.scan({ gte, lt, limit: DELETE_CHUNK })) keys.push(k) - if (keys.length === 0) return - await driver.deleteMany(keys) - if (keys.length < DELETE_CHUNK) return - } - } -} - -/** The root `KVNode` over a driver. */ -export class KVStorage extends KVNode { - constructor(driver: KVDriver, opts: KVStorageOptions = {}) { - const sep = opts.separator ?? DEFAULT_SEPARATOR - validateSeparator(sep) - const registry = new IndexRegistry(opts.indexes ?? {}, sep) - super({ driver, sep, registry, readRepair: opts.readRepair ?? true }, [], '', '', registry.trie.rootState, undefined) - } -} diff --git a/src/kv/sorted.ts b/src/kv/sorted.ts deleted file mode 100644 index 53d72d7..0000000 --- a/src/kv/sorted.ts +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Ordered string-keyed map used by the in-memory driver. Kept behind an interface so the - * implementation can be swapped (e.g. for a B-tree) without touching the driver. - */ -export interface SortedMap { - readonly size: number - get(key: string): V | undefined - has(key: string): boolean - set(key: string, value: V): void - /** Inserts many entries; may be faster than repeated `set`. */ - setMany(entries: Iterable<[string, V]>): void - delete(key: string): boolean - /** - * Keys in `[gte, lt)` in order (descending when `reverse`). The iterator stays valid across - * mutations between steps: it resumes after (before) the last key it returned. - */ - keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator -} - -/** - * Sorted chunks of at most `maxChunk` keys (split in half on overflow) plus a `Map` for values. - * Inserts and deletes cost O(log n + maxChunk); iteration is O(1) per step while the map - * is unchanged and re-seeks in O(log n) after a mutation. - */ -export class ChunkedSortedMap implements SortedMap { - constructor(readonly maxChunk = 1024) {} - - #chunks: string[][] = [[]] - readonly #values = new Map() - #version = 0 - - get size(): number { - return this.#values.size - } - - get(key: string): V | undefined { - return this.#values.get(key) - } - - has(key: string): boolean { - return this.#values.has(key) - } - - set(key: string, value: V): void { - if (!this.#values.has(key)) this.#insert(key) - this.#values.set(key, value) - } - - setMany(entries: Iterable<[string, V]>): void { - const fresh: string[] = [] - for (const [key, value] of entries) { - if (!this.#values.has(key)) fresh.push(key) - this.#values.set(key, value) - } - if (fresh.length === 0) return - if (fresh.length < 64 || fresh.length * 8 < this.#values.size) { - for (const key of fresh) this.#insert(key) - return - } - // Bulk path: merge and re-chunk, O(n + k log k). - fresh.sort(cmp) - const old = this.#chunks.flat() - const merged = new Array(old.length + fresh.length) - let i = 0 - let j = 0 - let k = 0 - while (i < old.length && j < fresh.length) merged[k++] = old[i]! < fresh[j]! ? old[i++]! : fresh[j++]! - while (i < old.length) merged[k++] = old[i++]! - while (j < fresh.length) merged[k++] = fresh[j++]! - const half = Math.max(1, this.maxChunk >>> 1) - this.#chunks = [] - for (let s = 0; s < merged.length; s += half) this.#chunks.push(merged.slice(s, s + half)) - this.#version++ - } - - delete(key: string): boolean { - if (!this.#values.delete(key)) return false - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - chunk.splice(lowerBound(chunk, key), 1) - if (chunk.length === 0 && this.#chunks.length > 1) this.#chunks.splice(c, 1) - this.#version++ - return true - } - - *keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator { - const { gte, lt, reverse = false } = range - let version = this.#version - let c: number - let i: number - // Position of the first candidate. - if (reverse) { - ;[c, i] = lt === undefined ? this.#end() : this.#seek(lt, false) - ;[c, i] = this.#prev(c, i) - } else { - ;[c, i] = gte === undefined ? [0, 0] : this.#seek(gte, false) - ;[c, i] = this.#normalize(c, i) - } - for (;;) { - const key = this.#chunks[c]?.[i] - if (key === undefined || c < 0) return - if (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt) return - yield key - if (this.#version !== version) { - version = this.#version - ;[c, i] = this.#seek(key, !reverse) - if (reverse) [c, i] = this.#prev(c, i) - else [c, i] = this.#normalize(c, i) - } else if (reverse) [c, i] = this.#prev(c, i) - else [c, i] = this.#normalize(c, i + 1) - } - } - - #insert(key: string): void { - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - chunk.splice(lowerBound(chunk, key), 0, key) - if (chunk.length > this.maxChunk) this.#chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1)) - this.#version++ - } - - /** Index of the last chunk whose first key is <= key (0 if none). */ - #chunkFor(key: string): number { - const chunks = this.#chunks - let lo = 0 - let hi = chunks.length - 1 - while (lo < hi) { - const mid = (lo + hi + 1) >>> 1 - if (chunks[mid]![0]! <= key) lo = mid - else hi = mid - 1 - } - return lo - } - - /** Position of the first key `>= key` (`> key` when exclusive); may point one past a chunk's end. */ - #seek(key: string, exclusive: boolean): [number, number] { - const c = this.#chunkFor(key) - const chunk = this.#chunks[c]! - let i = lowerBound(chunk, key) - if (exclusive && chunk[i] === key) i++ - return [c, i] - } - - #end(): [number, number] { - const c = this.#chunks.length - 1 - return [c, this.#chunks[c]!.length] - } - - /** Moves a position that is past its chunk's end to the start of the next chunk. */ - #normalize(c: number, i: number): [number, number] { - while (c < this.#chunks.length && i >= this.#chunks[c]!.length) { - c++ - i = 0 - } - return [c, i] - } - - #prev(c: number, i: number): [number, number] { - i-- - while (i < 0 && c > 0) { - c-- - i = this.#chunks[c]!.length - 1 - } - return i < 0 ? [-1, -1] : [c, i] - } -} - -function lowerBound(arr: string[], key: string): number { - let lo = 0 - let hi = arr.length - while (lo < hi) { - const mid = (lo + hi) >>> 1 - if (arr[mid]! < key) lo = mid + 1 - else hi = mid - } - return lo -} - -function cmp(a: string, b: string): number { - return a < b ? -1 : a > b ? 1 : 0 -} diff --git a/src/testing/index.ts b/src/testing/index.ts deleted file mode 100644 index 9629877..0000000 --- a/src/testing/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './kv-conformance.js' diff --git a/src/testing/kv-conformance.ts b/src/testing/kv-conformance.ts deleted file mode 100644 index 956c77a..0000000 --- a/src/testing/kv-conformance.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { beforeEach, describe, expect, it } from 'vitest' -import type { KVDriver, ScanOptions } from '../kv/driver.js' - -/** - * Reusable conformance suite for {@link KVDriver} implementations. Call it from a Vitest test file: - * - * ```ts - * runKVDriverConformance('MyDriver', () => new MyDriver()) - * ``` - * - * `makeDriver` must return an empty driver each time. - */ -export function runKVDriverConformance(name: string, makeDriver: () => KVDriver | Promise): void { - describe(`KVDriver conformance: ${name}`, () => { - let d: KVDriver - - beforeEach(async () => { - d = await makeDriver() - }) - - async function collect(opts: ScanOptions): Promise<[string, unknown][]> { - const out: [string, unknown][] = [] - for await (const e of d.scan(opts)) out.push(e) - return out - } - - const keysOf = async (opts: ScanOptions) => (await collect(opts)).map(([k]) => k) - - async function seed(): Promise { - // Deliberately inserted out of order. - await d.setMany([ - ['b', 2], - ['a/2', { n: 'a2' }], - ['c', [3]], - ['a', 1], - ['a/1', 'a1'], - ['a0', null], - ['10', 10], - ['9', 9], - ]) - } - - it('round-trips JSON values with getMany in key order, undefined for missing', async () => { - await seed() - expect(await d.getMany(['c', 'missing', 'a/2', 'a0', 'a'])).toEqual([[3], undefined, { n: 'a2' }, null, 1]) - expect(await d.getMany([])).toEqual([]) - }) - - it('overwrites on setMany and deletes with deleteMany', async () => { - await seed() - await d.setMany([['a', 'one']]) - await d.deleteMany(['b', 'missing']) - expect(await d.getMany(['a', 'b'])).toEqual(['one', undefined]) - }) - - it('scans all keys in lexicographic order', async () => { - await seed() - expect(await keysOf({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c']) - }) - - it('honours gte (inclusive) and lt (exclusive)', async () => { - await seed() - expect(await keysOf({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2']) - expect(await keysOf({ gte: 'a', lt: 'b' })).toEqual(['a', 'a/1', 'a/2', 'a0']) - expect(await keysOf({ gte: 'a0' })).toEqual(['a0', 'b', 'c']) - expect(await keysOf({ lt: 'a' })).toEqual(['10', '9']) - expect(await keysOf({ gte: 'zz' })).toEqual([]) - expect(await keysOf({ gte: 'b', lt: 'b' })).toEqual([]) - }) - - it('scans in reverse within bounds', async () => { - await seed() - expect(await keysOf({ reverse: true })).toEqual(['c', 'b', 'a0', 'a/2', 'a/1', 'a', '9', '10']) - expect(await keysOf({ reverse: true, gte: 'a', lt: 'a0' })).toEqual(['a/2', 'a/1', 'a']) - }) - - it('honours limit, forwards and in reverse', async () => { - await seed() - expect(await keysOf({ limit: 2 })).toEqual(['10', '9']) - expect(await keysOf({ limit: 2, reverse: true, lt: 'b' })).toEqual(['a0', 'a/2']) - expect(await keysOf({ limit: 0 })).toEqual([]) - }) - - it('yields values only when values: true', async () => { - await seed() - expect(await collect({ gte: 'a/', lt: 'a0' })).toEqual([ - ['a/1', undefined], - ['a/2', undefined], - ]) - expect(await collect({ gte: 'a/', lt: 'a0', values: true })).toEqual([ - ['a/1', 'a1'], - ['a/2', { n: 'a2' }], - ]) - }) - - it('tolerates writes while a scan is suspended', async () => { - await d.setMany([ - ['k1', 1], - ['k2', 2], - ['k3', 3], - ]) - const seen: string[] = [] - for await (const [key] of d.scan({})) { - seen.push(key) - if (key === 'k1') await d.deleteMany(['k1']) - } - expect(seen).toEqual(['k1', 'k2', 'k3']) - }) - - it('does not let callers mutate stored values through returned references', async () => { - await d.setMany([['o', { n: 1 }]]) - const [v] = (await d.getMany(['o'])) as [{ n: number }] - v.n = 2 - expect(await d.getMany(['o'])).toEqual([{ n: 1 }]) - }) - - it('stores unicode keys and orders them by code unit', async () => { - await d.setMany([ - ['é', 1], - ['z', 2], - ['日本', 3], - ]) - expect(await keysOf({})).toEqual(['z', 'é', '日本']) - }) - - describe('optional: setIfAbsent', () => { - it('writes only absent keys and reports which were written', async (ctx) => { - if (!d.setIfAbsent) return ctx.skip() - await d.setMany([['x', 1]]) - expect( - await d.setIfAbsent([ - ['x', 2], - ['y', 3], - ]), - ).toEqual([false, true]) - expect(await d.getMany(['x', 'y'])).toEqual([1, 3]) - }) - - it('only the first of concurrent claims wins', async (ctx) => { - if (!d.setIfAbsent) return ctx.skip() - const results = await Promise.all([1, 2, 3, 4].map((n) => d.setIfAbsent!([['claim', n]]))) - expect(results.flat().filter(Boolean)).toHaveLength(1) - }) - }) - - describe('optional: batch', () => { - it('applies sets and deletes in order', async (ctx) => { - if (!d.batch) return ctx.skip() - await d.setMany([['gone', 1]]) - await d.batch([ - { type: 'set', key: 'p', value: 1 }, - { type: 'delete', key: 'gone' }, - { type: 'set', key: 'q', value: { a: [1] } }, - { type: 'set', key: 'p', value: 2 }, - ]) - expect(await d.getMany(['p', 'q', 'gone'])).toEqual([2, { a: [1] }, undefined]) - }) - }) - - describe('optional: deleteRange', () => { - it('deletes exactly [gte, lt)', async (ctx) => { - if (!d.deleteRange) return ctx.skip() - await seed() - await d.deleteRange('a/', 'a0') - expect(await keysOf({})).toEqual(['10', '9', 'a', 'a0', 'b', 'c']) - await d.deleteRange('a', 'b') - expect(await keysOf({})).toEqual(['10', '9', 'b', 'c']) - }) - }) - }) -} diff --git a/src/utils/document-path.ts b/src/utils/document-path.ts new file mode 100644 index 0000000..5d2fb2a --- /dev/null +++ b/src/utils/document-path.ts @@ -0,0 +1,32 @@ +import { Json } from './json.js' + +export type Doc = Record + +/** Dot-path access into plain JSON documents (`'address.city'`, `'tags.0'`). */ +export class DocumentPath { + static get(doc: unknown, path: string): unknown { + let value = doc + for (const segment of path.split('.')) { + if (Array.isArray(value) || Json.isObject(value)) value = (value as Record)[segment] + else return undefined + } + return value + } + + /** Sets a path in place, creating intermediate objects. */ + static set(doc: Doc, path: string, value: unknown): void { + const segments = path.split('.') + let target = doc as Record + for (const segment of segments.slice(0, -1)) { + if (typeof target[segment] !== 'object' || target[segment] === null) target[segment] = {} + target = target[segment] as Record + } + target[segments.at(-1)!] = value + } + + static unset(doc: Doc, path: string): void { + const segments = path.split('.') + const parent = segments.length === 1 ? doc : DocumentPath.get(doc, segments.slice(0, -1).join('.')) + if (Json.isObject(parent)) delete parent[segments.at(-1)!] + } +} diff --git a/src/utils/document-query.ts b/src/utils/document-query.ts new file mode 100644 index 0000000..f030539 --- /dev/null +++ b/src/utils/document-query.ts @@ -0,0 +1,106 @@ +import { DocumentPath, type Doc } from './document-path.js' +import { Json } from './json.js' + +export type Comparison = { $eq?: unknown; $gt?: unknown; $gte?: unknown; $lt?: unknown; $lte?: unknown; $in?: unknown[] } + +/** Field (dot path) → value (equality) or comparison operators. All fields must match. */ +export type Filter = Record + +export interface FindOptions { + /** Field → 1 (ascending) or -1 (descending). */ + sort?: Record + limit?: number + skip?: number +} + +export interface FindResult { + key: string + data: T +} + +const OPERATORS = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in']) + +/** + * Filter evaluation, sorting and paging for document stores that don't query natively. + * + * - `{ field: value }` matches equal values; `null` also matches a missing field. + * - Comparisons only match values of the same type (`$gt: 5` never matches a string). + * - On an array field, a condition matches if the array itself or any one element satisfies it. + */ +export class DocumentQuery { + constructor( + readonly filter: Filter = {}, + readonly options: FindOptions = {}, + ) {} + + matches(doc: Doc): boolean { + return Object.entries(this.filter).every(([path, condition]) => DocumentQuery.matchesValue(DocumentPath.get(doc, path), condition)) + } + + /** Whether one field value satisfies a condition. */ + static matchesValue(value: unknown, condition: unknown): boolean { + const candidates = Array.isArray(value) ? [value, ...value] : [value] + if (!DocumentQuery.isComparison(condition)) return candidates.some((c) => DocumentQuery.equal(c, condition)) + const ops = Object.entries(condition) + return candidates.some((c) => ops.every(([op, arg]) => DocumentQuery.compare(op, c, arg))) + } + + /** Applies filter, sort, skip and limit to a stream of candidates. */ + async *run(source: AsyncIterable): AsyncGenerator { + const { sort, skip = 0, limit = Infinity } = this.options + if (limit <= 0) return + if (sort && Object.keys(sort).length) { + const all: FindResult[] = [] + for await (const result of source) if (this.matches(result.data)) all.push(result) + all.sort((a, b) => this.compareResults(a, b)) + yield* all.slice(skip, skip + limit) + return + } + let skipped = 0 + let emitted = 0 + for await (const result of source) { + if (!this.matches(result.data)) continue + if (skipped++ < skip) continue + yield result + if (++emitted >= limit) return + } + } + + private compareResults(a: FindResult, b: FindResult): number { + for (const [path, direction] of Object.entries(this.options.sort!)) { + const order = DocumentQuery.order(DocumentPath.get(a.data, path), DocumentPath.get(b.data, path)) + if (order !== 0) return order * direction + } + return a.key < b.key ? -1 : a.key > b.key ? 1 : 0 + } + + private static isComparison(condition: unknown): condition is Comparison { + if (!Json.isObject(condition)) return false + const keys = Object.keys(condition) + if (keys.length === 0 || !keys.every((k) => k.startsWith('$'))) return false + for (const k of keys) if (!OPERATORS.has(k)) throw new TypeError(`Unsupported operator ${k}`) + return true + } + + private static equal(value: unknown, expected: unknown): boolean { + return expected === null ? value === null || value === undefined : Json.equal(value, expected) + } + + private static compare(op: string, value: unknown, arg: unknown): boolean { + if (op === '$eq') return DocumentQuery.equal(value, arg) + if (op === '$in') return (arg as unknown[]).some((x) => DocumentQuery.equal(value, x)) + if (typeof value !== typeof arg || !['number', 'string', 'boolean'].includes(typeof value)) return false + const order = DocumentQuery.order(value, arg) + return op === '$gt' ? order > 0 : op === '$gte' ? order >= 0 : op === '$lt' ? order < 0 : order <= 0 + } + + /** Total order: missing/null < booleans < numbers < strings < anything else. */ + static order(a: unknown, b: unknown): number { + const rank = (v: unknown) => (v === undefined || v === null ? 0 : (({ boolean: 1, number: 2, string: 3 } as Record)[typeof v] ?? 4)) + const ra = rank(a) + const rb = rank(b) + if (ra !== rb) return ra - rb + if (ra === 4) [a, b] = [JSON.stringify(a), JSON.stringify(b)] + return a === b ? 0 : (a as number) < (b as number) ? -1 : 1 + } +} diff --git a/src/utils/document-update.ts b/src/utils/document-update.ts new file mode 100644 index 0000000..590fbc0 --- /dev/null +++ b/src/utils/document-update.ts @@ -0,0 +1,75 @@ +import { DocumentPath, type Doc } from './document-path.js' +import { DocumentQuery } from './document-query.js' +import { Json } from './json.js' + +/** Update operators. Paths may use dots. `$push`/`$addToSet` accept `{ $each: [...] }`. */ +export interface UpdateSpec { + $set?: Record + $unset?: Record + $inc?: Record + $push?: Record + $pull?: Record + $addToSet?: Record +} + +/** + * A resolved change to one document. Operators are resolved to final values (`$inc` becomes + * `set: { n: 6 }`), so replaying changes is deterministic. + */ +export type DocChange = { op: 'set'; doc: Doc } | { op: 'patch'; set: Record; unset: string[] } | { op: 'delete' } + +export class DocumentUpdate { + /** Resolves update operators against the current state into a patch. */ + static resolve(state: Doc, spec: UpdateSpec): DocChange { + const set: Record = {} + const unset: string[] = [] + const array = (path: string) => { + const current = DocumentPath.get(state, path) ?? [] + if (!Array.isArray(current)) throw new TypeError(`"${path}" is not an array`) + return Json.clone(current) + } + const items = (value: unknown) => (Json.isObject(value) && Array.isArray(value.$each) ? value.$each : [value]) + + for (const [op, fields] of Object.entries(spec)) { + for (const [path, value] of Object.entries(fields as Record)) { + if (path in set || unset.includes(path)) throw new TypeError(`"${path}" is updated twice`) + switch (op) { + case '$set': + set[path] = Json.clone(value) + break + case '$unset': + unset.push(path) + break + case '$inc': { + const current = DocumentPath.get(state, path) ?? 0 + if (typeof current !== 'number' || typeof value !== 'number') throw new TypeError(`$inc needs numbers at "${path}"`) + set[path] = current + value + break + } + case '$push': + set[path] = [...array(path), ...Json.clone(items(value))] + break + case '$addToSet': + set[path] = items(value).reduce((acc, item) => (acc.some((x) => Json.equal(x, item)) ? acc : [...acc, Json.clone(item)]), array(path)) + break + case '$pull': + set[path] = array(path).filter((item) => !DocumentQuery.matchesValue(item, value)) + break + default: + throw new TypeError(`Unsupported update operator ${op}`) + } + } + } + return { op: 'patch', set, unset } + } + + /** Applies a change to a state (`undefined` = no document). */ + static apply(state: Doc | undefined, change: DocChange): Doc | undefined { + if (change.op === 'delete') return undefined + if (change.op === 'set') return Json.clone(change.doc) + const next = Json.clone(state ?? {}) + for (const [path, value] of Object.entries(change.set)) DocumentPath.set(next, path, Json.clone(value)) + for (const path of change.unset) DocumentPath.unset(next, path) + return next + } +} diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 0000000..8b95d83 --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,19 @@ +/** An optimistic-concurrency check failed (stale `expectedVersion` or a lost commit race). */ +export class ConcurrencyError extends Error { + constructor( + readonly stream: string, + readonly expectedVersion: number | 'any', + readonly actualVersion: number, + ) { + super(`Concurrency conflict on "${stream}": expected version ${expectedVersion}, actual ${actualVersion}`) + this.name = 'ConcurrencyError' + } +} + +/** A key argument was passed but is `undefined`/`null`, empty, or not a string/number. */ +export class InvalidKeyError extends TypeError { + constructor(message: string) { + super(message) + this.name = 'InvalidKeyError' + } +} diff --git a/src/utils/json.ts b/src/utils/json.ts new file mode 100644 index 0000000..8c666b7 --- /dev/null +++ b/src/utils/json.ts @@ -0,0 +1,24 @@ +/** Helpers for JSON-compatible values (all stored values are JSON by contract). */ +export class Json { + /** Deep copy; much cheaper than `structuredClone` for plain JSON. */ + static clone(value: T): T { + if (value === null || typeof value !== 'object') return value + if (Array.isArray(value)) return value.map((v) => Json.clone(v)) as T + const out: Record = {} + for (const k in value) out[k] = Json.clone((value as Record)[k]) + return out as T + } + + static equal(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false + if (Array.isArray(a) !== Array.isArray(b)) return false + const ka = Object.keys(a) + const kb = Object.keys(b) + return ka.length === kb.length && ka.every((k) => Json.equal((a as Record)[k], (b as Record)[k])) + } + + static isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) + } +} diff --git a/src/utils/keys.ts b/src/utils/keys.ts new file mode 100644 index 0000000..e7a9149 --- /dev/null +++ b/src/utils/keys.ts @@ -0,0 +1,43 @@ +import { InvalidKeyError } from './errors.js' + +/** + * Key encoding. A node's key is its escaped path segments joined by `/`. + * + * Every character that sorts at or below `/` (and `~` itself) is escaped as `~XX` (hex). Escaped + * segments therefore only contain characters above `/`, so all keys of one child sort together: + * `p/a`, `p/a/...`, then siblings like `p/a~2D1` (`a-1`). That keeps child listing a simple skip-scan. + * + * Segments starting with `~` + a lowercase letter are reserved for internal use (e.g. `~e` for + * event commits); escaping only ever produces `~` + uppercase hex, so user keys can't collide. + */ +export class Keys { + static readonly separator = '/' + /** The character right after `/`: `prefix + segment + '0'` is the first key past a subtree. */ + static readonly afterSeparator = '0' + + /** Validates a user key and converts it to a segment. */ + static segment(key: unknown): string { + if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`) + if (typeof key === 'number' && Number.isFinite(key)) return String(key) + if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or finite number, got ${String(key)}`) + if (key === '') throw new InvalidKeyError('Key must not be empty') + return key + } + + static escape(segment: string): string { + if (!/[\x00-\x2f~]/.test(segment)) return segment + return segment.replace(/[\x00-\x2f~]/g, (c) => '~' + c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')) + } + + static unescape(segment: string): string { + return segment.includes('~') ? segment.replace(/~([0-9A-F]{2})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16))) : segment + } + + static join(path: readonly string[]): string { + return path.map((s) => Keys.escape(s)).join(Keys.separator) + } + + static isReserved(escapedSegment: string): boolean { + return /^~[a-z]/.test(escapedSegment) + } +} diff --git a/src/types/schema.ts b/src/utils/schema.ts similarity index 87% rename from src/types/schema.ts rename to src/utils/schema.ts index 80abbd2..71f340f 100644 --- a/src/types/schema.ts +++ b/src/utils/schema.ts @@ -15,13 +15,16 @@ export type ChildrenOf = S extends { children: infer C } ? C : {} export type KeyOf = Extract, string | number> export type ChildOf> = ChildrenOf[K] extends infer C extends Spec ? C : never +/** Value of this node (`K = undefined`) or of child `K`. */ +export type ValueAt = [K] extends [KeyOf] ? ValueOf> : ValueOf + /** Child keys as yielded at runtime (always strings). */ export type KeyString = `${KeyOf}` /** `[key, value]` for each child key, as a union of precise pairs. */ export type EntryOf = { [K in KeyOf]: [K, ValueOf>] }[KeyOf] -/** `[key, value]` as yielded at runtime (string keys). Dynamic levels collapse to `[string, V]`. */ +/** `[key, value]` as yielded at runtime (string keys). */ export type StringEntryOf = string extends KeyOf ? [string, ValueOf & string>>] : { [K in KeyOf]: [`${K}`, ValueOf>] }[KeyOf] diff --git a/src/utils/sorted-map.ts b/src/utils/sorted-map.ts new file mode 100644 index 0000000..aeee7e3 --- /dev/null +++ b/src/utils/sorted-map.ts @@ -0,0 +1,103 @@ +/** + * Ordered string-keyed map: sorted chunks of at most `maxChunk` keys (split in half when full) + * plus a `Map` for values. Inserts and deletes cost O(log n + maxChunk). + */ +export class SortedMap { + private chunks: string[][] = [[]] + private readonly values = new Map() + /** Bumped on every insert/delete so iterators know to re-seek. */ + private version = 0 + + constructor(private readonly maxChunk = 1024) {} + + get size(): number { + return this.values.size + } + + get(key: string): V | undefined { + return this.values.get(key) + } + + has(key: string): boolean { + return this.values.has(key) + } + + set(key: string, value: V): void { + if (!this.values.has(key)) { + const c = this.chunkFor(key) + const chunk = this.chunks[c]! + chunk.splice(SortedMap.lowerBound(chunk, key), 0, key) + if (chunk.length > this.maxChunk) this.chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1)) + this.version++ + } + this.values.set(key, value) + } + + delete(key: string): boolean { + if (!this.values.delete(key)) return false + const c = this.chunkFor(key) + const chunk = this.chunks[c]! + chunk.splice(SortedMap.lowerBound(chunk, key), 1) + if (chunk.length === 0 && this.chunks.length > 1) this.chunks.splice(c, 1) + this.version++ + return true + } + + /** + * Keys in `[gte, lt)`, descending when `reverse`. Safe across mutations between steps: + * after a change it re-seeks from the last key returned. + */ + *keys(gte?: string, lt?: string, reverse = false): IterableIterator { + let last: string | undefined + let version = -1 + let c = 0 + let i = 0 + for (;;) { + if (version !== this.version) { + version = this.version + const from = last ?? (reverse ? lt : gte) + if (from === undefined) { + c = reverse ? this.chunks.length - 1 : 0 + i = reverse ? this.chunks[c]!.length - 1 : 0 + } else { + c = this.chunkFor(from) + i = SortedMap.lowerBound(this.chunks[c]!, from) + // Forward: skip `last` itself. Reverse: step to the key before `from`. + if (reverse) i-- + else if (last !== undefined && this.chunks[c]![i] === last) i++ + } + } + // Move across chunk boundaries. + while (!reverse && c < this.chunks.length - 1 && i >= this.chunks[c]!.length) (c++, (i = 0)) + while (reverse && c > 0 && i < 0) i = this.chunks[--c]!.length - 1 + const key = this.chunks[c]?.[i] + if (key === undefined || (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt)) return + last = key + yield key + i += reverse ? -1 : 1 + } + } + + /** Index of the last chunk whose first key is <= key (0 if none). */ + private chunkFor(key: string): number { + let lo = 0 + let hi = this.chunks.length - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >>> 1 + if (this.chunks[mid]![0]! <= key) lo = mid + else hi = mid - 1 + } + return lo + } + + private static lowerBound(arr: string[], key: string): number { + let lo = 0 + let hi = arr.length + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (arr[mid]! < key) lo = mid + 1 + else hi = mid + } + return lo + } +} diff --git a/src/utils/ulid.ts b/src/utils/ulid.ts new file mode 100644 index 0000000..fb42411 --- /dev/null +++ b/src/utils/ulid.ts @@ -0,0 +1,36 @@ +const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32 + +/** + * ULID-like ids: 10 chars of millisecond time + 16 random chars, sortable by creation time. + * Ids from one generator within the same millisecond increment the random part, so they stay + * strictly increasing. + */ +export class Ulid { + private lastTime = -1 + private readonly random = new Uint8Array(16) // one base32 digit (0–31) per byte + + constructor(private readonly now: () => number = Date.now) {} + + generate(): string { + let time = this.now() + if (time <= this.lastTime) { + time = this.lastTime + let i = 15 + while (i >= 0 && this.random[i] === 31) this.random[i--] = 0 + if (i < 0) throw new Error('ULID overflow within one millisecond') + this.random[i]!++ + } else { + crypto.getRandomValues(this.random) + for (let i = 0; i < 16; i++) this.random[i]! &= 31 + this.random[0]! &= 15 // headroom for increments + this.lastTime = time + } + let out = '' + for (let i = 0; i < 10; i++) { + out = ALPHABET[time % 32]! + out + time = Math.floor(time / 32) + } + for (const digit of this.random) out += ALPHABET[digit] + return out + } +} diff --git a/test/core/encoding.test.ts b/test/core/encoding.test.ts deleted file mode 100644 index 409f07f..0000000 --- a/test/core/encoding.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { decodeIndexValue, encodeIndexValue, type IndexValue } from '../../src/core/encoding.js' - -const separators = fc.constantFrom('/', ':', '.', '|', '#', '!') - -const indexValue: fc.Arbitrary = fc.oneof( - fc.constant(null), - fc.boolean(), - fc.double({ noNaN: true }), - fc.integer(), - fc.string({ unit: 'binary' }), - fc.string({ unit: fc.constantFrom('/', ':', '0', '1', 'a', '%', '~', '\0', '!') }), -) - -const rank = (v: IndexValue) => (v === null ? 0 : v === false ? 1 : v === true ? 2 : typeof v === 'number' ? 3 : 4) - -function compare(a: IndexValue, b: IndexValue): number { - const ra = rank(a) - const rb = rank(b) - if (ra !== rb) return ra - rb - if (typeof a === 'number' && typeof b === 'number') return a === b ? 0 : a < b ? -1 : 1 - if (typeof a === 'string' && typeof b === 'string') return a === b ? 0 : a < b ? -1 : 1 - return 0 -} - -const sign = (n: number) => (n < 0 ? -1 : n > 0 ? 1 : 0) -const cmpStr = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0) - -describe('encodeIndexValue', () => { - it('round-trips (property)', () => { - fc.assert( - fc.property(indexValue, separators, (v, sep) => { - const decoded = decodeIndexValue(encodeIndexValue(v, sep), sep) - if (typeof v === 'number' && v === 0) - expect(decoded).toBe(0) // -0 normalised - else expect(decoded).toEqual(v) - }), - ) - }) - - it('emits only characters above the separator (property)', () => { - fc.assert( - fc.property(indexValue, separators, (v, sep) => { - for (const c of encodeIndexValue(v, sep)) expect(c > sep).toBe(true) - }), - ) - }) - - it('preserves order, including when followed by a separator and suffix (property)', () => { - fc.assert( - fc.property(indexValue, indexValue, fc.string(), fc.string(), separators, (a, b, sa, sb, sep) => { - const ea = encodeIndexValue(a, sep) - const eb = encodeIndexValue(b, sep) - const expected = sign(compare(a, b)) - expect(cmpStr(ea, eb)).toBe(expected) - if (expected !== 0) expect(cmpStr(ea + sep + sa, eb + sep + sb)).toBe(expected) - }), - { numRuns: 2000 }, - ) - }) - - it('orders types null < false < true < numbers < strings', () => { - const values: IndexValue[] = ['', 'a', -Infinity, -1, 0, 1e300, Infinity, true, false, null] - const sorted = [...values].sort((a, b) => cmpStr(encodeIndexValue(a), encodeIndexValue(b))) - expect(sorted).toEqual([null, false, true, -Infinity, -1, 0, 1e300, Infinity, '', 'a']) - }) - - it('rejects NaN and non-index values', () => { - expect(() => encodeIndexValue(NaN)).toThrow() - expect(() => encodeIndexValue({} as never)).toThrow() - }) -}) diff --git a/test/core/path.test.ts b/test/core/path.test.ts deleted file mode 100644 index 39279d8..0000000 --- a/test/core/path.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { escapeSegment, joinPath, pathKeys, splitPath, succ, toSegment, unescapeSegment, validateSeparator } from '../../src/core/path.js' -import { InvalidKeyError } from '../../src/errors.js' - -const separators = fc.constantFrom('/', ':', '.', '|', '#', '\\', '!') - -describe('escapeSegment', () => { - it('escapes %, the separator and a leading ~', () => { - expect(escapeSegment('a/b')).toBe('a%2Fb') - expect(escapeSegment('50%')).toBe('50%25') - expect(escapeSegment('~x')).toBe('%7Ex') - expect(escapeSegment('x~')).toBe('x~') - expect(escapeSegment('a:b', ':')).toBe('a%3Ab') - expect(escapeSegment('a/b', ':')).toBe('a/b') - expect(escapeSegment('plain')).toBe('plain') - }) - - it('round-trips (property)', () => { - fc.assert( - fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => { - expect(unescapeSegment(escapeSegment(s, sep))).toBe(s) - }), - ) - }) - - it('never emits a raw separator or a leading ~ (property)', () => { - fc.assert( - fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => { - const e = escapeSegment(s, sep) - expect(e.includes(sep)).toBe(false) - expect(e.startsWith('~')).toBe(false) - }), - ) - }) - - it('joins and splits paths (property)', () => { - fc.assert( - fc.property(fc.array(fc.string({ minLength: 1 }), { maxLength: 6 }), separators, (path, sep) => { - expect(splitPath(joinPath(path, sep), sep)).toEqual(path) - }), - ) - }) -}) - -describe('pathKeys', () => { - it('precomputes selfKey, prefix and subtree end', () => { - expect(pathKeys(['users', '123'], '/')).toMatchObject({ selfKey: 'users/123', prefix: 'users/123/', end: 'users/1230' }) - expect(pathKeys([], '/')).toMatchObject({ selfKey: '', prefix: '', end: undefined }) - expect(pathKeys(['a/b'], '/').selfKey).toBe('a%2Fb') - }) -}) - -describe('misc', () => { - it('succ returns the next char', () => { - expect(succ('/')).toBe('0') - }) - - it('toSegment rejects undefined, null, empty and non-finite', () => { - expect(() => toSegment(undefined)).toThrow(InvalidKeyError) - expect(() => toSegment(null)).toThrow(TypeError) - expect(() => toSegment('')).toThrow(InvalidKeyError) - expect(() => toSegment(NaN)).toThrow(InvalidKeyError) - expect(toSegment(12)).toBe('12') - }) - - it('validates separators', () => { - expect(() => validateSeparator('//')).toThrow() - expect(() => validateSeparator('%')).toThrow() - expect(() => validateSeparator('~')).toThrow() - expect(() => validateSeparator(':')).not.toThrow() - }) -}) diff --git a/test/document-storage/document-storage.suite.ts b/test/document-storage/document-storage.suite.ts new file mode 100644 index 0000000..dc489c4 --- /dev/null +++ b/test/document-storage/document-storage.suite.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest' +import type { BaseDocumentStorage } from '../../src/document-storage/base-document-storage.js' +import { collect } from '../helpers.js' + +type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } } + +/** Behaviour every document storage must have. `make` returns an empty storage. */ +export function runDocumentStorageSuite(name: string, make: () => BaseDocumentStorage): void { + describe(`document storage: ${name}`, () => { + const users = () => make().child('users') + + it('inserts with time-sortable ids, gets and lists', async () => { + const u = users() + const a = await u.insert({ name: 'Ada' }) + const b = await u.insert({ name: 'Bob' }) + expect(a < b).toBe(true) + expect(await u.get(a)).toEqual({ name: 'Ada' }) + expect(await collect(u.keys())).toEqual([a, b]) + }) + + it('set / get / delete in self and child forms', async () => { + const u = users() + await u.set('1', { name: 'Ada' }) + expect(await u.child('1').get()).toEqual({ name: 'Ada' }) + await u.child('1').set({ name: 'Ada L.' }) + expect(await u.getMany(['1', '2'])).toEqual([{ name: 'Ada L.' }, undefined]) + expect(await u.delete('1')).toBe(true) + expect(await u.child('1').delete()).toBe(false) + expect(await u.get('1')).toBeUndefined() + await u.set('1', { name: 'Again' }) + expect(await u.get('1')).toEqual({ name: 'Again' }) + }) + + it('updates with operators, returns the new state, and supports upsert', async () => { + const u = users() + await u.set('1', { name: 'Ada', tags: ['a'], logins: 1 }) + expect(await u.update('1', { $inc: { logins: 2 }, $push: { tags: 'b' }, $set: { 'address.city': 'London' } })).toEqual({ + name: 'Ada', + tags: ['a', 'b'], + logins: 3, + address: { city: 'London' }, + }) + expect(await u.child('1').update({ $unset: { address: 1 } })).toEqual({ name: 'Ada', tags: ['a', 'b'], logins: 3 }) + expect(await u.update('missing', { $set: { a: 1 } })).toBeUndefined() + expect(await u.update('missing', { $inc: { n: 1 } }, { upsert: true })).toEqual({ n: 1 }) + }) + + it('rejects bad keys and documents', async () => { + const u = users() + await expect(u.get(undefined as never)).rejects.toThrow(TypeError) + await expect(u.update(undefined as never, { $set: {} })).rejects.toThrow(TypeError) + await expect(make().get()).rejects.toThrow(/root/) + await expect(u.set('x', [] as never)).rejects.toThrow(/plain object/) + }) + + it('nests documents under documents', async () => { + const u = users() + await u.set('1', { name: 'Ada' }) + await u.child('1').child('posts').set('p1', { title: 'Hi' }) + expect(await u.child('1').child('posts').get('p1')).toEqual({ title: 'Hi' }) + expect((await collect(u.find())).map((r) => r.key)).toEqual(['1']) + }) + + it('finds with equality, comparisons, $in, dot paths, sort, skip and limit', async () => { + const u = users() + const rows: [string, User][] = [ + ['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }], + ['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'] }], + ['c', { name: 'Cy', age: 51, address: { city: 'London' } }], + ['d', { name: 'Di', age: 20 }], + ] + for (const [k, v] of rows) await u.set(k, v) + const keys = async (filter: Record, options = {}) => (await collect(u.find(filter, options))).map((r) => r.key) + expect(await keys({ age: 20 })).toEqual(['b', 'd']) + expect(await keys({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd']) + expect(await keys({ age: { $in: [36, 51] } })).toEqual(['a', 'c']) + expect(await keys({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c']) + expect(await keys({ tags: 'y' })).toEqual(['a', 'b']) + expect(await keys({ email: null })).toEqual(['c', 'd']) + expect(await keys({}, { sort: { age: -1 } })).toEqual(['c', 'a', 'b', 'd']) + expect(await keys({}, { sort: { age: 1 }, skip: 1, limit: 2 })).toEqual(['d', 'a']) + expect(await u.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: rows[1]![1] }) + expect(await u.findOne({ email: 'nobody' })).toBeUndefined() + }) + }) +} diff --git a/test/document-storage/document-storage.test.ts b/test/document-storage/document-storage.test.ts new file mode 100644 index 0000000..d4add11 --- /dev/null +++ b/test/document-storage/document-storage.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' +import { DocumentStorageInMemory } from '../../src/document-storage/document-storage-in-memory.js' +import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' +import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { collect, KVStorageCounting } from '../helpers.js' +import { runDocumentStorageSuite } from './document-storage.suite.js' + +const indexes = { users: ['email', 'age'] } + +runDocumentStorageSuite('DocumentStorageInMemory', () => new DocumentStorageInMemory()) +runDocumentStorageSuite('DocumentStorageInMemory (indexed)', () => new DocumentStorageInMemory({ indexes })) +runDocumentStorageSuite('DocumentStorageKV', () => new DocumentStorageKV(new KVStorageInMemory())) +runDocumentStorageSuite('EventSourced (in-memory projections)', () => { + const kv = new KVStorageInMemory() + return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageInMemory({ indexes })) +}) +runDocumentStorageSuite('EventSourced (KV projections)', () => { + const kv = new KVStorageInMemory() + return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))) +}) + +describe('DocumentStorageInMemory indexes', () => { + it('keeps the equality index in sync with updates and deletes', async () => { + const users = new DocumentStorageInMemory({ indexes }).child('users') + await users.set('1', { email: 'a', age: 1 }) + await users.set('2', { email: 'b', age: 1 }) + await users.update('1', { $set: { email: 'c' } }) + await users.delete('2') + expect(await collect(users.find({ email: 'a' }))).toEqual([]) + expect(await collect(users.find({ email: 'c' }))).toEqual([{ key: '1', data: { email: 'c', age: 1 } }]) + expect(await collect(users.find({ age: { $eq: 1 } }))).toHaveLength(1) + }) +}) + +describe('EventSourcedDocumentStorage', () => { + function setup() { + const kv = new KVStorageCounting() + const events = new EventStorageKV(kv) + const projections = new DocumentStorageKV(kv.child('projections')) + return { kv, events, projections, docs: new EventSourcedDocumentStorage(events, projections) } + } + + it('records resolved changes as history and replays any version', async () => { + const users = setup().docs.child('users') + await users.set('1', { n: 5 }) + await users.update('1', { $inc: { n: 1 } }) + await users.delete('1') + await users.set('1', { n: 10 }) + expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([ + { op: 'set', doc: { n: 5 } }, + { op: 'patch', set: { n: 6 }, unset: [] }, + { op: 'delete' }, + { op: 'set', doc: { n: 10 } }, + ]) + expect(await users.getAt('1', 2)).toEqual({ n: 6 }) + expect(await users.child('1').getAt(3)).toBeUndefined() + expect(await users.get('1')).toEqual({ n: 10 }) + }) + + it('heals a projection left behind by a crash between append and projection write', async () => { + const { docs, events } = setup() + const users = docs.child('users') + await users.set('1', { n: 1 }) + await events.child('users').append('1', [{ op: 'patch', set: { n: 2 }, unset: [] }]) // projection never written + expect(await users.get('1')).toEqual({ n: 1 }) + expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3 }) + await events.child('users').append('2', [{ op: 'set', doc: { n: 7 } }]) // first write crashed + expect(await users.catchUp('2')).toEqual({ n: 7 }) + expect(await users.get('2')).toEqual({ n: 7 }) + }) + + it('two instances incrementing the same document converge', async () => { + const kv = new KVStorageInMemory() + const make = () => new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))).child('users') + const a = make() + const b = make() + await a.set('1', { n: 0 }) + await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })])) + expect(await a.get('1')).toEqual({ n: 10 }) + }) + + it('a write is a projection read, an append and a projection write', async () => { + const { docs, kv } = setup() + await docs.child('users').set('1', { n: 1 }) + kv.reset() + await docs.child('users').update('1', { $inc: { n: 1 } }) + expect(kv.calls).toEqual({ readMany: 2, writeIfAbsent: 1, writeMany: 1 }) + }) +}) diff --git a/test/documents/event-sourced.test.ts b/test/documents/event-sourced.test.ts deleted file mode 100644 index 293bc1c..0000000 --- a/test/documents/event-sourced.test.ts +++ /dev/null @@ -1,379 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { EventSourcedDocumentStorage, type EventSourcedDocumentStorageOptions } from '../../src/documents/event-sourced.js' -import type { DocIndexDeclarations } from '../../src/documents/indexes.js' -import type { Filter } from '../../src/documents/filter.js' -import { CapabilityError, UniqueConstraintError } from '../../src/errors.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } } - -const userIndexes: DocIndexDeclarations = { - users: [{ fields: ['email'], unique: true }, { fields: ['age'] }, { name: 'cityAge', fields: ['address.city', 'age'] }, { fields: ['tags'] }], - 'users/*/posts': [{ fields: ['createdAt'] }], -} - -function setup(opts: { hide?: Capability[]; inner?: InMemoryKVDriver } & EventSourcedDocumentStorageOptions = {}) { - const { hide = [], inner = new InMemoryKVDriver(), ...docOpts } = opts - const driver = new CountingDriver(inner, hide) - const kv = new KVStorage(driver) - const events = new KVEventStorage(kv) - const docs = new EventSourcedDocumentStorage(events, { indexes: userIndexes, warnOnFullScan: false, ...docOpts }) - return { driver, kv, events, docs, users: docs.child('users') } -} - -afterEach(() => vi.restoreAllMocks()) - -describe.each([ - ['atomic batch', [] as Capability[]], - ['non-atomic', ['batch'] as Capability[]], -])('EventSourcedDocumentStorage (%s)', (_, hide) => { - describe('CRUD', () => { - it('inserts with time-sortable ids and reads back', async () => { - const { users } = setup({ hide }) - const a = await users.insert({ name: 'Ada', age: 36 }) - const b = await users.insert({ name: 'Bob', age: 20 }) - expect(a).toMatch(/^[0-9A-HJKMNP-TV-Z]{26}$/) - expect(a < b).toBe(true) - expect(await users.get(a)).toEqual({ name: 'Ada', age: 36 }) - expect(await collect(users.keys())).toEqual([a, b]) - }) - - it('set / get / delete in self and child forms', async () => { - const { users } = setup({ hide }) - await users.set('123', { name: 'Ada', age: 36 }) - expect(await users.child('123').get()).toEqual({ name: 'Ada', age: 36 }) - await users.child('123').set({ name: 'Ada L.' }) - expect(await users.get('123')).toEqual({ name: 'Ada L.' }) - expect(await users.delete('123')).toBe(true) - expect(await users.delete('123')).toBe(false) - expect(await users.get('123')).toBeUndefined() - expect(await collect(users.keys())).toEqual([]) - await users.set('123', { name: 'Again' }) - expect(await users.get('123')).toEqual({ name: 'Again' }) - }) - - it('applies arity rules', async () => { - const { users, docs } = setup({ hide }) - const id: string | undefined = undefined - await expect(users.get(id as never)).rejects.toThrow(TypeError) - await expect(users.update(id as never, { $set: { a: 1 } })).rejects.toThrow(TypeError) - await expect(users.delete(id as never)).rejects.toThrow(TypeError) - await expect(docs.get()).rejects.toThrow(/root/) - await expect(users.set('x', [] as never)).rejects.toThrow(/plain object/) - }) - - it('nests documents under documents', async () => { - const { users } = setup({ hide }) - await users.set('u1', { name: 'Ada' }) - const posts = users.child('u1').child('posts') - await posts.set('p1', { title: 'Hello', createdAt: '2024-01' }) - expect(await posts.get('p1')).toEqual({ title: 'Hello', createdAt: '2024-01' }) - expect((await collect(users.find())).map((r) => r.key)).toEqual(['u1']) - expect((await collect(posts.find({ createdAt: { $gte: '2024' } }))).map((r) => r.key)).toEqual(['p1']) - }) - }) - - describe('update', () => { - it('supports $set $unset $inc $push $pull $addToSet with dot paths', async () => { - const { users } = setup({ hide }) - await users.set('1', { name: 'Ada', tags: ['a'], address: { city: 'X' }, nickname: 'A' } as User) - const r = await users.update('1', { - $set: { name: 'Ada L.', 'address.zip': '123' }, - $inc: { logins: 1 }, - $push: { tags: { $each: ['b', 'c'] } }, - $unset: { nickname: true }, - }) - expect(r).toEqual({ name: 'Ada L.', tags: ['a', 'b', 'c'], address: { city: 'X', zip: '123' }, logins: 1 }) - await users.update('1', { $pull: { tags: { $in: ['a', 'c'] } }, $addToSet: { 'address.list': { $each: [1, 1, 2] } } }) - await users.child('1').update({ $inc: { logins: 5 }, $addToSet: { tags: 'b' } }) - expect(await users.get('1')).toEqual({ name: 'Ada L.', tags: ['b'], address: { city: 'X', zip: '123', list: [1, 2] }, logins: 6 }) - }) - - it('returns undefined for missing docs unless upsert', async () => { - const { users } = setup({ hide }) - expect(await users.update('nope', { $set: { a: 1 } })).toBeUndefined() - expect(await users.get('nope')).toBeUndefined() - expect(await users.update('nope', { $inc: { n: 2 } }, { upsert: true })).toEqual({ n: 2 }) - }) - - it('rejects conflicting paths and bad operators', async () => { - const { users } = setup({ hide }) - await users.set('1', { age: 1 }) - await expect(users.update('1', { $set: { a: 1 }, $unset: { 'a.b': 1 } })).rejects.toThrow(/conflict/) - await expect(users.update('1', { $rename: {} } as never)).rejects.toThrow(/Unsupported/) - await expect(users.update('1', { $inc: { name: 1 } }).then(() => users.update('1', { $push: { age: 1 } }))).rejects.toThrow(/non-array/) - }) - - it('stores resolved deltas, not operators, and skips no-op updates', async () => { - const { users } = setup({ hide }) - await users.set('1', { logins: 5 }) - await users.update('1', { $inc: { logins: 1 } }) - await users.update('1', { $set: { logins: 6 } }) // no-op - expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([ - { op: 'set', doc: { logins: 5 } }, - { op: 'patch', set: { logins: 6 } }, - ]) - }) - }) - - describe('history and getAt', () => { - it('replays to any version', async () => { - const { users } = setup({ hide }) - await users.set('1', { n: 1 }) - await users.update('1', { $inc: { n: 1 } }) - await users.delete('1') - await users.set('1', { n: 10 }) - expect(await users.getAt('1', 0)).toBeUndefined() - expect(await users.getAt('1', 1)).toEqual({ n: 1 }) - expect(await users.getAt('1', 2)).toEqual({ n: 2 }) - expect(await users.getAt('1', 3)).toBeUndefined() - expect(await users.child('1').getAt(4)).toEqual({ n: 10 }) - expect(await users.getAt('1', 99)).toEqual({ n: 10 }) - expect((await collect(users.child('1').history())).map((e) => e.version)).toEqual([1, 2, 3, 4]) - }) - }) - - describe('find', () => { - async function seed() { - const ctx = setup({ hide }) - const rows: [string, User][] = [ - ['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }], - ['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'], address: { city: 'Paris' } }], - ['c', { name: 'Cy', age: 51, tags: [], address: { city: 'London' } }], - ['d', { name: 'Di', age: 20, email: 'di@x' }], - ['e', { name: 'Ed', email: 'ed@x', address: { city: 'London' } }], - ] - for (const [k, v] of rows) await ctx.users.set(k, v) - await ctx.users.set('gone', { name: 'Gone', age: 20 }) - await ctx.users.delete('gone') - return ctx - } - const keysOf = async (it: AsyncIterable<{ key: string }>) => (await collect(it)).map((r) => r.key) - - it('evaluates comparison, logical and element operators', async () => { - const { users } = await seed() - // Unsorted results come in index or key order depending on the plan; compare as sets. - const q = async (f: Filter) => (await keysOf(users.find(f))).sort() - expect(await q({})).toEqual(['a', 'b', 'c', 'd', 'e']) - expect(await q({ age: 20 })).toEqual(['b', 'd']) - expect(await q({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd']) - expect(await q({ age: { $gt: 20 } })).toEqual(['a', 'c']) - expect(await q({ age: { $ne: 20 } })).toEqual(['a', 'c', 'e']) - expect(await q({ age: { $in: [36, 51] } })).toEqual(['a', 'c']) - expect(await q({ age: { $nin: [36, 51] } })).toEqual(['b', 'd', 'e']) - expect(await q({ age: { $exists: false } })).toEqual(['e']) - expect(await q({ age: null })).toEqual(['e']) - expect(await q({ email: { $exists: true }, age: { $lte: 20 } })).toEqual(['b', 'd']) - expect(await q({ 'address.city': 'London' })).toEqual(['a', 'c', 'e']) - expect(await q({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c']) - expect(await q({ tags: 'y' })).toEqual(['a', 'b']) - expect(await q({ tags: ['y'] })).toEqual(['b']) - expect(await q({ $or: [{ age: 51 }, { name: 'Bob' }] })).toEqual(['b', 'c']) - expect(await q({ $and: [{ age: { $gte: 20 } }, { age: { $lte: 20 } }] })).toEqual(['b', 'd']) - expect(await q({ $not: { age: 20 } })).toEqual(['a', 'c', 'e']) - expect(await q({ age: { $not: { $gte: 30 } } })).toEqual(['b', 'd', 'e']) - expect(await q({ name: { $gt: 5 } })).toEqual([]) // type-bracketed - }) - - it('sorts (index-backed and in-memory), skips, limits and projects', async () => { - const { users } = await seed() - expect(await keysOf(users.find({}, { sort: { age: 1 } }))).toEqual(['e', 'b', 'd', 'a', 'c']) - expect(await keysOf(users.find({}, { sort: { age: -1 } }))).toEqual(['c', 'a', 'd', 'b', 'e']) - expect(await keysOf(users.find({}, { sort: { name: -1 }, skip: 1, limit: 2 }))).toEqual(['d', 'c']) - expect(await keysOf(users.find({ age: { $gte: 20 } }, { sort: { age: -1 }, limit: 2 }))).toEqual(['c', 'a']) - expect(await keysOf(users.find({ 'address.city': 'London' }, { sort: { age: 1 } }))).toEqual(['e', 'a', 'c']) - expect(await collect(users.find({ age: 36 }, { projection: { name: 1, 'address.city': 1 } }))).toEqual([ - { key: 'a', data: { name: 'Ada', address: { city: 'London' } } }, - ]) - expect(await collect(users.find({ age: 36 }, { projection: { tags: 0, address: 0, email: 0 } }))).toEqual([{ key: 'a', data: { name: 'Ada', age: 36 } }]) - expect(await users.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: expect.objectContaining({ name: 'Bob' }) }) - expect(await users.findOne({ email: 'nobody' })).toBeUndefined() - }) - - it('uses an index: findOne costs one entry scan and one projection read', async () => { - const { users, driver } = await seed() - driver.reset() - await users.findOne({ email: 'di@x' }) - expect(driver.calls).toEqual({ scan: 1, getMany: 1 }) - }) - - it('warns once per filter shape on full scans', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const { users } = setup({ hide, warnOnFullScan: true }) - await collect(users.find({ name: 'x' })) - await collect(users.find({ name: 'y' })) - await collect(users.find({ age: 1 })) - expect(warn).toHaveBeenCalledTimes(1) - expect(warn.mock.calls[0]![0]).toMatch(/full scan for filter \{"name":"\?"\}/) - }) - }) - - describe('unique indexes', () => { - it('rejects duplicates and frees values on change and delete', async () => { - const { users } = setup({ hide }) - await users.set('1', { email: 'a@x' }) - await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError) - await expect(users.insert({ email: 'a@x' })).rejects.toMatchObject({ index: 'email', existingKey: '1' }) - expect(await collect(users.keys())).toEqual(['1']) - await users.update('1', { $set: { email: 'b@x' } }) - await users.set('2', { email: 'a@x' }) - await users.delete('2') - await users.set('3', { email: 'a@x' }) - await users.set('4', {}) // missing values are not claimed - await users.set('5', {}) - await users.set('3', { email: 'a@x', name: 'same value, same doc' }) - }) - - it('exactly one of concurrent writers claiming the same value wins', async () => { - const { users } = setup({ hide }) - const results = await Promise.allSettled(['1', '2', '3'].map((k) => users.set(k, { email: 'same' }))) - expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) - expect((await collect(users.find({ email: 'same' }))).length).toBe(1) - }) - - it('takes over a stale claim only after the grace period', async () => { - let t = 0 - const { users, driver } = setup({ hide, now: () => t, claimGraceMs: 1000 }) - // Leave an orphaned claim: doc 1's projection and events vanish but its claim remains. - await users.set('1', { email: 'a@x' }) - await driver.inner.deleteMany(['~p/users/1']) - await driver.inner.deleteMany((await collect(driver.inner.scan({ gte: 'users/1/', lt: 'users/10' }))).map(([k]) => k)) - await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError) - t = 5000 - await users.set('2', { email: 'a@x' }) - expect(await users.get('2')).toEqual({ email: 'a@x' }) - }) - - it('requires setIfAbsent', () => { - const kv = new KVStorage(new CountingDriver(new InMemoryKVDriver(), ['setIfAbsent'])) - expect(() => new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: userIndexes })).toThrow(CapabilityError) - }) - }) - - describe('projection lag (crash between append and projection write)', () => { - async function lagged() { - const ctx = setup({ hide }) - await ctx.users.set('1', { n: 1, email: 'a@x' }) - // Simulate a crash: the event is appended but the projection is never written. - await ctx.events.child('users').append('1', [{ op: 'patch', set: { n: 2, email: 'b@x' } }]) - return ctx - } - - it('reads see the stale projection; consistent reads catch up', async () => { - const { users } = await lagged() - expect(await users.get('1')).toEqual({ n: 1, email: 'a@x' }) - expect(await users.get('1', { consistent: true })).toEqual({ n: 2, email: 'b@x' }) - expect(await users.findOne({ email: 'b@x' })).toMatchObject({ key: '1' }) - expect(await users.findOne({ email: 'a@x' })).toBeUndefined() - }) - - it('the next write self-heals', async () => { - const { users } = await lagged() - expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3, email: 'b@x' }) - expect((await collect(users.history('1'))).map((e) => e.version)).toEqual([1, 2, 3]) - await users.set('2', { email: 'a@x' }) // old claim was released by the catch-up - }) - - it('catchUp() repairs a subtree', async () => { - const { users, docs, events } = await lagged() - await events - .child('users') - .child('9') - .child('posts') - .append('p', [{ op: 'set', doc: { createdAt: 'x' } }]) - expect(await docs.catchUp()).toBe(2) - expect(await users.get('1')).toEqual({ n: 2, email: 'b@x' }) - expect(await users.child('9').child('posts').get('p')).toEqual({ createdAt: 'x' }) - expect(await docs.catchUp()).toBe(0) - }) - }) - - describe('concurrency', () => { - it('two storage instances incrementing the same doc converge', async () => { - const inner = new InMemoryKVDriver() - const a = setup({ hide, inner }).users - const b = setup({ hide, inner }).users - await a.set('1', { n: 0 }) - await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })])) - expect(await a.get('1')).toEqual({ n: 10 }) - expect(await b.get('1', { consistent: true })).toEqual({ n: 10 }) - }) - }) -}) - -describe('costs', () => { - it('get is one driver call; update is projection read + append + projection write, even cold', async () => { - const inner = new InMemoryKVDriver() - await setup({ inner }).users.set('1', { name: 'a', age: 1 }) - const { users, driver } = setup({ inner }) // fresh instance: cold head cache - driver.reset() - await users.get('1') - expect(driver.calls).toEqual({ getMany: 1 }) - driver.reset() - await users.update('1', { $set: { age: 2 } }) - expect(driver.calls).toEqual({ getMany: 1, setIfAbsent: 1, batch: 1 }) - }) -}) - -describe('property: indexed find equals unindexed find', () => { - const doc = fc.record( - { - age: fc.option(fc.integer({ min: 0, max: 5 }), { nil: undefined }), - email: fc.constantFrom('a', 'b', 'c', 'd', 'e', 'f'), - tags: fc.array(fc.constantFrom('x', 'y', 'z'), { maxLength: 3 }), - address: fc.option(fc.record({ city: fc.constantFrom('L', 'P') }), { nil: undefined }), - }, - { requiredKeys: [] }, - ) - const bound = fc.integer({ min: -1, max: 6 }) - const filter: fc.Arbitrary = fc.oneof( - fc.record({ age: bound }), - fc.record({ age: fc.record({ $gte: bound, $lt: bound }, { requiredKeys: [] }) }), - fc.record({ age: fc.record({ $gt: bound, $lte: bound }, { requiredKeys: [] }) }), - fc.record({ tags: fc.constantFrom('x', 'y', 'z') }), - fc.record({ tags: fc.record({ $gte: fc.constantFrom('x', 'y'), $lt: fc.constantFrom('y', 'zz') }) }), - fc.record({ 'address.city': fc.constantFrom('L', 'P'), age: fc.record({ $gt: bound }) }), - fc.record({ 'address.city': fc.constantFrom('L', 'P') }), - fc.constant({ age: null }), - fc.constant({}), - ) - const sort = fc.option(fc.record({ age: fc.constantFrom(1 as const, -1 as const) }), { nil: undefined }) - - it('returns the same results in the same order', async () => { - await fc.assert( - fc.asyncProperty( - fc.array(fc.tuple(fc.constantFrom('1', '2', '3', '4', '5'), doc), { maxLength: 12 }), - filter, - sort, - fc.option(fc.integer({ min: 1, max: 4 }), { nil: undefined }), - async (writes, f, s, limit) => { - const plain = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { warnOnFullScan: false }).child('users') - const indexed = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { - warnOnFullScan: false, - indexes: { users: userIndexes.users!.map((i) => ({ ...i, unique: false })) }, - }).child('users') - for (const [k, v] of writes) { - await plain.set(k, v) - await indexed.set(k, v) - } - const opts = { ...(s ? { sort: s } : {}), ...(limit ? { limit } : {}) } - const a = await collect(plain.find(f, opts)) - const b = await collect(indexed.find(f, opts)) - if (s) { - expect(b).toEqual(a) - return - } - // Unsorted order depends on the plan: compare as sets (a limited result is any subset of that size). - const all = new Set((await collect(plain.find(f))).map((r) => r.key)) - expect(b).toHaveLength(a.length) - for (const r of b) expect(all.has(r.key)).toBe(true) - }, - ), - { numRuns: 300 }, - ) - }) -}) diff --git a/test/documents/units.test.ts b/test/documents/units.test.ts deleted file mode 100644 index 16f5018..0000000 --- a/test/documents/units.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createIdGenerator } from '../../src/documents/ulid.js' -import { matches } from '../../src/documents/filter.js' -import { applyPatch, resolveUpdate } from '../../src/documents/update.js' - -describe('ulid', () => { - it('is strictly increasing within one millisecond and sorts by time', () => { - let t = 1_700_000_000_000 - const gen = createIdGenerator(() => t) - const ids = Array.from({ length: 1000 }, () => gen()) - expect([...ids].sort()).toEqual(ids) - expect(new Set(ids).size).toBe(1000) - t += 1 - const later = gen() - expect(later > ids.at(-1)!).toBe(true) - expect(later.slice(0, 10) > ids[0]!.slice(0, 10)).toBe(true) - }) -}) - -describe('filter semantics', () => { - it('requires one array element to satisfy all positive operators of a condition', () => { - expect(matches({ s: [1, 20] }, { s: { $gt: 5, $lt: 10 } })).toBe(false) // MongoDB would match - expect(matches({ s: [1, 7] }, { s: { $gt: 5, $lt: 10 } })).toBe(true) - expect(matches({ s: [1, 20] }, { $and: [{ s: { $gt: 5 } }, { s: { $lt: 10 } }] })).toBe(true) - }) - - it('traverses arrays of sub-documents on dot paths', () => { - expect(matches({ a: [{ b: 1 }, { b: 2 }] }, { 'a.b': 2 })).toBe(true) - expect(matches({ a: [{ b: 1 }] }, { 'a.0.b': 1 })).toBe(true) - expect(matches({ a: [{ b: 1 }] }, { 'a.b': { $exists: true } })).toBe(true) - expect(matches({ a: [] }, { 'a.b': { $exists: true } })).toBe(false) - }) - - it('treats null as matching missing', () => { - expect(matches({}, { x: null })).toBe(true) - expect(matches({ x: null }, { x: { $ne: null } })).toBe(false) - expect(matches({ x: 0 }, { x: { $ne: null } })).toBe(true) - }) - - it('rejects unknown operators', () => { - expect(() => matches({}, { x: { $regex: 'a' } })).toThrow(/Unknown operator/) - expect(() => matches({}, { $nor: [] } as never)).toThrow(/Unknown top-level/) - }) -}) - -describe('update resolution', () => { - it('$pull by sub-document filter and by condition', () => { - const state = { - items: [ - { k: 1, v: 'a' }, - { k: 2, v: 'b' }, - ], - n: [1, 5, 9], - } - const patch = resolveUpdate(state, { $pull: { items: { k: 2 }, n: { $gte: 5 } } }) - expect(applyPatch(state, patch)).toEqual({ items: [{ k: 1, v: 'a' }], n: [1] }) - }) - - it('$unset of an array position nulls it', () => { - expect(applyPatch({ a: [1, 2] }, resolveUpdate({ a: [1, 2] }, { $unset: { 'a.0': true } }))).toEqual({ a: [null, 2] }) - }) -}) diff --git a/test/event-storage/event-storage-kv.test.ts b/test/event-storage/event-storage-kv.test.ts new file mode 100644 index 0000000..4b0af71 --- /dev/null +++ b/test/event-storage/event-storage-kv.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { ConcurrencyError } from '../../src/utils/errors.js' +import { collect, KVStorageCounting } from '../helpers.js' + +function setup(kv = new KVStorageCounting()) { + let t = 1000 + return { kv, events: new EventStorageKV(kv, { now: () => t++ }) } +} + +describe('EventStorageKV', () => { + it('appends and reads in self and child forms', async () => { + const orders = setup().events.child('orders') + expect(await orders.append('o1', [{ type: 'created' }], { expectedVersion: 0 })).toEqual({ version: 1 }) + expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3 }) + expect(await collect(orders.read('o1'))).toEqual([ + { version: 1, at: 1000, data: { type: 'created' } }, + { version: 2, at: 1001, data: { type: 'paid' } }, + { version: 3, at: 1001, data: { type: 'shipped' } }, + ]) + expect(await collect(orders.child('o1').read({ from: 3 }))).toHaveLength(1) + expect(await orders.version('o1')).toBe(3) + expect(await orders.version('none')).toBe(0) + }) + + it('stores one key per commit', async () => { + const { kv, events } = setup() + await events.child('s').append(['a']) + await events.child('s').append(['b', 'c']) + expect((await collect(kv.scan())).map(([k]) => k)).toEqual(['s/~e/000000000001', 's/~e/000000000002']) + }) + + it('enforces expectedVersion and the arity rules', async () => { + const { events } = setup() + const s = events.child('s') + await s.append(['a']) + await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError) + await expect(events.append(['x'])).rejects.toThrow(/root/) + await expect(events.child('o').append(undefined as never, ['x'] as never)).rejects.toThrow(TypeError) + await expect(events.child('o').version(undefined as never)).rejects.toThrow(TypeError) + }) + + it('lists child streams', async () => { + const orders = setup().events.child('orders') + await orders.append('o1', ['a']) + await orders.append('o2', ['b']) + await orders.child('o1').append('notes', ['n']) + expect(await collect(orders.keys())).toEqual(['o1', 'o2']) + expect(await collect(orders.child('o1').keys())).toEqual(['notes']) + }) + + it('a warm append is one writeIfAbsent; a cold one adds one reverse scan', async () => { + const { kv, events } = setup() + await events.child('s').append(['a']) + kv.reset() + await events.child('s').append(['b']) + expect(kv.calls).toEqual({ writeIfAbsent: 1 }) + const cold = setup(kv).events + kv.reset() + await cold.child('s').append(['c']) + expect(kv.calls).toEqual({ scan: 1, writeIfAbsent: 1 }) + }) + + it('a writer with a stale cache collides; with "any" it retries', async () => { + const kv = new KVStorageInMemory() + const a = setup(new KVStorageCounting(kv)).events.child('s') + const b = setup(new KVStorageCounting(kv)).events.child('s') + await a.append(['a1']) + await b.append(['b1']) + await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 }) + expect(await a.append(['a3'])).toEqual({ version: 3 }) + const results = await Promise.allSettled([a.append(['x'], { expectedVersion: 3 }), b.append(['y'], { expectedVersion: 3 })]) + expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) + }) + + it('reads any window correctly (property)', async () => { + await fc.assert( + fc.asyncProperty( + fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 20 }), + fc.record({ + from: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }), + to: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }), + limit: fc.option(fc.nat(10), { nil: undefined }), + }), + async (sizes, options) => { + const s = setup().events.child('s') + let n = 0 + for (const size of sizes) await s.append(Array.from({ length: size }, () => ++n)) + let expected = Array.from({ length: n }, (_, i) => i + 1).filter((v) => v >= (options.from ?? 1) && v <= (options.to ?? Infinity)) + if (options.limit !== undefined) expected = expected.slice(0, options.limit) + expect((await collect(s.read(options))).map((e) => e.data)).toEqual(expected) + }, + ), + { numRuns: 200 }, + ) + }) +}) diff --git a/test/events/kv.test.ts b/test/events/kv.test.ts deleted file mode 100644 index a1a393c..0000000 --- a/test/events/kv.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { ConcurrencyError } from '../../src/errors.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -function setup(hide: Capability[] = [], inner = new InMemoryKVDriver()) { - const driver = new CountingDriver(inner, hide) - let t = 1000 - const events = new KVEventStorage(new KVStorage(driver), { now: () => t++ }) - return { driver, events } -} - -describe('KVEventStorage', () => { - it('appends and reads with versions and metadata', async () => { - const { events } = setup() - const orders = events.child('orders') - expect(await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 })).toEqual({ version: 1, commit: 1 }) - expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3, commit: 2 }) - expect(await collect(orders.read('o1'))).toEqual([ - { version: 1, commit: 1, at: 1000, data: { type: 'created', total: 40 } }, - { version: 2, commit: 2, at: 1001, data: { type: 'paid' } }, - { version: 3, commit: 2, at: 1001, data: { type: 'shipped' } }, - ]) - expect(await orders.version('o1')).toBe(3) - expect(await orders.child('o1').version()).toBe(3) - expect(await orders.version('nope')).toBe(0) - }) - - it('stores one key per commit under /~e/', async () => { - const { events, driver } = setup() - await events.child('orders').append('o1', ['a']) - await events.child('orders').append('o1', ['b', 'c']) - expect(await collect(driver.inner.scan({ values: true }))).toEqual([ - ['orders/o1/~e/000000000001', { firstVersion: 1, at: 1000, events: ['a'] }], - ['orders/o1/~e/000000000002', { firstVersion: 2, at: 1001, events: ['b', 'c'] }], - ]) - }) - - it('enforces expectedVersion', async () => { - const { events } = setup() - const s = events.child('s') - await s.append(['a'], { expectedVersion: 0 }) - await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError) - await expect(s.append(['b'], { expectedVersion: 5 })).rejects.toMatchObject({ expectedVersion: 5, actualVersion: 1 }) - expect(await s.append(['b'], { expectedVersion: 1 })).toEqual({ version: 2, commit: 2 }) - await expect(s.append([], { expectedVersion: 1 })).rejects.toBeInstanceOf(ConcurrencyError) - expect(await s.append([])).toEqual({ version: 2, commit: 2 }) - }) - - it('applies the arity rules', async () => { - const { events } = setup() - const orders = events.child('orders') - const id: string | undefined = undefined - await expect(orders.append(id as never, ['x'] as never)).rejects.toThrow(TypeError) - await expect(orders.version(id as never)).rejects.toThrow(TypeError) - await expect(collect(orders.read(id as never))).rejects.toThrow(TypeError) - await expect(events.append(['x'])).rejects.toThrow(/root/) - await expect(orders.append('o1', 'x' as never)).rejects.toThrow(/array/) - }) - - it('lists child streams and keeps nested streams separate', async () => { - const { events } = setup() - const orders = events.child('orders') - await orders.append('o1', ['a']) - await orders.append('o2', ['b']) - await orders.child('o1').child('shipments').append('s1', ['ship']) - await orders.child('o1').append('notes', ['n']) - expect(await collect(orders.keys())).toEqual(['o1', 'o2']) - expect(await collect(orders.child('o1').keys())).toEqual(['notes', 'shipments']) - expect((await collect(orders.read('o1'))).map((e) => e.data)).toEqual(['a']) - }) -}) - -describe('read windows (property)', () => { - it('matches brute force for any commit layout', async () => { - await fc.assert( - fc.asyncProperty( - fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 25 }), - fc.record({ - from: fc.option(fc.integer({ min: -1, max: 70 }), { nil: undefined }), - to: fc.option(fc.integer({ min: 0, max: 70 }), { nil: undefined }), - reverse: fc.boolean(), - limit: fc.option(fc.integer({ min: 0, max: 10 }), { nil: undefined }), - }), - async (sizes, opts) => { - const { events } = setup() - const s = events.child('s') - const all: number[] = [] - for (const n of sizes) { - const batch = Array.from({ length: n }, (_, i) => all.length + i + 1) - await s.append(batch) - all.push(...batch) - } - let exp = all.filter((v) => v >= Math.max(1, opts.from ?? 1) && v <= (opts.to ?? Infinity)) - if (opts.reverse) exp.reverse() - if (opts.limit !== undefined) exp = exp.slice(0, opts.limit) - const got = await collect(s.read(opts)) - expect(got.map((e) => e.data)).toEqual(exp) - expect(got.map((e) => e.version)).toEqual(exp) - }, - ), - { numRuns: 300 }, - ) - }) -}) - -describe('costs', () => { - it('cached head: a warm append is a single setIfAbsent', async () => { - const { events, driver } = setup() - const s = events.child('s') - await s.append(['a']) - driver.reset() - await s.append(['b']) - await s.append(['c'], { expectedVersion: 2 }) - expect(driver.calls).toEqual({ setIfAbsent: 2 }) - }) - - it('cold head lookup is one reverse scan with limit 1', async () => { - const inner = new InMemoryKVDriver() - await setup([], inner).events.child('s').append(['a', 'b']) - const { events, driver } = setup([], inner) - expect(await events.child('s').append(['c'])).toEqual({ version: 3, commit: 2 }) - expect(driver.calls).toEqual({ scan: 1, setIfAbsent: 1 }) - }) - - it('read({ from }) finds single-event commits in one probe', async () => { - const { events, driver } = setup() - const s = events.child('s') - for (let i = 0; i < 200; i++) await s.append([i]) - driver.reset() - expect((await collect(s.read({ from: 150, limit: 1 }))).map((e) => e.version)).toEqual([150]) - expect(driver.calls).toEqual({ getMany: 1, scan: 1 }) - }) - - it('read({ from }) uses O(log commits) probes for multi-event commits', async () => { - const { events, driver } = setup() - const s = events.child('s') - for (let i = 0; i < 1000; i++) await s.append(i % 3 === 0 ? [i, i] : [i]) - driver.reset() - const [first] = await collect(s.read({ from: 777, limit: 1 })) - expect(first?.version).toBe(777) - // 1 guess probe + ceil(log8(1000)) search rounds, then the read scan. The guess hit an - // existing commit, so no head scan was needed. - expect(driver.calls.getMany).toBeLessThanOrEqual(1 + 4) - expect(driver.calls.scan).toBe(1) - }) -}) - -describe('concurrency', () => { - it('a stale writer collides with the existing commit key', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await a.append(['a1']) - await b.append(['b1']) // a's cached head is now stale - await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 }) - // 'any' retries after refreshing the head - expect(await a.append(['a3'])).toEqual({ version: 3, commit: 3 }) - expect((await collect(b.read())).map((e) => e.data)).toEqual(['a1', 'b1', 'a3']) - }) - - it('a stale cached head is refreshed before rejecting expectedVersion', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await a.append(['a1']) - await b.append(['b1']) - expect(await a.append(['a2'], { expectedVersion: 2 })).toEqual({ version: 3, commit: 3 }) - }) - - it('interleaved writers from two instances produce a dense, non-overlapping log', async () => { - const inner = new InMemoryKVDriver() - const a = setup([], inner).events.child('s') - const b = setup([], inner).events.child('s') - await Promise.all([a.append(['a1']), b.append(['b1']), a.append(['a2', 'a3']), b.append(['b2'])]) - const log = await collect(a.read()) - expect(log.map((e) => e.version)).toEqual([1, 2, 3, 4, 5]) - expect(new Set(log.map((e) => e.data))).toEqual(new Set(['a1', 'a2', 'a3', 'b1', 'b2'])) - }) - - it('exactly one of several optimistic writers at the same version wins', async () => { - const inner = new InMemoryKVDriver() - const writers = [0, 1, 2, 3].map(() => setup([], inner).events.child('s')) - const results = await Promise.allSettled(writers.map((w, i) => w.append([i], { expectedVersion: 0 }))) - expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1) - expect(results.filter((r) => r.status === 'rejected').every((r) => (r as PromiseRejectedResult).reason instanceof ConcurrencyError)).toBe(true) - }) - - it('without setIfAbsent, an in-process mutex serialises appends (single-writer mode)', async () => { - const { events, driver } = setup(['setIfAbsent']) - const s = events.child('s') - await Promise.all(Array.from({ length: 20 }, (_, i) => s.append([i]))) - expect((await collect(s.read())).map((e) => e.version)).toEqual(Array.from({ length: 20 }, (_, i) => i + 1)) - expect(driver.calls.setIfAbsent).toBeUndefined() - await expect(Promise.all([s.append(['x'], { expectedVersion: 20 }), s.append(['y'], { expectedVersion: 20 })])).rejects.toBeInstanceOf(ConcurrencyError) - }) -}) diff --git a/test/helpers.ts b/test/helpers.ts index fbb656f..0c00824 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,83 +1,45 @@ -import { BaseKVDriver, type BatchOp, type KVDriver, type ScanOptions } from '../src/kv/driver.js' -import { InMemoryKVDriver } from '../src/kv/memory.js' +import { BaseKVStorage, type ScanOptions } from '../src/kv-storage/base-kv-storage.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' -/** A minimal driver implementing only the four required methods, over a plain Map. */ -export class MinimalDriver extends BaseKVDriver { - readonly map = new Map() - async getMany(keys: string[]) { - return keys.map((k) => (this.map.has(k) ? JSON.parse(this.map.get(k)!) : undefined)) - } - async setMany(entries: [string, unknown][]) { - for (const [k, v] of entries) this.map.set(k, JSON.stringify(v)) - } - async deleteMany(keys: string[]) { - for (const k of keys) this.map.delete(k) - } - async *scan({ gte, lt, limit = Infinity, reverse, values }: ScanOptions): AsyncGenerator<[string, unknown]> { - let keys = [...this.map.keys()].filter((k) => (gte === undefined || k >= gte) && (lt === undefined || k < lt)).sort() - if (reverse) keys.reverse() - keys = keys.slice(0, limit) - for (const k of keys) { - if (!this.map.has(k)) continue - yield [k, values ? JSON.parse(this.map.get(k)!) : undefined] - } - } +export async function collect(iterable: AsyncIterable): Promise { + const out: T[] = [] + for await (const item of iterable) out.push(item) + return out } -export type Capability = 'setIfAbsent' | 'batch' | 'deleteRange' - /** - * Wraps a driver, counting calls per method and optionally hiding optional capabilities. - * `scan` counts each scan started (i.e. each seek). + * Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern + * (e.g. an encryption layer): extend BaseKVStorage and delegate the five primitives. */ -export class CountingDriver implements KVDriver { +export class KVStorageCounting extends BaseKVStorage { calls: Record = {} - setIfAbsent?: KVDriver['setIfAbsent'] - batch?: KVDriver['batch'] - deleteRange?: KVDriver['deleteRange'] - constructor( - readonly inner: KVDriver = new InMemoryKVDriver(), - hide: Capability[] = [], - ) { - if (inner.setIfAbsent && !hide.includes('setIfAbsent')) this.setIfAbsent = (e: [string, unknown][]) => (this.#count('setIfAbsent'), inner.setIfAbsent!(e)) - if (inner.batch && !hide.includes('batch')) this.batch = (ops: BatchOp[]) => (this.#count('batch'), inner.batch!(ops)) - if (inner.deleteRange && !hide.includes('deleteRange')) - this.deleteRange = (gte: string, lt: string) => (this.#count('deleteRange'), inner.deleteRange!(gte, lt)) - } - - #count(m: string): void { - this.calls[m] = (this.calls[m] ?? 0) + 1 + constructor(readonly inner: BaseKVStorage = new KVStorageInMemory()) { + super() } reset(): void { this.calls = {} } - get total(): number { - return Object.values(this.calls).reduce((a, b) => a + b, 0) + readMany(keys: string[]) { + return this.count('readMany', () => this.inner.readMany(keys)) + } + writeMany(entries: [string, unknown][]) { + return this.count('writeMany', () => this.inner.writeMany(entries)) + } + removeMany(keys: string[]) { + return this.count('removeMany', () => this.inner.removeMany(keys)) + } + writeIfAbsent(entries: [string, unknown][]) { + return this.count('writeIfAbsent', () => this.inner.writeIfAbsent(entries)) + } + scan(options?: ScanOptions) { + return this.count('scan', () => this.inner.scan(options)) } - getMany(keys: string[]) { - this.#count('getMany') - return this.inner.getMany(keys) - } - setMany(entries: [string, unknown][]) { - this.#count('setMany') - return this.inner.setMany(entries) - } - deleteMany(keys: string[]) { - this.#count('deleteMany') - return this.inner.deleteMany(keys) - } - scan(opts: ScanOptions) { - this.#count('scan') - return this.inner.scan(opts) + private count(name: string, fn: () => T): T { + this.calls[name] = (this.calls[name] ?? 0) + 1 + return fn() } } - -export async function collect(it: AsyncIterable): Promise { - const out: T[] = [] - for await (const x of it) out.push(x) - return out -} diff --git a/test/http/http.test.ts b/test/http/http.test.ts deleted file mode 100644 index 274efaf..0000000 --- a/test/http/http.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import type { KVDriver } from '../../src/kv/driver.js' -import { KVStorage } from '../../src/kv/node.js' -import { KVEventStorage } from '../../src/events/kv.js' -import { EventSourcedDocumentStorage } from '../../src/documents/event-sourced.js' -import { createKVHttpHandler } from '../../src/http/handler.js' -import { HttpKVDriver } from '../../src/http/client.js' -import { runKVDriverConformance } from '../../src/testing/kv-conformance.js' -import { collect, CountingDriver, MinimalDriver } from '../helpers.js' -import { serve } from './node-server.js' - -/** A client wired straight to a handler, no socket. */ -function inProcess(server: KVDriver = new InMemoryKVDriver(), caps?: ConstructorParameters[0]['capabilities']) { - const handler = createKVHttpHandler(server, { basePath: '/kv' }) - return new HttpKVDriver({ url: 'http://test/kv', capabilities: caps, fetch: async (input, init) => handler(new Request(input, init)) }) -} - -runKVDriverConformance('HttpKVDriver (in-process)', () => inProcess()) -runKVDriverConformance('HttpKVDriver (minimal server driver, discovered capabilities)', () => { - const handler = createKVHttpHandler(new MinimalDriver()) - return HttpKVDriver.connect({ url: 'http://test', fetch: async (input, init) => handler(new Request(input, init)) }) -}) - -describe('over a real socket', () => { - let server: Awaited> - const backends: InMemoryKVDriver[] = [] - beforeAll(async () => { - // One server; each driver gets a fresh backend selected by path. - server = await serve(async (req) => { - const m = /^\/db(\d+)/.exec(new URL(req.url).pathname) - const backend = backends[Number(m?.[1])] - if (!backend) return new Response('no db', { status: 404 }) - return createKVHttpHandler(backend, { basePath: `/db${m![1]}` })(req) - }) - }) - afterAll(() => server.close()) - - const fresh = () => { - backends.push(new InMemoryKVDriver()) - return HttpKVDriver.connect({ url: `${server.url}/db${backends.length - 1}` }) - } - - runKVDriverConformance('HttpKVDriver (node:http)', fresh) - - it('cancels a server-side scan when the client stops early', async () => { - const d = await fresh() - await d.setMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number])) - const first: string[] = [] - for await (const [k] of d.scan({})) { - first.push(k) - if (first.length === 3) break - } - expect(first).toEqual(['k00000', 'k00001', 'k00002']) - }) - - it('runs the whole stack (KV indexes, events, documents) over HTTP', async () => { - const kv = new KVStorage(await fresh(), { indexes: { users: { byAge: (u: { age: number }) => u.age } } }) - await kv.child('users').set('1', { age: 30 }) - expect(await collect(kv.child('users').findBy('byAge', { gte: 18 }))).toEqual([['1', { age: 30 }]]) - const docs = new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: { people: [{ fields: ['email'], unique: true }] } }) - const id = await docs.child('people').insert({ email: 'a@x', n: 1 }) - await docs.child('people').update(id, { $inc: { n: 1 } }) - await expect(docs.child('people').insert({ email: 'a@x' })).rejects.toThrow(/Unique/) - expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } }) - }) -}) - -describe('handler', () => { - it('reports capabilities and rejects unsupported operations with 501', async () => { - const handler = createKVHttpHandler(new MinimalDriver()) - const caps = await (await handler(new Request('http://x/capabilities'))).json() - expect(caps).toEqual({ setIfAbsent: false, batch: false, deleteRange: true }) - const res = await handler(new Request('http://x/batch', { method: 'POST', body: JSON.stringify({ ops: [] }) })) - expect(res.status).toBe(501) - }) - - it('maps driver errors to 500 and unknown routes to 404', async () => { - const broken = new CountingDriver(new InMemoryKVDriver()) - broken.getMany = () => Promise.reject(new Error('boom')) - const handler = createKVHttpHandler(broken) - expect((await handler(new Request('http://x/getMany', { method: 'POST', body: '{"keys":["a"]}' }))).status).toBe(500) - expect((await handler(new Request('http://x/nope'))).status).toBe(404) - await expect(inProcess(new MinimalDriver(), { batch: true }).batch!([])).rejects.toThrow(/HTTP 501/) - }) - - it('distinguishes null values from missing keys', async () => { - const d = inProcess() - await d.setMany([['n', null]]) - expect(await d.getMany(['n', 'missing'])).toEqual([null, undefined]) - }) -}) diff --git a/test/http/node-server.ts b/test/http/node-server.ts deleted file mode 100644 index 05b1551..0000000 --- a/test/http/node-server.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { createServer, type IncomingHttpHeaders } from 'node:http' -import type { AddressInfo } from 'node:net' - -/** Serves a `Request → Response` handler over a real socket (test helper). */ -export async function serve(handler: (req: Request) => Promise): Promise<{ url: string; close: () => Promise }> { - const server = createServer(async (req, res) => { - const chunks: Buffer[] = [] - for await (const c of req) chunks.push(c as Buffer) - const headers = Object.entries(req.headers as IncomingHttpHeaders).flatMap(([k, v]) => (v === undefined ? [] : [[k, String(v)] as [string, string]])) - const request = new Request(`http://localhost${req.url}`, { - method: req.method, - headers, - body: req.method === 'GET' || req.method === 'HEAD' ? undefined : Buffer.concat(chunks), - }) - const response = await handler(request) - res.writeHead(response.status, Object.fromEntries(response.headers)) - if (!response.body) return void res.end() - const reader = response.body.getReader() - res.on('close', () => void reader.cancel().catch(() => {})) - try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - res.write(value) - } - } finally { - res.end() - } - }) - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - const { port } = server.address() as AddressInfo - return { - url: `http://127.0.0.1:${port}`, - close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))), - } -} diff --git a/test/kv-storage/kv-storage-http.test.ts b/test/kv-storage/kv-storage-http.test.ts new file mode 100644 index 0000000..8a5a3b9 --- /dev/null +++ b/test/kv-storage/kv-storage-http.test.ts @@ -0,0 +1,54 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js' +import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js' +import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' +import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' +import { serve } from './node-server.js' + +runKVStorageConformance('KVStorageHttp (in-process)', () => { + const server = new KVStorageHttpServer(new KVStorageInMemory(), '/kv') + return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) }) +}) + +describe('over a real socket', () => { + let server: Awaited> + const stores: KVStorageHttpServer[] = [] + beforeAll(async () => { + // One socket; each test storage gets its own backend under /. + server = await serve((request) => stores[Number(new URL(request.url).pathname.split('/')[1])]!.handle(request)) + }) + afterAll(() => server.close()) + const fresh = () => { + stores.push(new KVStorageHttpServer(new KVStorageInMemory(), `/${stores.length}`)) + return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` }) + } + + runKVStorageConformance('KVStorageHttp (node:http)', fresh) + + it('stops a long scan early', async () => { + const db = fresh() + await db.writeMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number])) + const seen: string[] = [] + for await (const [key] of db.scan()) if (seen.push(key) === 3) break + expect(seen).toEqual(['k00000', 'k00001', 'k00002']) + }) + + it('runs events and event-sourced documents over HTTP', async () => { + const kv = fresh() + const docs = new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))) + const id = await docs.child('people').insert({ email: 'a@x', n: 1 }) + await docs.child('people').update(id, { $inc: { n: 1 } }) + expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } }) + }) +}) + +describe('server', () => { + it('maps errors to 404 / 500', async () => { + const server = new KVStorageHttpServer(new KVStorageInMemory()) + expect((await server.handle(new Request('http://x/nope', { method: 'POST', body: '{}' }))).status).toBe(404) + expect((await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys": 5}' }))).status).toBe(500) + }) +}) diff --git a/test/kv-storage/kv-storage-in-memory.test.ts b/test/kv-storage/kv-storage-in-memory.test.ts new file mode 100644 index 0000000..68c09a5 --- /dev/null +++ b/test/kv-storage/kv-storage-in-memory.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js' +import { collect, KVStorageCounting } from '../helpers.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' + +runKVStorageConformance('KVStorageInMemory', () => new KVStorageInMemory()) +runKVStorageConformance('wrapper (KVStorageCounting)', () => new KVStorageCounting()) + +describe('costs', () => { + it('single-key operations are one primitive call', async () => { + const db = new KVStorageCounting() + const users = db.child('users') + await users.set('1', 'x') + await users.get('1') + await users.child('1').get() + await users.delete('1') + expect(db.calls).toEqual({ writeMany: 1, readMany: 2, removeMany: 1 }) + }) + + it('keys() seeks once per child with descendants, not once per key', async () => { + const db = new KVStorageCounting() + const writes: [string, number][] = [] + for (let c = 0; c < 50; c++) for (let d = 0; d < 20; d++) writes.push([`n/c${c}/d${d}`, d]) + for (let c = 0; c < 50; c++) writes.push([`n/leaf${c}`, c]) + await db.writeMany(writes) + db.reset() + expect(await collect(db.child('n').keys())).toHaveLength(100) + expect(db.calls).toEqual({ scan: 51 }) + }) +}) + +describe('model-based: random writes vs a reference map', () => { + const segment = fc.constantFrom('a', 'a-1', 'a.b', 'b', 'x/y', '%', '~e', '1', '10', '9', 'ab', 'a!', 'é') + const path = fc.array(segment, { minLength: 1, maxLength: 3 }) + const op = fc.oneof( + fc.record({ t: fc.constant('set' as const), path, value: fc.integer() }), + fc.record({ t: fc.constant('delete' as const), path }), + fc.record({ t: fc.constant('deleteTree' as const), path }), + ) + const at = (db: BaseKVStorage, p: string[]) => p.reduce((n, s) => n.child(s), db) + + it('get, keys and entries agree with the model', async () => { + await fc.assert( + fc.asyncProperty(fc.array(op, { maxLength: 30 }), fc.array(segment, { maxLength: 2 }), async (ops, parent) => { + const db = new KVStorageInMemory() + const model = new Map() + const isUnder = (p: string[], q: string[]) => q.length >= p.length && p.every((s, i) => q[i] === s) + for (const o of ops) { + if (o.t === 'set') (await at(db, o.path).set(o.value), model.set(JSON.stringify(o.path), o.value)) + else if (o.t === 'delete') (await at(db, o.path).delete(), model.delete(JSON.stringify(o.path))) + else { + await at(db, o.path).deleteTree() + for (const k of [...model.keys()]) if (isUnder(o.path, JSON.parse(k))) model.delete(k) + } + } + for (const [k, v] of model) expect(await at(db, JSON.parse(k)).get()).toBe(v) + const children = new Map() + for (const [k, v] of model) { + const p = JSON.parse(k) as string[] + if (p.length <= parent.length || !isUnder(parent, p)) continue + const own = p.length === parent.length + 1 + if (own || !children.has(p[parent.length]!)) children.set(p[parent.length]!, own ? v : children.get(p[parent.length]!)) + } + const node = at(db, parent) + const listed = await collect(node.keys()) + expect([...listed].sort()).toEqual([...children.keys()].sort()) + expect(await collect(node.keys({ reverse: true }))).toEqual([...listed].reverse()) + const withValues = listed.filter((k) => children.get(k) !== undefined) + expect(await collect(node.entries())).toEqual(withValues.map((k) => [k, children.get(k)])) + expect(await collect(node.entries({ reverse: true }))).toEqual(withValues.reverse().map((k) => [k, children.get(k)])) + for (let i = 0; i < listed.length; i++) expect(await collect(node.keys({ after: listed[i]! }))).toEqual(listed.slice(i + 1)) + }), + { numRuns: 300 }, + ) + }) +}) diff --git a/test/kv-storage/kv-storage.conformance.ts b/test/kv-storage/kv-storage.conformance.ts new file mode 100644 index 0000000..939afe4 --- /dev/null +++ b/test/kv-storage/kv-storage.conformance.ts @@ -0,0 +1,168 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import type { BaseKVStorage, ScanOptions } from '../../src/kv-storage/base-kv-storage.js' +import { InvalidKeyError } from '../../src/utils/errors.js' +import { collect } from '../helpers.js' + +/** Behaviour every KV storage must have. `make` returns an empty storage. */ +export function runKVStorageConformance(name: string, make: () => BaseKVStorage | Promise): void { + describe(`KV storage conformance: ${name}`, () => { + let db: BaseKVStorage + beforeEach(async () => { + db = await make() + }) + const scanKeys = async (options: ScanOptions) => (await collect(db.scan(options))).map(([k]) => k) + + describe('primitives', () => { + beforeEach(async () => { + await db.writeMany([ + ['b', 2], + ['a/2', { n: 'a2' }], + ['c', [3]], + ['a', 1], + ['a/1', 'a1'], + ['a0', null], + ['10', 10], + ['9', 9], + ]) + }) + + it('reads in request order with undefined for missing keys and null preserved', async () => { + expect(await db.readMany(['c', 'missing', 'a/2', 'a0'])).toEqual([[3], undefined, { n: 'a2' }, null]) + }) + + it('scans in key order with bounds, reverse, limit and optional values', async () => { + expect(await scanKeys({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c']) + expect(await scanKeys({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2']) + expect(await scanKeys({ reverse: true, gte: 'a', lt: 'b' })).toEqual(['a0', 'a/2', 'a/1', 'a']) + expect(await scanKeys({ limit: 2, reverse: true })).toEqual(['c', 'b']) + expect(await scanKeys({ limit: 0 })).toEqual([]) + expect(await collect(db.scan({ gte: 'a/', lt: 'a0', values: true }))).toEqual([ + ['a/1', 'a1'], + ['a/2', { n: 'a2' }], + ]) + expect((await collect(db.scan({ gte: 'b' })))[0]).toEqual(['b', undefined]) + }) + + it('removes keys', async () => { + await db.removeMany(['b', 'missing']) + expect(await db.readMany(['b', 'c'])).toEqual([undefined, [3]]) + }) + + it('writeIfAbsent writes only absent keys, and only one concurrent claim wins', async () => { + expect( + await db.writeIfAbsent([ + ['a', 'x'], + ['new', 'y'], + ]), + ).toEqual([false, true]) + expect(await db.readMany(['a', 'new'])).toEqual([1, 'y']) + const results = await Promise.all([1, 2, 3].map((n) => db.writeIfAbsent([['claim', n]]))) + expect(results.flat().filter(Boolean)).toHaveLength(1) + }) + + it('keeps scanning correctly when keys change mid-scan', async () => { + // Keys removed mid-scan may or may not still appear (stores may read ahead), but every key + // present throughout must appear exactly once, in order. + const seen: string[] = [] + for await (const [key] of db.scan({})) { + seen.push(key) + if (key === 'a') await db.removeMany(['a', 'a/1']) + } + expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c']) + }) + + it('does not share references with callers', async () => { + const [value] = (await db.readMany(['a/2'])) as [{ n: string }] + value.n = 'changed' + expect(await db.readMany(['a/2'])).toEqual([{ n: 'a2' }]) + }) + }) + + describe('nodes', () => { + it('reads and writes own and child values at the same key', async () => { + const users = db.child('users') + await users.set('123', { name: 'Ada' }) + expect(await users.child('123').get()).toEqual({ name: 'Ada' }) + await users.child('123').set({ name: 'Bob' }) + expect(await users.get('123')).toEqual({ name: 'Bob' }) + expect(await db.readMany(['users/123'])).toEqual([{ name: 'Bob' }]) + await db.child('n').set(5, 'five') + expect(await db.child('n').get('5')).toBe('five') + }) + + it('batch reads, writes and deletes children', async () => { + const n = db.child('n') + await n.setMany([ + ['a', 1], + ['b', 2], + ]) + expect(await n.getMany(['b', 'x', 'a'])).toEqual([2, undefined, 1]) + await n.deleteMany(['a']) + expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2]) + }) + + it('keeps own values and children independent', async () => { + const user = db.child('users').child('1') + await user.set('own') + await user.set('name', 'Ada') + await user.delete() + expect(await user.get()).toBeUndefined() + expect(await user.get('name')).toBe('Ada') + }) + + it('rejects passed-but-undefined keys, root values and undefined values', async () => { + const users = db.child('users') + const id: string | undefined = undefined + await expect(users.get(id as never)).rejects.toThrow(InvalidKeyError) + await expect(users.delete(null as never)).rejects.toThrow(TypeError) + await expect(users.set(id as never, 1 as never)).rejects.toThrow(TypeError) + await expect(users.set('x', undefined)).rejects.toThrow(/undefined/) + await expect(db.get()).rejects.toThrow(/root/) + expect(() => db.child('')).toThrow(InvalidKeyError) + }) + + it('escapes keys so special characters round-trip', async () => { + const n = db.child('a/b') + for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) await n.set(key, key) + for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) expect(await n.get(key)).toBe(key) + expect((await collect(n.keys())).sort()).toEqual(['50%', 'a b', 'a-1', 'a.b', 'x/y', '~e', 'é'].sort()) + }) + + it('lists direct children (including ones with only descendants), skipping reserved keys', async () => { + const n = db.child('n') + await n.set('a', 1) + await n.child('b').set('deep', 1) + await n.set('c', 3) + await n.child('c').child('x').set('y', 1) + await n.set('a-1', 4) + await db.writeMany([['n/~e/000000000001', {}]]) + expect(await collect(n.keys())).toEqual(['a', 'a-1', 'b', 'c']) + expect(await collect(n.keys({ reverse: true }))).toEqual(['c', 'b', 'a-1', 'a']) + expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['a-1', 'b']) + expect(await collect(n.keys({ after: 'c', reverse: true }))).toEqual(['b', 'a-1', 'a']) + expect(await collect(n.entries())).toEqual([ + ['a', 1], + ['a-1', 4], + ['c', 3], + ]) + expect(await collect(n.entries({ reverse: true, limit: 2 }))).toEqual([ + ['c', 3], + ['a-1', 4], + ]) + expect(await collect(db.keys())).toEqual(['n']) + }) + + it('deleteTree removes a subtree but not lookalike siblings', async () => { + const n = db.child('n') + await n.set('a', 1) + await n.child('a').child('b').set('c', 3) + await n.set('a!', 2) + await n.set('a0', 3) + await n.deleteTree('a') + expect(await collect(n.keys())).toEqual(['a0', 'a!']) // 'a!' is stored as 'a~21' + await db.deleteTree() + expect(await collect(db.scan())).toEqual([]) + }) + }) + }) +} diff --git a/test/kv-storage/node-server.ts b/test/kv-storage/node-server.ts new file mode 100644 index 0000000..8660188 --- /dev/null +++ b/test/kv-storage/node-server.ts @@ -0,0 +1,24 @@ +import { createServer } from 'node:http' +import type { AddressInfo } from 'node:net' + +/** Serves a `Request → Response` handler over a real socket (test helper). */ +export async function serve(handle: (request: Request) => Promise): Promise<{ url: string; close: () => Promise }> { + const server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(chunk as Buffer) + const response = await handle( + new Request(`http://localhost${req.url}`, { method: req.method, body: req.method === 'POST' ? Buffer.concat(chunks) : undefined }), + ) + res.writeHead(response.status, Object.fromEntries(response.headers)) + if (!response.body) return void res.end() + const reader = response.body.getReader() + res.on('close', () => void reader.cancel().catch(() => {})) + for (let next = await reader.read(); !next.done; next = await reader.read()) res.write(next.value) + res.end() + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))), + } +} diff --git a/test/kv/indexes.test.ts b/test/kv/indexes.test.ts deleted file mode 100644 index ee57e37..0000000 --- a/test/kv/indexes.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage, type KVStorageOptions } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import type { KVDriver } from '../../src/kv/driver.js' -import { encodeIndexValue } from '../../src/core/encoding.js' -import { collect, CountingDriver, type Capability } from '../helpers.js' - -type User = { email?: string; age?: number; tags?: string[] } -type Post = { createdAt: string; tags: string[] } - -const indexes: KVStorageOptions['indexes'] = { - users: { - byEmail: (u: User) => u.email, - byAge: (u: User) => u.age, - byTag: (u: User) => u.tags, - }, - 'users/*/posts': { byDate: (p: Post) => p.createdAt, byTag: (p: Post) => p.tags }, -} - -function setup(hide: Capability[] = []) { - const driver = new CountingDriver(new InMemoryKVDriver(), hide) - return { driver, db: new KVStorage(driver, { indexes }) } -} - -const modes: [string, Capability[]][] = [ - ['atomic batch', []], - ['non-atomic', ['batch']], -] - -describe.each(modes)('indexes (%s)', (_, hide) => { - it('writes entries under /~i///', async () => { - const { db, driver } = setup(hide) - await db.child('users').set('123', { email: 'sada@x.io', age: 30 }) - const keys = (await collect(driver.inner.scan({}))).map(([k]) => k) - expect(keys).toEqual([`users/123`, `users/~i/byAge/${encodeIndexValue(30)}/123`, `users/~i/byEmail/${encodeIndexValue('sada@x.io')}/123`]) - }) - - it('finds by exact value and keeps entries in sync on update and delete', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('1', { email: 'a@x', age: 30 }) - await users.set('2', { email: 'b@x', age: 30 }) - expect(await collect(users.findBy('byEmail', 'a@x'))).toEqual([['1', { email: 'a@x', age: 30 }]]) - expect((await collect(users.findBy('byAge', 30))).map(([k]) => k)).toEqual(['1', '2']) - await users.set('1', { email: 'c@x', age: 31 }) - expect(await collect(users.findBy('byEmail', 'a@x'))).toEqual([]) - expect((await collect(users.findBy('byEmail', 'c@x'))).map(([k]) => k)).toEqual(['1']) - await users.delete('2') - expect(await collect(users.findBy('byAge', 30))).toEqual([]) - }) - - it('self-form writes on a child update the parent index', async () => { - const { db } = setup(hide) - const u = db.child('users').child('7') - await u.set({ email: 'self@x' }) - expect((await collect(db.child('users').findBy('byEmail', 'self@x'))).map(([k]) => k)).toEqual(['7']) - await u.delete() - expect(await collect(db.child('users').findBy('byEmail', 'self@x'))).toEqual([]) - }) - - it('deleteTree removes the child entry', async () => { - const { db, driver } = setup(hide) - await db.child('users').set('7', { email: 'e' }) - await db.child('users').child('7').child('posts').set('p', { createdAt: '2024', tags: [] }) - await db.child('users').deleteTree('7') - expect(await collect(driver.inner.scan({}))).toEqual([]) - }) - - it('supports multi-entry values, de-duplicated in range queries', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('1', { tags: ['a', 'b', 'a'] }) - await users.set('2', { tags: ['b'] }) - expect((await collect(users.findBy('byTag', 'b'))).map(([k]) => k)).toEqual(['1', '2']) - expect((await collect(users.findBy('byTag', { gte: 'a', lt: 'c' }))).map(([k]) => k)).toEqual(['1', '2']) - await users.set('1', { tags: ['c'] }) - expect((await collect(users.findBy('byTag', 'b'))).map(([k]) => k)).toEqual(['2']) - }) - - it('range queries are ordered by value and honour bounds, limit and reverse', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.setMany([ - ['a', { age: 10 }], - ['b', { age: 9 }], - ['c', { age: -1.5 }], - ['d', { age: 100 }], - ['e', { age: 20 }], - ]) - const ages = async (q: object, o = {}) => (await collect(users.findBy('byAge', q as never, o))).map(([, v]) => (v as User).age) - expect(await ages({})).toEqual([-1.5, 9, 10, 20, 100]) - expect(await ages({ gte: 9, lt: 20 })).toEqual([9, 10]) - expect(await ages({ gt: 9, lte: 20 })).toEqual([10, 20]) - expect(await ages({ gte: 0 }, { limit: 2 })).toEqual([9, 10]) - expect(await ages({ lt: 100 }, { reverse: true })).toEqual([20, 10, 9, -1.5]) - }) - - it('applies pattern indexes with * to nested nodes only', async () => { - const { db } = setup(hide) - const posts = db.child('users').child('u1').child('posts') - await posts.set('p1', { createdAt: '2024-02', tags: ['x'] }) - await posts.set('p2', { createdAt: '2024-01', tags: ['y'] }) - expect((await collect(posts.findBy('byDate', { gte: '2024' }))).map(([k]) => k)).toEqual(['p2', 'p1']) - await expect(db.child('other').findBy('byDate', 'x').next()).rejects.toThrow(/No index/) - }) - - it('handles child keys and values needing escaping', async () => { - const { db } = setup(hide) - const users = db.child('users') - await users.set('a/b%~', { email: 'we/ird%~@x' }) - expect(await collect(users.findBy('byEmail', 'we/ird%~@x'))).toEqual([['a/b%~', { email: 'we/ird%~@x' }]]) - }) -}) - -describe('index costs', () => { - it('non-indexed nodes do not read before writing', async () => { - const { db, driver } = setup() - driver.reset() - await db.child('other').set('1', { email: 'x' }) - await db.child('other').setMany([['2', {}]]) - expect(driver.calls).toEqual({ setMany: 2 }) - }) - - it('indexed writes: one getMany then one atomic batch', async () => { - const { db, driver } = setup() - driver.reset() - await db.child('users').setMany([ - ['1', { email: 'a' }], - ['2', { email: 'b' }], - ]) - expect(driver.calls).toEqual({ getMany: 1, batch: 1 }) - }) - - it('findBy with a small limit makes one value read', async () => { - const { db, driver } = setup() - await db.child('users').setMany(Array.from({ length: 300 }, (_, i) => [String(i), { age: 1 }] as [string, User])) - driver.reset() - expect(await collect(db.child('users').findBy('byAge', 1, { limit: 1 }))).toHaveLength(1) - expect(driver.calls).toEqual({ scan: 1, getMany: 1 }) - driver.reset() - expect(await collect(db.child('users').findBy('byAge', 1))).toHaveLength(300) - expect(driver.calls).toEqual({ scan: 1, getMany: 3 }) - }) -}) - -describe('non-atomic crash recovery (read-repair)', () => { - /** Fails the n-th mutating call. */ - class CrashingDriver extends CountingDriver { - failAt = Infinity - #n = 0 - override setMany(entries: [string, unknown][]) { - if (++this.#n === this.failAt) return Promise.reject(new Error('crash')) - return super.setMany(entries) - } - override deleteMany(keys: string[]) { - if (++this.#n === this.failAt) return Promise.reject(new Error('crash')) - return super.deleteMany(keys) - } - } - - it('skips and deletes stale entries left by a crash after the index add', async () => { - const driver = new CrashingDriver(new InMemoryKVDriver(), ['batch']) - const users = new KVStorage(driver, { indexes }).child('users') - await users.set('1', { email: 'old' }) - driver.failAt = 2 + 2 // adds succeed, value write crashes - await expect(users.set('1', { email: 'new' })).rejects.toThrow('crash') - // The 'new' entry is a dangling hint; the value still says 'old'. - expect(await collect(users.findBy('byEmail', 'new'))).toEqual([]) - expect(await collect(users.findBy('byEmail', 'old'))).toEqual([['1', { email: 'old' }]]) - const entries = (await collect(driver.inner.scan({ gte: 'users/~i/' }))).map(([k]) => k) - expect(entries).toEqual([`users/~i/byEmail/${encodeIndexValue('old')}/1`]) - }) - - it('skips stale entries left by a crash before the index remove', async () => { - const driver = new CrashingDriver(new InMemoryKVDriver(), ['batch']) - const users = new KVStorage(driver, { indexes }).child('users') - await users.set('1', { email: 'old' }) - driver.failAt = 2 + 3 // adds + value succeed, remove crashes - await expect(users.set('1', { email: 'new' })).rejects.toThrow('crash') - expect(await collect(users.findBy('byEmail', 'old'))).toEqual([]) - expect(await collect(users.findBy('byEmail', 'new'))).toEqual([['1', { email: 'new' }]]) - }) - - it('can leave stale entries in place with readRepair: false', async () => { - const driver = new InMemoryKVDriver() - await driver.setMany([[`users/~i/byEmail/${encodeIndexValue('x')}/1`, true]]) - const users = new KVStorage(driver, { indexes, readRepair: false }).child('users') - expect(await collect(users.findBy('byEmail', 'x'))).toEqual([]) - expect(await collect(driver.scan({}))).toHaveLength(1) - }) -}) - -describe('index declarations', () => { - it('rejects the same index name on overlapping patterns', () => { - expect(() => new KVStorage(new InMemoryKVDriver(), { indexes: { 'a/*': { i: () => 1 }, '*/b': { i: () => 2 } } })).toThrow(/overlapping/) - expect(() => new KVStorage(new InMemoryKVDriver(), { indexes: { 'a/*': { i: () => 1 }, '*/b': { j: () => 2 } } })).not.toThrow() - }) - - it('rejects non-indexable index function results', async () => { - const db = new KVStorage(new InMemoryKVDriver(), { indexes: { '': { bad: () => ({}) as never } } }) - await expect(db.set('x', 1)).rejects.toThrow(/non-indexable/) - }) -}) - -describe('property: findBy equals brute-force filter', () => { - const val = fc.record({ - age: fc.option(fc.integer({ min: -3, max: 3 }), { nil: undefined }), - tags: fc.array(fc.constantFrom('a', 'b', 'c/d'), { maxLength: 3 }), - }) - const op = fc.oneof( - fc.record({ t: fc.constant('set' as const), k: fc.constantFrom('1', '2', '3', 'x/y'), v: val }), - fc.record({ t: fc.constant('del' as const), k: fc.constantFrom('1', '2', '3', 'x/y') }), - ) - for (const [mode, hide] of modes) { - it(mode, async () => { - await fc.assert( - fc.asyncProperty(fc.array(op, { maxLength: 20 }), fc.integer({ min: -4, max: 4 }), fc.integer({ min: -4, max: 4 }), async (ops, lo, hi) => { - const driver: KVDriver = new CountingDriver(new InMemoryKVDriver(), hide) - const users = new KVStorage(driver, { indexes }).child('users') - const model = new Map() - for (const o of ops) { - if (o.t === 'set') { - await users.set(o.k, o.v) - model.set(o.k, o.v) - } else { - await users.delete(o.k) - model.delete(o.k) - } - } - const got = (await collect(users.findBy('byAge', { gte: lo, lt: hi }))).map(([k]) => k).sort() - const exp = [...model] - .filter(([, v]) => v.age !== undefined && v.age >= lo && v.age < hi) - .map(([k]) => k) - .sort() - expect(got).toEqual(exp) - for (const tag of ['a', 'b', 'c/d']) { - const g = (await collect(users.findBy('byTag', tag))).map(([k]) => k).sort() - expect(g).toEqual( - [...model] - .filter(([, v]) => v.tags?.includes(tag)) - .map(([k]) => k) - .sort(), - ) - } - // No stale entries are ever left behind by clean writes. - let entryCount = 0 - for await (const _ of driver.scan({ gte: 'users/~i/', lt: 'users/~i0' })) entryCount++ - const expectedEntries = [...model.values()].reduce((n, v) => n + (v.age !== undefined ? 1 : 0) + new Set(v.tags).size, 0) - expect(entryCount).toBe(expectedEntries) - }), - { numRuns: 200 }, - ) - }) - } -}) diff --git a/test/kv/memory-driver.test.ts b/test/kv/memory-driver.test.ts deleted file mode 100644 index df7fe71..0000000 --- a/test/kv/memory-driver.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { runKVDriverConformance } from '../../src/testing/kv-conformance.js' -import { CountingDriver, MinimalDriver } from '../helpers.js' - -runKVDriverConformance('InMemoryKVDriver', () => new InMemoryKVDriver()) -runKVDriverConformance('BaseKVDriver (minimal)', () => new MinimalDriver()) -runKVDriverConformance('CountingDriver(InMemory)', () => new CountingDriver()) - -describe('InMemoryKVDriver', () => { - it('keeps order after large bulk merges', async () => { - const d = new InMemoryKVDriver() - const keys = Array.from({ length: 5000 }, (_, i) => `k${(i * 7919) % 5000}`) - await d.setMany(keys.slice(0, 2500).map((k) => [k, 1])) - await d.setMany(keys.slice(2500).map((k) => [k, 2])) - const seen: string[] = [] - for await (const [k] of d.scan({})) seen.push(k) - expect(seen).toEqual([...new Set(keys)].sort()) - expect(d.size).toBe(5000) - }) - - it('can skip cloning', async () => { - const d = new InMemoryKVDriver({ clone: false }) - const v = { n: 1 } - await d.setMany([['k', v]]) - expect((await d.getMany(['k']))[0]).toBe(v) - }) -}) diff --git a/test/kv/node.test.ts b/test/kv/node.test.ts deleted file mode 100644 index 52af85f..0000000 --- a/test/kv/node.test.ts +++ /dev/null @@ -1,354 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { KVStorage } from '../../src/kv/node.js' -import { InMemoryKVDriver } from '../../src/kv/memory.js' -import { escapeSegment } from '../../src/core/path.js' -import { InvalidKeyError } from '../../src/errors.js' -import { collect, CountingDriver, MinimalDriver } from '../helpers.js' - -function setup() { - const driver = new CountingDriver() - const db = new KVStorage(driver) - return { driver, db } -} - -describe('KVNode basics', () => { - it('reads and writes own and child values at the same key', async () => { - const { db } = setup() - const users = db.child('users') - await users.set('123', { name: 'Ada' }) - expect(await users.get('123')).toEqual({ name: 'Ada' }) - expect(await users.child('123').get()).toEqual({ name: 'Ada' }) - await users.child('123').set({ name: 'Bob' }) - expect(await users.get('123')).toEqual({ name: 'Bob' }) - expect(users.child('123').selfKey).toBe('users/123') - }) - - it('treats number keys as strings', async () => { - const { db } = setup() - await db.child('n').set(5, 'five') - expect(await db.child('n').get('5')).toBe('five') - }) - - it('escapes keys containing the separator, % and leading ~', async () => { - const { db, driver } = setup() - const n = db.child('a/b') - await n.set('~x', 1) - await n.set('50%', 2) - expect(await collect(driver.inner.scan({}))).toEqual([ - ['a%2Fb/%7Ex', undefined], - ['a%2Fb/50%25', undefined], - ]) - // Ordered by escaped segment: '%7Ex' < '50%25'. - expect(await collect(n.keys())).toEqual(['~x', '50%']) - }) - - it('own value and children are independent keys', async () => { - const { db } = setup() - const u = db.child('users').child('1') - await u.set('own') - await u.set('name', 'Ada') - await u.delete() - expect(await u.get()).toBeUndefined() - expect(await u.get('name')).toBe('Ada') - }) - - it('getMany / setMany / deleteMany', async () => { - const { db, driver } = setup() - const n = db.child('n') - driver.reset() - await n.setMany([ - ['a', 1], - ['b', 2], - ]) - expect(await n.getMany(['b', 'x', 'a'])).toEqual([2, undefined, 1]) - await n.deleteMany(['a', 'x']) - expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2]) - expect(driver.calls).toEqual({ setMany: 1, getMany: 2, deleteMany: 1 }) - }) - - it('single-key ops make exactly one driver call', async () => { - const { db, driver } = setup() - const n = db.child('users') - driver.reset() - await n.set('1', 'x') - await n.get('1') - await n.child('1').get() - await n.delete('1') - expect(driver.calls).toEqual({ setMany: 1, getMany: 2, deleteMany: 1 }) - }) - - it('rejects setting undefined', async () => { - const { db } = setup() - await expect(db.child('a').set('b', undefined)).rejects.toThrow(TypeError) - }) -}) - -describe('arity dispatch', () => { - it('throws on passed-but-undefined or null keys instead of reading self', async () => { - const { db } = setup() - const users = db.child('users') - const maybe: string | undefined = undefined - await expect(users.get(maybe as never)).rejects.toThrow(InvalidKeyError) - await expect(users.get(null as never)).rejects.toThrow(TypeError) - await expect(users.set(maybe as never, 1 as never)).rejects.toThrow(TypeError) - await expect(users.delete(maybe as never)).rejects.toThrow(TypeError) - await expect(users.getMany([maybe as never])).rejects.toThrow(TypeError) - await expect(users.deleteTree(maybe as never)).rejects.toThrow(TypeError) - expect(() => users.child(maybe as never)).toThrow(TypeError) - }) - - it('root has no own value', async () => { - const { db } = setup() - await expect(db.get()).rejects.toThrow(TypeError) - await expect(db.set(1)).rejects.toThrow(TypeError) - await expect(db.delete()).rejects.toThrow(TypeError) - }) - - it('rejects empty keys', () => { - const { db } = setup() - expect(() => db.child('')).toThrow(InvalidKeyError) - }) -}) - -describe('keys / entries', () => { - it('lists direct children, including ones with only descendants, skipping reserved', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.child('b').set('deep', 1) - await n.set('c', 3) - await n.child('c').child('x').set('y', 1) - await db.driver.setMany([['n/~i/idx/v/a', true]]) - await n.set('\u{1F600}', 'emoji') - expect(await collect(n.keys())).toEqual(['a', 'b', 'c', '\u{1F600}']) - expect(await collect(n.keys({ reverse: true }))).toEqual(['\u{1F600}', 'c', 'b', 'a']) - expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['b', 'c']) - expect(await collect(n.keys({ after: 'c', reverse: true }))).toEqual(['b', 'a']) - expect(await collect(n.entries())).toEqual([ - ['a', 1], - ['c', 3], - ['\u{1F600}', 'emoji'], - ]) - expect(await collect(n.entries({ reverse: true, limit: 2 }))).toEqual([ - ['\u{1F600}', 'emoji'], - ['c', 3], - ]) - }) - - it('root keys skip the reserved ~p/~e roots', async () => { - const { db } = setup() - await db.driver.setMany([ - ['~p/users/1', {}], - ['users/1', {}], - ]) - expect(await collect(db.keys())).toEqual(['users']) - }) - - it('orders siblings that extend another with a below-separator char first', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.set('a-1', 2) - await n.child('a').set('x', 3) - expect(await collect(n.keys())).toEqual(['a-1', 'a']) - expect(await collect(n.keys({ reverse: true }))).toEqual(['a', 'a-1']) - expect(await collect(n.entries())).toEqual([ - ['a-1', 2], - ['a', 1], - ]) - }) - - it('is a skip-scan: seeks are O(children), not O(subtree)', async () => { - const { db, driver } = setup() - const n = db.child('n') - const writes: [string, unknown][] = [] - for (let c = 0; c < 50; c++) { - writes.push([`n/c${String(c).padStart(3, '0')}`, c]) - for (let d = 0; d < 100; d++) writes.push([`n/c${String(c).padStart(3, '0')}/d${d}`, d]) - } - await driver.inner.setMany(writes) - driver.reset() - const keys = await collect(n.keys()) - expect(keys).toHaveLength(50) - // One initial scan plus one seek per child with descendants. - expect(driver.calls.scan).toBe(51) - expect(driver.calls.getMany).toBeUndefined() - }) - - it('siblings without descendants share one iterator', async () => { - const { db, driver } = setup() - const n = db.child('n') - await n.setMany(Array.from({ length: 100 }, (_, i) => [`k${i}`, i] as [string, number])) - driver.reset() - expect(await collect(n.keys())).toHaveLength(100) - expect(driver.calls.scan).toBe(1) - }) -}) - -describe('deleteTree', () => { - it('deletes own value and all descendants but not lookalike siblings', async () => { - const { db } = setup() - const n = db.child('n') - await n.set('a', 1) - await n.child('a').set('b', 2) - await n.child('a').child('b').set('c', 3) - await n.set('a!', 'sibling') - await n.set('a0', 'sibling') - await n.deleteTree('a') - expect(await collect(n.keys())).toEqual(['a!', 'a0']) - await n.child('a0').deleteTree() - expect(await collect(n.keys())).toEqual(['a!']) - }) - - it('works without deleteRange, and on root', async () => { - const driver = new CountingDriver(new InMemoryKVDriver(), ['deleteRange']) - const db = new KVStorage(driver) - for (let i = 0; i < 2500; i++) await db.child('x').child(String(i)).set('v', i) - await db.child('y').set(1, 1) - await db.child('x').deleteTree() - expect(await collect(db.keys())).toEqual(['y']) - await db.deleteTree() - expect(await collect(driver.scan({}))).toEqual([]) - }) -}) - -describe('tree', () => { - it('assembles descendants up to depth, field-per-key style', async () => { - const { db } = setup() - const p = db.child('profile').child('9') - await p.set('username', 'ada') - await p.set('age', 36) - await p.child('address').set('city', 'London') - await p.child('address').child('geo').set('lat', 51) - await p.child('address').set('own') - await db.driver.setMany([['profile/9/~i/x/y', true]]) - expect(await p.tree({ depth: 1 })).toEqual({ username: 'ada', age: 36, address: 'own' }) - expect(await p.tree({ depth: 2 })).toEqual({ username: 'ada', age: 36, address: { '': 'own', city: 'London', geo: undefined } }) - expect(await p.tree({ depth: 3 })).toEqual({ username: 'ada', age: 36, address: { '': 'own', city: 'London', geo: { lat: 51 } } }) - }) -}) - -// ------------------------------------------------------------------ model-based - -type Path = string[] -const SEP = '/' -const segArb = fc.constantFrom('a', 'a-1', 'a.b', 'b', 'a/x', '%', '~t', '1', '10', '9', 'ab', 'a!', 'é') -const pathArb = fc.array(segArb, { minLength: 1, maxLength: 3 }) -const valArb = fc.oneof(fc.integer(), fc.string(), fc.constant(null), fc.record({ n: fc.integer() })) - -type Op = - | { t: 'set'; path: Path; value: unknown; selfForm: boolean } - | { t: 'delete'; path: Path; selfForm: boolean } - | { t: 'deleteTree'; path: Path } - | { t: 'setMany'; parent: Path; entries: [string, unknown][] } - -const opArb: fc.Arbitrary = fc.oneof( - { weight: 5, arbitrary: fc.record({ t: fc.constant('set' as const), path: pathArb, value: valArb, selfForm: fc.boolean() }) }, - { weight: 2, arbitrary: fc.record({ t: fc.constant('delete' as const), path: pathArb, selfForm: fc.boolean() }) }, - { weight: 1, arbitrary: fc.record({ t: fc.constant('deleteTree' as const), path: pathArb }) }, - { - weight: 1, - arbitrary: fc.record({ - t: fc.constant('setMany' as const), - parent: fc.array(segArb, { maxLength: 2 }), - entries: fc.array(fc.tuple(segArb, valArb), { maxLength: 4 }), - }), - }, -) - -const enc = (p: Path) => JSON.stringify(p) -const isPrefix = (p: Path, q: Path) => p.length <= q.length && p.every((s, i) => q[i] === s) - -/** Expected child order: escaped segment + separator, lexicographic. */ -function expectedChildren(model: Map, parent: Path): { key: string; own: unknown; hasOwn: boolean }[] { - const kids = new Map() - for (const [k, v] of model) { - const p = JSON.parse(k) as Path - if (p.length <= parent.length || !isPrefix(parent, p)) continue - const seg = p[parent.length]! - const e = kids.get(seg) ?? { own: undefined, hasOwn: false } - if (p.length === parent.length + 1) Object.assign(e, { own: v, hasOwn: true }) - kids.set(seg, e) - } - return [...kids] - .map(([key, e]) => ({ key, ...e })) - .sort((x, y) => { - const a = escapeSegment(x.key, SEP) + SEP - const b = escapeSegment(y.key, SEP) + SEP - return a < b ? -1 : a > b ? 1 : 0 - }) -} - -function nodeAt(db: KVStorage, path: Path) { - let n = db as ReturnType - for (const s of path) n = n.child(s) - return n -} - -describe('model-based (random ops vs reference Map)', () => { - for (const [name, make] of [ - ['InMemory', () => new InMemoryKVDriver()], - ['Minimal', () => new MinimalDriver()], - ] as const) { - it(`matches the model (${name})`, async () => { - await fc.assert( - fc.asyncProperty( - fc.array(opArb, { maxLength: 30 }), - fc.array(fc.array(segArb, { maxLength: 2 }), { minLength: 1, maxLength: 3 }), - async (ops, probes) => { - const db = new KVStorage(make()) - const model = new Map() - for (const op of ops) { - switch (op.t) { - case 'set': { - if (op.selfForm) await nodeAt(db, op.path).set(op.value) - else await nodeAt(db, op.path.slice(0, -1)).set(op.path.at(-1)!, op.value) - model.set(enc(op.path), op.value) - break - } - case 'delete': { - if (op.selfForm) await nodeAt(db, op.path).delete() - else await nodeAt(db, op.path.slice(0, -1)).delete(op.path.at(-1)!) - model.delete(enc(op.path)) - break - } - case 'deleteTree': { - await nodeAt(db, op.path).deleteTree() - for (const k of [...model.keys()]) if (isPrefix(op.path, JSON.parse(k))) model.delete(k) - break - } - case 'setMany': { - await nodeAt(db, op.parent).setMany(op.entries) - for (const [k, v] of op.entries) model.set(enc([...op.parent, k]), v) - break - } - } - } - for (const [k, v] of model) expect(await nodeAt(db, JSON.parse(k)).get()).toEqual(v) - for (const parent of probes) { - const node = nodeAt(db, parent) - const exp = expectedChildren(model, parent) - const keys = exp.map((e) => e.key) - expect(await collect(node.keys())).toEqual(keys) - expect(await collect(node.keys({ reverse: true }))).toEqual([...keys].reverse()) - expect(await collect(node.entries())).toEqual(exp.filter((e) => e.hasOwn).map((e) => [e.key, e.own])) - expect(await collect(node.entries({ reverse: true }))).toEqual( - exp - .filter((e) => e.hasOwn) - .map((e) => [e.key, e.own]) - .reverse(), - ) - for (let i = 0; i < keys.length; i++) { - expect(await collect(node.keys({ after: keys[i]! }))).toEqual(keys.slice(i + 1)) - expect(await collect(node.keys({ after: keys[i]!, reverse: true }))).toEqual(keys.slice(0, i).reverse()) - expect(await collect(node.keys({ limit: i + 1 }))).toEqual(keys.slice(0, i + 1)) - } - } - }, - ), - { numRuns: 300 }, - ) - }, 60_000) - } -}) diff --git a/test/kv/sorted.test.ts b/test/kv/sorted.test.ts deleted file mode 100644 index 7ded610..0000000 --- a/test/kv/sorted.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from 'vitest' -import fc from 'fast-check' -import { ChunkedSortedMap } from '../../src/kv/sorted.js' - -const key = fc.string({ unit: fc.constantFrom('a', 'b', 'c', '/', '0'), minLength: 1, maxLength: 4 }) -type Op = { t: 'set'; k: string } | { t: 'del'; k: string } | { t: 'bulk'; ks: string[] } -const op: fc.Arbitrary = fc.oneof( - { weight: 5, arbitrary: fc.record({ t: fc.constant('set' as const), k: key }) }, - { weight: 3, arbitrary: fc.record({ t: fc.constant('del' as const), k: key }) }, - { weight: 1, arbitrary: fc.record({ t: fc.constant('bulk' as const), ks: fc.array(key, { minLength: 60, maxLength: 120 }) }) }, -) - -describe('ChunkedSortedMap (model-based, tiny chunks)', () => { - it('matches a sorted reference for ranges in both directions', () => { - fc.assert( - fc.property(fc.array(op, { maxLength: 80 }), fc.integer({ min: 2, max: 6 }), key, key, (ops, chunk, a, b) => { - const m = new ChunkedSortedMap(chunk) - const ref = new Map() - ops.forEach((o, i) => { - if (o.t === 'set') (m.set(o.k, i), ref.set(o.k, i)) - else if (o.t === 'del') expect(m.delete(o.k)).toBe(ref.delete(o.k)) - else (m.setMany(o.ks.map((k) => [k, i])), o.ks.forEach((k) => ref.set(k, i))) - }) - const sorted = [...ref.keys()].sort() - const [gte, lt] = a < b ? [a, b] : [b, a] - expect(m.size).toBe(ref.size) - expect([...m.keys({})]).toEqual(sorted) - expect([...m.keys({ reverse: true })]).toEqual([...sorted].reverse()) - expect([...m.keys({ gte, lt })]).toEqual(sorted.filter((k) => k >= gte && k < lt)) - expect([...m.keys({ gte, lt, reverse: true })]).toEqual(sorted.filter((k) => k >= gte && k < lt).reverse()) - for (const k of sorted) expect(m.get(k)).toBe(ref.get(k)) - }), - { numRuns: 500 }, - ) - }) - - it('iterators resume correctly after mutations between steps', () => { - fc.assert( - fc.property( - fc.uniqueArray(key, { minLength: 1, maxLength: 60 }), - fc.array(fc.tuple(fc.boolean(), key), { maxLength: 40 }), - fc.boolean(), - (initial, muts, reverse) => { - const m = new ChunkedSortedMap(3) - initial.forEach((k) => m.set(k, 0)) - const live = new Set(initial) - const seen: string[] = [] - let step = 0 - for (const k of m.keys({ reverse })) { - seen.push(k) - const mut = muts[step++] - if (mut) { - const [add, mk] = mut - if (add) (m.set(mk, 1), live.add(mk)) - else (m.delete(mk), live.delete(mk)) - } - } - // Strictly monotone, and every key present throughout the iteration was visited. - for (let i = 1; i < seen.length; i++) expect(reverse ? seen[i]! < seen[i - 1]! : seen[i]! > seen[i - 1]!).toBe(true) - const stable = initial.filter((k) => !muts.some(([, mk]) => mk === k)) - for (const k of stable) expect(seen).toContain(k) - }, - ), - { numRuns: 500 }, - ) - }) -}) diff --git a/test/types.test-d.ts b/test/types.test-d.ts index 35fcfaa..7f589f6 100644 --- a/test/types.test-d.ts +++ b/test/types.test-d.ts @@ -1,123 +1,89 @@ import { describe, expectTypeOf, it } from 'vitest' -import { KVStorage, type KVNode } from '../src/kv/node.js' -import { InMemoryKVDriver } from '../src/kv/memory.js' -import { KVEventStorage } from '../src/events/kv.js' -import type { StoredEvent } from '../src/events/types.js' -import { EventSourcedDocumentStorage } from '../src/documents/event-sourced.js' -import type { FindResult } from '../src/documents/types.js' -import type { AnySpec, Fields, Many } from '../src/types/schema.js' +import type { BaseKVStorage } from '../src/kv-storage/base-kv-storage.js' +import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js' +import { EventStorageKV } from '../src/event-storage/event-storage-kv.js' +import type { StoredEvent } from '../src/event-storage/base-event-storage.js' +import { DocumentStorageInMemory } from '../src/document-storage/document-storage-in-memory.js' +import { EventSourcedDocumentStorage } from '../src/document-storage/event-sourced-document-storage.js' +import type { DocChange } from '../src/utils/document-update.js' +import type { FindResult } from '../src/utils/document-query.js' +import type { Fields, Many } from '../src/utils/schema.js' type User = { username: string; age: number } type Post = { title: string; createdAt: string } type Schema = { children: { users: Many }> - profile: { children: { [id: string]: { children: Fields } } } // field-per-key style + profile: { children: { [id: string]: { children: Fields } } } } } -const driver = new InMemoryKVDriver() +describe('KV types', () => { + const db = new KVStorageInMemory() -describe('KV types (section 7)', () => { - const db = new KVStorage(driver) - - it('reads child values at dynamic levels', () => { + it('types reads at dynamic and named levels', () => { expectTypeOf(db.child('users').get('123')).toEqualTypeOf>() expectTypeOf(db.child('users').child('123').get()).toEqualTypeOf>() - expectTypeOf(db.child('users').getMany(['1', '2'])).toEqualTypeOf>() - }) - - it('navigates named levels', () => { - expectTypeOf(db.child('users').child('123').child('posts')).toEqualTypeOf>>() - // @ts-expect-error unknown named child - db.child('users').child('123').child('postz') - // @ts-expect-error unknown top-level child - db.child('userz') - }) - - it('reads field-per-key values', () => { expectTypeOf(db.child('profile').child('9').get('age')).toEqualTypeOf>() - expectTypeOf(db.child('profile').child('9').get('username')).toEqualTypeOf>() + expectTypeOf(db.child('users').child('123').child('posts')).toEqualTypeOf>>() + // @ts-expect-error unknown child + db.child('users').child('123').child('postz') // @ts-expect-error unknown field db.child('profile').child('9').get('agee') }) - it('checks written values', () => { + it('types writes', () => { db.child('users').set('1', { username: 'a', age: 1 }) db.child('users').child('1').set({ username: 'a', age: 1 }) - // @ts-expect-error wrong value type + // @ts-expect-error missing field db.child('users').set('1', { username: 'a' }) db.child('profile') .child('9') .setMany([ ['age', 3], - ['username', 'x'], - ]) - db.child('profile') - .child('9') - .setMany([ // @ts-expect-error value doesn't match its key - ['age', 'x'], + ['username', 3], ]) }) it('types listings', () => { - expectTypeOf(db.child('users').keys()).toEqualTypeOf>() expectTypeOf(db.child('profile').child('9').keys()).toEqualTypeOf>() expectTypeOf(db.child('users').entries()).toEqualTypeOf>() }) - it('defaults to untyped and supports as()', () => { - const raw = new KVStorage(driver) - expectTypeOf(raw.child('anything').child(5).get('x')).toEqualTypeOf>() - const users = raw.child('users').as>() - expectTypeOf(users.get('1')).toEqualTypeOf>() - expectTypeOf(raw).toMatchTypeOf>() + it('is untyped by default', () => { + expectTypeOf(new KVStorageInMemory().child('a').child(1).get('b')).toEqualTypeOf>() }) }) -type OrderEvent = { type: 'created'; total: number } | { type: 'paid' } -type EventSchema = { children: { orders: Many }> } } +describe('event and document types', () => { + type OrderEvent = { type: 'created'; total: number } | { type: 'paid' } + const orders = new EventStorageKV<{ children: { orders: Many } }>(new KVStorageInMemory()).child('orders') + const users = new DocumentStorageInMemory<{ children: { users: Many }> } }>().child('users') -describe('event types', () => { - const events = new KVEventStorage(new KVStorage(driver)) - const orders = events.child('orders') - - it('checks appended events', () => { - orders.append('o1', [{ type: 'created', total: 40 }]) + it('types events', () => { + orders.append('o1', [{ type: 'created', total: 1 }]) orders.child('o1').append([{ type: 'paid' }]) // @ts-expect-error not an OrderEvent orders.append('o1', [{ type: 'refunded' }]) - orders - .child('o1') - .child('shipments') - .append('s1', [{ carrier: 'x' }]) + expectTypeOf(orders.read('o1')).toEqualTypeOf>>() }) - it('types read results', () => { - expectTypeOf(orders.read('o1')).toEqualTypeOf>>() - expectTypeOf(orders.version('o1')).toEqualTypeOf>() - }) -}) - -type DocSchema = { children: { users: Many }> } } - -describe('document types', () => { - const docs = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(driver))) - const users = docs.child('users') - - it('types reads and writes', () => { + it('types documents', () => { expectTypeOf(users.get('1')).toEqualTypeOf>() - expectTypeOf(users.child('1').get()).toEqualTypeOf>() - expectTypeOf(users.insert({ username: 'a', age: 1 })).toEqualTypeOf>() - // @ts-expect-error missing field - users.insert({ username: 'a' }) expectTypeOf(users.update('1', { $inc: { age: 1 } })).toEqualTypeOf>() expectTypeOf(users.child('1').child('posts').get('p')).toEqualTypeOf>() + expectTypeOf(users.find()).toEqualTypeOf>>() + // @ts-expect-error missing field + users.insert({ username: 'a' }) }) - it('types queries', () => { - expectTypeOf(users.find({ age: { $gte: 18 } })).toEqualTypeOf>>() - expectTypeOf(users.findOne()).toEqualTypeOf | undefined>>() + it('keeps event-sourced methods on children', () => { + const docs = new EventSourcedDocumentStorage<{ children: { users: Many } }>( + new EventStorageKV(new KVStorageInMemory()), + new DocumentStorageInMemory(), + ) + expectTypeOf(docs.child('users').history('1')).toEqualTypeOf>>() + expectTypeOf(docs.child('users').getAt('1', 3)).toEqualTypeOf>() }) }) diff --git a/test/utils/utils.test.ts b/test/utils/utils.test.ts new file mode 100644 index 0000000..85ac5db --- /dev/null +++ b/test/utils/utils.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { Keys } from '../../src/utils/keys.js' +import { SortedMap } from '../../src/utils/sorted-map.js' +import { Ulid } from '../../src/utils/ulid.js' +import { DocumentQuery } from '../../src/utils/document-query.js' +import { DocumentUpdate } from '../../src/utils/document-update.js' + +describe('Keys', () => { + it('round-trips any segment and never emits a character at or below the separator', () => { + fc.assert( + fc.property(fc.string({ unit: 'binary', minLength: 1 }), (s) => { + const escaped = Keys.escape(s) + expect(Keys.unescape(escaped)).toBe(s) + expect([...escaped].every((c) => c > '/')).toBe(true) + expect(Keys.isReserved(escaped)).toBe(false) + }), + ) + }) + + it('keeps every child subtree contiguous in key order', () => { + fc.assert( + fc.property( + fc.uniqueArray(fc.string({ unit: fc.constantFrom('a', '-', '.', '/', '~', ' ', '0'), minLength: 1, maxLength: 3 }), { minLength: 2 }), + (segments) => { + const keys = segments.flatMap((s) => [Keys.escape(s), Keys.escape(s) + '/x']).sort() + const order = keys.map((k) => k.split('/')[0]) + // Each segment's keys appear as one consecutive run. + expect(new Set(order).size).toBe(order.filter((s, i) => s !== order[i - 1]).length) + }, + ), + ) + }) +}) + +describe('SortedMap', () => { + it('matches a sorted reference, including iteration across mutations', () => { + fc.assert( + fc.property( + fc.array(fc.tuple(fc.boolean(), fc.string({ unit: fc.constantFrom('a', 'b', 'c'), maxLength: 4 })), { maxLength: 200 }), + fc.boolean(), + (ops, reverse) => { + const map = new SortedMap(4) + const ref = new Set() + for (const [add, key] of ops) add ? (map.set(key, 1), ref.add(key)) : (map.delete(key), ref.delete(key)) + const sorted = [...ref].sort() + expect([...map.keys(undefined, undefined, reverse)]).toEqual(reverse ? sorted.reverse() : sorted) + // Deleting each key as we pass it must not skip the rest. + const seen = [...map.keys()].length + let visited = 0 + for (const key of map.keys()) (visited++, map.delete(key)) + expect(visited).toBe(seen) + }, + ), + ) + }) +}) + +describe('Ulid', () => { + it('is strictly increasing within a millisecond and sorts by time', () => { + let t = 1_700_000_000_000 + const ulid = new Ulid(() => t) + const ids = Array.from({ length: 500 }, () => ulid.generate()) + expect([...ids].sort()).toEqual(ids) + t++ + expect(ulid.generate() > ids.at(-1)!).toBe(true) + }) +}) + +describe('DocumentQuery', () => { + const q = (filter: object, doc: object) => new DocumentQuery(filter as never).matches(doc as never) + it('matches equality, comparisons, $in, dot paths, arrays and null-as-missing', () => { + expect(q({ a: 1 }, { a: 1 })).toBe(true) + expect(q({ 'a.b': 'x' }, { a: { b: 'x' } })).toBe(true) + expect(q({ n: { $gte: 2, $lt: 5 } }, { n: 4 })).toBe(true) + expect(q({ n: { $gt: 2 } }, { n: '9' })).toBe(false) + expect(q({ n: { $in: [1, 2] } }, { n: 2 })).toBe(true) + expect(q({ tags: 'x' }, { tags: ['y', 'x'] })).toBe(true) + expect(q({ tags: ['x'] }, { tags: ['x'] })).toBe(true) + expect(q({ a: null }, {})).toBe(true) + expect(() => q({ a: { $regex: 'x' } }, {})).toThrow(/Unsupported/) + }) +}) + +describe('DocumentUpdate', () => { + it('resolves operators to final values', () => { + const state = { n: 5, tags: ['a', 'b'], nested: { x: 1 } } + const change = DocumentUpdate.resolve(state, { $inc: { n: 1 }, $push: { tags: { $each: ['c'] } }, $unset: { 'nested.x': 1 }, $set: { 'nested.y': 2 } }) + expect(change).toEqual({ op: 'patch', set: { n: 6, tags: ['a', 'b', 'c'], 'nested.y': 2 }, unset: ['nested.x'] }) + expect(DocumentUpdate.apply(state, change)).toEqual({ n: 6, tags: ['a', 'b', 'c'], nested: { y: 2 } }) + expect(DocumentUpdate.apply(state, DocumentUpdate.resolve(state, { $pull: { tags: 'a' }, $addToSet: { more: { $each: [1, 1] } } }))).toMatchObject({ + tags: ['b'], + more: [1], + }) + expect(() => DocumentUpdate.resolve(state, { $set: { n: 1 }, $inc: { n: 1 } })).toThrow(/twice/) + }) +}) -- 2.54.0 From 4ffbc7e78d52389d3cd21bd542e8c27b59f49ce1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:41:42 +0000 Subject: [PATCH 02/11] Add KVStorageComposite, IndexedDB storage and SSE change streams - Every KV storage emits change events (base class owns writeMany/removeMany/ writeIfAbsent; implementations provide protected apply* methods) - KVStorageHttpServer streams changes over SSE (GET /events) with a resumable history; KVStorageHttp.watch() subscribes and resumes via Last-Event-ID - KVStorageIndexedDB for local persistence - KVStorageComposite keeps replicas of a source in sync: local reads, resume instead of full sync, offline outbox replayed in order, echo tracking - Values may contain bigint/Uint8Array (Extended JSON over the network) - Vendor sse-session, event-emitter and extended-json (no runtime deps) - Freeze child nodes so shadowed state on children fails loudly - CI on Node 22, 24 and 26 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- .gitea/workflows/ci.yml | 2 +- CLAUDE.md | 8 +- DESIGN.md | 101 +++- package-lock.json | 19 +- package.json | 3 +- src/document-storage/base-document-storage.ts | 9 +- src/event-storage/base-event-storage.ts | 9 +- src/index.ts | 4 + src/kv-storage/base-kv-storage.ts | 80 ++- src/kv-storage/kv-storage-composite.ts | 379 ++++++++++++++ src/kv-storage/kv-storage-http-server.ts | 123 ++++- src/kv-storage/kv-storage-http.ts | 116 ++++- src/kv-storage/kv-storage-in-memory.ts | 6 +- src/kv-storage/kv-storage-indexeddb.ts | 129 +++++ src/utils/errors.ts | 8 + src/utils/event-emitter.ts | 134 +++++ src/utils/extended-json.ts | 51 ++ src/utils/json.ts | 12 +- src/utils/sse-session/async-push-iterator.ts | 85 ++++ src/utils/sse-session/exponential-backoff.ts | 251 ++++++++++ src/utils/sse-session/index.ts | 5 + src/utils/sse-session/sse-event-parser.ts | 214 ++++++++ src/utils/sse-session/sse-session.ts | 471 ++++++++++++++++++ src/utils/sse-session/types.ts | 145 ++++++ test/helpers.ts | 7 +- test/kv-storage/kv-storage-composite.test.ts | 144 ++++++ test/kv-storage/kv-storage-http-watch.test.ts | 61 +++ test/kv-storage/kv-storage-http.test.ts | 4 +- test/kv-storage/kv-storage-in-memory.test.ts | 9 + test/kv-storage/kv-storage-indexeddb.test.ts | 8 + test/kv-storage/kv-storage.conformance.ts | 22 + test/kv-storage/sync-helpers.ts | 35 ++ test/utils/event-emitter.test.ts | 58 +++ test/utils/extended-json.test.ts | 32 ++ test/utils/sse-session.test.ts | 32 ++ tsconfig.json | 2 +- 36 files changed, 2679 insertions(+), 99 deletions(-) create mode 100644 src/kv-storage/kv-storage-composite.ts create mode 100644 src/kv-storage/kv-storage-indexeddb.ts create mode 100644 src/utils/event-emitter.ts create mode 100644 src/utils/extended-json.ts create mode 100644 src/utils/sse-session/async-push-iterator.ts create mode 100644 src/utils/sse-session/exponential-backoff.ts create mode 100644 src/utils/sse-session/index.ts create mode 100644 src/utils/sse-session/sse-event-parser.ts create mode 100644 src/utils/sse-session/sse-session.ts create mode 100644 src/utils/sse-session/types.ts create mode 100644 test/kv-storage/kv-storage-composite.test.ts create mode 100644 test/kv-storage/kv-storage-http-watch.test.ts create mode 100644 test/kv-storage/kv-storage-indexeddb.test.ts create mode 100644 test/kv-storage/sync-helpers.ts create mode 100644 test/utils/event-emitter.test.ts create mode 100644 test/utils/extended-json.test.ts create mode 100644 test/utils/sse-session.test.ts diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4789154..abbcc9b 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - node: [20, 22] + node: [22, 24, 26] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 9d9b3f6..507cc91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ Layered TypeScript storage library: KV → Events → Documents. The design is i - `npm run typecheck`, `npm run build`, `npm run bench` - `npm run format` / `npm run format:check`: Prettier -CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, tests and build on Node 20 and 22. +CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, tests and build on Node 22, 24 and 26. ## Conventions @@ -18,7 +18,11 @@ CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, primitives, and the user-facing API lives in the base class. - Classes over loose functions; TS `private`, not `#`; no overloads (use `param?` or tuple-union rest params); don't override base methods in subclasses. -- No runtime dependencies. Tests live only in `test/`, mirroring `src/`. +- **Never reassign a field after construction.** `child()` nodes are frozen `Object.create(root)` + views, so `this.x = ...` on a child throws. Keep mutable state in a `private readonly state` + object or in Maps (see `KVStorageComposite`, `KVStorageHttp`). +- No runtime dependencies; third-party code is vendored (`src/utils/sse-session/`, `event-emitter.ts`, + `extended-json.ts`, with a header naming the source). Tests live only in `test/`, mirroring `src/`. - New KV/document implementations must pass `test/kv-storage/kv-storage.conformance.ts` / `test/document-storage/document-storage.suite.ts`. - Ask the owner before pushing. diff --git a/DESIGN.md b/DESIGN.md index 4021168..bca1a8c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -14,6 +14,9 @@ section 7 lists what changed and why. that delegates the primitives. 2. **One navigation primitive: `child(key)`.** Every instance is a node; `child` returns a view of the same storage rooted deeper (via `Object.create`, so it shares the instance's state). Sync, no I/O. + Child nodes are **frozen**: an implementation that reassigns a field while running on a child + would silently shadow the root's value, so that throws instead. Implementations keep mutable + state in a `readonly state` object or in Maps. 3. **Self or child by argument.** `get()` reads this node, `get(key)` a child. A key that is passed but `undefined`/`null` throws (`InvalidKeyError`) instead of silently meaning "self". Methods use optional parameters or tuple-union rest parameters, not overloads. @@ -21,16 +24,21 @@ section 7 lists what changed and why. suits the backend (in-memory equality maps, native Mongo/SQLite/IndexedDB indexes, or nothing). 5. **Simple over clever.** Code style: classes, TS `private` (not `#`), no method overrides in subclasses, tests only under `test/`. +6. **No runtime dependencies.** Third-party code is vendored into `src/utils/` with a header naming + its source: `sse-session/` (Harvmaster/sse-session), `event-emitter.ts` and `extended-json.ts` + (xo-cash-utils, branch installable-sse). Node ≥ 22; CI runs Node 22, 24 and 26. ## 2. Layout ``` src/ - kv-storage/ base-kv-storage.ts, kv-storage-in-memory.ts, kv-storage-http.ts, kv-storage-http-server.ts + kv-storage/ base-kv-storage.ts, kv-storage-in-memory.ts, kv-storage-indexeddb.ts, + kv-storage-http.ts, kv-storage-http-server.ts, kv-storage-composite.ts event-storage/ base-event-storage.ts, event-storage-kv.ts document-storage/ base-document-storage.ts, document-storage-in-memory.ts, document-storage-kv.ts, event-sourced-document-storage.ts - utils/ keys, sorted-map, json, ulid, errors, schema (types), document-path/query/update + utils/ keys, sorted-map, json, extended-json, event-emitter, ulid, errors, schema (types), + document-path/query/update, sse-session/ (vendored) test/ mirrors src/; shared suites: kv-storage.conformance.ts, document-storage.suite.ts bench/ ``` @@ -49,15 +57,24 @@ bench/ ## 4. KV storage -`BaseKVStorage` primitives (absolute keys): +`BaseKVStorage` primitives (absolute keys). Implementations provide the reads and the protected +`apply*` writes; the public `writeMany` / `removeMany` / `writeIfAbsent` live in the base class and +emit a `change` event after each write, so every store emits change events. -| Primitive | | -| ------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| `readMany(keys)` | values in order, `undefined` for missing | -| `writeMany(entries)` | | -| `removeMany(keys)` | | -| `writeIfAbsent(entries)` | `true` = written; atomic per key (needed by events) | -| `scan({ gte, lt, limit, reverse, values })` | key-ordered; must keep working if keys change mid-scan (keys present throughout appear exactly once) | +| Primitive | | +| ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| `readMany(keys)` | values in order, `undefined` for missing | +| `scan({ gte, lt, limit, reverse, values })` | key-ordered; must keep working if keys change mid-scan (keys present throughout appear exactly once) | +| `applyWrite(entries)` → public `writeMany` | | +| `applyRemove(keys)` → public `removeMany` | | +| `applyWriteIfAbsent(entries)` → public `writeIfAbsent` | `true` = written; atomic per key (needed by events) | + +**Values** are JSON plus `bigint` and `Uint8Array`. Anything crossing the network is encoded as +Extended JSON (``, ``); IndexedDB stores them natively. + +**Events** (`EventEmitter`, shared by all nodes of a storage): `change` → +`{ changes: { key, value? }[], origin: 'local' | 'remote', id? }` (no `value` = removed), and +`resync` → changes may have been missed, so reload. Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / `set(key, value)`, `setMany`, `delete(key?)`, `deleteMany`, `deleteTree(key?)`, `keys({ limit, after, reverse })`, @@ -68,14 +85,66 @@ Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / ` reserved segments. - **The only KV "index" is the ordered key space.** There is no `find`: with KV you know where to look, or you list a node's children. -- **Implementations:** `KVStorageInMemory` (`SortedMap`: sorted chunks of ≤1024 keys; values deep-copied); - `KVStorageHttp` (client) + `KVStorageHttpServer` (wraps any KV storage; `handle(Request) → Response`, - `POST /` with JSON bodies, `scan` streamed as NDJSON). The server exposes the whole - underlying store (primitives use absolute keys). +- **Implementations:** + - `KVStorageInMemory`: `SortedMap` (sorted chunks of ≤1024 keys); values deep-copied. + - `KVStorageIndexedDB`: one object store with string keys. Scans read in batches, each in its own + transaction, resuming after the last key (IndexedDB transactions close when the code awaits + anything else). `writeIfAbsent` runs in one readwrite transaction. + - `KVStorageHttp` (client) and `KVStorageHttpServer` (wraps any KV storage; + `handle(Request) → Response`): `POST /` with Extended JSON bodies; `scan` streams + NDJSON. Unreachable server (fetch failure, 502/503/504) → `StorageUnavailableError`. The server + exposes the whole underlying store (primitives use absolute keys). + - `KVStorageComposite`: see 4.1. +- **Change stream (SSE).** `GET /events` on the server streams every change to the wrapped storage + as `event: change` with id `-` (epoch = server instance). The server keeps the last + `historySize` (10 000) change batches. + - A client reconnecting with `Last-Event-ID` gets the batches it missed, or `event: resync` if it + can't be resumed (another epoch, or too old). + - Every catch-up ends with `event: ready`. + - Keep-alive comments are sent every 15 s. + - `KVStorageHttp.watch()` subscribes (via the vendored `SSESession`, which reconnects with + backoff), emits `change` events with `origin: 'remote'` and `resync`, and resolves once caught up. + - Limitation: a server only sees writes made through its own storage instance, so several server + processes over one database need a shared pub/sub (not built). - **Wrappers** (e.g. a future encryption layer) extend `BaseKVStorage` and delegate the primitives. Keys must stay plaintext (or be deterministically encrypted) for prefix scans to work; values can be encrypted freely. +### 4.1 `KVStorageComposite({ source, replicas, retry? })` + +Keeps full local copies of a source (e.g. `KVStorageHttp`) so reads are local and the app works +offline. Replicas are listed fastest first (e.g. `[inMemory, indexedDb]`). The last replica is the +persistent one: it seeds the others on start and stores the composite's internal keys under `~q/` +(hidden from reads): `~q/meta` (`{ lastEventId }`, present once a sync has completed) and the +outbox `~q/o/`. + +- **Start** (lazy, on first use): + 1. Load the outbox and copy the persistent replica into the faster ones. + 2. If a previous sync exists, serve reads locally straight away. + 3. Watch the source, resuming from `lastEventId`. + 4. Run a full sync only when there's no resume point, when the source sends `resync`, or when the + source can't be watched. + 5. Before the first sync completes, reads go to the source. +- **Remote changes** (and full syncs) are applied in order through one queue. A change that arrives + during a full sync waits behind it, so the end state always matches the source. `lastEventId` is + saved after each applied batch. Changes the replicas already hold are skipped. +- **Full sync** merge-scans the source against the persistent replica (both key-ordered) and applies + the difference to every replica in batches of 500. +- **Writes** go to the source, then every replica. + - On `StorageUnavailableError` (or while anything is queued, to keep order), the write is applied + locally and appended to the outbox. The outbox is replayed in order with exponential backoff, + and also whenever the source shows signs of life. Last writer wins on conflicts. + - Remote changes to keys with queued writes are ignored until those writes are replayed. + - `writeIfAbsent` always needs the source. +- **Echo tracking.** Each write sent to a watched source expects one echo per key. A remote change + to a key with an outstanding echo is consumed instead of applied. The stream is in server order, + so that change is either the echo or older than it, and this avoids briefly showing a stale value. + A full sync clears the counts. +- Write through the composite, not directly through the same source instance (its `local`-origin + changes are not mirrored). +- `start()`, `synced()` (complete copy, all received changes applied), `queued` (outbox size), + `close()`. + ## 5. Event storage `BaseEventStorage` primitives over stream paths: `appendTo`, `readFrom`, `versionOf`, `streamsOf`. @@ -153,6 +222,10 @@ Node API: `get(key?)`, `getMany`, `set(doc)` / `set(key, doc)`, `insert(doc)` ## 8. Open questions +- Composite: replicate a subtree instead of the whole store; conflict policies beyond last writer + wins for offline writes. +- Multi-process servers need a shared change feed (pub/sub) for SSE. + - Hard deletes from immutable event logs (privacy requests): compaction vs crypto-shredding. The encryption wrapper could support per-subject keys. - Order-preserving numeric keys (`9` before `10`) as an opt-in key encoding. diff --git a/package-lock.json b/package-lock.json index 1a4a335..40b5f5e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "devDependencies": { "@types/node": "^22.20.5", + "fake-indexeddb": "^6.2.6", "fast-check": "^4.10.2", "prettier": "^3.9.10", "typescript": "^5.9.3", @@ -16,14 +17,6 @@ }, "engines": { "node": ">=20" - }, - "peerDependencies": { - "vitest": ">=3" - }, - "peerDependenciesMeta": { - "vitest": { - "optional": true - } } }, "node_modules/@esbuild/aix-ppc64": { @@ -1136,6 +1129,16 @@ "node": ">=12.0.0" } }, + "node_modules/fake-indexeddb": { + "version": "6.2.6", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.6.tgz", + "integrity": "sha512-ihhmr0KoET51jT8R4YhbUYuLLvr1qqO6INISygtN16i8loR8kAxtAUr4+EKHTSMjL+lggaAn4OC7DRqzrNgnuQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/fast-check": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.10.2.tgz", diff --git a/package.json b/package.json index 1f1e75a..8c502b3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Layered TypeScript storage: KV -> Events -> Documents", "type": "module", "engines": { - "node": ">=20" + "node": ">=22" }, "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -29,6 +29,7 @@ }, "devDependencies": { "@types/node": "^22.20.5", + "fake-indexeddb": "^6.2.6", "fast-check": "^4.10.2", "prettier": "^3.9.10", "typescript": "^5.9.3", diff --git a/src/document-storage/base-document-storage.ts b/src/document-storage/base-document-storage.ts index 7730ebf..cf9bf09 100644 --- a/src/document-storage/base-document-storage.ts +++ b/src/document-storage/base-document-storage.ts @@ -27,7 +27,7 @@ export type ChildDoc = ValueOf>> */ export abstract class BaseDocumentStorage { /** Unescaped path of this node (`[]` for the root, which has no own document). */ - path: readonly string[] = [] + readonly path: readonly string[] = [] protected readonly ids = new Ulid() abstract readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> @@ -40,8 +40,11 @@ export abstract class BaseDocumentStorage { abstract query(parent: readonly string[], query: DocumentQuery): AsyncIterable child>(key: K): BaseDocumentStorage> { - const node = Object.create(this) as BaseDocumentStorage> - node.path = [...this.path, Keys.segment(key)] + const node = Object.create(this, { path: { value: [...this.path, Keys.segment(key)], enumerable: true } }) as BaseDocumentStorage> + // Frozen: child nodes share the root's state through the prototype, so an implementation that + // reassigns a field (`this.x = ...`) while running on a child would silently shadow it. + // Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw. + Object.freeze(node) return node } diff --git a/src/event-storage/base-event-storage.ts b/src/event-storage/base-event-storage.ts index 173e361..4daaabf 100644 --- a/src/event-storage/base-event-storage.ts +++ b/src/event-storage/base-event-storage.ts @@ -39,7 +39,7 @@ export interface StreamKeysOptions { */ export abstract class BaseEventStorage { /** Unescaped path of this node (`[]` for the root, which is not a stream). */ - path: readonly string[] = [] + readonly path: readonly string[] = [] abstract appendTo(stream: readonly string[], events: unknown[], expectedVersion: number | 'any'): Promise abstract readFrom(stream: readonly string[], options: ReadOptions): AsyncIterable> @@ -47,8 +47,11 @@ export abstract class BaseEventStorage { abstract streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncIterable child>(key: K): BaseEventStorage> { - const node = Object.create(this) as BaseEventStorage> - node.path = [...this.path, Keys.segment(key)] + const node = Object.create(this, { path: { value: [...this.path, Keys.segment(key)], enumerable: true } }) as BaseEventStorage> + // Frozen: child nodes share the root's state through the prototype, so an implementation that + // reassigns a field (`this.x = ...`) while running on a child would silently shadow it. + // Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw. + Object.freeze(node) return node } diff --git a/src/index.ts b/src/index.ts index 78841b9..4ef085e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,8 @@ export * from './kv-storage/base-kv-storage.js' export * from './kv-storage/kv-storage-in-memory.js' export * from './kv-storage/kv-storage-http.js' export * from './kv-storage/kv-storage-http-server.js' +export * from './kv-storage/kv-storage-indexeddb.js' +export * from './kv-storage/kv-storage-composite.js' export * from './event-storage/base-event-storage.js' export * from './event-storage/event-storage-kv.js' export * from './document-storage/base-document-storage.js' @@ -12,6 +14,8 @@ export * from './utils/document-path.js' export * from './utils/document-query.js' export * from './utils/document-update.js' export * from './utils/errors.js' +export * from './utils/event-emitter.js' +export * from './utils/extended-json.js' export * from './utils/json.js' export * from './utils/keys.js' export * from './utils/schema.js' diff --git a/src/kv-storage/base-kv-storage.ts b/src/kv-storage/base-kv-storage.ts index e13a874..08dc823 100644 --- a/src/kv-storage/base-kv-storage.ts +++ b/src/kv-storage/base-kv-storage.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from '../utils/event-emitter.js' import { Keys } from '../utils/keys.js' import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueAt, ValueOf } from '../utils/schema.js' @@ -19,39 +20,90 @@ export interface KeysOptions { reverse?: boolean } +/** One key's new value; `value: undefined` means it was removed. */ +export interface KVChange { + key: string + value?: unknown +} + +export interface KVChangeEvent { + changes: KVChange[] + /** `local`: made through this instance. `remote`: observed from elsewhere (e.g. pushed by a server). */ + origin: 'local' | 'remote' + /** Position in the source's change stream, when it has one (used to resume). */ + id?: string +} + +export type KVStorageEvents = { + change: KVChangeEvent + /** Changes may have been missed (e.g. a resume point was too old): consumers should reload. */ + resync: { id?: string } +} + /** - * Ordered key-value storage. Implementations provide five raw primitives over absolute keys; - * everything else (navigation, self/child reads and writes, child listing) is defined here once. + * Ordered key-value storage. Implementations provide five raw primitives over absolute keys + * (`readMany`, `scan`, and the protected `apply*` writes); everything else (navigation, self/child + * reads and writes, child listing, change events) is defined here once. + * + * Every write emits a `change` event from the storage (shared by all its child nodes). * * Every instance is a node: `child(key)` returns a view of the same storage rooted at that key. * Methods taking an optional `key` act on that child, or on this node's own value when omitted. * A key that is passed but `undefined`/`null` throws rather than silently meaning "self". */ -export abstract class BaseKVStorage { +export abstract class BaseKVStorage extends EventEmitter { /** Unescaped path of this node (`[]` for the root). */ - path: readonly string[] = [] + readonly path: readonly string[] = [] /** This node's own key (`''` for the root). */ - key = '' + readonly key: string = '' /** Prefix of every descendant key (`''` for the root). */ - prefix = '' + readonly prefix: string = '' /** Values for absolute keys, in order; `undefined` for missing. */ abstract readMany(keys: string[]): Promise - abstract writeMany(entries: [key: string, value: unknown][]): Promise - abstract removeMany(keys: string[]): Promise - /** Writes each entry only if its key is absent; `true` = written. Must be atomic per key. */ - abstract writeIfAbsent(entries: [key: string, value: unknown][]): Promise /** Entries in key order (descending when `reverse`). */ abstract scan(options?: ScanOptions): AsyncIterable<[key: string, value: unknown]> + protected abstract applyWrite(entries: [key: string, value: unknown][]): Promise + protected abstract applyRemove(keys: string[]): Promise + /** Writes each entry only if its key is absent; `true` = written. Must be atomic per key. */ + protected abstract applyWriteIfAbsent(entries: [key: string, value: unknown][]): Promise + + // ------------------------------------------------------------------ raw writes (absolute keys) + + async writeMany(entries: [key: string, value: unknown][]): Promise { + if (entries.length === 0) return + await this.applyWrite(entries) + this.emit('change', { changes: entries.map(([key, value]) => ({ key, value })), origin: 'local' }) + } + + async removeMany(keys: string[]): Promise { + if (keys.length === 0) return + await this.applyRemove(keys) + this.emit('change', { changes: keys.map((key) => ({ key })), origin: 'local' }) + } + + async writeIfAbsent(entries: [key: string, value: unknown][]): Promise { + if (entries.length === 0) return [] + const written = await this.applyWriteIfAbsent(entries) + const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value })) + if (changes.length) this.emit('change', { changes, origin: 'local' }) + return written + } // ------------------------------------------------------------------ navigation /** A view of this storage rooted at `key`. Synchronous; no I/O. */ child>(key: K): BaseKVStorage> { - const node = Object.create(this) as BaseKVStorage> - node.path = [...this.path, Keys.segment(key)] - node.key = this.prefix + Keys.escape(Keys.segment(key)) - node.prefix = node.key + Keys.separator + const own = this.prefix + Keys.escape(Keys.segment(key)) + const node = Object.create(this, { + path: { value: [...this.path, Keys.segment(key)], enumerable: true }, + key: { value: own, enumerable: true }, + prefix: { value: own + Keys.separator, enumerable: true }, + }) as BaseKVStorage> + // Frozen: child nodes share the root's state through the prototype, so an implementation that + // reassigns a field (`this.x = ...`) while running on a child would silently shadow it. + // Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw. + Object.freeze(node) return node } diff --git a/src/kv-storage/kv-storage-composite.ts b/src/kv-storage/kv-storage-composite.ts new file mode 100644 index 0000000..a61ab3d --- /dev/null +++ b/src/kv-storage/kv-storage-composite.ts @@ -0,0 +1,379 @@ +import { StorageUnavailableError } from '../utils/errors.js' +import type { OffCallback } from '../utils/event-emitter.js' +import { Json } from '../utils/json.js' +import type { AnySpec, Spec } from '../utils/schema.js' +import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js' +import { BaseKVStorage, type KVChange, type KVChangeEvent, type ScanOptions } from './base-kv-storage.js' + +export interface KVStorageCompositeOptions { + /** The source of truth (e.g. `KVStorageHttp`). */ + source: BaseKVStorage + /** + * Local full copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be + * persistent: it also stores the offline outbox and sync metadata, and seeds the others on start. + */ + replicas: BaseKVStorage[] + /** Delays between attempts to replay queued writes or re-sync. */ + retry?: Partial<{ baseDelay: number; maxDelay: number }> +} + +/** Implemented by sources that can push remote changes (e.g. `KVStorageHttp`). */ +interface WatchableSource { + watch(): Promise + unwatch(): Promise + lastEventId: string | undefined +} + +type OutboxEntry = { type: 'write'; entries: [string, unknown][] } | { type: 'remove'; keys: string[] } + +interface SyncMeta { + lastEventId?: string +} + +/** Internal keys in the persistent replica; hidden from reads. */ +const INTERNAL_PREFIX = '~q/' +const INTERNAL_END = '~q0' +const META_KEY = '~q/meta' +const OUTBOX_PREFIX = '~q/o/' +const OUTBOX_END = '~q/o0' +const SYNC_BATCH = 500 + +/** + * Keeps local replicas of a source in sync, so reads (including `keys()`, `entries()` and document + * queries) are served locally, and the app keeps working offline. + * + * - **Start** (lazily, on first use): the persistent replica seeds the faster ones. If it holds a + * previous sync, reads are served from it straight away (possibly stale until caught up). + * - **Staying in sync:** remote changes pushed by the source (`watch()`, e.g. SSE) are applied to + * every replica. On restart the source resumes from the saved `lastEventId`; a full sync (diffing + * the source against the replicas) only runs the first time, or when the source says it can't + * resume (`resync`). Without `watch()`, a full sync runs on start. + * - **Writes** go to the source, then the replicas. If the source is unreachable + * (`StorageUnavailableError`), they are applied locally and queued in a persisted outbox, then + * replayed in order when it's reachable again (last writer wins on conflict). While any write is + * queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source and + * fails offline. + * + * Write through the composite, not the source: changes made directly through the same source + * instance (`origin: 'local'`) are not mirrored. + */ +export class KVStorageComposite extends BaseKVStorage { + private readonly source: BaseKVStorage + private readonly replicas: BaseKVStorage[] + /** Persistent replica: seeds the others and holds the outbox and metadata. */ + private readonly persistent: BaseKVStorage + private readonly backoff: { baseDelay: number; maxDelay: number } + /** Queued-write count per key; remote changes to these keys are ignored until replayed. */ + private readonly pending = new Map() + /** + * Echoes still expected per key: one per write this client sent to a watchable source. A remote + * change to such a key is the echo or older than it (the stream is in server order), so it is + * consumed instead of applied; this avoids briefly showing a stale value. + */ + private readonly echoes = new Map() + private readonly offs: OffCallback[] = [] + /** Mutable state lives here: child nodes share this object (see `child()`). */ + private readonly state: { + started?: Promise + /** Replicas hold a complete (possibly stale) copy, so reads can be served locally. */ + hasSnapshot: boolean + snapshotReady: Promise + markSnapshot: () => void + /** + * Remote changes and full syncs are applied strictly in order through this chain. Changes that + * arrive during a full sync wait behind it, so the end state matches the source even if it + * changed during the scan. + */ + applying: Promise + replaying?: Promise + outboxSeq: number + retryTimer?: ReturnType + retryAttempt: number + closed: boolean + } + + constructor(options: KVStorageCompositeOptions) { + super() + if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica') + this.source = options.source + this.replicas = options.replicas + this.persistent = options.replicas.at(-1)! + this.backoff = { baseDelay: 1000, maxDelay: 30_000, ...options.retry } + let markSnapshot!: () => void + const snapshotReady = new Promise((resolve) => (markSnapshot = resolve)) + this.state = { hasSnapshot: false, snapshotReady, markSnapshot, applying: Promise.resolve(), outboxSeq: 0, retryAttempt: 0, closed: false } + } + + /** Starts syncing (also happens on first use). Resolves once local data is loaded. */ + start(): Promise { + return (this.state.started ??= this.open()) + } + + /** Resolves once the replicas hold a complete copy and all received remote changes are applied. */ + async synced(): Promise { + await this.start() + await this.state.snapshotReady + await this.state.applying + } + + /** Number of writes waiting to be replayed to the source. */ + get queued(): number { + let total = 0 + for (const count of this.pending.values()) total += count + return total + } + + /** Stops watching the source and cancels retries. */ + async close(): Promise { + this.state.closed = true + clearTimeout(this.state.retryTimer) + for (const off of this.offs.splice(0)) off() + if (this.watchable) await (this.source as unknown as WatchableSource).unwatch() + } + + async readMany(keys: string[]): Promise { + await this.start() + return (this.state.hasSnapshot ? this.replicas[0]! : this.source).readMany(keys) + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + await this.start() + if (!this.state.hasSnapshot) return yield* this.source.scan(options) + // Skip internal keys before applying the limit. + let remaining = options.limit ?? Infinity + if (remaining <= 0) return + for await (const entry of this.replicas[0]!.scan({ ...options, limit: undefined })) { + if (entry[0] >= INTERNAL_PREFIX && entry[0] < INTERNAL_END) continue + yield entry + if (--remaining <= 0) return + } + } + + protected async applyWrite(entries: [string, unknown][]): Promise { + await this.start() + await this.writeThrough({ type: 'write', entries }) + } + + protected async applyRemove(keys: string[]): Promise { + await this.start() + await this.writeThrough({ type: 'remove', keys }) + } + + protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { + await this.start() + const written = await this.source.writeIfAbsent(entries) + await this.applyToReplicas(entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value }))) + return written + } + + // ------------------------------------------------------------------ start + + private get watchable(): boolean { + return typeof (this.source as unknown as Partial).watch === 'function' + } + + private async open(): Promise { + const [meta] = (await this.persistent.readMany([META_KEY])) as (SyncMeta | undefined)[] + for await (const [key, value] of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, values: true })) { + this.state.outboxSeq = Number(key.slice(OUTBOX_PREFIX.length)) + for (const k of KVStorageComposite.keysOf(value as OutboxEntry)) this.pending.set(k, (this.pending.get(k) ?? 0) + 1) + } + await this.seedFasterReplicas() + if (meta) this.snapshotTaken() + + this.offs.push( + this.source.on('change', (event) => { + if (event.origin === 'remote') this.enqueueApply(() => this.applyRemote(event)) + if (this.queued > 0) void this.replay() + }), + this.source.on('resync', () => this.enqueueApply(() => this.fullSync())), + ) + + if (this.watchable) { + const source = this.source as unknown as WatchableSource + source.lastEventId = meta?.lastEventId + // Without a resume point, catch up on the stream first so no change after the snapshot is missed. + const watching = source.watch() + if (!meta?.lastEventId) void watching.then(() => this.enqueueApply(() => this.fullSync())) + } else { + this.enqueueApply(() => this.fullSync()) + } + if (this.queued > 0) void this.replay() + } + + /** Copies the persistent replica into the faster ones. */ + private async seedFasterReplicas(): Promise { + const faster = this.replicas.slice(0, -1) + if (faster.length === 0) return + let batch: [string, unknown][] = [] + const flush = async () => { + await Promise.all(faster.map((replica) => replica.writeMany(batch))) + batch = [] + } + for await (const entry of this.persistent.scan({ values: true })) { + if (!entry[0].startsWith(INTERNAL_PREFIX) && batch.push(entry) >= SYNC_BATCH) await flush() + } + await flush() + } + + private snapshotTaken(): void { + this.state.hasSnapshot = true + this.state.markSnapshot() + } + + // ------------------------------------------------------------------ remote → replicas + + private enqueueApply(task: () => Promise): void { + this.state.applying = this.state.applying.then(task).catch((error) => { + if (!(error instanceof StorageUnavailableError)) console.error(error) + }) + } + + private async applyRemote(event: KVChangeEvent): Promise { + const candidates = event.changes.filter(({ key }) => !this.consumeEcho(key) && !this.pending.has(key) && !key.startsWith(INTERNAL_PREFIX)) + // Skip changes the replicas already reflect. + const current = await this.replicas[0]!.readMany(candidates.map(({ key }) => key)) + const changes = candidates.filter(({ value }, i) => !Json.equal(value, current[i])) + await this.applyToReplicas(changes) + if (event.id) await this.persistent.writeMany([[META_KEY, { lastEventId: event.id } satisfies SyncMeta]]) + if (changes.length) this.emit('change', { changes, origin: 'remote', id: event.id }) + } + + /** Diffs the source against the persistent replica and applies the differences to every replica. */ + private async fullSync(): Promise { + // The diff re-reads the source, so outstanding echoes no longer matter (and may never arrive). + this.echoes.clear() + try { + const id = this.watchable ? (this.source as unknown as WatchableSource).lastEventId : undefined + let changes: KVChange[] = [] + const flush = async () => { + const applied = changes.filter(({ key }) => !this.pending.has(key)) + await this.applyToReplicas(applied) + if (applied.length) this.emit('change', { changes: applied, origin: 'remote' }) + changes = [] + } + for await (const change of this.diff()) if (changes.push(change) >= SYNC_BATCH) await flush() + await flush() + await this.persistent.writeMany([[META_KEY, { lastEventId: id } satisfies SyncMeta]]) + this.snapshotTaken() + this.state.retryAttempt = 0 + } catch (error) { + if (!(error instanceof StorageUnavailableError)) throw error + this.scheduleRetry(() => this.enqueueApply(() => this.fullSync())) + } + } + + /** Changes that turn the persistent replica into the source (both scanned in key order). */ + private async *diff(): AsyncGenerator { + const remote = this.source.scan({ values: true })[Symbol.asyncIterator]() + const local = this.persistent.scan({ values: true })[Symbol.asyncIterator]() + const nextLocal = async () => { + for (let next = await local.next(); ; next = await local.next()) if (next.done || !next.value[0].startsWith(INTERNAL_PREFIX)) return next + } + let r = await remote.next() + let l = await nextLocal() + while (!r.done || !l.done) { + if (l.done || (!r.done && r.value[0] < l.value[0])) { + yield { key: r.value![0], value: r.value![1] } + r = await remote.next() + } else if (r.done || l.value[0] < r.value[0]) { + yield { key: l.value[0] } + l = await nextLocal() + } else { + if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] } + r = await remote.next() + l = await nextLocal() + } + } + } + + private async applyToReplicas(changes: KVChange[]): Promise { + const writes = changes.filter((c) => c.value !== undefined).map((c): [string, unknown] => [c.key, c.value]) + const removes = changes.filter((c) => c.value === undefined).map((c) => c.key) + await Promise.all(this.replicas.map(async (replica) => (await replica.writeMany(writes), await replica.removeMany(removes)))) + } + + // ------------------------------------------------------------------ local writes → source + + private async writeThrough(entry: OutboxEntry): Promise { + if (this.queued === 0) { + try { + await this.send(entry) + return this.applyToReplicas(KVStorageComposite.changesOf(entry)) + } catch (error) { + if (!(error instanceof StorageUnavailableError)) throw error + } + } + // Offline, or earlier writes are still queued: apply locally and queue for the source. + const seq = ++this.state.outboxSeq + for (const key of KVStorageComposite.keysOf(entry)) this.pending.set(key, (this.pending.get(key) ?? 0) + 1) + await this.persistent.writeMany([[OUTBOX_PREFIX + String(seq).padStart(12, '0'), entry]]) + await this.applyToReplicas(KVStorageComposite.changesOf(entry)) + void this.replay() + } + + /** Sends queued writes to the source in order; stops (and retries later) if it's unreachable. */ + private replay(): Promise { + const run = async () => { + try { + for (;;) { + let next: [string, unknown] | undefined + for await (const entry of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, limit: 1, values: true })) next = entry + if (!next) return + const [key, entry] = next as [string, OutboxEntry] + await this.send(entry) + await this.persistent.removeMany([key]) + for (const k of KVStorageComposite.keysOf(entry)) { + const count = this.pending.get(k)! - 1 + if (count > 0) this.pending.set(k, count) + else this.pending.delete(k) + } + this.state.retryAttempt = 0 + } + } catch (error) { + if (!(error instanceof StorageUnavailableError)) console.error(error) + this.scheduleRetry(() => void this.replay()) + } finally { + this.state.replaying = undefined + } + } + return (this.state.replaying ??= run()) + } + + /** Sends a write to the source, expecting its echo when the source is watched. */ + private async send(entry: OutboxEntry): Promise { + const keys = this.watchable ? KVStorageComposite.keysOf(entry) : [] + for (const key of keys) this.echoes.set(key, (this.echoes.get(key) ?? 0) + 1) + try { + await (entry.type === 'write' ? this.source.writeMany(entry.entries) : this.source.removeMany(entry.keys)) + } catch (error) { + for (const key of keys) this.consumeEcho(key) + throw error + } + } + + /** Whether `key` had an outstanding echo (now consumed). */ + private consumeEcho(key: string): boolean { + const count = this.echoes.get(key) + if (!count) return false + if (count > 1) this.echoes.set(key, count - 1) + else this.echoes.delete(key) + return true + } + + private scheduleRetry(task: () => void): void { + if (this.state.closed) return + clearTimeout(this.state.retryTimer) + const delay = ExponentialBackoff.calculateDelay({ ...this.backoff, growthRate: 2, jitter: 0.2, maxAttempts: 0 }, this.state.retryAttempt++) + this.state.retryTimer = setTimeout(task, delay) + ;(this.state.retryTimer as { unref?: () => void }).unref?.() + } + + private static keysOf(entry: OutboxEntry): string[] { + return entry.type === 'write' ? entry.entries.map(([key]) => key) : entry.keys + } + + private static changesOf(entry: OutboxEntry): KVChange[] { + return entry.type === 'write' ? entry.entries.map(([key, value]) => ({ key, value })) : entry.keys.map((key) => ({ key })) + } +} diff --git a/src/kv-storage/kv-storage-http-server.ts b/src/kv-storage/kv-storage-http-server.ts index 54be60d..208262a 100644 --- a/src/kv-storage/kv-storage-http-server.ts +++ b/src/kv-storage/kv-storage-http-server.ts @@ -1,36 +1,75 @@ -import type { BaseKVStorage, ScanOptions } from './base-kv-storage.js' +import { ExtendedJson } from '../utils/extended-json.js' +import type { BaseKVStorage, KVChange, ScanOptions } from './base-kv-storage.js' + +export interface KVStorageHttpServerOptions { + /** Path prefix to strip, e.g. `/kv`. Default `''`. */ + basePath?: string + /** Recent change batches kept for clients resuming with `Last-Event-ID`. Default 10 000. */ + historySize?: number + /** Interval of SSE keep-alive comments. Default 15 000 ms. */ + keepAliveMs?: number +} + +interface HistoryEntry { + seq: number + changes: KVChange[] +} /** * Serves any KV storage over HTTP for `KVStorageHttp`. Framework-agnostic: `handle(Request)` - * returns a `Response`. All routes are `POST /` with a JSON body; `scan` streams NDJSON. - * The raw primitives use absolute keys, so this exposes the whole store behind `storage`. + * returns a `Response`. + * + * - `POST /` with an Extended JSON body (`scan` streams NDJSON). + * - `GET /events`: Server-Sent Events of every change to the storage. Event ids are + * `-` (epoch = this server instance). A client sending `Last-Event-ID` gets the + * changes it missed if they are still in the history; otherwise it gets a `resync` event. + * A client without one gets a `ready` event carrying the current id. + * + * Only changes made through the wrapped storage instance are seen; several server processes over + * one database each see only their own writes. */ export class KVStorageHttpServer { + private readonly basePath: string + private readonly historySize: number + private readonly keepAliveMs: number + private readonly epoch = Math.random().toString(36).slice(2, 10) + private seq = 0 + private readonly history: HistoryEntry[] = [] + private readonly subscribers = new Set<(frame: string) => void>() + constructor( private readonly storage: BaseKVStorage, - private readonly basePath = '', - ) {} + options: KVStorageHttpServerOptions = {}, + ) { + this.basePath = options.basePath ?? '' + this.historySize = options.historySize ?? 10_000 + this.keepAliveMs = options.keepAliveMs ?? 15_000 + storage.on('change', ({ changes }) => this.publish(changes)) + } async handle(request: Request): Promise { const path = new URL(request.url).pathname - if (request.method !== 'POST' || !path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') + if (!path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') + const route = `${request.method} ${path.slice(this.basePath.length)}` + if (route === 'GET /events') return this.events(request.headers.get('last-event-id')) + if (request.method !== 'POST') return KVStorageHttpServer.error(404, 'Not found') try { - const body = (await request.json()) as { keys: string[]; entries: [string, unknown][] } & ScanOptions - switch (path.slice(this.basePath.length)) { - case '/readMany': { + const body = ExtendedJson.parse(await request.text()) as { keys: string[]; entries: [string, unknown][] } & ScanOptions + switch (route) { + case 'POST /readMany': { const values = await this.storage.readMany(body.keys) const missing = values.flatMap((v, i) => (v === undefined ? [i] : [])) - return Response.json({ values: values.map((v) => v ?? null), missing }) + return KVStorageHttpServer.json({ values: values.map((v) => v ?? null), missing }) } - case '/writeMany': + case 'POST /writeMany': await this.storage.writeMany(body.entries) return new Response(null, { status: 204 }) - case '/removeMany': + case 'POST /removeMany': await this.storage.removeMany(body.keys) return new Response(null, { status: 204 }) - case '/writeIfAbsent': - return Response.json(await this.storage.writeIfAbsent(body.entries)) - case '/scan': + case 'POST /writeIfAbsent': + return KVStorageHttpServer.json(await this.storage.writeIfAbsent(body.entries)) + case 'POST /scan': return this.scan(body) default: return KVStorageHttpServer.error(404, 'Not found') @@ -40,6 +79,54 @@ export class KVStorageHttpServer { } } + private publish(changes: KVChange[]): void { + const entry = { seq: ++this.seq, changes } + this.history.push(entry) + if (this.history.length > this.historySize) this.history.splice(0, this.history.length - this.historySize) + const frame = this.frame('change', entry.seq, changes) + for (const send of this.subscribers) send(frame) + } + + private events(lastEventId: string | null): Response { + const encoder = new TextEncoder() + let send: ((frame: string) => void) | undefined + let keepAlive: ReturnType | undefined + const stream = new ReadableStream({ + start: (controller) => { + send = (frame) => controller.enqueue(encoder.encode(frame)) + send(this.catchUpFrames(lastEventId)) + this.subscribers.add(send) + keepAlive = setInterval(() => send!(': keep-alive\n\n'), this.keepAliveMs) + ;(keepAlive as { unref?: () => void }).unref?.() + }, + cancel: () => { + if (send) this.subscribers.delete(send) + clearInterval(keepAlive) + }, + }) + return new Response(stream, { headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' } }) + } + + /** + * What a (re)connecting client gets before live changes: the changes it missed, or `resync` when + * they can't be replayed; then always `ready` (the client is caught up as of that id). + */ + private catchUpFrames(lastEventId: string | null): string { + const ready = this.frame('ready', this.seq, null) + if (lastEventId === null) return ready + const [epoch, seqText] = lastEventId.split('-') + const seq = Number(seqText) + const oldest = this.history[0]?.seq ?? this.seq + 1 + // Resumable only if it's our epoch and nothing after `seq` has been dropped from the history. + if (epoch !== this.epoch || !Number.isInteger(seq) || seq > this.seq || seq < oldest - 1) return this.frame('resync', this.seq, null) + ready + const missed = this.history.filter((entry) => entry.seq > seq).map((entry) => this.frame('change', entry.seq, entry.changes)) + return missed.join('') + ready + } + + private frame(event: string, seq: number, data: unknown): string { + return `id: ${this.epoch}-${seq}\nevent: ${event}\ndata: ${ExtendedJson.stringify(data)}\n\n` + } + private scan(options: ScanOptions): Response { const iterator = this.storage.scan(options)[Symbol.asyncIterator]() const encoder = new TextEncoder() @@ -53,7 +140,7 @@ export class KVStorageHttpServer { if (chunk) controller.enqueue(encoder.encode(chunk)) return controller.close() } - chunk += JSON.stringify(options.values ? next.value : [next.value[0]]) + '\n' + chunk += ExtendedJson.stringify(options.values ? next.value : [next.value[0]]) + '\n' } controller.enqueue(encoder.encode(chunk)) }, @@ -64,6 +151,10 @@ export class KVStorageHttpServer { return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } }) } + private static json(body: unknown): Response { + return new Response(ExtendedJson.stringify(body), { headers: { 'content-type': 'application/json' } }) + } + private static error(status: number, message: string): Response { return Response.json({ error: message }, { status }) } diff --git a/src/kv-storage/kv-storage-http.ts b/src/kv-storage/kv-storage-http.ts index ced0b4e..4e43026 100644 --- a/src/kv-storage/kv-storage-http.ts +++ b/src/kv-storage/kv-storage-http.ts @@ -1,24 +1,48 @@ +import { StorageUnavailableError } from '../utils/errors.js' +import { ExtendedJson } from '../utils/extended-json.js' import type { AnySpec, Spec } from '../utils/schema.js' -import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' +import { SSESession } from '../utils/sse-session/sse-session.js' +import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js' export interface KVStorageHttpOptions { /** Base URL of a `KVStorageHttpServer`, e.g. `https://host/kv`. */ url: string headers?: Record - fetch?: typeof fetch + fetch?: (url: string, init: RequestInit) => Promise + /** Called when the change stream fails (it keeps reconnecting). Default: ignore. */ + onWatchError?: (error: Error) => void } -/** KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage. */ +/** + * KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage. + * + * `watch()` subscribes to the server's change stream (SSE): changes made by anyone are emitted + * here as `change` events with `origin: 'remote'`, and `resync` is emitted when the server can't + * resume from `lastEventId`. Values travel as Extended JSON (`bigint` and `Uint8Array` survive). + */ export class KVStorageHttp extends BaseKVStorage { private readonly url: string private readonly headers: Record - private readonly fetcher: typeof fetch + private readonly fetcher: (url: string, init: RequestInit) => Promise + private readonly onWatchError: (error: Error) => void + /** Mutable state lives here: child nodes share this object (see `child()`). */ + private readonly state: { session?: SSESession; lastEventId?: string } = {} constructor(options: KVStorageHttpOptions) { super() this.url = options.url.replace(/\/$/, '') this.headers = options.headers ?? {} - this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis) + this.fetcher = options.fetch ?? ((url, init) => globalThis.fetch(url, init)) + this.onWatchError = options.onWatchError ?? (() => {}) + } + + /** Position in the server's change stream; sent as `Last-Event-ID` when (re)connecting. */ + get lastEventId(): string | undefined { + return this.state.lastEventId + } + + set lastEventId(id: string | undefined) { + this.state.lastEventId = id } async readMany(keys: string[]): Promise { @@ -28,18 +52,6 @@ export class KVStorageHttp extends BaseKVStorage { return values } - async writeMany(entries: [string, unknown][]): Promise { - if (entries.length) await this.post('writeMany', { entries }) - } - - async removeMany(keys: string[]): Promise { - if (keys.length) await this.post('removeMany', { keys }) - } - - async writeIfAbsent(entries: [string, unknown][]): Promise { - return entries.length ? ((await this.post('writeIfAbsent', { entries })) as boolean[]) : [] - } - async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { if (options.limit !== undefined && options.limit <= 0) return const response = await this.request('scan', options) @@ -53,8 +65,8 @@ export class KVStorageHttp extends BaseKVStorage { const lines = buffer.split('\n') buffer = lines.pop()! for (const line of lines) { - const [key, value] = JSON.parse(line) as [string, unknown?] - yield [key, value] + const [key, entryValue] = ExtendedJson.parse(line) as [string, unknown?] + yield [key, entryValue] } } } finally { @@ -62,17 +74,71 @@ export class KVStorageHttp extends BaseKVStorage { } } + /** + * Subscribes to the server's change stream. Resolves once connected and caught up (missed + * changes replayed, or `resync` emitted); keeps reconnecting and resuming from `lastEventId` + * until `unwatch()`. While the server is unreachable this stays pending, so don't await it on a + * path that must work offline. + */ + async watch(): Promise { + if (this.state.session) return + const session = new SSESession(`${this.url}/events`, { + fetch: this.fetcher, + headers: this.headers, + persistent: true, + onError: this.onWatchError, + onRequest: async (request) => + this.lastEventId ? { ...request, headers: { ...(request.headers as Record), 'Last-Event-ID': this.lastEventId } } : request, + }) + session.on('message', (message) => { + if (message.id) this.lastEventId = message.id + if (message.event === 'change') { + this.emit('change', { changes: ExtendedJson.parse(message.data) as KVChange[], origin: 'remote', id: message.id }) + } else if (message.event === 'resync') { + this.emit('resync', { id: message.id }) + } + }) + this.state.session = session + const caughtUp = session.waitFor('message', (message) => message.event === 'ready') + await session.connect() + await caughtUp + } + + async unwatch(): Promise { + const session = this.state.session + this.state.session = undefined + await session?.disconnect() + } + + protected async applyWrite(entries: [string, unknown][]): Promise { + await this.post('writeMany', { entries }) + } + + protected async applyRemove(keys: string[]): Promise { + await this.post('removeMany', { keys }) + } + + protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { + return (await this.post('writeIfAbsent', { entries })) as boolean[] + } + private async post(path: string, body: unknown): Promise { const response = await this.request(path, body) - return response.status === 204 ? undefined : response.json() + return response.status === 204 ? undefined : ExtendedJson.parse(await response.text()) } private async request(path: string, body: unknown): Promise { - const response = await this.fetcher(`${this.url}/${path}`, { - method: 'POST', - headers: { ...this.headers, 'content-type': 'application/json' }, - body: JSON.stringify(body), - }) + let response: Response + try { + response = await this.fetcher(`${this.url}/${path}`, { + method: 'POST', + headers: { ...this.headers, 'content-type': 'application/json' }, + body: ExtendedJson.stringify(body), + }) + } catch (error) { + throw new StorageUnavailableError(`Cannot reach ${this.url}`, { cause: error }) + } + if ([502, 503, 504].includes(response.status)) throw new StorageUnavailableError(`HTTP ${response.status} from ${this.url}`) if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`) return response } diff --git a/src/kv-storage/kv-storage-in-memory.ts b/src/kv-storage/kv-storage-in-memory.ts index 6883e1a..3ac5f6e 100644 --- a/src/kv-storage/kv-storage-in-memory.ts +++ b/src/kv-storage/kv-storage-in-memory.ts @@ -11,15 +11,15 @@ export class KVStorageInMemory extends BaseKVStorage Json.clone(this.data.get(k))) } - async writeMany(entries: [string, unknown][]): Promise { + protected async applyWrite(entries: [string, unknown][]): Promise { for (const [k, v] of entries) this.data.set(k, Json.clone(v)) } - async removeMany(keys: string[]): Promise { + protected async applyRemove(keys: string[]): Promise { for (const k of keys) this.data.delete(k) } - async writeIfAbsent(entries: [string, unknown][]): Promise { + protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { return entries.map(([k, v]) => !this.data.has(k) && (this.data.set(k, Json.clone(v)), true)) } diff --git a/src/kv-storage/kv-storage-indexeddb.ts b/src/kv-storage/kv-storage-indexeddb.ts new file mode 100644 index 0000000..4c0e7c7 --- /dev/null +++ b/src/kv-storage/kv-storage-indexeddb.ts @@ -0,0 +1,129 @@ +import type { AnySpec, Spec } from '../utils/schema.js' +import { BaseKVStorage, type ScanOptions } from './base-kv-storage.js' + +export interface KVStorageIndexedDBOptions { + /** Database name. */ + name: string + /** Object store name. Default `kv`. */ + storeName?: string + /** IndexedDB factory. Default `globalThis.indexedDB`. */ + indexedDB?: IDBFactory + /** Entries read per transaction while scanning. Default 256. */ + scanBatchSize?: number +} + +/** + * KV storage in the browser's IndexedDB (one object store, string keys). Values are stored with the + * structured clone algorithm, so `bigint` and `Uint8Array` survive. IndexedDB orders string keys by + * UTF-16 code unit, like JavaScript string comparison. + */ +export class KVStorageIndexedDB extends BaseKVStorage { + private readonly storeName: string + private readonly scanBatchSize: number + private readonly db: Promise + + constructor(options: KVStorageIndexedDBOptions) { + super() + this.storeName = options.storeName ?? 'kv' + this.scanBatchSize = options.scanBatchSize ?? 256 + const factory = options.indexedDB ?? globalThis.indexedDB + const open = factory.open(options.name, 1) + open.onupgradeneeded = () => open.result.createObjectStore(this.storeName) + this.db = KVStorageIndexedDB.done(open) + } + + async readMany(keys: string[]): Promise { + const store = await this.store('readonly') + return Promise.all(keys.map((key) => KVStorageIndexedDB.done(store.get(key)))) + } + + async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { + let remaining = options.limit ?? Infinity + let last: string | undefined + while (remaining > 0) { + const batch = await this.readBatch(options, last, Math.min(remaining, this.scanBatchSize)) + for (const entry of batch) { + yield entry + remaining-- + } + if (batch.length < this.scanBatchSize) return + last = batch.at(-1)![0] + } + } + + /** Closes the database connection. */ + async close(): Promise { + ;(await this.db).close() + } + + protected async applyWrite(entries: [string, unknown][]): Promise { + const store = await this.store('readwrite') + for (const [key, value] of entries) store.put(value, key) + await KVStorageIndexedDB.committed(store.transaction) + } + + protected async applyRemove(keys: string[]): Promise { + const store = await this.store('readwrite') + for (const key of keys) store.delete(key) + await KVStorageIndexedDB.committed(store.transaction) + } + + protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { + // One readwrite transaction: checks and writes can't interleave with other writers. + const store = await this.store('readwrite') + const written = await Promise.all( + entries.map(async ([key, value]) => { + if ((await KVStorageIndexedDB.done(store.getKey(key))) !== undefined) return false + store.add(value, key) + return true + }), + ) + await KVStorageIndexedDB.committed(store.transaction) + return written + } + + /** Up to `count` entries after `last` (exclusive), in scan order, within one transaction. */ + private async readBatch(options: ScanOptions, last: string | undefined, count: number): Promise<[string, unknown][]> { + const reverse = options.reverse ?? false + // The upper bound is always exclusive (`lt`, or the last key seen in reverse); the lower bound + // is exclusive only when resuming forwards after `last`. + const lower = reverse ? options.gte : (last ?? options.gte) + const upper = reverse ? (last ?? options.lt) : options.lt + const lowerOpen = !reverse && last !== undefined + if (lower !== undefined && upper !== undefined && lower >= upper) return [] + let range: IDBKeyRange | undefined + if (lower !== undefined && upper !== undefined) range = IDBKeyRange.bound(lower, upper, lowerOpen, true) + else if (lower !== undefined) range = IDBKeyRange.lowerBound(lower, lowerOpen) + else if (upper !== undefined) range = IDBKeyRange.upperBound(upper, true) + const store = await this.store('readonly') + const out: [string, unknown][] = [] + const request = options.values ? store.openCursor(range, reverse ? 'prev' : 'next') : store.openKeyCursor(range, reverse ? 'prev' : 'next') + return new Promise((resolve, reject) => { + request.onerror = () => reject(request.error) + request.onsuccess = () => { + const cursor = request.result + if (!cursor || out.length >= count) return resolve(out) + out.push([cursor.key as string, options.values ? (cursor as IDBCursorWithValue).value : undefined]) + cursor.continue() + } + }) + } + + private async store(mode: IDBTransactionMode): Promise { + return (await this.db).transaction(this.storeName, mode).objectStore(this.storeName) + } + + private static done(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result) + request.onerror = () => reject(request.error) + }) + } + + private static committed(transaction: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve() + transaction.onerror = transaction.onabort = () => reject(transaction.error) + }) + } +} diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 8b95d83..c92957d 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -17,3 +17,11 @@ export class InvalidKeyError extends TypeError { this.name = 'InvalidKeyError' } } + +/** A remote storage could not be reached (network failure or 502/503/504). Writes may be retried later. */ +export class StorageUnavailableError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options) + this.name = 'StorageUnavailableError' + } +} diff --git a/src/utils/event-emitter.ts b/src/utils/event-emitter.ts new file mode 100644 index 0000000..6ef1090 --- /dev/null +++ b/src/utils/event-emitter.ts @@ -0,0 +1,134 @@ +// Adapted from xo-cash-utils `source/event-emitter.ts` (branch installable-sse): payloads are passed +// through as-is (no cloning, freezing or DeeplyReadonly), and timeouts reject with a plain Error. + +export type EventMap = Record + +type Listener = (detail: T) => void + +/** Lets listeners for individual payloads live in one collection typed with the union of payloads. */ +type StoredListener = { + bivarianceHack(detail: T): void +}['bivarianceHack'] + +interface ListenerEntry { + listener: StoredListener + wrappedListener: StoredListener + cancel: () => void +} + +/** Returned by {@link EventEmitter.on} and {@link EventEmitter.once} to remove the listener. */ +export type OffCallback = () => void + +/** A small typed event emitter. */ +export class EventEmitter { + private readonly listeners: Map>> = new Map() + + /** + * Adds a listener. + * @param debounceMilliseconds - When > 0, only the last event within this window is delivered. + */ + on(type: K, listener: Listener, debounceMilliseconds: number = 0): OffCallback { + const { cancel, listener: cancellableListener } = this.cancellable(listener) + const wrappedListener = debounceMilliseconds > 0 ? this.debounce(cancellableListener, debounceMilliseconds) : cancellableListener + this.entriesFor(type).add({ listener, wrappedListener, cancel } as ListenerEntry) + return () => this.off(type, listener) + } + + /** Adds a listener that is removed after its first call. */ + once(type: K, listener: Listener, debounceMilliseconds: number = 0): OffCallback { + const wrappedListener: Listener = (detail) => { + this.off(type, listener) + listener(detail) + } + const { cancel, listener: cancellableListener } = this.cancellable(wrappedListener) + const debouncedListener = debounceMilliseconds > 0 ? this.debounce(cancellableListener, debounceMilliseconds) : cancellableListener + this.entriesFor(type).add({ listener, wrappedListener: debouncedListener, cancel } as ListenerEntry) + return () => this.off(type, listener) + } + + /** Removes one listener, or all listeners for `type` when none is given. */ + off(type: K, listener?: Listener): void { + const listeners = this.listeners.get(type) + if (!listeners) return + for (const entry of Array.from(listeners)) { + if (listener && entry.listener !== listener && entry.wrappedListener !== listener) continue + // Cancel so a pending debounced call doesn't fire after removal. + entry.cancel() + listeners.delete(entry) + } + if (!listener || listeners.size === 0) this.listeners.delete(type) + } + + /** + * Calls every listener for `type` with `payload`. Listener errors are logged, not thrown. + * @returns Whether any listener was registered. + */ + emit(type: K, payload: T[K]): boolean { + const listeners = this.listeners.get(type) + if (!listeners) return false + for (const entry of Array.from(listeners)) { + try { + entry.wrappedListener(payload) + } catch (error) { + console.error(error) + } + } + return listeners.size > 0 + } + + removeAllListeners(): void { + for (const type of Array.from(this.listeners.keys())) this.off(type) + } + + /** Resolves with the first payload of `type` matching `predicate`; rejects after `timeoutMs` if given. */ + waitFor(type: K, predicate: (payload: T[K]) => boolean, timeoutMs?: number): Promise { + return new Promise((resolve, reject) => { + let timeoutId: ReturnType | undefined + const cleanup = (): void => { + this.off(type, listener) + if (timeoutId !== undefined) clearTimeout(timeoutId) + } + const listener = (payload: T[K]): void => { + try { + if (!predicate(payload)) return + cleanup() + resolve(payload) + } catch (error) { + cleanup() + reject(error) + } + } + if (timeoutMs !== undefined) { + timeoutId = setTimeout(() => { + this.off(type, listener) + reject(new Error(`Timed out waiting for "${String(type)}"`)) + }, timeoutMs) + } + this.on(type, listener) + }) + } + + private entriesFor(type: K): Set> { + let entries = this.listeners.get(type) + if (!entries) this.listeners.set(type, (entries = new Set())) + return entries + } + + private debounce(func: Listener, wait: number): Listener { + let timeout: ReturnType | undefined + return (detail) => { + if (timeout !== undefined) clearTimeout(timeout) + timeout = setTimeout(() => func(detail), wait) + } + } + + private cancellable(func: Listener): { cancel: () => void; listener: Listener } { + let cancelled = false + return { + cancel: () => void (cancelled = true), + listener: (detail) => { + if (!cancelled) func(detail) + }, + } + } +} diff --git a/src/utils/extended-json.ts b/src/utils/extended-json.ts new file mode 100644 index 0000000..13888e3 --- /dev/null +++ b/src/utils/extended-json.ts @@ -0,0 +1,51 @@ +// Adapted from xo-cash-utils `source/extended-json.ts` (branch installable-sse), with local hex +// encoding instead of @bitauth/libauth. + +/** Matches a bigint encoded as ``. */ +const BIGINT_PATTERN = /^[+-]?[0-9]+)n>$/u + +/** Matches a Uint8Array encoded as ``. */ +const UINT8ARRAY_PATTERN = /^[0-9a-f]*)>$/u + +/** + * JSON that also round-trips `bigint` (``) and `Uint8Array` (``). + * Used for every value that crosses the network. + */ +export class ExtendedJson { + static stringify(value: unknown): string { + return JSON.stringify(value, ExtendedJson.replacer) + } + + static parse(text: string): unknown { + return JSON.parse(text, ExtendedJson.reviver) + } + + /** `JSON.stringify` replacer. */ + static replacer(this: void, _key: string, value: unknown): unknown { + if (value instanceof Uint8Array) return `` + if (typeof value === 'bigint') return `` + return value + } + + /** `JSON.parse` reviver. */ + static reviver(this: void, _key: string, value: unknown): unknown { + if (typeof value !== 'string') return value + const bigint = value.match(BIGINT_PATTERN) + if (bigint) return BigInt(bigint.groups!.bigint!) + const bytes = value.match(UINT8ARRAY_PATTERN) + if (bytes) return ExtendedJson.fromHex(bytes.groups!.hex!) + return value + } + + private static toHex(bytes: Uint8Array): string { + let hex = '' + for (const byte of bytes) hex += byte.toString(16).padStart(2, '0') + return hex + } + + private static fromHex(hex: string): Uint8Array { + const bytes = new Uint8Array(hex.length / 2) + for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + return bytes + } +} diff --git a/src/utils/json.ts b/src/utils/json.ts index 8c666b7..84d13eb 100644 --- a/src/utils/json.ts +++ b/src/utils/json.ts @@ -1,8 +1,11 @@ -/** Helpers for JSON-compatible values (all stored values are JSON by contract). */ +/** + * Helpers for stored values: JSON plus `bigint` and `Uint8Array` (see `ExtendedJson`). + */ export class Json { - /** Deep copy; much cheaper than `structuredClone` for plain JSON. */ + /** Deep copy; much cheaper than `structuredClone` for plain data. */ static clone(value: T): T { if (value === null || typeof value !== 'object') return value + if (value instanceof Uint8Array) return value.slice() as T if (Array.isArray(value)) return value.map((v) => Json.clone(v)) as T const out: Record = {} for (const k in value) out[k] = Json.clone((value as Record)[k]) @@ -12,6 +15,9 @@ export class Json { static equal(a: unknown, b: unknown): boolean { if (a === b) return true if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false + if (a instanceof Uint8Array || b instanceof Uint8Array) { + return a instanceof Uint8Array && b instanceof Uint8Array && a.length === b.length && a.every((byte, i) => byte === b[i]) + } if (Array.isArray(a) !== Array.isArray(b)) return false const ka = Object.keys(a) const kb = Object.keys(b) @@ -19,6 +25,6 @@ export class Json { } static isObject(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) + return typeof value === 'object' && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array) } } diff --git a/src/utils/sse-session/async-push-iterator.ts b/src/utils/sse-session/async-push-iterator.ts new file mode 100644 index 0000000..462285f --- /dev/null +++ b/src/utils/sse-session/async-push-iterator.ts @@ -0,0 +1,85 @@ +// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c). +// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession. + +/** + * An async iterable queue that bridges push-based producers and pull-based consumers. + * + * Composes an internal {@link ReadableStream} instead of extending it, so producers + * call {@link push} while consumers use standard async iteration (`for await...of`). + * + * ```ts + * const messages = new AsyncPushIterator(); + * + * // Producer (elsewhere) + * messages.push(event); + * + * // Consumer + * for await (const event of messages) { + * handle(event); + * } + * ``` + * + * {@link Symbol.asyncIterator} returns `stream.values({ preventCancel: true })` so + * breaking out of `for await...of` does not cancel the underlying stream. That + * matters for long-lived sessions where the producer keeps pushing after a consumer + * stops reading early (for example, test helpers that only collect a fixed count). + */ +export class AsyncPushIterator { + /** ReadableStream backing the async iterator returned from {@link Symbol.asyncIterator}. */ + private readonly stream: ReadableStream + + /** Controller used to enqueue values and close the stream from {@link push} and {@link close}. */ + private controller: ReadableStreamDefaultController | undefined + + /** When true, no more values are accepted and iteration eventually completes. */ + public closed = false + + public constructor() { + // `start`'s `this` is the underlying source object when using a plain method. + // An arrow function captures the class instance so the controller is stored here. + this.stream = new ReadableStream({ + start: (controller: ReadableStreamDefaultController): void => { + this.controller = controller + }, + }) + } + + /** + * Enqueues a value for the consumer. + * + * After {@link close}, pushes are silently dropped. + * + * @param value - The next value to yield from the iterator. + */ + push(value: T): void { + if (this.closed) return + + this.controller?.enqueue(value) + } + + /** + * Ends the stream. + * + * Marks the iterator closed so future {@link push} calls are ignored. + * Buffered values are still yielded before iteration completes. + */ + close(): void { + this.closed = true + + try { + this.controller?.close() + } catch { + // The reader may already have released or cancelled the stream. + } + } + + /** + * Returns an async iterator over the composed stream. + * + * Uses `preventCancel: true` so early `break` from `for await...of` does not + * close the stream and block later pushes. + */ + [Symbol.asyncIterator](): AsyncIterableIterator { + return this.stream.values({ preventCancel: true }) + } +} diff --git a/src/utils/sse-session/exponential-backoff.ts b/src/utils/sse-session/exponential-backoff.ts new file mode 100644 index 0000000..ca95335 --- /dev/null +++ b/src/utils/sse-session/exponential-backoff.ts @@ -0,0 +1,251 @@ +// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c). +// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession. + +/** + * Exponential backoff is a technique used to retry a function after a delay. + * + * The delay increases exponentially with each attempt, up to a maximum delay. + * + * The jitter is a random amount of time added to the delay to prevent thundering herd problems. + * + * The growth rate is the factor by which the delay increases with each attempt. + */ +export class ExponentialBackoff { + /** + * Create a new ExponentialBackoff instance + * + * @param config - The configuration for the exponential backoff + * @returns The ExponentialBackoff instance + */ + static from(config?: Partial): ExponentialBackoff { + const backoff = new ExponentialBackoff(config) + + return backoff + } + + /** + * Run the function with exponential backoff + * + * @param fn - The function to run + * @param onError - The callback to call when an error occurs + * @param options - The configuration for the exponential backoff + * + * @throws An AggregateError with all the errors that were thrown by the task function + * + * @returns The result of the function + */ + static run( + fn: (context: ExpontialBackoffCallbackContext) => Promise, + onError = (_error: Error, _context: ExpontialBackoffCallbackContext): void => {}, + options?: Partial, + ): Promise { + const backoff = ExponentialBackoff.from(options) + + return backoff.run(fn, onError) + } + + private readonly options: ExponentialBackoffOptions + + /** + * Creates a new exponential-backoff instance. + * + * Unspecified options use the defaults listed below. + * + * @param options - Exponential-backoff configuration overrides. + * @param options.maxDelay - Maximum delay between retries. Default: `10_000` ms. + * @param options.maxAttempts - Maximum number of attempts; `0` retries indefinitely. Default: `10`. + * @param options.baseDelay - Delay used as the basis for the first retry. Default: `1_000` ms. + * @param options.growthRate - Multiplier applied to the delay after each attempt. Default: `2`. + * @param options.jitter - Random proportional variation applied to each delay. Default: `0.1`. + */ + constructor(options: Partial = {}) { + this.options = { + maxDelay: 10_000, + maxAttempts: 10, + baseDelay: 1_000, + growthRate: 2, + jitter: 0.1, + ...options, + } + + ExponentialBackoff.validateOptions(this.options) + } + + /** + * Run the function with exponential backoff + * + * If the function fails but we have not hit the max attempts, the error will be passed to the onError callback + * and the function will be retried with an exponential delay + * + * If the function fails and we have hit the max attempts, an AggregateError will be thrown with all the errors that were thrown by the task function + * + * @param fn - The function to run + * @param onError - The callback to call when an error occurs + * + * @throws An AggregateError with all the errors that were thrown by the task function + * + * @returns The result of the function + */ + async run( + fn: (context: ExpontialBackoffCallbackContext) => Promise, + onError = (_error: Error, _context: ExpontialBackoffCallbackContext): void => {}, + ): Promise { + const abortController = new AbortController() + + const context: ExpontialBackoffCallbackContext = { + errors: [], + attempt: 1, + options: { ...this.options }, + nextDelay: 0, + signal: abortController.signal, + abort: (reason?: unknown) => abortController.abort(reason), + } + + while (context.options.maxAttempts === 0 || context.attempt <= context.options.maxAttempts) { + context.nextDelay = ExponentialBackoff.calculateDelay(context.options, context.attempt - 1) + + try { + // Await the promise before returning so its execution context remains in the try-catch + // If we didn't await, this `run` function would successfully return and any errors would not be caught here. + return await fn(context) + } catch (error) { + // Store the error in case we fail every attempt + const errorInstance = error instanceof Error ? error : new Error(`${error}`) + context.errors.push(errorInstance) + await onError(errorInstance, context) + } + + // If the abort controller is aborted, throw an error + if (abortController.signal.aborted) { + const reasonError = abortController.signal.reason instanceof Error ? abortController.signal.reason : new Error(`${abortController.signal.reason}`) + throw new Error('Exponential backoff aborted', { cause: reasonError }) + } + + // If we have hit the max attempts, break out of the loop + const unlimitedAttempts = context.options.maxAttempts === 0 + if (!unlimitedAttempts && context.attempt >= context.options.maxAttempts) { + break + } + + // Wait before going to the next attempt + await this.wait(context.nextDelay, abortController.signal) + + context.attempt++ + } + + // We completed the loop without ever succeeding. Throw an ExponentialBackoffMaxRetriesHitError with all the errors we got + throw new AggregateError(context.errors, 'Exponential backoff max retries hit') + } + + private wait(delay: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + // Set a timeout to resolve the promise + const timeout = setTimeout(() => { + resolve() + }, delay) + + // If the signal is aborted, reject the promise + signal.addEventListener('abort', () => { + clearTimeout(timeout) + reject(new Error('Exponential backoff aborted')) + }) + }) + } + + /** + * Calculate the delay before we should attempt to retry + * + * NOTE: The maximum delay is (maxDelay * (1 + jitter)) + * + * @param attempt + * @returns The time in milliseconds before another attempt should be made + */ + public static calculateDelay(options: ExponentialBackoffOptions, attempt: number): number { + // Get the power of the growth rate + const power = options.growthRate ** attempt + + // Get the delay before jitter or limit + const rawDelay = options.baseDelay * power + + // Cap the delay to the maximum. Do this before the jitter so jitter does not become larger than delay + const cappedDelay = Math.min(rawDelay, options.maxDelay) + + // Get a random number for the amount to "jitter" the delay by + const jitterAmount = Math.random() + + // Calculate the jitter + const jitter = jitterAmount * options.jitter * cappedDelay + + // Subtract the jitter from the delay + return cappedDelay - jitter + } + + /** + * Validate the options, ensure that no values are NaN + * @param options + */ + public static validateOptions(options: ExponentialBackoffOptions): void { + const keys = Object.keys(options) as (keyof ExponentialBackoffOptions)[] + + for (const key of keys) { + if (!isFinite(options[key])) { + throw new Error(`Invalid option: ${key} is not finite`) + } + + if (key === 'jitter' && (options[key] < 0 || options[key] > 1)) { + throw new Error(`Invalid option: jitter is not between 0 and 1`) + } + + if (options[key] < 0) { + throw new Error(`Invalid option: ${key} is less than 0`) + } + } + } +} + +export type ExponentialBackoffOptions = { + /** + * The maximum delay between attempts in milliseconds + */ + maxDelay: number + + /** + * The maximum number of attempts. Passing 0 will result in infinite attempts. + */ + maxAttempts: number + + /** + * The base delay between attempts in milliseconds + */ + baseDelay: number + + /** + * The growth rate of the delay + */ + growthRate: number + + /** + * The jitter of the delay as a percentage of growthRate + */ + jitter: number +} + +export type ExpontialBackoffCallbackContext = { + /** The errors that have occurred */ + errors: Error[] + + /** The current attempt */ + attempt: number + + /** The options for the exponential backoff */ + options: ExponentialBackoffOptions + + /** The next delay before the next attempt */ + nextDelay: number + + /** The abort signal */ + signal: AbortSignal + + /** The abort function */ + abort: (reason?: unknown) => void +} diff --git a/src/utils/sse-session/index.ts b/src/utils/sse-session/index.ts new file mode 100644 index 0000000..02b167c --- /dev/null +++ b/src/utils/sse-session/index.ts @@ -0,0 +1,5 @@ +export * from './types.js' +export * from './sse-session.js' +export * from './sse-event-parser.js' +export * from './async-push-iterator.js' +export * from './exponential-backoff.js' diff --git a/src/utils/sse-session/sse-event-parser.ts b/src/utils/sse-session/sse-event-parser.ts new file mode 100644 index 0000000..a445348 --- /dev/null +++ b/src/utils/sse-session/sse-event-parser.ts @@ -0,0 +1,214 @@ +// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c). +// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession. + +import type { SSEvent } from './types.js' + +/** + * Optional encoders used when decoding incoming SSE bytes and re-encoding + * any buffered remainder between chunks. + */ +export interface SSEEventParserOptions { + /** Decodes raw stream bytes into text. Defaults to a new `TextDecoder`. */ + textDecoder: TextDecoder +} + +/** + * Regex that splits decoded SSE text into lines. + * + * The SSE wire format is line-oriented (`field: value` per line). Servers may + * send `\r\n` (HTTP default), `\n` (Unix), or `\r` (legacy Mac). Matching all + * three keeps parsing correct regardless of platform or server implementation. + */ +export const SSE_LINE_ENDINGS = /\r\n|\r|\n/ + +/** + * Regex that matches the single optional leading space in an SSE field value. + * + * Per the SSE spec, `field: value` may include one space immediately after the + * colon; that space is not part of the value. Used with `.replace()` to strip + * it when parsing lines such as `data: hello` → `hello`. + */ +export const SSE_FIELD_VALUE_REGEX = /^ / + +/** + * The newline character used when normalizing SSE text internally. + * + * Used to join consecutive `data:` lines into one payload and to reassemble + * buffered partial lines between streamed chunks before the next parse call. + */ +export const NEW_LINE = '\n' + +/** + * Incrementally parses Server-Sent Events (SSE) from streamed byte chunks. + * + * SSE payloads are line-oriented: each event is a sequence of `field: value` + * lines terminated by a blank line. This parser accepts arbitrary chunk + * boundaries from a live HTTP response body and emits only complete events. + * + * Typical usage is one parser instance per connection, calling {@link parseEvents} + * for each chunk received from the stream: + * + * ```ts + * const parser = new SSEEventParser(); + * + * for await (const chunk of response.body) { + * for (const event of parser.parseEvents(chunk)) { + * // handle event.data, event.event, event.id, event.retry + * } + * } + * ``` + * + * Supported fields follow the SSE spec: `data`, `event`, `id`, and `retry`. + * Multiple `data:` lines in one event are joined with `\n`. An event is only + * emitted once a blank line is seen and at least one `data` field was collected. + */ +export class SSEEventParser { + private readonly textDecoder: TextDecoder + + /** Bytes from a partial line or incomplete event, carried over to the next chunk. */ + private messageBuffer: string = '' + + /** + * Creates a parser for one SSE stream. + * + * Inject custom encoders in tests or when a non-default character encoding + * is required; production callers can rely on the defaults. + * + * @param options - Optional text encoders for decode/encode of stream bytes. + */ + constructor(options: Partial = {}) { + this.textDecoder = options.textDecoder ?? new TextDecoder() + } + + /** + * Clears any buffered bytes from a partial line or incomplete event. + * + * Call when abandoning a transport so the next connection does not prepend + * stale bytes to incoming chunks. + */ + public reset(): void { + // Clear the message buffer + this.messageBuffer = '' + + // Reset the decoder to clear any buffered bytes + this.textDecoder.decode() + } + + /** + * Parses all complete SSE events contained in a newly received chunk. + * + * The chunk is appended to any bytes buffered from earlier calls. Complete + * events (blank-line delimited blocks with at least one `data` field) are + * returned immediately; any trailing partial line or in-progress event stays + * in the internal buffer until a later chunk completes it. + * + * @param chunk - Newly received SSE stream bytes. + * @returns Zero or more complete parsed SSE events from this chunk. + */ + public parseEvents(chunk: Uint8Array): SSEvent[] { + const lines = this.getBufferedLines(chunk) + + // The final split item is the incomplete remainder after the final line + // ending. It must not be processed as a complete SSE line. + const completeLines = lines.slice(0, -1) + + const events: SSEvent[] = [] + let event: Partial = {} + let processedLineCount = 0 + + for (const [index, line] of completeLines.entries()) { + // A blank line indicates the end of an event. If we have received data, we can complete the event + if (line === '') { + if (event.data !== undefined) { + // Cast event when pushing to an SSE Event so it changes from Partial to SSEvent + // The "Partial" can safely be removed as we have guaranteed that `data` is defined + events.push(event as SSEvent) + } + + event = {} + processedLineCount = index + 1 + + continue + } + + this.parseLine(line, event) + } + + this.storeRemainingLines(lines, processedLineCount) + + return events + } + + /** + * Appends a new chunk to the buffered bytes and splits the combined payload + * into lines. + * + * Accepts `\r\n`, `\r`, and `\n` line endings so events parse correctly + * regardless of server or platform conventions. + */ + private getBufferedLines(chunk: Uint8Array): string[] { + this.messageBuffer += this.textDecoder.decode(chunk, { stream: true }) + + return this.messageBuffer.split(SSE_LINE_ENDINGS) + } + + /** + * Parses one SSE field line into an in-progress event. + * + * Lines starting with a colon are ignored as comments. A single optional space after the colon + * is stripped from the field value, per the SSE spec. + */ + private parseLine(line: string, event: Partial): void { + // Split the line at the first colon. Before the first colon is the field, after the first colon is the value. + const [field, ...valueArray] = line.split(':') + + // Join the rest of the array back together to get the value as it was before the split. + // If the value has a space at the start, remove it (according to the SSE spec) + const value = valueArray.join(':').replace(SSE_FIELD_VALUE_REGEX, '') + + switch (field) { + case 'data': + event.data = event.data !== undefined ? `${event.data}${NEW_LINE}${value}` : value + + return + + case 'event': + event.event = value + + return + + case 'id': + event.id = value + + return + + case 'retry': + this.parseRetry(value, event) + + return + } + } + + /** + * Applies a numeric `retry:` field to an in-progress event. + * + * Non-numeric values are ignored rather than failing the parse. + */ + private parseRetry(value: string, event: Partial): void { + const retry = parseInt(value, 10) + + if (!isNaN(retry)) { + event.retry = retry + } + } + + /** + * Preserves incomplete trailing lines for the next received chunk. + * + * Only lines that were fully processed (through a completed event boundary) + * are discarded; the remainder is re-encoded into {@link messageBuffer}. + */ + private storeRemainingLines(lines: string[], processedLineCount: number): void { + this.messageBuffer = lines.slice(processedLineCount).join(NEW_LINE) + } +} diff --git a/src/utils/sse-session/sse-session.ts b/src/utils/sse-session/sse-session.ts new file mode 100644 index 0000000..9211cae --- /dev/null +++ b/src/utils/sse-session/sse-session.ts @@ -0,0 +1,471 @@ +// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c). +// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession. + +import type { SSESessionOptions, SSESessionEventMap, SSEvent } from './types.js' + +import { EventEmitter } from '../event-emitter.js' +import { AsyncPushIterator } from './async-push-iterator.js' +import { ExponentialBackoff } from './exponential-backoff.js' + +import { SSEEventParser } from './sse-event-parser.js' + +/** + * A fetch-based Server-Sent Events (SSE) client with reconnect and optional + * browser tab visibility handling. + * + * Each session maintains one HTTP streaming connection at a time. Incoming + * bytes are parsed into {@link SSEvent} objects and delivered through two + * surfaces: + * + * - **Events** — `"connected"`, `"message"`, `"disconnected"`, `"error"`, + * and `"closed"` on the session itself (extends {@link EventEmitter}). + * - **Messages** — {@link messages}, an async iterable for `for await...of` + * consumers. + * + * Typical usage: + * + * ```ts + * const session = await SSESession.create("/events"); + * + * session.on("message", (event) => console.log(event.data)); + * + * for await (const event of session.messages) { + * handle(event); + * } + * ``` + * + * ## Lifecycle + * + * - {@link connect} opens (or reopens) the transport. It resolves once the + * HTTP stream is established; reading continues in the background. + * - {@link abort} stops the in-flight fetch without ending the session. + * Used internally for tab visibility. The {@link messages} iterator stays + * open so an existing consumer resumes when the tab becomes visible again. + * - {@link disconnect} aborts the transport, closes {@link messages}, emits + * `"closed"`, and disables attached visibility handlers until the next + * manual {@link connect}. + * + * Automatic reconnect is controlled by {@link SSESessionOptions.persistent} + * (server closed the stream) and + * {@link SSESessionOptions.attemptReconnect} (transport error). + * + * ## Connection supersession + * + * Each {@link connect} or {@link abort} bumps an internal `connectionId`. + * Background read loops capture their id at start and exit quietly when a + * newer connection supersedes them, avoiding duplicate events or errors from + * stale transports. + */ +export class SSESession extends EventEmitter { + /** + * Creates a session and waits until the first connection is established. + * + * @param url - The SSE endpoint URL. + * @param options - Configuration merged with instance defaults. + * @returns A connected session. + * @throws When the initial connection cannot be established. + */ + static async create(url: string, options: Partial = {}): Promise { + const client = new SSESession(url, options) + await client.connect() + + return client + } + + /** + * Enables SSE resume semantics by sending `Last-Event-ID` on reconnect. + * + * Listens for incoming `"message"` events and remembers the most recent + * {@link SSEvent.id}. On every subsequent connect or reconnect, the session's + * {@link onRequest} hook is wrapped so that header is attached when an id is + * known, allowing the server to replay only events the client has not yet + * received. + * + * The existing {@link onRequest} callback is preserved and runs after the + * header is applied, so auth or other header mutations continue to work. + * + * Attach as early in the session lifetime as possible. When added after + * {@link create}, the initial connection omits the header (no id yet); + * all later reconnects include it. To instrument before the first connect, + * call this on the session returned from {@link withBrowserVisibility} + * before awaiting a separate {@link connect} when the tab starts hidden. + * + * ```ts + * const session = await SSESession.create(url); + * await SSESession.addLastEventIdReconnect(session); + * // Reconnects send Last-Event-ID once an event with an id is received. + * ``` + * + * @param client - The session to instrument. + * @returns The same session, for chaining. + */ + static async addLastEventIdReconnect(client: SSESession): Promise { + let lastEventId: string | undefined + + client.on('message', (event) => { + // If the event has no id, we should not update the lastEventId and instead keep the old one. + lastEventId = event.id ?? lastEventId + }) + + const originalOnRequest = client.options.onRequest + + client.options.onRequest = async (request: RequestInit): Promise => { + if (lastEventId) { + request.headers = { ...request.headers, 'Last-Event-ID': lastEventId } + } + + return originalOnRequest(request) + } + + return client + } + + /** + * Pauses and resumes a session based on browser tab visibility. + * + * Uses the Page Visibility API (`document.visibilitychange`): + * + * - **hidden** — {@link abort} stops the active fetch. {@link messages} + * stays open; `"disconnected"` fires but `"closed"` does not. + * - **visible** — {@link connect} re-establishes the stream if needed. + * + * The listener is removed when {@link disconnect} emits `"closed"`, and + * re-attached automatically on the next `"connected"` event. + * + * No-op in non-browser environments where `document` is undefined. + * + * @param client - The session to manage. + */ + static addBrowserVisibilityHandler(client: SSESession): SSESession { + if (typeof document === 'undefined') return client + + const handleVisibilityChange = (): void => { + if (document.visibilityState === 'hidden') { + void client.abort() + + return + } + + client.connect().catch(() => { + // connect() reports failures via onError and the "error" event. + }) + } + + document.addEventListener('visibilitychange', handleVisibilityChange) + + // Stop managing visibility after an explicit disconnect; re-register + // when the same instance is manually connected again. + client.once('closed', () => { + document.removeEventListener('visibilitychange', handleVisibilityChange) + + client.once('connected', () => { + SSESession.addBrowserVisibilityHandler(client) + }) + }) + + return client + } + + /** SSE endpoint URL for this session. */ + private readonly url: string + + /** + * Per-instance configuration. + * + * Defaults live on the instance field (not a shared static) so each session + * gets its own {@link SSEEventParser} and {@link ExponentialBackoff}. + */ + public options: SSESessionOptions = { + fetch: (...args) => fetch(...args), + method: 'GET', + headers: { + Accept: 'text/event-stream', + 'Cache-Control': 'no-cache', + }, + body: new FormData(), + + onRequest: (request) => Promise.resolve(request), + onConnected: () => {}, + onDisconnected: () => {}, + onError: (error) => console.error('SSEClient error:', error), + + // Retry the initial fetch until it succeeds (maxAttempts: 0 = unlimited). + retry: new ExponentialBackoff({ + baseDelay: 1000, + maxDelay: 10000, + maxAttempts: 0, + growthRate: 1.3, + jitter: 0.3, + }), + + attemptReconnect: true, + persistent: false, + + eventParser: new SSEEventParser(), + } + + /** AbortController for the currently active fetch, if any. */ + private controller: AbortController | null = null + + /** + * Asynchronous stream of parsed SSE events for the active connection. + * + * Stays open across {@link abort} and automatic reconnects so an existing + * `for await` consumer keeps receiving events after visibility resumes. + * + * Closes when: + * - the server ends the stream and {@link SSESessionOptions.persistent} + * is false, + * - {@link disconnect} is called, or + * - a transport error occurs with + * {@link SSESessionOptions.attemptReconnect} disabled. + * + * A later {@link connect} replaces this with a new iterator when the + * previous one was closed. Consumers should read from `session.messages` + * rather than caching a reference across terminal disconnects. + */ + public messages: AsyncPushIterator = new AsyncPushIterator() + + public constructor(url: string, options: Partial = {}) { + super() + + this.url = url + this.options = { + ...this.options, + ...options, + // Shallow merge would drop default headers when options.headers is set. + headers: { ...this.options.headers, ...options.headers }, + } + } + + /** + * Connects or reconnects to the SSE endpoint. + * + * Resolves once the HTTP stream is established and `"connected"` has been + * emitted. Body reading continues asynchronously in the background via + * {@link readStream}. + * + * @throws When the fetch retry policy exhausts attempts or the connection + * is superseded before the reader is handed off (in the latter case the + * promise resolves without throwing). + */ + public async connect(): Promise { + // If there is already a controller present, we are already connected. + if (this.controller) return + + // Prepare for a fresh transport. Parser state from an abandoned connection + // must not bleed into the next one; reopen messages if a prior terminal + // close ended the consumer's iteration loop. + // NOTE: The resetEventParser may be out of spec and should not be done here: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation + this.options.eventParser.reset() + + // If the last messages iterator wasnt closed, we keep it around as this is likely a reconnect. + if (this.messages.closed) { + this.messages = new AsyncPushIterator() + } + + const controller = new AbortController() + this.controller = controller + + const { method, headers, body } = this.options + + const fetchBody = method === 'POST' ? body : null + + const fetchOptions: RequestInit = { + method, + headers: headers || {}, + body: fetchBody ?? null, + signal: controller.signal, + cache: 'no-store', + } + + let reader: ReadableStreamDefaultReader + + try { + reader = await this.options.retry.run(() => this.createReader(fetchOptions)) + } catch (error) { + // A newer abort/connect superseded this attempt — leave state to the winner. + if (this.controller !== controller) return + + this.controller = null + + await this.notifyDisconnected() + await this.notifyError(error) + this.messages.close() + + throw error + } + + // Connection succeeded but was already replaced (for example abort during fetch). + if (this.controller !== controller) { + await reader.cancel() + + return + } + + await SSESession.tryAsync( + () => this.options.onConnected(), + (error) => this.notifyError(error), + ) + this.emit('connected', undefined) + + // Fire-and-forget: connect() resolves while the stream is consumed. + this.readStream(reader, controller).catch((error) => { + this.options.onError(error) + }) + } + + /** + * Aborts only the currently active transport. + * + * The session remains reusable: {@link messages} stays open, visibility + * handling stays attached, and {@link connect} can reopen the stream. + * Partial parser state from the abandoned transport is discarded. + * + * Emits `"disconnected"` but not `"closed"`. + */ + public async abort(): Promise { + if (!this.controller) return + + // Grab the current controller to ensure we are aborting the correct one. + const controller = this.controller + this.controller = null + + // Invalidate any in-flight read loop and fetch for this transport. + controller.abort() + this.options.eventParser.reset() + + await this.notifyDisconnected() + } + + /** + * Terminates the session and disables attached visibility handling until + * the same instance is manually {@link connect connected} again. + * + * Closes {@link messages} and emits `"closed"`. + */ + public async disconnect(): Promise { + this.messages.close() + this.emit('closed', undefined) + + if (this.controller) { + await this.abort() + } else { + this.options.eventParser.reset() + } + } + + /** + * Performs the HTTP request and returns a reader for the response body. + * + * {@link SSESessionOptions.onRequest} may mutate headers (for example auth + * tokens or `Last-Event-ID`) before the fetch runs. + */ + private async createReader(fetchOptions: RequestInit): Promise> { + const requestOptions = await this.options.onRequest(fetchOptions) + const response = await this.options.fetch(this.url, requestOptions) + + if (!response.ok) { + const responseCode = response.status + const responseText = await response.text() + + const error = new Error(`HTTP error: ${responseCode} ${responseText}`) + void this.notifyError(error) + throw error + } + + if (!response.body) { + const error = new Error('HTTP error: Response body is null') + void this.notifyError(error) + throw error + } + + return response.body.getReader() + } + + /** + * Reads bytes from an established stream until it ends, errors, or is + * superseded by a newer connection. + */ + private async readStream(reader: ReadableStreamDefaultReader, controller: AbortController): Promise { + try { + while (this.controller === controller) { + const { done, value } = await reader.read() + + // abort() or a newer connect() may have landed while we were awaiting. + if (this.controller !== controller) return + + if (done) { + this.controller = null + + await this.notifyDisconnected() + + if (this.options.persistent) { + // Server closed gracefully — reopen unless the consumer opted out. + await this.connect() + } else { + this.messages.close() + } + + return + } + + // Some environments yield `{ done: false, value: undefined }`. + if (!value) continue + + for (const event of this.options.eventParser.parseEvents(value)) { + this.emit('message', event) + this.messages.push(event) + } + } + } catch (error) { + // If the controller is different, we already started a new connection and it would be confusing to handle this error. + if (controller !== this.controller) return + + // Invalidate the current controller to allow for reconnection if needed + this.controller = null + + await this.notifyDisconnected() + + // Expected path for abort() — do not treat as an error or reconnect. + if (controller.signal.aborted) return + + await this.notifyError(error) + + if (this.options.attemptReconnect) { + await this.connect() + } else { + this.messages.close() + } + } + } + + /** Invokes {@link SSESessionOptions.onDisconnected} and emits `"disconnected"`. */ + private async notifyDisconnected(): Promise { + await SSESession.tryAsync( + () => this.options.onDisconnected(), + (error) => this.options.onError(error), + ) + this.emit('disconnected', undefined) + } + + /** Invokes {@link SSESessionOptions.onError} and emits `"error"`. */ + private async notifyError(error: unknown): Promise { + const errorInstance = error instanceof Error ? error : new Error(String(error)) + + await SSESession.tryAsync( + () => this.options.onError(errorInstance), + (callbackError) => console.error('SSESession error:', callbackError), + ) + this.emit('error', errorInstance) + } + + /** Runs `fn`, passing any error to `onError` instead of throwing. */ + private static async tryAsync(fn: () => unknown, onError?: (error: Error) => void): Promise { + try { + await fn() + } catch (error) { + onError?.(error instanceof Error ? error : new Error(`${error}`)) + } + } +} diff --git a/src/utils/sse-session/types.ts b/src/utils/sse-session/types.ts new file mode 100644 index 0000000..997b0ce --- /dev/null +++ b/src/utils/sse-session/types.ts @@ -0,0 +1,145 @@ +// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c). +// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession. + +export type SSERequestInit = { + /** + * The HTTP method to use. + */ + method: 'GET' | 'POST' + + /** + * Request headers sent on every connect and reconnect. + */ + headers?: Record + + /** + * Request body for POST-based SSE endpoints. + */ + body?: string | FormData +} + +/** + * The fetch function to use. + * + * NOTE: This is compatible with Browser/Node's native "fetch" function. + * We use this in place of "typeof fetch" so that we can accept non-standard URLs ("url" is a "string" here). + * For example, a LibP2P adapter might not use a standardized URL format (and might only include "path"). + * This would cause a type error as native fetch expects type "URL". + */ +export type SSERequestFunction = { + fetch: (url: string, options: RequestInit) => Promise +} + +/** + * Lifecycle hooks invoked by {@link SSESession} during connect, read, and teardown. + */ +export type SSESessionCallbacks = { + /** + * Called before each fetch so callers can attach auth headers, cookies, or + * a `Last-Event-ID` for resume semantics. + */ + onRequest: (request: RequestInit) => Promise + + /** + * Called after the HTTP stream is established and before body reading begins. + */ + onConnected: () => void + + /** + * Called when the active transport ends — including {@link SSESession.abort}, + * server stream completion, and errors. Not paired with {@link SSESessionCallbacks.onConnected} + * when the initial connect never succeeds. + */ + onDisconnected: () => void + + /** + * Called on fetch or read failures. Not invoked for intentional + * {@link SSESession.abort} aborts. + */ + onError: (error: Error) => void +} + +export type SSESessionRetryInterface = { + /** + * Retry policy used while establishing the HTTP connection in + * {@link SSESession.connect}. Defaults to {@link ExponentialBackoff} with + * unlimited attempts. + */ + retry: { + run(fn: () => Promise, onError?: (error: Error) => void): Promise + } +} + +export interface SSEParser { + /** + * Incremental SSE frame parser for the response body. + * + * {@link SSEEventParser.reset} is called by the session when abandoning a + * transport so partial frames do not carry over to the next connection. + */ + eventParser: { + parseEvents(buffer: Uint8Array): SSEvent[] + reset(): void + } +} + +export type SSELifecycleOptions = { + /** + * When true, {@link SSESession} calls {@link SSESession.connect} again after + * a transport **error** (not an intentional abort). + */ + attemptReconnect: boolean + + /** + * When true, {@link SSESession} calls {@link SSESession.connect} again after + * the **server** closes the stream normally (`done`). + */ + persistent: boolean +} + +/** + * Events emitted by {@link SSESession}. + * + * - `"connected"` — HTTP stream established. + * - `"message"` — A complete SSE event was parsed. + * - `"disconnected"` — The active transport ended (including {@link SSESession.abort}). + * - `"error"` — An unexpected fetch or read failure. + * - `"closed"` — {@link SSESession.disconnect} was called; visibility handling is detached. + */ +export type SSESessionEventMap = { + connected: void + disconnected: void + error: Error + message: SSEvent + closed: void +} + +/** + * Configuration for {@link SSESession}. + */ +export type SSESessionOptions = SSESessionCallbacks & SSERequestInit & SSERequestFunction & SSESessionRetryInterface & SSELifecycleOptions & SSEParser + +/** + * Represents a Server-Sent Event. + */ +export interface SSEvent { + /** + * Event data. + */ + data: string + + /** + * Event type. + */ + event?: string + + /** + * Event ID. + */ + id?: string + + /** + * Reconnection time in milliseconds. + */ + retry?: number +} diff --git a/test/helpers.ts b/test/helpers.ts index 0c00824..65550dc 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -10,6 +10,7 @@ export async function collect(iterable: AsyncIterable): Promise { /** * Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern * (e.g. an encryption layer): extend BaseKVStorage and delegate the five primitives. + * Counts are reported under the public method names (writeMany, removeMany, writeIfAbsent). */ export class KVStorageCounting extends BaseKVStorage { calls: Record = {} @@ -25,13 +26,13 @@ export class KVStorageCounting extends BaseKVStorage { readMany(keys: string[]) { return this.count('readMany', () => this.inner.readMany(keys)) } - writeMany(entries: [string, unknown][]) { + protected applyWrite(entries: [string, unknown][]) { return this.count('writeMany', () => this.inner.writeMany(entries)) } - removeMany(keys: string[]) { + protected applyRemove(keys: string[]) { return this.count('removeMany', () => this.inner.removeMany(keys)) } - writeIfAbsent(entries: [string, unknown][]) { + protected applyWriteIfAbsent(entries: [string, unknown][]) { return this.count('writeIfAbsent', () => this.inner.writeIfAbsent(entries)) } scan(options?: ScanOptions) { diff --git a/test/kv-storage/kv-storage-composite.test.ts b/test/kv-storage/kv-storage-composite.test.ts new file mode 100644 index 0000000..c6bbd22 --- /dev/null +++ b/test/kv-storage/kv-storage-composite.test.ts @@ -0,0 +1,144 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js' +import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' +import { collect } from '../helpers.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' +import { TestNetwork } from './sync-helpers.js' + +runKVStorageConformance( + 'KVStorageComposite (in-memory source, 1 replica)', + () => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [new KVStorageInMemory()] }), +) +runKVStorageConformance('KVStorageComposite (http source, 2 replicas)', () => { + network = new TestNetwork() + return new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), new KVStorageInMemory()] }) +}) + +let network: TestNetwork +const composites: KVStorageComposite[] = [] +afterEach(async () => { + await Promise.all(composites.splice(0).map((c) => c.close())) + await network?.close() +}) + +function composite(persistent: BaseKVStorage = new KVStorageInMemory()) { + const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], retry: { baseDelay: 5, maxDelay: 20 } }) + composites.push(c) + return c +} + +describe('KVStorageComposite sync', () => { + it('loads existing data with one full sync, then serves reads locally', async () => { + network = new TestNetwork() + await network.store.writeMany([ + ['users/1', { name: 'Ada' }], + ['users/2', { name: 'Bob' }], + ]) + const db = composite() + await db.synced() + network.store.reset() + expect(await db.child('users').get('1')).toEqual({ name: 'Ada' }) + expect(await collect(db.child('users').keys())).toEqual(['1', '2']) + expect(network.store.calls).toEqual({}) + }) + + it('applies changes pushed from other clients to every replica', async () => { + network = new TestNetwork() + const memory = new KVStorageInMemory() + const db = composite(memory) + await db.synced() + const seen = db.waitFor('change', (e) => e.origin === 'remote') + await network.client().child('users').set('9', { name: 'Remote' }) + expect((await seen).changes).toEqual([{ key: 'users/9', value: { name: 'Remote' } }]) + expect(await db.child('users').get('9')).toEqual({ name: 'Remote' }) + expect(await memory.readMany(['users/9'])).toEqual([{ name: 'Remote' }]) + }) + + it('queues writes while offline and replays them in order when back online', async () => { + network = new TestNetwork() + const db = composite() + await db.synced() + network.online = false + await db.child('users').set('1', { v: 1 }) + await db.child('users').set('1', { v: 2 }) + await db.child('users').delete('1') + await db.child('users').set('2', { v: 3 }) + expect(db.queued).toBe(4) + expect(await db.child('users').get('2')).toEqual({ v: 3 }) // served locally + await expect(db.writeIfAbsent([['x', 1]])).rejects.toThrow(/Cannot reach/) + network.online = true + await vi.waitFor(() => expect(db.queued).toBe(0)) + expect(await network.store.readMany(['users/1', 'users/2'])).toEqual([undefined, { v: 3 }]) + }) + + it('keeps queued local values when the server pushes an older value for the same key', async () => { + network = new TestNetwork() + const db = composite() + await db.synced() + network.online = false + await db.child('k').set({ from: 'local' }) + await network.store.writeMany([['k', { from: 'remote' }]]) // pushed over the still-open stream + await new Promise((r) => setTimeout(r, 20)) + expect(await db.child('k').get()).toEqual({ from: 'local' }) + network.online = true + await vi.waitFor(() => expect(db.queued).toBe(0)) + expect(await network.store.readMany(['k'])).toEqual([{ from: 'local' }]) + }) + + it('restarts from the persistent replica and resumes without a full sync', async () => { + network = new TestNetwork() + const persistent = new KVStorageInMemory() + const first = composite(persistent) + await first.child('a').set(1) + await first.synced() + await first.close() + await network.client().writeMany([['b', 2]]) // missed while "closed" + + network.store.reset() + const second = composite(persistent) + expect(await second.child('a').get()).toBe(1) // available before contacting the server + await vi.waitFor(async () => expect(await second.child('b').get()).toBe(2)) + expect(network.store.calls.scan).toBeUndefined() // resumed via Last-Event-ID, no full scan + }) + + it('falls back to a full sync when the server cannot resume', async () => { + network = new TestNetwork({ historySize: 1 }) + const persistent = new KVStorageInMemory() + const first = composite(persistent) + await first.child('a').set(1) + await first.synced() + await first.close() + await network.client().writeMany([['b', 2]]) + await network.client().writeMany([['c', 3]]) + await network.client().removeMany(['a']) + const second = composite(persistent) + await vi.waitFor(async () => expect(await collect(second.keys())).toEqual(['b', 'c'])) + }) + + it('replays an outbox left by a previous session', async () => { + network = new TestNetwork() + const persistent = new KVStorageInMemory() + const first = composite(persistent) + await first.synced() + network.online = false + await first.child('offline').set('queued') + await first.close() + network.online = true + const second = composite(persistent) + await second.start() + await vi.waitFor(async () => expect(await network.store.readMany(['offline'])).toEqual(['queued'])) + }) + + it('serves document queries from the local replica, offline too', async () => { + network = new TestNetwork() + const db = composite() + const users = new DocumentStorageKV(db).child('users') + await users.set('1', { age: 30 }) + await users.set('2', { age: 40 }) + await db.synced() + network.online = false + expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2']) + }) +}) diff --git a/test/kv-storage/kv-storage-http-watch.test.ts b/test/kv-storage/kv-storage-http-watch.test.ts new file mode 100644 index 0000000..889a9e9 --- /dev/null +++ b/test/kv-storage/kv-storage-http-watch.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js' +import { StorageUnavailableError } from '../../src/utils/errors.js' +import { TestNetwork } from './sync-helpers.js' + +let network: TestNetwork +afterEach(() => network?.close()) + +describe('KVStorageHttp.watch (SSE)', () => { + it('receives changes made by other clients as remote change events', async () => { + network = new TestNetwork() + const watcher = network.client() + await watcher.watch() + const received = watcher.waitFor('change', (e) => e.origin === 'remote') + await network.client().child('users').set('1', { name: 'Ada', n: 1n }) + const event = await received + expect(event.changes).toEqual([{ key: 'users/1', value: { name: 'Ada', n: 1n } }]) + expect(event.id).toMatch(/^\w+-1$/) + expect(watcher.lastEventId).toBe(event.id) + }) + + it('resumes from lastEventId and replays missed changes', async () => { + network = new TestNetwork() + const writer = network.client() + const watcher = network.client() + await watcher.watch() + const first = watcher.waitFor('change', (e) => e.origin === 'remote') + await writer.writeMany([['a', 1]]) + await first + await watcher.unwatch() + await writer.writeMany([['b', 2]]) + await writer.writeMany([['c', 3]]) + const replayed: KVChangeEvent[] = [] + watcher.on('change', (e) => replayed.push(e)) + await watcher.watch() // resolves once caught up, so the missed changes have been emitted + expect(replayed.map((e) => e.changes)).toEqual([[{ key: 'b', value: 2 }], [{ key: 'c', value: 3 }]]) + }) + + it('asks for a resync when the missed changes are no longer in the history', async () => { + network = new TestNetwork({ historySize: 1 }) + const writer = network.client() + const watcher = network.client() + await watcher.watch() + const seen = watcher.waitFor('change', () => true) + await writer.writeMany([['a', 1]]) + await seen + await watcher.unwatch() + await writer.writeMany([['b', 2]]) + await writer.writeMany([['c', 3]]) + const resync = watcher.waitFor('resync', () => true) + await watcher.watch() + expect(await resync).toEqual({ id: expect.stringMatching(/-3$/) }) + }) + + it('reports an unreachable server as StorageUnavailableError', async () => { + network = new TestNetwork() + const client = network.client() + network.online = false + await expect(client.readMany(['x'])).rejects.toBeInstanceOf(StorageUnavailableError) + }) +}) diff --git a/test/kv-storage/kv-storage-http.test.ts b/test/kv-storage/kv-storage-http.test.ts index 8a5a3b9..2dc295e 100644 --- a/test/kv-storage/kv-storage-http.test.ts +++ b/test/kv-storage/kv-storage-http.test.ts @@ -9,7 +9,7 @@ import { runKVStorageConformance } from './kv-storage.conformance.js' import { serve } from './node-server.js' runKVStorageConformance('KVStorageHttp (in-process)', () => { - const server = new KVStorageHttpServer(new KVStorageInMemory(), '/kv') + const server = new KVStorageHttpServer(new KVStorageInMemory(), { basePath: '/kv' }) return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) }) }) @@ -22,7 +22,7 @@ describe('over a real socket', () => { }) afterAll(() => server.close()) const fresh = () => { - stores.push(new KVStorageHttpServer(new KVStorageInMemory(), `/${stores.length}`)) + stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { basePath: `/${stores.length}` })) return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` }) } diff --git a/test/kv-storage/kv-storage-in-memory.test.ts b/test/kv-storage/kv-storage-in-memory.test.ts index 68c09a5..20f5e2a 100644 --- a/test/kv-storage/kv-storage-in-memory.test.ts +++ b/test/kv-storage/kv-storage-in-memory.test.ts @@ -76,3 +76,12 @@ describe('model-based: random writes vs a reference map', () => { ) }) }) + +describe('child nodes', () => { + it('are frozen, so implementations cannot accidentally shadow shared state', () => { + const child = new KVStorageInMemory().child('a').child('b') + expect(Object.isFrozen(child)).toBe(true) + expect(() => ((child as unknown as { state: number }).state = 1)).toThrow(TypeError) + expect(child.key).toBe('a/b') + }) +}) diff --git a/test/kv-storage/kv-storage-indexeddb.test.ts b/test/kv-storage/kv-storage-indexeddb.test.ts new file mode 100644 index 0000000..8fe639c --- /dev/null +++ b/test/kv-storage/kv-storage-indexeddb.test.ts @@ -0,0 +1,8 @@ +import 'fake-indexeddb/auto' +import { KVStorageIndexedDB } from '../../src/kv-storage/kv-storage-indexeddb.js' +import { runKVStorageConformance } from './kv-storage.conformance.js' + +let n = 0 +runKVStorageConformance('KVStorageIndexedDB', () => new KVStorageIndexedDB({ name: `db-${n++}` })) +// Tiny batches exercise resuming scans across transactions. +runKVStorageConformance('KVStorageIndexedDB (scan batches of 2)', () => new KVStorageIndexedDB({ name: `db-${n++}`, scanBatchSize: 2 })) diff --git a/test/kv-storage/kv-storage.conformance.ts b/test/kv-storage/kv-storage.conformance.ts index 939afe4..f5814bd 100644 --- a/test/kv-storage/kv-storage.conformance.ts +++ b/test/kv-storage/kv-storage.conformance.ts @@ -71,6 +71,28 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c']) }) + it('emits change events for writes and removals', async () => { + const events: unknown[] = [] + const off = db.on('change', (event) => events.push(event)) + await db.writeMany([['x', 1]]) + await db.removeMany(['x']) + await db.writeIfAbsent([ + ['a', 'no'], + ['y', 2], + ]) + off() + expect(events).toEqual([ + { changes: [{ key: 'x', value: 1 }], origin: 'local' }, + { changes: [{ key: 'x' }], origin: 'local' }, + { changes: [{ key: 'y', value: 2 }], origin: 'local' }, + ]) + }) + + it('stores bigint and Uint8Array values', async () => { + await db.writeMany([['bin', { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }]]) + expect(await db.readMany(['bin'])).toEqual([{ n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }]) + }) + it('does not share references with callers', async () => { const [value] = (await db.readMany(['a/2'])) as [{ n: string }] value.n = 'changed' diff --git a/test/kv-storage/sync-helpers.ts b/test/kv-storage/sync-helpers.ts new file mode 100644 index 0000000..e37e1a3 --- /dev/null +++ b/test/kv-storage/sync-helpers.ts @@ -0,0 +1,35 @@ +import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js' +import { KVStorageHttpServer, type KVStorageHttpServerOptions } from '../../src/kv-storage/kv-storage-http-server.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { KVStorageCounting } from '../helpers.js' + +/** + * An in-process server with a network switch. While `online` is false, new requests fail like a + * dropped connection (open SSE streams stay up, so pushes can still be simulated). + */ +export class TestNetwork { + online = true + readonly store = new KVStorageCounting(new KVStorageInMemory()) + readonly server: KVStorageHttpServer + private readonly clients: KVStorageHttp[] = [] + + constructor(options: KVStorageHttpServerOptions = {}) { + this.server = new KVStorageHttpServer(this.store, options) + } + + client(): KVStorageHttp { + const client = new KVStorageHttp({ + url: 'http://server', + fetch: async (url, init) => { + if (!this.online) throw new TypeError('fetch failed') + return this.server.handle(new Request(url, init)) + }, + }) + this.clients.push(client) + return client + } + + async close(): Promise { + await Promise.all(this.clients.map((c) => c.unwatch())) + } +} diff --git a/test/utils/event-emitter.test.ts b/test/utils/event-emitter.test.ts new file mode 100644 index 0000000..f6b47f5 --- /dev/null +++ b/test/utils/event-emitter.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventEmitter } from '../../src/utils/event-emitter.js' + +type Events = { tick: number; done: void } + +describe('EventEmitter', () => { + it('delivers payloads as-is (no cloning or freezing) and supports off', () => { + const emitter = new EventEmitter() + const seen: unknown[] = [] + const payload = { n: 1 } + const off = emitter.on('obj', (p) => seen.push(p)) + expect(emitter.emit('obj', payload)).toBe(true) + off() + expect(emitter.emit('obj', payload)).toBe(false) + expect(seen[0]).toBe(payload) + expect(Object.isFrozen(payload)).toBe(false) + }) + + it('once fires a single time; listener errors are logged, not thrown', () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const emitter = new EventEmitter() + const once = vi.fn() + emitter.once('tick', once) + emitter.on('tick', () => { + throw new Error('boom') + }) + emitter.emit('tick', 1) + emitter.emit('tick', 2) + expect(once).toHaveBeenCalledTimes(1) + expect(error).toHaveBeenCalledTimes(2) + error.mockRestore() + }) + + it('debounces and cancels pending debounced calls on off', async () => { + vi.useFakeTimers() + const emitter = new EventEmitter() + const listener = vi.fn() + const off = emitter.on('tick', listener, 50) + emitter.emit('tick', 1) + emitter.emit('tick', 2) + vi.advanceTimersByTime(60) + expect(listener).toHaveBeenCalledExactlyOnceWith(2) + emitter.emit('tick', 3) + off() + vi.advanceTimersByTime(60) + expect(listener).toHaveBeenCalledTimes(1) + vi.useRealTimers() + }) + + it('waitFor resolves on a matching payload and rejects on timeout', async () => { + const emitter = new EventEmitter() + const waiting = emitter.waitFor('tick', (n) => n > 1) + emitter.emit('tick', 1) + emitter.emit('tick', 2) + expect(await waiting).toBe(2) + await expect(emitter.waitFor('done', () => true, 5)).rejects.toThrow(/Timed out waiting for "done"/) + }) +}) diff --git a/test/utils/extended-json.test.ts b/test/utils/extended-json.test.ts new file mode 100644 index 0000000..c4dd21f --- /dev/null +++ b/test/utils/extended-json.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { ExtendedJson } from '../../src/utils/extended-json.js' + +describe('ExtendedJson', () => { + it('round-trips bigint and Uint8Array, nested anywhere', () => { + const value = { a: 1n, b: [new Uint8Array([0, 1, 255]), -(2n ** 70n)], c: { d: new Uint8Array(0), e: 'plain', f: null } } + expect(ExtendedJson.stringify({ n: 5n, bytes: new Uint8Array([171, 205]) })).toBe('{"n":"","bytes":""}') + expect(ExtendedJson.parse(ExtendedJson.stringify(value))).toEqual(value) + }) + + it('round-trips arbitrary data (property)', () => { + const data = fc.letrec((tie) => ({ + value: fc.oneof( + { depthSize: 'small' }, + fc.bigInt(), + fc.uint8Array(), + fc.string(), + fc.double({ noNaN: true, noDefaultInfinity: true }), + fc.boolean(), + fc.constant(null), + fc.array(tie('value')), + fc.dictionary(fc.string(), tie('value')), + ), + })).value + fc.assert( + fc.property(data, (v) => { + expect(ExtendedJson.parse(ExtendedJson.stringify(v))).toEqual(v) + }), + ) + }) +}) diff --git a/test/utils/sse-session.test.ts b/test/utils/sse-session.test.ts new file mode 100644 index 0000000..38cba5b --- /dev/null +++ b/test/utils/sse-session.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { SSEEventParser } from '../../src/utils/sse-session/sse-event-parser.js' +import { SSESession } from '../../src/utils/sse-session/sse-session.js' + +const encode = (text: string) => new TextEncoder().encode(text) + +describe('vendored sse-session', () => { + it('parses events across arbitrary chunk boundaries', () => { + const parser = new SSEEventParser() + const text = 'id: 1\nevent: change\ndata: {"a":1}\n\n: comment\ndata: line1\ndata: line2\n\n' + const events = [] + for (let i = 0; i < text.length; i += 3) events.push(...parser.parseEvents(encode(text.slice(i, i + 3)))) + expect(events).toEqual([{ id: '1', event: 'change', data: '{"a":1}' }, { data: 'line1\nline2' }]) + }) + + it('connects, delivers messages and reconnects when persistent', async () => { + let connections = 0 + const session = new SSESession('http://test/events', { + persistent: true, + fetch: async () => { + connections++ + return new Response(encode(`data: hello ${connections}\n\n`)) + }, + }) + const received: string[] = [] + session.on('message', (m) => received.push(m.data)) + await session.connect() + await session.waitFor('message', (m) => m.data === 'hello 2') + await session.disconnect() + expect(received.slice(0, 2)).toEqual(['hello 1', 'hello 2']) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index a33d2d0..568f991 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], "types": ["node"], "strict": true, "noUncheckedIndexedAccess": true, -- 2.54.0 From df7de27e20b1e0fb836b69d6c1fddf73385873e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:47:18 +0000 Subject: [PATCH 03/11] Composite: don't undo writes made while a full sync is scanning A write made during a full sync could be scanned in the replica but not in the (read-ahead) source scan, so the diff removed it, and echo tracking then consumed the server's echo. Keys written during a sync are now left out of the diff. Deterministic regression test with a read-ahead source. Also exclude -0 from the Extended JSON property test (JSON has no -0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- DESIGN.md | 2 +- src/kv-storage/kv-storage-composite.ts | 16 +++++++-- test/kv-storage/kv-storage-composite.test.ts | 37 +++++++++++++++++++- test/utils/extended-json.test.ts | 2 +- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index bca1a8c..9725b38 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -69,7 +69,7 @@ emit a `change` event after each write, so every store emits change events. | `applyRemove(keys)` → public `removeMany` | | | `applyWriteIfAbsent(entries)` → public `writeIfAbsent` | `true` = written; atomic per key (needed by events) | -**Values** are JSON plus `bigint` and `Uint8Array`. Anything crossing the network is encoded as +**Values** are JSON plus `bigint` and `Uint8Array` (as in JSON, `-0` becomes `0` and `undefined` properties are dropped). Anything crossing the network is encoded as Extended JSON (``, ``); IndexedDB stores them natively. **Events** (`EventEmitter`, shared by all nodes of a storage): `change` → diff --git a/src/kv-storage/kv-storage-composite.ts b/src/kv-storage/kv-storage-composite.ts index a61ab3d..3b7175e 100644 --- a/src/kv-storage/kv-storage-composite.ts +++ b/src/kv-storage/kv-storage-composite.ts @@ -85,6 +85,12 @@ export class KVStorageComposite extends BaseKVStorage< * changed during the scan. */ applying: Promise + /** + * Keys written through the composite while a full sync runs. The diff may have scanned the + * source before such a write and the replica after it; these keys are left out of the diff + * (the write already reached the source and the replicas). + */ + writtenDuringSync?: Set replaying?: Promise outboxSeq: number retryTimer?: ReturnType @@ -162,7 +168,9 @@ export class KVStorageComposite extends BaseKVStorage< protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { await this.start() const written = await this.source.writeIfAbsent(entries) - await this.applyToReplicas(entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value }))) + const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value })) + for (const { key } of changes) this.state.writtenDuringSync?.add(key) + await this.applyToReplicas(changes) return written } @@ -243,11 +251,12 @@ export class KVStorageComposite extends BaseKVStorage< private async fullSync(): Promise { // The diff re-reads the source, so outstanding echoes no longer matter (and may never arrive). this.echoes.clear() + const writtenDuringSync = (this.state.writtenDuringSync = new Set()) try { const id = this.watchable ? (this.source as unknown as WatchableSource).lastEventId : undefined let changes: KVChange[] = [] const flush = async () => { - const applied = changes.filter(({ key }) => !this.pending.has(key)) + const applied = changes.filter(({ key }) => !this.pending.has(key) && !writtenDuringSync.has(key)) await this.applyToReplicas(applied) if (applied.length) this.emit('change', { changes: applied, origin: 'remote' }) changes = [] @@ -260,6 +269,8 @@ export class KVStorageComposite extends BaseKVStorage< } catch (error) { if (!(error instanceof StorageUnavailableError)) throw error this.scheduleRetry(() => this.enqueueApply(() => this.fullSync())) + } finally { + this.state.writtenDuringSync = undefined } } @@ -296,6 +307,7 @@ export class KVStorageComposite extends BaseKVStorage< // ------------------------------------------------------------------ local writes → source private async writeThrough(entry: OutboxEntry): Promise { + for (const key of KVStorageComposite.keysOf(entry)) this.state.writtenDuringSync?.add(key) if (this.queued === 0) { try { await this.send(entry) diff --git a/test/kv-storage/kv-storage-composite.test.ts b/test/kv-storage/kv-storage-composite.test.ts index c6bbd22..aef18cb 100644 --- a/test/kv-storage/kv-storage-composite.test.ts +++ b/test/kv-storage/kv-storage-composite.test.ts @@ -3,7 +3,7 @@ import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js' import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js' -import { collect } from '../helpers.js' +import { collect, KVStorageCounting } from '../helpers.js' import { runKVStorageConformance } from './kv-storage.conformance.js' import { TestNetwork } from './sync-helpers.js' @@ -141,4 +141,39 @@ describe('KVStorageComposite sync', () => { network.online = false expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2']) }) + + it('does not undo writes made while a full sync is scanning', async () => { + // A source whose scan reads ahead (like the HTTP stream) and pauses after its first entry. + let release!: () => void + let reached!: () => void + const gate = new Promise((r) => (release = r)) + const paused = new Promise((r) => (reached = r)) + const inner = new KVStorageInMemory() + await inner.writeMany([ + ['a', 1], + ['c', 3], + ]) + const source = new KVStorageCounting(inner) + const scan = source.scan.bind(source) + source.scan = async function* (options) { + let first = true + for (const entry of await collect(scan(options))) { + yield entry + if (first && options?.values) ((first = false), reached(), await gate) + } + } + const persistent = new KVStorageInMemory() + await persistent.writeMany([['a0', 'stale']]) // a replica copy from before, without sync metadata + const db = new KVStorageComposite({ source, replicas: [persistent] }) + void db.start() + await paused // the diff has passed 'b' in the source but not in the replica + await db.writeMany([['b', 2]]) + release() + await db.synced() + expect(await collect(db.scan({ values: true }))).toEqual([ + ['a', 1], + ['b', 2], + ['c', 3], + ]) + }) }) diff --git a/test/utils/extended-json.test.ts b/test/utils/extended-json.test.ts index c4dd21f..e85dc28 100644 --- a/test/utils/extended-json.test.ts +++ b/test/utils/extended-json.test.ts @@ -16,7 +16,7 @@ describe('ExtendedJson', () => { fc.bigInt(), fc.uint8Array(), fc.string(), - fc.double({ noNaN: true, noDefaultInfinity: true }), + fc.double({ noNaN: true, noDefaultInfinity: true }).map((x) => (x === 0 ? 0 : x)) /* JSON has no -0 */, fc.boolean(), fc.constant(null), fc.array(tie('value')), -- 2.54.0 From a8c9539ad924d5e96d66f501f7bd72fe086b3323 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:01:06 +0000 Subject: [PATCH 04/11] Authenticated, scoped sync: signed requests and per-path subscriptions - RequestAuth: clients sign `${timestamp}:${path}:${body}` (as in sync-server-example); headers carry public key, timestamp, signature and algorithm. Pluggable signers/verifiers; Ed25519 via WebCrypto built in. Timestamp window and replay protection. - KVStorageHttpServer requires auth ('none' must be explicit). With auth, every key, scan range and subscription path must be under the caller's public key. - POST /events with { paths }: subscribers only receive (and on resume only replay) changes under their authorized paths. - KVStorageHttp: signer option; subscribe(path)/unsubscribe(path)/close() replace watch(); 401/403 reject instead of retrying. - KVStorageComposite: prefix option scopes sync, cache and subscription. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- DESIGN.md | 53 +++++- src/index.ts | 2 + src/kv-storage/kv-storage-composite.ts | 67 +++++-- src/kv-storage/kv-storage-http-server.ts | 141 +++++++++++---- src/kv-storage/kv-storage-http.ts | 171 +++++++++++++----- src/utils/errors.ts | 11 ++ src/utils/extended-json.ts | 18 +- src/utils/hex.ts | 16 ++ src/utils/request-auth.ts | 150 +++++++++++++++ test/kv-storage/kv-storage-http-auth.test.ts | 95 ++++++++++ test/kv-storage/kv-storage-http-watch.test.ts | 16 +- test/kv-storage/kv-storage-http.test.ts | 6 +- test/kv-storage/sync-helpers.ts | 10 +- test/utils/request-auth.test.ts | 69 +++++++ 14 files changed, 696 insertions(+), 129 deletions(-) create mode 100644 src/utils/hex.ts create mode 100644 src/utils/request-auth.ts create mode 100644 test/kv-storage/kv-storage-http-auth.test.ts create mode 100644 test/utils/request-auth.test.ts diff --git a/DESIGN.md b/DESIGN.md index 9725b38..b919ff4 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -94,26 +94,58 @@ Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / ` `handle(Request) → Response`): `POST /` with Extended JSON bodies; `scan` streams NDJSON. Unreachable server (fetch failure, 502/503/504) → `StorageUnavailableError`. The server exposes the whole underlying store (primitives use absolute keys). - - `KVStorageComposite`: see 4.1. -- **Change stream (SSE).** `GET /events` on the server streams every change to the wrapped storage - as `event: change` with id `-` (epoch = server instance). The server keeps the last + - `KVStorageComposite`: see 4.2. +- **Change stream (SSE).** `POST /events` with `{ paths }` streams changes to those keys and their + subtrees as `event: change` with id `-` (epoch = server instance). The server keeps the last `historySize` (10 000) change batches. - A client reconnecting with `Last-Event-ID` gets the batches it missed, or `event: resync` if it can't be resumed (another epoch, or too old). - Every catch-up ends with `event: ready`. - Keep-alive comments are sent every 15 s. - - `KVStorageHttp.watch()` subscribes (via the vendored `SSESession`, which reconnects with - backoff), emits `change` events with `origin: 'remote'` and `resync`, and resolves once caught up. + - `KVStorageHttp.subscribe(path = this.key)` / `unsubscribe(path)` manage the subscribed paths. + Each change reconnects the stream (vendored `SSESession`, backoff) with the new list, resuming + from `lastEventId`. The client emits `change` (`origin: 'remote'`) and `resync`, and `subscribe` + resolves once caught up. A 401/403 rejects `subscribe` instead of retrying. - Limitation: a server only sees writes made through its own storage instance, so several server processes over one database need a shared pub/sub (not built). - **Wrappers** (e.g. a future encryption layer) extend `BaseKVStorage` and delegate the primitives. Keys must stay plaintext (or be deterministically encrypted) for prefix scans to work; values can be encrypted freely. -### 4.1 `KVStorageComposite({ source, replicas, retry? })` +### 4.1 Authentication (`utils/request-auth.ts`) -Keeps full local copies of a source (e.g. `KVStorageHttp`) so reads are local and the app works -offline. Replicas are listed fastest first (e.g. `[inMemory, indexedDb]`). The last replica is the +The server requires `auth`: either `{ verifiers, maxSkewMs?, now? }`, or `'none'` (open, for local +development and tests). It can't be left open by omission. + +- **Signed requests.** The client's `RequestSigner` signs `${timestamp}:${path}:${body}` (path = + URL pathname, body = the exact request body; same scheme as Harvmaster/sync-server-example). + Headers: `X-Public-Key` (lowercase hex), `X-Timestamp` (ms), `X-Signature` (hex) and + `X-Signature-Algorithm`. Every request is signed, and so is each SSE (re)connect, with a fresh + timestamp. +- **Verification (401).** The server picks the `SignatureVerifier` registered for the algorithm + (unknown → 401) and checks the timestamp is within `maxSkewMs` (default 60 s). It then rejects a + signature already accepted within that window (replay protection), and verifies the signature + over the received path and raw body. The body is parsed only afterwards. +- **Algorithms are pluggable.** Built in: `Ed25519Signer` / `Ed25519Verifier` (WebCrypto, so no + dependencies; Node 22+ and current browsers; keys persist as PKCS #8). Others, e.g. secp256k1 via + libauth, implement `RequestSigner` (`algorithm`, `publicKey`, `sign`) and `SignatureVerifier` + (`algorithm`, `verify`). The server can accept several algorithms at once. +- **Authorization (403): the public key is the user id.** Every key in `readMany`, `writeMany`, + `removeMany` and `writeIfAbsent`, and every subscription path, must be `` or under + `/`. A scan must stay within `[, 0)`, i.e. the key and its + subtree (escaping guarantees no other key falls in that range, including lookalikes such as + `x`). +- **Scoped events.** Each subscriber only receives changes under its own (authorized) paths, live + and when resuming. Event ids are a global sequence, so a client can infer the server's overall + write volume, though nothing about the contents. +- Proxies must preserve the URL path, because the signature covers it. + +### 4.2 `KVStorageComposite({ source, prefix?, replicas, retry? })` + +Keeps local copies of a source (e.g. `KVStorageHttp`) so reads are local and the app works +offline. With `prefix` (e.g. the user's public key) only that key and its subtree are synced, cached +and subscribed to; keys outside it throw `RangeError`. Without it, the whole source is copied, which +is only appropriate when the client may read all of it. Replicas are listed fastest first (e.g. `[inMemory, indexedDb]`). The last replica is the persistent one: it seeds the others on start and stores the composite's internal keys under `~q/` (hidden from reads): `~q/meta` (`{ lastEventId }`, present once a sync has completed) and the outbox `~q/o/`. @@ -222,8 +254,9 @@ Node API: `get(key?)`, `getMany`, `set(doc)` / `set(key, doc)`, `insert(doc)` ## 8. Open questions -- Composite: replicate a subtree instead of the whole store; conflict policies beyond last writer - wins for offline writes. +- Composite: lazily syncing parts of a large subtree on first read; conflict policies beyond last + writer wins for offline writes. +- Authorization beyond "public key = user id" (sharing a subtree with other keys). - Multi-process servers need a shared change feed (pub/sub) for SSE. - Hard deletes from immutable event logs (privacy requests): compaction vs crypto-shredding. The diff --git a/src/index.ts b/src/index.ts index 4ef085e..966cb34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,8 @@ export * from './utils/document-update.js' export * from './utils/errors.js' export * from './utils/event-emitter.js' export * from './utils/extended-json.js' +export * from './utils/hex.js' +export * from './utils/request-auth.js' export * from './utils/json.js' export * from './utils/keys.js' export * from './utils/schema.js' diff --git a/src/kv-storage/kv-storage-composite.ts b/src/kv-storage/kv-storage-composite.ts index 3b7175e..c698f80 100644 --- a/src/kv-storage/kv-storage-composite.ts +++ b/src/kv-storage/kv-storage-composite.ts @@ -1,13 +1,20 @@ import { StorageUnavailableError } from '../utils/errors.js' import type { OffCallback } from '../utils/event-emitter.js' import { Json } from '../utils/json.js' -import type { AnySpec, Spec } from '../utils/schema.js' +import { Keys } from '../utils/keys.js' +import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js' import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js' import { BaseKVStorage, type KVChange, type KVChangeEvent, type ScanOptions } from './base-kv-storage.js' export interface KVStorageCompositeOptions { /** The source of truth (e.g. `KVStorageHttp`). */ source: BaseKVStorage + /** + * Only this key and its subtree are synced, cached and subscribed to (e.g. the user's public + * key). Keys outside it are rejected. Default `''`: the whole source (only for sources the + * client may read entirely). + */ + prefix?: string /** * Local full copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be * persistent: it also stores the offline outbox and sync metadata, and seeds the others on start. @@ -19,8 +26,8 @@ export interface KVStorageCompositeOptions { /** Implemented by sources that can push remote changes (e.g. `KVStorageHttp`). */ interface WatchableSource { - watch(): Promise - unwatch(): Promise + subscribe(path: string): Promise + unsubscribe(path: string): Promise lastEventId: string | undefined } @@ -38,16 +45,22 @@ const OUTBOX_PREFIX = '~q/o/' const OUTBOX_END = '~q/o0' const SYNC_BATCH = 500 +// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates them with +// Object.create, so they share this class's prototype). +export interface KVStorageComposite { + child>(key: K): KVStorageComposite> +} + /** * Keeps local replicas of a source in sync, so reads (including `keys()`, `entries()` and document * queries) are served locally, and the app keeps working offline. * * - **Start** (lazily, on first use): the persistent replica seeds the faster ones. If it holds a * previous sync, reads are served from it straight away (possibly stale until caught up). - * - **Staying in sync:** remote changes pushed by the source (`watch()`, e.g. SSE) are applied to + * - **Staying in sync:** remote changes pushed by the source (`subscribe(prefix)`, e.g. SSE) are applied to * every replica. On restart the source resumes from the saved `lastEventId`; a full sync (diffing * the source against the replicas) only runs the first time, or when the source says it can't - * resume (`resync`). Without `watch()`, a full sync runs on start. + * resume (`resync`). Without `subscribe()`, a full sync runs on start. * - **Writes** go to the source, then the replicas. If the source is unreachable * (`StorageUnavailableError`), they are applied locally and queued in a persisted outbox, then * replayed in order when it's reachable again (last writer wins on conflict). While any write is @@ -59,6 +72,8 @@ const SYNC_BATCH = 500 */ export class KVStorageComposite extends BaseKVStorage { private readonly source: BaseKVStorage + /** Synced key range: `[prefix, prefix + '0')` is the prefix's own key and its subtree. */ + private readonly scope: { prefix: string; gte?: string; lt?: string } private readonly replicas: BaseKVStorage[] /** Persistent replica: seeds the others and holds the outbox and metadata. */ private readonly persistent: BaseKVStorage @@ -102,6 +117,8 @@ export class KVStorageComposite extends BaseKVStorage< super() if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica') this.source = options.source + const prefix = options.prefix ?? '' + this.scope = prefix === '' ? { prefix } : { prefix, gte: prefix, lt: prefix + Keys.afterSeparator } this.replicas = options.replicas this.persistent = options.replicas.at(-1)! this.backoff = { baseDelay: 1000, maxDelay: 30_000, ...options.retry } @@ -134,21 +151,27 @@ export class KVStorageComposite extends BaseKVStorage< this.state.closed = true clearTimeout(this.state.retryTimer) for (const off of this.offs.splice(0)) off() - if (this.watchable) await (this.source as unknown as WatchableSource).unwatch() + if (this.watchable) await (this.source as unknown as WatchableSource).unsubscribe(this.scope.prefix) } async readMany(keys: string[]): Promise { + this.checkScope(keys) await this.start() return (this.state.hasSnapshot ? this.replicas[0]! : this.source).readMany(keys) } async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { await this.start() - if (!this.state.hasSnapshot) return yield* this.source.scan(options) + // Narrow the range to the synced scope (and, against an authenticated server, to what it allows). + const gte = this.scope.gte !== undefined && (options.gte === undefined || options.gte < this.scope.gte) ? this.scope.gte : options.gte + const lt = this.scope.lt !== undefined && (options.lt === undefined || options.lt > this.scope.lt) ? this.scope.lt : options.lt + if (gte !== undefined && lt !== undefined && gte >= lt) return + const scoped = { ...options, gte, lt } + if (!this.state.hasSnapshot) return yield* this.source.scan(scoped) // Skip internal keys before applying the limit. let remaining = options.limit ?? Infinity if (remaining <= 0) return - for await (const entry of this.replicas[0]!.scan({ ...options, limit: undefined })) { + for await (const entry of this.replicas[0]!.scan({ ...scoped, limit: undefined })) { if (entry[0] >= INTERNAL_PREFIX && entry[0] < INTERNAL_END) continue yield entry if (--remaining <= 0) return @@ -156,16 +179,19 @@ export class KVStorageComposite extends BaseKVStorage< } protected async applyWrite(entries: [string, unknown][]): Promise { + this.checkScope(entries.map(([key]) => key)) await this.start() await this.writeThrough({ type: 'write', entries }) } protected async applyRemove(keys: string[]): Promise { + this.checkScope(keys) await this.start() await this.writeThrough({ type: 'remove', keys }) } protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { + this.checkScope(entries.map(([key]) => key)) await this.start() const written = await this.source.writeIfAbsent(entries) const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value })) @@ -176,8 +202,17 @@ export class KVStorageComposite extends BaseKVStorage< // ------------------------------------------------------------------ start + private inScope(key: string): boolean { + const { prefix } = this.scope + return prefix === '' || key === prefix || key.startsWith(prefix + Keys.separator) + } + + private checkScope(keys: string[]): void { + for (const key of keys) if (!this.inScope(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.scope.prefix}"`) + } + private get watchable(): boolean { - return typeof (this.source as unknown as Partial).watch === 'function' + return typeof (this.source as unknown as Partial).subscribe === 'function' } private async open(): Promise { @@ -201,8 +236,9 @@ export class KVStorageComposite extends BaseKVStorage< const source = this.source as unknown as WatchableSource source.lastEventId = meta?.lastEventId // Without a resume point, catch up on the stream first so no change after the snapshot is missed. - const watching = source.watch() - if (!meta?.lastEventId) void watching.then(() => this.enqueueApply(() => this.fullSync())) + const watching = source.subscribe(this.scope.prefix) + if (!meta?.lastEventId) void watching.then(() => this.enqueueApply(() => this.fullSync())).catch((error) => console.error(error)) + else void watching.catch((error) => console.error(error)) } else { this.enqueueApply(() => this.fullSync()) } @@ -238,7 +274,9 @@ export class KVStorageComposite extends BaseKVStorage< } private async applyRemote(event: KVChangeEvent): Promise { - const candidates = event.changes.filter(({ key }) => !this.consumeEcho(key) && !this.pending.has(key) && !key.startsWith(INTERNAL_PREFIX)) + const candidates = event.changes.filter( + ({ key }) => this.inScope(key) && !this.consumeEcho(key) && !this.pending.has(key) && !key.startsWith(INTERNAL_PREFIX), + ) // Skip changes the replicas already reflect. const current = await this.replicas[0]!.readMany(candidates.map(({ key }) => key)) const changes = candidates.filter(({ value }, i) => !Json.equal(value, current[i])) @@ -276,8 +314,9 @@ export class KVStorageComposite extends BaseKVStorage< /** Changes that turn the persistent replica into the source (both scanned in key order). */ private async *diff(): AsyncGenerator { - const remote = this.source.scan({ values: true })[Symbol.asyncIterator]() - const local = this.persistent.scan({ values: true })[Symbol.asyncIterator]() + const range = { gte: this.scope.gte, lt: this.scope.lt, values: true } + const remote = this.source.scan(range)[Symbol.asyncIterator]() + const local = this.persistent.scan(range)[Symbol.asyncIterator]() const nextLocal = async () => { for (let next = await local.next(); ; next = await local.next()) if (next.done || !next.value[0].startsWith(INTERNAL_PREFIX)) return next } diff --git a/src/kv-storage/kv-storage-http-server.ts b/src/kv-storage/kv-storage-http-server.ts index 208262a..d785a2b 100644 --- a/src/kv-storage/kv-storage-http-server.ts +++ b/src/kv-storage/kv-storage-http-server.ts @@ -1,7 +1,16 @@ +import { AuthError } from '../utils/errors.js' import { ExtendedJson } from '../utils/extended-json.js' +import { Keys } from '../utils/keys.js' +import { RequestAuth, type RequestAuthOptions, type SignatureVerifier } from '../utils/request-auth.js' import type { BaseKVStorage, KVChange, ScanOptions } from './base-kv-storage.js' export interface KVStorageHttpServerOptions { + /** + * `{ verifiers }`: requests must be signed (see `RequestAuth`), and a caller may only touch keys + * under its own public key (`` and `/...`). + * `'none'`: no authentication, everything is open; for local development and tests only. + */ + auth: 'none' | ({ verifiers: SignatureVerifier[] } & RequestAuthOptions) /** Path prefix to strip, e.g. `/kv`. Default `''`. */ basePath?: string /** Recent change batches kept for clients resuming with `Last-Event-ID`. Default 10 000. */ @@ -15,32 +24,45 @@ interface HistoryEntry { changes: KVChange[] } +interface Subscriber { + /** Keys whose own value and subtree this subscriber receives (`''` = everything). */ + paths: string[] + send: (frame: string) => void +} + +type RequestBody = { keys: string[]; entries: [string, unknown][]; paths: string[] } & ScanOptions + /** * Serves any KV storage over HTTP for `KVStorageHttp`. Framework-agnostic: `handle(Request)` - * returns a `Response`. + * returns a `Response`. Every route is `POST` with an Extended JSON body: * - * - `POST /` with an Extended JSON body (`scan` streams NDJSON). - * - `GET /events`: Server-Sent Events of every change to the storage. Event ids are - * `-` (epoch = this server instance). A client sending `Last-Event-ID` gets the - * changes it missed if they are still in the history; otherwise it gets a `resync` event. - * A client without one gets a `ready` event carrying the current id. + * - `/readMany`, `/writeMany`, `/removeMany`, `/writeIfAbsent`, `/scan` (streams NDJSON). + * - `/events` with `{ paths }`: Server-Sent Events of changes to those keys and their subtrees. + * Event ids are `-` (epoch = this server instance). A client sending + * `Last-Event-ID` gets the matching changes it missed if they are still in the history, + * otherwise a `resync` event; either way a `ready` event ends the catch-up. + * + * With authentication, each request is checked first (401), then every key, scan range and + * subscription path must be the caller's public key or under it (403). * * Only changes made through the wrapped storage instance are seen; several server processes over * one database each see only their own writes. */ export class KVStorageHttpServer { + private readonly auth: RequestAuth | undefined private readonly basePath: string private readonly historySize: number private readonly keepAliveMs: number private readonly epoch = Math.random().toString(36).slice(2, 10) private seq = 0 private readonly history: HistoryEntry[] = [] - private readonly subscribers = new Set<(frame: string) => void>() + private readonly subscribers = new Set() constructor( private readonly storage: BaseKVStorage, - options: KVStorageHttpServerOptions = {}, + options: KVStorageHttpServerOptions, ) { + this.auth = options.auth === 'none' ? undefined : new RequestAuth(options.auth.verifiers, options.auth) this.basePath = options.basePath ?? '' this.historySize = options.historySize ?? 10_000 this.keepAliveMs = options.keepAliveMs ?? 15_000 @@ -49,58 +71,99 @@ export class KVStorageHttpServer { async handle(request: Request): Promise { const path = new URL(request.url).pathname - if (!path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') - const route = `${request.method} ${path.slice(this.basePath.length)}` - if (route === 'GET /events') return this.events(request.headers.get('last-event-id')) - if (request.method !== 'POST') return KVStorageHttpServer.error(404, 'Not found') + if (request.method !== 'POST' || !path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found') try { - const body = ExtendedJson.parse(await request.text()) as { keys: string[]; entries: [string, unknown][] } & ScanOptions - switch (route) { - case 'POST /readMany': { + const text = await request.text() + const owner = this.auth ? await this.auth.verify(request.headers, path, text) : undefined + const body = ExtendedJson.parse(text) as RequestBody + switch (path.slice(this.basePath.length)) { + case '/readMany': { + this.authorize(owner, body.keys) const values = await this.storage.readMany(body.keys) const missing = values.flatMap((v, i) => (v === undefined ? [i] : [])) return KVStorageHttpServer.json({ values: values.map((v) => v ?? null), missing }) } - case 'POST /writeMany': + case '/writeMany': + this.authorize( + owner, + body.entries.map(([key]) => key), + ) await this.storage.writeMany(body.entries) return new Response(null, { status: 204 }) - case 'POST /removeMany': + case '/removeMany': + this.authorize(owner, body.keys) await this.storage.removeMany(body.keys) return new Response(null, { status: 204 }) - case 'POST /writeIfAbsent': + case '/writeIfAbsent': + this.authorize( + owner, + body.entries.map(([key]) => key), + ) return KVStorageHttpServer.json(await this.storage.writeIfAbsent(body.entries)) - case 'POST /scan': + case '/scan': + this.authorizeRange(owner, body) return this.scan(body) + case '/events': + if (!Array.isArray(body.paths) || body.paths.length === 0) return KVStorageHttpServer.error(400, 'Subscribe to at least one path') + this.authorize(owner, body.paths) + return this.events(body.paths, request.headers.get('last-event-id')) default: return KVStorageHttpServer.error(404, 'Not found') } } catch (error) { + if (error instanceof AuthError) return KVStorageHttpServer.error(error.status, error.message) return KVStorageHttpServer.error(500, error instanceof Error ? error.message : String(error)) } } + // ------------------------------------------------------------------ authorization + + /** Every key must be the owner's key or under it (no-op without authentication). */ + private authorize(owner: string | undefined, keys: string[]): void { + if (owner === undefined) return + for (const key of keys) { + if (typeof key !== 'string' || !KVStorageHttpServer.within(owner, key)) throw new AuthError(403, `No access to "${key}"`) + } + } + + /** A scan must stay inside `[owner, owner + '0')`: the owner's key and its subtree. */ + private authorizeRange(owner: string | undefined, options: ScanOptions): void { + if (owner === undefined) return + const { gte, lt } = options + if (gte === undefined || lt === undefined || gte < owner || lt > owner + Keys.afterSeparator) throw new AuthError(403, 'Scan outside your keys') + } + + /** Whether `key` is `path` or below it (`''` covers everything). */ + private static within(path: string, key: string): boolean { + return path === '' || key === path || key.startsWith(path + Keys.separator) + } + + // ------------------------------------------------------------------ change stream + private publish(changes: KVChange[]): void { const entry = { seq: ++this.seq, changes } this.history.push(entry) if (this.history.length > this.historySize) this.history.splice(0, this.history.length - this.historySize) - const frame = this.frame('change', entry.seq, changes) - for (const send of this.subscribers) send(frame) + for (const subscriber of this.subscribers) { + const frame = this.changeFrame(entry, subscriber.paths) + if (frame) subscriber.send(frame) + } } - private events(lastEventId: string | null): Response { + private events(paths: string[], lastEventId: string | null): Response { const encoder = new TextEncoder() - let send: ((frame: string) => void) | undefined + let subscriber: Subscriber | undefined let keepAlive: ReturnType | undefined const stream = new ReadableStream({ start: (controller) => { - send = (frame) => controller.enqueue(encoder.encode(frame)) - send(this.catchUpFrames(lastEventId)) - this.subscribers.add(send) - keepAlive = setInterval(() => send!(': keep-alive\n\n'), this.keepAliveMs) + subscriber = { paths, send: (frame) => controller.enqueue(encoder.encode(frame)) } + subscriber.send(this.catchUpFrames(paths, lastEventId)) + this.subscribers.add(subscriber) + keepAlive = setInterval(() => subscriber!.send(': keep-alive\n\n'), this.keepAliveMs) ;(keepAlive as { unref?: () => void }).unref?.() }, cancel: () => { - if (send) this.subscribers.delete(send) + if (subscriber) this.subscribers.delete(subscriber) clearInterval(keepAlive) }, }) @@ -108,10 +171,10 @@ export class KVStorageHttpServer { } /** - * What a (re)connecting client gets before live changes: the changes it missed, or `resync` when - * they can't be replayed; then always `ready` (the client is caught up as of that id). + * What a (re)connecting client gets before live changes: the matching changes it missed, or + * `resync` when they can't be replayed; then always `ready` (caught up as of that id). */ - private catchUpFrames(lastEventId: string | null): string { + private catchUpFrames(paths: string[], lastEventId: string | null): string { const ready = this.frame('ready', this.seq, null) if (lastEventId === null) return ready const [epoch, seqText] = lastEventId.split('-') @@ -119,14 +182,26 @@ export class KVStorageHttpServer { const oldest = this.history[0]?.seq ?? this.seq + 1 // Resumable only if it's our epoch and nothing after `seq` has been dropped from the history. if (epoch !== this.epoch || !Number.isInteger(seq) || seq > this.seq || seq < oldest - 1) return this.frame('resync', this.seq, null) + ready - const missed = this.history.filter((entry) => entry.seq > seq).map((entry) => this.frame('change', entry.seq, entry.changes)) - return missed.join('') + ready + return ( + this.history + .filter((entry) => entry.seq > seq) + .map((entry) => this.changeFrame(entry, paths)) + .join('') + ready + ) + } + + /** A `change` frame with only the changes under `paths`, or `''` if none match. */ + private changeFrame(entry: HistoryEntry, paths: string[]): string { + const changes = entry.changes.filter(({ key }) => paths.some((path) => KVStorageHttpServer.within(path, key))) + return changes.length ? this.frame('change', entry.seq, changes) : '' } private frame(event: string, seq: number, data: unknown): string { return `id: ${this.epoch}-${seq}\nevent: ${event}\ndata: ${ExtendedJson.stringify(data)}\n\n` } + // ------------------------------------------------------------------ scan + private scan(options: ScanOptions): Response { const iterator = this.storage.scan(options)[Symbol.asyncIterator]() const encoder = new TextEncoder() diff --git a/src/kv-storage/kv-storage-http.ts b/src/kv-storage/kv-storage-http.ts index 4e43026..9cfe6cc 100644 --- a/src/kv-storage/kv-storage-http.ts +++ b/src/kv-storage/kv-storage-http.ts @@ -1,39 +1,58 @@ -import { StorageUnavailableError } from '../utils/errors.js' +import { AuthError, StorageUnavailableError } from '../utils/errors.js' import { ExtendedJson } from '../utils/extended-json.js' -import type { AnySpec, Spec } from '../utils/schema.js' +import { RequestAuth, type RequestSigner } from '../utils/request-auth.js' +import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js' +import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js' import { SSESession } from '../utils/sse-session/sse-session.js' import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js' export interface KVStorageHttpOptions { /** Base URL of a `KVStorageHttpServer`, e.g. `https://host/kv`. */ url: string + /** Signs every request (required by servers with authentication). */ + signer?: RequestSigner headers?: Record fetch?: (url: string, init: RequestInit) => Promise /** Called when the change stream fails (it keeps reconnecting). Default: ignore. */ - onWatchError?: (error: Error) => void + onSubscriptionError?: (error: Error) => void +} + +// Types only: children of a KVStorageHttp are KVStorageHttp nodes too (child() creates them with +// Object.create, so they share this class's prototype). +export interface KVStorageHttp { + child>(key: K): KVStorageHttp> } /** * KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage. * - * `watch()` subscribes to the server's change stream (SSE): changes made by anyone are emitted - * here as `change` events with `origin: 'remote'`, and `resync` is emitted when the server can't - * resume from `lastEventId`. Values travel as Extended JSON (`bigint` and `Uint8Array` survive). + * `subscribe(path)` receives changes under a key (pushed over SSE) as `change` events with + * `origin: 'remote'`; `resync` is emitted when the server can't resume from `lastEventId`. With a + * `signer`, every request is signed, and an authenticated server only allows keys under the + * signer's public key. Values travel as Extended JSON (`bigint` and `Uint8Array` survive). */ export class KVStorageHttp extends BaseKVStorage { private readonly url: string + private readonly signer: RequestSigner | undefined private readonly headers: Record private readonly fetcher: (url: string, init: RequestInit) => Promise - private readonly onWatchError: (error: Error) => void + private readonly onSubscriptionError: (error: Error) => void /** Mutable state lives here: child nodes share this object (see `child()`). */ - private readonly state: { session?: SSESession; lastEventId?: string } = {} + private readonly state: { + paths: Set + session?: SSESession + lastEventId?: string + /** Subscription changes are applied one at a time. */ + updating: Promise + } = { paths: new Set(), updating: Promise.resolve() } constructor(options: KVStorageHttpOptions) { super() this.url = options.url.replace(/\/$/, '') + this.signer = options.signer this.headers = options.headers ?? {} this.fetcher = options.fetch ?? ((url, init) => globalThis.fetch(url, init)) - this.onWatchError = options.onWatchError ?? (() => {}) + this.onSubscriptionError = options.onSubscriptionError ?? (() => {}) } /** Position in the server's change stream; sent as `Last-Event-ID` when (re)connecting. */ @@ -45,6 +64,11 @@ export class KVStorageHttp extends BaseKVStorage { this.state.lastEventId = id } + /** Keys currently subscribed to. */ + get subscriptions(): ReadonlySet { + return this.state.paths + } + async readMany(keys: string[]): Promise { if (keys.length === 0) return [] const { values, missing } = (await this.post('readMany', { keys })) as { values: unknown[]; missing: number[] } @@ -75,39 +99,22 @@ export class KVStorageHttp extends BaseKVStorage { } /** - * Subscribes to the server's change stream. Resolves once connected and caught up (missed - * changes replayed, or `resync` emitted); keeps reconnecting and resuming from `lastEventId` - * until `unwatch()`. While the server is unreachable this stays pending, so don't await it on a - * path that must work offline. + * Receives changes to `path` (default: this node's key) and everything under it. Resolves once + * the stream is connected and caught up (missed changes replayed, or `resync` emitted). While + * the server is unreachable this stays pending, so don't await it on a path that must work offline. */ - async watch(): Promise { - if (this.state.session) return - const session = new SSESession(`${this.url}/events`, { - fetch: this.fetcher, - headers: this.headers, - persistent: true, - onError: this.onWatchError, - onRequest: async (request) => - this.lastEventId ? { ...request, headers: { ...(request.headers as Record), 'Last-Event-ID': this.lastEventId } } : request, - }) - session.on('message', (message) => { - if (message.id) this.lastEventId = message.id - if (message.event === 'change') { - this.emit('change', { changes: ExtendedJson.parse(message.data) as KVChange[], origin: 'remote', id: message.id }) - } else if (message.event === 'resync') { - this.emit('resync', { id: message.id }) - } - }) - this.state.session = session - const caughtUp = session.waitFor('message', (message) => message.event === 'ready') - await session.connect() - await caughtUp + subscribe(path: string = this.key): Promise { + return this.updateSubscriptions(() => this.state.paths.add(path)) } - async unwatch(): Promise { - const session = this.state.session - this.state.session = undefined - await session?.disconnect() + /** Stops receiving changes to `path` (default: this node's key). */ + unsubscribe(path: string = this.key): Promise { + return this.updateSubscriptions(() => this.state.paths.delete(path)) + } + + /** Drops every subscription and closes the stream. */ + close(): Promise { + return this.updateSubscriptions(() => this.state.paths.clear()) } protected async applyWrite(entries: [string, unknown][]): Promise { @@ -122,24 +129,102 @@ export class KVStorageHttp extends BaseKVStorage { return (await this.post('writeIfAbsent', { entries })) as boolean[] } + // ------------------------------------------------------------------ subscriptions + + /** Applies a subscription change, then reconnects the stream with the new path list. */ + private updateSubscriptions(change: () => unknown): Promise { + const run = async () => { + const before = JSON.stringify([...this.state.paths]) + change() + if (JSON.stringify([...this.state.paths]) === before && (this.state.session || this.state.paths.size === 0)) return + await this.state.session?.disconnect() + this.state.session = undefined + if (this.state.paths.size > 0) await this.connect([...this.state.paths]) + } + const result = this.state.updating.then(run) + this.state.updating = result.catch(() => {}) + return result + } + + private async connect(paths: string[]): Promise { + const url = `${this.url}/events` + const body = ExtendedJson.stringify({ paths }) + const session = new SSESession(url, { + method: 'POST', + body, + fetch: this.fetcher, + headers: { ...this.headers, 'content-type': 'application/json' }, + persistent: true, + onError: this.onSubscriptionError, + retry: KVStorageHttp.retryUnlessDenied(), + // Signed on every (re)connect, so the timestamp is always fresh. + onRequest: async (request) => ({ + ...request, + headers: { + ...(request.headers as Record), + ...(this.state.lastEventId ? { 'Last-Event-ID': this.state.lastEventId } : {}), + ...(await this.authHeaders(url, body)), + }, + }), + }) + session.on('message', (message) => { + if (message.id) this.state.lastEventId = message.id + if (message.event === 'change') { + this.emit('change', { changes: ExtendedJson.parse(message.data) as KVChange[], origin: 'remote', id: message.id }) + } else if (message.event === 'resync') { + this.emit('resync', { id: message.id }) + } + }) + this.state.session = session + const caughtUp = session.waitFor('message', (message) => message.event === 'ready') + try { + await session.connect() + } catch (error) { + this.state.session = undefined + throw (error as Error).cause instanceof AuthError ? (error as Error).cause : error + } + await caughtUp + } + + // ------------------------------------------------------------------ requests + private async post(path: string, body: unknown): Promise { const response = await this.request(path, body) return response.status === 204 ? undefined : ExtendedJson.parse(await response.text()) } - private async request(path: string, body: unknown): Promise { + private async request(path: string, payload: unknown): Promise { + const url = `${this.url}/${path}` + const body = ExtendedJson.stringify(payload) let response: Response try { - response = await this.fetcher(`${this.url}/${path}`, { + response = await this.fetcher(url, { method: 'POST', - headers: { ...this.headers, 'content-type': 'application/json' }, - body: ExtendedJson.stringify(body), + headers: { ...this.headers, 'content-type': 'application/json', ...(await this.authHeaders(url, body)) }, + body, }) } catch (error) { throw new StorageUnavailableError(`Cannot reach ${this.url}`, { cause: error }) } if ([502, 503, 504].includes(response.status)) throw new StorageUnavailableError(`HTTP ${response.status} from ${this.url}`) + if (response.status === 401 || response.status === 403) throw new AuthError(response.status, await response.text()) if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`) return response } + + /** Retries connecting with backoff, except when the server denies access (401/403): that won't fix itself. */ + private static retryUnlessDenied(): { run(fn: () => Promise): Promise } { + const backoff = new ExponentialBackoff({ baseDelay: 1000, maxDelay: 10_000, maxAttempts: 0, growthRate: 1.3, jitter: 0.3 }) + return { + run: (fn) => + backoff.run(fn, (error, context) => { + const status = /^HTTP error: (40[13])/.exec(error.message)?.[1] + if (status) context.abort(new AuthError(Number(status) as 401 | 403, error.message)) + }), + } + } + + private async authHeaders(url: string, body: string): Promise> { + return this.signer ? RequestAuth.headers(this.signer, new URL(url).pathname, body) : {} + } } diff --git a/src/utils/errors.ts b/src/utils/errors.ts index c92957d..5c15807 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -25,3 +25,14 @@ export class StorageUnavailableError extends Error { this.name = 'StorageUnavailableError' } } + +/** A request was not authenticated (401) or not allowed for this caller (403). */ +export class AuthError extends Error { + constructor( + readonly status: 401 | 403, + message: string, + ) { + super(message) + this.name = 'AuthError' + } +} diff --git a/src/utils/extended-json.ts b/src/utils/extended-json.ts index 13888e3..f7ada03 100644 --- a/src/utils/extended-json.ts +++ b/src/utils/extended-json.ts @@ -1,6 +1,8 @@ // Adapted from xo-cash-utils `source/extended-json.ts` (branch installable-sse), with local hex // encoding instead of @bitauth/libauth. +import { Hex } from './hex.js' + /** Matches a bigint encoded as ``. */ const BIGINT_PATTERN = /^[+-]?[0-9]+)n>$/u @@ -22,7 +24,7 @@ export class ExtendedJson { /** `JSON.stringify` replacer. */ static replacer(this: void, _key: string, value: unknown): unknown { - if (value instanceof Uint8Array) return `` + if (value instanceof Uint8Array) return `` if (typeof value === 'bigint') return `` return value } @@ -33,19 +35,7 @@ export class ExtendedJson { const bigint = value.match(BIGINT_PATTERN) if (bigint) return BigInt(bigint.groups!.bigint!) const bytes = value.match(UINT8ARRAY_PATTERN) - if (bytes) return ExtendedJson.fromHex(bytes.groups!.hex!) + if (bytes) return Hex.decode(bytes.groups!.hex!) return value } - - private static toHex(bytes: Uint8Array): string { - let hex = '' - for (const byte of bytes) hex += byte.toString(16).padStart(2, '0') - return hex - } - - private static fromHex(hex: string): Uint8Array { - const bytes = new Uint8Array(hex.length / 2) - for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) - return bytes - } } diff --git a/src/utils/hex.ts b/src/utils/hex.ts new file mode 100644 index 0000000..5d7737a --- /dev/null +++ b/src/utils/hex.ts @@ -0,0 +1,16 @@ +/** Lowercase hex encoding of bytes. */ +export class Hex { + static encode(bytes: Uint8Array): string { + let hex = '' + for (const byte of bytes) hex += byte.toString(16).padStart(2, '0') + return hex + } + + /** Throws on odd length or non-hex characters. */ + static decode(hex: string): Uint8Array { + if (hex.length % 2 !== 0 || !/^[0-9a-fA-F]*$/.test(hex)) throw new TypeError('Invalid hex string') + const bytes = new Uint8Array(hex.length / 2) + for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) + return bytes + } +} diff --git a/src/utils/request-auth.ts b/src/utils/request-auth.ts new file mode 100644 index 0000000..87737be --- /dev/null +++ b/src/utils/request-auth.ts @@ -0,0 +1,150 @@ +import { AuthError } from './errors.js' +import { Hex } from './hex.js' + +/** Signs requests. Implement this to use another algorithm (e.g. secp256k1 via libauth). */ +export interface RequestSigner { + /** Algorithm name sent in `X-Signature-Algorithm`; the server must have a matching verifier. */ + readonly algorithm: string + /** Public key as lowercase hex. Also the owner segment of the caller's keys (`//...`). */ + readonly publicKey: string + sign(payload: Uint8Array): Promise +} + +/** Verifies signatures for one algorithm. */ +export interface SignatureVerifier { + readonly algorithm: string + verify(publicKey: string, payload: Uint8Array, signature: Uint8Array): Promise +} + +export const AUTH_HEADERS = { + publicKey: 'X-Public-Key', + timestamp: 'X-Timestamp', + signature: 'X-Signature', + algorithm: 'X-Signature-Algorithm', +} as const + +export interface RequestAuthOptions { + /** Accepted difference between the request timestamp and the server clock. Default 60 000 ms. */ + maxSkewMs?: number + now?: () => number +} + +/** + * Signed-request authentication. The client signs `${timestamp}:${path}:${body}` (path = URL + * pathname, body = the exact request body) and sends the public key, timestamp, signature (hex) + * and algorithm as headers. The server checks the timestamp is recent, verifies the signature + * with the verifier for that algorithm, and rejects a signature it has already seen within the + * timestamp window (replay protection). + */ +export class RequestAuth { + private readonly verifiers: Map + private readonly maxSkewMs: number + private readonly now: () => number + /** Signatures accepted within the window → when they expire; insertion order ≈ expiry order. */ + private readonly seen = new Map() + + constructor(verifiers: SignatureVerifier[], options: RequestAuthOptions = {}) { + if (verifiers.length === 0) throw new TypeError('RequestAuth needs at least one verifier') + this.verifiers = new Map(verifiers.map((v) => [v.algorithm, v])) + this.maxSkewMs = options.maxSkewMs ?? 60_000 + this.now = options.now ?? Date.now + } + + static payload(timestamp: string, path: string, body: string): Uint8Array { + return new TextEncoder().encode(`${timestamp}:${path}:${body}`) + } + + /** Auth headers for a request to `path` with `body`. */ + static async headers(signer: RequestSigner, path: string, body: string, now: number = Date.now()): Promise> { + const timestamp = String(now) + const signature = await signer.sign(RequestAuth.payload(timestamp, path, body)) + return { + [AUTH_HEADERS.publicKey]: signer.publicKey, + [AUTH_HEADERS.timestamp]: timestamp, + [AUTH_HEADERS.signature]: Hex.encode(signature), + [AUTH_HEADERS.algorithm]: signer.algorithm, + } + } + + /** Returns the caller's public key, or throws `AuthError` (401). */ + async verify(headers: Headers, path: string, body: string): Promise { + const publicKey = headers.get(AUTH_HEADERS.publicKey) + const timestamp = headers.get(AUTH_HEADERS.timestamp) + const signature = headers.get(AUTH_HEADERS.signature) + const algorithm = headers.get(AUTH_HEADERS.algorithm) + if (!publicKey || !timestamp || !signature || !algorithm) throw new AuthError(401, 'Missing authentication headers') + const verifier = this.verifiers.get(algorithm) + if (!verifier) throw new AuthError(401, `Unsupported signature algorithm "${algorithm}"`) + if (!/^[0-9a-f]+$/.test(publicKey)) throw new AuthError(401, 'Public key must be lowercase hex') + const now = this.now() + const time = Number(timestamp) + if (!Number.isInteger(time) || Math.abs(now - time) > this.maxSkewMs) throw new AuthError(401, 'Timestamp outside the accepted window') + this.forgetExpired(now) + if (this.seen.has(signature)) throw new AuthError(401, 'Replayed request') + let valid = false + try { + valid = await verifier.verify(publicKey, RequestAuth.payload(timestamp, path, body), Hex.decode(signature)) + } catch { + valid = false + } + if (!valid) throw new AuthError(401, 'Invalid signature') + this.seen.set(signature, time + this.maxSkewMs) + return publicKey + } + + private forgetExpired(now: number): void { + for (const [signature, expires] of this.seen) { + if (expires >= now) break + this.seen.delete(signature) + } + } +} + +/** Ed25519 signer using WebCrypto (Node 22+ and current browsers; no dependencies). */ +export class Ed25519Signer implements RequestSigner { + readonly algorithm = 'ed25519' + + private constructor( + private readonly privateKey: CryptoKey, + readonly publicKey: string, + ) {} + + /** A new random key pair. */ + static async generate(): Promise { + const pair = (await crypto.subtle.generateKey({ name: 'Ed25519' }, true, ['sign', 'verify'])) as CryptoKeyPair + return new Ed25519Signer(pair.privateKey, Hex.encode(new Uint8Array(await crypto.subtle.exportKey('raw', pair.publicKey)))) + } + + /** Restores a key exported with `exportPrivateKey()` (PKCS #8). */ + static async fromPrivateKey(pkcs8: Uint8Array): Promise { + const privateKey = await crypto.subtle.importKey('pkcs8', pkcs8 as BufferSource, { name: 'Ed25519' }, true, ['sign']) + // WebCrypto can't derive the public key directly; the JWK export of a private key includes it (`x`). + const jwk = await crypto.subtle.exportKey('jwk', privateKey) + const publicKey = Uint8Array.from(atob(jwk.x!.replace(/-/g, '+').replace(/_/g, '/')), (c) => c.charCodeAt(0)) + return new Ed25519Signer(privateKey, Hex.encode(publicKey)) + } + + async exportPrivateKey(): Promise { + return new Uint8Array(await crypto.subtle.exportKey('pkcs8', this.privateKey)) + } + + async sign(payload: Uint8Array): Promise { + return new Uint8Array(await crypto.subtle.sign({ name: 'Ed25519' }, this.privateKey, payload as BufferSource)) + } +} + +/** Ed25519 verifier using WebCrypto. */ +export class Ed25519Verifier implements SignatureVerifier { + readonly algorithm = 'ed25519' + private readonly keys = new Map>() + + async verify(publicKey: string, payload: Uint8Array, signature: Uint8Array): Promise { + let key = this.keys.get(publicKey) + if (!key) { + key = crypto.subtle.importKey('raw', Hex.decode(publicKey) as BufferSource, { name: 'Ed25519' }, false, ['verify']) + this.keys.set(publicKey, key) + if (this.keys.size > 10_000) this.keys.delete(this.keys.keys().next().value!) + } + return crypto.subtle.verify({ name: 'Ed25519' }, await key, signature as BufferSource, payload as BufferSource) + } +} diff --git a/test/kv-storage/kv-storage-http-auth.test.ts b/test/kv-storage/kv-storage-http-auth.test.ts new file mode 100644 index 0000000..79575a8 --- /dev/null +++ b/test/kv-storage/kv-storage-http-auth.test.ts @@ -0,0 +1,95 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js' +import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { AuthError } from '../../src/utils/errors.js' +import { Ed25519Signer, Ed25519Verifier } from '../../src/utils/request-auth.js' +import { collect } from '../helpers.js' +import { TestNetwork } from './sync-helpers.js' + +let network: TestNetwork +let alice: Ed25519Signer +let bob: Ed25519Signer + +beforeEach(async () => { + network = new TestNetwork({ auth: { verifiers: [new Ed25519Verifier()] } }) + alice = await Ed25519Signer.generate() + bob = await Ed25519Signer.generate() + await network.store.writeMany([ + [`${alice.publicKey}/profile`, { name: 'Alice' }], + [`${bob.publicKey}/profile`, { name: 'Bob' }], + [`${bob.publicKey}/posts/1`, { title: 'secret' }], + ]) +}) +afterEach(() => network.close()) + +describe('authenticated KVStorageHttpServer', () => { + it('lets a user read, write and list their own keys', async () => { + const mine = network.client(alice).child(alice.publicKey) + expect(await mine.get('profile')).toEqual({ name: 'Alice' }) + await mine.child('posts').set('1', { title: 'hello' }) + expect(await collect(mine.keys())).toEqual(['posts', 'profile']) + expect(await collect(mine.child('posts').entries())).toEqual([['1', { title: 'hello' }]]) + }) + + it("forbids reading, writing or scanning another user's keys", async () => { + const client = network.client(alice) + const theirs = client.child(bob.publicKey) + await expect(theirs.get('profile')).rejects.toMatchObject({ status: 403 }) + await expect(theirs.set('profile', { name: 'hacked' })).rejects.toBeInstanceOf(AuthError) + await expect(theirs.delete('profile')).rejects.toMatchObject({ status: 403 }) + await expect(collect(theirs.keys())).rejects.toMatchObject({ status: 403 }) + await expect(collect(client.keys())).rejects.toMatchObject({ status: 403 }) // root listing + await expect(client.readMany([alice.publicKey + 'x/a'])).rejects.toMatchObject({ status: 403 }) // lookalike prefix + expect(await network.store.readMany([`${bob.publicKey}/profile`])).toEqual([{ name: 'Bob' }]) + }) + + it('rejects unsigned requests', async () => { + await expect(network.client().child(alice.publicKey).get('profile')).rejects.toMatchObject({ status: 401 }) + }) + + it('delivers only subscribed, authorized changes', async () => { + const watcher = network.client(alice) + const received: KVChangeEvent[] = [] + watcher.on('change', (e) => e.origin === 'remote' && received.push(e)) + await watcher.child(alice.publicKey).child('posts').subscribe() + const writer = network.client(alice).child(alice.publicKey) + await network.client(bob).child(bob.publicKey).child('posts').set('2', { title: 'bob only' }) + await writer.set('profile', { name: 'Alice 2' }) // not subscribed + const arrived = watcher.waitFor('change', () => true) + await writer.child('posts').set('1', { title: 'mine' }) + await arrived + expect(received.flatMap((e) => e.changes)).toEqual([{ key: `${alice.publicKey}/posts/1`, value: { title: 'mine' } }]) + }) + + it("refuses subscriptions to another user's keys without retrying forever", async () => { + await expect(network.client(alice).child(bob.publicKey).subscribe()).rejects.toMatchObject({ status: 403 }) + await expect(network.client(alice).subscribe('')).rejects.toMatchObject({ status: 403 }) + }) + + it('replays only matching changes when resuming', async () => { + const watcher = network.client(alice) + const mine = watcher.child(alice.publicKey) + await mine.subscribe() + await mine.unsubscribe() + await network.client(bob).child(bob.publicKey).set('x', 1) + await network.client(alice).child(alice.publicKey).set('y', 2) + const replayed: KVChangeEvent[] = [] + watcher.on('change', (e) => replayed.push(e)) + await mine.subscribe() + expect(replayed.flatMap((e) => e.changes)).toEqual([{ key: `${alice.publicKey}/y`, value: 2 }]) + }) + + it('a composite scoped to the user syncs only their data', async () => { + const db = new KVStorageComposite({ source: network.client(alice), prefix: alice.publicKey, replicas: [new KVStorageInMemory()] }) + await db.synced() + expect(await collect(db.scan({ values: true }))).toEqual([[`${alice.publicKey}/profile`, { name: 'Alice' }]]) + expect(await collect(db.keys())).toEqual([alice.publicKey]) + await expect(db.child(bob.publicKey).get('profile')).rejects.toBeInstanceOf(RangeError) + const pushed = db.waitFor('change', (e) => e.origin === 'remote') + await network.client(bob).child(bob.publicKey).set('ignored', 1) + await network.client(alice).child(alice.publicKey).set('fromElsewhere', 1) + expect((await pushed).changes).toEqual([{ key: `${alice.publicKey}/fromElsewhere`, value: 1 }]) + await db.close() + }) +}) diff --git a/test/kv-storage/kv-storage-http-watch.test.ts b/test/kv-storage/kv-storage-http-watch.test.ts index 889a9e9..31dc777 100644 --- a/test/kv-storage/kv-storage-http-watch.test.ts +++ b/test/kv-storage/kv-storage-http-watch.test.ts @@ -6,11 +6,11 @@ import { TestNetwork } from './sync-helpers.js' let network: TestNetwork afterEach(() => network?.close()) -describe('KVStorageHttp.watch (SSE)', () => { +describe('KVStorageHttp.subscribe (SSE)', () => { it('receives changes made by other clients as remote change events', async () => { network = new TestNetwork() const watcher = network.client() - await watcher.watch() + await watcher.subscribe('') const received = watcher.waitFor('change', (e) => e.origin === 'remote') await network.client().child('users').set('1', { name: 'Ada', n: 1n }) const event = await received @@ -23,16 +23,16 @@ describe('KVStorageHttp.watch (SSE)', () => { network = new TestNetwork() const writer = network.client() const watcher = network.client() - await watcher.watch() + await watcher.subscribe('') const first = watcher.waitFor('change', (e) => e.origin === 'remote') await writer.writeMany([['a', 1]]) await first - await watcher.unwatch() + await watcher.unsubscribe('') await writer.writeMany([['b', 2]]) await writer.writeMany([['c', 3]]) const replayed: KVChangeEvent[] = [] watcher.on('change', (e) => replayed.push(e)) - await watcher.watch() // resolves once caught up, so the missed changes have been emitted + await watcher.subscribe('') // resolves once caught up, so the missed changes have been emitted expect(replayed.map((e) => e.changes)).toEqual([[{ key: 'b', value: 2 }], [{ key: 'c', value: 3 }]]) }) @@ -40,15 +40,15 @@ describe('KVStorageHttp.watch (SSE)', () => { network = new TestNetwork({ historySize: 1 }) const writer = network.client() const watcher = network.client() - await watcher.watch() + await watcher.subscribe('') const seen = watcher.waitFor('change', () => true) await writer.writeMany([['a', 1]]) await seen - await watcher.unwatch() + await watcher.unsubscribe('') await writer.writeMany([['b', 2]]) await writer.writeMany([['c', 3]]) const resync = watcher.waitFor('resync', () => true) - await watcher.watch() + await watcher.subscribe('') expect(await resync).toEqual({ id: expect.stringMatching(/-3$/) }) }) diff --git a/test/kv-storage/kv-storage-http.test.ts b/test/kv-storage/kv-storage-http.test.ts index 2dc295e..50d6410 100644 --- a/test/kv-storage/kv-storage-http.test.ts +++ b/test/kv-storage/kv-storage-http.test.ts @@ -9,7 +9,7 @@ import { runKVStorageConformance } from './kv-storage.conformance.js' import { serve } from './node-server.js' runKVStorageConformance('KVStorageHttp (in-process)', () => { - const server = new KVStorageHttpServer(new KVStorageInMemory(), { basePath: '/kv' }) + const server = new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none', basePath: '/kv' }) return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) }) }) @@ -22,7 +22,7 @@ describe('over a real socket', () => { }) afterAll(() => server.close()) const fresh = () => { - stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { basePath: `/${stores.length}` })) + stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none', basePath: `/${stores.length}` })) return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` }) } @@ -47,7 +47,7 @@ describe('over a real socket', () => { describe('server', () => { it('maps errors to 404 / 500', async () => { - const server = new KVStorageHttpServer(new KVStorageInMemory()) + const server = new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none' }) expect((await server.handle(new Request('http://x/nope', { method: 'POST', body: '{}' }))).status).toBe(404) expect((await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys": 5}' }))).status).toBe(500) }) diff --git a/test/kv-storage/sync-helpers.ts b/test/kv-storage/sync-helpers.ts index e37e1a3..8b3eece 100644 --- a/test/kv-storage/sync-helpers.ts +++ b/test/kv-storage/sync-helpers.ts @@ -1,6 +1,7 @@ import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js' import { KVStorageHttpServer, type KVStorageHttpServerOptions } from '../../src/kv-storage/kv-storage-http-server.js' import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import type { RequestSigner } from '../../src/utils/request-auth.js' import { KVStorageCounting } from '../helpers.js' /** @@ -13,13 +14,14 @@ export class TestNetwork { readonly server: KVStorageHttpServer private readonly clients: KVStorageHttp[] = [] - constructor(options: KVStorageHttpServerOptions = {}) { - this.server = new KVStorageHttpServer(this.store, options) + constructor(options: Partial = {}) { + this.server = new KVStorageHttpServer(this.store, { auth: 'none', ...options }) } - client(): KVStorageHttp { + client(signer?: RequestSigner): KVStorageHttp { const client = new KVStorageHttp({ url: 'http://server', + signer, fetch: async (url, init) => { if (!this.online) throw new TypeError('fetch failed') return this.server.handle(new Request(url, init)) @@ -30,6 +32,6 @@ export class TestNetwork { } async close(): Promise { - await Promise.all(this.clients.map((c) => c.unwatch())) + await Promise.all(this.clients.map((c) => c.close())) } } diff --git a/test/utils/request-auth.test.ts b/test/utils/request-auth.test.ts new file mode 100644 index 0000000..532f10d --- /dev/null +++ b/test/utils/request-auth.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { AUTH_HEADERS, Ed25519Signer, Ed25519Verifier, RequestAuth, type RequestSigner, type SignatureVerifier } from '../../src/utils/request-auth.js' +import { AuthError } from '../../src/utils/errors.js' + +const NOW = 1_700_000_000_000 + +async function signed(signer: RequestSigner, path = '/kv/readMany', body = '{"keys":["a"]}', at = NOW) { + return new Headers(await RequestAuth.headers(signer, path, body, at)) +} + +describe('RequestAuth', () => { + it('accepts a correctly signed request and returns the public key', async () => { + const signer = await Ed25519Signer.generate() + const auth = new RequestAuth([new Ed25519Verifier()], { now: () => NOW }) + const headers = await signed(signer) + expect(headers.get(AUTH_HEADERS.algorithm)).toBe('ed25519') + expect(signer.publicKey).toMatch(/^[0-9a-f]{64}$/) + expect(await auth.verify(headers, '/kv/readMany', '{"keys":["a"]}')).toBe(signer.publicKey) + }) + + it('rejects tampered bodies, paths and timestamps', async () => { + const signer = await Ed25519Signer.generate() + const auth = new RequestAuth([new Ed25519Verifier()], { now: () => NOW }) + await expect(auth.verify(await signed(signer), '/kv/readMany', '{"keys":["b"]}')).rejects.toThrow(/Invalid signature/) + await expect(auth.verify(await signed(signer), '/kv/writeMany', '{"keys":["a"]}')).rejects.toThrow(/Invalid signature/) + const headers = await signed(signer) + headers.set(AUTH_HEADERS.timestamp, String(NOW + 1)) + await expect(auth.verify(headers, '/kv/readMany', '{"keys":["a"]}')).rejects.toThrow(/Invalid signature/) + const other = await Ed25519Signer.generate() + const swapped = await signed(signer) + swapped.set(AUTH_HEADERS.publicKey, other.publicKey) + await expect(auth.verify(swapped, '/kv/readMany', '{"keys":["a"]}')).rejects.toThrow(/Invalid signature/) + }) + + it('rejects old or future timestamps, replays, missing headers and unknown algorithms', async () => { + const signer = await Ed25519Signer.generate() + const auth = new RequestAuth([new Ed25519Verifier()], { now: () => NOW, maxSkewMs: 1000 }) + await expect(auth.verify(await signed(signer, '/p', 'b', NOW - 5000), '/p', 'b')).rejects.toThrow(/window/) + await expect(auth.verify(await signed(signer, '/p', 'b', NOW + 5000), '/p', 'b')).rejects.toThrow(/window/) + const once = await signed(signer, '/p', 'b') + await auth.verify(once, '/p', 'b') + await expect(auth.verify(once, '/p', 'b')).rejects.toThrow(/Replayed/) + await expect(auth.verify(new Headers(), '/p', 'b')).rejects.toBeInstanceOf(AuthError) + const unknown = await signed(signer, '/p', 'c') + unknown.set(AUTH_HEADERS.algorithm, 'rot13') + await expect(auth.verify(unknown, '/p', 'c')).rejects.toThrow(/Unsupported signature algorithm "rot13"/) + }) + + it('restores an Ed25519 key from its exported private key', async () => { + const signer = await Ed25519Signer.generate() + const restored = await Ed25519Signer.fromPrivateKey(await signer.exportPrivateKey()) + expect(restored.publicKey).toBe(signer.publicKey) + const auth = new RequestAuth([new Ed25519Verifier()], { now: () => NOW }) + expect(await auth.verify(await signed(restored), '/kv/readMany', '{"keys":["a"]}')).toBe(signer.publicKey) + }) + + it('supports other algorithms through the verifier registry', async () => { + // A toy algorithm (signature = sha256(publicKey + payload)) standing in for e.g. secp256k1 via libauth. + const digest = async (publicKey: string, payload: Uint8Array) => + new Uint8Array(await crypto.subtle.digest('SHA-256', new Uint8Array([...new TextEncoder().encode(publicKey), ...payload]))) + const toySigner: RequestSigner = { algorithm: 'toy', publicKey: 'abcd', sign: (payload) => digest('abcd', payload) } + const toyVerifier: SignatureVerifier = { + algorithm: 'toy', + verify: async (publicKey, payload, signature) => (await digest(publicKey, payload)).every((byte, i) => byte === signature[i]), + } + const auth = new RequestAuth([new Ed25519Verifier(), toyVerifier], { now: () => NOW }) + expect(await auth.verify(await signed(toySigner), '/kv/readMany', '{"keys":["a"]}')).toBe('abcd') + }) +}) -- 2.54.0 From 1f82ec9110dccd51948675da947e7772e9ce832c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:43:49 +0000 Subject: [PATCH 05/11] Break KVStorageComposite into focused classes - KVReplicaSet: seed, apply, local scan, diff against the source, sync meta - KVOutbox: persisted offline queue with in-order replay (load() resumes it) - utils: KeyRange (also used for server authorization), KeyCounter, SerialQueue (also used by KVStorageHttp and the event-sourced store), RetryTimer (separate timers for sync and replay; previously one shared timer could cancel the other's retry) The composite now only orchestrates. Behaviour unchanged; new unit tests for the building blocks and the outbox. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE --- DESIGN.md | 12 + .../event-sourced-document-storage.ts | 12 +- src/index.ts | 6 + src/kv-storage/kv-outbox.ts | 91 ++++ src/kv-storage/kv-replica-set.ts | 105 +++++ src/kv-storage/kv-storage-composite.ts | 405 ++++++------------ src/kv-storage/kv-storage-http-server.ts | 37 +- src/kv-storage/kv-storage-http.ts | 15 +- src/utils/key-counter.ts | 36 ++ src/utils/key-range.ts | 36 ++ src/utils/retry-timer.ts | 36 ++ src/utils/serial-queue.ts | 17 + test/kv-storage/kv-outbox.test.ts | 36 ++ test/utils/building-blocks.test.ts | 89 ++++ 14 files changed, 616 insertions(+), 317 deletions(-) create mode 100644 src/kv-storage/kv-outbox.ts create mode 100644 src/kv-storage/kv-replica-set.ts create mode 100644 src/utils/key-counter.ts create mode 100644 src/utils/key-range.ts create mode 100644 src/utils/retry-timer.ts create mode 100644 src/utils/serial-queue.ts create mode 100644 test/kv-storage/kv-outbox.test.ts create mode 100644 test/utils/building-blocks.test.ts diff --git a/DESIGN.md b/DESIGN.md index b919ff4..fbd9a6b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -177,6 +177,18 @@ outbox `~q/o/`. - `start()`, `synced()` (complete copy, all received changes applied), `queued` (outbox size), `close()`. +**Structure.** The composite only orchestrates (what to read from, when to sync, whether to queue a +write). The work is done by small classes: + +| Class | Job | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `KVReplicaSet` | the replicas of one `KeyRange`: seed the faster ones, apply changes, scan locally (hiding `~q/` bookkeeping), diff against the source, sync metadata | +| `KVOutbox` | the persisted offline queue (`~q/o/`), its pending-key counts, and in-order replay with backoff | +| `KeyRange` (utils) | a key plus its subtree: `contains`, `clamp` scan bounds, `covers` a scan; also used by the server's authorization | +| `KeyCounter` (utils) | per-key counts: pending outbox writes and expected echoes | +| `SerialQueue` (utils) | one-at-a-time tasks, optionally per key: applying remote changes, subscription updates, event-sourced writes | +| `RetryTimer` (utils) | backoff retries, one timer per job (full sync and outbox replay retry independently) | + ## 5. Event storage `BaseEventStorage` primitives over stream paths: `appendTo`, `readFrom`, `versionOf`, `streamsOf`. diff --git a/src/document-storage/event-sourced-document-storage.ts b/src/document-storage/event-sourced-document-storage.ts index d9f27c8..1a3f957 100644 --- a/src/document-storage/event-sourced-document-storage.ts +++ b/src/document-storage/event-sourced-document-storage.ts @@ -3,6 +3,7 @@ import type { Doc } from '../utils/document-path.js' import type { DocumentQuery, FindResult } from '../utils/document-query.js' import { DocumentUpdate, type DocChange } from '../utils/document-update.js' import { ConcurrencyError } from '../utils/errors.js' +import { SerialQueue } from '../utils/serial-queue.js' import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt } from '../utils/schema.js' import { BaseDocumentStorage } from './base-document-storage.js' @@ -29,8 +30,8 @@ export interface EventSourcedDocumentStorage { */ export class EventSourcedDocumentStorage extends BaseDocumentStorage { private readonly maxRetries: number - /** Tail of the pending writes per document, so writes in one process don't conflict with each other. */ - private readonly locks = new Map>() + /** Writes to one document run one at a time, so writes in one process don't conflict with each other. */ + private readonly writes = new SerialQueue() constructor( private readonly events: BaseEventStorage, @@ -50,12 +51,7 @@ export class EventSourcedDocumentStorage extends BaseD } mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { - const id = JSON.stringify(path) - const result = (this.locks.get(id) ?? Promise.resolve()).then(() => this.write(path, change)) - const tail = result.catch(() => {}) - this.locks.set(id, tail) - void tail.then(() => this.locks.get(id) === tail && this.locks.delete(id)) - return result + return this.writes.run(() => this.write(path, change), JSON.stringify(path)) } private async write(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise { diff --git a/src/index.ts b/src/index.ts index 966cb34..29b67db 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,8 @@ export * from './kv-storage/kv-storage-http.js' export * from './kv-storage/kv-storage-http-server.js' export * from './kv-storage/kv-storage-indexeddb.js' export * from './kv-storage/kv-storage-composite.js' +export * from './kv-storage/kv-replica-set.js' +export * from './kv-storage/kv-outbox.js' export * from './event-storage/base-event-storage.js' export * from './event-storage/event-storage-kv.js' export * from './document-storage/base-document-storage.js' @@ -17,6 +19,10 @@ export * from './utils/errors.js' export * from './utils/event-emitter.js' export * from './utils/extended-json.js' export * from './utils/hex.js' +export * from './utils/key-counter.js' +export * from './utils/key-range.js' +export * from './utils/retry-timer.js' +export * from './utils/serial-queue.js' export * from './utils/request-auth.js' export * from './utils/json.js' export * from './utils/keys.js' diff --git a/src/kv-storage/kv-outbox.ts b/src/kv-storage/kv-outbox.ts new file mode 100644 index 0000000..ead5a91 --- /dev/null +++ b/src/kv-storage/kv-outbox.ts @@ -0,0 +1,91 @@ +import { StorageUnavailableError } from '../utils/errors.js' +import { KeyCounter } from '../utils/key-counter.js' +import { KeyRange } from '../utils/key-range.js' +import type { RetryTimer } from '../utils/retry-timer.js' +import type { BaseKVStorage, KVChange } from './base-kv-storage.js' + +/** A write waiting to be sent to the source. */ +export type OutboxEntry = { type: 'write'; entries: [string, unknown][] } | { type: 'remove'; keys: string[] } + +const ENTRIES = new KeyRange('~q/o') + +/** + * Writes made while the source was unreachable, persisted as `~q/o/` in a store and + * replayed to the source in order. Replay stops at the first `StorageUnavailableError` and retries + * with backoff; any other error is logged and also retried. + */ +export class KVOutbox { + /** Queued writes per key. */ + readonly pending = new KeyCounter() + private seq = 0 + private replaying: Promise | undefined + + constructor( + private readonly store: BaseKVStorage, + private readonly send: (entry: OutboxEntry) => Promise, + private readonly retry: RetryTimer, + ) {} + + /** Number of queued writes. */ + get size(): number { + return this.pending.size + } + + /** Restores the queue left by a previous session and starts replaying it. */ + async load(): Promise { + for await (const [key, entry] of this.entries()) { + this.seq = Number(key.slice(ENTRIES.prefix.length + 1)) + this.pending.add(KVOutbox.keysOf(entry)) + } + if (this.size > 0) void this.replay() + } + + /** Queues a write and starts replaying. */ + async push(entry: OutboxEntry): Promise { + const key = `${ENTRIES.prefix}/${String(++this.seq).padStart(12, '0')}` + this.pending.add(KVOutbox.keysOf(entry)) + await this.store.writeMany([[key, entry]]) + void this.replay() + } + + /** Sends queued writes in order until the queue is empty or the source is unreachable. */ + replay(): Promise { + const run = async () => { + try { + for (let next = await this.first(); next; next = await this.first()) { + const [key, entry] = next + await this.send(entry) + await this.store.removeMany([key]) + for (const k of KVOutbox.keysOf(entry)) this.pending.take(k) + this.retry.succeeded() + } + } catch (error) { + if (!(error instanceof StorageUnavailableError)) console.error(error) + this.retry.schedule(() => void this.replay()) + } finally { + this.replaying = undefined + } + } + return (this.replaying ??= run()) + } + + static keysOf(entry: OutboxEntry): string[] { + return entry.type === 'write' ? entry.entries.map(([key]) => key) : entry.keys + } + + static changesOf(entry: OutboxEntry): KVChange[] { + return entry.type === 'write' ? entry.entries.map(([key, value]) => ({ key, value })) : entry.keys.map((key) => ({ key })) + } + + private async first(): Promise<[string, OutboxEntry] | undefined> { + for await (const entry of this.entries(1)) return entry + return undefined + } + + private async *entries(limit?: number): AsyncGenerator<[string, OutboxEntry]> { + // Only the entries themselves (`~q/o/...`), not `~q/o` itself. + for await (const [key, value] of this.store.scan({ gte: ENTRIES.prefix + '/', lt: ENTRIES.lt, limit, values: true })) { + yield [key, value as OutboxEntry] + } + } +} diff --git a/src/kv-storage/kv-replica-set.ts b/src/kv-storage/kv-replica-set.ts new file mode 100644 index 0000000..75a1409 --- /dev/null +++ b/src/kv-storage/kv-replica-set.ts @@ -0,0 +1,105 @@ +import { Json } from '../utils/json.js' +import { KeyRange } from '../utils/key-range.js' +import type { BaseKVStorage, KVChange, ScanOptions } from './base-kv-storage.js' + +/** Sync bookkeeping kept in the persistent replica. */ +export interface SyncMeta { + /** Resume position in the source's change stream. */ + lastEventId?: string +} + +/** Keys reserved for sync bookkeeping (`~q/...`); never synced or shown. */ +export const SYNC_INTERNAL = new KeyRange('~q') +const META_KEY = '~q/meta' +const BATCH = 500 + +/** + * Local copies of one key range of a source, fastest first. The last replica is the persistent + * one: it seeds the others and holds the sync metadata (and the outbox, see `KVOutbox`). + */ +export class KVReplicaSet { + readonly fastest: BaseKVStorage + readonly persistent: BaseKVStorage + + constructor( + private readonly replicas: BaseKVStorage[], + readonly range: KeyRange, + ) { + if (replicas.length === 0) throw new TypeError('At least one replica is required') + this.fastest = replicas[0]! + this.persistent = replicas.at(-1)! + } + + async readMeta(): Promise { + return (await this.persistent.readMany([META_KEY]))[0] as SyncMeta | undefined + } + + async writeMeta(meta: SyncMeta): Promise { + await this.persistent.writeMany([[META_KEY, meta]]) + } + + /** Copies the persistent replica into the faster ones. */ + async seed(): Promise { + const faster = this.replicas.slice(0, -1) + if (faster.length === 0) return + let batch: [string, unknown][] = [] + const flush = async () => { + await Promise.all(faster.map((replica) => replica.writeMany(batch))) + batch = [] + } + for await (const entry of this.scanPersistent()) if (batch.push(entry) >= BATCH) await flush() + await flush() + } + + async apply(changes: KVChange[]): Promise { + const writes = changes.filter((c) => c.value !== undefined).map((c): [string, unknown] => [c.key, c.value]) + const removes = changes.filter((c) => c.value === undefined).map((c) => c.key) + await Promise.all(this.replicas.map(async (replica) => (await replica.writeMany(writes), await replica.removeMany(removes)))) + } + + /** The changes whose values differ from what the replicas hold. */ + async unapplied(changes: KVChange[]): Promise { + const current = await this.fastest.readMany(changes.map(({ key }) => key)) + return changes.filter(({ value }, i) => !Json.equal(value, current[i])) + } + + /** Scans the fastest replica within the range, hiding bookkeeping keys. */ + async *scan(options: ScanOptions): AsyncGenerator<[string, unknown]> { + const scoped = this.range.clamp(options) + let remaining = options.limit ?? Infinity + if (!scoped || remaining <= 0) return + // Filter before applying the limit. + for await (const entry of this.fastest.scan({ ...scoped, limit: undefined })) { + if (SYNC_INTERNAL.contains(entry[0])) continue + yield entry + if (--remaining <= 0) return + } + } + + /** Changes that turn the replicas into `source` within the range (merge of two ordered scans). */ + async *diff(source: BaseKVStorage): AsyncGenerator { + const remote = source.scan({ gte: this.range.gte, lt: this.range.lt, values: true })[Symbol.asyncIterator]() + const local = this.scanPersistent() + let r = await remote.next() + let l = await local.next() + while (!r.done || !l.done) { + if (l.done || (!r.done && r.value[0] < l.value[0])) { + yield { key: r.value![0], value: r.value![1] } + r = await remote.next() + } else if (r.done || l.value[0] < r.value[0]) { + yield { key: l.value[0] } + l = await local.next() + } else { + if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] } + r = await remote.next() + l = await local.next() + } + } + } + + private async *scanPersistent(): AsyncGenerator<[string, unknown]> { + for await (const entry of this.persistent.scan({ gte: this.range.gte, lt: this.range.lt, values: true })) { + if (!SYNC_INTERNAL.contains(entry[0])) yield entry + } + } +} diff --git a/src/kv-storage/kv-storage-composite.ts b/src/kv-storage/kv-storage-composite.ts index c698f80..1595bfc 100644 --- a/src/kv-storage/kv-storage-composite.ts +++ b/src/kv-storage/kv-storage-composite.ts @@ -1,10 +1,13 @@ import { StorageUnavailableError } from '../utils/errors.js' import type { OffCallback } from '../utils/event-emitter.js' -import { Json } from '../utils/json.js' -import { Keys } from '../utils/keys.js' +import { KeyCounter } from '../utils/key-counter.js' +import { KeyRange } from '../utils/key-range.js' +import { RetryTimer, type RetryTimerOptions } from '../utils/retry-timer.js' import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js' -import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js' +import { SerialQueue } from '../utils/serial-queue.js' import { BaseKVStorage, type KVChange, type KVChangeEvent, type ScanOptions } from './base-kv-storage.js' +import { KVOutbox, type OutboxEntry } from './kv-outbox.js' +import { KVReplicaSet, SYNC_INTERNAL } from './kv-replica-set.js' export interface KVStorageCompositeOptions { /** The source of truth (e.g. `KVStorageHttp`). */ @@ -16,115 +19,85 @@ export interface KVStorageCompositeOptions { */ prefix?: string /** - * Local full copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be - * persistent: it also stores the offline outbox and sync metadata, and seeds the others on start. + * Local copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be persistent: + * it seeds the others on start and holds the offline outbox and sync metadata. */ replicas: BaseKVStorage[] - /** Delays between attempts to replay queued writes or re-sync. */ - retry?: Partial<{ baseDelay: number; maxDelay: number }> + /** Backoff for replaying queued writes and retrying a failed sync. */ + retry?: RetryTimerOptions } -/** Implemented by sources that can push remote changes (e.g. `KVStorageHttp`). */ -interface WatchableSource { +/** Implemented by sources that push remote changes (e.g. `KVStorageHttp`). */ +interface SubscribableSource { subscribe(path: string): Promise unsubscribe(path: string): Promise lastEventId: string | undefined } -type OutboxEntry = { type: 'write'; entries: [string, unknown][] } | { type: 'remove'; keys: string[] } - -interface SyncMeta { - lastEventId?: string -} - -/** Internal keys in the persistent replica; hidden from reads. */ -const INTERNAL_PREFIX = '~q/' -const INTERNAL_END = '~q0' -const META_KEY = '~q/meta' -const OUTBOX_PREFIX = '~q/o/' -const OUTBOX_END = '~q/o0' -const SYNC_BATCH = 500 - -// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates them with -// Object.create, so they share this class's prototype). +// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates +// them with Object.create, so they share this class's prototype). export interface KVStorageComposite { child>(key: K): KVStorageComposite> } /** - * Keeps local replicas of a source in sync, so reads (including `keys()`, `entries()` and document - * queries) are served locally, and the app keeps working offline. + * Keeps local replicas of (part of) a source in sync, so reads, including `keys()`, `entries()` + * and document queries, are local, and the app keeps working offline. * * - **Start** (lazily, on first use): the persistent replica seeds the faster ones. If it holds a - * previous sync, reads are served from it straight away (possibly stale until caught up). - * - **Staying in sync:** remote changes pushed by the source (`subscribe(prefix)`, e.g. SSE) are applied to - * every replica. On restart the source resumes from the saved `lastEventId`; a full sync (diffing - * the source against the replicas) only runs the first time, or when the source says it can't - * resume (`resync`). Without `subscribe()`, a full sync runs on start. - * - **Writes** go to the source, then the replicas. If the source is unreachable - * (`StorageUnavailableError`), they are applied locally and queued in a persisted outbox, then - * replayed in order when it's reachable again (last writer wins on conflict). While any write is - * queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source and - * fails offline. + * previous sync, reads are served locally straight away (possibly stale until caught up). + * - **Staying in sync:** remote changes pushed by the source are applied to every replica, in + * order. On restart the source resumes from the saved `lastEventId`; a full sync (diff of the + * source against the replicas) only runs the first time, when the source can't resume + * (`resync`), or when it can't push changes at all. + * - **Writes** go to the source, then the replicas. When the source is unreachable they are + * applied locally and queued in the outbox (`KVOutbox`), which replays them in order. While + * anything is queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source. * * Write through the composite, not the source: changes made directly through the same source * instance (`origin: 'local'`) are not mirrored. */ export class KVStorageComposite extends BaseKVStorage { private readonly source: BaseKVStorage - /** Synced key range: `[prefix, prefix + '0')` is the prefix's own key and its subtree. */ - private readonly scope: { prefix: string; gte?: string; lt?: string } - private readonly replicas: BaseKVStorage[] - /** Persistent replica: seeds the others and holds the outbox and metadata. */ - private readonly persistent: BaseKVStorage - private readonly backoff: { baseDelay: number; maxDelay: number } - /** Queued-write count per key; remote changes to these keys are ignored until replayed. */ - private readonly pending = new Map() + private readonly range: KeyRange + private readonly replicas: KVReplicaSet + private readonly outbox: KVOutbox + /** Remote changes and full syncs are applied strictly in order. */ + private readonly applying = new SerialQueue() + private readonly syncRetry: RetryTimer + private readonly outboxRetry: RetryTimer /** - * Echoes still expected per key: one per write this client sent to a watchable source. A remote - * change to such a key is the echo or older than it (the stream is in server order), so it is - * consumed instead of applied; this avoids briefly showing a stale value. + * Echoes still expected per key: one per write sent to a subscribable source. A remote change to + * such a key is the echo or older than it (the stream is in server order), so it is consumed + * instead of applied; this avoids briefly showing a stale value. */ - private readonly echoes = new Map() + private readonly echoes = new KeyCounter() private readonly offs: OffCallback[] = [] /** Mutable state lives here: child nodes share this object (see `child()`). */ private readonly state: { started?: Promise - /** Replicas hold a complete (possibly stale) copy, so reads can be served locally. */ + /** The replicas hold a complete (possibly stale) copy, so reads can be served locally. */ hasSnapshot: boolean - snapshotReady: Promise - markSnapshot: () => void + snapshotTaken: () => void + snapshot: Promise /** - * Remote changes and full syncs are applied strictly in order through this chain. Changes that - * arrive during a full sync wait behind it, so the end state matches the source even if it - * changed during the scan. - */ - applying: Promise - /** - * Keys written through the composite while a full sync runs. The diff may have scanned the - * source before such a write and the replica after it; these keys are left out of the diff - * (the write already reached the source and the replicas). + * Keys written while a full sync runs. Its diff may have scanned the source before such a + * write and the replica after it, so these keys are left out of the diff. */ writtenDuringSync?: Set - replaying?: Promise - outboxSeq: number - retryTimer?: ReturnType - retryAttempt: number - closed: boolean } constructor(options: KVStorageCompositeOptions) { super() - if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica') this.source = options.source - const prefix = options.prefix ?? '' - this.scope = prefix === '' ? { prefix } : { prefix, gte: prefix, lt: prefix + Keys.afterSeparator } - this.replicas = options.replicas - this.persistent = options.replicas.at(-1)! - this.backoff = { baseDelay: 1000, maxDelay: 30_000, ...options.retry } - let markSnapshot!: () => void - const snapshotReady = new Promise((resolve) => (markSnapshot = resolve)) - this.state = { hasSnapshot: false, snapshotReady, markSnapshot, applying: Promise.resolve(), outboxSeq: 0, retryAttempt: 0, closed: false } + this.range = new KeyRange(options.prefix ?? '') + this.replicas = new KVReplicaSet(options.replicas, this.range) + this.syncRetry = new RetryTimer(options.retry) + this.outboxRetry = new RetryTimer(options.retry) + this.outbox = new KVOutbox(this.replicas.persistent, (entry) => this.send(entry), this.outboxRetry) + let snapshotTaken!: () => void + const snapshot = new Promise((resolve) => (snapshotTaken = resolve)) + this.state = { hasSnapshot: false, snapshot, snapshotTaken } } /** Starts syncing (also happens on first use). Resolves once local data is loaded. */ @@ -135,296 +108,172 @@ export class KVStorageComposite extends BaseKVStorage< /** Resolves once the replicas hold a complete copy and all received remote changes are applied. */ async synced(): Promise { await this.start() - await this.state.snapshotReady - await this.state.applying + await this.state.snapshot + await this.applying.idle() } - /** Number of writes waiting to be replayed to the source. */ + /** Number of writes waiting to be sent to the source. */ get queued(): number { - let total = 0 - for (const count of this.pending.values()) total += count - return total + return this.outbox.size } - /** Stops watching the source and cancels retries. */ + /** Stops following the source and cancels retries. */ async close(): Promise { - this.state.closed = true - clearTimeout(this.state.retryTimer) + this.syncRetry.stop() + this.outboxRetry.stop() for (const off of this.offs.splice(0)) off() - if (this.watchable) await (this.source as unknown as WatchableSource).unsubscribe(this.scope.prefix) + if (this.subscribable) await this.subscribable.unsubscribe(this.range.prefix) } async readMany(keys: string[]): Promise { - this.checkScope(keys) + this.checkRange(keys) await this.start() - return (this.state.hasSnapshot ? this.replicas[0]! : this.source).readMany(keys) + return (this.state.hasSnapshot ? this.replicas.fastest : this.source).readMany(keys) } async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> { await this.start() - // Narrow the range to the synced scope (and, against an authenticated server, to what it allows). - const gte = this.scope.gte !== undefined && (options.gte === undefined || options.gte < this.scope.gte) ? this.scope.gte : options.gte - const lt = this.scope.lt !== undefined && (options.lt === undefined || options.lt > this.scope.lt) ? this.scope.lt : options.lt - if (gte !== undefined && lt !== undefined && gte >= lt) return - const scoped = { ...options, gte, lt } - if (!this.state.hasSnapshot) return yield* this.source.scan(scoped) - // Skip internal keys before applying the limit. - let remaining = options.limit ?? Infinity - if (remaining <= 0) return - for await (const entry of this.replicas[0]!.scan({ ...scoped, limit: undefined })) { - if (entry[0] >= INTERNAL_PREFIX && entry[0] < INTERNAL_END) continue - yield entry - if (--remaining <= 0) return - } + if (this.state.hasSnapshot) return yield* this.replicas.scan(options) + const scoped = this.range.clamp(options) + if (scoped) yield* this.source.scan(scoped) } protected async applyWrite(entries: [string, unknown][]): Promise { - this.checkScope(entries.map(([key]) => key)) - await this.start() - await this.writeThrough({ type: 'write', entries }) + await this.write({ type: 'write', entries }) } protected async applyRemove(keys: string[]): Promise { - this.checkScope(keys) - await this.start() - await this.writeThrough({ type: 'remove', keys }) + await this.write({ type: 'remove', keys }) } protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise { - this.checkScope(entries.map(([key]) => key)) + this.checkRange(entries.map(([key]) => key)) await this.start() const written = await this.source.writeIfAbsent(entries) const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value })) for (const { key } of changes) this.state.writtenDuringSync?.add(key) - await this.applyToReplicas(changes) + await this.replicas.apply(changes) return written } // ------------------------------------------------------------------ start - private inScope(key: string): boolean { - const { prefix } = this.scope - return prefix === '' || key === prefix || key.startsWith(prefix + Keys.separator) - } - - private checkScope(keys: string[]): void { - for (const key of keys) if (!this.inScope(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.scope.prefix}"`) - } - - private get watchable(): boolean { - return typeof (this.source as unknown as Partial).subscribe === 'function' - } - private async open(): Promise { - const [meta] = (await this.persistent.readMany([META_KEY])) as (SyncMeta | undefined)[] - for await (const [key, value] of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, values: true })) { - this.state.outboxSeq = Number(key.slice(OUTBOX_PREFIX.length)) - for (const k of KVStorageComposite.keysOf(value as OutboxEntry)) this.pending.set(k, (this.pending.get(k) ?? 0) + 1) - } - await this.seedFasterReplicas() + const meta = await this.replicas.readMeta() + await this.outbox.load() + await this.replicas.seed() if (meta) this.snapshotTaken() this.offs.push( this.source.on('change', (event) => { - if (event.origin === 'remote') this.enqueueApply(() => this.applyRemote(event)) - if (this.queued > 0) void this.replay() + if (event.origin === 'remote') this.enqueue(() => this.applyRemote(event)) + if (this.outbox.size > 0) void this.outbox.replay() // the source is reachable again }), - this.source.on('resync', () => this.enqueueApply(() => this.fullSync())), + this.source.on('resync', () => this.enqueue(() => this.fullSync())), ) - if (this.watchable) { - const source = this.source as unknown as WatchableSource + const source = this.subscribable + if (source) { source.lastEventId = meta?.lastEventId // Without a resume point, catch up on the stream first so no change after the snapshot is missed. - const watching = source.subscribe(this.scope.prefix) - if (!meta?.lastEventId) void watching.then(() => this.enqueueApply(() => this.fullSync())).catch((error) => console.error(error)) - else void watching.catch((error) => console.error(error)) + const subscribed = source.subscribe(this.range.prefix) + if (!meta?.lastEventId) void subscribed.then(() => this.enqueue(() => this.fullSync())).catch((error) => console.error(error)) + else void subscribed.catch((error) => console.error(error)) } else { - this.enqueueApply(() => this.fullSync()) + this.enqueue(() => this.fullSync()) } - if (this.queued > 0) void this.replay() } - /** Copies the persistent replica into the faster ones. */ - private async seedFasterReplicas(): Promise { - const faster = this.replicas.slice(0, -1) - if (faster.length === 0) return - let batch: [string, unknown][] = [] - const flush = async () => { - await Promise.all(faster.map((replica) => replica.writeMany(batch))) - batch = [] - } - for await (const entry of this.persistent.scan({ values: true })) { - if (!entry[0].startsWith(INTERNAL_PREFIX) && batch.push(entry) >= SYNC_BATCH) await flush() - } - await flush() - } - - private snapshotTaken(): void { - this.state.hasSnapshot = true - this.state.markSnapshot() - } - - // ------------------------------------------------------------------ remote → replicas - - private enqueueApply(task: () => Promise): void { - this.state.applying = this.state.applying.then(task).catch((error) => { + /** Queues a sync task; failures are logged (an unreachable source is retried elsewhere). */ + private enqueue(task: () => Promise): void { + this.applying.run(task).catch((error) => { if (!(error instanceof StorageUnavailableError)) console.error(error) }) } + private snapshotTaken(): void { + this.state.hasSnapshot = true + this.state.snapshotTaken() + } + + private get subscribable(): SubscribableSource | undefined { + const source = this.source as unknown as Partial + return typeof source.subscribe === 'function' ? (source as SubscribableSource) : undefined + } + + private checkRange(keys: string[]): void { + for (const key of keys) if (!this.range.contains(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.range.prefix}"`) + } + + // ------------------------------------------------------------------ source → replicas + private async applyRemote(event: KVChangeEvent): Promise { - const candidates = event.changes.filter( - ({ key }) => this.inScope(key) && !this.consumeEcho(key) && !this.pending.has(key) && !key.startsWith(INTERNAL_PREFIX), + const relevant = event.changes.filter( + ({ key }) => this.range.contains(key) && !SYNC_INTERNAL.contains(key) && !this.echoes.take(key) && !this.outbox.pending.has(key), ) - // Skip changes the replicas already reflect. - const current = await this.replicas[0]!.readMany(candidates.map(({ key }) => key)) - const changes = candidates.filter(({ value }, i) => !Json.equal(value, current[i])) - await this.applyToReplicas(changes) - if (event.id) await this.persistent.writeMany([[META_KEY, { lastEventId: event.id } satisfies SyncMeta]]) + const changes = await this.replicas.unapplied(relevant) + await this.replicas.apply(changes) + if (event.id) await this.replicas.writeMeta({ lastEventId: event.id }) if (changes.length) this.emit('change', { changes, origin: 'remote', id: event.id }) } - /** Diffs the source against the persistent replica and applies the differences to every replica. */ + /** Makes the replicas match the source (within the range). */ private async fullSync(): Promise { // The diff re-reads the source, so outstanding echoes no longer matter (and may never arrive). this.echoes.clear() const writtenDuringSync = (this.state.writtenDuringSync = new Set()) + const skip = ({ key }: KVChange) => this.outbox.pending.has(key) || writtenDuringSync.has(key) try { - const id = this.watchable ? (this.source as unknown as WatchableSource).lastEventId : undefined - let changes: KVChange[] = [] + const lastEventId = this.subscribable?.lastEventId + let batch: KVChange[] = [] const flush = async () => { - const applied = changes.filter(({ key }) => !this.pending.has(key) && !writtenDuringSync.has(key)) - await this.applyToReplicas(applied) - if (applied.length) this.emit('change', { changes: applied, origin: 'remote' }) - changes = [] + const changes = batch.filter((change) => !skip(change)) + await this.replicas.apply(changes) + if (changes.length) this.emit('change', { changes, origin: 'remote' }) + batch = [] } - for await (const change of this.diff()) if (changes.push(change) >= SYNC_BATCH) await flush() + for await (const change of this.replicas.diff(this.source)) if (batch.push(change) >= 500) await flush() await flush() - await this.persistent.writeMany([[META_KEY, { lastEventId: id } satisfies SyncMeta]]) + await this.replicas.writeMeta({ lastEventId }) this.snapshotTaken() - this.state.retryAttempt = 0 + this.syncRetry.succeeded() } catch (error) { if (!(error instanceof StorageUnavailableError)) throw error - this.scheduleRetry(() => this.enqueueApply(() => this.fullSync())) + this.syncRetry.schedule(() => this.enqueue(() => this.fullSync())) } finally { this.state.writtenDuringSync = undefined } } - /** Changes that turn the persistent replica into the source (both scanned in key order). */ - private async *diff(): AsyncGenerator { - const range = { gte: this.scope.gte, lt: this.scope.lt, values: true } - const remote = this.source.scan(range)[Symbol.asyncIterator]() - const local = this.persistent.scan(range)[Symbol.asyncIterator]() - const nextLocal = async () => { - for (let next = await local.next(); ; next = await local.next()) if (next.done || !next.value[0].startsWith(INTERNAL_PREFIX)) return next - } - let r = await remote.next() - let l = await nextLocal() - while (!r.done || !l.done) { - if (l.done || (!r.done && r.value[0] < l.value[0])) { - yield { key: r.value![0], value: r.value![1] } - r = await remote.next() - } else if (r.done || l.value[0] < r.value[0]) { - yield { key: l.value[0] } - l = await nextLocal() - } else { - if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] } - r = await remote.next() - l = await nextLocal() - } - } - } + // ------------------------------------------------------------------ writes → source - private async applyToReplicas(changes: KVChange[]): Promise { - const writes = changes.filter((c) => c.value !== undefined).map((c): [string, unknown] => [c.key, c.value]) - const removes = changes.filter((c) => c.value === undefined).map((c) => c.key) - await Promise.all(this.replicas.map(async (replica) => (await replica.writeMany(writes), await replica.removeMany(removes)))) - } - - // ------------------------------------------------------------------ local writes → source - - private async writeThrough(entry: OutboxEntry): Promise { - for (const key of KVStorageComposite.keysOf(entry)) this.state.writtenDuringSync?.add(key) - if (this.queued === 0) { + private async write(entry: OutboxEntry): Promise { + const keys = KVOutbox.keysOf(entry) + this.checkRange(keys) + await this.start() + for (const key of keys) this.state.writtenDuringSync?.add(key) + if (this.outbox.size === 0) { try { await this.send(entry) - return this.applyToReplicas(KVStorageComposite.changesOf(entry)) + return this.replicas.apply(KVOutbox.changesOf(entry)) } catch (error) { if (!(error instanceof StorageUnavailableError)) throw error } } // Offline, or earlier writes are still queued: apply locally and queue for the source. - const seq = ++this.state.outboxSeq - for (const key of KVStorageComposite.keysOf(entry)) this.pending.set(key, (this.pending.get(key) ?? 0) + 1) - await this.persistent.writeMany([[OUTBOX_PREFIX + String(seq).padStart(12, '0'), entry]]) - await this.applyToReplicas(KVStorageComposite.changesOf(entry)) - void this.replay() + await this.outbox.push(entry) + await this.replicas.apply(KVOutbox.changesOf(entry)) } - /** Sends queued writes to the source in order; stops (and retries later) if it's unreachable. */ - private replay(): Promise { - const run = async () => { - try { - for (;;) { - let next: [string, unknown] | undefined - for await (const entry of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, limit: 1, values: true })) next = entry - if (!next) return - const [key, entry] = next as [string, OutboxEntry] - await this.send(entry) - await this.persistent.removeMany([key]) - for (const k of KVStorageComposite.keysOf(entry)) { - const count = this.pending.get(k)! - 1 - if (count > 0) this.pending.set(k, count) - else this.pending.delete(k) - } - this.state.retryAttempt = 0 - } - } catch (error) { - if (!(error instanceof StorageUnavailableError)) console.error(error) - this.scheduleRetry(() => void this.replay()) - } finally { - this.state.replaying = undefined - } - } - return (this.state.replaying ??= run()) - } - - /** Sends a write to the source, expecting its echo when the source is watched. */ + /** Sends a write to the source, expecting its echo when the source pushes changes. */ private async send(entry: OutboxEntry): Promise { - const keys = this.watchable ? KVStorageComposite.keysOf(entry) : [] - for (const key of keys) this.echoes.set(key, (this.echoes.get(key) ?? 0) + 1) + const keys = this.subscribable ? KVOutbox.keysOf(entry) : [] + this.echoes.add(keys) try { await (entry.type === 'write' ? this.source.writeMany(entry.entries) : this.source.removeMany(entry.keys)) } catch (error) { - for (const key of keys) this.consumeEcho(key) + for (const key of keys) this.echoes.take(key) throw error } } - - /** Whether `key` had an outstanding echo (now consumed). */ - private consumeEcho(key: string): boolean { - const count = this.echoes.get(key) - if (!count) return false - if (count > 1) this.echoes.set(key, count - 1) - else this.echoes.delete(key) - return true - } - - private scheduleRetry(task: () => void): void { - if (this.state.closed) return - clearTimeout(this.state.retryTimer) - const delay = ExponentialBackoff.calculateDelay({ ...this.backoff, growthRate: 2, jitter: 0.2, maxAttempts: 0 }, this.state.retryAttempt++) - this.state.retryTimer = setTimeout(task, delay) - ;(this.state.retryTimer as { unref?: () => void }).unref?.() - } - - private static keysOf(entry: OutboxEntry): string[] { - return entry.type === 'write' ? entry.entries.map(([key]) => key) : entry.keys - } - - private static changesOf(entry: OutboxEntry): KVChange[] { - return entry.type === 'write' ? entry.entries.map(([key, value]) => ({ key, value })) : entry.keys.map((key) => ({ key })) - } } diff --git a/src/kv-storage/kv-storage-http-server.ts b/src/kv-storage/kv-storage-http-server.ts index d785a2b..32ee001 100644 --- a/src/kv-storage/kv-storage-http-server.ts +++ b/src/kv-storage/kv-storage-http-server.ts @@ -1,6 +1,6 @@ import { AuthError } from '../utils/errors.js' import { ExtendedJson } from '../utils/extended-json.js' -import { Keys } from '../utils/keys.js' +import { KeyRange } from '../utils/key-range.js' import { RequestAuth, type RequestAuthOptions, type SignatureVerifier } from '../utils/request-auth.js' import type { BaseKVStorage, KVChange, ScanOptions } from './base-kv-storage.js' @@ -25,8 +25,8 @@ interface HistoryEntry { } interface Subscriber { - /** Keys whose own value and subtree this subscriber receives (`''` = everything). */ - paths: string[] + /** Keys (with their subtrees) this subscriber receives; `''` = everything. */ + ranges: KeyRange[] send: (frame: string) => void } @@ -121,21 +121,15 @@ export class KVStorageHttpServer { /** Every key must be the owner's key or under it (no-op without authentication). */ private authorize(owner: string | undefined, keys: string[]): void { if (owner === undefined) return + const allowed = new KeyRange(owner) for (const key of keys) { - if (typeof key !== 'string' || !KVStorageHttpServer.within(owner, key)) throw new AuthError(403, `No access to "${key}"`) + if (typeof key !== 'string' || !allowed.contains(key)) throw new AuthError(403, `No access to "${key}"`) } } - /** A scan must stay inside `[owner, owner + '0')`: the owner's key and its subtree. */ + /** A scan must stay inside the owner's key and its subtree. */ private authorizeRange(owner: string | undefined, options: ScanOptions): void { - if (owner === undefined) return - const { gte, lt } = options - if (gte === undefined || lt === undefined || gte < owner || lt > owner + Keys.afterSeparator) throw new AuthError(403, 'Scan outside your keys') - } - - /** Whether `key` is `path` or below it (`''` covers everything). */ - private static within(path: string, key: string): boolean { - return path === '' || key === path || key.startsWith(path + Keys.separator) + if (owner !== undefined && !new KeyRange(owner).covers(options.gte, options.lt)) throw new AuthError(403, 'Scan outside your keys') } // ------------------------------------------------------------------ change stream @@ -145,7 +139,7 @@ export class KVStorageHttpServer { this.history.push(entry) if (this.history.length > this.historySize) this.history.splice(0, this.history.length - this.historySize) for (const subscriber of this.subscribers) { - const frame = this.changeFrame(entry, subscriber.paths) + const frame = this.changeFrame(entry, subscriber.ranges) if (frame) subscriber.send(frame) } } @@ -156,8 +150,9 @@ export class KVStorageHttpServer { let keepAlive: ReturnType | undefined const stream = new ReadableStream({ start: (controller) => { - subscriber = { paths, send: (frame) => controller.enqueue(encoder.encode(frame)) } - subscriber.send(this.catchUpFrames(paths, lastEventId)) + const ranges = paths.map((path) => new KeyRange(path)) + subscriber = { ranges, send: (frame) => controller.enqueue(encoder.encode(frame)) } + subscriber.send(this.catchUpFrames(ranges, lastEventId)) this.subscribers.add(subscriber) keepAlive = setInterval(() => subscriber!.send(': keep-alive\n\n'), this.keepAliveMs) ;(keepAlive as { unref?: () => void }).unref?.() @@ -174,7 +169,7 @@ export class KVStorageHttpServer { * What a (re)connecting client gets before live changes: the matching changes it missed, or * `resync` when they can't be replayed; then always `ready` (caught up as of that id). */ - private catchUpFrames(paths: string[], lastEventId: string | null): string { + private catchUpFrames(ranges: KeyRange[], lastEventId: string | null): string { const ready = this.frame('ready', this.seq, null) if (lastEventId === null) return ready const [epoch, seqText] = lastEventId.split('-') @@ -185,14 +180,14 @@ export class KVStorageHttpServer { return ( this.history .filter((entry) => entry.seq > seq) - .map((entry) => this.changeFrame(entry, paths)) + .map((entry) => this.changeFrame(entry, ranges)) .join('') + ready ) } - /** A `change` frame with only the changes under `paths`, or `''` if none match. */ - private changeFrame(entry: HistoryEntry, paths: string[]): string { - const changes = entry.changes.filter(({ key }) => paths.some((path) => KVStorageHttpServer.within(path, key))) + /** A `change` frame with only the changes inside `ranges`, or `''` if none match. */ + private changeFrame(entry: HistoryEntry, ranges: KeyRange[]): string { + const changes = entry.changes.filter(({ key }) => ranges.some((range) => range.contains(key))) return changes.length ? this.frame('change', entry.seq, changes) : '' } diff --git a/src/kv-storage/kv-storage-http.ts b/src/kv-storage/kv-storage-http.ts index 9cfe6cc..fd8bb64 100644 --- a/src/kv-storage/kv-storage-http.ts +++ b/src/kv-storage/kv-storage-http.ts @@ -3,6 +3,7 @@ import { ExtendedJson } from '../utils/extended-json.js' import { RequestAuth, type RequestSigner } from '../utils/request-auth.js' import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js' import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js' +import { SerialQueue } from '../utils/serial-queue.js' import { SSESession } from '../utils/sse-session/sse-session.js' import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js' @@ -37,14 +38,10 @@ export class KVStorageHttp extends BaseKVStorage { private readonly headers: Record private readonly fetcher: (url: string, init: RequestInit) => Promise private readonly onSubscriptionError: (error: Error) => void + /** Subscription changes are applied one at a time. */ + private readonly updates = new SerialQueue() /** Mutable state lives here: child nodes share this object (see `child()`). */ - private readonly state: { - paths: Set - session?: SSESession - lastEventId?: string - /** Subscription changes are applied one at a time. */ - updating: Promise - } = { paths: new Set(), updating: Promise.resolve() } + private readonly state: { paths: Set; session?: SSESession; lastEventId?: string } = { paths: new Set() } constructor(options: KVStorageHttpOptions) { super() @@ -141,9 +138,7 @@ export class KVStorageHttp extends BaseKVStorage { this.state.session = undefined if (this.state.paths.size > 0) await this.connect([...this.state.paths]) } - const result = this.state.updating.then(run) - this.state.updating = result.catch(() => {}) - return result + return this.updates.run(run) } private async connect(paths: string[]): Promise { diff --git a/src/utils/key-counter.ts b/src/utils/key-counter.ts new file mode 100644 index 0000000..9e5e016 --- /dev/null +++ b/src/utils/key-counter.ts @@ -0,0 +1,36 @@ +/** A count per key (a multiset). */ +export class KeyCounter { + private readonly counts = new Map() + private total = 0 + + /** Sum of all counts. */ + get size(): number { + return this.total + } + + has(key: string): boolean { + return this.counts.has(key) + } + + add(keys: Iterable): void { + for (const key of keys) { + this.counts.set(key, (this.counts.get(key) ?? 0) + 1) + this.total++ + } + } + + /** Decrements `key`; returns whether it had a count. */ + take(key: string): boolean { + const count = this.counts.get(key) + if (!count) return false + if (count > 1) this.counts.set(key, count - 1) + else this.counts.delete(key) + this.total-- + return true + } + + clear(): void { + this.counts.clear() + this.total = 0 + } +} diff --git a/src/utils/key-range.ts b/src/utils/key-range.ts new file mode 100644 index 0000000..642a558 --- /dev/null +++ b/src/utils/key-range.ts @@ -0,0 +1,36 @@ +import type { ScanOptions } from '../kv-storage/base-kv-storage.js' +import { Keys } from './keys.js' + +/** + * A key and its whole subtree: `[prefix, prefix + '0')`. Escaping guarantees nothing else sorts + * in that range (not even lookalikes such as `prefix-x`). The empty prefix is every key. + */ +export class KeyRange { + /** Inclusive lower bound (`undefined` = unbounded). */ + readonly gte: string | undefined + /** Exclusive upper bound (`undefined` = unbounded). */ + readonly lt: string | undefined + + constructor(readonly prefix: string) { + this.gte = prefix === '' ? undefined : prefix + this.lt = prefix === '' ? undefined : prefix + Keys.afterSeparator + } + + contains(key: string): boolean { + return this.prefix === '' || key === this.prefix || key.startsWith(this.prefix + Keys.separator) + } + + /** Whether a scan over `[gte, lt)` stays inside this range. */ + covers(gte: string | undefined, lt: string | undefined): boolean { + if (this.prefix === '') return true + return gte !== undefined && lt !== undefined && gte >= this.gte! && lt <= this.lt! + } + + /** Scan options narrowed to this range, or `undefined` if nothing is left. */ + clamp(options: ScanOptions): ScanOptions | undefined { + const gte = this.gte !== undefined && (options.gte === undefined || options.gte < this.gte) ? this.gte : options.gte + const lt = this.lt !== undefined && (options.lt === undefined || options.lt > this.lt) ? this.lt : options.lt + if (gte !== undefined && lt !== undefined && gte >= lt) return undefined + return { ...options, gte, lt } + } +} diff --git a/src/utils/retry-timer.ts b/src/utils/retry-timer.ts new file mode 100644 index 0000000..ac04b2b --- /dev/null +++ b/src/utils/retry-timer.ts @@ -0,0 +1,36 @@ +import { ExponentialBackoff } from './sse-session/exponential-backoff.js' + +export interface RetryTimerOptions { + baseDelay?: number + maxDelay?: number +} + +/** Schedules one pending retry at a time, with exponential backoff that resets on success. */ +export class RetryTimer { + private readonly options: { baseDelay: number; maxDelay: number } + private timer: ReturnType | undefined + private attempt = 0 + private stopped = false + + constructor(options: RetryTimerOptions = {}) { + this.options = { baseDelay: 1000, maxDelay: 30_000, ...options } + } + + /** Runs `task` after the next backoff delay, replacing any retry already scheduled. */ + schedule(task: () => void): void { + if (this.stopped) return + clearTimeout(this.timer) + const delay = ExponentialBackoff.calculateDelay({ ...this.options, growthRate: 2, jitter: 0.2, maxAttempts: 0 }, this.attempt++) + this.timer = setTimeout(task, delay) + ;(this.timer as { unref?: () => void }).unref?.() + } + + succeeded(): void { + this.attempt = 0 + } + + stop(): void { + this.stopped = true + clearTimeout(this.timer) + } +} diff --git a/src/utils/serial-queue.ts b/src/utils/serial-queue.ts new file mode 100644 index 0000000..943a634 --- /dev/null +++ b/src/utils/serial-queue.ts @@ -0,0 +1,17 @@ +/** Runs tasks one at a time, in the order given; independently per key when a key is passed. */ +export class SerialQueue { + private readonly tails = new Map>() + + run(task: () => Promise, key: string = ''): Promise { + const result = (this.tails.get(key) ?? Promise.resolve()).then(task) + const tail = result.catch(() => {}) + this.tails.set(key, tail) + void tail.then(() => this.tails.get(key) === tail && this.tails.delete(key)) + return result + } + + /** Resolves once every task queued so far (for `key`) has settled. */ + async idle(key: string = ''): Promise { + while (this.tails.has(key)) await this.tails.get(key) + } +} diff --git a/test/kv-storage/kv-outbox.test.ts b/test/kv-storage/kv-outbox.test.ts new file mode 100644 index 0000000..e4dc134 --- /dev/null +++ b/test/kv-storage/kv-outbox.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it, vi } from 'vitest' +import { KVOutbox, type OutboxEntry } from '../../src/kv-storage/kv-outbox.js' +import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js' +import { RetryTimer } from '../../src/utils/retry-timer.js' +import { StorageUnavailableError } from '../../src/utils/errors.js' + +describe('KVOutbox', () => { + it('replays entries in order once the source is reachable, and survives a restart', async () => { + const store = new KVStorageInMemory() + let online = false + const sent: OutboxEntry[] = [] + const send = async (entry: OutboxEntry) => { + if (!online) throw new StorageUnavailableError('offline') + sent.push(entry) + } + const firstRetry = new RetryTimer({ baseDelay: 5, maxDelay: 5 }) + const first = new KVOutbox(store, send, firstRetry) + await first.push({ type: 'write', entries: [['a', 1]] }) + await first.push({ type: 'remove', keys: ['a'] }) + expect(first.size).toBe(2) + expect(first.pending.has('a')).toBe(true) + + firstRetry.stop() // the first session ends... + await first.replay() // ...once its in-flight replay attempt has failed + // A new session over the same store picks up the queue. + const second = new KVOutbox(store, send, new RetryTimer({ baseDelay: 5, maxDelay: 5 })) + online = true + await second.load() + await vi.waitFor(() => expect(second.size).toBe(0)) + expect(sent).toEqual([ + { type: 'write', entries: [['a', 1]] }, + { type: 'remove', keys: ['a'] }, + ]) + expect(await store.readMany(['~q/o/000000000001', '~q/o/000000000002'])).toEqual([undefined, undefined]) + }) +}) diff --git a/test/utils/building-blocks.test.ts b/test/utils/building-blocks.test.ts new file mode 100644 index 0000000..0c5eb21 --- /dev/null +++ b/test/utils/building-blocks.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' +import { KeyRange } from '../../src/utils/key-range.js' +import { KeyCounter } from '../../src/utils/key-counter.js' +import { SerialQueue } from '../../src/utils/serial-queue.js' +import { RetryTimer } from '../../src/utils/retry-timer.js' +import { Keys } from '../../src/utils/keys.js' + +describe('KeyRange', () => { + it('contains a key and its subtree, not lookalikes', () => { + const range = new KeyRange('ab') + expect(['ab', 'ab/c', 'ab/c/d'].every((k) => range.contains(k))).toBe(true) + expect(['a', 'abc', 'ab0', 'ab~2D1', 'b'].some((k) => range.contains(k))).toBe(false) + expect(new KeyRange('').contains('anything')).toBe(true) + }) + + it('agrees with its scan bounds for escaped keys (property)', () => { + const path = fc.array(fc.string({ unit: fc.constantFrom('a', 'b', '-', '/', '~', '0', '.'), minLength: 1, maxLength: 3 }), { minLength: 1, maxLength: 3 }) + fc.assert( + fc.property(path, path, (p, q) => { + const range = new KeyRange(Keys.join(p)) + const key = Keys.join(q) + expect(range.contains(key)).toBe(key >= range.gte! && key < range.lt!) + }), + ) + }) + + it('clamps and checks scan ranges', () => { + const range = new KeyRange('ab') + expect(range.clamp({ limit: 5 })).toEqual({ limit: 5, gte: 'ab', lt: 'ab0' }) + expect(range.clamp({ gte: 'ab/x', lt: 'zz' })).toEqual({ gte: 'ab/x', lt: 'ab0' }) + expect(range.clamp({ gte: 'b' })).toBeUndefined() + expect(range.covers('ab/', 'ab0')).toBe(true) + expect(range.covers('a', 'ab0')).toBe(false) + expect(range.covers(undefined, 'ab0')).toBe(false) + expect(new KeyRange('').covers(undefined, undefined)).toBe(true) + }) +}) + +describe('KeyCounter', () => { + it('counts per key and in total', () => { + const counter = new KeyCounter() + counter.add(['a', 'a', 'b']) + expect(counter.size).toBe(3) + expect(counter.take('a')).toBe(true) + expect(counter.has('a')).toBe(true) + expect(counter.take('a')).toBe(true) + expect(counter.has('a')).toBe(false) + expect(counter.take('a')).toBe(false) + expect(counter.size).toBe(1) + counter.clear() + expect(counter.size).toBe(0) + }) +}) + +describe('SerialQueue', () => { + it('runs tasks in order per key, keys independently, and survives failures', async () => { + const queue = new SerialQueue() + const log: string[] = [] + const task = (name: string, ms: number) => () => new Promise((r) => setTimeout(() => (log.push(name), r()), ms)) + const a1 = queue.run(task('a1', 20), 'a') + const failed = queue.run(() => Promise.reject(new Error('boom')), 'a') + const a2 = queue.run(task('a2', 1), 'a') + const b1 = queue.run(task('b1', 1), 'b') + await expect(failed).rejects.toThrow('boom') + await Promise.all([a1, a2, b1]) + expect(log).toEqual(['b1', 'a1', 'a2']) + await queue.idle('a') + }) +}) + +describe('RetryTimer', () => { + it('replaces a pending retry and stops cleanly', () => { + vi.useFakeTimers() + const timer = new RetryTimer({ baseDelay: 10, maxDelay: 10 }) + const first = vi.fn() + const second = vi.fn() + timer.schedule(first) + timer.schedule(second) + vi.advanceTimersByTime(20) + expect(first).not.toHaveBeenCalled() + expect(second).toHaveBeenCalledOnce() + timer.schedule(first) + timer.stop() + vi.advanceTimersByTime(20) + expect(first).not.toHaveBeenCalled() + vi.useRealTimers() + }) +}) -- 2.54.0 From 631c75428bfb2e65243e84d8f734dea974c13b9e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 13:33:07 +0000 Subject: [PATCH 06/11] Plain operations: last write wins, hash-keyed events, simpler composite - KV: one write primitive, write(changes) with { key, value? }; writeIfAbsent removed. The server executes exactly what clients send and never reads values, so they can be encrypted end to end. Adds setFields/getFields on the base class. - HTTP: no server history or event ids; SSE relays every change (own writes included) and the client emits resync when a dropped stream reconnects. - Composite: writes apply locally, then go to the source (undone locally if refused); pushed changes are applied and announced only where they differ from local values; offline writes go to a queue store (any KV storage); sync() = replay, diff, replay. Removes KVReplicaSet, KVOutbox, KeyCounter, SerialQueue and RetryTimer. - Events: id =