Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Claude
|
cb9ff514a0 | ||
|
Claude
|
7d755cecae | ||
|
Claude
|
68d1a1c8e9 | ||
|
Claude
|
3525dbc622 | ||
|
Claude
|
ad517965ee | ||
|
Claude
|
631c75428b | ||
|
Claude
|
1f82ec9110 | ||
|
Claude
|
a8c9539ad9 | ||
|
Claude
|
df7de27e20 | ||
|
Claude
|
4ffbc7e78d | ||
|
Claude
|
08fb78721f |
+43
-2
@@ -2,9 +2,12 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: ['**'] # not tags
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
NPM_REGISTRY: https://verdaccio.harvmaster.com/
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Node ${{ matrix.node }}
|
||||
@@ -12,7 +15,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: [20, 22]
|
||||
node: [22, 24, 26]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
@@ -28,3 +31,41 @@ jobs:
|
||||
run: npm run test:only
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
browser:
|
||||
name: Chromium
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Install Chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Browser tests
|
||||
run: npm run test:browser
|
||||
|
||||
# Pushes only (not pull requests): main → `latest`, next → `next`, other branches → `dev-<branch>`.
|
||||
# See scripts/publish.mjs. Needs the NPM_REGISTRY_TOKEN secret.
|
||||
staging:
|
||||
name: Publish to staging registry
|
||||
needs: [check, browser]
|
||||
if: github.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Configure registry
|
||||
run: |
|
||||
host="${NPM_REGISTRY#https://}"
|
||||
printf 'registry=%s\n//%s:_authToken=${NPM_REGISTRY_TOKEN}\n' "$NPM_REGISTRY" "$host" > .npmrc
|
||||
- name: Publish
|
||||
run: node scripts/publish.mjs "${{ github.ref_name }}"
|
||||
env:
|
||||
NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }}
|
||||
|
||||
@@ -1,24 +1,33 @@
|
||||
# PotentialStorage
|
||||
|
||||
Layered TypeScript storage library: KV → Events → Documents. The full design, milestones and
|
||||
implementation decisions are in [DESIGN.md](./DESIGN.md) — read it before changing anything.
|
||||
Record new decisions in DESIGN.md section 12.
|
||||
Layered TypeScript storage library: KV → Events → Documents. The design is in
|
||||
[DESIGN.md](./DESIGN.md); read it before changing anything and keep it up to date.
|
||||
|
||||
## Commands
|
||||
|
||||
- `npm test` — `tsc --noEmit` then all Vitest tests (including `*.test-d.ts` type tests)
|
||||
- `npm run test:only -- <file>` — Vitest only, e.g. a single file
|
||||
- `npm run typecheck` — `tsc --noEmit` over src, test and bench
|
||||
- `npm run bench` — `vitest bench`
|
||||
- `npm run build` — emit `dist/`
|
||||
- `npm run format` / `npm run format:check` — Prettier
|
||||
- `npm test`: `tsc --noEmit`, then all Vitest tests (including `test/types.test-d.ts`)
|
||||
- `npm run test:only -- <file>`: Vitest only
|
||||
- `npm run test:browser`: `test/browser/` in headless Chromium (Vitest browser mode + Playwright)
|
||||
against a real server on another origin (`test/browser/global-setup.ts`)
|
||||
- `npm run typecheck`, `npm run build`, `npm run bench`
|
||||
- `npm run format` / `npm run format:check`: Prettier
|
||||
|
||||
CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs format check, typecheck, tests and build on Node 20 and 22.
|
||||
CI (`.gitea/workflows/ci.yml`, Gitea Actions) runs the format check, typecheck, tests and build on
|
||||
Node 22, 24 and 26, and the Chromium tests. On pushes it then publishes to the staging registry
|
||||
(`scripts/publish.mjs`: `main` → `latest`, `next` → `next`, others → `dev-<branch>`).
|
||||
|
||||
## Rules
|
||||
## Conventions
|
||||
|
||||
- No runtime dependencies in `src/`. Dev dependencies only.
|
||||
- Drivers stay flat and string-keyed; all path/index/escaping logic lives in the node classes.
|
||||
- Self vs child dispatch is by `arguments.length`; a passed-but-`undefined`/`null` key throws.
|
||||
- Multi-result reads are `AsyncIterable`; don't build large arrays internally.
|
||||
- Keep tests green and commit per milestone. Ask the owner before pushing.
|
||||
- One abstract base class per layer (`src/<layer>/base-*.ts`); implementations only provide the raw
|
||||
primitives, and the user-facing API lives in the base class.
|
||||
- Classes over loose functions; TS `private`, not `#`; no overloads (use `param?` or tuple-union
|
||||
rest params); don't override base methods in subclasses.
|
||||
- **Never reassign a field after construction.** `child()` nodes are frozen `Object.create(root)`
|
||||
views, so `this.x = ...` on a child throws. Keep mutable state in a `private readonly state`
|
||||
object or in Maps (see `KVStorageComposite`, `KVStorageHttp`).
|
||||
- No runtime dependencies; third-party code is vendored (`src/utils/sse-session/`, `event-emitter.ts`,
|
||||
`extended-json.ts`, with a header naming the source). Tests live only in `test/`, mirroring `src/`.
|
||||
- `README.md` examples are mirrored in `test/readme.test.ts`; change both together.
|
||||
- New KV/document implementations must pass `test/kv-storage/kv-storage.conformance.ts` /
|
||||
`test/document-storage/document-storage.suite.ts`.
|
||||
- Ask the owner before pushing.
|
||||
|
||||
@@ -1,655 +1,321 @@
|
||||
# PotentialStorage — Design
|
||||
|
||||
This document is the design brief agreed with the project owner (sections 0–11), followed by
|
||||
implementation decisions made where the brief was silent (section 12) and benchmark baselines
|
||||
(section 13).
|
||||
Layered TypeScript storage: **KV → Events → Documents**. Each layer is an abstract base class
|
||||
that implementations extend; each is usable on its own. No runtime dependencies.
|
||||
|
||||
## 0. Setup and workflow
|
||||
|
||||
- Repository: `git@git.harvmaster.com:Harvmaster/PotentialStorage.git` (SSH; self-hosted Git, not GitHub). Clone it. If you can't reach it, stop and tell the owner rather than starting a fresh repo elsewhere.
|
||||
- Inspect what's already in the repo first. If it has tooling (package.json, tsconfig, test runner), keep it. If it's empty, set up:
|
||||
- TypeScript (`strict: true`, `noUncheckedIndexedAccess: true`), ESM, Node ≥ 20
|
||||
- Vitest for tests and `vitest bench` for benchmarks; `expectTypeOf` for type tests
|
||||
- Package name `potential-storage` unless one already exists
|
||||
- **No runtime dependencies in the core.** Dev dependencies are fine (e.g. `fast-check` for property tests).
|
||||
- Copy this brief into the repo as `DESIGN.md` and create a short `CLAUDE.md` pointing at it plus the build/test commands.
|
||||
- Work milestone by milestone (section 9). Commit after each milestone with a clear message, and keep tests green. Ask the owner before pushing.
|
||||
This supersedes the original implementation brief (see git history before the `simplify` branch);
|
||||
section 7 lists what changed and why.
|
||||
|
||||
## 1. Principles
|
||||
|
||||
1. **Three layers**, each usable on its own: **KV** → **Event** (append-only, built on KV) → **Document** (Mongo-like, built on Events or native).
|
||||
2. **Drivers vs stores.** Backends implement a tiny, flat, string-keyed _driver_. All path, child, index, escaping and typing logic lives once in the user-facing _node_ classes, so every backend gets identical behaviour.
|
||||
3. **One navigation primitive: `child(key)`.** There are no collection/document classes. Every node is a store over its direct children, and `child(key)` re-roots to that key. Navigation is synchronous and does no I/O.
|
||||
4. **No methods on data.** Reads return plain values.
|
||||
5. **Performance is a requirement.** Batch operations are the primitive; scans are ordered and streamed; nothing replays or scans more than it must (section 8).
|
||||
1. **One abstract class per layer.** Implementations provide a handful of raw primitives; the
|
||||
whole user-facing API is written once in the base class. A client (e.g. HTTP) and a server-side
|
||||
store are the same kind of object, and wrappers (e.g. encryption) are just another subclass
|
||||
that delegates the primitives.
|
||||
2. **One navigation primitive: `child(key)`.** Every instance is a node; `child` returns a view of
|
||||
the same storage rooted deeper (via `Object.create`, so it shares the instance's state). Sync, no I/O.
|
||||
Child nodes are **frozen**: an implementation that reassigns a field while running on a child
|
||||
would silently shadow the root's value, so that throws instead. Implementations keep mutable
|
||||
state in a `readonly state` object or in Maps.
|
||||
3. **Self or child by argument.** `get()` reads this node, `get(key)` a child. A key that is passed
|
||||
but `undefined`/`null` throws (`InvalidKeyError`) instead of silently meaning "self". Methods use
|
||||
optional parameters or tuple-union rest parameters, not overloads.
|
||||
4. **Indexes belong to the store.** Document stores receive the fields to index and use them however
|
||||
suits the backend (in-memory equality maps, native Mongo/SQLite/IndexedDB indexes, or nothing).
|
||||
5. **Simple over clever.** Code style: classes, TS `private` (not `#`), no method overrides in
|
||||
subclasses, tests only under `test/`.
|
||||
6. **No runtime dependencies.** Third-party code is vendored into `src/utils/` with a header naming
|
||||
its source: `sse-session/` (Harvmaster/sse-session), `event-emitter.ts` and `extended-json.ts`
|
||||
(xo-cash-utils, branch installable-sse). Node ≥ 22; CI runs Node 22, 24 and 26.
|
||||
|
||||
## 2. Paths and key encoding (`src/core/path.ts`)
|
||||
|
||||
- A node's path is `readonly string[]` of **unescaped** segments. Number keys are converted with `String()`.
|
||||
- The separator is a single character, configurable, default `/`.
|
||||
- **Escaping** (per segment): percent-encode `%`, the separator, and a **leading** `~`. Escaped segments never contain a raw separator. Write `escapeSegment` / `unescapeSegment` with round-trip property tests.
|
||||
- **Reserved segments** start with `~` (user segments can't, because a leading `~` is escaped):
|
||||
- `~i` index entries
|
||||
- `~u` unique-index claims
|
||||
- `~e` event commits
|
||||
- `~p` document projections root
|
||||
- Each node precomputes in its constructor:
|
||||
- `selfKey` = escaped segments joined by the separator (e.g. `users/123`)
|
||||
- `prefix` = `selfKey + sep` (`''` for the root)
|
||||
- No string work per call beyond `prefix + escapeSegment(key)`.
|
||||
- Keys are ordered **lexicographically** (so `10` sorts before `9`). Document this. An order-preserving numeric key encoding is a possible later option, not part of v1.
|
||||
|
||||
Example layout:
|
||||
## 2. Layout
|
||||
|
||||
```
|
||||
users/123 → KV value of users/123
|
||||
users/123/posts/p1 → KV value of a nested child
|
||||
users/~i/byEmail/sada@x.io/123 → index entry (value: empty / true)
|
||||
orders/o1/~e/000000000007 → event commit #7 of stream orders/o1
|
||||
~p/users/123 → document projection { state, version, commit }
|
||||
src/
|
||||
kv-storage/ base-kv-storage.ts, kv-storage-in-memory.ts, kv-storage-indexeddb.ts,
|
||||
kv-storage-http.ts, kv-storage-http-server.ts, kv-storage-composite.ts,
|
||||
kv-storage-encrypted.ts
|
||||
event-storage/ base-event-storage.ts, event-storage-kv.ts
|
||||
document-storage/ base-document-storage.ts, document-storage-in-memory.ts, document-storage-kv.ts,
|
||||
event-sourced-document-storage.ts
|
||||
utils/ keys, sorted-map, json, extended-json, event-emitter, ulid, errors, schema (types),
|
||||
document-path/query/update, sse-session/ (vendored)
|
||||
test/ mirrors src/; shared suites: kv-storage.conformance.ts, document-storage.suite.ts;
|
||||
browser/ runs in headless Chromium against a real cross-origin server;
|
||||
readme.test.ts runs the README's examples
|
||||
bench/
|
||||
```
|
||||
|
||||
## 3. Layer 1: KV
|
||||
|
||||
### 3.1 Driver interface (`src/kv/driver.ts`)
|
||||
|
||||
```ts
|
||||
export interface ScanOptions {
|
||||
gte?: string
|
||||
lt?: string
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
values?: boolean // when false/omitted, yield value as undefined
|
||||
}
|
||||
|
||||
export interface KVDriver {
|
||||
getMany(keys: string[]): Promise<(unknown | undefined)[]> // same order as keys
|
||||
setMany(entries: [key: string, value: unknown][]): Promise<void>
|
||||
deleteMany(keys: string[]): Promise<void>
|
||||
scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]> // MUST be key-ordered
|
||||
|
||||
// Optional capabilities. Stores feature-detect them and fall back where possible.
|
||||
setIfAbsent?(entries: [string, unknown][]): Promise<boolean[]> // true = written
|
||||
batch?(ops: BatchOp[]): Promise<void> // atomic multi-op write
|
||||
deleteRange?(gte: string, lt: string): Promise<void>
|
||||
}
|
||||
|
||||
export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string }
|
||||
```
|
||||
|
||||
- Values are passed through as JSON-serialisable values. Drivers that need bytes (HTTP, Redis, files) serialise internally via a pluggable `Codec` (default JSON).
|
||||
- Provide an abstract `BaseKVDriver` with default implementations of `deleteRange` (scan + deleteMany) and helpers, so a minimal driver implements only the four required methods.
|
||||
- **`InMemoryKVDriver`**: a sorted structure (sorted array with binary search is fine for v1; keep it behind an interface so it can become a B-tree). It implements every optional capability: `setIfAbsent`, atomic `batch` and `deleteRange`.
|
||||
- Write a reusable **driver conformance suite** `runKVDriverConformance(name, makeDriver)` covering ordering, `gte`/`lt` bounds, reverse, limit, `values` on/off, `getMany` order with missing keys, and each optional capability when present.
|
||||
|
||||
### 3.2 `KVStorage` / `KVNode` (`src/kv/node.ts`)
|
||||
|
||||
```ts
|
||||
const db = new KVStorage(driver, { separator: '/', indexes: { ... } }) // db is the root KVNode
|
||||
const users = db.child('users')
|
||||
const user = users.child('123')
|
||||
```
|
||||
|
||||
API of every `KVNode` (types in section 7):
|
||||
|
||||
| Method | Meaning |
|
||||
| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `child(key)` | Sync. Returns a node at `path + key`. |
|
||||
| `as<S>()` | Type assertion for untyped use; returns the same node retyped. |
|
||||
| `get()` / `get(key)` | Own value at `selfKey` / value at child key. |
|
||||
| `getMany(keys)` | Batch child read, same order, `undefined` for missing. |
|
||||
| `set(value)` / `set(key, value)` | Write own value / child value. |
|
||||
| `setMany(entries)` | Batch child write. |
|
||||
| `delete()` / `delete(key)` | Delete own value / child value only (not descendants). |
|
||||
| `deleteMany(keys)` | Batch child delete. |
|
||||
| `deleteTree()` / `deleteTree(key)` | Delete own value **and all descendants** (uses `deleteRange` if available). |
|
||||
| `keys(opts?)` | `AsyncIterable<string>` of **direct** child keys (unescaped). Options: `limit`, `after`, `reverse`. |
|
||||
| `entries(opts?)` | `AsyncIterable<[key, value]>` of direct children that have an own value. |
|
||||
| `tree({ depth })` | Assemble an object from descendant keys up to `depth` (for field-per-key style). Bounded and explicit; never the default. |
|
||||
| `findBy(index, value \| { gte?, lt? }, opts?)` | Index lookup over direct children (section 3.3). |
|
||||
|
||||
**Arity dispatch (important):**
|
||||
|
||||
- Self vs child is decided by `arguments.length`, never by checking for `undefined`.
|
||||
- If a key argument is passed but is `undefined` or `null`, throw a `TypeError` (this prevents `users.get(maybeId)` silently reading self).
|
||||
- The root node (`path = []`) has no own value: `get()`, `set(v)` and `delete()` on root throw.
|
||||
- Implement one private `#resolve(args)` that every method uses.
|
||||
|
||||
**Direct-children listing (`keys`)** uses a skip-scan:
|
||||
|
||||
1. Scan `gte: prefix`, `lt: prefix + succ(sep)`.
|
||||
2. For each key, take the first segment after `prefix`. Skip reserved (`~…`) segments. Yield the segment (unescaped) if it differs from the previous one.
|
||||
3. When the next key is a descendant of the segment just yielded, re-seek the driver to `gte: prefix + escSeg + succ(sep)` (where `succ(c)` is the next char code) instead of iterating through the subtree. Keep using the same iterator while keys are siblings.
|
||||
|
||||
Listing must be O(direct children × seek cost), not O(subtree). A child that has no own value but has descendants **is** listed.
|
||||
|
||||
### 3.3 Indexes (`src/kv/indexes.ts`)
|
||||
|
||||
Indexes are declared on **path patterns** in the `KVStorage` constructor. A pattern names a parent node, and its index applies to that node's direct children's values:
|
||||
|
||||
```ts
|
||||
new KVStorage(driver, {
|
||||
indexes: {
|
||||
users: { byEmail: (u: User) => u.email },
|
||||
'users/*/posts': { byDate: (p: Post) => p.createdAt, byTag: (p: Post) => p.tags },
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
- An index function returns `IndexValue | IndexValue[] | undefined`, where `IndexValue = string | number | boolean | null`. An array means multi-entry; `undefined` means not indexed.
|
||||
- Compile patterns into a **segment trie** (`*` matches one segment). A node resolves its index set **once, at construction**, in O(depth).
|
||||
- Entry key: `<nodePrefix>~i/<indexName>/<encodedValue>/<escapedChildKey>` with a `true` value.
|
||||
- **`encodeIndexValue` must be order-preserving and separator-safe:**
|
||||
- Type tag first, so values order by type: `null` < `false` < `true` < numbers < strings.
|
||||
- Numbers use an order-preserving float64 encoding (flip the sign bit for positives and all bits for negatives, then fixed-width hex).
|
||||
- Strings are escaped like segments.
|
||||
- This enables `findBy(index, { gte, lt })` range scans.
|
||||
- **Writes to an indexed node:**
|
||||
1. Read the old value(s) (`getMany`) and compute the diff of index entries.
|
||||
2. If the driver has `batch`, write the value and the index adds/removes atomically.
|
||||
3. Otherwise, write index **adds first**, then the value, then index **removes**. Treat index entries as hints: `findBy` re-reads each value, re-checks the index function, and lazily deletes stale entries (read-repair).
|
||||
- Non-indexed nodes must **not** pay the read-before-write cost.
|
||||
|
||||
## 4. Layer 2: Events (`src/events/`)
|
||||
|
||||
An append-only log per stream. A stream is identified by a node path. Same `child()` navigation as KV.
|
||||
|
||||
```ts
|
||||
const events = new EventStorage(kv) // takes a KVStorage (or a KVNode as root)
|
||||
const orders = events.child('orders')
|
||||
|
||||
await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 })
|
||||
await orders.child('o1').append([{ type: 'paid' }]) // self form
|
||||
for await (const e of orders.read('o1', { from: 1 })) {
|
||||
/* { version, commit, at, data } */
|
||||
}
|
||||
await orders.version('o1')
|
||||
for await (const id of orders.keys()) {
|
||||
} // child streams
|
||||
orders.child('o1').child('shipments')
|
||||
```
|
||||
|
||||
- `EventStorage` is the abstract interface. `KVEventStorage` is the KV-backed implementation. Leave room for native backends (EventStoreDB, Kafka) implementing the same interface.
|
||||
- **Arity:** `append(events, opts?)` vs `append(key, events, opts?)`. Dispatch on `Array.isArray(args[0])`, plus the undefined-key rule from 3.2. `read`, `version` and `keys` follow the 3.2 rules.
|
||||
- **Versions:** an empty stream has version 0, and the first event has version 1. `expectedVersion: number | 'any'` (default `'any'`).
|
||||
- **Storage: one key per commit, densely numbered.**
|
||||
- Key: `<streamSelfKey>/~e/<commitSeq zero-padded to 12>`
|
||||
- Value: `{ firstVersion, at, events: E[] }`
|
||||
- The commit sequence (not the event version) is the key so that **any** stale writer collides with the existing key. Keying by version would let a stale writer create overlapping version ranges.
|
||||
- **Append:**
|
||||
1. Get the head `{ commitSeq, version }` (from the cache, or a reverse scan with `limit: 1`).
|
||||
2. If `expectedVersion` is a number and doesn't match the head, throw `ConcurrencyError`.
|
||||
3. Call `setIfAbsent(nextCommitKey)`.
|
||||
4. On `false` (collision): refresh the head. If `expectedVersion` was given, throw `ConcurrencyError`; otherwise retry up to 5 times.
|
||||
5. Without `setIfAbsent`, fall back to a per-stream in-process mutex. Document that this mode is single-writer only.
|
||||
- **Head cache:** a bounded LRU keyed by stream path. Update it on successful append; invalidate it on collision.
|
||||
- **`read({ from, to, reverse, limit })`** streams events with metadata. To start at version `from`, locate the containing commit by binary search over commit numbers using `getMany` probes (O(log commits) round trips), then forward-scan.
|
||||
- There is no update or delete. Keys inside `~e` are never rewritten.
|
||||
|
||||
## 5. Layer 3: Documents (`src/documents/`)
|
||||
|
||||
`DocumentStorage` is the interface. There are two implementations:
|
||||
|
||||
- `EventSourcedDocumentStorage(events, { projections?: KVNode })` is the v1 target. Projections default to the `~p` root on the same KV.
|
||||
- `MongoDocumentStorage(client)` is a later milestone (section 9).
|
||||
|
||||
```ts
|
||||
const docs = new EventSourcedDocumentStorage(events, { indexes: { users: [{ fields: ['email'], unique: true }, { fields: ['age'] }] } })
|
||||
const users = docs.child('users')
|
||||
|
||||
const id = await users.insert({ name: 'Ada', age: 36 }) // generates a time-sortable id
|
||||
await users.set('123', { name: 'Ada', age: 36 })
|
||||
await users.update('123', { $set: { name: 'Ada L.' }, $inc: { logins: 1 } })
|
||||
await users.child('123').update({ $unset: { nickname: true } }) // self form
|
||||
await users.get('123')
|
||||
await users.delete('123')
|
||||
for await (const { key, data } of users.find({ age: { $gte: 18 } }, { sort: { age: 1 }, limit: 20 })) {
|
||||
}
|
||||
await users.findOne({ email: 'ada@x.io' })
|
||||
await users.history('123') // event-sourced only
|
||||
await users.getAt('123', 17) // state as of version 17
|
||||
users.child('123').child('posts') // nested docs, same as KV
|
||||
```
|
||||
|
||||
### 5.1 Events and projections
|
||||
|
||||
- The doc at path `P` has its event stream at `P` in `EventStorage`. Its projection is at `~p/P` in the projections KV: `{ state, version }` (and `deleted: true` for tombstones).
|
||||
- Event types: `{ op: 'set', doc }`, `{ op: 'patch', set?: Record<path, value>, unset?: path[] }`, `{ op: 'delete' }`.
|
||||
- **Store resolved deltas, not operators.** For example, `$inc` resolves to `{ set: { logins: 6 } }`, so replay is deterministic.
|
||||
- Supported update operators in v1: `$set`, `$unset`, `$inc`, `$push`, `$pull`, `$addToSet`, with dot-path support.
|
||||
|
||||
**Write path:**
|
||||
|
||||
1. Read the projection.
|
||||
2. Compute the new state and patch.
|
||||
3. `append` with `expectedVersion = projection.version`.
|
||||
4. Write the projection and index entries (atomic `batch` if available).
|
||||
5. On `ConcurrencyError`: catch up the projection from the stream, recompute, and retry (bounded).
|
||||
|
||||
**Reads** are a single projection `get`. Never replay on read.
|
||||
|
||||
**Crash between append and projection write:**
|
||||
|
||||
- The projection lags the stream. The next write's `expectedVersion` fails, which triggers catch-up, so the system self-heals.
|
||||
- Provide `get(key, { consistent: true })` that checks the stream head and catches up first.
|
||||
- Provide a `catchUp(node?)` maintenance method to repair a subtree.
|
||||
- `insert` ids are time-sortable (ULID-like). Implement a small generator, with no dependency.
|
||||
|
||||
### 5.2 Queries and document indexes
|
||||
|
||||
- Filter subset:
|
||||
- `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$exists`
|
||||
- `$and`, `$or`, `$not`
|
||||
- implicit equality and dot paths
|
||||
- Options: `sort`, `limit`, `skip`, `projection` (fields).
|
||||
- `find` on a node queries that node's **direct children**.
|
||||
- Indexes are declared per path pattern (same trie mechanism as 3.3).
|
||||
- Each index has `fields` (compound allowed); entries go under `~p/<node>/~i/<name>/<enc(field1)>/<enc(field2)>/…/<childKey>`, using `encodeIndexValue`.
|
||||
- `unique: true` additionally claims `~u/<name>/<enc(values)>` via `setIfAbsent`. Throw at construction if the driver lacks `setIfAbsent`.
|
||||
- **Planner:**
|
||||
- Pick the index that covers the most leading equality fields, then at most one range field.
|
||||
- Use the index order for `sort` when it matches.
|
||||
- Otherwise do a full scan of children projections, with a one-time dev-mode `console.warn` naming the filter.
|
||||
|
||||
## 6. Shared conventions
|
||||
|
||||
- Every layer uses the same navigation (`child`, `as`, `keys`) and the same self/child arity rules.
|
||||
- All multi-result reads are `AsyncIterable` (streamed, cancellable). Never build large arrays internally.
|
||||
- Errors are typed classes exported from `src/errors.ts`: `ConcurrencyError`, `UniqueConstraintError`, `CapabilityError`, `InvalidKeyError`.
|
||||
- Public exports go through `src/index.ts`, with subpath exports per layer if convenient.
|
||||
|
||||
## 7. Types (`src/types/schema.ts`)
|
||||
|
||||
Each node is typed by a single **Spec** that describes its own value and its children by key:
|
||||
|
||||
```ts
|
||||
export type Spec = { value?: unknown; children?: Record<string, Spec> }
|
||||
export type AnySpec = { value: unknown; children: Record<string, AnySpec> } // untyped default
|
||||
|
||||
export type ValueOf<S> = S extends { value: infer V } ? V : never
|
||||
export type ChildrenOf<S> = S extends { children: infer C } ? C : {}
|
||||
export type KeyOf<S> = Extract<keyof ChildrenOf<S>, string | number>
|
||||
export type ChildOf<S, K extends KeyOf<S>> = ChildrenOf<S>[K]
|
||||
|
||||
// Helpers
|
||||
export type Many<V, C extends Record<string, Spec> = {}> = { children: { [id: string]: { value: V; children: C } } }
|
||||
export type Fields<T> = { [K in keyof T & string]: { value: T[K] } }
|
||||
```
|
||||
|
||||
Node signatures:
|
||||
|
||||
```ts
|
||||
class KVNode<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): KVNode<ChildOf<S, K>>
|
||||
get(): Promise<ValueOf<S> | undefined>
|
||||
get<K extends KeyOf<S>>(key: K): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
// set/delete/getMany/etc. follow the same pattern
|
||||
}
|
||||
```
|
||||
|
||||
Example to cover with `expectTypeOf` tests:
|
||||
|
||||
```ts
|
||||
type User = { username: string; age: number }
|
||||
type Post = { title: string; createdAt: string }
|
||||
type Schema = {
|
||||
children: {
|
||||
users: Many<User, { posts: Many<Post> }>
|
||||
profile: { children: { [id: string]: { children: Fields<User> } } } // field-per-key style
|
||||
}
|
||||
}
|
||||
|
||||
const db = new KVStorage<Schema>(driver)
|
||||
db.child('users').get('123') // User | undefined
|
||||
db.child('users').child('123').get() // User | undefined (same key)
|
||||
db.child('users').child('123').child('posts') // KVNode<Many<Post>>
|
||||
db.child('users').child('123').child('postz') // ✗ type error
|
||||
db.child('profile').child('9').get('age') // number | undefined
|
||||
db.child('profile').child('9').get('agee') // ✗ type error
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- **Named levels** check keys at compile time; **dynamic `[id: string]` levels** accept any string.
|
||||
- One node's children can't mix an index signature with named keys (a TS restriction), so schemas alternate between dynamic and named levels. That alternation exists only in types; the runtime API is just `child()`.
|
||||
- `EventStorage` and `DocumentStorage` nodes use the same Spec machinery (the value type is the event type or the document type respectively).
|
||||
|
||||
## 8. Performance requirements
|
||||
|
||||
- Single-key ops: exactly one driver call (no hidden reads), except indexed writes (one `getMany` before the write) and document writes (projection read plus append plus projection write).
|
||||
- `*Many` ops: one driver call each (plus index work), never N calls.
|
||||
- `keys()`: skip-scan, O(direct children) seeks.
|
||||
- Event head lookup: cached; cold lookup is a single reverse scan with `limit: 1`.
|
||||
- Document `get`: one driver `get`.
|
||||
- Add `vitest bench` benchmarks for:
|
||||
- KV `get`/`set`/`getMany` at 10k/100k keys
|
||||
- `keys()` on a node with 1k children each having 100 descendants
|
||||
- event append and read
|
||||
- document `get`/`update`/indexed `find` vs full-scan `find`
|
||||
- Record baseline numbers in `DESIGN.md`.
|
||||
|
||||
## 9. Milestones
|
||||
|
||||
1. **Core:** `path.ts` (escape, unescape, encode, `succ`), `encodeIndexValue`, errors. Property tests for round-trips and order preservation.
|
||||
2. **KV driver:** `KVDriver`, `BaseKVDriver`, `InMemoryKVDriver`, conformance suite.
|
||||
3. **KV node:** full API from 3.2, including arity rules, skip-scan `keys`, `deleteTree`, `tree`. Add a model-based property test: random ops vs a reference `Map`.
|
||||
4. **KV indexes:** pattern trie, atomic and non-atomic write paths, read-repair, range `findBy`.
|
||||
5. **Events:** `KVEventStorage` with commit keys, `setIfAbsent` and mutex modes, head cache, binary-search `read({ from })`, concurrency tests (simulated interleaving with two storage instances on one driver).
|
||||
6. **Documents:** `EventSourcedDocumentStorage`, update operators, projections, catch-up, queries, planner, unique indexes, `history` and `getAt`.
|
||||
7. **Types:** Spec machinery wired through all three layers, plus `expectTypeOf` tests from section 7.
|
||||
8. **HTTP driver:**
|
||||
- A `createKVHttpHandler(driver)` (framework-agnostic `Request → Response`) plus an `HttpKVDriver` client.
|
||||
- Endpoints: `POST /getMany`, `POST /setMany`, `POST /deleteMany`, `POST /batch`, `POST /setIfAbsent`, `GET /scan?gte<&limit&reverse&values` (NDJSON stream).
|
||||
- It must pass the conformance suite end-to-end.
|
||||
9. **Benchmarks** and a `DESIGN.md` update with results.
|
||||
10. **(Later, ask first)** `MongoDocumentStorage`:
|
||||
- Path pattern maps to a collection (e.g. `users/*/posts` → `users.posts`), with a `_parent` field and a compound index `(_parent, _id)`.
|
||||
- Native indexes map from the same index declarations.
|
||||
- It should pass the document test suite where semantics allow (no `history`/`getAt`).
|
||||
|
||||
## 10. Decisions already made (don't revisit without asking)
|
||||
|
||||
- `child()`-only navigation; no collection/doc classes.
|
||||
- `get()` with no argument reads the node's **own key**. It never assembles the subtree; `tree()` does that explicitly.
|
||||
- A node's own value and its children are independent keys.
|
||||
- Self/child dispatch by arity; a passed-but-undefined key throws.
|
||||
- Indexes are declared on path patterns at construction.
|
||||
- Events use one key per commit, densely numbered.
|
||||
- Documents store resolved deltas, and reads hit projections only.
|
||||
- Keys sort lexicographically.
|
||||
- Values are JSON-serialisable; byte drivers use a codec.
|
||||
|
||||
## 11. Open questions to raise with the owner (don't block on them)
|
||||
|
||||
- Order-preserving numeric **keys** (not just index values)?
|
||||
- Should multi-writer safety be **required** (a hard error without `setIfAbsent`) rather than a documented single-writer fallback?
|
||||
- The hard-delete story for immutable event logs (compaction or crypto-shredding) for privacy deletion requests.
|
||||
|
||||
## 12. Implementation decisions
|
||||
|
||||
Decisions taken where the brief was silent. Each is the simplest option consistent with sections 1–10.
|
||||
|
||||
### M1 — Core
|
||||
|
||||
- **Separator** must be a single printable ASCII character other than `%` and `~`.
|
||||
- **Empty keys are rejected** (`InvalidKeyError`), as are `NaN`/`±Infinity` number keys. An empty segment
|
||||
would produce `a//b`, which is ambiguous with the subtree layout.
|
||||
- **Separator escaping** uses `%` + two uppercase hex digits (`/` → `%2F`, `%` → `%25`, leading `~` → `%7E`).
|
||||
- **`InvalidKeyError` extends `TypeError`**, so "passed-but-undefined key throws a `TypeError`" (3.2) and
|
||||
the typed error class (6) are the same throw.
|
||||
- **Index string encoding deviates from "escaped like segments".** Percent-escaping is not
|
||||
order-preserving (`"a/"` → `"a%2F"` sorts below `"a&"`, and `"a"` + sep sorts above `"a!"`), which
|
||||
would break `findBy` range scans. Instead every character the index encoder emits sorts strictly
|
||||
above the separator: code units `<= succ(sep)` are written as `succ(sep)` followed by
|
||||
`chr(succ(sep) + 1 + c)`. This is monotone and prefix-safe, so `a < b ⇔ enc(a)+sep+… < enc(b)+sep+…`
|
||||
exactly (property-tested). Encodings are still separator-free.
|
||||
- **Type tags and number digits**: tags are the first five digits of the digit alphabet; numbers use
|
||||
16 digits of `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16 characters
|
||||
just above the separator. `-0` is stored as `0`; `NaN` cannot be indexed.
|
||||
- **Ordering** of strings is by UTF-16 code unit (JS `<`). Drivers that compare UTF-8 bytes agree
|
||||
except for astral-plane characters vs. U+E000–U+FFFF; noted as a driver-author caveat.
|
||||
|
||||
### M2 — KV driver
|
||||
|
||||
- **Conformance suite location**: `src/testing/kv-conformance.ts`, exported as `potential-storage/testing`.
|
||||
It imports `vitest`, which is an _optional peer dependency_; the core entry point never imports it.
|
||||
- **`InMemoryKVDriver` clones values** on write and read (`structuredClone`, skipped for primitives) so
|
||||
stored state can't be mutated through shared references, matching serialising drivers. Opt out
|
||||
with `{ clone: false }`.
|
||||
- **Scans tolerate concurrent writes**: the in-memory scan remembers the last key yielded and re-seeks
|
||||
if the map changed while suspended. Read-repair and `deleteTree` rely on this. The conformance suite
|
||||
requires it of every driver (deleting an already-yielded key mid-scan must not skip keys).
|
||||
- **`setMany` bulk path**: ≥32 new keys are sorted and merged (O(n + k log k)); fewer are spliced.
|
||||
- **`BaseKVDriver.deleteRange`** deletes in chunks of 1000 (scan with `limit`, then `deleteMany`,
|
||||
repeat), so it never deletes from under its own live iterator. It also provides `get`/`set`/`delete`
|
||||
single-key helpers.
|
||||
- **No fake `batch`**: `BaseKVDriver` deliberately does not provide a non-atomic `batch`, since stores
|
||||
use its presence to mean "atomic".
|
||||
|
||||
### M3 — KV node
|
||||
|
||||
- **Child order is "subtree order", not plain segment order.** The brief's skip-scan assumes a
|
||||
child's keys are contiguous, but they aren't when a sibling extends a segment with a character
|
||||
that sorts below the separator (for `/`: space and `!"#$%&'()*+,-.`). With children `a` and `a-1`
|
||||
the key space is `p/a`, `p/a-1`, `p/a-1/x`, `p/a/x`, so `a`'s descendants sort _after_ `a-1`, and
|
||||
"yield if different from the previous segment" would emit `a` twice. `keys()`/`entries()` therefore
|
||||
order children by **escaped segment followed by the separator** (`a-1/` < `a/`), which is the order
|
||||
of their subtrees in key space. For keys without such characters this is plain lexicographic order.
|
||||
- Forward listing defers a child seen only through its own key on a small stack (bounded by
|
||||
segment length) until the scan passes `p/<seg>/`. It costs no extra driver calls beyond the
|
||||
brief's one seek per child with descendants.
|
||||
- Reverse listing needs one batched `getMany` probe per new child whose segment contains a
|
||||
below-separator character, to find shorter siblings that only have an own key.
|
||||
- `after` is exclusive in the direction of travel and follows the same order.
|
||||
- Implemented once in `src/kv/children.ts` (`scanChildren`) and reused by the event and document
|
||||
layers. A model-based property test checks forward, reverse, `after` and `limit` against
|
||||
brute force with keys like `a`, `a-1`, `a.b`, `a/x`, `%`, `~t`.
|
||||
- **Open question for the owner:** keep subtree order, or change segment escaping so the key
|
||||
layout nests exactly. That would mean escaping every below-separator character with an escape
|
||||
character above the separator, at the cost of the readable percent-encoding.
|
||||
- **Errors from async methods are rejections.** Every async method (`get`, `set`, …) is an `async`
|
||||
function, so a bad key yields a rejected promise rather than a synchronous throw. `child()` is sync
|
||||
and throws synchronously.
|
||||
- **`set(undefined)` is rejected** (`TypeError`: use `delete()`); `undefined` means "missing" in reads.
|
||||
- **`deleteTree()` on the root** is allowed and clears the whole store (chunked scan + delete, since
|
||||
the root range has no upper bound). Subtree deletion is `delete(selfKey)` plus
|
||||
`deleteRange(prefix, selfKey + succ(sep))`. It is not `[selfKey, …)`, which would also catch
|
||||
siblings such as `a!`.
|
||||
- **`tree({ depth })`** builds a plain object: leaves are values, interior nodes are objects. A node
|
||||
with both an own value and children exposes its own value under the `''` property (empty keys are
|
||||
invalid, so it can't collide). Reserved subtrees are skipped. Deeper levels are skipped by seeking
|
||||
rather than reading.
|
||||
- **`keys()` and `entries()` yield string keys** (number keys come back as strings).
|
||||
- **Exposed low-level properties** on every node: `path`, `selfKey`, `prefix`, `end` (subtree upper
|
||||
bound), `driver`, `separator`, `isRoot`. The event and document layers build on these.
|
||||
|
||||
### M4 — KV indexes
|
||||
|
||||
- **Pattern syntax**: segments separated by the storage separator; `*` matches one segment; `''`
|
||||
names the root. A literal `*` key can't be targeted by a pattern.
|
||||
- **Overlapping patterns** that declare the same index name throw at construction.
|
||||
- **Index trie state is stepped per `child()` call** (O(1) in patterns), so each node resolves its
|
||||
indexes in O(depth) overall. A node holds two bound index sets: one over its children (for child
|
||||
writes) and its parent's (for self-form writes such as `users.child('1').set(v)`).
|
||||
- **Index function results**: `undefined` → not indexed; arrays are multi-entry and de-duplicated
|
||||
per child; anything that is not `string | number | boolean | null` (or `NaN`) throws a `TypeError`
|
||||
at write time. Index functions are not called for missing (`undefined`) values.
|
||||
- **`findBy` results** are `[key, value]` pairs (the value is re-read anyway for verification).
|
||||
Range queries accept `gt`/`gte`/`lt`/`lte`. Results are in index-value order, then child-key order,
|
||||
and range queries de-duplicate multi-entry hits.
|
||||
- **`findBy` batching**: entry keys are read in chunks of 128 (or of the remaining `limit`, if
|
||||
smaller) and values fetched with one `getMany` per chunk. Stale entries are deleted with one
|
||||
`deleteMany` per chunk.
|
||||
- **Read-repair race (non-atomic drivers only)**: a reader can see a concurrent writer's fresh
|
||||
index add before that writer's value write lands, judge it stale, and delete it. With multiple
|
||||
writers on a driver without `batch`, this can lose an index entry. Mitigation: `readRepair: false`
|
||||
on `KVStorage` (stale entries are then only skipped, never deleted). With `batch`, entries never go
|
||||
stale, so the race doesn't arise.
|
||||
- **`set` of an indexed child with duplicate keys in one `setMany`**: last write wins, and index
|
||||
diffs are computed once per key.
|
||||
|
||||
### M5 — Events
|
||||
|
||||
- **`EventStorage` is an interface (type), not a class.** The brief's sample says `new EventStorage(kv)`
|
||||
but also says it is the abstract interface; the concrete class is `KVEventStorage(kv, opts?)`, which
|
||||
accepts a `KVStorage` or any `KVNode` as root. Options: `headCacheSize` (default 10 000),
|
||||
`maxRetries` (default 5) and `now` (clock, default `Date.now`).
|
||||
- **Commit value**: `{ firstVersion, at, events }`, with `at` in ms since epoch. Read results are
|
||||
`{ version, commit, at, data }`.
|
||||
- **The root node is not a stream** (consistent with the root having no own KV value). Non-root
|
||||
nodes are both a stream and a container of child streams; `keys()` lists child streams, and a
|
||||
node's `~e` subtree is reserved, so streams and children never mix.
|
||||
- **`version()` always reads storage** (one reverse scan with `limit: 1`) and refreshes the head cache.
|
||||
Only `append` trusts the cache. When a cached head disagrees with a numeric `expectedVersion`,
|
||||
the head is re-read before throwing, so a stale cache never produces a false `ConcurrencyError`.
|
||||
- **`append([])`** writes nothing; it still checks `expectedVersion` against a fresh head and returns it.
|
||||
- **Read arity**: `read()` and `read(opts)` are the self form (the first argument is a non-null
|
||||
object). Any other first argument is a key, so `read(undefined)` throws like every other
|
||||
passed-but-undefined key.
|
||||
- **Locating `from`/`to`**: commit `n` has `firstVersion >= n`, so the commit containing version `v` has
|
||||
`seq <= v`. `read` probes `seq = v` first; with single-event commits that is the answer, in one
|
||||
`getMany`. Otherwise it runs an 8-ary search, one `getMany` of up to 8 probes per round, which is
|
||||
O(log₈ commits) round trips. The head is only read when the guess is past the end.
|
||||
- **Mutex mode** (driver without `setIfAbsent`): a per-stream in-process promise chain serialises
|
||||
appends. It is safe only with a single writer process.
|
||||
|
||||
### M6 — Documents
|
||||
|
||||
- **Projection value** is `{ state, version, commit }`, or `{ deleted: true, version, commit }` for a
|
||||
tombstone. Tombstones keep the version, so re-creating a document continues its stream.
|
||||
- **`expectedCommit` append hint.** Writes pass the projection's `commit` to `append` as
|
||||
`expectedCommit`, so a write is exactly projection `getMany` + `setIfAbsent` + projection `batch`
|
||||
(3 driver calls), even with a cold head cache. A hint taken from a projection can only be stale
|
||||
(which causes a collision, then catch-up and retry), never ahead of the stream, so it cannot
|
||||
create a gap. Non-KV backends may ignore it.
|
||||
- **Per-document in-process write lock.** Writes to the same document within one storage instance
|
||||
are serialised (`KeyedMutex`), so optimistic retries (default 5) are spent only on cross-process
|
||||
contention.
|
||||
- **Return values**: `set` → `void`; `insert` → id; `update` → new state, or `undefined` if the document
|
||||
doesn't exist (pass `{ upsert: true }` to treat missing as `{}`); `delete` → whether a document
|
||||
was deleted. No-op updates (resulting state equal to current) append nothing.
|
||||
- **Arity for documents**: in forms with an optional leading key (`get`, `set`, `update`), a non-null
|
||||
object first argument selects the self form, and any other first argument is a key (so `undefined`
|
||||
throws). `delete`/`history` use argument count. `getAt(version)` vs `getAt(key, version)` also uses
|
||||
argument count, because a version and a numeric key are both numbers.
|
||||
- **The root document node has no own document** (consistent with KV).
|
||||
- **`keys()` on a document node** lists children holding a live document. Unlike KV `keys()`, it skips
|
||||
children that only have nested documents or tombstones.
|
||||
- **`find` results** are `{ key, data }`. `findOne` returns the same shape or `undefined`.
|
||||
- **Filter semantics vs MongoDB** (see `src/documents/filter.ts`):
|
||||
- Comparisons are type-bracketed (`$gt: 5` matches only numbers), and cross-type order follows
|
||||
the index encoding: `null/missing < false < true < numbers < strings < other`.
|
||||
- On array fields, every positive operator in one condition (`$eq $gt $gte $lt $lte $in`) must be
|
||||
satisfied by the **same element** (like `$elemMatch`). In MongoDB, `{ s: { $gt: 5, $lt: 10 } }`
|
||||
matches `[1, 20]`; here it doesn't. This makes index range scans exact. To get MongoDB's
|
||||
behaviour, split the condition with `$and`.
|
||||
- `{ f: null }` matches missing fields. `$not` works both at top level (negating a filter) and at
|
||||
field level. Unknown operators throw.
|
||||
- `$in` is not index-accelerated in v1.
|
||||
- **Index entries** for each field value: scalars as-is, arrays as their scalar elements (multikey;
|
||||
empty arrays → `null`), and missing or non-scalar values as `null`. Every live document therefore
|
||||
has at least one entry per index, so a full index scan sees all documents. Indexing a document
|
||||
where more than one field of a compound index has multiple values throws (as MongoDB does).
|
||||
- **Index names** default to the fields joined by `,`.
|
||||
- **Unique indexes**: claims are scoped **per parent node** (`<node>/~u/<name>/<values>`), like one
|
||||
MongoDB collection per parent. Value tuples where every field is missing/null are not claimed
|
||||
(sparse). Claims are taken before appending, then released if the append fails. A claim value is
|
||||
`[ownerKey, claimedAtMs]`. A claim held by another document is taken over only if that
|
||||
document, after catching up from its stream, no longer holds the value, **and** the claim is older
|
||||
than `claimGraceMs` (default 30 s). The owner might otherwise be between claiming and appending.
|
||||
The residual risk is a writer stalling for longer than the grace period between claiming and
|
||||
appending.
|
||||
- **Planner**: per index, count leading equality fields (k), then whether field k has a range;
|
||||
score = 2k + (range ? 1 : 0). Ties prefer an index whose order satisfies `sort` after dropping
|
||||
equality-constrained sort fields. With no usable predicate, an index matching `sort` is still used
|
||||
as an ordered full index scan, which streams and stops early with `limit`. Range bounds are clamped
|
||||
to the bound's type class. Index-backed results are re-read from projections, re-checked against the
|
||||
full filter, de-duplicated, and stale entries are read-repaired.
|
||||
- **Result order** when `sort` is not given is the plan's order (index order, or child-key order for
|
||||
a full scan). Ties in sorted results are broken by child key in the direction of the first sort
|
||||
field, both for index-ordered and in-memory sorts.
|
||||
- **In-memory sort** keeps a bounded top `skip + limit` buffer when a limit is given. Without a limit,
|
||||
it buffers all matches; this is the one place results are materialised.
|
||||
- **Full-scan warning**: `console.warn` once per filter _shape_ (values replaced by `?`) per storage
|
||||
instance, only for non-empty filters. On by default unless `NODE_ENV === 'production'`; set it with
|
||||
`warnOnFullScan`.
|
||||
- **Catch-up** re-asserts unique claims for the caught-up state with plain writes, since the log is
|
||||
the source of truth. A slow catch-up can briefly overwrite a newer projection with an older one;
|
||||
the next write's `expectedVersion` check detects this and heals it. `catchUp()` walks the subtree
|
||||
via event-stream `keys()` and returns the number of documents repaired.
|
||||
- **Projections for non-KV event stores** must be passed explicitly (`projections: KVNode`); the
|
||||
projection key of a document is then `projections.prefix + docKey`.
|
||||
|
||||
### M7 — Types
|
||||
|
||||
- **`AnySpec` uses an index signature** (`{ [key: string]: AnySpec }`), not `Record<string, …>`, so
|
||||
`KeyOf<AnySpec>` is `string | number` and number keys type-check on untyped stores.
|
||||
- **`ChildOf` guards with `infer C extends Spec`** so that `KVNode<ChildOf<S, K>>` satisfies the `Spec`
|
||||
constraint generically.
|
||||
- **`setMany` entries are a union of precise `[key, value]` pairs** (`EntryOf<S>`), so on named levels a
|
||||
value must match its own key (`['age', 'x']` is rejected).
|
||||
- **Listings**: `keys()` yields `` `${KeyOf<S>}` `` (string literals on named levels, `string` on dynamic
|
||||
ones). `entries()` and `findBy()` yield `[string, V]` on dynamic levels.
|
||||
- **"Accept any node" parameters** are typed `KVNode<AnySpec>` / `EventNode<AnySpec>`. A typed node is
|
||||
assignable to these, but not to `KVNode<any>`, because `KeyOf<any>` collapses to `never`.
|
||||
- **Type tests** live in `test/types.test-d.ts` and are enforced by `tsc --noEmit` (part of `npm test`),
|
||||
including the `@ts-expect-error` negative cases from section 7.
|
||||
|
||||
### M8 — HTTP driver
|
||||
|
||||
- **Extra endpoints** beyond the brief's list: `POST /deleteRange {gte, lt}` and
|
||||
`GET /capabilities`. Unsupported optional operations return `501`, driver errors `500`, and unknown
|
||||
routes `404`.
|
||||
- **Missing vs `null`**: `/getMany` returns `{ values, missing }`, where `missing` lists the indices of
|
||||
absent keys, because `null` is a legitimate stored value.
|
||||
- **Codec scope**: the codec serialises whole request/response bodies and each NDJSON scan line
|
||||
(`[key]` or `[key, value]`), so it must not emit newlines. The default is JSON.
|
||||
- **Capabilities on the client**: `HttpKVDriver` defines `setIfAbsent`/`batch` only when the server
|
||||
supports them, so stores' feature detection (e.g. unique indexes requiring `setIfAbsent`, atomic
|
||||
index writes using `batch`) behaves as it would locally. `HttpKVDriver.connect()` discovers them via
|
||||
`/capabilities`. The constructor takes them explicitly (default: all). Without native
|
||||
`deleteRange` the client falls back to `BaseKVDriver`'s chunked delete.
|
||||
- **Scan streaming**: the server pulls the driver iterator in chunks that grow from 16 to 256
|
||||
entries, so short reads (skip-scan re-seeks) don't make it run far ahead. Cancelling the client
|
||||
iterator cancels the response body, which calls `return()` on the server-side iterator.
|
||||
- **Optional `headers`** (e.g. auth) and an injectable `fetch` on the client; an optional `basePath` on
|
||||
the handler.
|
||||
- The conformance suite runs end-to-end in-process (fetch → handler), over a real `node:http`
|
||||
socket, and against a minimal server driver with discovered capabilities. A smoke test runs KV
|
||||
indexes, events and unique-indexed documents over HTTP.
|
||||
|
||||
### M9 — Benchmarks and in-memory driver performance
|
||||
|
||||
- **The in-memory driver's sorted array was replaced by `ChunkedSortedMap`** (sorted chunks of ≤1024
|
||||
keys, split in half on overflow; values in a `Map`). Profiling showed 59% of document-update time
|
||||
in the sorted array's O(n) `splice` once the store held ~70k keys (each write adds commit, index
|
||||
and claim keys). Inserts and deletes are now O(log n + 1024). Iteration is O(1) per step and
|
||||
re-seeks after a mutation, which replaces the old version-counter revalidation in the driver.
|
||||
The structure stays behind the `SortedMap` interface; a B-tree remains possible.
|
||||
- **Value cloning uses a hand-written JSON deep copy** instead of `structuredClone` (about 10× cheaper
|
||||
per value), in the in-memory driver and the document layer. Values are JSON by contract (§10).
|
||||
|
||||
<!-- decisions appended per milestone -->
|
||||
|
||||
## 13. Benchmark baselines
|
||||
|
||||
`npm run bench` (Vitest bench), in-memory driver with default cloning, Node 22.22, Linux x64 cloud
|
||||
container (shared CPU, so expect ±10–20% run-to-run). Mean time per operation.
|
||||
|
||||
**KV** (`bench/kv.bench.ts`): values are small objects.
|
||||
|
||||
| Operation | 10k keys | 100k keys |
|
||||
| -------------------- | -------- | --------- |
|
||||
| `get` | 1.2 µs | 1.6 µs |
|
||||
| `set` (overwrite) | 1.7 µs | 2.0 µs |
|
||||
| `getMany` × 100 keys | 21 µs | 22 µs |
|
||||
|
||||
**`keys()`** (`bench/keys.bench.ts`): node with 1 000 children, each with 100 descendants (101k keys).
|
||||
|
||||
| Operation | Mean |
|
||||
| -------------------------------- | ------ |
|
||||
| `keys()` skip-scan (1 000 seeks) | 2.9 ms |
|
||||
| Reference: full subtree scan | 36 ms |
|
||||
|
||||
**Events** (`bench/events.bench.ts`): single-event commits.
|
||||
|
||||
| Operation | Mean |
|
||||
| ---------------------------------------------- | ------ |
|
||||
| `append` 1 event (warm head cache) | 6.3 µs |
|
||||
| `read` 100 events from version 5 000 of 10 000 | 174 µs |
|
||||
| `read` full 10 000-event stream | 21 ms |
|
||||
| `version()` (reverse scan, `limit: 1`) | 3.9 µs |
|
||||
|
||||
**Documents** (`bench/documents.bench.ts`): 10 000 docs, indexes on `age` and unique `email`.
|
||||
|
||||
| Operation | Mean |
|
||||
| ----------------------------------------------------------------- | ------- |
|
||||
| `get` | 2.6 µs |
|
||||
| `update` (`$inc`, indexed node: projection read + append + batch) | 22 µs |
|
||||
| `findOne` by unique index (`email`) | 16 µs |
|
||||
| `find` indexed (`age = k`, ~100 hits) | 0.57 ms |
|
||||
| `find` full scan (`city = k`, ~200 hits of 10k) | 28 ms |
|
||||
|
||||
For comparison, the first run with the sorted-array map and `structuredClone` measured: KV `get`
|
||||
3.7 µs, `getMany`×100 186 µs, `append` 34 µs, document `update` 110 µs, full scan 57 ms, and
|
||||
full stream read 62 ms.
|
||||
## 3. Keys (`utils/keys.ts`)
|
||||
|
||||
- A node's key is its escaped path segments joined by `/` (`users/123`). The separator is fixed.
|
||||
- **Escaping:** every character at or below `/` (space, `!"#$%&'()*+,-.`, `/`, control characters)
|
||||
and `~` itself becomes `~XX` (uppercase hex). Escaped segments contain only characters above `/`,
|
||||
so all keys of one child sort together (`p/a`, `p/a/…`, then `p/a~2D1` for `a-1`). That keeps
|
||||
child listing a plain skip-scan.
|
||||
- **Reserved segments** are `~` + a lowercase letter (`~e` = event commits). Escaping only produces
|
||||
`~` + uppercase hex, so user keys can't collide.
|
||||
- Keys sort lexicographically by escaped form (`10` before `9`; `a-1` → `a~2D1` after `a0`).
|
||||
- Empty keys, `NaN`/`Infinity` and non-string/number keys are rejected.
|
||||
|
||||
## 4. KV storage
|
||||
|
||||
`BaseKVStorage` primitives (absolute keys). Implementations provide two reads and one protected
|
||||
write; the public `writeMany(changes)` / `removeMany(keys)` live in the base class and emit a
|
||||
`change` event after each write, so every store emits change events.
|
||||
|
||||
| Primitive | |
|
||||
| ------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
|
||||
| `readMany(keys)` | values in order, `undefined` for missing |
|
||||
| `scan({ gte, lt, limit, reverse, values })` | key-ordered; must keep working if keys change mid-scan (keys present throughout appear exactly once) |
|
||||
| `write(changes)` → public `writeMany` | `{ key, value }` sets, `{ key }` deletes, in one batch |
|
||||
|
||||
There is no conditional write. Stores execute exactly the operations they are given, and the last
|
||||
write wins. Data that can be written concurrently belongs in event storage (section 5), where
|
||||
concurrent appends never touch the same key. Data whose key is the hash of its content is
|
||||
immutable, so concurrent writes of it are identical.
|
||||
|
||||
**Values** are JSON plus `bigint` and `Uint8Array` (as in JSON, `-0` becomes `0` and `undefined` properties are dropped). Anything crossing the network is encoded as
|
||||
Extended JSON (`<bigint: 123n>`, `<uint8array: abcd>`); IndexedDB stores them natively.
|
||||
|
||||
**Events** (`EventEmitter`, shared by all nodes of a storage): `change` →
|
||||
`{ changes: { key, value? }[], origin: 'local' | 'remote' }` (no `value` = removed), and
|
||||
`resync` → changes may have been missed, so re-read.
|
||||
|
||||
Node API (defined once): `child`, `as`, `get(key?)`, `getMany`, `set(value)` / `set(key, value)`,
|
||||
`setMany`, `setFields({ a, b })` / `getFields()` (direct children as an object), `delete(key?)`,
|
||||
`deleteMany`, `deleteTree(key?)`, `keys({ limit, after, reverse })`, `entries(...)`. The root has
|
||||
no own value. A node's own value and its children are independent keys.
|
||||
|
||||
- **`keys()`** is a skip-scan: one seek per child that has descendants; siblings without
|
||||
descendants share one iterator. It lists children that only have descendants, and skips
|
||||
reserved segments.
|
||||
- **The only KV "index" is the ordered key space.** There is no `find`: with KV you know where to look,
|
||||
or you list a node's children.
|
||||
- **Implementations:**
|
||||
- `KVStorageInMemory`: `SortedMap` (sorted chunks of ≤1024 keys); values deep-copied.
|
||||
- `KVStorageIndexedDB`: one object store with string keys. Scans read in batches, each in its own
|
||||
transaction, resuming after the last key (IndexedDB transactions close when the code awaits
|
||||
anything else). A write is one readwrite transaction.
|
||||
- `KVStorageHttp` (client) and `KVStorageHttpServer` (wraps any KV storage;
|
||||
`handle(Request) → Response`): `POST /readMany`, `/write`, `/scan` with Extended JSON bodies;
|
||||
`scan` streams NDJSON. Unreachable server (fetch failure, 502/503/504) →
|
||||
`StorageUnavailableError`. The server executes exactly what clients send and never reads values
|
||||
for decisions, so values can be encrypted end to end.
|
||||
- `KVStorageComposite`: see 4.2. `KVStorageEncrypted`: see 4.3.
|
||||
- **CORS.** `allowOrigins` (`['https://app…']` or `'*'`; default none) answers preflights and adds
|
||||
`Access-Control-Allow-Origin` for allowed origins. Allowed request headers: `content-type`,
|
||||
`cache-control` (sent by `SSESession`), `last-event-id` and the auth headers.
|
||||
- **Change stream (SSE).** `POST /events` with `{ paths }` streams every change to those keys and
|
||||
their subtrees as `event: change` (data: the changes), including the subscriber's own writes.
|
||||
`event: ready` follows each connection; keep-alive comments are sent every 15 s. The server keeps
|
||||
no history: nothing is replayed after a reconnect.
|
||||
- `KVStorageHttp.subscribe(path = this.key)` / `unsubscribe(path)` / `close()` manage the
|
||||
subscribed paths. Each change reconnects the stream (vendored `SSESession`, backoff) with the
|
||||
new list; `subscribe` resolves once connected. A 401/403 rejects instead of retrying.
|
||||
- The client emits each pushed batch as `change` (`origin: 'remote'`), and `resync` whenever the
|
||||
stream reconnects after a drop, since changes may have been missed meanwhile.
|
||||
- One SSE connection per `KVStorageHttp` instance. Browsers allow 6 HTTP/1.1 connections per
|
||||
host, so apps should share one client per server (or use HTTP/2); `close()` releases it.
|
||||
- Limitation: a server only sees writes made through its own storage instance, so several server
|
||||
processes over one database need a shared pub/sub (not built).
|
||||
- **Wrappers** (e.g. a future encryption layer) extend `BaseKVStorage` and delegate the primitives.
|
||||
Keys must stay plaintext (or be deterministically encrypted) for prefix scans to work; values can be
|
||||
encrypted freely.
|
||||
|
||||
### 4.1 Authentication (`utils/request-auth.ts`)
|
||||
|
||||
The server requires `auth`: either `{ verifiers, maxSkewMs?, now? }`, or `'none'` (open, for local
|
||||
development and tests). It can't be left open by omission.
|
||||
|
||||
- **Signed requests.** The client's `RequestSigner` signs `${timestamp}:${nonce}:${path}:${body}`
|
||||
(nonce = 16 random bytes as hex, fresh per request; path = URL pathname; body = the exact request
|
||||
body). Based on Harvmaster/sync-server-example, plus the nonce. Headers: `X-Public-Key`
|
||||
(lowercase hex), `X-Timestamp` (ms), `X-Nonce`, `X-Signature` (hex) and `X-Signature-Algorithm`.
|
||||
Every request is signed, and so is each SSE (re)connect, with a fresh timestamp and nonce.
|
||||
- **Verification (401).** The server picks the `SignatureVerifier` registered for the algorithm
|
||||
(unknown → 401), checks the nonce format (16–128 characters of `[0-9A-Za-z_-]`) and that the
|
||||
timestamp is within `maxSkewMs` (default 60 s). It then rejects a nonce the same public key
|
||||
already used within that window (replay protection), and verifies the signature over the
|
||||
received timestamp, nonce, path and raw body. The body is parsed only afterwards.
|
||||
- **Why a nonce.** Ed25519 signatures are deterministic, so without it two identical requests
|
||||
from one key at the same millisecond (two tabs or devices subscribing to the same path, parallel
|
||||
identical reads) would be indistinguishable, and the second rejected as a replay.
|
||||
- **Algorithms are pluggable.** Built in: `Ed25519Signer` / `Ed25519Verifier` (WebCrypto, so no
|
||||
dependencies; Node 22+ and current browsers; keys persist as PKCS #8). Others, e.g. secp256k1 via
|
||||
libauth, implement `RequestSigner` (`algorithm`, `publicKey`, `sign`) and `SignatureVerifier`
|
||||
(`algorithm`, `verify`). The server can accept several algorithms at once.
|
||||
- **Authorization (403): the public key is the user id.** Every key in `readMany` and `write`, and
|
||||
every subscription path, must be `<publicKey>` or under
|
||||
`<publicKey>/`. A scan must stay within `[<publicKey>, <publicKey>0)`, i.e. the key and its
|
||||
subtree (escaping guarantees no other key falls in that range, including lookalikes such as
|
||||
`<publicKey>x`).
|
||||
- **Scoped events.** Each subscriber only receives changes under its own (authorized) paths.
|
||||
- Proxies must preserve the URL path, because the signature covers it.
|
||||
|
||||
### 4.2 `KVStorageComposite({ source, replicas, queue?, prefix?, retryMs? })`
|
||||
|
||||
Keeps local copies of a source (e.g. `KVStorageHttp`) so reads are local and the app works
|
||||
offline. With `prefix` (e.g. the user's public key) only that key and its subtree are synced, cached
|
||||
and subscribed to; keys outside it throw `RangeError`. Without it, the whole source is copied, which
|
||||
is only appropriate when the client may read all of it.
|
||||
|
||||
- **Replicas** are listed fastest first (e.g. `[inMemory, indexedDb]`); the last one seeds the
|
||||
others on start. Reads come from the first.
|
||||
- **The queue** is any KV store (default: in memory). It holds writes the source couldn't take
|
||||
(`writes/<ulid>`) and whether a sync has completed (`synced`). Use a persistent node outside the
|
||||
synced range, e.g. `indexedDb.child('~sync')` with `prefix: userKey`; a queue inside the range is
|
||||
refused, since a sync would delete it.
|
||||
- **Writes** are applied to every replica, then sent to the source, exactly as given.
|
||||
- Applying locally first means the source relaying the write back finds nothing new.
|
||||
- A write the source refuses (e.g. 403) is undone locally and rethrown.
|
||||
- On `StorageUnavailableError`, or while a sync runs or writes are queued (to keep order), the
|
||||
write goes to the queue and is sent later. The last write to reach the source wins.
|
||||
- A queued write the source refuses on replay is dropped (retrying would block every later
|
||||
write), the source's values for its keys are restored locally, and it is reported to `onError`.
|
||||
- **Remote changes** pushed by the source are applied and announced (`change`, `origin: 'remote'`)
|
||||
only where they differ from what the first replica holds. No write tagging or echo tracking is
|
||||
needed: echoes of this client's writes change nothing.
|
||||
- **`sync()`**: replay the queue in order (send, then re-apply locally) → merge-scan the source
|
||||
against the last replica (both key-ordered), applying the difference in batches of 500, except
|
||||
for keys with local writes the source hasn't confirmed yet (in flight or queued; counted per
|
||||
key, and reloaded from a persistent queue on start), which would otherwise briefly revert →
|
||||
replay again (writes made meanwhile) → mark `synced`. It runs on start, on `resync` (the stream
|
||||
reconnected), and `retryMs` after a failure while offline. Called while a sync is running, it
|
||||
schedules one more after it (shared by all callers meanwhile), so changes after the running
|
||||
sync's scan aren't missed; e.g. the sync after subscribing, which closes the gap between the
|
||||
first scan and the stream connecting.
|
||||
- **Reads** wait for the first sync. Until one has completed, they go to the source. With a
|
||||
persistent queue, a restart reads locally straight away (offline too).
|
||||
- `start()` (also lazy on first use), `sync()`, `queued`, `close()`; `onError` receives errors no
|
||||
caller can be rejected with (refused queued writes, failed syncs). Default `console.error`.
|
||||
- Limitations: catching up re-scans the whole range (no server history); a remote change can
|
||||
overwrite a value whose write is still queued until that write is replayed (last write wins);
|
||||
replicas, source and queue must be root stores (the primitives use absolute keys).
|
||||
|
||||
### 4.3 `KVStorageEncrypted(inner, { key })`
|
||||
|
||||
Encrypts values before they reach another KV storage (usually `KVStorageHttp`), so the server only
|
||||
stores ciphertext.
|
||||
|
||||
- AES-GCM (WebCrypto) with a fresh 96-bit random IV per write. Stored value: `Uint8Array`
|
||||
`[1 (format), iv (12), ciphertext + tag]`. The storage key is the additional authenticated data,
|
||||
so a value moved to another key, or altered, fails to decrypt (`Error`, with the cause).
|
||||
- Keys stay plaintext (the server needs them for authorization, ordering and prefix scans); value
|
||||
sizes and timing are visible.
|
||||
- Remote `change` events from the inner storage are decrypted (in order) and re-emitted; `resync`
|
||||
and `subscribe` / `unsubscribe` pass through, so it works as a composite `source`.
|
||||
- Keys: `generateKey()` (extractable), `exportKey` / `importKey` (raw), `deriveKey(password, salt,
|
||||
iterations = 600 000)` (PBKDF2-SHA-256, not extractable). No key rotation yet.
|
||||
|
||||
## 5. Event storage
|
||||
|
||||
`BaseEventStorage` primitives over stream paths: `appendTo(stream, events)` → ids,
|
||||
`readFrom(stream, { after, limit, reverse })`, `streamsOf`. Node API: `append(events)` /
|
||||
`append(key, events)` → ids, `read(opts?)` / `read(key, opts?)` → `{ id, at, data }`, `keys()`.
|
||||
The root is not a stream.
|
||||
|
||||
**Appends never conflict.** Concurrent writers each add their own events, and every reader sees all
|
||||
of them in the same order. There are no versions and no `expectedVersion`; folding events into
|
||||
state is where concurrent changes meet (see event-sourced documents).
|
||||
|
||||
**`EventStorageKV`** (on any KV storage): each event is one plain KV write,
|
||||
`<stream>/~e/<time:15>_<sha256(time + canonical Extended JSON of the event)>` → the event.
|
||||
|
||||
- Keys sort by time, so a stream reads in time order with one scan; `after` is an exclusive id.
|
||||
- Events from different writers in the same millisecond order by hash: arbitrary, but the same for
|
||||
every reader. One instance never reuses a time, so its own appends keep their order.
|
||||
- The id can be recomputed from the content, and re-sending an event (e.g. a retried request)
|
||||
writes the same key, so it can't be duplicated.
|
||||
- An append is one `writeMany`, so it works through `KVStorageComposite` offline and syncs like any
|
||||
other data.
|
||||
|
||||
## 6. Document storage
|
||||
|
||||
`BaseDocumentStorage` primitives over document paths:
|
||||
|
||||
| Primitive | |
|
||||
| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `readMany(paths)` | |
|
||||
| `mutate(path, change)` | maps current state → `DocChange` (or `null` = no-op), applies it, returns the new state; the implementation decides how it's made safe |
|
||||
| `query(parent, DocumentQuery)` | children of `parent`, filtered, sorted and paged |
|
||||
|
||||
Node API: `get(key?)`, `getMany`, `set(doc)` / `set(key, doc)`, `insert(doc)` → ULID-like id,
|
||||
`update(spec, opts?)` / `update(key, spec, opts?)` → new state or `undefined` (`{ upsert }`),
|
||||
`delete(key?)` → existed, `find(filter, { sort, limit, skip })`, `findOne`, `keys()`.
|
||||
|
||||
- **Queries** (`utils/document-query.ts`): `{ field: value }` (dot paths; `null` also matches missing),
|
||||
`$eq $gt $gte $lt $lte $in`. Comparisons are type-bracketed. On arrays a condition matches the array
|
||||
or any one element. Sorting order: missing/null < booleans < numbers < strings < other.
|
||||
- **Updates** (`utils/document-update.ts`): `$set $unset $inc $push $pull $addToSet` (`$each` for
|
||||
push/addToSet). They resolve to a `DocChange` with final values (`$inc` → `set: { n: 6 }`), so
|
||||
changes replay deterministically.
|
||||
- **Indexes:** `{ indexes: { users: ['email', 'age'], 'users/*/posts': ['createdAt'] } }`, passed to
|
||||
the implementation.
|
||||
|
||||
**Implementations**
|
||||
|
||||
- `DocumentStorageInMemory`: maps per collection; equality index per declared field (used for
|
||||
`{ f: v }` / `{ f: { $eq: v } }`), otherwise scans. Writes are atomic in-process.
|
||||
- `DocumentStorageKV`: each document is the KV value at its path; queries scan the collection's
|
||||
children; no indexes; concurrent writes to one document are last-writer-wins.
|
||||
- `EventSourcedDocumentStorage(events, projections)`: each document's `DocChange`s are appended to
|
||||
its own stream. Current state lives in **any** `BaseDocumentStorage` used as the projection store,
|
||||
which serves all reads, queries and indexes.
|
||||
- A write reads the projection, resolves the change (`$inc` → `set: { n: 6 }`), appends it, and
|
||||
applies it to the projection.
|
||||
- Concurrent edits to different fields both survive; the same field ends with the later event's
|
||||
value. (Increments from two devices don't add up: each resolves to a final value.)
|
||||
- Events written elsewhere (another device, arriving through sync) are folded in by
|
||||
`catchUp(key?)`, which rebuilds the projection from the whole stream. It also repairs a
|
||||
projection left behind by a crash between append and projection write.
|
||||
- Extras: `history(key?)`, `getAt(eventId)` / `getAt(key, eventId)`. Children keep these methods
|
||||
in their types through interface declaration merging (types only).
|
||||
|
||||
## 7. Changes from the original brief
|
||||
|
||||
- Drivers + node classes → one abstract class per layer (client and server share an API).
|
||||
- KV indexes (`findBy`, pattern trie, order-preserving value encoding, read-repair) removed.
|
||||
Document indexes are declared as fields and handled by each store; unique indexes and the query
|
||||
planner were removed.
|
||||
- Queries: `$ne $nin $exists $and $or $not` and projections dropped.
|
||||
- Separator fixed to `/`, and every character ≤ `/` is escaped so child keys stay contiguous. This
|
||||
removed the subtree-order workaround in child listing.
|
||||
- Event-sourced documents are events + a projection store instead of a bespoke projection format.
|
||||
- No conditional writes and no versions: KV stores execute exactly what they're given (so a server
|
||||
never needs to read values, which may be encrypted), and events get time + content-hash ids so
|
||||
concurrent appends never collide. Optimistic concurrency (`expectedVersion`) was dropped.
|
||||
- `tree()` removed (use `entries()` for field-per-key reads).
|
||||
|
||||
## 8. Open questions
|
||||
|
||||
- Composite: lazily syncing parts of a large subtree on first read; catching up without a full
|
||||
re-scan (would need server-side history or a change log in event storage).
|
||||
- Event-sourced documents: subscribing to event streams to run `catchUp` automatically.
|
||||
- Authorization beyond "public key = user id" (sharing a subtree with other keys).
|
||||
- Encryption: key rotation (re-encrypt, or a key id in the value format); sharing encrypted data
|
||||
between users.
|
||||
- Multi-process servers need a shared change feed (pub/sub) for SSE.
|
||||
|
||||
- Hard deletes from immutable event logs (privacy requests): compaction vs crypto-shredding. The
|
||||
encryption wrapper could support per-subject keys.
|
||||
- Order-preserving numeric keys (`9` before `10`) as an opt-in key encoding.
|
||||
|
||||
## 9. Benchmark baselines
|
||||
|
||||
`npm run bench`, in-memory stores, Node 22, Linux x64 cloud container (shared CPU, ±10–20%). Mean per operation.
|
||||
|
||||
| Benchmark | Mean |
|
||||
| -------------------------------------------------------------------- | --------------- |
|
||||
| KV `get` / `set` @ 10k keys | 1.5 / 1.7 µs |
|
||||
| KV `get` / `set` @ 100k keys | 2.0 / 2.3 µs |
|
||||
| KV `getMany` × 100 | 25–35 µs |
|
||||
| `keys()` on 1k children × 100 descendants (skip-scan vs full scan) | 2.8 ms vs 34 ms |
|
||||
| Event `append` (one SHA-256 via WebCrypto + one write) | 72 µs |
|
||||
| Event `read` 100 from the middle of 10k | 119 µs |
|
||||
| Event `read` full 10k stream | 11 ms |
|
||||
| Event-sourced document `update ($inc)` | 86 µs |
|
||||
| Documents in-memory @ 10k: `get` / `update` | 1.8 / 6.6 µs |
|
||||
| Documents in-memory: `find` indexed (~100 hits) / scan (~200 of 10k) | 0.20 / 11 ms |
|
||||
| Event-sourced @ 10k: `get` / `update` | 1.6 / 17 µs |
|
||||
| Event-sourced: `find` indexed / scan | 0.38 / 12 ms |
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# @harvmaster/storage
|
||||
|
||||
Layered TypeScript storage that works the same in the browser and on the server:
|
||||
|
||||
- **KV storage**: ordered keys and values, navigated as a tree (`db.child('users').get('1')`).
|
||||
In memory, IndexedDB, or over HTTP, with live updates over Server-Sent Events.
|
||||
- **Event storage**: append-only streams on top of any KV storage.
|
||||
- **Document storage**: JSON documents with queries, updates (`$set`, `$inc`, …) and indexes, on
|
||||
top of any KV storage, or event-sourced.
|
||||
- **Sync**: local replicas that work offline and stay up to date with a server
|
||||
(`KVStorageComposite`), end-to-end encryption (`KVStorageEncrypted`) and signed requests where
|
||||
the user's public key is their id.
|
||||
|
||||
No runtime dependencies. Node 22+ and current browsers. Values are JSON plus `bigint` and
|
||||
`Uint8Array`.
|
||||
|
||||
> **Status: experimental (0.x).** The API may change between minor versions. See
|
||||
> [DESIGN.md](./DESIGN.md) for how everything works and its known limitations.
|
||||
|
||||
## Install
|
||||
|
||||
The package is published to a private registry. Add to your project's `.npmrc`:
|
||||
|
||||
```ini
|
||||
@harvmaster:registry=https://verdaccio.harvmaster.com/
|
||||
```
|
||||
|
||||
```sh
|
||||
npm install @harvmaster/storage # latest release
|
||||
npm install @harvmaster/storage@next # `next` branch
|
||||
npm install @harvmaster/storage@dev-sync # a development branch (`dev-<branch>`)
|
||||
```
|
||||
|
||||
## KV storage
|
||||
|
||||
Every store is a tree of nodes. `child(key)` returns a view rooted deeper (no I/O); methods take
|
||||
an optional key to act on a child instead of the node itself.
|
||||
|
||||
```ts
|
||||
import { KVStorageInMemory } from '@harvmaster/storage'
|
||||
|
||||
const db = new KVStorageInMemory()
|
||||
const users = db.child('users')
|
||||
|
||||
await users.set('1', { name: 'Ada' })
|
||||
await users.child('1').get() // { name: 'Ada' }
|
||||
await users.get('1') // the same
|
||||
await users.setMany([
|
||||
['2', { name: 'Bob' }],
|
||||
['3', { name: 'Cy' }],
|
||||
])
|
||||
await users.getMany(['1', 'missing']) // [{ name: 'Ada' }, undefined]
|
||||
|
||||
for await (const key of users.keys({ limit: 10 })) console.log(key) // '1', '2', '3'
|
||||
for await (const [key, user] of users.entries({ reverse: true })) console.log(key, user)
|
||||
|
||||
await users.delete('3')
|
||||
await users.deleteTree() // users and everything under it
|
||||
```
|
||||
|
||||
A node's own value and its children are independent, so an object can also be stored field per key:
|
||||
|
||||
```ts
|
||||
const settings = db.child('settings')
|
||||
await settings.setFields({ theme: 'dark', fontSize: 14 })
|
||||
await settings.get('theme') // 'dark'
|
||||
await settings.getFields() // { fontSize: 14, theme: 'dark' }
|
||||
```
|
||||
|
||||
Keys sort as strings (`'10'` before `'9'`), and any string or number is a valid key: special
|
||||
characters are escaped. Every write emits a `change` event:
|
||||
|
||||
```ts
|
||||
db.on('change', ({ changes, origin }) => console.log(origin, changes)) // 'local', [{ key: 'users/1', value: … }]
|
||||
```
|
||||
|
||||
### Types
|
||||
|
||||
Describe the tree once to get typed keys and values:
|
||||
|
||||
```ts
|
||||
import { KVStorageInMemory, type Many, type Fields } from '@harvmaster/storage'
|
||||
|
||||
type User = { name: string }
|
||||
type Schema = { children: { users: Many<User>; settings: { children: Fields<{ theme: string; fontSize: number }> } } }
|
||||
|
||||
const typed = new KVStorageInMemory<Schema>()
|
||||
await typed.child('users').set('1', { name: 'Ada' }) // checked
|
||||
const theme = await typed.child('settings').get('theme') // string | undefined
|
||||
```
|
||||
|
||||
### IndexedDB
|
||||
|
||||
```ts
|
||||
import { KVStorageIndexedDB } from '@harvmaster/storage'
|
||||
|
||||
const local = new KVStorageIndexedDB({ name: 'my-app' })
|
||||
```
|
||||
|
||||
## Event storage
|
||||
|
||||
Append-only streams. Every node is a stream (its own events) and a parent of other streams.
|
||||
Appends never conflict: concurrent writers each add their own events, and every reader sees all of
|
||||
them in the same (time) order.
|
||||
|
||||
```ts
|
||||
import { EventStorageKV, KVStorageInMemory } from '@harvmaster/storage'
|
||||
|
||||
const events = new EventStorageKV(new KVStorageInMemory())
|
||||
const orders = events.child('orders')
|
||||
|
||||
const [id] = await orders.append('o1', [{ type: 'created', total: 30 }])
|
||||
await orders.child('o1').append([{ type: 'paid' }])
|
||||
|
||||
for await (const event of orders.read('o1')) console.log(event.id, event.at, event.data)
|
||||
for await (const event of orders.read('o1', { after: id })) console.log(event.data) // { type: 'paid' }
|
||||
for await (const key of orders.keys()) console.log(key) // 'o1'
|
||||
```
|
||||
|
||||
Event ids are `<time>_<hash of the time and content>`, so writing the same event again (e.g. a
|
||||
retried request) can't duplicate it.
|
||||
|
||||
## Document storage
|
||||
|
||||
```ts
|
||||
import { DocumentStorageKV, KVStorageInMemory } from '@harvmaster/storage'
|
||||
|
||||
const docs = new DocumentStorageKV(new KVStorageInMemory())
|
||||
const people = docs.child('people')
|
||||
|
||||
const id = await people.insert({ name: 'Ada', age: 36, tags: ['math'] })
|
||||
await people.set('grace', { name: 'Grace', age: 45, tags: ['navy', 'cobol'] })
|
||||
await people.update(id, { $inc: { age: 1 }, $push: { tags: 'engines' } })
|
||||
|
||||
for await (const { key, data } of people.find({ age: { $gte: 37 } }, { sort: { age: -1 }, limit: 10 })) console.log(key, data)
|
||||
await people.findOne({ tags: 'cobol' }) // { key: 'grace', data: { … } }
|
||||
await people.delete('grace') // true
|
||||
```
|
||||
|
||||
Filters: `{ field: value }` (dot paths; arrays match any element), `$eq $gt $gte $lt $lte $in`.
|
||||
Updates: `$set $unset $inc $push $pull $addToSet` (`$each` for push/addToSet).
|
||||
|
||||
- `DocumentStorageKV` stores each document as one KV value, so it works over any KV storage
|
||||
(including a synced one, below). Queries scan the collection.
|
||||
- `DocumentStorageInMemory({ indexes: { people: ['age'] } })` keeps equality indexes.
|
||||
- `EventSourcedDocumentStorage(events, projections)` records every change as an event (with
|
||||
`history(key)` and `getAt(key, eventId)`), and keeps the current state in any document store.
|
||||
|
||||
## Sync, offline and encryption
|
||||
|
||||
The pieces below combine into an offline-first app whose server only stores ciphertext.
|
||||
|
||||
### Server
|
||||
|
||||
`KVStorageHttpServer` serves any KV storage over HTTP. `handle(request)` takes a Fetch `Request`
|
||||
and returns a `Response`, so it works with `Deno.serve`, `Bun.serve`, Cloudflare Workers, Hono,
|
||||
and so on (in Node, through an adapter such as `@hono/node-server`).
|
||||
|
||||
```ts
|
||||
import { Ed25519Verifier, KVStorageHttpServer, KVStorageInMemory } from '@harvmaster/storage'
|
||||
|
||||
const server = new KVStorageHttpServer(new KVStorageInMemory(), {
|
||||
// Requests must be signed, and a user may only touch keys under their public key.
|
||||
auth: { verifiers: [new Ed25519Verifier()] },
|
||||
basePath: '/kv',
|
||||
allowOrigins: ['https://app.example.com'], // CORS, if the app is on another origin
|
||||
})
|
||||
|
||||
Deno.serve((request) => server.handle(request))
|
||||
```
|
||||
|
||||
`auth: 'none'` turns authentication off (local development only). The server keeps its data in
|
||||
whichever KV storage it wraps, and executes exactly the writes clients send (last write wins).
|
||||
|
||||
### Client
|
||||
|
||||
```ts
|
||||
import { Ed25519Signer, KVStorageComposite, KVStorageEncrypted, KVStorageHttp, KVStorageInMemory, KVStorageIndexedDB } from '@harvmaster/storage'
|
||||
|
||||
// The user's identity: keep the private key (e.g. in IndexedDB or the OS keychain).
|
||||
const signer = await Ed25519Signer.generate() // or Ed25519Signer.fromPrivateKey(saved)
|
||||
const saved = await signer.exportPrivateKey()
|
||||
|
||||
// The encryption key: generate one and store it like the private key, or derive it from a password
|
||||
// with a random salt created once and stored with the user's data (the salt isn't secret).
|
||||
const salt = crypto.getRandomValues(new Uint8Array(16))
|
||||
const key = await KVStorageEncrypted.deriveKey('correct horse battery staple', salt)
|
||||
|
||||
const indexedDb = new KVStorageIndexedDB({ name: 'my-app' })
|
||||
const db = new KVStorageComposite({
|
||||
source: new KVStorageEncrypted(new KVStorageHttp({ url: 'https://api.example.com/kv', signer }), { key }),
|
||||
replicas: [new KVStorageInMemory(), indexedDb], // fastest first; the last one persists
|
||||
queue: indexedDb.child('~sync'), // offline writes, kept across reloads (outside `prefix`)
|
||||
prefix: signer.publicKey, // only this user's data is synced
|
||||
})
|
||||
|
||||
const me = db.child(signer.publicKey)
|
||||
await me.child('notes').set('1', { text: 'Works offline' }) // local immediately, then sent
|
||||
await me.child('notes').get('1') // read locally
|
||||
|
||||
db.on('change', ({ changes, origin }) => {
|
||||
if (origin === 'remote') render(changes) // made on another device
|
||||
})
|
||||
```
|
||||
|
||||
What the composite does:
|
||||
|
||||
- **Reads** come from the first replica once the first sync has completed; after a reload, straight
|
||||
from IndexedDB, even offline.
|
||||
- **Writes** go to every replica, then to the server. If the server is unreachable, they're queued
|
||||
and replayed in order later; `db.queued` is the number waiting. A write the server refuses (e.g. 403) is undone locally and rejected.
|
||||
- **Changes from other devices** arrive over SSE and are applied, and announced as `change`
|
||||
events, only where they differ from local values.
|
||||
- **`db.sync()`** replays queued writes and catches up with the server. It runs on start, whenever
|
||||
the live connection is restored, and periodically while offline.
|
||||
- `onError` receives errors with no caller to reject, e.g. a queued write the server later refused
|
||||
(it's dropped, and the server's value restored).
|
||||
- `db.close()` stops syncing and closes the live connection.
|
||||
|
||||
Documents and events work on top of the composite like any other KV storage:
|
||||
|
||||
```ts
|
||||
type Todo = { title: string; done: boolean }
|
||||
const todos = new DocumentStorageKV<{ children: { todos: Many<Todo> } }>(me).child('todos')
|
||||
await todos.insert({ title: 'Ship it', done: false })
|
||||
for await (const { data } of todos.find({ done: false })) console.log(data.title)
|
||||
```
|
||||
|
||||
### Encryption
|
||||
|
||||
`KVStorageEncrypted` wraps any KV storage and AES-GCM encrypts each value with a fresh IV; the
|
||||
key a value is stored under is authenticated too, so the server can't move values around.
|
||||
|
||||
- **Keys are not encrypted**: the server needs them for authorization and ordering. Don't put
|
||||
secrets in keys. Value sizes and write times are visible as well.
|
||||
- `generateKey()`, `exportKey(key)` / `importKey(raw)`, or `deriveKey(password, salt)` (PBKDF2;
|
||||
store the random salt alongside the data, it isn't secret).
|
||||
- Losing the key means losing the data. There is no recovery.
|
||||
|
||||
### Signed requests
|
||||
|
||||
Every request is signed with the user's key: `X-Public-Key`, `X-Timestamp`, `X-Nonce` (random
|
||||
per request), `X-Signature` and `X-Signature-Algorithm` headers, over
|
||||
`${timestamp}:${nonce}:${path}:${body}`. The server checks the signature, a 60 s timestamp
|
||||
window and that the nonce wasn't used before (replay protection), then that every key is the
|
||||
public key or under it. Ed25519 is built in (WebCrypto); other algorithms plug in by implementing
|
||||
`RequestSigner` / `SignatureVerifier`.
|
||||
|
||||
### Live updates without the composite
|
||||
|
||||
```ts
|
||||
const http = new KVStorageHttp({ url: 'https://api.example.com/kv', signer })
|
||||
http.on('change', ({ changes }) => console.log(changes))
|
||||
http.on('resync', () => reload()) // reconnected: changes may have been missed
|
||||
await http.child(signer.publicKey).child('notes').subscribe()
|
||||
```
|
||||
|
||||
Each `KVStorageHttp` instance keeps one SSE connection for all its subscriptions. Browsers allow
|
||||
only 6 HTTP/1.1 connections per host, so share one client per server (or serve over HTTP/2).
|
||||
|
||||
## Writing your own storage
|
||||
|
||||
Implement the primitives of the base class for a layer; everything else is inherited. For KV:
|
||||
|
||||
```ts
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from '@harvmaster/storage'
|
||||
|
||||
class MyKVStorage extends BaseKVStorage {
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
/* values in order, undefined for missing */
|
||||
}
|
||||
async *scan(options?: ScanOptions): AsyncGenerator<[string, unknown]> {
|
||||
/* key-ordered entries within { gte, lt }, honouring limit/reverse/values */
|
||||
}
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
/* set each key to its value, or delete it when value is undefined */
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The repository's conformance suite (`test/kv-storage/kv-storage.conformance.ts`) checks the
|
||||
behaviour every KV storage must have.
|
||||
|
||||
## Development
|
||||
|
||||
```sh
|
||||
npm test # typecheck + Node tests
|
||||
npm run test:browser # Chromium tests (real IndexedDB, WebCrypto, CORS, SSE)
|
||||
npm run bench
|
||||
npm run format
|
||||
```
|
||||
|
||||
CI runs formatting, typechecking, Node 22/24/26, Chromium and the build on every push, and then
|
||||
publishes: `main` → `latest`, `next` → `next`, other branches → `dev-<branch>`.
|
||||
@@ -0,0 +1,33 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { DocumentStorageInMemory } from '../src/document-storage/document-storage-in-memory.js'
|
||||
import { EventSourcedDocumentStorage } from '../src/document-storage/event-sourced-document-storage.js'
|
||||
import { EventStorageKV } from '../src/event-storage/event-storage-kv.js'
|
||||
import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js'
|
||||
import type { BaseDocumentStorage } from '../src/document-storage/base-document-storage.js'
|
||||
|
||||
const N = 10_000
|
||||
const stores: [string, BaseDocumentStorage][] = [
|
||||
['in-memory', new DocumentStorageInMemory({ indexes: { users: ['age'] } })],
|
||||
['event-sourced', new EventSourcedDocumentStorage(new EventStorageKV(new KVStorageInMemory()), new DocumentStorageInMemory({ indexes: { users: ['age'] } }))],
|
||||
]
|
||||
for (const [name, docs] of stores) {
|
||||
const users = docs.child('users')
|
||||
for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, age: i % 100, city: `c${i % 50}` })
|
||||
let i = 0
|
||||
describe(`documents (${name}) @ 10k`, () => {
|
||||
bench('get', async () => {
|
||||
await users.get(`u${i++ % N}`)
|
||||
})
|
||||
bench('update ($inc)', async () => {
|
||||
await users.update(`u${i++ % N}`, { $inc: { logins: 1 } })
|
||||
})
|
||||
bench('find by indexed field (~100 hits)', async () => {
|
||||
for await (const _ of users.find({ age: i++ % 100 })) {
|
||||
}
|
||||
})
|
||||
bench('find by unindexed field (scan, ~200 hits)', async () => {
|
||||
for await (const _ of users.find({ city: `c${i++ % 50}` })) {
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { InMemoryKVDriver } from '../src/kv/memory.js'
|
||||
import { KVStorage } from '../src/kv/node.js'
|
||||
import { KVEventStorage } from '../src/events/kv.js'
|
||||
import { EventSourcedDocumentStorage } from '../src/documents/event-sourced.js'
|
||||
|
||||
const N = 10_000
|
||||
const docs = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), {
|
||||
indexes: { users: [{ fields: ['age'] }, { fields: ['email'], unique: true }] },
|
||||
warnOnFullScan: false,
|
||||
})
|
||||
const users = docs.child('users')
|
||||
for (let i = 0; i < N; i++) await users.set(`u${i}`, { name: `user ${i}`, email: `u${i}@x.io`, age: i % 100, city: `c${i % 50}` })
|
||||
let i = 0
|
||||
|
||||
describe(`documents @ ${N / 1000}k docs`, () => {
|
||||
bench('get', async () => {
|
||||
await users.get(`u${i++ % N}`)
|
||||
})
|
||||
bench('update ($inc, indexed node)', async () => {
|
||||
await users.update(`u${i++ % N}`, { $inc: { logins: 1 } })
|
||||
})
|
||||
bench('findOne by unique index (email)', async () => {
|
||||
await users.findOne({ email: `u${i++ % N}@x.io` })
|
||||
})
|
||||
bench('find indexed (age = k, ~100 hits)', async () => {
|
||||
for await (const _ of users.find({ age: i++ % 100 })) {
|
||||
}
|
||||
})
|
||||
bench('find full scan (city = k, ~200 hits)', async () => {
|
||||
for await (const _ of users.find({ city: `c${i++ % 50}` })) {
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,26 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { EventStorageKV } from '../src/event-storage/event-storage-kv.js'
|
||||
import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js'
|
||||
|
||||
const events = new EventStorageKV(new KVStorageInMemory())
|
||||
const streams = events.child('streams')
|
||||
const ids = await streams.append(
|
||||
'long',
|
||||
Array.from({ length: 10_000 }, (_, i) => ({ i })),
|
||||
)
|
||||
const middle = ids[5000]
|
||||
let s = 0
|
||||
|
||||
describe('events', () => {
|
||||
bench('append 1 event', async () => {
|
||||
await streams.append(`s${s++ % 1000}`, [{ type: 'e' }])
|
||||
})
|
||||
bench('read 100 events from the middle of 10k', async () => {
|
||||
for await (const _ of streams.read('long', { after: middle, limit: 100 })) {
|
||||
}
|
||||
})
|
||||
bench('read full 10k-event stream', async () => {
|
||||
for await (const _ of streams.read('long')) {
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,27 +0,0 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { InMemoryKVDriver } from '../src/kv/memory.js'
|
||||
import { KVStorage } from '../src/kv/node.js'
|
||||
import { KVEventStorage } from '../src/events/kv.js'
|
||||
|
||||
const events = new KVEventStorage(new KVStorage(new InMemoryKVDriver()))
|
||||
const streams = events.child('streams')
|
||||
const long = streams.child('long')
|
||||
for (let i = 0; i < 10_000; i++) await long.append([{ type: 'tick', i }])
|
||||
let s = 0
|
||||
|
||||
describe('events', () => {
|
||||
bench('append 1 event (warm head cache)', async () => {
|
||||
await streams.append(`s${s++ % 1000}`, [{ type: 'e' }])
|
||||
})
|
||||
bench('read 100 events from version 5000 of 10k', async () => {
|
||||
for await (const _ of long.read({ from: 5000, limit: 100 })) {
|
||||
}
|
||||
})
|
||||
bench('read full 10k-event stream', async () => {
|
||||
for await (const _ of long.read()) {
|
||||
}
|
||||
})
|
||||
bench('version() (cold head lookup)', async () => {
|
||||
await long.version()
|
||||
})
|
||||
})
|
||||
@@ -1,26 +0,0 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { InMemoryKVDriver } from '../src/kv/memory.js'
|
||||
import { KVStorage } from '../src/kv/node.js'
|
||||
|
||||
// 1k children, each with 100 descendants (100k keys under the node).
|
||||
const db = new KVStorage(new InMemoryKVDriver({ clone: false }))
|
||||
const node = db.child('n')
|
||||
const writes: [string, number][] = []
|
||||
for (let c = 0; c < 1000; c++) {
|
||||
writes.push([`n/c${c}`, c])
|
||||
for (let d = 0; d < 100; d++) writes.push([`n/c${c}/d${d}`, d])
|
||||
}
|
||||
await db.driver.setMany(writes)
|
||||
|
||||
describe('keys() on 1k children x 100 descendants', () => {
|
||||
bench('keys() skip-scan', async () => {
|
||||
let n = 0
|
||||
for await (const _ of node.keys()) n++
|
||||
if (n !== 1000) throw new Error(`expected 1000, got ${n}`)
|
||||
})
|
||||
bench('naive full subtree scan (reference)', async () => {
|
||||
let n = 0
|
||||
for await (const _ of db.driver.scan({ gte: 'n/', lt: 'n0' })) n++
|
||||
if (n !== 101_000) throw new Error(`expected 101000, got ${n}`)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import type { KVChange } from '../src/kv-storage/base-kv-storage.js'
|
||||
import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js'
|
||||
|
||||
for (const n of [10_000, 100_000]) {
|
||||
const db = new KVStorageInMemory()
|
||||
await db.writeMany(Array.from({ length: n }, (_, i) => ({ key: `users/u${i}`, value: { name: `user ${i}`, n: i } })))
|
||||
const users = db.child('users')
|
||||
const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`)
|
||||
let i = 0
|
||||
describe(`KV @ ${n / 1000}k keys`, () => {
|
||||
bench('get', async () => {
|
||||
await users.get(`u${i++ % n}`)
|
||||
})
|
||||
bench('set', async () => {
|
||||
await users.set(`u${i++ % n}`, { name: 'x', n: i })
|
||||
})
|
||||
bench('getMany x100', async () => {
|
||||
await users.getMany(keys)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const tree = new KVStorageInMemory()
|
||||
const writes: KVChange[] = []
|
||||
for (let c = 0; c < 1000; c++) {
|
||||
writes.push({ key: `n/c${c}`, value: c })
|
||||
for (let d = 0; d < 100; d++) writes.push({ key: `n/c${c}/d${d}`, value: d })
|
||||
}
|
||||
await tree.writeMany(writes)
|
||||
|
||||
describe('keys() on 1k children x 100 descendants', () => {
|
||||
bench('keys() skip-scan', async () => {
|
||||
for await (const _ of tree.child('n').keys()) {
|
||||
}
|
||||
})
|
||||
bench('full subtree scan (reference)', async () => {
|
||||
for await (const _ of tree.scan({ gte: 'n/', lt: 'n0' })) {
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,29 +0,0 @@
|
||||
import { bench, describe } from 'vitest'
|
||||
import { InMemoryKVDriver } from '../src/kv/memory.js'
|
||||
import { KVStorage } from '../src/kv/node.js'
|
||||
|
||||
async function populated(n: number) {
|
||||
const db = new KVStorage(new InMemoryKVDriver())
|
||||
const users = db.child('users')
|
||||
const batch: [string, { name: string; n: number }][] = []
|
||||
for (let i = 0; i < n; i++) batch.push([`u${i}`, { name: `user ${i}`, n: i }])
|
||||
await users.setMany(batch)
|
||||
return users
|
||||
}
|
||||
|
||||
for (const n of [10_000, 100_000]) {
|
||||
const users = await populated(n)
|
||||
const keys = Array.from({ length: 100 }, (_, i) => `u${(i * 7919) % n}`)
|
||||
let i = 0
|
||||
describe(`KV @ ${n / 1000}k keys`, () => {
|
||||
bench('get', async () => {
|
||||
await users.get(`u${i++ % n}`)
|
||||
})
|
||||
bench('set (overwrite)', async () => {
|
||||
await users.set(`u${i++ % n}`, { name: 'x', n: i })
|
||||
})
|
||||
bench('getMany x100', async () => {
|
||||
await users.getMany(keys)
|
||||
})
|
||||
})
|
||||
}
|
||||
Generated
+327
-10
@@ -1,29 +1,66 @@
|
||||
{
|
||||
"name": "potential-storage",
|
||||
"name": "@harvmaster/storage",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "potential-storage",
|
||||
"name": "@harvmaster/storage",
|
||||
"version": "0.1.0",
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.20.5",
|
||||
"@vitest/browser": "^3.2.7",
|
||||
"fake-indexeddb": "^6.2.6",
|
||||
"fast-check": "^4.10.2",
|
||||
"playwright": "^1.56.1",
|
||||
"prettier": "^3.9.10",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^3.2.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"vitest": ">=3"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"vitest": {
|
||||
"optional": true
|
||||
"node": ">=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz",
|
||||
"integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-validator-identifier": "^7.29.7",
|
||||
"js-tokens": "^4.0.0",
|
||||
"picocolors": "^1.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame/node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@babel/helper-validator-identifier": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz",
|
||||
"integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/runtime": {
|
||||
"version": "7.29.10",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.10.tgz",
|
||||
"integrity": "sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
@@ -492,6 +529,13 @@
|
||||
"node": "^22.20 || ^24.12 || >=25"
|
||||
}
|
||||
},
|
||||
"node_modules/@polka/url": {
|
||||
"version": "1.0.0-next.29",
|
||||
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
|
||||
"integrity": "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm-eabi": {
|
||||
"version": "4.65.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.65.0.tgz",
|
||||
@@ -842,6 +886,47 @@
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@testing-library/dom": {
|
||||
"version": "10.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.2.tgz",
|
||||
"integrity": "sha512-yzr2S9HyAIdhz2/6qHgbs665Q7PKVcDF05vsOlHPxG1mo36gKVesdYVeDLnXgfjJ03CrKRk08knc6+E/9m8v2Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.10.4",
|
||||
"@babel/runtime": "^7.12.5",
|
||||
"@types/aria-query": "^5.0.1",
|
||||
"aria-query": "5.3.0",
|
||||
"dom-accessibility-api": "^0.5.9",
|
||||
"lz-string": "^1.5.0",
|
||||
"picocolors": "1.1.1",
|
||||
"pretty-format": "^27.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@testing-library/user-event": {
|
||||
"version": "14.6.8",
|
||||
"resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.8.tgz",
|
||||
"integrity": "sha512-vgSSqnANMnkcYR/vUNDbwS7aM3OErRLNUG8vCVVlj4oc6cKffUrccX7UAS5AmJQQNZTSibi9z5GvvRZ0aYgUCQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12",
|
||||
"npm": ">=6"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@testing-library/dom": ">=7.21.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/aria-query": {
|
||||
"version": "5.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz",
|
||||
"integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/chai": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
|
||||
@@ -877,6 +962,42 @@
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/browser": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/browser/-/browser-3.2.7.tgz",
|
||||
"integrity": "sha512-gIzazUkbQfv6T1rJHOLhMMKQnplKAvvQ7QNGaFwI6oCsp4z2aSDZCojGpX3QX3+MYsvJdyy/8BRIYVEbAkMkEA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/user-event": "^14.6.1",
|
||||
"@vitest/mocker": "3.2.7",
|
||||
"@vitest/utils": "3.2.7",
|
||||
"magic-string": "^0.30.17",
|
||||
"sirv": "^3.0.1",
|
||||
"tinyrainbow": "^2.0.0",
|
||||
"ws": "^8.18.2"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/vitest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright": "*",
|
||||
"vitest": "3.2.7",
|
||||
"webdriverio": "^7.0.0 || ^8.0.0 || ^9.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"playwright": {
|
||||
"optional": true
|
||||
},
|
||||
"safaridriver": {
|
||||
"optional": true
|
||||
},
|
||||
"webdriverio": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/expect": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz",
|
||||
@@ -992,6 +1113,39 @@
|
||||
"url": "https://opencollective.com/vitest"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-regex": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
|
||||
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
|
||||
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/aria-query": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
|
||||
"integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"dequal": "^2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/assertion-error": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
|
||||
@@ -1067,6 +1221,23 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/dom-accessibility-api": {
|
||||
"version": "0.5.16",
|
||||
"resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz",
|
||||
"integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/es-module-lexer": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz",
|
||||
@@ -1136,6 +1307,16 @@
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fake-indexeddb": {
|
||||
"version": "6.2.6",
|
||||
"resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.6.tgz",
|
||||
"integrity": "sha512-ihhmr0KoET51jT8R4YhbUYuLLvr1qqO6INISygtN16i8loR8kAxtAUr4+EKHTSMjL+lggaAn4OC7DRqzrNgnuQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-check": {
|
||||
"version": "4.10.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.10.2.tgz",
|
||||
@@ -1206,6 +1387,16 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lz-string": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz",
|
||||
"integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"lz-string": "bin/bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "0.30.21",
|
||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
||||
@@ -1216,6 +1407,16 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/mrmime": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz",
|
||||
"integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
@@ -1279,6 +1480,53 @@
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.56.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz",
|
||||
"integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.56.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.56.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz",
|
||||
"integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright/node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.31",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.31.tgz",
|
||||
@@ -1324,6 +1572,21 @@
|
||||
"url": "https://github.com/prettier/prettier?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/pretty-format": {
|
||||
"version": "27.5.1",
|
||||
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz",
|
||||
"integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-regex": "^5.0.1",
|
||||
"ansi-styles": "^5.0.0",
|
||||
"react-is": "^17.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pure-rand": {
|
||||
"version": "8.4.2",
|
||||
"resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz",
|
||||
@@ -1341,6 +1604,13 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-is": {
|
||||
"version": "17.0.2",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
|
||||
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.65.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.65.0.tgz",
|
||||
@@ -1394,6 +1664,21 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/sirv": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz",
|
||||
"integrity": "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@polka/url": "^1.0.0-next.24",
|
||||
"mrmime": "^2.0.0",
|
||||
"totalist": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz",
|
||||
@@ -1492,6 +1777,16 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/totalist": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz",
|
||||
"integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
@@ -1700,6 +1995,28 @@
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.22.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz",
|
||||
"integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
-31
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"name": "potential-storage",
|
||||
"name": "@harvmaster/storage",
|
||||
"version": "0.1.0",
|
||||
"description": "Layered TypeScript storage: KV -> Events -> Documents",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
"node": ">=22"
|
||||
},
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
@@ -12,54 +12,34 @@
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
},
|
||||
"./kv": {
|
||||
"types": "./dist/kv/index.d.ts",
|
||||
"import": "./dist/kv/index.js"
|
||||
},
|
||||
"./events": {
|
||||
"types": "./dist/events/index.d.ts",
|
||||
"import": "./dist/events/index.js"
|
||||
},
|
||||
"./documents": {
|
||||
"types": "./dist/documents/index.d.ts",
|
||||
"import": "./dist/documents/index.js"
|
||||
},
|
||||
"./http": {
|
||||
"types": "./dist/http/index.d.ts",
|
||||
"import": "./dist/http/index.js"
|
||||
},
|
||||
"./testing": {
|
||||
"types": "./dist/testing/index.d.ts",
|
||||
"import": "./dist/testing/index.js"
|
||||
}
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://verdaccio.harvmaster.com/"
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"prepack": "npm run build",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "tsc --noEmit && vitest run",
|
||||
"test:watch": "vitest",
|
||||
"bench": "vitest bench --run",
|
||||
"test:only": "vitest run",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
"format:check": "prettier --check .",
|
||||
"test:browser": "vitest run --config vitest.browser.config.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.20.5",
|
||||
"@vitest/browser": "^3.2.7",
|
||||
"fake-indexeddb": "^6.2.6",
|
||||
"fast-check": "^4.10.2",
|
||||
"playwright": "^1.56.1",
|
||||
"prettier": "^3.9.10",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^3.2.7"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"vitest": ">=3"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"vitest": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// Publishes the package for a branch: `main` → `latest`, `next` → `next`, any other branch →
|
||||
// `dev-<branch>`. Usage: node scripts/publish.mjs <branch> [--dry-run]
|
||||
//
|
||||
// `main` publishes the version in package.json as is, and skips if it is already published (bump
|
||||
// the version to release). Other branches publish a prerelease, `<version>-<tag>.<timestamp>`,
|
||||
// so every push gets its own version. The registry and its token come from `.npmrc` / npm config.
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
const [branch, ...flags] = process.argv.slice(2)
|
||||
if (!branch) throw new Error('Usage: node scripts/publish.mjs <branch> [--dry-run]')
|
||||
const dryRun = flags.includes('--dry-run')
|
||||
|
||||
const npm = (...args) => execFileSync('npm', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] }).trim()
|
||||
|
||||
const pkg = JSON.parse(readFileSync('package.json', 'utf8'))
|
||||
const base = pkg.version.replace(/\+.*/, '')
|
||||
const tag = branch === 'main' ? 'latest' : branch === 'next' ? 'next' : `dev-${branch.replace(/[^0-9A-Za-z-]/g, '-')}`
|
||||
|
||||
if (tag === 'latest') {
|
||||
let published = ''
|
||||
try {
|
||||
published = npm('view', `${pkg.name}@${base}`, 'version')
|
||||
} catch {
|
||||
// Not found (or the package doesn't exist yet): publish it.
|
||||
}
|
||||
if (published) {
|
||||
console.log(`${pkg.name}@${base} is already published; bump the version in package.json to release.`)
|
||||
process.exit(0)
|
||||
}
|
||||
} else {
|
||||
const timestamp = new Date().toISOString().replace(/\D/g, '')
|
||||
pkg.version = `${base}${base.includes('-') ? '.' : '-'}${tag}.${timestamp}`
|
||||
writeFileSync('package.json', `${JSON.stringify(pkg, null, 2)}\n`)
|
||||
}
|
||||
|
||||
console.log(`Publishing ${pkg.name}@${pkg.version} with tag "${tag}"`)
|
||||
execFileSync('npm', ['publish', '--tag', tag, ...(dryRun ? ['--dry-run'] : [])], { stdio: 'inherit' })
|
||||
@@ -1,176 +0,0 @@
|
||||
import { DEFAULT_SEPARATOR, succ } from './path.js'
|
||||
|
||||
export type IndexValue = string | number | boolean | null
|
||||
|
||||
/**
|
||||
* Order-preserving, separator-safe encoding of index values.
|
||||
*
|
||||
* Every character the encoder emits sorts strictly above the separator. That makes encodings
|
||||
* prefix-safe inside keys: `enc(a) + sep + …` sorts below `enc(b)` whenever `enc(a)` is a proper
|
||||
* prefix of `enc(b)`, so `a < b ⇔ key(a, …) < key(b, …)` and range scans over index entries are exact.
|
||||
*
|
||||
* Layout: one type-tag character, then a body.
|
||||
* - Type tags order `null < false < true < number < string`.
|
||||
* - Numbers: IEEE-754 float64, big-endian; positive values have the sign bit flipped, negative
|
||||
* values have all bits flipped; written as 16 fixed-width "hex" digits. `-0` is normalised to `0`;
|
||||
* `NaN` is rejected.
|
||||
* - Strings: each code unit `c` with `c <= E` (where `E = succ(sep)`) is written as `E` followed by
|
||||
* `chr(E + 1 + c)`; every other code unit is written as-is. This is monotone, so order is kept.
|
||||
*
|
||||
* The 16 digits are `0-9a-f` when the separator sorts below `'0'` (e.g. `/`), otherwise the 16
|
||||
* characters immediately after the separator.
|
||||
*/
|
||||
export interface IndexValueCodec {
|
||||
readonly sep: string
|
||||
encode(value: IndexValue): string
|
||||
decode(encoded: string): IndexValue
|
||||
}
|
||||
|
||||
const codecs = new Map<string, IndexValueCodec>()
|
||||
|
||||
export function indexValueCodec(sep: string = DEFAULT_SEPARATOR): IndexValueCodec {
|
||||
let codec = codecs.get(sep)
|
||||
if (!codec) {
|
||||
codec = makeCodec(sep)
|
||||
codecs.set(sep, codec)
|
||||
}
|
||||
return codec
|
||||
}
|
||||
|
||||
export function encodeIndexValue(value: IndexValue, sep: string = DEFAULT_SEPARATOR): string {
|
||||
return indexValueCodec(sep).encode(value)
|
||||
}
|
||||
|
||||
export function decodeIndexValue(encoded: string, sep: string = DEFAULT_SEPARATOR): IndexValue {
|
||||
return indexValueCodec(sep).decode(encoded)
|
||||
}
|
||||
|
||||
/** Encodes a compound value as `enc(v1) + sep + enc(v2) + …`. */
|
||||
export function encodeIndexValues(values: readonly IndexValue[], sep: string = DEFAULT_SEPARATOR): string {
|
||||
const codec = indexValueCodec(sep)
|
||||
let out = ''
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
if (i > 0) out += sep
|
||||
out += codec.encode(values[i]!)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function isIndexValue(v: unknown): v is IndexValue {
|
||||
return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v))
|
||||
}
|
||||
|
||||
function makeCodec(sep: string): IndexValueCodec {
|
||||
const sepCode = sep.charCodeAt(0)
|
||||
const digits = sepCode < 0x30 ? '0123456789abcdef' : Array.from({ length: 16 }, (_, i) => String.fromCharCode(sepCode + 1 + i)).join('')
|
||||
const digitValue = new Map<string, number>([...digits].map((d, i) => [d, i]))
|
||||
const TAG_NULL = digits[0]!
|
||||
const TAG_FALSE = digits[1]!
|
||||
const TAG_TRUE = digits[2]!
|
||||
const TAG_NUMBER = digits[3]!
|
||||
const TAG_STRING = digits[4]!
|
||||
const esc = succ(sep)
|
||||
const escCode = esc.charCodeAt(0)
|
||||
|
||||
const view = new DataView(new ArrayBuffer(8))
|
||||
|
||||
function hex32(n: number): string {
|
||||
let out = ''
|
||||
for (let shift = 28; shift >= 0; shift -= 4) out += digits[(n >>> shift) & 0xf]
|
||||
return out
|
||||
}
|
||||
|
||||
function encodeNumber(n: number): string {
|
||||
if (Number.isNaN(n)) throw new TypeError('NaN cannot be indexed')
|
||||
view.setFloat64(0, n === 0 ? 0 : n) // normalise -0
|
||||
let hi = view.getUint32(0)
|
||||
let lo = view.getUint32(4)
|
||||
if (hi & 0x80000000) {
|
||||
hi = ~hi >>> 0
|
||||
lo = ~lo >>> 0
|
||||
} else {
|
||||
hi = (hi ^ 0x80000000) >>> 0
|
||||
}
|
||||
return hex32(hi) + hex32(lo)
|
||||
}
|
||||
|
||||
function decodeNumber(body: string): number {
|
||||
if (body.length !== 16) throw new TypeError(`Bad encoded number ${JSON.stringify(body)}`)
|
||||
let hi = 0
|
||||
let lo = 0
|
||||
for (let i = 0; i < 8; i++) hi = (hi << 4) | digitValue.get(body[i]!)!
|
||||
for (let i = 8; i < 16; i++) lo = (lo << 4) | digitValue.get(body[i]!)!
|
||||
hi >>>= 0
|
||||
lo >>>= 0
|
||||
if (hi & 0x80000000) {
|
||||
hi = (hi ^ 0x80000000) >>> 0
|
||||
} else {
|
||||
hi = ~hi >>> 0
|
||||
lo = ~lo >>> 0
|
||||
}
|
||||
view.setUint32(0, hi)
|
||||
view.setUint32(4, lo)
|
||||
return view.getFloat64(0)
|
||||
}
|
||||
|
||||
function encodeString(s: string): string {
|
||||
let out = ''
|
||||
let start = 0
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
const c = s.charCodeAt(i)
|
||||
if (c <= escCode) {
|
||||
out += s.slice(start, i) + esc + String.fromCharCode(escCode + 1 + c)
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
return start === 0 ? s : out + s.slice(start)
|
||||
}
|
||||
|
||||
function decodeString(body: string): string {
|
||||
if (body.indexOf(esc) === -1) return body
|
||||
let out = ''
|
||||
for (let i = 0; i < body.length; i++) {
|
||||
const c = body[i]!
|
||||
if (c === esc) {
|
||||
const next = body.charCodeAt(++i)
|
||||
out += String.fromCharCode(next - escCode - 1)
|
||||
} else out += c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
return {
|
||||
sep,
|
||||
encode(value) {
|
||||
if (value === null) return TAG_NULL
|
||||
switch (typeof value) {
|
||||
case 'boolean':
|
||||
return value ? TAG_TRUE : TAG_FALSE
|
||||
case 'number':
|
||||
return TAG_NUMBER + encodeNumber(value)
|
||||
case 'string':
|
||||
return TAG_STRING + encodeString(value)
|
||||
default:
|
||||
throw new TypeError(`Cannot index value of type ${typeof value}`)
|
||||
}
|
||||
},
|
||||
decode(encoded) {
|
||||
const tag = encoded[0]
|
||||
const body = encoded.slice(1)
|
||||
switch (tag) {
|
||||
case TAG_NULL:
|
||||
return null
|
||||
case TAG_FALSE:
|
||||
return false
|
||||
case TAG_TRUE:
|
||||
return true
|
||||
case TAG_NUMBER:
|
||||
return decodeNumber(body)
|
||||
case TAG_STRING:
|
||||
return decodeString(body)
|
||||
default:
|
||||
throw new TypeError(`Bad encoded index value ${JSON.stringify(encoded)}`)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
/** Minimal bounded LRU over `Map` insertion order. */
|
||||
export class LRU<K, V> {
|
||||
readonly #map = new Map<K, V>()
|
||||
|
||||
constructor(readonly capacity: number) {}
|
||||
|
||||
get(key: K): V | undefined {
|
||||
const v = this.#map.get(key)
|
||||
if (v !== undefined) {
|
||||
this.#map.delete(key)
|
||||
this.#map.set(key, v)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
set(key: K, value: V): void {
|
||||
if (this.capacity <= 0) return
|
||||
this.#map.delete(key)
|
||||
this.#map.set(key, value)
|
||||
if (this.#map.size > this.capacity) this.#map.delete(this.#map.keys().next().value as K)
|
||||
}
|
||||
|
||||
delete(key: K): void {
|
||||
this.#map.delete(key)
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.#map.size
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
/** Per-key in-process mutex: calls with the same key run one at a time, in order. */
|
||||
export class KeyedMutex {
|
||||
readonly #tails = new Map<string, Promise<unknown>>()
|
||||
|
||||
run<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
const prev = this.#tails.get(key) ?? Promise.resolve()
|
||||
const result = prev.then(fn, fn)
|
||||
const tail = result.catch(() => {})
|
||||
this.#tails.set(key, tail)
|
||||
void tail.then(() => {
|
||||
if (this.#tails.get(key) === tail) this.#tails.delete(key)
|
||||
})
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
import { InvalidKeyError } from '../errors.js'
|
||||
|
||||
export const DEFAULT_SEPARATOR = '/'
|
||||
|
||||
/** Reserved segments. User segments can never start with `~` (a leading `~` is escaped). */
|
||||
export const RESERVED = {
|
||||
index: '~i',
|
||||
unique: '~u',
|
||||
events: '~e',
|
||||
projections: '~p',
|
||||
} as const
|
||||
|
||||
export type Key = string | number
|
||||
|
||||
/** The character immediately after `c` (by UTF-16 code unit). */
|
||||
export function succ(c: string): string {
|
||||
return String.fromCharCode(c.charCodeAt(0) + 1)
|
||||
}
|
||||
|
||||
export function validateSeparator(sep: string): void {
|
||||
if (sep.length !== 1) throw new TypeError(`Separator must be a single character, got ${JSON.stringify(sep)}`)
|
||||
const code = sep.charCodeAt(0)
|
||||
if (code <= 0x20 || code >= 0x7f || sep === '%' || sep === '~') {
|
||||
throw new TypeError(`Separator must be printable ASCII other than "%" and "~", got ${JSON.stringify(sep)}`)
|
||||
}
|
||||
}
|
||||
|
||||
function hex2(code: number): string {
|
||||
return '%' + code.toString(16).toUpperCase().padStart(2, '0')
|
||||
}
|
||||
|
||||
/** Converts a user key to a segment string, rejecting `undefined`/`null`/empty. */
|
||||
export function toSegment(key: unknown): string {
|
||||
if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`)
|
||||
if (typeof key === 'number') {
|
||||
if (!Number.isFinite(key)) throw new InvalidKeyError(`Numeric key must be finite, got ${key}`)
|
||||
return String(key)
|
||||
}
|
||||
if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or number, got ${typeof key}`)
|
||||
if (key === '') throw new InvalidKeyError('Key must not be empty')
|
||||
return key
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes one path segment: percent-encodes `%`, the separator, and a leading `~`.
|
||||
* The result never contains a raw separator and never starts with `~`.
|
||||
*/
|
||||
export function escapeSegment(segment: string, sep: string = DEFAULT_SEPARATOR): string {
|
||||
// Fast path: nothing to escape (the overwhelmingly common case).
|
||||
if (segment.indexOf('%') === -1 && segment.indexOf(sep) === -1 && segment.charCodeAt(0) !== 0x7e) return segment
|
||||
let out = ''
|
||||
for (let i = 0; i < segment.length; i++) {
|
||||
const c = segment[i]!
|
||||
if (c === '%' || c === sep || (i === 0 && c === '~')) out += hex2(c.charCodeAt(0))
|
||||
else out += c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function unescapeSegment(escaped: string): string {
|
||||
if (escaped.indexOf('%') === -1) return escaped
|
||||
return escaped.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16)))
|
||||
}
|
||||
|
||||
export function isReservedSegment(escaped: string): boolean {
|
||||
return escaped.charCodeAt(0) === 0x7e // '~'
|
||||
}
|
||||
|
||||
/** Joins unescaped segments into an escaped key. */
|
||||
export function joinPath(path: readonly string[], sep: string = DEFAULT_SEPARATOR): string {
|
||||
let out = ''
|
||||
for (let i = 0; i < path.length; i++) {
|
||||
if (i > 0) out += sep
|
||||
out += escapeSegment(path[i]!, sep)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Splits an escaped key into unescaped segments. */
|
||||
export function splitPath(key: string, sep: string = DEFAULT_SEPARATOR): string[] {
|
||||
if (key === '') return []
|
||||
return key.split(sep).map(unescapeSegment)
|
||||
}
|
||||
|
||||
/**
|
||||
* Precomputed key strings for a node path.
|
||||
* - `selfKey`: escaped segments joined by the separator (`''` for the root)
|
||||
* - `prefix`: `selfKey + sep` (`''` for the root) — every descendant key starts with it
|
||||
* - `end`: exclusive upper bound of the subtree range (`selfKey + succ(sep)`; `undefined` for the root)
|
||||
*/
|
||||
export interface PathKeys {
|
||||
readonly path: readonly string[]
|
||||
readonly sep: string
|
||||
readonly selfKey: string
|
||||
readonly prefix: string
|
||||
readonly end: string | undefined
|
||||
}
|
||||
|
||||
export function pathKeys(path: readonly string[], sep: string): PathKeys {
|
||||
const selfKey = joinPath(path, sep)
|
||||
const root = path.length === 0
|
||||
return {
|
||||
path,
|
||||
sep,
|
||||
selfKey,
|
||||
prefix: root ? '' : selfKey + sep,
|
||||
end: root ? undefined : selfKey + succ(sep),
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
/**
|
||||
* Segment trie over path patterns. `*` matches exactly one segment.
|
||||
* A node's matching state is computed incrementally: `step(parentState, segment)`,
|
||||
* so resolving a path is O(depth) and `child()` is O(1) in the number of patterns.
|
||||
*/
|
||||
export interface TrieNode<T> {
|
||||
readonly literal: Map<string, TrieNode<T>>
|
||||
star: TrieNode<T> | undefined
|
||||
values: { pattern: string; value: T }[]
|
||||
}
|
||||
|
||||
export type TrieState<T> = readonly TrieNode<T>[]
|
||||
|
||||
export class PatternTrie<T> {
|
||||
readonly root: TrieNode<T> = newNode()
|
||||
|
||||
constructor(readonly sep: string) {}
|
||||
|
||||
/** `''` names the root node. */
|
||||
add(pattern: string, value: T): void {
|
||||
let node = this.root
|
||||
for (const seg of splitPattern(pattern, this.sep)) {
|
||||
if (seg === '*') node = node.star ??= newNode()
|
||||
else {
|
||||
let next = node.literal.get(seg)
|
||||
if (!next) node.literal.set(seg, (next = newNode()))
|
||||
node = next
|
||||
}
|
||||
}
|
||||
node.values.push({ pattern, value })
|
||||
}
|
||||
|
||||
get rootState(): TrieState<T> {
|
||||
return [this.root]
|
||||
}
|
||||
|
||||
step(state: TrieState<T>, segment: string): TrieState<T> {
|
||||
if (state.length === 0) return state
|
||||
if (state.length === 1) {
|
||||
const n = state[0]!
|
||||
const lit = n.literal.get(segment)
|
||||
if (lit && n.star) return [lit, n.star]
|
||||
return lit ? [lit] : n.star ? [n.star] : EMPTY
|
||||
}
|
||||
const out: TrieNode<T>[] = []
|
||||
for (const n of state) {
|
||||
const lit = n.literal.get(segment)
|
||||
if (lit) out.push(lit)
|
||||
if (n.star) out.push(n.star)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
resolve(path: readonly string[]): TrieState<T> {
|
||||
let state = this.rootState
|
||||
for (const seg of path) state = this.step(state, seg)
|
||||
return state
|
||||
}
|
||||
|
||||
/** Values attached to every pattern matching the state's path. */
|
||||
static values<T>(state: TrieState<T>): { pattern: string; value: T }[] {
|
||||
if (state.length === 1) return state[0]!.values
|
||||
return state.flatMap((n) => n.values)
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY: TrieState<never> = []
|
||||
|
||||
function newNode<T>(): TrieNode<T> {
|
||||
return { literal: new Map(), star: undefined, values: [] }
|
||||
}
|
||||
|
||||
export function splitPattern(pattern: string, sep: string): string[] {
|
||||
return pattern === '' ? [] : pattern.split(sep)
|
||||
}
|
||||
|
||||
/** Whether two patterns can match the same path. */
|
||||
export function patternsOverlap(a: string, b: string, sep: string): boolean {
|
||||
const sa = splitPattern(a, sep)
|
||||
const sb = splitPattern(b, sep)
|
||||
return sa.length === sb.length && sa.every((s, i) => s === '*' || sb[i] === '*' || s === sb[i])
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import type { Doc } from '../utils/document-path.js'
|
||||
import { DocumentQuery, type Filter, type FindOptions, type FindResult } from '../utils/document-query.js'
|
||||
import { DocumentUpdate, type DocChange, type UpdateSpec } from '../utils/document-update.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt, ValueOf } from '../utils/schema.js'
|
||||
import { Ulid } from '../utils/ulid.js'
|
||||
|
||||
/**
|
||||
* Fields to index, per collection path pattern (`*` matches one segment), e.g.
|
||||
* `{ users: ['email', 'age'], 'users/*\/posts': ['createdAt'] }`. Each implementation decides
|
||||
* how (or whether) to use them; a native database maps them onto its own indexes.
|
||||
*/
|
||||
export type IndexDeclarations = Record<string, string[]>
|
||||
|
||||
export interface UpdateOptions {
|
||||
/** Treat a missing document as `{}`. */
|
||||
upsert?: boolean
|
||||
}
|
||||
|
||||
/** Document type of a node's children. */
|
||||
export type ChildDoc<S> = ValueOf<ChildOf<S, KeyOf<S>>>
|
||||
|
||||
/**
|
||||
* Document storage. Every node is a document (its own) and a collection of child documents.
|
||||
* Implementations provide three primitives over document paths; the user-facing API is
|
||||
* defined here once.
|
||||
*/
|
||||
export abstract class BaseDocumentStorage<S extends Spec = AnySpec> {
|
||||
/** Unescaped path of this node (`[]` for the root, which has no own document). */
|
||||
readonly path: readonly string[] = []
|
||||
protected readonly ids = new Ulid()
|
||||
|
||||
abstract readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]>
|
||||
/**
|
||||
* Maps the current state of one document to a change (`null` = no change), applies it and
|
||||
* returns the new state. Implementations decide how this is made safe against concurrent writers.
|
||||
*/
|
||||
abstract mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise<Doc | undefined>
|
||||
/** Children of `parent` matching the query, filtered, sorted and paged. */
|
||||
abstract query(parent: readonly string[], query: DocumentQuery): AsyncIterable<FindResult>
|
||||
|
||||
child<K extends KeyOf<S>>(key: K): BaseDocumentStorage<ChildOf<S, K>> {
|
||||
const node = Object.create(this, { path: { value: [...this.path, Keys.segment(key)], enumerable: true } }) as BaseDocumentStorage<ChildOf<S, K>>
|
||||
// Frozen: child nodes share the root's state through the prototype, so an implementation that
|
||||
// reassigns a field (`this.x = ...`) while running on a child would silently shadow it.
|
||||
// Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw.
|
||||
Object.freeze(node)
|
||||
return node
|
||||
}
|
||||
|
||||
as<S2 extends Spec>(): BaseDocumentStorage<S2> {
|
||||
return this as unknown as BaseDocumentStorage<S2>
|
||||
}
|
||||
|
||||
async get<K extends KeyOf<S> | undefined = undefined>(key?: K): Promise<ValueAt<S, K> | undefined> {
|
||||
const [doc] = await this.readMany([this.pathOf(arguments.length > 0, key)])
|
||||
return doc as ValueAt<S, K> | undefined
|
||||
}
|
||||
|
||||
async getMany<K extends KeyOf<S>>(keys: readonly K[]): Promise<(ValueOf<ChildOf<S, K>> | undefined)[]> {
|
||||
return (await this.readMany(keys.map((k) => this.pathOf(true, k)))) as (ValueOf<ChildOf<S, K>> | undefined)[]
|
||||
}
|
||||
|
||||
/** `set(doc)` replaces this document; `set(key, doc)` a child document. */
|
||||
async set<K extends KeyOf<S>>(...args: [doc: ValueOf<S>] | [key: K, doc: ValueOf<ChildOf<S, K>>]): Promise<void> {
|
||||
const doc = args[args.length - 1]
|
||||
if (typeof doc !== 'object' || doc === null || Array.isArray(doc)) throw new TypeError('A document must be a plain object')
|
||||
await this.mutate(this.pathOf(args.length > 1, args[0]), () => ({ op: 'set', doc: doc as Doc }))
|
||||
}
|
||||
|
||||
/** Adds a child document under a new time-sortable id and returns the id. */
|
||||
async insert(doc: ChildDoc<S>): Promise<string> {
|
||||
const id = this.ids.generate()
|
||||
await this.set(id as KeyOf<S>, doc as never)
|
||||
return id
|
||||
}
|
||||
|
||||
/**
|
||||
* `update(spec, options?)` this document; `update(key, spec, options?)` a child.
|
||||
* Returns the new state, or `undefined` if the document doesn't exist (unless `upsert`).
|
||||
*/
|
||||
async update<K extends KeyOf<S>>(
|
||||
...args: [spec: UpdateSpec, options?: UpdateOptions] | [key: K, spec: UpdateSpec, options?: UpdateOptions]
|
||||
): Promise<ValueAt<S, K> | undefined> {
|
||||
const hasKey = typeof args[0] !== 'object' || args[0] === null
|
||||
const spec = args[hasKey ? 1 : 0] as UpdateSpec
|
||||
const options = (args[hasKey ? 2 : 1] ?? {}) as UpdateOptions
|
||||
const next = await this.mutate(this.pathOf(hasKey, args[0]), (current) => (current || options.upsert ? DocumentUpdate.resolve(current ?? {}, spec) : null))
|
||||
return next as ValueAt<S, K> | undefined
|
||||
}
|
||||
|
||||
/** Deletes this document (or a child). Returns whether one existed. */
|
||||
async delete(key?: KeyOf<S>): Promise<boolean> {
|
||||
let existed = false
|
||||
await this.mutate(this.pathOf(arguments.length > 0, key), (current) => ((existed = current !== undefined) ? { op: 'delete' } : null))
|
||||
return existed
|
||||
}
|
||||
|
||||
/** Child documents matching `filter`. */
|
||||
find(filter: Filter = {}, options: FindOptions = {}): AsyncIterable<FindResult<ChildDoc<S>>> {
|
||||
return this.query(this.path, new DocumentQuery(filter, options)) as AsyncIterable<FindResult<ChildDoc<S>>>
|
||||
}
|
||||
|
||||
async findOne(filter: Filter = {}, options: FindOptions = {}): Promise<FindResult<ChildDoc<S>> | undefined> {
|
||||
for await (const result of this.find(filter, { ...options, limit: 1 })) return result
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Keys of child documents. */
|
||||
async *keys(): AsyncGenerator<string> {
|
||||
for await (const { key } of this.find()) yield key
|
||||
}
|
||||
|
||||
/** Path of a child (`hasKey`) or this node; validates keys. */
|
||||
protected pathOf(hasKey: boolean, key: unknown): readonly string[] {
|
||||
if (hasKey) return [...this.path, Keys.segment(key)]
|
||||
if (this.path.length === 0) throw new TypeError('The root node has no own document')
|
||||
return this.path
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { Doc } from '../utils/document-path.js'
|
||||
import { DocumentPath } from '../utils/document-path.js'
|
||||
import type { DocumentQuery, FindResult } from '../utils/document-query.js'
|
||||
import { DocumentUpdate, type DocChange } from '../utils/document-update.js'
|
||||
import { Json } from '../utils/json.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, Spec } from '../utils/schema.js'
|
||||
import { BaseDocumentStorage, type IndexDeclarations } from './base-document-storage.js'
|
||||
|
||||
export interface DocumentStorageInMemoryOptions {
|
||||
/** Fields with an equality index, per collection pattern. */
|
||||
indexes?: IndexDeclarations
|
||||
}
|
||||
|
||||
/** Field → JSON-encoded value → child keys. */
|
||||
type CollectionIndex = Map<string, Map<string, Set<string>>>
|
||||
|
||||
/**
|
||||
* In-memory document storage. Declared index fields get an equality index per collection, used
|
||||
* when a filter has `{ field: value }` or `{ field: { $eq: value } }`; everything else is a scan.
|
||||
*/
|
||||
export class DocumentStorageInMemory<S extends Spec = AnySpec> extends BaseDocumentStorage<S> {
|
||||
private readonly collections = new Map<string, Map<string, Doc>>()
|
||||
private readonly indexes = new Map<string, CollectionIndex>()
|
||||
private readonly declarations: [pattern: string[], fields: string[]][]
|
||||
|
||||
constructor(options: DocumentStorageInMemoryOptions = {}) {
|
||||
super()
|
||||
this.declarations = Object.entries(options.indexes ?? {}).map(([pattern, fields]) => [pattern === '' ? [] : pattern.split('/'), fields])
|
||||
}
|
||||
|
||||
async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> {
|
||||
return paths.map((path) => Json.clone(this.collection(path.slice(0, -1)).get(path.at(-1)!)))
|
||||
}
|
||||
|
||||
async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise<Doc | undefined> {
|
||||
const parent = path.slice(0, -1)
|
||||
const key = path.at(-1)!
|
||||
const docs = this.collection(parent)
|
||||
const current = docs.get(key)
|
||||
const resolved = change(Json.clone(current))
|
||||
if (!resolved) return Json.clone(current)
|
||||
const next = DocumentUpdate.apply(current, resolved)
|
||||
if (next) docs.set(key, next)
|
||||
else docs.delete(key)
|
||||
this.reindex(parent, key, current, next)
|
||||
return Json.clone(next)
|
||||
}
|
||||
|
||||
async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator<FindResult> {
|
||||
const docs = this.collection(parent)
|
||||
const keys = [...(this.lookup(parent, query) ?? docs.keys())].sort()
|
||||
yield* query.run(
|
||||
(async function* () {
|
||||
for (const key of keys) {
|
||||
const doc = docs.get(key)
|
||||
if (doc) yield { key, data: Json.clone(doc) }
|
||||
}
|
||||
})(),
|
||||
)
|
||||
}
|
||||
|
||||
private collection(parent: readonly string[]): Map<string, Doc> {
|
||||
const id = Keys.join(parent)
|
||||
let docs = this.collections.get(id)
|
||||
if (!docs) this.collections.set(id, (docs = new Map()))
|
||||
return docs
|
||||
}
|
||||
|
||||
/** Candidate keys from an equality index, if the filter allows one. */
|
||||
private lookup(parent: readonly string[], query: DocumentQuery): Iterable<string> | undefined {
|
||||
const index = this.indexes.get(Keys.join(parent))
|
||||
for (const field of this.fieldsFor(parent)) {
|
||||
let value = query.filter[field]
|
||||
if (Json.isObject(value) && Object.keys(value).length === 1 && '$eq' in value) value = value.$eq
|
||||
if (value === undefined || typeof value === 'object') continue
|
||||
return index?.get(field)?.get(JSON.stringify(value)) ?? []
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
private reindex(parent: readonly string[], key: string, before: Doc | undefined, after: Doc | undefined): void {
|
||||
const fields = this.fieldsFor(parent)
|
||||
if (fields.length === 0) return
|
||||
const id = Keys.join(parent)
|
||||
let index = this.indexes.get(id)
|
||||
if (!index) this.indexes.set(id, (index = new Map()))
|
||||
for (const field of fields) {
|
||||
let byValue = index.get(field)
|
||||
if (!byValue) index.set(field, (byValue = new Map()))
|
||||
for (const value of DocumentStorageInMemory.indexValues(before, field)) byValue.get(value)?.delete(key)
|
||||
for (const value of DocumentStorageInMemory.indexValues(after, field)) {
|
||||
let keys = byValue.get(value)
|
||||
if (!keys) byValue.set(value, (keys = new Set()))
|
||||
keys.add(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fieldsFor(parent: readonly string[]): string[] {
|
||||
return this.declarations
|
||||
.filter(([pattern]) => pattern.length === parent.length && pattern.every((s, i) => s === '*' || s === parent[i]))
|
||||
.flatMap(([, fields]) => fields)
|
||||
}
|
||||
|
||||
/** Index keys for a field: its value, plus each element of an array (missing indexes as `null`). */
|
||||
private static indexValues(doc: Doc | undefined, field: string): string[] {
|
||||
if (!doc) return []
|
||||
const value = DocumentPath.get(doc, field) ?? null
|
||||
return Array.isArray(value) ? value.map((v) => JSON.stringify(v)) : [JSON.stringify(value)]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js'
|
||||
import type { Doc } from '../utils/document-path.js'
|
||||
import type { DocumentQuery, FindResult } from '../utils/document-query.js'
|
||||
import { DocumentUpdate, type DocChange } from '../utils/document-update.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, Spec } from '../utils/schema.js'
|
||||
import { BaseDocumentStorage } from './base-document-storage.js'
|
||||
|
||||
/**
|
||||
* Document storage on any KV storage: each document is the value at its path. Queries scan the
|
||||
* collection's direct children (no indexes). Writes are read-modify-write, so concurrent writers
|
||||
* to the same document are last-writer-wins; use `EventSourcedDocumentStorage` if that matters.
|
||||
*/
|
||||
export class DocumentStorageKV<S extends Spec = AnySpec> extends BaseDocumentStorage<S> {
|
||||
constructor(private readonly kv: BaseKVStorage) {
|
||||
super()
|
||||
}
|
||||
|
||||
async readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> {
|
||||
return (await this.kv.readMany(paths.map((p) => this.keyOf(p)))) as (Doc | undefined)[]
|
||||
}
|
||||
|
||||
async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise<Doc | undefined> {
|
||||
const key = this.keyOf(path)
|
||||
const [current] = (await this.kv.readMany([key])) as (Doc | undefined)[]
|
||||
const resolved = change(current)
|
||||
if (!resolved) return current
|
||||
const next = DocumentUpdate.apply(current, resolved)
|
||||
if (next) await this.kv.writeMany([{ key, value: next }])
|
||||
else await this.kv.removeMany([key])
|
||||
return next
|
||||
}
|
||||
|
||||
async *query(parent: readonly string[], query: DocumentQuery): AsyncGenerator<FindResult> {
|
||||
const collection = parent.reduce<BaseKVStorage>((node, segment) => node.child(segment), this.kv)
|
||||
yield* query.run(
|
||||
(async function* () {
|
||||
for await (const [key, data] of collection.entries()) yield { key, data: data as Doc }
|
||||
})(),
|
||||
)
|
||||
}
|
||||
|
||||
private keyOf(path: readonly string[]): string {
|
||||
return this.kv.prefix + Keys.join(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { BaseEventStorage, StoredEvent } from '../event-storage/base-event-storage.js'
|
||||
import type { Doc } from '../utils/document-path.js'
|
||||
import type { DocumentQuery, FindResult } from '../utils/document-query.js'
|
||||
import { DocumentUpdate, type DocChange } from '../utils/document-update.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec, ValueAt } from '../utils/schema.js'
|
||||
import { BaseDocumentStorage } from './base-document-storage.js'
|
||||
|
||||
// Types only: children of an event-sourced store are event-sourced stores too (child() creates
|
||||
// them with Object.create, so they share this class's prototype).
|
||||
export interface EventSourcedDocumentStorage<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): EventSourcedDocumentStorage<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* Documents as event streams: each change to a document is appended to its own stream, and the
|
||||
* current state is kept in a projection store (any `BaseDocumentStorage`), which serves all reads
|
||||
* and queries.
|
||||
*
|
||||
* A document's state is its events folded in order. Changes are stored as resolved values
|
||||
* (`$inc` becomes `set: { n: 6 }`), so two devices editing the same field concurrently end with
|
||||
* the later event's value; edits to different fields both survive.
|
||||
*
|
||||
* The projection is updated by this store's own writes. Events written elsewhere (another device,
|
||||
* arriving through sync) are folded in by `catchUp(key)`, which rebuilds the document from its
|
||||
* whole stream; call it when such events arrive.
|
||||
*/
|
||||
export class EventSourcedDocumentStorage<S extends Spec = AnySpec> extends BaseDocumentStorage<S> {
|
||||
constructor(
|
||||
private readonly events: BaseEventStorage,
|
||||
private readonly projections: BaseDocumentStorage,
|
||||
) {
|
||||
super()
|
||||
}
|
||||
|
||||
readMany(paths: (readonly string[])[]): Promise<(Doc | undefined)[]> {
|
||||
return this.projections.readMany(paths)
|
||||
}
|
||||
|
||||
query(parent: readonly string[], query: DocumentQuery): AsyncIterable<FindResult> {
|
||||
return this.projections.query(parent, query)
|
||||
}
|
||||
|
||||
async mutate(path: readonly string[], change: (current: Doc | undefined) => DocChange | null): Promise<Doc | undefined> {
|
||||
const [current] = await this.projections.readMany([path])
|
||||
const resolved = change(current)
|
||||
if (!resolved) return current
|
||||
await this.events.appendTo(path, [resolved])
|
||||
// Apply the change itself (not the state computed above), so concurrent writes in this
|
||||
// process each land on the latest projection.
|
||||
return this.projections.mutate(path, () => resolved)
|
||||
}
|
||||
|
||||
/** Events of this document (or a child), oldest first. */
|
||||
history(key?: KeyOf<S>): AsyncIterable<StoredEvent<DocChange>> {
|
||||
return this.events.readFrom(this.pathOf(arguments.length > 0, key), {}) as AsyncIterable<StoredEvent<DocChange>>
|
||||
}
|
||||
|
||||
/** State right after event `eventId`: `getAt(eventId)` for this document, `getAt(key, eventId)` for a child. */
|
||||
async getAt<K extends KeyOf<S>>(...args: [eventId: string] | [key: K, eventId: string]): Promise<ValueAt<S, K> | undefined> {
|
||||
const until = args[args.length - 1] as string
|
||||
let state: Doc | undefined
|
||||
for await (const event of this.events.readFrom(this.pathOf(args.length > 1, args[0]), {})) {
|
||||
if (event.id > until) break
|
||||
state = DocumentUpdate.apply(state, event.data as DocChange)
|
||||
}
|
||||
return state as ValueAt<S, K> | undefined
|
||||
}
|
||||
|
||||
/** Rebuilds the projection of this document (or a child) from all its events. Returns the state. */
|
||||
async catchUp(key?: KeyOf<S>): Promise<Doc | undefined> {
|
||||
const path = this.pathOf(arguments.length > 0, key)
|
||||
let state: Doc | undefined
|
||||
for await (const event of this.events.readFrom(path, {})) state = DocumentUpdate.apply(state, event.data as DocChange)
|
||||
await this.projections.mutate(path, () => (state ? { op: 'set', doc: state } : { op: 'delete' }))
|
||||
return state
|
||||
}
|
||||
}
|
||||
@@ -1,606 +0,0 @@
|
||||
import type { IndexValue } from '../core/encoding.js'
|
||||
import { escapeSegment, RESERVED, succ, toSegment, unescapeSegment } from '../core/path.js'
|
||||
import type { TrieState } from '../core/trie.js'
|
||||
import { CapabilityError, ConcurrencyError, UniqueConstraintError } from '../errors.js'
|
||||
import { KeyedMutex } from '../core/mutex.js'
|
||||
import { KVEventNode } from '../events/kv.js'
|
||||
import type { EventNode, ReadOptions, StoredEvent } from '../events/types.js'
|
||||
import { scanChildren } from '../kv/children.js'
|
||||
import type { BatchOp, KVDriver } from '../kv/driver.js'
|
||||
import type { KVNode } from '../kv/node.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js'
|
||||
import { analyze, matches, type Filter } from './filter.js'
|
||||
import { DocIndexes, DocIndexRegistry, type DocIndexDeclarations, type Plan } from './indexes.js'
|
||||
import type { ChildDoc, DocKeysOptions, EventSourcedDocumentNode, FindOptions, FindResult, GetOptions, UpdateOptions } from './types.js'
|
||||
import { ulid } from './ulid.js'
|
||||
import { applyEvent, applyPatch, resolveUpdate, toPatchEvent, type DocEvent, type UpdateSpec } from './update.js'
|
||||
import { clone, compareValues, deepEqual, getPath, isPlainObject, setPath, sortKey, unsetPath, type Doc } from './values.js'
|
||||
|
||||
export interface EventSourcedDocumentStorageOptions {
|
||||
/** Where projections live. Default: the `~p` root of the event store's KV. Required for non-KV event stores. */
|
||||
projections?: KVNode<AnySpec>
|
||||
/** Path pattern → indexes over that node's direct children. */
|
||||
indexes?: DocIndexDeclarations
|
||||
/** Delete stale index entries found by `find`. Default `true`. */
|
||||
readRepair?: boolean
|
||||
/** Retries after a write conflict (each preceded by a catch-up). Default 5. */
|
||||
maxRetries?: number
|
||||
/** Warn once per filter shape when `find` falls back to a full scan. Default: `NODE_ENV !== 'production'`. */
|
||||
warnOnFullScan?: boolean
|
||||
/** Id generator for `insert`. Default: ULID-like. */
|
||||
newId?: () => string
|
||||
/**
|
||||
* A unique claim whose owner no longer holds the value is only taken over once it is older
|
||||
* than this, since its owner may be mid-write. Default 30 000 ms.
|
||||
*/
|
||||
claimGraceMs?: number
|
||||
/** Clock for claim timestamps. Default `Date.now`. */
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/** Stored projection of one document. */
|
||||
export type Projection =
|
||||
{ state: Doc; version: number; commit: number; deleted?: undefined } | { deleted: true; version: number; commit: number; state?: undefined }
|
||||
|
||||
interface DocContext {
|
||||
readonly driver: KVDriver
|
||||
readonly sep: string
|
||||
/** Projection-space prefix: projection key of a doc = `projBase + docSelfKey`. */
|
||||
readonly projBase: string
|
||||
readonly registry: DocIndexRegistry
|
||||
readonly readRepair: boolean
|
||||
readonly maxRetries: number
|
||||
readonly warnOnFullScan: boolean
|
||||
readonly warned: Set<string>
|
||||
readonly newId: () => string
|
||||
readonly claimGraceMs: number
|
||||
readonly now: () => number
|
||||
/** Serialises writes to one document within this process, so retries only arise across processes. */
|
||||
readonly writeLock: KeyedMutex
|
||||
}
|
||||
|
||||
/** Stored value of a unique claim: owner child segment (escaped) and claim time. */
|
||||
type Claim = [owner: string, at: number]
|
||||
|
||||
interface Target {
|
||||
ev: EventNode<AnySpec>
|
||||
parentEv: EventNode<AnySpec>
|
||||
esc: string
|
||||
/** Projection key of the document. */
|
||||
projKey: string
|
||||
/** Projection-space prefix of its parent (sibling projection key = `parentProj + esc`). */
|
||||
parentProj: string
|
||||
idx: DocIndexes | undefined
|
||||
}
|
||||
|
||||
type Hit = { esc: string; state: Doc }
|
||||
|
||||
const FIND_CHUNK = 128
|
||||
|
||||
/**
|
||||
* Document node backed by an event stream per document and a projection per document.
|
||||
* Writes: read projection → resolve delta → append (expectedVersion = projection.version) →
|
||||
* write projection + index entries. Reads hit the projection only.
|
||||
*/
|
||||
export class EventSourcedDocumentNodeImpl<S extends Spec = AnySpec> implements EventSourcedDocumentNode<S> {
|
||||
readonly path: readonly string[]
|
||||
/** Document-space key (escaped), relative to the store root. */
|
||||
readonly selfKey: string
|
||||
readonly #ctx: DocContext
|
||||
readonly #ev: EventNode<AnySpec>
|
||||
readonly #parentEv: EventNode<AnySpec> | undefined
|
||||
readonly #esc: string
|
||||
readonly #prefix: string
|
||||
readonly #state: TrieState<any>
|
||||
readonly #selfIdx: DocIndexes | undefined
|
||||
readonly #childIdx: DocIndexes | undefined
|
||||
|
||||
/** @internal Use `EventSourcedDocumentStorage` and `child()`. */
|
||||
constructor(
|
||||
ctx: DocContext,
|
||||
ev: EventNode<AnySpec>,
|
||||
parentEv: EventNode<AnySpec> | undefined,
|
||||
path: readonly string[],
|
||||
selfKey: string,
|
||||
esc: string,
|
||||
state: TrieState<any>,
|
||||
selfIdx: DocIndexes | undefined,
|
||||
) {
|
||||
this.#ctx = ctx
|
||||
this.#ev = ev
|
||||
this.#parentEv = parentEv
|
||||
this.path = path
|
||||
this.selfKey = selfKey
|
||||
this.#esc = esc
|
||||
this.#prefix = path.length === 0 ? '' : selfKey + ctx.sep
|
||||
this.#state = state
|
||||
this.#selfIdx = selfIdx
|
||||
this.#childIdx = ctx.registry.bind(state, ctx.projBase + this.#prefix)
|
||||
}
|
||||
|
||||
child<K extends KeyOf<S>>(key: K): EventSourcedDocumentNodeImpl<ChildOf<S, K>> {
|
||||
const seg = toSegment(key)
|
||||
const esc = escapeSegment(seg, this.#ctx.sep)
|
||||
return new EventSourcedDocumentNodeImpl(
|
||||
this.#ctx,
|
||||
this.#ev.child(seg),
|
||||
this.#ev,
|
||||
[...this.path, seg],
|
||||
this.#prefix + esc,
|
||||
esc,
|
||||
this.#ctx.registry.trie.step(this.#state, seg),
|
||||
this.#childIdx,
|
||||
)
|
||||
}
|
||||
|
||||
as<S2 extends Spec>(): EventSourcedDocumentNodeImpl<S2> {
|
||||
return this as unknown as EventSourcedDocumentNodeImpl<S2>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- reads
|
||||
|
||||
get(opts?: GetOptions): Promise<ValueOf<S> | undefined>
|
||||
get<K extends KeyOf<S>>(key: K, opts?: GetOptions): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
async get(...args: unknown[]): Promise<unknown> {
|
||||
const [t, rest] = this.#split(args)
|
||||
const opts = (rest[0] ?? {}) as GetOptions
|
||||
let proj = await this.#projection(t)
|
||||
if (opts.consistent && (await t.ev.version()) > (proj?.version ?? 0)) {
|
||||
await this.#catchUpDoc(t)
|
||||
proj = await this.#projection(t)
|
||||
}
|
||||
return proj?.state
|
||||
}
|
||||
|
||||
history(): AsyncIterable<StoredEvent<DocEvent>>
|
||||
history<K extends KeyOf<S>>(key: K): AsyncIterable<StoredEvent<DocEvent>>
|
||||
history(...args: unknown[]): AsyncIterable<StoredEvent<DocEvent>> {
|
||||
const t = args.length === 0 ? this.#self() : this.#child(args[0])
|
||||
return t.ev.read() as AsyncIterable<StoredEvent<DocEvent>>
|
||||
}
|
||||
|
||||
getAt(version: number): Promise<ValueOf<S> | undefined>
|
||||
getAt<K extends KeyOf<S>>(key: K, version: number): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
async getAt(...args: unknown[]): Promise<unknown> {
|
||||
const t = args.length <= 1 ? this.#self() : this.#child(args[0])
|
||||
const version = args[args.length - 1] as number
|
||||
if (!Number.isInteger(version) || version < 0) throw new TypeError(`Invalid version ${String(version)}`)
|
||||
let state: Doc | undefined
|
||||
for await (const e of t.ev.read({ to: version } satisfies ReadOptions)) state = applyEvent(state, e.data as DocEvent)
|
||||
return state
|
||||
}
|
||||
|
||||
async *find(filter: Filter = {}, opts: FindOptions = {}): AsyncGenerator<FindResult<ChildDoc<S>>> {
|
||||
const skip = opts.skip ?? 0
|
||||
const limit = opts.limit ?? Infinity
|
||||
if (limit <= 0) return
|
||||
const sort = Object.entries(opts.sort ?? {}) as [string, 1 | -1][]
|
||||
const plan = this.#childIdx?.plan(analyze(filter), sort)
|
||||
if (!plan && Object.keys(filter).length > 0) this.#warnFullScan(filter)
|
||||
const source = plan ? this.#indexScan(plan, skip + limit) : this.#projectionScan()
|
||||
const project = opts.projection ? projector(opts.projection) : (d: Doc) => d
|
||||
|
||||
if (sort.length && !plan?.sorted) {
|
||||
// In-memory sort, bounded to skip + limit when a limit is given.
|
||||
const cap = skip + limit
|
||||
const cmp = comparator(sort)
|
||||
const top: Hit[] = []
|
||||
for await (const hit of source) {
|
||||
if (!matches(hit.state, filter)) continue
|
||||
if (top.length >= cap && cmp(hit, top[top.length - 1]!) >= 0) continue
|
||||
let lo = 0
|
||||
let hi = top.length
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1
|
||||
if (cmp(top[mid]!, hit) <= 0) lo = mid + 1
|
||||
else hi = mid
|
||||
}
|
||||
top.splice(lo, 0, hit)
|
||||
if (top.length > cap) top.pop()
|
||||
}
|
||||
for (let i = skip; i < top.length; i++) yield { key: unescapeSegment(top[i]!.esc), data: project(top[i]!.state) as ChildDoc<S> }
|
||||
return
|
||||
}
|
||||
|
||||
let skipped = 0
|
||||
let emitted = 0
|
||||
for await (const hit of source) {
|
||||
if (!matches(hit.state, filter)) continue
|
||||
if (skipped < skip) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
yield { key: unescapeSegment(hit.esc), data: project(hit.state) as ChildDoc<S> }
|
||||
if (++emitted >= limit) return
|
||||
}
|
||||
}
|
||||
|
||||
async findOne(filter: Filter = {}, opts: FindOptions = {}): Promise<FindResult<ChildDoc<S>> | undefined> {
|
||||
for await (const r of this.find(filter, { ...opts, limit: 1 })) return r
|
||||
return undefined
|
||||
}
|
||||
|
||||
async *keys(opts: DocKeysOptions = {}): AsyncGenerator<KeyString<S>> {
|
||||
let remaining = opts.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
const after = opts.after === undefined ? undefined : toSegment(opts.after)
|
||||
const { prefix, end } = this.#childRange()
|
||||
for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { after, reverse: opts.reverse, values: true })) {
|
||||
if (!hit.hasOwn || (hit.value as Projection).deleted) continue
|
||||
yield unescapeSegment(hit.esc) as KeyString<S>
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- writes
|
||||
|
||||
set(doc: ValueOf<S>): Promise<void>
|
||||
set<K extends KeyOf<S>>(key: K, doc: ValueOf<ChildOf<S, K>>): Promise<void>
|
||||
async set(...args: unknown[]): Promise<void> {
|
||||
const [t, rest] = this.#split(args)
|
||||
const doc = rest[0]
|
||||
if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object')
|
||||
const snapshot = clone(doc)
|
||||
await this.#write(t, () => ({ op: 'set', doc: snapshot }))
|
||||
}
|
||||
|
||||
async insert(doc: ChildDoc<S>): Promise<string> {
|
||||
if (!isPlainObject(doc)) throw new TypeError('A document must be a plain object')
|
||||
const id = this.#ctx.newId()
|
||||
const t = this.#child(id)
|
||||
const snapshot = clone(doc)
|
||||
await this.#write(t, (cur) => {
|
||||
if (cur) throw new Error(`Document "${id}" already exists`)
|
||||
return { op: 'set', doc: snapshot }
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
update(spec: UpdateSpec, opts?: UpdateOptions): Promise<ValueOf<S> | undefined>
|
||||
update<K extends KeyOf<S>>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
async update(...args: unknown[]): Promise<unknown> {
|
||||
const [t, rest] = this.#split(args)
|
||||
const spec = rest[0] as UpdateSpec
|
||||
const opts = (rest[1] ?? {}) as UpdateOptions
|
||||
if (!isPlainObject(spec)) throw new TypeError('update() expects an update spec')
|
||||
const result = await this.#write(t, (cur) => {
|
||||
if (!cur && !opts.upsert) return null
|
||||
const base = cur ?? {}
|
||||
const patch = resolveUpdate(base, spec)
|
||||
if (cur && deepEqual(applyPatch(base, patch), cur)) return null // no-op: no event
|
||||
return toPatchEvent(patch)
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
delete(): Promise<boolean>
|
||||
delete<K extends KeyOf<S>>(key: K): Promise<boolean>
|
||||
async delete(...args: unknown[]): Promise<boolean> {
|
||||
const t = args.length === 0 ? this.#self() : this.#child(args[0])
|
||||
let deleted = false
|
||||
await this.#write(t, (cur) => {
|
||||
deleted = cur !== undefined
|
||||
return cur ? { op: 'delete' } : null
|
||||
})
|
||||
return deleted
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- maintenance
|
||||
|
||||
async catchUp(): Promise<number> {
|
||||
let repaired = 0
|
||||
if (this.path.length > 0 && (await this.#catchUpDoc(this.#self()))) repaired++
|
||||
for await (const key of this.#ev.keys()) repaired += await this.child(key as KeyOf<S>).catchUp()
|
||||
return repaired
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- internals
|
||||
|
||||
#self(): Target {
|
||||
if (this.path.length === 0) throw new TypeError('The root node has no own document')
|
||||
const parentProj = this.#ctx.projBase + this.selfKey.slice(0, this.selfKey.length - this.#esc.length)
|
||||
return { ev: this.#ev, parentEv: this.#parentEv!, esc: this.#esc, projKey: this.#ctx.projBase + this.selfKey, parentProj, idx: this.#selfIdx }
|
||||
}
|
||||
|
||||
#child(key: unknown): Target {
|
||||
const seg = toSegment(key)
|
||||
const esc = escapeSegment(seg, this.#ctx.sep)
|
||||
const parentProj = this.#ctx.projBase + this.#prefix
|
||||
return { ev: this.#ev.child(seg), parentEv: this.#ev, esc, projKey: parentProj + esc, parentProj, idx: this.#childIdx }
|
||||
}
|
||||
|
||||
/** Target for a sibling of `t` given its escaped segment. */
|
||||
#sibling(t: Target, esc: string): Target {
|
||||
return { ev: t.parentEv.child(unescapeSegment(esc)), parentEv: t.parentEv, esc, projKey: t.parentProj + esc, parentProj: t.parentProj, idx: t.idx }
|
||||
}
|
||||
|
||||
/** Self form when there are no args or the first is a non-null object; otherwise key form. */
|
||||
#split(args: unknown[]): [Target, unknown[]] {
|
||||
if (args.length === 0 || (typeof args[0] === 'object' && args[0] !== null)) return [this.#self(), args]
|
||||
return [this.#child(args[0]), args.slice(1)]
|
||||
}
|
||||
|
||||
#childRange(): { prefix: string; end: string | undefined } {
|
||||
const prefix = this.#ctx.projBase + this.#prefix
|
||||
return { prefix, end: prefix === '' ? undefined : prefix.slice(0, -1) + succ(this.#ctx.sep) }
|
||||
}
|
||||
|
||||
async #projection(t: Target): Promise<Projection | undefined> {
|
||||
return (await this.#ctx.driver.getMany([t.projKey]))[0] as Projection | undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* The write path. `compute` maps the current state to an event (or `null` for a no-op).
|
||||
* Returns the resulting state.
|
||||
*/
|
||||
#write(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise<Doc | undefined> {
|
||||
return this.#ctx.writeLock.run(t.projKey, () => this.#writeUnlocked(t, compute))
|
||||
}
|
||||
|
||||
async #writeUnlocked(t: Target, compute: (cur: Doc | undefined) => DocEvent | null): Promise<Doc | undefined> {
|
||||
const { maxRetries } = this.#ctx
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const proj = await this.#projection(t)
|
||||
const cur = proj?.state
|
||||
const event = compute(cur)
|
||||
if (!event) return cur
|
||||
const next = applyEvent(cur, event)
|
||||
|
||||
const oldClaims = t.idx?.claimsFor(cur)
|
||||
const newClaims = t.idx?.claimsFor(next)
|
||||
const acquired: string[] = []
|
||||
if (newClaims?.size) {
|
||||
const wanted = new Map([...newClaims].filter(([k]) => !oldClaims!.has(k)))
|
||||
if (wanted.size) await this.#claim(t, wanted, acquired)
|
||||
}
|
||||
|
||||
let result
|
||||
try {
|
||||
result = await t.ev.append([event], { expectedVersion: proj?.version ?? 0, expectedCommit: proj?.commit ?? 0 })
|
||||
} catch (err) {
|
||||
if (acquired.length) await this.#ctx.driver.deleteMany(acquired)
|
||||
if (err instanceof ConcurrencyError && attempt < maxRetries) {
|
||||
await this.#catchUpDoc(t)
|
||||
continue
|
||||
}
|
||||
throw err
|
||||
}
|
||||
const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : []
|
||||
const projection: Projection = next
|
||||
? { state: next, version: result.version, commit: result.commit }
|
||||
: { deleted: true, version: result.version, commit: result.commit }
|
||||
await this.#writeProjection(t, cur, next, projection, releases)
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims unique values for `t`; records successful claims in `acquired`. Throws
|
||||
* `UniqueConstraintError` on conflict. A claim held by another document is taken over only if
|
||||
* that document, caught up to its stream head, no longer holds the value and the claim is older
|
||||
* than `claimGraceMs` (its owner might otherwise be between claiming and appending).
|
||||
*/
|
||||
async #claim(t: Target, claims: Map<string, { index: string; values: IndexValue[] }>, acquired: string[]): Promise<void> {
|
||||
const { driver, now, claimGraceMs } = this.#ctx
|
||||
const keys = [...claims.keys()]
|
||||
const written = await driver.setIfAbsent!(keys.map((k) => [k, [t.esc, now()] satisfies Claim]))
|
||||
const lost = keys.filter((k, i) => {
|
||||
if (written[i]) acquired.push(k)
|
||||
return !written[i]
|
||||
})
|
||||
if (!lost.length) return
|
||||
const holders = (await driver.getMany(lost)) as (Claim | undefined)[]
|
||||
const fail = async (k: string, owner?: string): Promise<never> => {
|
||||
if (acquired.length) await driver.deleteMany(acquired.splice(0))
|
||||
const c = claims.get(k)!
|
||||
throw new UniqueConstraintError(c.index, c.values, owner === undefined ? undefined : unescapeSegment(owner))
|
||||
}
|
||||
for (let i = 0; i < lost.length; i++) {
|
||||
const k = lost[i]!
|
||||
const holder = holders[i]
|
||||
if (holder?.[0] === t.esc) continue // already ours (e.g. left by an interrupted write)
|
||||
if (holder) {
|
||||
const owner = this.#sibling(t, holder[0])
|
||||
await this.#catchUpDoc(owner)
|
||||
const ownerProj = await this.#projection(owner)
|
||||
if (t.idx!.claimsFor(ownerProj?.state).has(k)) await fail(k, holder[0])
|
||||
if (now() - holder[1] < claimGraceMs) await fail(k, holder[0])
|
||||
await driver.deleteMany([k])
|
||||
}
|
||||
const [ok] = await driver.setIfAbsent!([[k, [t.esc, now()] satisfies Claim]])
|
||||
if (!ok) await fail(k)
|
||||
acquired.push(k)
|
||||
}
|
||||
}
|
||||
|
||||
async #writeProjection(t: Target, cur: Doc | undefined, next: Doc | undefined, projection: Projection, releases: string[]): Promise<void> {
|
||||
const { driver } = this.#ctx
|
||||
if (!t.idx) {
|
||||
await driver.setMany([[t.projKey, projection]])
|
||||
return
|
||||
}
|
||||
const before = t.idx.entriesFor(t.esc, cur)
|
||||
const after = t.idx.entriesFor(t.esc, next)
|
||||
const adds = [...after].filter((e) => !before.has(e))
|
||||
const removes = [...before].filter((e) => !after.has(e)).concat(releases)
|
||||
if (driver.batch) {
|
||||
const ops: BatchOp[] = adds.map((key) => ({ type: 'set', key, value: true }))
|
||||
ops.push({ type: 'set', key: t.projKey, value: projection })
|
||||
for (const key of removes) ops.push({ type: 'delete', key })
|
||||
await driver.batch(ops)
|
||||
return
|
||||
}
|
||||
if (adds.length) await driver.setMany(adds.map((k) => [k, true]))
|
||||
await driver.setMany([[t.projKey, projection]])
|
||||
if (removes.length) await driver.deleteMany(removes)
|
||||
}
|
||||
|
||||
/** Replays events newer than the projection into it. Returns whether anything changed. */
|
||||
async #catchUpDoc(t: Target): Promise<boolean> {
|
||||
const proj = await this.#projection(t)
|
||||
const cur = proj?.state
|
||||
let state = cur
|
||||
let version = proj?.version ?? 0
|
||||
let commit = proj?.commit ?? 0
|
||||
for await (const e of t.ev.read({ from: version + 1 })) {
|
||||
state = applyEvent(state, e.data as DocEvent)
|
||||
version = e.version
|
||||
commit = e.commit
|
||||
}
|
||||
if (version === (proj?.version ?? 0)) return false
|
||||
const oldClaims = t.idx?.claimsFor(cur)
|
||||
const newClaims = t.idx?.claimsFor(state)
|
||||
if (newClaims?.size) {
|
||||
// The log is the source of truth: (re)assert claims for the caught-up state.
|
||||
const missing = [...newClaims.keys()].filter((k) => !oldClaims!.has(k))
|
||||
if (missing.length) await this.#ctx.driver.setMany(missing.map((k) => [k, [t.esc, this.#ctx.now()] satisfies Claim]))
|
||||
}
|
||||
const releases = oldClaims ? [...oldClaims.keys()].filter((k) => !newClaims!.has(k)) : []
|
||||
const projection: Projection = state ? { state, version, commit } : { deleted: true, version, commit }
|
||||
await this.#writeProjection(t, cur, state, projection, releases)
|
||||
return true
|
||||
}
|
||||
|
||||
async *#projectionScan(): AsyncGenerator<Hit> {
|
||||
const { prefix, end } = this.#childRange()
|
||||
for await (const hit of scanChildren(this.#ctx.driver, prefix, end, this.#ctx.sep, { values: true })) {
|
||||
const p = hit.value as Projection | undefined
|
||||
if (hit.hasOwn && p?.state) yield { esc: hit.esc, state: p.state }
|
||||
}
|
||||
}
|
||||
|
||||
async *#indexScan(plan: Plan, want: number): AsyncGenerator<Hit> {
|
||||
const idx = this.#childIdx!
|
||||
const { driver, readRepair } = this.#ctx
|
||||
const parentProj = this.#ctx.projBase + this.#prefix
|
||||
const seen = new Set<string>()
|
||||
let chunk: string[] = []
|
||||
let chunkSize = Math.min(FIND_CHUNK, Math.max(1, want))
|
||||
const it = driver.scan({ gte: plan.gte, lt: plan.lt, reverse: plan.reverse })[Symbol.asyncIterator]()
|
||||
|
||||
async function* flush(): AsyncGenerator<Hit> {
|
||||
const entries = chunk
|
||||
chunk = []
|
||||
chunkSize = FIND_CHUNK
|
||||
const escs = entries.map((e) => idx.childOf(plan.index, e))
|
||||
const projs = (await driver.getMany(escs.map((e) => parentProj + e))) as (Projection | undefined)[]
|
||||
const stale: string[] = []
|
||||
try {
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const esc = escs[i]!
|
||||
const state = projs[i]?.state
|
||||
if (!idx.entriesFor(esc, state).has(entries[i]!)) {
|
||||
stale.push(entries[i]!)
|
||||
continue
|
||||
}
|
||||
if (seen.has(esc)) continue
|
||||
seen.add(esc)
|
||||
yield { esc, state: state! }
|
||||
}
|
||||
} finally {
|
||||
if (readRepair && stale.length) await driver.deleteMany(stale)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const r = await it.next()
|
||||
if (r.done) break
|
||||
chunk.push(r.value[0])
|
||||
if (chunk.length >= chunkSize) yield* flush()
|
||||
}
|
||||
if (chunk.length) yield* flush()
|
||||
} finally {
|
||||
await it.return?.()
|
||||
}
|
||||
}
|
||||
|
||||
#warnFullScan(filter: Filter): void {
|
||||
if (!this.#ctx.warnOnFullScan) return
|
||||
const shape = JSON.stringify(filterShape(filter))
|
||||
if (this.#ctx.warned.has(shape)) return
|
||||
this.#ctx.warned.add(shape)
|
||||
console.warn(`[potential-storage] find() on "${this.selfKey || '(root)'}" has no usable index; full scan for filter ${shape}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Filter with values replaced by `?`, for de-duplicating warnings. */
|
||||
function filterShape(f: unknown): unknown {
|
||||
if (Array.isArray(f)) return f.map(filterShape)
|
||||
if (isPlainObject(f))
|
||||
return Object.fromEntries(
|
||||
Object.entries(f).map(([k, v]) => [
|
||||
k,
|
||||
k.startsWith('$') && (k === '$and' || k === '$or' || k === '$not') ? filterShape(v) : isPlainObject(v) ? filterShape(v) : '?',
|
||||
]),
|
||||
)
|
||||
return '?'
|
||||
}
|
||||
|
||||
function comparator(sort: [string, 1 | -1][]): (a: Hit, b: Hit) => number {
|
||||
const tieDir = sort[0]?.[1] ?? 1
|
||||
return (a, b) => {
|
||||
for (const [field, dir] of sort) {
|
||||
const d = compareValues(sortKey(a.state, field, dir), sortKey(b.state, field, dir))
|
||||
if (d !== 0) return d * dir
|
||||
}
|
||||
return a.esc === b.esc ? 0 : (a.esc < b.esc ? -1 : 1) * tieDir
|
||||
}
|
||||
}
|
||||
|
||||
function projector(spec: Record<string, 0 | 1 | boolean>): (d: Doc) => Doc {
|
||||
const fields = Object.entries(spec)
|
||||
const include = fields.some(([, v]) => v === 1 || v === true)
|
||||
if (include && fields.some(([, v]) => v === 0 || v === false)) throw new TypeError('Projection cannot mix inclusion and exclusion')
|
||||
return (d) => {
|
||||
if (include) {
|
||||
const out: Doc = {}
|
||||
for (const [f] of fields) {
|
||||
const v = getPath(d, f)
|
||||
if (v !== undefined) setPath(out, f, clone(v))
|
||||
}
|
||||
return out
|
||||
}
|
||||
const out = clone(d)
|
||||
for (const [f] of fields) unsetPath(out, f)
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
/** Event-sourced document storage: the root document node. */
|
||||
export class EventSourcedDocumentStorage<S extends Spec = AnySpec> extends EventSourcedDocumentNodeImpl<S> {
|
||||
constructor(events: EventNode<AnySpec>, opts: EventSourcedDocumentStorageOptions = {}) {
|
||||
let driver: KVDriver
|
||||
let sep: string
|
||||
let projBase: string
|
||||
if (opts.projections) {
|
||||
driver = opts.projections.driver
|
||||
sep = opts.projections.separator
|
||||
projBase = opts.projections.prefix
|
||||
} else if (events instanceof KVEventNode) {
|
||||
const kv = events.kv
|
||||
driver = kv.driver
|
||||
sep = kv.separator
|
||||
projBase = kv.prefix + RESERVED.projections + sep
|
||||
} else {
|
||||
throw new TypeError('EventSourcedDocumentStorage needs `projections` when the event store is not KV-backed')
|
||||
}
|
||||
const registry = new DocIndexRegistry(opts.indexes ?? {}, sep)
|
||||
if (registry.hasUnique && !driver.setIfAbsent) throw new CapabilityError('setIfAbsent', 'unique document indexes require it')
|
||||
const ctx: DocContext = {
|
||||
driver,
|
||||
sep,
|
||||
projBase,
|
||||
registry,
|
||||
readRepair: opts.readRepair ?? true,
|
||||
maxRetries: opts.maxRetries ?? 5,
|
||||
warnOnFullScan: opts.warnOnFullScan ?? (typeof process === 'undefined' || process.env?.NODE_ENV !== 'production'),
|
||||
warned: new Set(),
|
||||
newId: opts.newId ?? ulid,
|
||||
claimGraceMs: opts.claimGraceMs ?? 30_000,
|
||||
now: opts.now ?? Date.now,
|
||||
writeLock: new KeyedMutex(),
|
||||
}
|
||||
super(ctx, events, undefined, [], '', '', registry.trie.rootState, undefined)
|
||||
}
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
import type { IndexValue } from '../core/encoding.js'
|
||||
import { compareValues, deepEqual, isPlainObject, isScalar, resolvePath, typeRank, type Doc } from './values.js'
|
||||
|
||||
/**
|
||||
* Query filter (Mongo subset): field conditions with implicit equality and dot paths,
|
||||
* `$eq $ne $gt $gte $lt $lte $in $nin $exists`, and `$and $or $not`.
|
||||
*
|
||||
* Semantics differences from MongoDB (see DESIGN.md §12, M6):
|
||||
* - Comparisons are type-bracketed: `$gt: 5` only matches numbers.
|
||||
* - On array fields, all positive operators in one condition (`$eq $gt $gte $lt $lte $in`) must be
|
||||
* satisfied by the same element (like `$elemMatch`), so index range scans are exact.
|
||||
*/
|
||||
export type Filter = { [field: string]: unknown; $and?: Filter[]; $or?: Filter[]; $not?: Filter }
|
||||
|
||||
const POSITIVE = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in'])
|
||||
const FIELD_LEVEL = new Set(['$ne', '$nin', '$exists', '$not'])
|
||||
|
||||
export function isOperatorObject(v: unknown): v is Record<string, unknown> {
|
||||
if (!isPlainObject(v)) return false
|
||||
const keys = Object.keys(v)
|
||||
return keys.length > 0 && keys.every((k) => k.startsWith('$'))
|
||||
}
|
||||
|
||||
export function matches(doc: unknown, filter: Filter): boolean {
|
||||
for (const key of Object.keys(filter)) {
|
||||
const cond = filter[key]
|
||||
switch (key) {
|
||||
case '$and':
|
||||
if (!(cond as Filter[]).every((f) => matches(doc, f))) return false
|
||||
break
|
||||
case '$or':
|
||||
if (!(cond as Filter[]).some((f) => matches(doc, f))) return false
|
||||
break
|
||||
case '$not':
|
||||
if (matches(doc, cond as Filter)) return false
|
||||
break
|
||||
default:
|
||||
if (key.startsWith('$')) throw new TypeError(`Unknown top-level operator ${key}`)
|
||||
if (!matchesField(resolvePath(doc, key), cond)) return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** Whether the values found at a path satisfy a field condition. */
|
||||
export function matchesField(values: unknown[], cond: unknown): boolean {
|
||||
if (!isOperatorObject(cond)) return someCandidate(values, (c) => eq(c, cond))
|
||||
const positive: [string, unknown][] = []
|
||||
for (const [op, arg] of Object.entries(cond)) {
|
||||
if (POSITIVE.has(op)) positive.push([op, arg])
|
||||
else if (!FIELD_LEVEL.has(op)) throw new TypeError(`Unknown operator ${op}`)
|
||||
}
|
||||
for (const [op, arg] of Object.entries(cond)) {
|
||||
switch (op) {
|
||||
case '$ne':
|
||||
if (someCandidate(values, (c) => eq(c, arg))) return false
|
||||
break
|
||||
case '$nin':
|
||||
if (someCandidate(values, (c) => (arg as unknown[]).some((x) => eq(c, x)))) return false
|
||||
break
|
||||
case '$exists':
|
||||
if (values.length > 0 !== Boolean(arg)) return false
|
||||
break
|
||||
case '$not':
|
||||
if (matchesField(values, arg)) return false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (positive.length === 0) return true
|
||||
return someCandidate(values, (c) => positive.every(([op, arg]) => applyPositive(op, c, arg)))
|
||||
}
|
||||
|
||||
/** Tests each value and, for arrays, each element. Missing paths test `undefined` once. */
|
||||
function someCandidate(values: unknown[], test: (c: unknown) => boolean): boolean {
|
||||
if (values.length === 0) return test(undefined)
|
||||
for (const v of values) {
|
||||
if (test(v)) return true
|
||||
if (Array.isArray(v) && v.some(test)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function eq(c: unknown, x: unknown): boolean {
|
||||
if (x === null) return c === null || c === undefined
|
||||
return deepEqual(c, x)
|
||||
}
|
||||
|
||||
function applyPositive(op: string, c: unknown, arg: unknown): boolean {
|
||||
switch (op) {
|
||||
case '$eq':
|
||||
return eq(c, arg)
|
||||
case '$in':
|
||||
if (!Array.isArray(arg)) throw new TypeError('$in expects an array')
|
||||
return arg.some((x) => eq(c, x))
|
||||
default: {
|
||||
const rc = typeRank(c)
|
||||
if (rc !== typeRank(arg) || rc === 4) return false
|
||||
const d = compareValues(c, arg)
|
||||
return op === '$gt' ? d > 0 : op === '$gte' ? d >= 0 : op === '$lt' ? d < 0 : d <= 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ planner analysis
|
||||
|
||||
export interface RangePred {
|
||||
gt?: IndexValue
|
||||
gte?: IndexValue
|
||||
lt?: IndexValue
|
||||
lte?: IndexValue
|
||||
}
|
||||
|
||||
export interface FieldPred {
|
||||
eq?: IndexValue
|
||||
range?: RangePred
|
||||
}
|
||||
|
||||
/**
|
||||
* Index-usable predicates per field, from the top level and top-level `$and`.
|
||||
* Only scalar `$eq`/implicit equality and same-typed scalar range bounds qualify.
|
||||
*/
|
||||
export function analyze(filter: Filter, out = new Map<string, FieldPred>()): Map<string, FieldPred> {
|
||||
for (const key of Object.keys(filter)) {
|
||||
const cond = filter[key]
|
||||
if (key === '$and') {
|
||||
for (const f of cond as Filter[]) analyze(f, out)
|
||||
continue
|
||||
}
|
||||
if (key.startsWith('$')) continue
|
||||
const pred = out.get(key) ?? {}
|
||||
if (!isOperatorObject(cond)) {
|
||||
if (isScalar(cond)) pred.eq ??= cond
|
||||
} else {
|
||||
if ('$eq' in cond && isScalar(cond.$eq)) pred.eq ??= cond.$eq
|
||||
const range: RangePred = {}
|
||||
let rank = -1
|
||||
let ok = false
|
||||
for (const op of ['$gt', '$gte', '$lt', '$lte'] as const) {
|
||||
if (!(op in cond)) continue
|
||||
const v = cond[op]
|
||||
if (!isScalar(v)) continue
|
||||
const r = typeRank(v)
|
||||
if (rank !== -1 && r !== rank) continue
|
||||
rank = r
|
||||
range[op.slice(1) as keyof RangePred] = v
|
||||
ok = true
|
||||
}
|
||||
if (ok && !pred.range) pred.range = range
|
||||
}
|
||||
if (pred.eq !== undefined || pred.range) out.set(key, pred)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export type { Doc }
|
||||
@@ -1,7 +0,0 @@
|
||||
export * from './types.js'
|
||||
export * from './event-sourced.js'
|
||||
export * from './filter.js'
|
||||
export * from './update.js'
|
||||
export * from './indexes.js'
|
||||
export * from './ulid.js'
|
||||
export { compareValues, deepEqual, type Doc } from './values.js'
|
||||
@@ -1,220 +0,0 @@
|
||||
import { encodeIndexValues, indexValueCodec, type IndexValue } from '../core/encoding.js'
|
||||
import { escapeSegment, RESERVED, succ } from '../core/path.js'
|
||||
import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js'
|
||||
import type { FieldPred, RangePred } from './filter.js'
|
||||
import { indexScalars, resolvePath, typeRank, type Doc } from './values.js'
|
||||
|
||||
export interface DocIndexSpec {
|
||||
/** Default: the fields joined with `,`. */
|
||||
name?: string
|
||||
fields: string[]
|
||||
/** Claims `~u/<name>/<values>` per value tuple. Documents where every field is missing/null are not claimed. */
|
||||
unique?: boolean
|
||||
}
|
||||
|
||||
/** Path pattern (`*` = one segment, `''` = root) → indexes over that node's direct children. */
|
||||
export type DocIndexDeclarations = Record<string, DocIndexSpec[]>
|
||||
|
||||
export interface SortSpec {
|
||||
[field: string]: 1 | -1
|
||||
}
|
||||
|
||||
interface CompiledDocIndex {
|
||||
name: string
|
||||
fields: string[]
|
||||
segs: string[][]
|
||||
unique: boolean
|
||||
/** `escapeSegment(name) + sep` */
|
||||
head: string
|
||||
}
|
||||
|
||||
export interface Plan {
|
||||
index: string
|
||||
gte: string
|
||||
lt: string
|
||||
reverse: boolean
|
||||
/** Index order satisfies the requested sort. */
|
||||
sorted: boolean
|
||||
}
|
||||
|
||||
/** Document indexes over the children of one node. */
|
||||
export class DocIndexes {
|
||||
readonly #list: CompiledDocIndex[]
|
||||
readonly #byName: Map<string, CompiledDocIndex>
|
||||
readonly #enc: (v: IndexValue) => string
|
||||
|
||||
constructor(
|
||||
list: CompiledDocIndex[],
|
||||
/** `<projBase><nodePrefix>~i<sep>` */
|
||||
readonly base: string,
|
||||
/** `<projBase><nodePrefix>~u<sep>` */
|
||||
readonly uniqueBase: string,
|
||||
readonly sep: string,
|
||||
) {
|
||||
this.#list = list
|
||||
this.#byName = new Map(list.map((i) => [i.name, i]))
|
||||
const codec = indexValueCodec(sep)
|
||||
this.#enc = (v) => codec.encode(v)
|
||||
}
|
||||
|
||||
get hasUnique(): boolean {
|
||||
return this.#list.some((i) => i.unique)
|
||||
}
|
||||
|
||||
/** Value tuples a state produces for one index (cartesian over multikey fields). */
|
||||
#tuples(idx: CompiledDocIndex, state: Doc): IndexValue[][] {
|
||||
let tuples: IndexValue[][] = [[]]
|
||||
let multi = 0
|
||||
for (const segs of idx.segs) {
|
||||
const found = resolvePath(state, segs)
|
||||
const scalars =
|
||||
found.length === 0 ? [null] : found.length === 1 && !Array.isArray(found[0]) ? indexScalars(found[0]) : [...new Set(found.flatMap(indexScalars))]
|
||||
if (tuples.length === 1 && scalars.length === 1) {
|
||||
tuples[0]!.push(scalars[0]!)
|
||||
continue
|
||||
}
|
||||
if (scalars.length > 1 && ++multi > 1) throw new TypeError(`Index "${idx.name}": cannot index more than one array field per document`)
|
||||
tuples = tuples.flatMap((t) => scalars.map((s) => [...t, s]))
|
||||
}
|
||||
return tuples
|
||||
}
|
||||
|
||||
entriesFor(escChild: string, state: Doc | undefined, out: Set<string> = new Set()): Set<string> {
|
||||
if (!state) return out
|
||||
for (const idx of this.#list) {
|
||||
const stem = this.base + idx.head
|
||||
for (const t of this.#tuples(idx, state)) out.add(stem + encodeIndexValues(t, this.sep) + this.sep + escChild)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Unique claim key → index name and values. */
|
||||
claimsFor(state: Doc | undefined): Map<string, { index: string; values: IndexValue[] }> {
|
||||
const out = new Map<string, { index: string; values: IndexValue[] }>()
|
||||
if (!state) return out
|
||||
for (const idx of this.#list) {
|
||||
if (!idx.unique) continue
|
||||
for (const t of this.#tuples(idx, state)) {
|
||||
if (t.every((v) => v === null)) continue
|
||||
out.set(this.uniqueBase + idx.head + encodeIndexValues(t, this.sep), { index: idx.name, values: t })
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Escaped child segment of an entry key of index `name`. */
|
||||
childOf(name: string, entryKey: string): string {
|
||||
const idx = this.#byName.get(name)!
|
||||
let pos = this.base.length + idx.head.length
|
||||
for (let i = 0; i < idx.fields.length; i++) pos = entryKey.indexOf(this.sep, pos) + 1
|
||||
return entryKey.slice(pos)
|
||||
}
|
||||
|
||||
/**
|
||||
* Chooses the index covering the most leading equality fields, then at most one range field.
|
||||
* Ties prefer an index whose order satisfies `sort`. With no usable predicate, an index whose
|
||||
* order satisfies `sort` is still used (a full, ordered index scan). Returns `undefined` for a
|
||||
* full projection scan.
|
||||
*/
|
||||
plan(preds: Map<string, FieldPred>, sort: [string, 1 | -1][]): Plan | undefined {
|
||||
let best: { idx: CompiledDocIndex; k: number; range: RangePred | undefined; sorted: boolean; dir: 1 | -1; score: number } | undefined
|
||||
for (const idx of this.#list) {
|
||||
let k = 0
|
||||
while (k < idx.fields.length && preds.get(idx.fields[k]!)?.eq !== undefined) k++
|
||||
const range = k < idx.fields.length ? preds.get(idx.fields[k]!)?.range : undefined
|
||||
// Sort fields constrained by equality are constant; drop them before matching.
|
||||
const eqFields = new Set(idx.fields.slice(0, k))
|
||||
const rest = sort.filter(([f]) => !eqFields.has(f))
|
||||
// Ties are broken by child key in the direction of the first sort field.
|
||||
const dir = rest[0]?.[1] ?? sort[0]?.[1] ?? 1
|
||||
const sorted = rest.length === 0 || (rest.every(([f, d], i) => idx.fields[k + i] === f && d === dir) && rest.length <= idx.fields.length - k)
|
||||
const score = k * 2 + (range ? 1 : 0)
|
||||
if (score === 0 && !(sorted && rest.length > 0)) continue
|
||||
if (!best || score > best.score || (score === best.score && sorted && !best.sorted)) best = { idx, k, range, sorted, dir, score }
|
||||
}
|
||||
if (!best) return undefined
|
||||
const { idx, k, range, sorted, dir } = best
|
||||
const succSep = succ(this.sep)
|
||||
let prefix = this.base + idx.head
|
||||
for (let i = 0; i < k; i++) prefix += this.#enc(preds.get(idx.fields[i]!)!.eq!) + this.sep
|
||||
// prefix ends with sep: the range of everything under it is [prefix, prefix-without-sep + succ(sep)).
|
||||
const all = { gte: prefix, lt: prefix.slice(0, -1) + succSep }
|
||||
let { gte, lt } = all
|
||||
if (range) {
|
||||
const sample = (range.gt ?? range.gte ?? range.lt ?? range.lte) as IndexValue
|
||||
const [lo, hi] = typeBounds(typeRank(sample), this.#enc)
|
||||
gte = prefix + lo
|
||||
lt = hi === undefined ? all.lt : prefix + hi
|
||||
if (range.gte !== undefined) gte = max(gte, prefix + this.#enc(range.gte))
|
||||
if (range.gt !== undefined) gte = max(gte, prefix + this.#enc(range.gt) + succSep)
|
||||
if (range.lt !== undefined) lt = min(lt, prefix + this.#enc(range.lt))
|
||||
if (range.lte !== undefined) lt = min(lt, prefix + this.#enc(range.lte) + succSep)
|
||||
}
|
||||
return { index: idx.name, gte, lt, reverse: sorted && dir === -1, sorted }
|
||||
}
|
||||
}
|
||||
|
||||
/** Encoded `[lower, upper)` of one type class; `undefined` upper = end of index. */
|
||||
function typeBounds(rank: number, enc: (v: IndexValue) => string): [string, string | undefined] {
|
||||
switch (rank) {
|
||||
case 0:
|
||||
return [enc(null), enc(false)]
|
||||
case 1:
|
||||
return [enc(false), enc(-Infinity)]
|
||||
case 2:
|
||||
return [enc(-Infinity), enc('')]
|
||||
default:
|
||||
return [enc(''), undefined]
|
||||
}
|
||||
}
|
||||
|
||||
const max = (a: string, b: string) => (a > b ? a : b)
|
||||
const min = (a: string, b: string) => (a < b ? a : b)
|
||||
|
||||
export class DocIndexRegistry {
|
||||
readonly trie: PatternTrie<CompiledDocIndex[]>
|
||||
readonly empty: boolean
|
||||
readonly hasUnique: boolean
|
||||
|
||||
constructor(
|
||||
decls: DocIndexDeclarations,
|
||||
readonly sep: string,
|
||||
) {
|
||||
this.trie = new PatternTrie(sep)
|
||||
const patterns = Object.keys(decls)
|
||||
const compiled = new Map<string, CompiledDocIndex[]>()
|
||||
for (const p of patterns) {
|
||||
const list = decls[p]!.map((spec): CompiledDocIndex => {
|
||||
if (!spec.fields.length) throw new TypeError(`Index on "${p}" has no fields`)
|
||||
const name = spec.name ?? spec.fields.join(',')
|
||||
return { name, fields: spec.fields, segs: spec.fields.map((f) => f.split('.')), unique: spec.unique === true, head: escapeSegment(name, sep) + sep }
|
||||
})
|
||||
const names = list.map((i) => i.name)
|
||||
const dup = names.find((n, i) => names.indexOf(n) !== i)
|
||||
if (dup) throw new TypeError(`Duplicate index name "${dup}" on "${p}"`)
|
||||
compiled.set(p, list)
|
||||
}
|
||||
for (let i = 0; i < patterns.length; i++) {
|
||||
for (let j = i + 1; j < patterns.length; j++) {
|
||||
const a = patterns[i]!
|
||||
const b = patterns[j]!
|
||||
if (!patternsOverlap(a, b, sep)) continue
|
||||
const bn = new Set(compiled.get(b)!.map((x) => x.name))
|
||||
const clash = compiled.get(a)!.filter((x) => bn.has(x.name))
|
||||
if (clash.length) throw new TypeError(`Index name(s) ${clash.map((c) => c.name).join(', ')} declared on overlapping patterns "${a}" and "${b}"`)
|
||||
}
|
||||
}
|
||||
for (const p of patterns) this.trie.add(p, compiled.get(p)!)
|
||||
this.empty = patterns.length === 0
|
||||
this.hasUnique = [...compiled.values()].some((l) => l.some((i) => i.unique))
|
||||
}
|
||||
|
||||
/** `projPrefix` is the projection-space prefix of the node (`<projBase><nodePrefix>`). */
|
||||
bind(state: TrieState<CompiledDocIndex[]>, projPrefix: string): DocIndexes | undefined {
|
||||
if (this.empty) return undefined
|
||||
const found = PatternTrie.values(state)
|
||||
if (found.length === 0) return undefined
|
||||
const list = found.flatMap((f) => f.value)
|
||||
return new DocIndexes(list, projPrefix + RESERVED.index + this.sep, projPrefix + RESERVED.unique + this.sep, this.sep)
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
import type { StoredEvent } from '../events/types.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js'
|
||||
import type { Filter } from './filter.js'
|
||||
import type { SortSpec } from './indexes.js'
|
||||
import type { DocEvent, UpdateSpec } from './update.js'
|
||||
|
||||
export interface FindOptions {
|
||||
sort?: SortSpec
|
||||
limit?: number
|
||||
skip?: number
|
||||
/** Inclusion (`{ a: 1 }`) or exclusion (`{ a: 0 }`) of dot-path fields. */
|
||||
projection?: Record<string, 0 | 1 | boolean>
|
||||
}
|
||||
|
||||
export interface FindResult<T> {
|
||||
key: string
|
||||
data: T
|
||||
}
|
||||
|
||||
export interface GetOptions {
|
||||
/** Check the stream head and catch the projection up first (one extra scan). */
|
||||
consistent?: boolean
|
||||
}
|
||||
|
||||
export interface UpdateOptions {
|
||||
/** Treat a missing document as `{}`. */
|
||||
upsert?: boolean
|
||||
}
|
||||
|
||||
export interface DocKeysOptions {
|
||||
limit?: number
|
||||
after?: string | number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
/** Document type of a node's children. */
|
||||
export type ChildDoc<S> = ValueOf<ChildOf<S, KeyOf<S>>>
|
||||
|
||||
/**
|
||||
* One node of a document store: a document (its own) and a collection of child documents.
|
||||
* Same navigation and self/child arity rules as `KVNode`; in forms with an optional leading key,
|
||||
* a non-null object first argument selects the self form.
|
||||
*/
|
||||
export interface DocumentNode<S extends Spec = AnySpec> {
|
||||
readonly path: readonly string[]
|
||||
child<K extends KeyOf<S>>(key: K): DocumentNode<ChildOf<S, K>>
|
||||
as<S2 extends Spec>(): DocumentNode<S2>
|
||||
|
||||
get(opts?: GetOptions): Promise<ValueOf<S> | undefined>
|
||||
get<K extends KeyOf<S>>(key: K, opts?: GetOptions): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
|
||||
set(doc: ValueOf<S>): Promise<void>
|
||||
set<K extends KeyOf<S>>(key: K, doc: ValueOf<ChildOf<S, K>>): Promise<void>
|
||||
|
||||
/** Inserts a child with a generated time-sortable id; returns the id. */
|
||||
insert(doc: ChildDoc<S>): Promise<string>
|
||||
|
||||
/** Returns the new state, or `undefined` if the document doesn't exist (and no `upsert`). */
|
||||
update(spec: UpdateSpec, opts?: UpdateOptions): Promise<ValueOf<S> | undefined>
|
||||
update<K extends KeyOf<S>>(key: K, spec: UpdateSpec, opts?: UpdateOptions): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
|
||||
/** Returns whether a document was deleted. */
|
||||
delete(): Promise<boolean>
|
||||
delete<K extends KeyOf<S>>(key: K): Promise<boolean>
|
||||
|
||||
/** Queries direct children. */
|
||||
find(filter?: Filter, opts?: FindOptions): AsyncIterable<FindResult<ChildDoc<S>>>
|
||||
findOne(filter?: Filter, opts?: FindOptions): Promise<FindResult<ChildDoc<S>> | undefined>
|
||||
|
||||
/** Keys of direct children that hold a (non-deleted) document. */
|
||||
keys(opts?: DocKeysOptions): AsyncIterable<KeyString<S>>
|
||||
}
|
||||
|
||||
/** Document store backed by an event log: adds history and point-in-time reads. */
|
||||
export interface EventSourcedDocumentNode<S extends Spec = AnySpec> extends DocumentNode<S> {
|
||||
child<K extends KeyOf<S>>(key: K): EventSourcedDocumentNode<ChildOf<S, K>>
|
||||
as<S2 extends Spec>(): EventSourcedDocumentNode<S2>
|
||||
|
||||
history(): AsyncIterable<StoredEvent<DocEvent>>
|
||||
history<K extends KeyOf<S>>(key: K): AsyncIterable<StoredEvent<DocEvent>>
|
||||
|
||||
/** State as of `version` (`undefined` if missing or deleted then). */
|
||||
getAt(version: number): Promise<ValueOf<S> | undefined>
|
||||
getAt<K extends KeyOf<S>>(key: K, version: number): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
|
||||
/** Rebuilds lagging projections for this node and all descendants; returns how many were repaired. */
|
||||
catchUp(): Promise<number>
|
||||
}
|
||||
|
||||
export type DocumentStorage<S extends Spec = AnySpec> = DocumentNode<S>
|
||||
@@ -1,37 +0,0 @@
|
||||
const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32
|
||||
|
||||
/**
|
||||
* ULID-like id generator: 10 chars of millisecond timestamp + 16 chars of randomness,
|
||||
* lexicographically sortable by creation time. Ids generated within the same millisecond
|
||||
* increment the random part, so they stay strictly increasing per generator.
|
||||
*/
|
||||
export function createIdGenerator(now: () => number = Date.now): () => string {
|
||||
let lastTime = -1
|
||||
const rand = new Uint8Array(16) // one base32 digit (0–31) per byte
|
||||
return () => {
|
||||
let t = now()
|
||||
if (t <= lastTime) {
|
||||
t = lastTime
|
||||
// Increment the random part (base 32, right to left).
|
||||
let i = 15
|
||||
while (i >= 0 && rand[i] === 31) rand[i--] = 0
|
||||
if (i < 0) throw new Error('ULID random overflow within one millisecond')
|
||||
rand[i]!++
|
||||
} else {
|
||||
crypto.getRandomValues(rand)
|
||||
for (let i = 0; i < 16; i++) rand[i]! &= 31
|
||||
// Leave headroom so increments rarely overflow.
|
||||
rand[0]! &= 15
|
||||
lastTime = t
|
||||
}
|
||||
let out = ''
|
||||
for (let i = 9; i >= 0; i--) {
|
||||
out = ALPHABET[t % 32]! + out
|
||||
t = Math.floor(t / 32)
|
||||
}
|
||||
for (let i = 0; i < 16; i++) out += ALPHABET[rand[i]!]
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
export const ulid = createIdGenerator()
|
||||
@@ -1,109 +0,0 @@
|
||||
import { isOperatorObject, matches, matchesField, type Filter } from './filter.js'
|
||||
import { clone, deepEqual, getPath, isPlainObject, setPath, unsetPath, type Doc } from './values.js'
|
||||
|
||||
/** Update operators supported in v1. Paths may use dots. */
|
||||
export interface UpdateSpec {
|
||||
$set?: Record<string, unknown>
|
||||
$unset?: Record<string, unknown>
|
||||
$inc?: Record<string, number>
|
||||
$push?: Record<string, unknown>
|
||||
$pull?: Record<string, unknown>
|
||||
$addToSet?: Record<string, unknown>
|
||||
}
|
||||
|
||||
/** Stored document events: resolved deltas, never operators, so replay is deterministic. */
|
||||
export type DocEvent = { op: 'set'; doc: Doc } | { op: 'patch'; set?: Record<string, unknown>; unset?: string[] } | { op: 'delete' }
|
||||
|
||||
export interface Patch {
|
||||
set: Record<string, unknown>
|
||||
unset: string[]
|
||||
}
|
||||
|
||||
const OPS = new Set(['$set', '$unset', '$inc', '$push', '$pull', '$addToSet'])
|
||||
|
||||
/** Resolves operators against `state` into a patch of final values. */
|
||||
export function resolveUpdate(state: Doc, spec: UpdateSpec): Patch {
|
||||
const keys = Object.keys(spec)
|
||||
if (keys.length === 0) throw new TypeError('Empty update')
|
||||
for (const k of keys) if (!OPS.has(k)) throw new TypeError(`Unsupported update operator ${k}`)
|
||||
|
||||
const patch: Patch = { set: {}, unset: [] }
|
||||
const touched: string[] = []
|
||||
const touch = (path: string) => {
|
||||
for (const t of touched) {
|
||||
if (t === path || t.startsWith(path + '.') || path.startsWith(t + '.')) throw new TypeError(`Update paths conflict: "${t}" and "${path}"`)
|
||||
}
|
||||
touched.push(path)
|
||||
}
|
||||
const arrayAt = (path: string, op: string): unknown[] => {
|
||||
const cur = getPath(state, path)
|
||||
if (cur === undefined || cur === null) return []
|
||||
if (!Array.isArray(cur)) throw new TypeError(`${op} on non-array field "${path}"`)
|
||||
return cur
|
||||
}
|
||||
const eachOf = (v: unknown): unknown[] => (isPlainObject(v) && Object.keys(v).length === 1 && Array.isArray(v.$each) ? (v.$each as unknown[]) : [v])
|
||||
|
||||
for (const [path, v] of Object.entries(spec.$set ?? {})) {
|
||||
touch(path)
|
||||
patch.set[path] = clone(v)
|
||||
}
|
||||
for (const path of Object.keys(spec.$unset ?? {})) {
|
||||
touch(path)
|
||||
patch.unset.push(path)
|
||||
}
|
||||
for (const [path, by] of Object.entries(spec.$inc ?? {})) {
|
||||
touch(path)
|
||||
if (typeof by !== 'number') throw new TypeError(`$inc by non-number at "${path}"`)
|
||||
const cur = getPath(state, path)
|
||||
if (cur !== undefined && cur !== null && typeof cur !== 'number') throw new TypeError(`$inc on non-number field "${path}"`)
|
||||
patch.set[path] = ((cur as number | undefined) ?? 0) + by
|
||||
}
|
||||
for (const [path, v] of Object.entries(spec.$push ?? {})) {
|
||||
touch(path)
|
||||
patch.set[path] = [...clone(arrayAt(path, '$push')), ...clone(eachOf(v))]
|
||||
}
|
||||
for (const [path, v] of Object.entries(spec.$addToSet ?? {})) {
|
||||
touch(path)
|
||||
const next = clone(arrayAt(path, '$addToSet'))
|
||||
for (const item of eachOf(v)) if (!next.some((x) => deepEqual(x, item))) next.push(clone(item))
|
||||
patch.set[path] = next
|
||||
}
|
||||
for (const [path, cond] of Object.entries(spec.$pull ?? {})) {
|
||||
touch(path)
|
||||
patch.set[path] = clone(arrayAt(path, '$pull')).filter((el) => !pullMatches(el, cond))
|
||||
}
|
||||
return patch
|
||||
}
|
||||
|
||||
function pullMatches(el: unknown, cond: unknown): boolean {
|
||||
if (isOperatorObject(cond)) return matchesField([el], cond)
|
||||
if (isPlainObject(cond) && isPlainObject(el)) return matches(el, cond as Filter)
|
||||
return deepEqual(el, cond)
|
||||
}
|
||||
|
||||
/** Applies a resolved patch to a copy of `state`. */
|
||||
export function applyPatch(state: Doc, patch: { set?: Record<string, unknown>; unset?: string[] }): Doc {
|
||||
const next = clone(state)
|
||||
for (const [path, v] of Object.entries(patch.set ?? {})) setPath(next, path, clone(v))
|
||||
for (const path of patch.unset ?? []) unsetPath(next, path)
|
||||
return next
|
||||
}
|
||||
|
||||
/** Folds one event into a state (`undefined` = no document). */
|
||||
export function applyEvent(state: Doc | undefined, e: DocEvent): Doc | undefined {
|
||||
switch (e.op) {
|
||||
case 'set':
|
||||
return clone(e.doc)
|
||||
case 'patch':
|
||||
return applyPatch(state ?? {}, e)
|
||||
case 'delete':
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export function toPatchEvent(p: Patch): DocEvent {
|
||||
const e: { op: 'patch'; set?: Record<string, unknown>; unset?: string[] } = { op: 'patch' }
|
||||
if (Object.keys(p.set).length) e.set = p.set
|
||||
if (p.unset.length) e.unset = p.unset
|
||||
return e
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
import type { IndexValue } from '../core/encoding.js'
|
||||
|
||||
/** Plain JSON object. */
|
||||
export type Doc = Record<string, unknown>
|
||||
|
||||
export function isPlainObject(v: unknown): v is Doc {
|
||||
return typeof v === 'object' && v !== null && !Array.isArray(v)
|
||||
}
|
||||
|
||||
export function isScalar(v: unknown): v is IndexValue {
|
||||
return v === null || typeof v === 'string' || typeof v === 'boolean' || (typeof v === 'number' && !Number.isNaN(v))
|
||||
}
|
||||
|
||||
export function splitPath(path: string): string[] {
|
||||
return path.split('.')
|
||||
}
|
||||
|
||||
/**
|
||||
* Values reachable at a dot path. Arrays are traversed: a non-numeric segment applied to an
|
||||
* array maps over its elements. Missing paths yield no values.
|
||||
*/
|
||||
export function resolvePath(doc: unknown, path: string | readonly string[]): unknown[] {
|
||||
const segs = typeof path === 'string' ? splitPath(path) : path
|
||||
const out: unknown[] = []
|
||||
walk(doc, segs, 0, out)
|
||||
return out
|
||||
}
|
||||
|
||||
function walk(v: unknown, segs: readonly string[], i: number, out: unknown[]): void {
|
||||
if (i === segs.length) {
|
||||
if (v !== undefined) out.push(v)
|
||||
return
|
||||
}
|
||||
const seg = segs[i]!
|
||||
if (Array.isArray(v)) {
|
||||
if (/^\d+$/.test(seg)) walk(v[Number(seg)], segs, i + 1, out)
|
||||
else for (const el of v) if (isPlainObject(el) || Array.isArray(el)) walk(el, segs, i, out)
|
||||
return
|
||||
}
|
||||
if (isPlainObject(v) && Object.hasOwn(v, seg)) walk(v[seg], segs, i + 1, out)
|
||||
}
|
||||
|
||||
/** Value at a dot path without array traversal (numeric segments index arrays). */
|
||||
export function getPath(doc: unknown, path: string): unknown {
|
||||
let v = doc
|
||||
for (const seg of splitPath(path)) {
|
||||
if (Array.isArray(v) && /^\d+$/.test(seg)) v = v[Number(seg)]
|
||||
else if (isPlainObject(v) && Object.hasOwn(v, seg)) v = v[seg]
|
||||
else return undefined
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
/** Sets a dot path in place, creating intermediate objects. */
|
||||
export function setPath(doc: Doc, path: string, value: unknown): void {
|
||||
const segs = splitPath(path)
|
||||
let v: unknown = doc
|
||||
for (let i = 0; i < segs.length - 1; i++) {
|
||||
const seg = segs[i]!
|
||||
const container = v as Record<string, unknown>
|
||||
let next = Array.isArray(container) && /^\d+$/.test(seg) ? container[Number(seg)] : container[seg]
|
||||
if (next === undefined || next === null) {
|
||||
next = {}
|
||||
if (Array.isArray(container)) container[Number(seg)] = next
|
||||
else container[seg] = next
|
||||
} else if (typeof next !== 'object') {
|
||||
throw new TypeError(`Cannot set "${path}": "${segs.slice(0, i + 1).join('.')}" is not an object`)
|
||||
}
|
||||
v = next
|
||||
}
|
||||
const last = segs.at(-1)!
|
||||
if (Array.isArray(v) && /^\d+$/.test(last)) v[Number(last)] = value
|
||||
else (v as Doc)[last] = value
|
||||
}
|
||||
|
||||
export function unsetPath(doc: Doc, path: string): void {
|
||||
const segs = splitPath(path)
|
||||
let v: unknown = doc
|
||||
for (let i = 0; i < segs.length - 1; i++) {
|
||||
v = isPlainObject(v) ? v[segs[i]!] : Array.isArray(v) && /^\d+$/.test(segs[i]!) ? v[Number(segs[i])] : undefined
|
||||
if (typeof v !== 'object' || v === null) return
|
||||
}
|
||||
const last = segs.at(-1)!
|
||||
if (Array.isArray(v) && /^\d+$/.test(last))
|
||||
v[Number(last)] = null // Mongo semantics: keep positions
|
||||
else if (isPlainObject(v)) delete v[last]
|
||||
}
|
||||
|
||||
export function deepEqual(a: unknown, b: unknown): boolean {
|
||||
if (a === b) return true
|
||||
if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false
|
||||
if (Array.isArray(a) !== Array.isArray(b)) return false
|
||||
if (Array.isArray(a)) {
|
||||
const bb = b as unknown[]
|
||||
return a.length === bb.length && a.every((x, i) => deepEqual(x, bb[i]))
|
||||
}
|
||||
const ka = Object.keys(a)
|
||||
const kb = Object.keys(b)
|
||||
return ka.length === kb.length && ka.every((k) => Object.hasOwn(b, k) && deepEqual((a as Doc)[k], (b as Doc)[k]))
|
||||
}
|
||||
|
||||
/** Deep copy of a JSON-compatible value (documents are JSON). */
|
||||
export function clone<T>(v: T): T {
|
||||
if (v === null || typeof v !== 'object') return v
|
||||
if (Array.isArray(v)) return v.map(clone) as T
|
||||
const out: Doc = {}
|
||||
for (const k in v) out[k] = clone((v as Doc)[k])
|
||||
return out as T
|
||||
}
|
||||
|
||||
/** Type class used for bracketing comparisons, in index-encoding order. */
|
||||
export function typeRank(v: unknown): number {
|
||||
if (v === null || v === undefined) return 0
|
||||
if (typeof v === 'boolean') return 1
|
||||
if (typeof v === 'number') return 2
|
||||
if (typeof v === 'string') return 3
|
||||
return 4
|
||||
}
|
||||
|
||||
/**
|
||||
* Total order matching the index encoding for scalars:
|
||||
* `null/missing < false < true < numbers < strings < anything else`.
|
||||
*/
|
||||
export function compareValues(a: unknown, b: unknown): number {
|
||||
const ra = typeRank(a)
|
||||
const rb = typeRank(b)
|
||||
if (ra !== rb) return ra < rb ? -1 : 1
|
||||
if (ra === 1 || ra === 2 || ra === 3) return a === b ? 0 : (a as number) < (b as number) ? -1 : 1
|
||||
if (ra === 4) {
|
||||
const sa = JSON.stringify(a)
|
||||
const sb = JSON.stringify(b)
|
||||
return sa === sb ? 0 : sa < sb ? -1 : 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* The scalar(s) an index stores for a field value: scalars as-is, arrays as their scalar
|
||||
* elements (multikey; empty → null), and missing or non-scalar values as `null`.
|
||||
*/
|
||||
export function indexScalars(v: unknown): IndexValue[] {
|
||||
if (Array.isArray(v)) {
|
||||
const out: IndexValue[] = []
|
||||
for (const el of v) out.push(isScalar(el) ? el : null)
|
||||
return out.length ? out : [null]
|
||||
}
|
||||
return [isScalar(v) ? v : null]
|
||||
}
|
||||
|
||||
/** Sort key for a field under a direction; arrays sort by their min (asc) or max (desc) element. */
|
||||
export function sortKey(doc: unknown, path: string, dir: 1 | -1): unknown {
|
||||
const vals = resolvePath(doc, path)
|
||||
if (vals.length === 0) return null
|
||||
let best: unknown
|
||||
let first = true
|
||||
for (const v of vals) {
|
||||
for (const s of indexScalars(v)) {
|
||||
if (first || compareValues(s, best) * dir < 0) best = s
|
||||
first = false
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/** Base class for all PotentialStorage errors. */
|
||||
export class PotentialStorageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = new.target.name
|
||||
}
|
||||
}
|
||||
|
||||
/** An optimistic-concurrency check failed (stale `expectedVersion` or a lost commit race). */
|
||||
export class ConcurrencyError extends PotentialStorageError {
|
||||
constructor(
|
||||
readonly stream: string,
|
||||
readonly expectedVersion: number | 'any',
|
||||
readonly actualVersion: number,
|
||||
) {
|
||||
super(`Concurrency conflict on "${stream}": expected version ${expectedVersion}, actual ${actualVersion}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** A unique index already holds the given value for another key. */
|
||||
export class UniqueConstraintError extends PotentialStorageError {
|
||||
constructor(
|
||||
readonly index: string,
|
||||
readonly values: readonly unknown[],
|
||||
readonly existingKey?: string,
|
||||
) {
|
||||
super(`Unique index "${index}" already contains ${JSON.stringify(values)}${existingKey === undefined ? '' : ` (key "${existingKey}")`}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** The driver lacks an optional capability required by the requested feature. */
|
||||
export class CapabilityError extends PotentialStorageError {
|
||||
constructor(
|
||||
readonly capability: string,
|
||||
detail?: string,
|
||||
) {
|
||||
super(`Driver lacks capability "${capability}"${detail ? `: ${detail}` : ''}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A key argument is invalid: passed but `undefined`/`null`, empty, or otherwise unusable.
|
||||
* Extends `TypeError` so callers can catch either.
|
||||
*/
|
||||
export class InvalidKeyError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'InvalidKeyError'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueAt, ValueOf } from '../utils/schema.js'
|
||||
|
||||
export interface ReadOptions {
|
||||
/** Start after this event id (exclusive), in the direction of travel. */
|
||||
after?: string
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
export interface StoredEvent<E> {
|
||||
/** Unique, time-ordered id (see the implementation for its format). */
|
||||
id: string
|
||||
/** Append time (ms since epoch). */
|
||||
at: number
|
||||
data: E
|
||||
}
|
||||
|
||||
export interface StreamKeysOptions {
|
||||
limit?: number
|
||||
after?: string | number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Append-only event streams. Every node is a stream (its own events) and a parent of child
|
||||
* streams. Implementations provide three primitives over stream paths; the user-facing,
|
||||
* self-or-child API is defined here once.
|
||||
*
|
||||
* Appends never conflict: concurrent writers each add their own events, and every reader sees all
|
||||
* of them in the same (time) order. Folding events into state is where concurrent changes meet.
|
||||
*/
|
||||
export abstract class BaseEventStorage<S extends Spec = AnySpec> {
|
||||
/** Unescaped path of this node (`[]` for the root, which is not a stream). */
|
||||
readonly path: readonly string[] = []
|
||||
|
||||
/** Appends events in order; returns their ids. */
|
||||
abstract appendTo(stream: readonly string[], events: unknown[]): Promise<string[]>
|
||||
/** Events in id (time) order. */
|
||||
abstract readFrom(stream: readonly string[], options: ReadOptions): AsyncIterable<StoredEvent<unknown>>
|
||||
abstract streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncIterable<string>
|
||||
|
||||
child<K extends KeyOf<S>>(key: K): BaseEventStorage<ChildOf<S, K>> {
|
||||
const node = Object.create(this, { path: { value: [...this.path, Keys.segment(key)], enumerable: true } }) as BaseEventStorage<ChildOf<S, K>>
|
||||
// Frozen: child nodes share the root's state through the prototype, so an implementation that
|
||||
// reassigns a field (`this.x = ...`) while running on a child would silently shadow it.
|
||||
// Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw.
|
||||
Object.freeze(node)
|
||||
return node
|
||||
}
|
||||
|
||||
as<S2 extends Spec>(): BaseEventStorage<S2> {
|
||||
return this as unknown as BaseEventStorage<S2>
|
||||
}
|
||||
|
||||
/** `append(events)` to this stream; `append(key, events)` to a child stream. Returns the event ids. */
|
||||
append<K extends KeyOf<S>>(...args: [events: ValueOf<S>[]] | [key: K, events: ValueOf<ChildOf<S, K>>[]]): Promise<string[]> {
|
||||
const hasKey = !Array.isArray(args[0])
|
||||
const events = args[hasKey ? 1 : 0]
|
||||
if (!Array.isArray(events)) return Promise.reject(new TypeError('append() expects an array of events'))
|
||||
return this.withStream(hasKey, args[0], (stream) => this.appendTo(stream, events))
|
||||
}
|
||||
|
||||
/** `read(options?)` this stream; `read(key, options?)` a child stream. */
|
||||
async *read<K extends KeyOf<S>>(...args: [options?: ReadOptions] | [key: K, options?: ReadOptions]): AsyncGenerator<StoredEvent<ValueAt<S, K>>> {
|
||||
const hasKey = typeof args[0] !== 'object' && args.length > 0
|
||||
const stream = this.streamPath(hasKey, args[0])
|
||||
yield* this.readFrom(stream, ((hasKey ? args[1] : args[0]) ?? {}) as ReadOptions) as AsyncIterable<StoredEvent<ValueAt<S, K>>>
|
||||
}
|
||||
|
||||
/** Keys of child streams. */
|
||||
keys(options: StreamKeysOptions = {}): AsyncIterable<KeyString<S>> {
|
||||
return this.streamsOf(this.path, options) as AsyncIterable<KeyString<S>>
|
||||
}
|
||||
|
||||
private withStream<T>(hasKey: boolean, key: unknown, fn: (stream: readonly string[]) => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return fn(this.streamPath(hasKey, key))
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
}
|
||||
}
|
||||
|
||||
private streamPath(hasKey: boolean, key: unknown): readonly string[] {
|
||||
if (hasKey) return [...this.path, Keys.segment(key)]
|
||||
if (this.path.length === 0) throw new TypeError('The root node is not a stream')
|
||||
return this.path
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { BaseKVStorage } from '../kv-storage/base-kv-storage.js'
|
||||
import { ExtendedJson } from '../utils/extended-json.js'
|
||||
import { Hex } from '../utils/hex.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { BaseEventStorage, type ReadOptions, type StoredEvent, type StreamKeysOptions } from './base-event-storage.js'
|
||||
|
||||
export interface EventStorageKVOptions {
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
// Types only: children of an EventStorageKV are EventStorageKV nodes too (child() creates them with
|
||||
// Object.create, so they share this class's prototype).
|
||||
export interface EventStorageKV<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): EventStorageKV<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* Event storage on any KV storage. Each event is one plain write:
|
||||
* `<stream>/~e/<time:15>_<hash>` → the event, where `hash` is the SHA-256 (hex) of the time and
|
||||
* the event's canonical Extended JSON.
|
||||
*
|
||||
* - Concurrent appends from different writers get different keys, so all are kept.
|
||||
* - Keys sort by time, so a stream reads in time order; equal times order by hash, the same way
|
||||
* for every reader. One instance never reuses a time, so its own appends keep their order.
|
||||
* - The key can be recomputed from the content, and writing the same event again writes the same
|
||||
* key (e.g. a retried request), so it can't be duplicated.
|
||||
*/
|
||||
export class EventStorageKV<S extends Spec = AnySpec> extends BaseEventStorage<S> {
|
||||
private readonly now: () => number
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state = { lastTime: 0 }
|
||||
|
||||
constructor(
|
||||
private readonly kv: BaseKVStorage,
|
||||
options: EventStorageKVOptions = {},
|
||||
) {
|
||||
super()
|
||||
this.now = options.now ?? Date.now
|
||||
}
|
||||
|
||||
async appendTo(stream: readonly string[], events: unknown[]): Promise<string[]> {
|
||||
const base = this.eventsPrefix(stream)
|
||||
const entries = await Promise.all(
|
||||
events.map(async (data) => {
|
||||
this.state.lastTime = Math.max(this.now(), this.state.lastTime + 1)
|
||||
const id = await EventStorageKV.id(this.state.lastTime, data)
|
||||
return { key: base + id, value: data }
|
||||
}),
|
||||
)
|
||||
await this.kv.writeMany(entries)
|
||||
return entries.map(({ key }) => key.slice(base.length))
|
||||
}
|
||||
|
||||
async *readFrom(stream: readonly string[], options: ReadOptions): AsyncGenerator<StoredEvent<unknown>> {
|
||||
const base = this.eventsPrefix(stream)
|
||||
const { after, reverse = false, limit } = options
|
||||
// `after` is exclusive: start at it and skip it.
|
||||
const bounds = reverse
|
||||
? { gte: base, lt: after === undefined ? base.slice(0, -1) + Keys.afterSeparator : base + after }
|
||||
: { gte: base + (after ?? ''), lt: base.slice(0, -1) + Keys.afterSeparator }
|
||||
let remaining = limit ?? Infinity
|
||||
for await (const [key, data] of this.kv.scan({ ...bounds, reverse, values: true })) {
|
||||
if (remaining <= 0) return
|
||||
const id = key.slice(base.length)
|
||||
if (id === after) continue
|
||||
remaining--
|
||||
yield { id, at: Number(id.slice(0, 15)), data }
|
||||
}
|
||||
}
|
||||
|
||||
async *streamsOf(parent: readonly string[], options: StreamKeysOptions): AsyncGenerator<string> {
|
||||
yield* parent.reduce<BaseKVStorage>((node, segment) => node.child(segment), this.kv).keys(options)
|
||||
}
|
||||
|
||||
/** `<time:15>_<sha256(time + canonical data)>`. */
|
||||
static async id(time: number, data: unknown): Promise<string> {
|
||||
const stamp = String(time).padStart(15, '0')
|
||||
const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(stamp + ExtendedJson.canonical(data)))
|
||||
return `${stamp}_${Hex.encode(new Uint8Array(digest))}`
|
||||
}
|
||||
|
||||
private eventsPrefix(stream: readonly string[]): string {
|
||||
return `${this.kv.prefix}${Keys.join(stream)}/~e/`
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export * from './types.js'
|
||||
export * from './kv.js'
|
||||
@@ -1,289 +0,0 @@
|
||||
import { LRU } from '../core/lru.js'
|
||||
import { KeyedMutex } from '../core/mutex.js'
|
||||
import { escapeSegment, RESERVED, succ, toSegment } from '../core/path.js'
|
||||
import { ConcurrencyError } from '../errors.js'
|
||||
import type { KVDriver } from '../kv/driver.js'
|
||||
import type { KVNode } from '../kv/node.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js'
|
||||
import type { AppendOptions, AppendResult, EventNode, ReadOptions, StoredEvent, StreamKeysOptions } from './types.js'
|
||||
|
||||
export interface KVEventStorageOptions {
|
||||
/** Max streams whose head is cached. Default 10 000. */
|
||||
headCacheSize?: number
|
||||
/** Retries after a commit collision when `expectedVersion` is `'any'`. Default 5. */
|
||||
maxRetries?: number
|
||||
/** Clock for commit timestamps. Default `Date.now`. */
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/** Stored value of one commit key. */
|
||||
export interface CommitRecord<E = unknown> {
|
||||
firstVersion: number
|
||||
at: number
|
||||
events: E[]
|
||||
}
|
||||
|
||||
interface Head {
|
||||
commit: number
|
||||
version: number
|
||||
}
|
||||
|
||||
interface Stream {
|
||||
/** Stream's own key (escaped); identifies it in the cache and in errors. */
|
||||
key: string
|
||||
/** `<key>/~e/` — commit keys are `base + pad12(seq)`. */
|
||||
base: string
|
||||
/** Exclusive upper bound of commit keys. */
|
||||
end: string
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
interface EventContext {
|
||||
readonly driver: KVDriver
|
||||
readonly sep: string
|
||||
readonly heads: LRU<string, Head>
|
||||
readonly mutex: KeyedMutex
|
||||
readonly maxRetries: number
|
||||
readonly now: () => number
|
||||
}
|
||||
|
||||
const SEQ_WIDTH = 12
|
||||
const SEARCH_FANOUT = 8
|
||||
const EMPTY_HEAD: Head = { commit: 0, version: 0 }
|
||||
|
||||
const pad = (seq: number) => String(seq).padStart(SEQ_WIDTH, '0')
|
||||
|
||||
/**
|
||||
* Event node over a `KVNode`. Each append is one commit key, densely numbered:
|
||||
* `<stream>/~e/<commitSeq:12>` → `{ firstVersion, at, events }`. Commits are claimed with
|
||||
* `setIfAbsent`, so any stale writer collides with the existing key. Without `setIfAbsent`,
|
||||
* appends are serialised by an in-process per-stream mutex (single-writer only).
|
||||
*/
|
||||
export class KVEventNode<S extends Spec = AnySpec> implements EventNode<S> {
|
||||
readonly #ctx: EventContext
|
||||
readonly #kv: KVNode<AnySpec>
|
||||
|
||||
/** @internal Use `KVEventStorage` and `child()`. */
|
||||
constructor(ctx: EventContext, kv: KVNode<AnySpec>) {
|
||||
this.#ctx = ctx
|
||||
this.#kv = kv
|
||||
}
|
||||
|
||||
get path(): readonly string[] {
|
||||
return this.#kv.path
|
||||
}
|
||||
|
||||
/** The underlying KV node. */
|
||||
get kv(): KVNode<AnySpec> {
|
||||
return this.#kv
|
||||
}
|
||||
|
||||
child<K extends KeyOf<S>>(key: K): KVEventNode<ChildOf<S, K>> {
|
||||
return new KVEventNode(this.#ctx, this.#kv.child(key as string).as<AnySpec>())
|
||||
}
|
||||
|
||||
as<S2 extends Spec>(): KVEventNode<S2> {
|
||||
return this as unknown as KVEventNode<S2>
|
||||
}
|
||||
|
||||
append(events: readonly ValueOf<S>[], opts?: AppendOptions): Promise<AppendResult>
|
||||
append<K extends KeyOf<S>>(key: K, events: readonly ValueOf<ChildOf<S, K>>[], opts?: AppendOptions): Promise<AppendResult>
|
||||
async append(...args: unknown[]): Promise<AppendResult> {
|
||||
const self = Array.isArray(args[0])
|
||||
const stream = self ? this.#self() : this.#child(args[0])
|
||||
const events = args[self ? 0 : 1]
|
||||
const opts = (args[self ? 1 : 2] ?? {}) as AppendOptions
|
||||
if (!Array.isArray(events)) throw new TypeError('append() expects an array of events')
|
||||
const expected = opts.expectedVersion ?? 'any'
|
||||
if (expected !== 'any' && !(Number.isInteger(expected) && expected >= 0)) throw new TypeError(`Invalid expectedVersion ${String(expected)}`)
|
||||
const hint = expected !== 'any' && Number.isInteger(opts.expectedCommit) ? { commit: opts.expectedCommit!, version: expected } : undefined
|
||||
return this.#append(stream, events, expected, hint)
|
||||
}
|
||||
|
||||
read(opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<S>>>
|
||||
read<K extends KeyOf<S>>(key: K, opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<ChildOf<S, K>>>>
|
||||
async *read(...args: unknown[]): AsyncGenerator<StoredEvent<any>> {
|
||||
const self = args.length === 0 || (typeof args[0] === 'object' && args[0] !== null)
|
||||
const stream = self ? this.#self() : this.#child(args[0])
|
||||
yield* this.#read(stream, ((self ? args[0] : args[1]) ?? {}) as ReadOptions)
|
||||
}
|
||||
|
||||
version(): Promise<number>
|
||||
version<K extends KeyOf<S>>(key: K): Promise<number>
|
||||
async version(...args: unknown[]): Promise<number> {
|
||||
const stream = args.length === 0 ? this.#self() : this.#child(args[0])
|
||||
return (await this.#loadHead(stream)).version
|
||||
}
|
||||
|
||||
keys(opts: StreamKeysOptions = {}): AsyncIterable<KeyString<S>> {
|
||||
return this.#kv.keys(opts) as AsyncIterable<KeyString<S>>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- internals
|
||||
|
||||
#self(): Stream {
|
||||
if (this.#kv.isRoot) throw new TypeError('The root node is not a stream')
|
||||
return this.#streamAt(this.#kv.selfKey)
|
||||
}
|
||||
|
||||
#child(key: unknown): Stream {
|
||||
return this.#streamAt(this.#kv.prefix + escapeSegment(toSegment(key), this.#ctx.sep))
|
||||
}
|
||||
|
||||
#streamAt(key: string): Stream {
|
||||
const sep = this.#ctx.sep
|
||||
const head = key + sep + RESERVED.events
|
||||
return { key, base: head + sep, end: head + succ(sep) }
|
||||
}
|
||||
|
||||
async #loadHead(s: Stream): Promise<Head> {
|
||||
for await (const [key, value] of this.#ctx.driver.scan({ gte: s.base, lt: s.end, reverse: true, limit: 1, values: true })) {
|
||||
const c = value as CommitRecord
|
||||
const head = { commit: Number(key.slice(s.base.length)), version: c.firstVersion + c.events.length - 1 }
|
||||
this.#ctx.heads.set(s.key, head)
|
||||
return head
|
||||
}
|
||||
return EMPTY_HEAD
|
||||
}
|
||||
|
||||
async #append(s: Stream, events: readonly unknown[], expected: number | 'any', hint?: Head): Promise<AppendResult> {
|
||||
const { driver, heads, maxRetries } = this.#ctx
|
||||
if (events.length === 0) {
|
||||
const head = await this.#loadHead(s)
|
||||
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
||||
return { version: head.version, commit: head.commit }
|
||||
}
|
||||
if (!driver.setIfAbsent) return this.#ctx.mutex.run(s.key, () => this.#appendSingleWriter(s, events, expected))
|
||||
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
let head = heads.get(s.key) ?? (attempt === 0 ? hint : undefined)
|
||||
if (!head) head = await this.#loadHead(s)
|
||||
else if (expected !== 'any' && head.version !== expected) head = await this.#loadHead(s) // cache may be stale
|
||||
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
||||
|
||||
const seq = head.commit + 1
|
||||
const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] }
|
||||
const [written] = await driver.setIfAbsent([[s.base + pad(seq), record]])
|
||||
if (written) {
|
||||
const next = { commit: seq, version: head.version + events.length }
|
||||
heads.set(s.key, next)
|
||||
return next
|
||||
}
|
||||
// Lost the race for this commit number.
|
||||
heads.delete(s.key)
|
||||
if (expected !== 'any' || attempt >= maxRetries) {
|
||||
throw new ConcurrencyError(s.key, expected, (await this.#loadHead(s)).version)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #appendSingleWriter(s: Stream, events: readonly unknown[], expected: number | 'any'): Promise<AppendResult> {
|
||||
const head = this.#ctx.heads.get(s.key) ?? (await this.#loadHead(s))
|
||||
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
||||
const seq = head.commit + 1
|
||||
const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] }
|
||||
await this.#ctx.driver.setMany([[s.base + pad(seq), record]])
|
||||
const next = { commit: seq, version: head.version + events.length }
|
||||
this.#ctx.heads.set(s.key, next)
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
* Sequence number of the commit containing `version`, or 0 if the stream hasn't reached it.
|
||||
* Commit `n` has `firstVersion >= n`, so the answer is at most `version`: probe that first
|
||||
* (one round trip when commits hold single events), then k-ary search with `getMany` probes.
|
||||
*/
|
||||
async #locate(s: Stream, version: number): Promise<number> {
|
||||
const { driver } = this.#ctx
|
||||
const [guess] = (await driver.getMany([s.base + pad(version)])) as (CommitRecord | undefined)[]
|
||||
if (guess && guess.firstVersion <= version) return version
|
||||
let hi: number
|
||||
if (guess) hi = version - 1
|
||||
else {
|
||||
const head = await this.#loadHead(s)
|
||||
if (head.version < version) return 0
|
||||
hi = head.commit
|
||||
}
|
||||
let lo = 1 // commit 1 starts at version 1
|
||||
while (lo < hi) {
|
||||
const probes: number[] = []
|
||||
for (let i = 1; i <= SEARCH_FANOUT; i++) {
|
||||
const p = lo + Math.ceil(((hi - lo) * i) / SEARCH_FANOUT)
|
||||
if (p !== probes.at(-1)) probes.push(p)
|
||||
}
|
||||
const found = (await driver.getMany(probes.map((p) => s.base + pad(p)))) as (CommitRecord | undefined)[]
|
||||
for (let i = 0; i < probes.length; i++) {
|
||||
const c = found[i]
|
||||
if (c && c.firstVersion <= version) lo = probes[i]!
|
||||
else {
|
||||
hi = probes[i]! - 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return lo
|
||||
}
|
||||
|
||||
async *#read(s: Stream, opts: ReadOptions): AsyncGenerator<StoredEvent<unknown>> {
|
||||
const from = Math.max(1, opts.from ?? 1)
|
||||
const to = opts.to ?? Infinity
|
||||
let remaining = opts.limit ?? Infinity
|
||||
if (to < from || remaining <= 0) return
|
||||
const { driver } = this.#ctx
|
||||
|
||||
if (!opts.reverse) {
|
||||
let gte = s.base
|
||||
if (from > 1) {
|
||||
const seq = await this.#locate(s, from)
|
||||
if (seq === 0) return
|
||||
gte = s.base + pad(seq)
|
||||
}
|
||||
for await (const [key, value] of driver.scan({ gte, lt: s.end, values: true })) {
|
||||
const c = value as CommitRecord
|
||||
const commit = Number(key.slice(s.base.length))
|
||||
for (let i = 0; i < c.events.length; i++) {
|
||||
const version = c.firstVersion + i
|
||||
if (version < from) continue
|
||||
if (version > to) return
|
||||
yield { version, commit, at: c.at, data: c.events[i] }
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
let lt = s.end
|
||||
if (to !== Infinity) {
|
||||
const seq = await this.#locate(s, to)
|
||||
if (seq !== 0) lt = s.base + pad(seq + 1)
|
||||
}
|
||||
for await (const [key, value] of driver.scan({ gte: s.base, lt, reverse: true, values: true })) {
|
||||
const c = value as CommitRecord
|
||||
const commit = Number(key.slice(s.base.length))
|
||||
for (let i = c.events.length - 1; i >= 0; i--) {
|
||||
const version = c.firstVersion + i
|
||||
if (version > to) continue
|
||||
if (version < from) return
|
||||
yield { version, commit, at: c.at, data: c.events[i] }
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Event storage over a `KVStorage` (or any `KVNode` as root). */
|
||||
export class KVEventStorage<S extends Spec = AnySpec> extends KVEventNode<S> {
|
||||
constructor(kv: KVNode<AnySpec>, opts: KVEventStorageOptions = {}) {
|
||||
super(
|
||||
{
|
||||
driver: kv.driver,
|
||||
sep: kv.separator,
|
||||
heads: new LRU(opts.headCacheSize ?? 10_000),
|
||||
mutex: new KeyedMutex(),
|
||||
maxRetries: opts.maxRetries ?? 5,
|
||||
now: opts.now ?? Date.now,
|
||||
},
|
||||
kv,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js'
|
||||
|
||||
export interface AppendOptions {
|
||||
/** Version the stream must be at (0 = empty). Default `'any'`. */
|
||||
expectedVersion?: number | 'any'
|
||||
/**
|
||||
* Commit sequence of the head at `expectedVersion`, if the caller knows it (e.g. from a
|
||||
* projection). Lets a cold append skip the head lookup. Must come from the stream itself: a
|
||||
* stale hint only causes a collision, but an invented one could leave a gap. Ignored unless
|
||||
* `expectedVersion` is a number; backends may ignore it.
|
||||
*/
|
||||
expectedCommit?: number
|
||||
}
|
||||
|
||||
export interface AppendResult {
|
||||
/** Stream version after the append (the last event's version). */
|
||||
version: number
|
||||
/** Commit sequence number of this append (the current head commit when no events were given). */
|
||||
commit: number
|
||||
}
|
||||
|
||||
export interface ReadOptions {
|
||||
/** First version to read (inclusive). Default 1. */
|
||||
from?: number
|
||||
/** Last version to read (inclusive). Default: the head. */
|
||||
to?: number
|
||||
reverse?: boolean
|
||||
limit?: number
|
||||
}
|
||||
|
||||
export interface StoredEvent<E> {
|
||||
version: number
|
||||
commit: number
|
||||
/** Commit timestamp (ms since epoch). */
|
||||
at: number
|
||||
data: E
|
||||
}
|
||||
|
||||
export interface StreamKeysOptions {
|
||||
limit?: number
|
||||
after?: string | number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* One node of an event store: a stream (its own events) and a container of child streams.
|
||||
* Same navigation and self/child arity rules as `KVNode`.
|
||||
*/
|
||||
export interface EventNode<S extends Spec = AnySpec> {
|
||||
readonly path: readonly string[]
|
||||
child<K extends KeyOf<S>>(key: K): EventNode<ChildOf<S, K>>
|
||||
as<S2 extends Spec>(): EventNode<S2>
|
||||
|
||||
append(events: readonly ValueOf<S>[], opts?: AppendOptions): Promise<AppendResult>
|
||||
append<K extends KeyOf<S>>(key: K, events: readonly ValueOf<ChildOf<S, K>>[], opts?: AppendOptions): Promise<AppendResult>
|
||||
|
||||
read(opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<S>>>
|
||||
read<K extends KeyOf<S>>(key: K, opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<ChildOf<S, K>>>>
|
||||
|
||||
/** Current version (0 = empty). Always reads storage. */
|
||||
version(): Promise<number>
|
||||
version<K extends KeyOf<S>>(key: K): Promise<number>
|
||||
|
||||
/** Child stream keys. */
|
||||
keys(opts?: StreamKeysOptions): AsyncIterable<KeyString<S>>
|
||||
}
|
||||
|
||||
/** The root of an event store. Native backends (EventStoreDB, Kafka, …) implement this too. */
|
||||
export type EventStorage<S extends Spec = AnySpec> = EventNode<S>
|
||||
@@ -1,115 +0,0 @@
|
||||
import { BaseKVDriver, JsonCodec, type BatchOp, type Codec, type ScanOptions } from '../kv/driver.js'
|
||||
import type { HttpCapabilities } from './handler.js'
|
||||
|
||||
export interface HttpKVDriverOptions {
|
||||
/** Base URL of a `createKVHttpHandler` endpoint, e.g. `https://host/kv`. */
|
||||
url: string
|
||||
/** Optional capabilities the server's driver supports. Default: all. `connect()` discovers them. */
|
||||
capabilities?: Partial<HttpCapabilities>
|
||||
codec?: Codec
|
||||
headers?: Record<string, string>
|
||||
fetch?: typeof fetch
|
||||
}
|
||||
|
||||
/**
|
||||
* Driver client for `createKVHttpHandler`. Optional methods exist only when the server
|
||||
* supports them, so stores' feature detection works across HTTP.
|
||||
*/
|
||||
export class HttpKVDriver extends BaseKVDriver {
|
||||
readonly #url: string
|
||||
readonly #codec: Codec
|
||||
readonly #headers: Record<string, string>
|
||||
readonly #fetch: typeof fetch
|
||||
readonly #nativeDeleteRange: boolean
|
||||
|
||||
setIfAbsent?: (entries: [string, unknown][]) => Promise<boolean[]>
|
||||
batch?: (ops: BatchOp[]) => Promise<void>
|
||||
|
||||
/** Creates a client after asking the server which optional capabilities it has. */
|
||||
static async connect(opts: Omit<HttpKVDriverOptions, 'capabilities'>): Promise<HttpKVDriver> {
|
||||
const probe = new HttpKVDriver({ ...opts, capabilities: {} })
|
||||
const capabilities = (await probe.#request('GET', '/capabilities')) as HttpCapabilities
|
||||
return new HttpKVDriver({ ...opts, capabilities })
|
||||
}
|
||||
|
||||
constructor(opts: HttpKVDriverOptions) {
|
||||
super()
|
||||
this.#url = opts.url.replace(/\/$/, '')
|
||||
this.#codec = opts.codec ?? JsonCodec
|
||||
this.#headers = opts.headers ?? {}
|
||||
this.#fetch = opts.fetch ?? globalThis.fetch.bind(globalThis)
|
||||
const caps = { setIfAbsent: true, batch: true, deleteRange: true, ...opts.capabilities }
|
||||
this.#nativeDeleteRange = caps.deleteRange
|
||||
if (caps.setIfAbsent) this.setIfAbsent = async (entries) => ((await this.#request('POST', '/setIfAbsent', { entries })) as { written: boolean[] }).written
|
||||
if (caps.batch) this.batch = async (ops) => void (await this.#request('POST', '/batch', { ops }))
|
||||
}
|
||||
|
||||
async getMany(keys: string[]): Promise<unknown[]> {
|
||||
if (keys.length === 0) return []
|
||||
const { values, missing } = (await this.#request('POST', '/getMany', { keys })) as { values: unknown[]; missing: number[] }
|
||||
for (const i of missing) values[i] = undefined
|
||||
return values
|
||||
}
|
||||
|
||||
async setMany(entries: [string, unknown][]): Promise<void> {
|
||||
if (entries.length) await this.#request('POST', '/setMany', { entries })
|
||||
}
|
||||
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (keys.length) await this.#request('POST', '/deleteMany', { keys })
|
||||
}
|
||||
|
||||
override async deleteRange(gte: string, lt: string): Promise<void> {
|
||||
if (!this.#nativeDeleteRange) return super.deleteRange(gte, lt)
|
||||
await this.#request('POST', '/deleteRange', { gte, lt })
|
||||
}
|
||||
|
||||
async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> {
|
||||
if (opts.limit !== undefined && opts.limit <= 0) return
|
||||
const q = new URLSearchParams()
|
||||
if (opts.gte !== undefined) q.set('gte', opts.gte)
|
||||
if (opts.lt !== undefined) q.set('lt', opts.lt)
|
||||
if (opts.limit !== undefined && opts.limit !== Infinity) q.set('limit', String(opts.limit))
|
||||
if (opts.reverse) q.set('reverse', 'true')
|
||||
if (opts.values) q.set('values', 'true')
|
||||
const res = await this.#fetch(`${this.#url}/scan?${q}`, { headers: this.#headers })
|
||||
if (!res.ok || !res.body) throw await httpError(res)
|
||||
const reader = res.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buf = ''
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
buf += decoder.decode(value, { stream: true })
|
||||
let nl: number
|
||||
while ((nl = buf.indexOf('\n')) !== -1) {
|
||||
const line = buf.slice(0, nl)
|
||||
buf = buf.slice(nl + 1)
|
||||
const [key, v] = this.#codec.decode(line) as [string, unknown?]
|
||||
yield [key, v]
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
async #request(method: 'GET' | 'POST', path: string, body?: unknown): Promise<unknown> {
|
||||
const res = await this.#fetch(this.#url + path, {
|
||||
method,
|
||||
headers: { ...this.#headers, ...(body === undefined ? {} : { 'content-type': 'application/json' }) },
|
||||
body: body === undefined ? undefined : this.#codec.encode(body),
|
||||
})
|
||||
if (!res.ok) throw await httpError(res)
|
||||
return res.status === 204 ? undefined : this.#codec.decode(await res.text())
|
||||
}
|
||||
}
|
||||
|
||||
async function httpError(res: Response): Promise<Error> {
|
||||
let detail = ''
|
||||
try {
|
||||
detail = await res.text()
|
||||
} catch {}
|
||||
return new Error(`HTTP ${res.status}: ${detail}`)
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
import { JsonCodec, type BatchOp, type Codec, type KVDriver, type ScanOptions } from '../kv/driver.js'
|
||||
|
||||
export interface KVHttpHandlerOptions {
|
||||
/** Serialises request/response bodies and NDJSON scan lines. Must not emit newlines. Default JSON. */
|
||||
codec?: Codec
|
||||
/** Path prefix to strip, e.g. `/kv`. Default `''`. */
|
||||
basePath?: string
|
||||
}
|
||||
|
||||
export interface HttpCapabilities {
|
||||
setIfAbsent: boolean
|
||||
batch: boolean
|
||||
deleteRange: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Framework-agnostic HTTP front for a driver (`Request → Response`).
|
||||
*
|
||||
* - `POST /getMany {keys}` → `{values, missing}` (`missing` = indices of absent keys)
|
||||
* - `POST /setMany {entries}`, `POST /deleteMany {keys}`, `POST /batch {ops}` → 204
|
||||
* - `POST /setIfAbsent {entries}` → `{written}`
|
||||
* - `POST /deleteRange {gte, lt}` → 204
|
||||
* - `GET /scan?gte<&limit&reverse&values` → NDJSON, one `[key]` or `[key, value]` per line
|
||||
* - `GET /capabilities` → `{setIfAbsent, batch, deleteRange}`
|
||||
*/
|
||||
export function createKVHttpHandler(driver: KVDriver, opts: KVHttpHandlerOptions = {}): (req: Request) => Promise<Response> {
|
||||
const codec = opts.codec ?? JsonCodec
|
||||
const base = opts.basePath ?? ''
|
||||
const reply = (body: unknown, status = 200) => new Response(codec.encode(body), { status, headers: { 'content-type': 'application/json' } })
|
||||
const noContent = () => new Response(null, { status: 204 })
|
||||
const fail = (status: number, message: string) => reply({ error: message }, status)
|
||||
|
||||
return async (req) => {
|
||||
const url = new URL(req.url)
|
||||
if (!url.pathname.startsWith(base)) return fail(404, 'Not found')
|
||||
const route = `${req.method} ${url.pathname.slice(base.length)}`
|
||||
try {
|
||||
const body = async <T>() => codec.decode(await req.text()) as T
|
||||
switch (route) {
|
||||
case 'POST /getMany': {
|
||||
const { keys } = await body<{ keys: string[] }>()
|
||||
const found = await driver.getMany(keys)
|
||||
const missing: number[] = []
|
||||
const values = found.map((v, i) => (v === undefined ? (missing.push(i), null) : v))
|
||||
return reply({ values, missing })
|
||||
}
|
||||
case 'POST /setMany':
|
||||
await driver.setMany((await body<{ entries: [string, unknown][] }>()).entries)
|
||||
return noContent()
|
||||
case 'POST /deleteMany':
|
||||
await driver.deleteMany((await body<{ keys: string[] }>()).keys)
|
||||
return noContent()
|
||||
case 'POST /setIfAbsent':
|
||||
if (!driver.setIfAbsent) return fail(501, 'setIfAbsent not supported')
|
||||
return reply({ written: await driver.setIfAbsent((await body<{ entries: [string, unknown][] }>()).entries) })
|
||||
case 'POST /batch':
|
||||
if (!driver.batch) return fail(501, 'batch not supported')
|
||||
await driver.batch((await body<{ ops: BatchOp[] }>()).ops)
|
||||
return noContent()
|
||||
case 'POST /deleteRange': {
|
||||
if (!driver.deleteRange) return fail(501, 'deleteRange not supported')
|
||||
const { gte, lt } = await body<{ gte: string; lt: string }>()
|
||||
await driver.deleteRange(gte, lt)
|
||||
return noContent()
|
||||
}
|
||||
case 'GET /capabilities':
|
||||
return reply({ setIfAbsent: !!driver.setIfAbsent, batch: !!driver.batch, deleteRange: !!driver.deleteRange } satisfies HttpCapabilities)
|
||||
case 'GET /scan':
|
||||
return scanResponse(driver, parseScan(url.searchParams), codec)
|
||||
default:
|
||||
return fail(404, `No route ${route}`)
|
||||
}
|
||||
} catch (err) {
|
||||
return fail(500, err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseScan(q: URLSearchParams): ScanOptions {
|
||||
const opts: ScanOptions = {}
|
||||
if (q.has('gte')) opts.gte = q.get('gte')!
|
||||
if (q.has('lt')) opts.lt = q.get('lt')!
|
||||
if (q.has('limit')) opts.limit = Number(q.get('limit'))
|
||||
if (q.get('reverse') === 'true') opts.reverse = true
|
||||
if (q.get('values') === 'true') opts.values = true
|
||||
return opts
|
||||
}
|
||||
|
||||
function scanResponse(driver: KVDriver, opts: ScanOptions, codec: Codec): Response {
|
||||
const it = driver.scan(opts)[Symbol.asyncIterator]()
|
||||
const encoder = new TextEncoder()
|
||||
// Chunks grow so short (skip-scan) reads don't make the server run far ahead.
|
||||
let chunk = 16
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
let buf = ''
|
||||
for (let i = 0; i < chunk; i++) {
|
||||
const r = await it.next()
|
||||
if (r.done) {
|
||||
if (buf) controller.enqueue(encoder.encode(buf))
|
||||
controller.close()
|
||||
return
|
||||
}
|
||||
const [key, value] = r.value
|
||||
buf += codec.encode(opts.values ? [key, value] : [key]) + '\n'
|
||||
}
|
||||
chunk = Math.min(chunk * 2, 256)
|
||||
controller.enqueue(encoder.encode(buf))
|
||||
},
|
||||
async cancel() {
|
||||
await it.return?.()
|
||||
},
|
||||
})
|
||||
return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } })
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export * from './handler.js'
|
||||
export * from './client.js'
|
||||
+26
-8
@@ -1,8 +1,26 @@
|
||||
export * from './errors.js'
|
||||
export * from './core/path.js'
|
||||
export * from './core/encoding.js'
|
||||
export * from './kv/index.js'
|
||||
export * from './types/schema.js'
|
||||
export * from './events/index.js'
|
||||
export * from './documents/index.js'
|
||||
export * from './http/index.js'
|
||||
export * from './kv-storage/base-kv-storage.js'
|
||||
export * from './kv-storage/kv-storage-in-memory.js'
|
||||
export * from './kv-storage/kv-storage-http.js'
|
||||
export * from './kv-storage/kv-storage-http-server.js'
|
||||
export * from './kv-storage/kv-storage-indexeddb.js'
|
||||
export * from './kv-storage/kv-storage-composite.js'
|
||||
export * from './kv-storage/kv-storage-encrypted.js'
|
||||
export * from './event-storage/base-event-storage.js'
|
||||
export * from './event-storage/event-storage-kv.js'
|
||||
export * from './document-storage/base-document-storage.js'
|
||||
export * from './document-storage/document-storage-in-memory.js'
|
||||
export * from './document-storage/document-storage-kv.js'
|
||||
export * from './document-storage/event-sourced-document-storage.js'
|
||||
export * from './utils/document-path.js'
|
||||
export * from './utils/document-query.js'
|
||||
export * from './utils/document-update.js'
|
||||
export * from './utils/errors.js'
|
||||
export * from './utils/event-emitter.js'
|
||||
export * from './utils/extended-json.js'
|
||||
export * from './utils/hex.js'
|
||||
export * from './utils/key-range.js'
|
||||
export * from './utils/request-auth.js'
|
||||
export * from './utils/json.js'
|
||||
export * from './utils/keys.js'
|
||||
export * from './utils/schema.js'
|
||||
export * from './utils/ulid.js'
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
import { EventEmitter } from '../utils/event-emitter.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueAt, ValueOf } from '../utils/schema.js'
|
||||
|
||||
export interface ScanOptions {
|
||||
/** Inclusive lower bound. */
|
||||
gte?: string
|
||||
/** Exclusive upper bound. */
|
||||
lt?: string
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
/** Include values; otherwise they are `undefined`. */
|
||||
values?: boolean
|
||||
}
|
||||
|
||||
export interface KeysOptions {
|
||||
limit?: number
|
||||
/** Start after this child key (exclusive), in the direction of travel. */
|
||||
after?: string | number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
/** One key's new value; `value: undefined` means it was removed. */
|
||||
export interface KVChange {
|
||||
key: string
|
||||
value?: unknown
|
||||
}
|
||||
|
||||
export interface KVChangeEvent {
|
||||
changes: KVChange[]
|
||||
/** `local`: made through this instance. `remote`: observed from elsewhere (e.g. pushed by a server). */
|
||||
origin: 'local' | 'remote'
|
||||
}
|
||||
|
||||
export type KVStorageEvents = {
|
||||
change: KVChangeEvent
|
||||
/** Changes may have been missed (e.g. a dropped connection was restored): consumers should re-read. */
|
||||
resync: void
|
||||
}
|
||||
|
||||
/**
|
||||
* Ordered key-value storage. Implementations provide three primitives over absolute keys:
|
||||
* `readMany`, `scan` and the protected `write`. Everything else (navigation, self/child reads and
|
||||
* writes, child listing, change events) is defined here once.
|
||||
*
|
||||
* Writes are plain operations: set a key, or delete it. There are no versions or conditions;
|
||||
* concurrent writers to one key end with whichever write arrived last. Use event storage when
|
||||
* concurrent writes must all be kept.
|
||||
*
|
||||
* Every write emits a `change` event from the storage (shared by all its child nodes).
|
||||
*
|
||||
* Every instance is a node: `child(key)` returns a view of the same storage rooted at that key.
|
||||
* Methods taking an optional `key` act on that child, or on this node's own value when omitted.
|
||||
* A key that is passed but `undefined`/`null` throws rather than silently meaning "self".
|
||||
*/
|
||||
export abstract class BaseKVStorage<S extends Spec = AnySpec> extends EventEmitter<KVStorageEvents> {
|
||||
/** Unescaped path of this node (`[]` for the root). */
|
||||
readonly path: readonly string[] = []
|
||||
/** This node's own key (`''` for the root). */
|
||||
readonly key: string = ''
|
||||
/** Prefix of every descendant key (`''` for the root). */
|
||||
readonly prefix: string = ''
|
||||
|
||||
/** Values for absolute keys, in order; `undefined` for missing. */
|
||||
abstract readMany(keys: string[]): Promise<unknown[]>
|
||||
/** Entries in key order (descending when `reverse`). */
|
||||
abstract scan(options?: ScanOptions): AsyncIterable<[key: string, value: unknown]>
|
||||
/** Applies changes in order: sets each key to its value, or deletes it when the value is `undefined`. */
|
||||
protected abstract write(changes: KVChange[]): Promise<void>
|
||||
|
||||
// ------------------------------------------------------------------ raw writes (absolute keys)
|
||||
|
||||
/** Applies changes (`value: undefined` deletes) and emits them as a `change` event. */
|
||||
async writeMany(changes: KVChange[]): Promise<void> {
|
||||
if (changes.length === 0) return
|
||||
await this.write(changes)
|
||||
this.emit('change', { changes, origin: 'local' })
|
||||
}
|
||||
|
||||
async removeMany(keys: string[]): Promise<void> {
|
||||
await this.writeMany(keys.map((key) => ({ key })))
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ navigation
|
||||
|
||||
/** A view of this storage rooted at `key`. Synchronous; no I/O. */
|
||||
child<K extends KeyOf<S>>(key: K): BaseKVStorage<ChildOf<S, K>> {
|
||||
const own = this.prefix + Keys.escape(Keys.segment(key))
|
||||
const node = Object.create(this, {
|
||||
path: { value: [...this.path, Keys.segment(key)], enumerable: true },
|
||||
key: { value: own, enumerable: true },
|
||||
prefix: { value: own + Keys.separator, enumerable: true },
|
||||
}) as BaseKVStorage<ChildOf<S, K>>
|
||||
// Frozen: child nodes share the root's state through the prototype, so an implementation that
|
||||
// reassigns a field (`this.x = ...`) while running on a child would silently shadow it.
|
||||
// Keep mutable state in objects (Maps, a `state` object) instead; this makes mistakes throw.
|
||||
Object.freeze(node)
|
||||
return node
|
||||
}
|
||||
|
||||
/** Re-types this node. No runtime effect. */
|
||||
as<S2 extends Spec>(): BaseKVStorage<S2> {
|
||||
return this as unknown as BaseKVStorage<S2>
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ reads and writes
|
||||
|
||||
async get<K extends KeyOf<S> | undefined = undefined>(key?: K): Promise<ValueAt<S, K> | undefined> {
|
||||
const [value] = await this.readMany([this.keyOf(arguments.length > 0, key)])
|
||||
return value as ValueAt<S, K> | undefined
|
||||
}
|
||||
|
||||
async getMany<K extends KeyOf<S>>(keys: readonly K[]): Promise<(ValueOf<ChildOf<S, K>> | undefined)[]> {
|
||||
return (await this.readMany(keys.map((k) => this.keyOf(true, k)))) as (ValueOf<ChildOf<S, K>> | undefined)[]
|
||||
}
|
||||
|
||||
/** `set(value)` writes this node's own value; `set(key, value)` a child's. */
|
||||
async set<K extends KeyOf<S>>(...args: [value: ValueOf<S>] | [key: K, value: ValueOf<ChildOf<S, K>>]): Promise<void> {
|
||||
const value = args[args.length - 1]
|
||||
if (value === undefined) throw new TypeError('Cannot set undefined; use delete()')
|
||||
await this.writeMany([{ key: this.keyOf(args.length > 1, args[0]), value }])
|
||||
}
|
||||
|
||||
async setMany(entries: readonly EntryOf<S>[]): Promise<void> {
|
||||
if (entries.some(([, v]) => v === undefined)) throw new TypeError('Cannot set undefined; use delete()')
|
||||
await this.writeMany(entries.map(([k, value]) => ({ key: this.keyOf(true, k), value })))
|
||||
}
|
||||
|
||||
/** Writes each property of `fields` as a child key (one write). Nested objects stay single values. */
|
||||
async setFields(fields: { [K in KeyString<S>]?: ValueOf<ChildOf<S, K & KeyOf<S>>> }): Promise<void> {
|
||||
await this.setMany(Object.entries(fields).filter(([, v]) => v !== undefined) as unknown as EntryOf<S>[])
|
||||
}
|
||||
|
||||
/** Reads the direct children that have a value into an object. */
|
||||
async getFields(): Promise<{ [K in KeyString<S>]?: ValueOf<ChildOf<S, K & KeyOf<S>>> }> {
|
||||
const fields: Record<string, unknown> = {}
|
||||
for await (const [key, value] of this.entries()) fields[key] = value
|
||||
return fields as { [K in KeyString<S>]?: ValueOf<ChildOf<S, K & KeyOf<S>>> }
|
||||
}
|
||||
|
||||
/** Deletes this node's own value (or a child's), not descendants. */
|
||||
async delete(key?: KeyOf<S>): Promise<void> {
|
||||
await this.removeMany([this.keyOf(arguments.length > 0, key)])
|
||||
}
|
||||
|
||||
async deleteMany(keys: readonly KeyOf<S>[]): Promise<void> {
|
||||
await this.removeMany(keys.map((k) => this.keyOf(true, k)))
|
||||
}
|
||||
|
||||
/** Deletes this node's own value (or a child's) and all descendants. */
|
||||
async deleteTree(key?: KeyOf<S>): Promise<void> {
|
||||
const own = this.keyOf(arguments.length > 0, key, true)
|
||||
if (own !== '') await this.removeMany([own])
|
||||
const prefix = own === '' ? '' : own + Keys.separator
|
||||
for (;;) {
|
||||
const batch: string[] = []
|
||||
for await (const [k] of this.scan({ gte: prefix, lt: own === '' ? undefined : own + Keys.afterSeparator, limit: 1000 })) batch.push(k)
|
||||
if (batch.length === 0) return
|
||||
await this.removeMany(batch)
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ listing
|
||||
|
||||
/**
|
||||
* Direct child keys in key order, including children that only have descendants.
|
||||
* Skip-scan: one seek per child that has descendants, not one read per descendant.
|
||||
*/
|
||||
async *keys(options: KeysOptions = {}): AsyncGenerator<KeyString<S>> {
|
||||
for await (const [key] of this.children(options, false)) yield key as KeyString<S>
|
||||
}
|
||||
|
||||
/** Direct children that have their own value, as `[key, value]`. */
|
||||
async *entries(options: KeysOptions = {}): AsyncGenerator<StringEntryOf<S>> {
|
||||
let remaining = options.limit ?? Infinity
|
||||
for await (const [key, value, hasValue] of this.children({ ...options, limit: undefined }, true)) {
|
||||
if (remaining <= 0) return
|
||||
if (!hasValue) continue
|
||||
remaining--
|
||||
yield [key, value] as StringEntryOf<S>
|
||||
}
|
||||
}
|
||||
|
||||
private async *children(options: KeysOptions, values: boolean): AsyncGenerator<[key: string, value: unknown, hasValue: boolean]> {
|
||||
const { reverse = false } = options
|
||||
let remaining = options.limit ?? Infinity
|
||||
let gte = this.prefix
|
||||
let lt = this.key === '' ? undefined : this.key + Keys.afterSeparator
|
||||
if (options.after !== undefined) {
|
||||
const after = this.prefix + Keys.escape(Keys.segment(options.after))
|
||||
if (reverse) lt = after
|
||||
else gte = after + Keys.afterSeparator
|
||||
}
|
||||
let last: string | undefined
|
||||
let restart = true
|
||||
let iterator: AsyncIterator<[string, unknown]> | undefined
|
||||
try {
|
||||
while (remaining > 0) {
|
||||
if (restart) {
|
||||
await iterator?.return?.()
|
||||
iterator = this.scan({ gte, lt, reverse, values })[Symbol.asyncIterator]()
|
||||
restart = false
|
||||
}
|
||||
const next = await iterator!.next()
|
||||
if (next.done) return
|
||||
const [key, value] = next.value
|
||||
const end = key.indexOf(Keys.separator, this.prefix.length)
|
||||
const segment = key.slice(this.prefix.length, end === -1 ? undefined : end)
|
||||
const isOwn = end === -1
|
||||
if (segment !== last && !Keys.isReserved(segment)) {
|
||||
// Forward, a child's own key comes before its descendants; in reverse it comes after
|
||||
// them and is skipped by the jump below, so read it directly when values are wanted.
|
||||
let own = isOwn ? value : undefined
|
||||
if (!isOwn && reverse && values) [own] = await this.readMany([this.prefix + segment])
|
||||
yield [Keys.unescape(segment), own, isOwn || own !== undefined]
|
||||
remaining--
|
||||
}
|
||||
last = segment
|
||||
// Jump over the rest of this child's subtree instead of reading it.
|
||||
if (!isOwn || Keys.isReserved(segment)) {
|
||||
if (reverse) lt = this.prefix + segment
|
||||
else gte = this.prefix + segment + Keys.afterSeparator
|
||||
restart = true
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await iterator?.return?.()
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ internals
|
||||
|
||||
/** Absolute key for a child (`hasKey`) or this node; validates keys. */
|
||||
protected keyOf(hasKey: boolean, key: unknown, allowRoot = false): string {
|
||||
if (hasKey) return this.prefix + Keys.escape(Keys.segment(key))
|
||||
if (this.key === '' && !allowRoot) throw new TypeError('The root node has no own value')
|
||||
return this.key
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
import { StorageUnavailableError } from '../utils/errors.js'
|
||||
import type { OffCallback } from '../utils/event-emitter.js'
|
||||
import { Json } from '../utils/json.js'
|
||||
import { KeyRange } from '../utils/key-range.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { Ulid } from '../utils/ulid.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
import { KVStorageInMemory } from './kv-storage-in-memory.js'
|
||||
|
||||
export interface KVStorageCompositeOptions {
|
||||
/** The source of truth, e.g. `KVStorageHttp`. */
|
||||
source: BaseKVStorage
|
||||
/** Local copies, fastest first, e.g. `[inMemory, indexedDb]`. The last one seeds the others on start. */
|
||||
replicas: BaseKVStorage[]
|
||||
/**
|
||||
* Holds writes made while the source is unreachable (under its `writes` child) and whether a
|
||||
* sync has completed (`synced`). Use a persistent node outside `prefix`, e.g.
|
||||
* `indexedDb.child('sync')`, so queued writes survive a reload. Default: in memory.
|
||||
*/
|
||||
queue?: BaseKVStorage
|
||||
/** Only this key and its subtree are synced (e.g. the user's public key). Default: everything. */
|
||||
prefix?: string
|
||||
/** Delay before retrying when the source is unreachable. Default 5 000 ms. */
|
||||
retryMs?: number
|
||||
/**
|
||||
* Called with errors that have no caller to reject: a queued write the source refused (it is
|
||||
* dropped, and the source's values restored locally), or a failed sync. Default: `console.error`.
|
||||
*/
|
||||
onError?: (error: Error) => void
|
||||
}
|
||||
|
||||
/** Sources that push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface Subscribable {
|
||||
subscribe(path: string): Promise<void>
|
||||
unsubscribe(path: string): Promise<void>
|
||||
}
|
||||
|
||||
// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates
|
||||
// them with Object.create, so they share this class's prototype).
|
||||
export interface KVStorageComposite<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): KVStorageComposite<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* Combines a source with local replicas: reads come from the first replica, writes go to every
|
||||
* replica and then the source, exactly as given (no merging: the last write wins). A write the
|
||||
* source refuses (e.g. 403) is undone locally and rethrown.
|
||||
*
|
||||
* - Changes pushed by the source (`subscribe`) are applied to the replicas, and announced as
|
||||
* `change` events, only where they differ from what the replicas hold. The source relaying this
|
||||
* client's own writes therefore changes nothing.
|
||||
* - A write the source can't take (`StorageUnavailableError`) is applied locally and stored in
|
||||
* the `queue` store; later writes queue behind it, so order is kept.
|
||||
* - `sync()` replays the queue, then copies what changed on the source into the replicas, except
|
||||
* keys with local writes the source hasn't confirmed yet (in flight or queued). It runs
|
||||
* on start, when the source reconnects (`resync`), and after a retry delay while offline.
|
||||
* - Reads wait for the first sync (until one succeeds, they go to the source). Whether a sync has
|
||||
* completed is kept in the `queue` store, so with a persistent one a restart reads offline.
|
||||
*/
|
||||
export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly source: BaseKVStorage
|
||||
private readonly replicas: BaseKVStorage[]
|
||||
private readonly queue: BaseKVStorage
|
||||
private readonly writes: BaseKVStorage
|
||||
private readonly range: KeyRange
|
||||
private readonly retryMs: number
|
||||
private readonly onError: (error: Error) => void
|
||||
private readonly ids = new Ulid()
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state: {
|
||||
started?: Promise<void>
|
||||
syncing?: Promise<void>
|
||||
/** A sync requested while one was running. */
|
||||
again?: Promise<void>
|
||||
/** A sync has completed at some point, so the replicas hold a full copy. */
|
||||
synced: boolean
|
||||
/** Writes waiting in the queue. */
|
||||
queued: number
|
||||
/** Remote changes are applied one after another. */
|
||||
applying: Promise<unknown>
|
||||
/** Keys with local writes the source hasn't confirmed yet (in flight or queued) → count. */
|
||||
pending: Map<string, number>
|
||||
retry?: ReturnType<typeof setTimeout>
|
||||
offs: OffCallback[]
|
||||
} = { synced: false, queued: 0, applying: Promise.resolve(), pending: new Map(), offs: [] }
|
||||
|
||||
constructor(options: KVStorageCompositeOptions) {
|
||||
super()
|
||||
if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica')
|
||||
this.source = options.source
|
||||
this.replicas = options.replicas
|
||||
this.queue = options.queue ?? new KVStorageInMemory()
|
||||
this.writes = this.queue.child('writes')
|
||||
this.range = new KeyRange(options.prefix ?? '')
|
||||
this.retryMs = options.retryMs ?? 5000
|
||||
this.onError = options.onError ?? ((error) => console.error(error))
|
||||
// A full sync makes the replicas match the source within the range, which would delete queued
|
||||
// writes stored inside that range.
|
||||
const queueRoot = KVStorageComposite.rootOf(this.queue)
|
||||
if (this.replicas.some((replica) => KVStorageComposite.rootOf(replica) === queueRoot) && this.range.contains(this.queue.key)) {
|
||||
throw new TypeError('The queue must be outside the synced range (e.g. a separate store, or a node outside `prefix`)')
|
||||
}
|
||||
}
|
||||
|
||||
/** Loads local data and starts syncing (also happens on first use). */
|
||||
start(): Promise<void> {
|
||||
return (this.state.started ??= this.open())
|
||||
}
|
||||
|
||||
/**
|
||||
* Replays queued writes, then brings the replicas up to date with the source (starts first if
|
||||
* needed). Called while a sync is running, it resolves after one more sync that starts after it.
|
||||
*/
|
||||
sync(): Promise<void> {
|
||||
// During a sync, schedule one more after it (shared by every caller meanwhile): what changed
|
||||
// since the running one scanned that part of the source must be picked up too.
|
||||
if (this.state.syncing) return (this.state.again ??= this.state.syncing.then(() => ((this.state.again = undefined), this.sync())))
|
||||
return (this.state.syncing = this.runSync().finally(() => (this.state.syncing = undefined)))
|
||||
}
|
||||
|
||||
/** Number of writes waiting for the source. */
|
||||
get queued(): number {
|
||||
return this.state.queued
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
clearTimeout(this.state.retry)
|
||||
for (const off of this.state.offs.splice(0)) off()
|
||||
if (KVStorageComposite.isSubscribable(this.source)) await this.source.unsubscribe(this.range.prefix)
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
this.checkRange(keys)
|
||||
return (await this.reader()).readMany(keys)
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
const scoped = this.range.clamp(options)
|
||||
if (scoped) yield* (await this.reader()).scan(scoped)
|
||||
}
|
||||
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
const keys = changes.map(({ key }) => key)
|
||||
this.checkRange(keys)
|
||||
await this.start()
|
||||
// Applied locally first, so the source relaying this write back finds nothing new. Until the
|
||||
// source confirms it, a sync must not replace it with the source's (older) value.
|
||||
const previous = await this.replicas[0]!.readMany(keys)
|
||||
this.markPending(keys, 1)
|
||||
await this.apply(changes)
|
||||
// While syncing or with writes queued, queue this one too so the source sees writes in order.
|
||||
if (this.state.queued > 0 || this.state.syncing) return this.enqueue(changes)
|
||||
try {
|
||||
await this.source.writeMany(changes)
|
||||
} catch (error) {
|
||||
if (error instanceof StorageUnavailableError) return this.enqueue(changes)
|
||||
this.markPending(keys, -1)
|
||||
await this.apply(keys.map((key, i) => ({ key, value: previous[i] })))
|
||||
throw error
|
||||
}
|
||||
this.markPending(keys, -1)
|
||||
}
|
||||
|
||||
private async enqueue(changes: KVChange[]): Promise<void> {
|
||||
await this.writes.set(this.ids.generate(), changes)
|
||||
this.state.queued++
|
||||
this.retryLater()
|
||||
}
|
||||
|
||||
/** The first replica once it holds a full copy; until then, a sync is attempted first and the source is the fallback. */
|
||||
private async reader(): Promise<BaseKVStorage> {
|
||||
if (!this.state.synced) await this.sync()
|
||||
return this.state.synced ? this.replicas[0]! : this.source
|
||||
}
|
||||
|
||||
private async open(): Promise<void> {
|
||||
this.state.synced = (await this.queue.get('synced')) === true
|
||||
for await (const [, changes] of this.writes.entries()) {
|
||||
this.state.queued++
|
||||
this.markPending(
|
||||
(changes as KVChange[]).map(({ key }) => key),
|
||||
1,
|
||||
)
|
||||
}
|
||||
await this.seed()
|
||||
this.state.offs.push(
|
||||
this.source.on('change', (event) => {
|
||||
if (event.origin !== 'remote') return
|
||||
const changes = event.changes.filter(({ key }) => this.range.contains(key))
|
||||
this.state.applying = this.state.applying.then(() => this.applyRemote(changes)).catch((error: Error) => this.onError(error))
|
||||
}),
|
||||
this.source.on('resync', () => void this.sync()),
|
||||
)
|
||||
if (KVStorageComposite.isSubscribable(this.source)) {
|
||||
void this.source
|
||||
.subscribe(this.range.prefix)
|
||||
.then(() => this.sync())
|
||||
.catch((error: Error) => this.onError(error))
|
||||
} else {
|
||||
void this.sync()
|
||||
}
|
||||
}
|
||||
|
||||
private async runSync(): Promise<void> {
|
||||
await this.start()
|
||||
try {
|
||||
await this.replay()
|
||||
let batch: KVChange[] = []
|
||||
for await (const change of this.diff()) {
|
||||
if (this.state.pending.has(change.key)) continue // a newer local write, not yet sent
|
||||
if (batch.push(change) >= 500) await this.apply(batch.splice(0), true)
|
||||
}
|
||||
await this.apply(batch, true)
|
||||
await this.replay() // writes made during the diff: send them, and re-apply them over it
|
||||
await this.queue.set('synced', true)
|
||||
this.state.synced = true
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) this.onError(error as Error)
|
||||
this.retryLater()
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends queued writes to the source in order, and re-applies each locally (a remote change may have overwritten it meanwhile). */
|
||||
private async replay(): Promise<void> {
|
||||
for (;;) {
|
||||
let next: [string, unknown] | undefined
|
||||
for await (const entry of this.writes.entries({ limit: 1 })) next = entry
|
||||
if (!next) return
|
||||
const changes = next[1] as KVChange[]
|
||||
try {
|
||||
await this.source.writeMany(changes)
|
||||
await this.apply(changes)
|
||||
} catch (error) {
|
||||
if (error instanceof StorageUnavailableError) throw error
|
||||
// Refused (e.g. 403): retrying won't help, and would block every later write. Drop it and
|
||||
// restore what the source holds.
|
||||
this.onError(new Error('A queued write was refused by the source and dropped', { cause: error }))
|
||||
const keys = changes.map(({ key }) => key)
|
||||
const current = await this.source.readMany(keys).catch(() => undefined)
|
||||
if (current) await this.applyRemote(keys.map((key, i) => ({ key, value: current[i] })))
|
||||
}
|
||||
await this.writes.delete(next[0])
|
||||
this.state.queued--
|
||||
this.markPending(
|
||||
changes.map(({ key }) => key),
|
||||
-1,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** What the source has that differs from the last replica (both scanned in key order). */
|
||||
private async *diff(): AsyncGenerator<KVChange> {
|
||||
const options = { gte: this.range.gte, lt: this.range.lt, values: true }
|
||||
const remote = this.source.scan(options)[Symbol.asyncIterator]()
|
||||
const local = this.replicas.at(-1)!.scan(options)[Symbol.asyncIterator]()
|
||||
let r = await remote.next()
|
||||
let l = await local.next()
|
||||
while (!r.done || !l.done) {
|
||||
if (l.done || (!r.done && r.value[0] < l.value[0])) {
|
||||
yield { key: r.value![0], value: r.value![1] }
|
||||
r = await remote.next()
|
||||
} else if (r.done || l.value[0] < r.value[0]) {
|
||||
yield { key: l.value[0] }
|
||||
l = await local.next()
|
||||
} else {
|
||||
if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] }
|
||||
;[r, l] = await Promise.all([remote.next(), local.next()])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Applies pushed changes that differ from what the replicas hold. */
|
||||
private async applyRemote(changes: KVChange[]): Promise<void> {
|
||||
const current = await this.replicas[0]!.readMany(changes.map(({ key }) => key))
|
||||
await this.apply(
|
||||
changes.filter(({ value }, i) => !Json.equal(value, current[i])),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
/** Writes changes to every replica; `remote` ones are also announced as `change` events. */
|
||||
private async apply(changes: KVChange[], remote = false): Promise<void> {
|
||||
if (changes.length === 0) return
|
||||
await Promise.all(this.replicas.map((replica) => replica.writeMany(changes)))
|
||||
if (remote) this.emit('change', { changes, origin: 'remote' })
|
||||
}
|
||||
|
||||
/** Copies the last (persistent) replica into the faster ones. */
|
||||
private async seed(): Promise<void> {
|
||||
if (this.replicas.length < 2) return
|
||||
let batch: KVChange[] = []
|
||||
const flush = () => Promise.all(this.replicas.slice(0, -1).map((replica) => replica.writeMany(batch.splice(0))))
|
||||
for await (const [key, value] of this.replicas.at(-1)!.scan({ gte: this.range.gte, lt: this.range.lt, values: true })) {
|
||||
if (batch.push({ key, value }) >= 500) await flush()
|
||||
}
|
||||
await flush()
|
||||
}
|
||||
|
||||
private markPending(keys: string[], delta: 1 | -1): void {
|
||||
for (const key of keys) {
|
||||
const count = (this.state.pending.get(key) ?? 0) + delta
|
||||
if (count > 0) this.state.pending.set(key, count)
|
||||
else this.state.pending.delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
private retryLater(): void {
|
||||
clearTimeout(this.state.retry)
|
||||
this.state.retry = setTimeout(() => void this.sync(), this.retryMs)
|
||||
;(this.state.retry as { unref?: () => void }).unref?.()
|
||||
}
|
||||
|
||||
private checkRange(keys: string[]): void {
|
||||
for (const key of keys) if (!this.range.contains(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.range.prefix}"`)
|
||||
}
|
||||
|
||||
/** The storage instance a node belongs to (child nodes are frozen views over it). */
|
||||
private static rootOf(node: BaseKVStorage): BaseKVStorage {
|
||||
while (Object.isFrozen(node)) node = Object.getPrototypeOf(node) as BaseKVStorage
|
||||
return node
|
||||
}
|
||||
|
||||
private static isSubscribable(source: BaseKVStorage): source is BaseKVStorage & Subscribable {
|
||||
return typeof (source as Partial<Subscribable>).subscribe === 'function'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import { ExtendedJson } from '../utils/extended-json.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
|
||||
export interface KVStorageEncryptedOptions {
|
||||
/** An AES-GCM key, e.g. from `KVStorageEncrypted.generateKey()` or `deriveKey(password, salt)`. */
|
||||
key: CryptoKey
|
||||
/** Called when a pushed change can't be decrypted (it is dropped). Default: `console.error`. */
|
||||
onError?: (error: Error) => void
|
||||
}
|
||||
|
||||
/** Sources that push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface Subscribable {
|
||||
subscribe(path: string): Promise<void>
|
||||
unsubscribe(path: string): Promise<void>
|
||||
}
|
||||
|
||||
// Types only: children of a KVStorageEncrypted are KVStorageEncrypted nodes too (child() creates
|
||||
// them with Object.create, so they share this class's prototype).
|
||||
export interface KVStorageEncrypted<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): KVStorageEncrypted<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypts values (not keys) before they reach another KV storage, e.g. `KVStorageHttp`, so the
|
||||
* server only ever stores ciphertext.
|
||||
*
|
||||
* - Each value is AES-GCM encrypted with a fresh random IV and stored as a `Uint8Array`:
|
||||
* `[1 (format), iv (12 bytes), ciphertext]`. The key it is stored under is authenticated too, so
|
||||
* a value moved to another key fails to decrypt.
|
||||
* - Keys stay readable: the server needs them for authorization, ordering and prefix scans. Don't
|
||||
* put secrets in keys. Value sizes and write times are visible as well.
|
||||
* - Changes the inner storage pushes (`origin: 'remote'`) are decrypted and re-emitted, and
|
||||
* `subscribe` / `unsubscribe` are passed through, so it can be a `KVStorageComposite` source.
|
||||
*/
|
||||
export class KVStorageEncrypted<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly cryptoKey: CryptoKey
|
||||
private readonly onError: (error: Error) => void
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state = { relaying: Promise.resolve() }
|
||||
|
||||
constructor(
|
||||
private readonly inner: BaseKVStorage,
|
||||
options: KVStorageEncryptedOptions,
|
||||
) {
|
||||
super()
|
||||
this.cryptoKey = options.key
|
||||
this.onError = options.onError ?? ((error) => console.error(error))
|
||||
inner.on('change', ({ changes, origin }) => {
|
||||
if (origin !== 'remote') return
|
||||
// Decrypted one batch at a time, so changes are re-emitted in the order they arrived.
|
||||
this.state.relaying = this.state.relaying
|
||||
.then(async () => {
|
||||
this.emit('change', { changes: await Promise.all(changes.map(({ key, value }) => this.decryptChange(key, value))), origin: 'remote' })
|
||||
})
|
||||
.catch((error: Error) => this.onError(error))
|
||||
})
|
||||
inner.on('resync', () => this.emit('resync', undefined))
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
const values = await this.inner.readMany(keys)
|
||||
return Promise.all(values.map((value, i) => (value === undefined ? undefined : this.decrypt(keys[i]!, value))))
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
for await (const [key, value] of this.inner.scan(options)) yield [key, options.values ? await this.decrypt(key, value) : undefined]
|
||||
}
|
||||
|
||||
/** Passes through to the inner storage, when it supports subscriptions. */
|
||||
async subscribe(path: string = this.key): Promise<void> {
|
||||
await (this.inner as Partial<Subscribable>).subscribe?.(path)
|
||||
}
|
||||
|
||||
async unsubscribe(path: string = this.key): Promise<void> {
|
||||
await (this.inner as Partial<Subscribable>).unsubscribe?.(path)
|
||||
}
|
||||
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
await this.inner.writeMany(
|
||||
await Promise.all(changes.map(async ({ key, value }) => (value === undefined ? { key } : { key, value: await this.encrypt(key, value) }))),
|
||||
)
|
||||
}
|
||||
|
||||
private async encrypt(key: string, value: unknown): Promise<Uint8Array> {
|
||||
const iv = crypto.getRandomValues(new Uint8Array(12))
|
||||
const plaintext = new TextEncoder().encode(ExtendedJson.stringify(value))
|
||||
const ciphertext = new Uint8Array(
|
||||
await crypto.subtle.encrypt({ name: 'AES-GCM', iv, additionalData: KVStorageEncrypted.encodeKey(key) }, this.cryptoKey, plaintext),
|
||||
)
|
||||
const out = new Uint8Array(1 + iv.length + ciphertext.length)
|
||||
out[0] = 1
|
||||
out.set(iv, 1)
|
||||
out.set(ciphertext, 1 + iv.length)
|
||||
return out
|
||||
}
|
||||
|
||||
private async decrypt(key: string, value: unknown): Promise<unknown> {
|
||||
if (!(value instanceof Uint8Array) || value[0] !== 1 || value.length < 13) throw new TypeError(`The value at "${key}" is not encrypted`)
|
||||
const iv = value.subarray(1, 13)
|
||||
try {
|
||||
const plaintext = await crypto.subtle.decrypt(
|
||||
{ name: 'AES-GCM', iv: iv as BufferSource, additionalData: KVStorageEncrypted.encodeKey(key) },
|
||||
this.cryptoKey,
|
||||
value.subarray(13) as BufferSource,
|
||||
)
|
||||
return ExtendedJson.parse(new TextDecoder().decode(plaintext))
|
||||
} catch (error) {
|
||||
throw new Error(`Cannot decrypt the value at "${key}" (wrong key, or the data was changed)`, { cause: error })
|
||||
}
|
||||
}
|
||||
|
||||
private async decryptChange(key: string, value: unknown): Promise<KVChange> {
|
||||
return value === undefined ? { key } : { key, value: await this.decrypt(key, value) }
|
||||
}
|
||||
|
||||
/** A new random AES-GCM 256 key (extractable, so it can be saved with `exportKey`). */
|
||||
static generateKey(): Promise<CryptoKey> {
|
||||
return crypto.subtle.generateKey({ name: 'AES-GCM', length: 256 }, true, ['encrypt', 'decrypt'])
|
||||
}
|
||||
|
||||
/** The raw bytes of a key, to store it (e.g. in a password manager or the OS keychain). */
|
||||
static async exportKey(key: CryptoKey): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.exportKey('raw', key))
|
||||
}
|
||||
|
||||
static importKey(raw: Uint8Array): Promise<CryptoKey> {
|
||||
return crypto.subtle.importKey('raw', raw as BufferSource, 'AES-GCM', true, ['encrypt', 'decrypt'])
|
||||
}
|
||||
|
||||
/**
|
||||
* A key derived from a password with PBKDF2-SHA-256. Use the same `salt` (random, stored with the
|
||||
* data, not secret) on every device. Not extractable.
|
||||
*/
|
||||
static async deriveKey(password: string, salt: Uint8Array, iterations = 600_000): Promise<CryptoKey> {
|
||||
const material = await crypto.subtle.importKey('raw', new TextEncoder().encode(password), 'PBKDF2', false, ['deriveKey'])
|
||||
return crypto.subtle.deriveKey(
|
||||
{ name: 'PBKDF2', hash: 'SHA-256', salt: salt as BufferSource, iterations },
|
||||
material,
|
||||
{ name: 'AES-GCM', length: 256 },
|
||||
false,
|
||||
['encrypt', 'decrypt'],
|
||||
)
|
||||
}
|
||||
|
||||
private static encodeKey(key: string): Uint8Array<ArrayBuffer> {
|
||||
return new TextEncoder().encode(key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { AuthError } from '../utils/errors.js'
|
||||
import { ExtendedJson } from '../utils/extended-json.js'
|
||||
import { KeyRange } from '../utils/key-range.js'
|
||||
import { AUTH_HEADERS, RequestAuth, type RequestAuthOptions, type SignatureVerifier } from '../utils/request-auth.js'
|
||||
import type { BaseKVStorage, KVChange, ScanOptions } from './base-kv-storage.js'
|
||||
|
||||
export interface KVStorageHttpServerOptions {
|
||||
/**
|
||||
* `{ verifiers }`: requests must be signed (see `RequestAuth`), and a caller may only touch keys
|
||||
* under its own public key (`<publicKey>` and `<publicKey>/...`).
|
||||
* `'none'`: no authentication, everything is open; for local development and tests only.
|
||||
*/
|
||||
auth: 'none' | ({ verifiers: SignatureVerifier[] } & RequestAuthOptions)
|
||||
/** Path prefix to strip, e.g. `/kv`. Default `''`. */
|
||||
basePath?: string
|
||||
/** Interval of SSE keep-alive comments. Default 15 000 ms. */
|
||||
keepAliveMs?: number
|
||||
/**
|
||||
* Origins allowed to call the server from a browser on another origin (CORS), e.g.
|
||||
* `['https://app.example.com']`, or `'*'` for any. Default: none (same origin only).
|
||||
*/
|
||||
allowOrigins?: string[] | '*'
|
||||
}
|
||||
|
||||
type RequestBody = { keys: string[]; changes: KVChange[]; paths: string[] } & ScanOptions
|
||||
|
||||
/**
|
||||
* Serves any KV storage over HTTP for `KVStorageHttp`. Framework-agnostic: `handle(Request)`
|
||||
* returns a `Response`. It executes exactly the operations clients send; it doesn't know about
|
||||
* versions or conflicts. Every route is `POST` with an Extended JSON body:
|
||||
*
|
||||
* - `/readMany { keys }`, `/write { changes }`, `/scan { gte, lt, limit, reverse, values }` (NDJSON).
|
||||
* - `/events { paths }`: Server-Sent Events relaying each change to those keys (and their
|
||||
* subtrees) as it happens, including the subscriber's own writes. A `ready` event follows the
|
||||
* connection. Nothing is replayed after a reconnect; clients catch up by reading.
|
||||
*
|
||||
* With authentication, each request is checked first (401), then every key, scan range and
|
||||
* subscription path must be the caller's public key or under it (403).
|
||||
*
|
||||
* Only changes made through the wrapped storage instance are relayed; several server processes
|
||||
* over one database each relay only their own writes.
|
||||
*/
|
||||
export class KVStorageHttpServer {
|
||||
private readonly auth: RequestAuth | undefined
|
||||
private readonly basePath: string
|
||||
private readonly keepAliveMs: number
|
||||
private readonly allowOrigins: string[] | '*'
|
||||
private readonly subscribers = new Set<{ ranges: KeyRange[]; send: (frame: string) => void }>()
|
||||
|
||||
constructor(
|
||||
private readonly storage: BaseKVStorage,
|
||||
options: KVStorageHttpServerOptions,
|
||||
) {
|
||||
this.auth = options.auth === 'none' ? undefined : new RequestAuth(options.auth.verifiers, options.auth)
|
||||
this.basePath = options.basePath ?? ''
|
||||
this.keepAliveMs = options.keepAliveMs ?? 15_000
|
||||
this.allowOrigins = options.allowOrigins ?? []
|
||||
storage.on('change', ({ changes }) => {
|
||||
for (const subscriber of this.subscribers) {
|
||||
const matching = changes.filter(({ key }) => subscriber.ranges.some((range) => range.contains(key)))
|
||||
if (matching.length) subscriber.send(`event: change\ndata: ${ExtendedJson.stringify(matching)}\n\n`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async handle(request: Request): Promise<Response> {
|
||||
const response = await this.route(request)
|
||||
const origin = request.headers.get('origin')
|
||||
if (!origin || !(this.allowOrigins === '*' || this.allowOrigins.includes(origin))) return response
|
||||
response.headers.set('access-control-allow-origin', origin)
|
||||
response.headers.append('vary', 'origin')
|
||||
return response
|
||||
}
|
||||
|
||||
private async route(request: Request): Promise<Response> {
|
||||
const path = new URL(request.url).pathname
|
||||
if (request.method === 'OPTIONS') {
|
||||
// CORS preflight (signed requests carry custom headers). `handle` adds the allowed origin.
|
||||
const headers = ['content-type', 'cache-control', 'last-event-id', ...Object.values(AUTH_HEADERS)].join(', ')
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: { 'access-control-allow-methods': 'POST', 'access-control-allow-headers': headers, 'access-control-max-age': '600' },
|
||||
})
|
||||
}
|
||||
if (request.method !== 'POST' || !path.startsWith(this.basePath)) return KVStorageHttpServer.error(404, 'Not found')
|
||||
try {
|
||||
const text = await request.text()
|
||||
const owner = this.auth ? await this.auth.verify(request.headers, path, text) : undefined
|
||||
const body = ExtendedJson.parse(text) as RequestBody
|
||||
switch (path.slice(this.basePath.length)) {
|
||||
case '/readMany': {
|
||||
this.authorize(owner, body.keys)
|
||||
const values = await this.storage.readMany(body.keys)
|
||||
const missing = values.flatMap((v, i) => (v === undefined ? [i] : []))
|
||||
return new Response(ExtendedJson.stringify({ values: values.map((v) => v ?? null), missing }), { headers: { 'content-type': 'application/json' } })
|
||||
}
|
||||
case '/write':
|
||||
this.authorize(
|
||||
owner,
|
||||
body.changes.map(({ key }) => key),
|
||||
)
|
||||
await this.storage.writeMany(body.changes)
|
||||
return new Response(null, { status: 204 })
|
||||
case '/scan':
|
||||
if (owner !== undefined && !new KeyRange(owner).covers(body.gte, body.lt)) throw new AuthError(403, 'Scan outside your keys')
|
||||
return this.scan(body)
|
||||
case '/events':
|
||||
if (!Array.isArray(body.paths) || body.paths.length === 0) return KVStorageHttpServer.error(400, 'Subscribe to at least one path')
|
||||
this.authorize(owner, body.paths)
|
||||
return this.events(body.paths)
|
||||
default:
|
||||
return KVStorageHttpServer.error(404, 'Not found')
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof AuthError) return KVStorageHttpServer.error(error.status, error.message)
|
||||
return KVStorageHttpServer.error(500, error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
}
|
||||
|
||||
/** Every key must be the owner's key or under it (no-op without authentication). */
|
||||
private authorize(owner: string | undefined, keys: string[]): void {
|
||||
if (owner === undefined) return
|
||||
const allowed = new KeyRange(owner)
|
||||
for (const key of keys) {
|
||||
if (typeof key !== 'string' || !allowed.contains(key)) throw new AuthError(403, `No access to "${key}"`)
|
||||
}
|
||||
}
|
||||
|
||||
private events(paths: string[]): Response {
|
||||
const encoder = new TextEncoder()
|
||||
let subscriber: { ranges: KeyRange[]; send: (frame: string) => void } | undefined
|
||||
let keepAlive: ReturnType<typeof setInterval> | undefined
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start: (controller) => {
|
||||
subscriber = { ranges: paths.map((path) => new KeyRange(path)), send: (frame) => controller.enqueue(encoder.encode(frame)) }
|
||||
this.subscribers.add(subscriber)
|
||||
subscriber.send('event: ready\ndata: null\n\n')
|
||||
keepAlive = setInterval(() => subscriber!.send(': keep-alive\n\n'), this.keepAliveMs)
|
||||
;(keepAlive as { unref?: () => void }).unref?.()
|
||||
},
|
||||
cancel: () => {
|
||||
if (subscriber) this.subscribers.delete(subscriber)
|
||||
clearInterval(keepAlive)
|
||||
},
|
||||
})
|
||||
return new Response(stream, { headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' } })
|
||||
}
|
||||
|
||||
private scan(options: ScanOptions): Response {
|
||||
const iterator = this.storage.scan(options)[Symbol.asyncIterator]()
|
||||
const encoder = new TextEncoder()
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
// Send in small batches so short reads (skip-scan seeks) don't make the server run far ahead.
|
||||
let chunk = ''
|
||||
for (let i = 0; i < 32; i++) {
|
||||
const next = await iterator.next()
|
||||
if (next.done) {
|
||||
if (chunk) controller.enqueue(encoder.encode(chunk))
|
||||
return controller.close()
|
||||
}
|
||||
chunk += ExtendedJson.stringify(options.values ? next.value : [next.value[0]]) + '\n'
|
||||
}
|
||||
controller.enqueue(encoder.encode(chunk))
|
||||
},
|
||||
async cancel() {
|
||||
await iterator.return?.()
|
||||
},
|
||||
})
|
||||
return new Response(stream, { headers: { 'content-type': 'application/x-ndjson' } })
|
||||
}
|
||||
|
||||
private static error(status: number, message: string): Response {
|
||||
return Response.json({ error: message }, { status })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { AuthError, StorageUnavailableError } from '../utils/errors.js'
|
||||
import { ExtendedJson } from '../utils/extended-json.js'
|
||||
import { RequestAuth, type RequestSigner } from '../utils/request-auth.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js'
|
||||
import { SSESession } from '../utils/sse-session/sse-session.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
|
||||
export interface KVStorageHttpOptions {
|
||||
/** Base URL of a `KVStorageHttpServer`, e.g. `https://host/kv`. */
|
||||
url: string
|
||||
/** Signs every request (required by servers with authentication). */
|
||||
signer?: RequestSigner
|
||||
headers?: Record<string, string>
|
||||
fetch?: (url: string, init: RequestInit) => Promise<Response>
|
||||
/** Called when the change stream fails (it keeps reconnecting). Default: ignore. */
|
||||
onSubscriptionError?: (error: Error) => void
|
||||
}
|
||||
|
||||
// Types only: children of a KVStorageHttp are KVStorageHttp nodes too (child() creates them with
|
||||
// Object.create, so they share this class's prototype).
|
||||
export interface KVStorageHttp<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): KVStorageHttp<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* KV storage client for a `KVStorageHttpServer`. Same API as any other KV storage.
|
||||
*
|
||||
* `subscribe(path)` relays every change the server pushes for a key and its subtree (including
|
||||
* this client's own writes) as `change` events with `origin: 'remote'`. When a dropped stream reconnects, `resync` is emitted: changes may have been
|
||||
* missed, so re-read what you need. With a `signer`, every request is signed. Values travel as
|
||||
* Extended JSON (`bigint` and `Uint8Array` survive).
|
||||
*/
|
||||
export class KVStorageHttp<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly url: string
|
||||
private readonly signer: RequestSigner | undefined
|
||||
private readonly headers: Record<string, string>
|
||||
private readonly fetcher: (url: string, init: RequestInit) => Promise<Response>
|
||||
private readonly onSubscriptionError: (error: Error) => void
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state: { paths: Set<string>; session?: SSESession; updating: Promise<unknown> } = { paths: new Set(), updating: Promise.resolve() }
|
||||
|
||||
constructor(options: KVStorageHttpOptions) {
|
||||
super()
|
||||
this.url = options.url.replace(/\/$/, '')
|
||||
this.signer = options.signer
|
||||
this.headers = options.headers ?? {}
|
||||
this.fetcher = options.fetch ?? ((url, init) => globalThis.fetch(url, init))
|
||||
this.onSubscriptionError = options.onSubscriptionError ?? (() => {})
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
if (keys.length === 0) return []
|
||||
const { values, missing } = ExtendedJson.parse(await (await this.request('readMany', { keys })).text()) as { values: unknown[]; missing: number[] }
|
||||
for (const i of missing) values[i] = undefined
|
||||
return values
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
if (options.limit !== undefined && options.limit <= 0) return
|
||||
const response = await this.request('scan', options)
|
||||
const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader()
|
||||
let buffer = ''
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) return
|
||||
buffer += value
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop()!
|
||||
for (const line of lines) {
|
||||
const [key, entryValue] = ExtendedJson.parse(line) as [string, unknown?]
|
||||
yield [key, entryValue]
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Relays changes to `path` (default: this node's key) and everything under it. Resolves once
|
||||
* the stream is connected. While the server is unreachable this stays pending, so don't await it
|
||||
* on a path that must work offline. A 401/403 rejects instead of retrying.
|
||||
*/
|
||||
subscribe(path: string = this.key): Promise<void> {
|
||||
return this.updateSubscriptions(() => this.state.paths.add(path))
|
||||
}
|
||||
|
||||
/** Stops relaying changes to `path` (default: this node's key). */
|
||||
unsubscribe(path: string = this.key): Promise<void> {
|
||||
return this.updateSubscriptions(() => this.state.paths.delete(path))
|
||||
}
|
||||
|
||||
/** Drops every subscription and closes the stream. */
|
||||
close(): Promise<void> {
|
||||
return this.updateSubscriptions(() => this.state.paths.clear())
|
||||
}
|
||||
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
await this.request('write', { changes })
|
||||
}
|
||||
|
||||
/** Applies a subscription change (one at a time), then reconnects with the new path list. */
|
||||
private updateSubscriptions(change: () => unknown): Promise<void> {
|
||||
const run = async () => {
|
||||
const before = JSON.stringify([...this.state.paths])
|
||||
change()
|
||||
if (JSON.stringify([...this.state.paths]) === before && (this.state.session || this.state.paths.size === 0)) return
|
||||
await this.state.session?.disconnect()
|
||||
this.state.session = undefined
|
||||
if (this.state.paths.size > 0) await this.connect([...this.state.paths])
|
||||
}
|
||||
const result = this.state.updating.then(run)
|
||||
this.state.updating = result.catch(() => {})
|
||||
return result
|
||||
}
|
||||
|
||||
private async connect(paths: string[]): Promise<void> {
|
||||
const url = `${this.url}/events`
|
||||
const body = ExtendedJson.stringify({ paths })
|
||||
const session = new SSESession(url, {
|
||||
method: 'POST',
|
||||
body,
|
||||
fetch: this.fetcher,
|
||||
headers: { ...this.headers, 'content-type': 'application/json' },
|
||||
persistent: true,
|
||||
onError: this.onSubscriptionError,
|
||||
retry: KVStorageHttp.retryUnlessDenied(),
|
||||
// Signed on every (re)connect, so the timestamp is always fresh.
|
||||
onRequest: async (request) => ({ ...request, headers: { ...(request.headers as Record<string, string>), ...(await this.authHeaders(url, body)) } }),
|
||||
})
|
||||
let readies = 0
|
||||
session.on('message', (message) => {
|
||||
if (message.event === 'change') this.emit('change', { changes: ExtendedJson.parse(message.data) as KVChange[], origin: 'remote' })
|
||||
// Every `ready` after the first is a reconnect: changes may have been missed meanwhile.
|
||||
if (message.event === 'ready' && readies++ > 0) this.emit('resync', undefined)
|
||||
})
|
||||
this.state.session = session
|
||||
const ready = session.waitFor('message', (message) => message.event === 'ready')
|
||||
try {
|
||||
await session.connect()
|
||||
} catch (error) {
|
||||
this.state.session = undefined
|
||||
throw (error as Error).cause instanceof AuthError ? (error as Error).cause : error
|
||||
}
|
||||
await ready
|
||||
}
|
||||
|
||||
private async request(path: string, payload: unknown): Promise<Response> {
|
||||
const url = `${this.url}/${path}`
|
||||
const body = ExtendedJson.stringify(payload)
|
||||
let response: Response
|
||||
try {
|
||||
response = await this.fetcher(url, {
|
||||
method: 'POST',
|
||||
headers: { ...this.headers, 'content-type': 'application/json', ...(await this.authHeaders(url, body)) },
|
||||
body,
|
||||
})
|
||||
} catch (error) {
|
||||
throw new StorageUnavailableError(`Cannot reach ${this.url}`, { cause: error })
|
||||
}
|
||||
if ([502, 503, 504].includes(response.status)) throw new StorageUnavailableError(`HTTP ${response.status} from ${this.url}`)
|
||||
if (response.status === 401 || response.status === 403) throw new AuthError(response.status, await response.text())
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`)
|
||||
return response
|
||||
}
|
||||
|
||||
private async authHeaders(url: string, body: string): Promise<Record<string, string>> {
|
||||
return this.signer ? RequestAuth.headers(this.signer, new URL(url).pathname, body) : {}
|
||||
}
|
||||
|
||||
/** Retries connecting with backoff, except when the server denies access (401/403): that won't fix itself. */
|
||||
private static retryUnlessDenied(): { run<T>(fn: () => Promise<T>): Promise<T> } {
|
||||
const backoff = new ExponentialBackoff({ baseDelay: 1000, maxDelay: 10_000, maxAttempts: 0, growthRate: 1.3, jitter: 0.3 })
|
||||
return {
|
||||
run: (fn) =>
|
||||
backoff.run(fn, (error, context) => {
|
||||
const status = /^HTTP error: (40[13])/.exec(error.message)?.[1]
|
||||
if (status) context.abort(new AuthError(Number(status) as 401 | 403, error.message))
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { Json } from '../utils/json.js'
|
||||
import { SortedMap } from '../utils/sorted-map.js'
|
||||
import type { AnySpec, Spec } from '../utils/schema.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
|
||||
/** In-memory KV storage. Values are deep-copied on write and read. */
|
||||
export class KVStorageInMemory<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly data = new SortedMap<unknown>()
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
return keys.map((k) => Json.clone(this.data.get(k)))
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
let remaining = options.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
for (const key of this.data.keys(options.gte, options.lt, options.reverse)) {
|
||||
yield [key, options.values ? Json.clone(this.data.get(key)) : undefined]
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
for (const { key, value } of changes) {
|
||||
if (value === undefined) this.data.delete(key)
|
||||
else this.data.set(key, Json.clone(value))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { AnySpec, Spec } from '../utils/schema.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
|
||||
export interface KVStorageIndexedDBOptions {
|
||||
/** Database name. */
|
||||
name: string
|
||||
/** Object store name. Default `kv`. */
|
||||
storeName?: string
|
||||
/** IndexedDB factory. Default `globalThis.indexedDB`. */
|
||||
indexedDB?: IDBFactory
|
||||
/** Entries read per transaction while scanning. Default 256. */
|
||||
scanBatchSize?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* KV storage in the browser's IndexedDB (one object store, string keys). Values are stored with the
|
||||
* structured clone algorithm, so `bigint` and `Uint8Array` survive. IndexedDB orders string keys by
|
||||
* UTF-16 code unit, like JavaScript string comparison.
|
||||
*/
|
||||
export class KVStorageIndexedDB<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly storeName: string
|
||||
private readonly scanBatchSize: number
|
||||
private readonly db: Promise<IDBDatabase>
|
||||
|
||||
constructor(options: KVStorageIndexedDBOptions) {
|
||||
super()
|
||||
this.storeName = options.storeName ?? 'kv'
|
||||
this.scanBatchSize = options.scanBatchSize ?? 256
|
||||
const factory = options.indexedDB ?? globalThis.indexedDB
|
||||
const open = factory.open(options.name, 1)
|
||||
open.onupgradeneeded = () => open.result.createObjectStore(this.storeName)
|
||||
this.db = KVStorageIndexedDB.done(open)
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
const store = await this.store('readonly')
|
||||
return Promise.all(keys.map((key) => KVStorageIndexedDB.done(store.get(key))))
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
let remaining = options.limit ?? Infinity
|
||||
let last: string | undefined
|
||||
while (remaining > 0) {
|
||||
const batch = await this.readBatch(options, last, Math.min(remaining, this.scanBatchSize))
|
||||
for (const entry of batch) {
|
||||
yield entry
|
||||
remaining--
|
||||
}
|
||||
if (batch.length < this.scanBatchSize) return
|
||||
last = batch.at(-1)![0]
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the database connection. */
|
||||
async close(): Promise<void> {
|
||||
;(await this.db).close()
|
||||
}
|
||||
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
const store = await this.store('readwrite')
|
||||
for (const { key, value } of changes) {
|
||||
if (value === undefined) store.delete(key)
|
||||
else store.put(value, key)
|
||||
}
|
||||
await KVStorageIndexedDB.committed(store.transaction)
|
||||
}
|
||||
|
||||
/** Up to `count` entries after `last` (exclusive), in scan order, within one transaction. */
|
||||
private async readBatch(options: ScanOptions, last: string | undefined, count: number): Promise<[string, unknown][]> {
|
||||
const reverse = options.reverse ?? false
|
||||
// The upper bound is always exclusive (`lt`, or the last key seen in reverse); the lower bound
|
||||
// is exclusive only when resuming forwards after `last`.
|
||||
const lower = reverse ? options.gte : (last ?? options.gte)
|
||||
const upper = reverse ? (last ?? options.lt) : options.lt
|
||||
const lowerOpen = !reverse && last !== undefined
|
||||
if (lower !== undefined && upper !== undefined && lower >= upper) return []
|
||||
let range: IDBKeyRange | undefined
|
||||
if (lower !== undefined && upper !== undefined) range = IDBKeyRange.bound(lower, upper, lowerOpen, true)
|
||||
else if (lower !== undefined) range = IDBKeyRange.lowerBound(lower, lowerOpen)
|
||||
else if (upper !== undefined) range = IDBKeyRange.upperBound(upper, true)
|
||||
const store = await this.store('readonly')
|
||||
const out: [string, unknown][] = []
|
||||
const request = options.values ? store.openCursor(range, reverse ? 'prev' : 'next') : store.openKeyCursor(range, reverse ? 'prev' : 'next')
|
||||
return new Promise((resolve, reject) => {
|
||||
request.onerror = () => reject(request.error)
|
||||
request.onsuccess = () => {
|
||||
const cursor = request.result
|
||||
if (!cursor || out.length >= count) return resolve(out)
|
||||
out.push([cursor.key as string, options.values ? (cursor as IDBCursorWithValue).value : undefined])
|
||||
cursor.continue()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private async store(mode: IDBTransactionMode): Promise<IDBObjectStore> {
|
||||
return (await this.db).transaction(this.storeName, mode).objectStore(this.storeName)
|
||||
}
|
||||
|
||||
private static done<T>(request: IDBRequest<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
request.onsuccess = () => resolve(request.result)
|
||||
request.onerror = () => reject(request.error)
|
||||
})
|
||||
}
|
||||
|
||||
private static committed(transaction: IDBTransaction): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
transaction.oncomplete = () => resolve()
|
||||
transaction.onerror = transaction.onabort = () => reject(transaction.error)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
import { escapeSegment, succ } from '../core/path.js'
|
||||
import type { KVDriver } from './driver.js'
|
||||
|
||||
/**
|
||||
* Direct-children skip-scan, shared by every layer.
|
||||
*
|
||||
* ## Child order
|
||||
*
|
||||
* Children are ordered by **escaped segment followed by the separator** ("subtree order": the
|
||||
* order in which their descendants appear in key space). For segments without characters that
|
||||
* sort below the separator this is plain lexicographic order. When one segment extends another
|
||||
* with such a character (`a` vs `a-1` with `/`), the longer one comes first (`a-1/` < `a/`).
|
||||
*
|
||||
* This is the one order that can be produced without extra reads: a child's own key
|
||||
* (`p/a`) sorts *before* siblings that extend it (`p/a-1`), while its descendants (`p/a/…`) sort
|
||||
* after them. The forward scan therefore defers yielding a child seen only through its own key
|
||||
* until the scan passes `p/a/`, using a stack bounded by segment length.
|
||||
*
|
||||
* ## Cost
|
||||
*
|
||||
* - Forward: one seek per child that has descendants (to jump over its subtree), one at the start,
|
||||
* and one per reserved (`~…`) region. Siblings without descendants share an iterator.
|
||||
* `after` adds one `getMany` when the `after` segment contains a below-separator character.
|
||||
* - Reverse: as forward, plus one `getMany` per newly found child whose segment contains a
|
||||
* below-separator character (to find shorter siblings that only have an own key), or that was
|
||||
* found through a descendant when `own` is requested.
|
||||
*/
|
||||
export interface ChildHit {
|
||||
/** Escaped segment. */
|
||||
esc: string
|
||||
/** Whether the child has an own value. Only accurate when `own` was requested. */
|
||||
hasOwn: boolean
|
||||
/** The own value, when `values` was requested and `hasOwn`. */
|
||||
value: unknown
|
||||
}
|
||||
|
||||
export interface ChildScanOptions {
|
||||
/** Unescaped segment to start after (exclusive), in the direction of travel. */
|
||||
after?: string
|
||||
reverse?: boolean
|
||||
limit?: number
|
||||
/** Fetch own values. Implies `own`. */
|
||||
values?: boolean
|
||||
/** Require accurate `hasOwn`. */
|
||||
own?: boolean
|
||||
}
|
||||
|
||||
const RESERVED_START = '~'
|
||||
const RESERVED_END = succ('~')
|
||||
|
||||
export async function* scanChildren(
|
||||
driver: KVDriver,
|
||||
prefix: string,
|
||||
end: string | undefined,
|
||||
sep: string,
|
||||
opts: ChildScanOptions = {},
|
||||
): AsyncGenerator<ChildHit> {
|
||||
const limit = opts.limit ?? Infinity
|
||||
if (limit <= 0) return
|
||||
if (opts.reverse) yield* reverseScan(driver, prefix, end, sep, opts, limit)
|
||||
else yield* forwardScan(driver, prefix, end, sep, opts, limit)
|
||||
}
|
||||
|
||||
/** Prefixes `E[0..j)` (j ≥ 1) where `E[j]` sorts below the separator. */
|
||||
function lowPrefixes(esc: string, sepCode: number): string[] {
|
||||
let out: string[] | undefined
|
||||
for (let j = 1; j < esc.length; j++) {
|
||||
if (esc.charCodeAt(j) < sepCode) (out ??= []).push(esc.slice(0, j))
|
||||
}
|
||||
return out ?? NONE
|
||||
}
|
||||
|
||||
const NONE: string[] = []
|
||||
|
||||
async function* forwardScan(
|
||||
driver: KVDriver,
|
||||
prefix: string,
|
||||
end: string | undefined,
|
||||
sep: string,
|
||||
opts: ChildScanOptions,
|
||||
limit: number,
|
||||
): AsyncGenerator<ChildHit> {
|
||||
const sepCode = sep.charCodeAt(0)
|
||||
const succSep = succ(sep)
|
||||
const values = opts.values === true
|
||||
const pending: ChildHit[] = []
|
||||
let emitted = 0
|
||||
let gte = prefix
|
||||
|
||||
if (opts.after !== undefined) {
|
||||
const escA = escapeSegment(opts.after, sep)
|
||||
gte = prefix + escA + succSep
|
||||
// Shorter siblings that extend into `after` with a low char come after it, but their own
|
||||
// keys sort before `gte`; probe for them.
|
||||
const cands = lowPrefixes(escA, sepCode)
|
||||
if (cands.length) {
|
||||
const found = await driver.getMany(cands.map((g) => prefix + g))
|
||||
for (let i = 0; i < cands.length; i++) {
|
||||
if (found[i] !== undefined) pending.push({ esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const prefixLen = prefix.length
|
||||
let it = driver.scan({ gte, lt: end, values })[Symbol.asyncIterator]()
|
||||
const reseek = async (from: string) => {
|
||||
await it.return?.()
|
||||
it = driver.scan({ gte: from, lt: end, values })[Symbol.asyncIterator]()
|
||||
}
|
||||
|
||||
try {
|
||||
outer: for (;;) {
|
||||
const r = await it.next()
|
||||
if (r.done) {
|
||||
while (pending.length) {
|
||||
yield pending.pop()!
|
||||
if (++emitted >= limit) return
|
||||
}
|
||||
return
|
||||
}
|
||||
const key = r.value[0]
|
||||
const cut = key.indexOf(sep, prefixLen)
|
||||
const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut)
|
||||
|
||||
while (pending.length) {
|
||||
const top = pending[pending.length - 1]!
|
||||
if (top.esc === esc) {
|
||||
// First descendant of a deferred child: emit it and jump over its subtree.
|
||||
pending.pop()
|
||||
yield top
|
||||
if (++emitted >= limit) return
|
||||
await reseek(prefix + esc + succSep)
|
||||
continue outer
|
||||
}
|
||||
if (top.esc + sep < esc + sep) {
|
||||
pending.pop()
|
||||
yield top
|
||||
if (++emitted >= limit) return
|
||||
} else break
|
||||
}
|
||||
|
||||
if (esc.startsWith(RESERVED_START)) {
|
||||
await reseek(prefix + RESERVED_END)
|
||||
continue
|
||||
}
|
||||
if (cut === -1) {
|
||||
// Own key: defer until we know no sibling extending it with a low char comes next.
|
||||
pending.push({ esc, hasOwn: true, value: r.value[1] })
|
||||
continue
|
||||
}
|
||||
// Descendant of a child with no own value (an own key would have sorted first).
|
||||
yield { esc, hasOwn: false, value: undefined }
|
||||
if (++emitted >= limit) return
|
||||
await reseek(prefix + esc + succSep)
|
||||
}
|
||||
} finally {
|
||||
await it.return?.()
|
||||
}
|
||||
}
|
||||
|
||||
async function* reverseScan(
|
||||
driver: KVDriver,
|
||||
prefix: string,
|
||||
end: string | undefined,
|
||||
sep: string,
|
||||
opts: ChildScanOptions,
|
||||
limit: number,
|
||||
): AsyncGenerator<ChildHit> {
|
||||
const sepCode = sep.charCodeAt(0)
|
||||
const values = opts.values === true
|
||||
const needOwn = values || opts.own === true
|
||||
// Segments already emitted whose own key may still appear (it sorts after everything else of theirs).
|
||||
const done: string[] = []
|
||||
let emitted = 0
|
||||
let lt = end
|
||||
|
||||
if (opts.after !== undefined) {
|
||||
const escA = escapeSegment(opts.after, sep)
|
||||
lt = prefix + escA + sep
|
||||
// `after` itself and shorter siblings extending into it with a low char sort after it in
|
||||
// child order but their own keys fall below `lt`: mark them done.
|
||||
done.push(...lowPrefixes(escA, sepCode), escA)
|
||||
}
|
||||
|
||||
const prefixLen = prefix.length
|
||||
let it = driver.scan({ gte: prefix, lt, reverse: true, values })[Symbol.asyncIterator]()
|
||||
const reseek = async (below: string) => {
|
||||
await it.return?.()
|
||||
it = driver.scan({ gte: prefix, lt: below, reverse: true, values })[Symbol.asyncIterator]()
|
||||
}
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const r = await it.next()
|
||||
if (r.done) return
|
||||
const key = r.value[0]
|
||||
const cut = key.indexOf(sep, prefixLen)
|
||||
const esc = cut === -1 ? key.slice(prefixLen) : key.slice(prefixLen, cut)
|
||||
|
||||
if (esc.startsWith(RESERVED_START)) {
|
||||
await reseek(prefix + RESERVED_START)
|
||||
continue
|
||||
}
|
||||
while (done.length && !esc.startsWith(done[done.length - 1]!)) done.pop()
|
||||
if (done.length && done[done.length - 1] === esc) {
|
||||
done.pop()
|
||||
if (cut !== -1) await reseek(prefix + esc + sep)
|
||||
continue
|
||||
}
|
||||
|
||||
// New child. Shorter siblings that it extends with a low char come before it in reverse
|
||||
// order; any with descendants were already emitted, so only own-key-only ones are unknown.
|
||||
const cands = lowPrefixes(esc, sepCode).filter((g) => !done.includes(g))
|
||||
const probeOwn = cut !== -1 && needOwn
|
||||
let found: unknown[] = NONE
|
||||
if (cands.length || probeOwn) {
|
||||
const probe = cands.map((g) => prefix + g)
|
||||
if (probeOwn) probe.push(prefix + esc)
|
||||
found = await driver.getMany(probe)
|
||||
}
|
||||
for (let i = 0; i < cands.length; i++) {
|
||||
if (found[i] === undefined) continue
|
||||
yield { esc: cands[i]!, hasOwn: true, value: values ? found[i] : undefined }
|
||||
done.push(cands[i]!)
|
||||
if (++emitted >= limit) return
|
||||
}
|
||||
const own = cut === -1 ? r.value[1] : probeOwn ? found[cands.length] : undefined
|
||||
yield { esc, hasOwn: cut === -1 || own !== undefined, value: values ? own : undefined }
|
||||
done.push(esc)
|
||||
if (++emitted >= limit) return
|
||||
if (cut !== -1) await reseek(prefix + esc + sep)
|
||||
}
|
||||
} finally {
|
||||
await it.return?.()
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
export interface ScanOptions {
|
||||
gte?: string
|
||||
lt?: string
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
/** When false/omitted, yielded values are `undefined`. */
|
||||
values?: boolean
|
||||
}
|
||||
|
||||
export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string }
|
||||
|
||||
/**
|
||||
* A flat, string-keyed backend. Values are JSON-serialisable; byte-oriented backends serialise
|
||||
* internally with a {@link Codec}. All path, index and typing logic lives above this interface.
|
||||
*/
|
||||
export interface KVDriver {
|
||||
/** Same order as `keys`; `undefined` for missing keys. */
|
||||
getMany(keys: string[]): Promise<(unknown | undefined)[]>
|
||||
setMany(entries: [key: string, value: unknown][]): Promise<void>
|
||||
deleteMany(keys: string[]): Promise<void>
|
||||
/** MUST yield in key order (descending when `reverse`). `gte` is inclusive, `lt` exclusive. */
|
||||
scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]>
|
||||
|
||||
// Optional capabilities. Stores feature-detect them and fall back where possible.
|
||||
/** Writes each entry only if its key is absent. `true` = written. */
|
||||
setIfAbsent?(entries: [string, unknown][]): Promise<boolean[]>
|
||||
/** Applies all ops atomically. */
|
||||
batch?(ops: BatchOp[]): Promise<void>
|
||||
deleteRange?(gte: string, lt: string): Promise<void>
|
||||
}
|
||||
|
||||
/** Serialises values for byte/string-oriented backends. */
|
||||
export interface Codec {
|
||||
encode(value: unknown): string
|
||||
decode(encoded: string): unknown
|
||||
}
|
||||
|
||||
export const JsonCodec: Codec = {
|
||||
encode: (value) => JSON.stringify(value),
|
||||
decode: (encoded) => JSON.parse(encoded),
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for drivers: implement the four required methods and get `deleteRange`
|
||||
* (scan + deleteMany in chunks) and single-key helpers for free.
|
||||
*/
|
||||
export abstract class BaseKVDriver implements KVDriver {
|
||||
abstract getMany(keys: string[]): Promise<(unknown | undefined)[]>
|
||||
abstract setMany(entries: [string, unknown][]): Promise<void>
|
||||
abstract deleteMany(keys: string[]): Promise<void>
|
||||
abstract scan(opts: ScanOptions): AsyncIterable<[string, unknown]>
|
||||
|
||||
/** Chunk size used by the default `deleteRange`. */
|
||||
protected deleteRangeChunk = 1000
|
||||
|
||||
async deleteRange(gte: string, lt: string): Promise<void> {
|
||||
for (;;) {
|
||||
const keys: string[] = []
|
||||
for await (const [key] of this.scan({ gte, lt, limit: this.deleteRangeChunk })) keys.push(key)
|
||||
if (keys.length === 0) return
|
||||
await this.deleteMany(keys)
|
||||
if (keys.length < this.deleteRangeChunk) return
|
||||
}
|
||||
}
|
||||
|
||||
async get(key: string): Promise<unknown | undefined> {
|
||||
return (await this.getMany([key]))[0]
|
||||
}
|
||||
|
||||
set(key: string, value: unknown): Promise<void> {
|
||||
return this.setMany([[key, value]])
|
||||
}
|
||||
|
||||
delete(key: string): Promise<void> {
|
||||
return this.deleteMany([key])
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
export * from './driver.js'
|
||||
export * from './memory.js'
|
||||
export * from './sorted.js'
|
||||
export * from './node.js'
|
||||
export * from './indexes.js'
|
||||
export * from './children.js'
|
||||
@@ -1,157 +0,0 @@
|
||||
import { indexValueCodec, isIndexValue, type IndexValue, type IndexValueCodec } from '../core/encoding.js'
|
||||
import { escapeSegment, RESERVED, succ } from '../core/path.js'
|
||||
import { PatternTrie, patternsOverlap, type TrieState } from '../core/trie.js'
|
||||
|
||||
export type { IndexValue }
|
||||
|
||||
/** Maps a child's value to index value(s). An array means multi-entry; `undefined` means not indexed. */
|
||||
export type IndexFn<V = any> = (value: V) => IndexValue | readonly IndexValue[] | undefined
|
||||
|
||||
/** Index name → function, for the direct children of nodes matching one pattern. */
|
||||
export type IndexDefs = Record<string, IndexFn>
|
||||
|
||||
/** Path pattern (`*` = one segment, `''` = root) → index definitions. */
|
||||
export type IndexDeclarations = Record<string, IndexDefs>
|
||||
|
||||
/** A value or a range over index values. Bounds combine (`{ gte, lt }`). */
|
||||
export type IndexQuery = IndexValue | IndexRange
|
||||
|
||||
export interface IndexRange {
|
||||
gt?: IndexValue
|
||||
gte?: IndexValue
|
||||
lt?: IndexValue
|
||||
lte?: IndexValue
|
||||
}
|
||||
|
||||
export function isIndexRange(q: unknown): q is IndexRange {
|
||||
return typeof q === 'object' && q !== null
|
||||
}
|
||||
|
||||
interface CompiledIndex {
|
||||
name: string
|
||||
/** `escapeSegment(name) + sep` */
|
||||
head: string
|
||||
fn: IndexFn
|
||||
}
|
||||
|
||||
/** Indexes over the direct children of one concrete node. Entry keys are rooted at `base`. */
|
||||
export class BoundIndexes {
|
||||
readonly #byName: Map<string, CompiledIndex>
|
||||
readonly #list: CompiledIndex[]
|
||||
readonly #codec: IndexValueCodec
|
||||
|
||||
constructor(
|
||||
defs: CompiledIndex[],
|
||||
/** `<nodePrefix>~i<sep>` */
|
||||
readonly base: string,
|
||||
readonly sep: string,
|
||||
) {
|
||||
this.#list = defs
|
||||
this.#byName = new Map(defs.map((d) => [d.name, d]))
|
||||
this.#codec = indexValueCodec(sep)
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.#byName.has(name)
|
||||
}
|
||||
|
||||
/** All index entry keys for a child (escaped segment) holding `value`. */
|
||||
entriesFor(escChild: string, value: unknown, out: Set<string> = new Set()): Set<string> {
|
||||
if (value === undefined) return out
|
||||
for (const idx of this.#list) this.#entriesOf(idx, escChild, value, out)
|
||||
return out
|
||||
}
|
||||
|
||||
#entriesOf(idx: CompiledIndex, escChild: string, value: unknown, out: Set<string>): void {
|
||||
const r = idx.fn(value)
|
||||
if (r === undefined) return
|
||||
const stem = this.base + idx.head
|
||||
const tail = this.sep + escChild
|
||||
if (Array.isArray(r)) {
|
||||
for (const v of r as IndexValue[]) out.add(stem + this.#encode(idx.name, v) + tail)
|
||||
} else out.add(stem + this.#encode(idx.name, r as IndexValue) + tail)
|
||||
}
|
||||
|
||||
#encode(name: string, v: unknown): string {
|
||||
if (!isIndexValue(v)) throw new TypeError(`Index "${name}" returned a non-indexable value: ${String(v)}`)
|
||||
return this.#codec.encode(v)
|
||||
}
|
||||
|
||||
/** Whether `entryKey` is among the entries `value` currently produces for index `name`. */
|
||||
matches(name: string, escChild: string, value: unknown, entryKey: string): boolean {
|
||||
if (value === undefined) return false
|
||||
const out = new Set<string>()
|
||||
this.#entriesOf(this.#get(name), escChild, value, out)
|
||||
return out.has(entryKey)
|
||||
}
|
||||
|
||||
/** Scan bounds over the entries of index `name` for a query. */
|
||||
range(name: string, q: IndexQuery): { gte: string; lt: string } {
|
||||
const stem = this.base + this.#get(name).head
|
||||
const succSep = succ(this.sep)
|
||||
const enc = (v: IndexValue) => stem + this.#encode(name, v)
|
||||
if (!isIndexRange(q)) {
|
||||
const k = enc(q)
|
||||
return { gte: k + this.sep, lt: k + succSep }
|
||||
}
|
||||
let gte = stem
|
||||
let lt = stem.slice(0, -1) + succSep
|
||||
if (q.gte !== undefined) gte = maxStr(gte, enc(q.gte))
|
||||
if (q.gt !== undefined) gte = maxStr(gte, enc(q.gt) + succSep)
|
||||
if (q.lt !== undefined) lt = minStr(lt, enc(q.lt))
|
||||
if (q.lte !== undefined) lt = minStr(lt, enc(q.lte) + succSep)
|
||||
return { gte, lt }
|
||||
}
|
||||
|
||||
/** Splits an entry key under index `name` into the escaped child segment. */
|
||||
childOf(name: string, entryKey: string): string {
|
||||
const start = this.base.length + this.#get(name).head.length
|
||||
return entryKey.slice(entryKey.indexOf(this.sep, start) + 1)
|
||||
}
|
||||
|
||||
#get(name: string): CompiledIndex {
|
||||
const idx = this.#byName.get(name)
|
||||
if (!idx) throw new TypeError(`Unknown index "${name}"`)
|
||||
return idx
|
||||
}
|
||||
}
|
||||
|
||||
const maxStr = (a: string, b: string) => (a > b ? a : b)
|
||||
const minStr = (a: string, b: string) => (a < b ? a : b)
|
||||
|
||||
/** Compiles index declarations into a pattern trie and binds them to nodes. */
|
||||
export class IndexRegistry {
|
||||
readonly trie: PatternTrie<IndexDefs>
|
||||
readonly empty: boolean
|
||||
|
||||
constructor(
|
||||
decls: IndexDeclarations,
|
||||
readonly sep: string,
|
||||
) {
|
||||
this.trie = new PatternTrie(sep)
|
||||
const patterns = Object.keys(decls)
|
||||
for (let i = 0; i < patterns.length; i++) {
|
||||
for (let j = i + 1; j < patterns.length; j++) {
|
||||
const a = patterns[i]!
|
||||
const b = patterns[j]!
|
||||
if (!patternsOverlap(a, b, sep)) continue
|
||||
const clash = Object.keys(decls[a]!).filter((n) => n in decls[b]!)
|
||||
if (clash.length) throw new TypeError(`Index name(s) ${clash.join(', ')} declared on overlapping patterns "${a}" and "${b}"`)
|
||||
}
|
||||
}
|
||||
for (const p of patterns) this.trie.add(p, decls[p]!)
|
||||
this.empty = patterns.length === 0
|
||||
}
|
||||
|
||||
/** Indexes over the children of the node with trie state `state` and key prefix `prefix`. */
|
||||
bind(state: TrieState<IndexDefs>, prefix: string): BoundIndexes | undefined {
|
||||
if (this.empty) return undefined
|
||||
const found = PatternTrie.values(state)
|
||||
if (found.length === 0) return undefined
|
||||
const defs: CompiledIndex[] = []
|
||||
for (const { value } of found) {
|
||||
for (const [name, fn] of Object.entries(value)) defs.push({ name, head: escapeSegment(name, this.sep) + this.sep, fn })
|
||||
}
|
||||
return new BoundIndexes(defs, prefix + RESERVED.index + this.sep, this.sep)
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import { BaseKVDriver, type BatchOp, type ScanOptions } from './driver.js'
|
||||
import { ChunkedSortedMap, type SortedMap } from './sorted.js'
|
||||
|
||||
export interface InMemoryKVDriverOptions {
|
||||
/**
|
||||
* Deep-copy values on write and read so callers can't mutate stored state through a shared
|
||||
* reference (matching serialising drivers). Values must be JSON-compatible. Default `true`.
|
||||
* Disable for speed when values are treated as immutable.
|
||||
*/
|
||||
clone?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory driver over a sorted map. Implements every optional capability:
|
||||
* `setIfAbsent`, atomic `batch` and `deleteRange`.
|
||||
*/
|
||||
export class InMemoryKVDriver extends BaseKVDriver {
|
||||
readonly #data: SortedMap<unknown> = new ChunkedSortedMap()
|
||||
readonly #clone: (v: unknown) => unknown
|
||||
|
||||
constructor(opts: InMemoryKVDriverOptions = {}) {
|
||||
super()
|
||||
this.#clone = opts.clone === false ? identity : cloneValue
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.#data.size
|
||||
}
|
||||
|
||||
async getMany(keys: string[]): Promise<unknown[]> {
|
||||
const out = new Array<unknown>(keys.length)
|
||||
for (let i = 0; i < keys.length; i++) out[i] = this.#clone(this.#data.get(keys[i]!))
|
||||
return out
|
||||
}
|
||||
|
||||
async setMany(entries: [string, unknown][]): Promise<void> {
|
||||
const clone = this.#clone
|
||||
this.#data.setMany(entries.map(([k, v]) => [k, clone(v)]))
|
||||
}
|
||||
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
for (const key of keys) this.#data.delete(key)
|
||||
}
|
||||
|
||||
async setIfAbsent(entries: [string, unknown][]): Promise<boolean[]> {
|
||||
return entries.map(([key, value]) => {
|
||||
if (this.#data.has(key)) return false
|
||||
this.#data.set(key, this.#clone(value))
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
async batch(ops: BatchOp[]): Promise<void> {
|
||||
// Applied synchronously within one microtask, so no other operation can observe a partial batch.
|
||||
for (const op of ops) {
|
||||
if (op.type === 'set') this.#data.set(op.key, this.#clone(op.value))
|
||||
else this.#data.delete(op.key)
|
||||
}
|
||||
}
|
||||
|
||||
override async deleteRange(gte: string, lt: string): Promise<void> {
|
||||
const doomed = [...this.#data.keys({ gte, lt })]
|
||||
for (const key of doomed) this.#data.delete(key)
|
||||
}
|
||||
|
||||
async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> {
|
||||
const { gte, lt, reverse = false, values = false } = opts
|
||||
let remaining = opts.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
// The key iterator re-seeks by itself if the map changes while we're suspended.
|
||||
for (const key of this.#data.keys({ gte, lt, reverse })) {
|
||||
yield [key, values ? this.#clone(this.#data.get(key)) : undefined]
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function identity(v: unknown): unknown {
|
||||
return v
|
||||
}
|
||||
|
||||
/** Deep copy of a JSON-compatible value (much faster than `structuredClone`). */
|
||||
function cloneValue(v: unknown): unknown {
|
||||
if (v === null || typeof v !== 'object') return v
|
||||
if (Array.isArray(v)) {
|
||||
const out = new Array<unknown>(v.length)
|
||||
for (let i = 0; i < v.length; i++) out[i] = cloneValue(v[i])
|
||||
return out
|
||||
}
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const k in v) out[k] = cloneValue((v as Record<string, unknown>)[k])
|
||||
return out
|
||||
}
|
||||
-402
@@ -1,402 +0,0 @@
|
||||
import { DEFAULT_SEPARATOR, escapeSegment, succ, toSegment, unescapeSegment, validateSeparator } from '../core/path.js'
|
||||
import type { TrieState } from '../core/trie.js'
|
||||
import type { AnySpec, ChildOf, EntryOf, KeyOf, KeyString, Spec, StringEntryOf, ValueOf } from '../types/schema.js'
|
||||
import { scanChildren } from './children.js'
|
||||
import type { BatchOp, KVDriver } from './driver.js'
|
||||
import { BoundIndexes, IndexRegistry, isIndexRange, type IndexDeclarations, type IndexDefs, type IndexQuery } from './indexes.js'
|
||||
|
||||
export interface KVStorageOptions {
|
||||
/** Single printable ASCII character other than `%` and `~`. Default `/`. */
|
||||
separator?: string
|
||||
/** Path pattern → index definitions over that node's direct children. */
|
||||
indexes?: IndexDeclarations
|
||||
/**
|
||||
* Delete stale index entries found by `findBy` (read-repair). Default `true`.
|
||||
* See DESIGN.md §12 (M4) for the race with concurrent non-atomic writers.
|
||||
*/
|
||||
readRepair?: boolean
|
||||
}
|
||||
|
||||
export interface KeysOptions {
|
||||
limit?: number
|
||||
/** Start after this child key (exclusive), in the direction of travel. */
|
||||
after?: string | number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
export interface TreeOptions {
|
||||
/** Levels below this node to include; `1` = direct children only. */
|
||||
depth: number
|
||||
}
|
||||
|
||||
export interface FindByOptions {
|
||||
limit?: number
|
||||
reverse?: boolean
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface KVContext {
|
||||
readonly driver: KVDriver
|
||||
readonly sep: string
|
||||
readonly registry: IndexRegistry
|
||||
readonly readRepair: boolean
|
||||
}
|
||||
|
||||
interface Slot {
|
||||
key: string
|
||||
esc: string
|
||||
value: unknown
|
||||
}
|
||||
|
||||
const FIND_CHUNK = 128
|
||||
const DELETE_CHUNK = 1000
|
||||
|
||||
/**
|
||||
* A store over its direct children. `child(key)` re-roots to that key (sync, no I/O).
|
||||
* Methods take an optional leading key: without it they act on this node's own value,
|
||||
* with it on that child's. The choice is made by argument count; a passed-but-`undefined`
|
||||
* key throws.
|
||||
*/
|
||||
export class KVNode<S extends Spec = AnySpec> {
|
||||
/** Unescaped path segments. */
|
||||
readonly path: readonly string[]
|
||||
/** This node's own key (escaped). `''` for the root. */
|
||||
readonly selfKey: string
|
||||
/** Prefix of every descendant key. `''` for the root. */
|
||||
readonly prefix: string
|
||||
/** Exclusive upper bound of the subtree range; `undefined` for the root. */
|
||||
readonly end: string | undefined
|
||||
|
||||
readonly #ctx: KVContext
|
||||
readonly #esc: string
|
||||
readonly #state: TrieState<IndexDefs>
|
||||
readonly #selfIdx: BoundIndexes | undefined
|
||||
readonly #childIdx: BoundIndexes | undefined
|
||||
|
||||
/** @internal Use `KVStorage` and `child()`. */
|
||||
constructor(ctx: KVContext, path: readonly string[], selfKey: string, esc: string, state: TrieState<IndexDefs>, selfIdx: BoundIndexes | undefined) {
|
||||
this.#ctx = ctx
|
||||
this.path = path
|
||||
this.selfKey = selfKey
|
||||
this.#esc = esc
|
||||
const root = path.length === 0
|
||||
this.prefix = root ? '' : selfKey + ctx.sep
|
||||
this.end = root ? undefined : selfKey + succ(ctx.sep)
|
||||
this.#state = state
|
||||
this.#selfIdx = selfIdx
|
||||
this.#childIdx = ctx.registry.bind(state, this.prefix)
|
||||
}
|
||||
|
||||
get driver(): KVDriver {
|
||||
return this.#ctx.driver
|
||||
}
|
||||
|
||||
get separator(): string {
|
||||
return this.#ctx.sep
|
||||
}
|
||||
|
||||
get isRoot(): boolean {
|
||||
return this.path.length === 0
|
||||
}
|
||||
|
||||
child<K extends KeyOf<S>>(key: K): KVNode<ChildOf<S, K>> {
|
||||
const seg = toSegment(key)
|
||||
const esc = escapeSegment(seg, this.#ctx.sep)
|
||||
return new KVNode(this.#ctx, [...this.path, seg], this.prefix + esc, esc, this.#ctx.registry.trie.step(this.#state, seg), this.#childIdx)
|
||||
}
|
||||
|
||||
/** Re-types this node. No runtime effect. */
|
||||
as<S2 extends Spec>(): KVNode<S2> {
|
||||
return this as unknown as KVNode<S2>
|
||||
}
|
||||
|
||||
get(): Promise<ValueOf<S> | undefined>
|
||||
get<K extends KeyOf<S>>(key: K): Promise<ValueOf<ChildOf<S, K>> | undefined>
|
||||
async get(...args: unknown[]): Promise<unknown> {
|
||||
const t = this.#resolve(args, 0)
|
||||
return (await this.#ctx.driver.getMany([t.key]))[0]
|
||||
}
|
||||
|
||||
async getMany<K extends KeyOf<S>>(keys: readonly K[]): Promise<(ValueOf<ChildOf<S, K>> | undefined)[]> {
|
||||
if (keys.length === 0) return []
|
||||
return (await this.#ctx.driver.getMany(keys.map((k) => this.#childKey(k)))) as (ValueOf<ChildOf<S, K>> | undefined)[]
|
||||
}
|
||||
|
||||
set(value: ValueOf<S>): Promise<void>
|
||||
set<K extends KeyOf<S>>(key: K, value: ValueOf<ChildOf<S, K>>): Promise<void>
|
||||
async set(...args: unknown[]): Promise<void> {
|
||||
const t = this.#resolve(args, 1)
|
||||
const value = args[args.length - 1]
|
||||
if (value === undefined) throw new TypeError('Cannot set undefined; use delete()')
|
||||
return this.#write(t.idx, [{ key: t.key, esc: t.esc, value }])
|
||||
}
|
||||
|
||||
async setMany(entries: readonly EntryOf<S>[]): Promise<void> {
|
||||
if (entries.length === 0) return
|
||||
const slots = entries.map(([k, value]): Slot => {
|
||||
if (value === undefined) throw new TypeError('Cannot set undefined; use delete()')
|
||||
const esc = escapeSegment(toSegment(k), this.#ctx.sep)
|
||||
return { key: this.prefix + esc, esc, value }
|
||||
})
|
||||
return this.#write(this.#childIdx, slots)
|
||||
}
|
||||
|
||||
/** Deletes the own value (or a child's own value) only, not descendants. */
|
||||
delete(): Promise<void>
|
||||
delete<K extends KeyOf<S>>(key: K): Promise<void>
|
||||
async delete(...args: unknown[]): Promise<void> {
|
||||
const t = this.#resolve(args, 0)
|
||||
return this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }])
|
||||
}
|
||||
|
||||
async deleteMany(keys: readonly KeyOf<S>[]): Promise<void> {
|
||||
if (keys.length === 0) return
|
||||
const slots = keys.map((k): Slot => {
|
||||
const esc = escapeSegment(toSegment(k), this.#ctx.sep)
|
||||
return { key: this.prefix + esc, esc, value: undefined }
|
||||
})
|
||||
return this.#write(this.#childIdx, slots)
|
||||
}
|
||||
|
||||
/** Deletes the own value and all descendants (or a child's). On the root, deletes everything. */
|
||||
deleteTree(): Promise<void>
|
||||
deleteTree<K extends KeyOf<S>>(key: K): Promise<void>
|
||||
async deleteTree(...args: unknown[]): Promise<void> {
|
||||
if (args.length === 0 && this.isRoot) return this.#deleteRange('', undefined)
|
||||
const t = this.#resolve(args, 0)
|
||||
await this.#write(t.idx, [{ key: t.key, esc: t.esc, value: undefined }])
|
||||
await this.#deleteRange(t.key + this.#ctx.sep, t.key + succ(this.#ctx.sep))
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct child keys (unescaped), including children that only have descendants.
|
||||
* Order: see `scanChildren` — lexicographic by escaped segment followed by the separator.
|
||||
*/
|
||||
async *keys(opts: KeysOptions = {}): AsyncGenerator<KeyString<S>> {
|
||||
const after = opts.after === undefined ? undefined : toSegment(opts.after)
|
||||
for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, {
|
||||
after,
|
||||
reverse: opts.reverse,
|
||||
limit: opts.limit,
|
||||
})) {
|
||||
yield unescapeSegment(hit.esc) as KeyString<S>
|
||||
}
|
||||
}
|
||||
|
||||
/** Direct children that have an own value, as `[key, value]`. */
|
||||
async *entries(opts: KeysOptions = {}): AsyncGenerator<StringEntryOf<S>> {
|
||||
const after = opts.after === undefined ? undefined : toSegment(opts.after)
|
||||
let remaining = opts.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
for await (const hit of scanChildren(this.#ctx.driver, this.prefix, this.end, this.#ctx.sep, {
|
||||
after,
|
||||
reverse: opts.reverse,
|
||||
values: true,
|
||||
})) {
|
||||
if (!hit.hasOwn) continue
|
||||
yield [unescapeSegment(hit.esc), hit.value] as StringEntryOf<S>
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles descendants up to `depth` levels into a plain object: leaves become values,
|
||||
* nodes with children become objects. A node with both an own value and children exposes the
|
||||
* own value under the `''` property (empty keys are invalid, so it can't collide).
|
||||
* Reserved (`~…`) subtrees are skipped. This node's own value is not included.
|
||||
*/
|
||||
async tree<T = Record<string, unknown>>(opts: TreeOptions): Promise<T> {
|
||||
const { depth } = opts
|
||||
if (!(depth >= 1)) throw new RangeError('tree() depth must be >= 1')
|
||||
const { driver, sep } = this.#ctx
|
||||
type TNode = { own?: unknown; has: boolean; kids?: Map<string, TNode> }
|
||||
const root: TNode = { has: false }
|
||||
const prefix = this.prefix
|
||||
let it = driver.scan({ gte: prefix, lt: this.end, values: true })[Symbol.asyncIterator]()
|
||||
const reseek = async (gte: string) => {
|
||||
await it.return?.()
|
||||
it = driver.scan({ gte, lt: this.end, values: true })[Symbol.asyncIterator]()
|
||||
}
|
||||
try {
|
||||
outer: for (;;) {
|
||||
const r = await it.next()
|
||||
if (r.done) break
|
||||
const [key, value] = r.value
|
||||
const segs = key.slice(prefix.length).split(sep)
|
||||
let base = prefix
|
||||
for (let i = 0; i < segs.length && i < depth; i++) {
|
||||
if (segs[i]!.startsWith('~')) {
|
||||
await reseek(base + succ('~'))
|
||||
continue outer
|
||||
}
|
||||
base += segs[i]! + sep
|
||||
}
|
||||
if (segs.length > depth) {
|
||||
await reseek(prefix + segs.slice(0, depth).join(sep) + succ(sep))
|
||||
continue
|
||||
}
|
||||
let node = root
|
||||
for (const s of segs) {
|
||||
const kids = (node.kids ??= new Map())
|
||||
const seg = unescapeSegment(s)
|
||||
let next = kids.get(seg)
|
||||
if (!next) kids.set(seg, (next = { has: false }))
|
||||
node = next
|
||||
}
|
||||
node.own = value
|
||||
node.has = true
|
||||
}
|
||||
} finally {
|
||||
await it.return?.()
|
||||
}
|
||||
const build = (n: TNode): unknown => {
|
||||
if (!n.kids) return n.own
|
||||
const obj: Record<string, unknown> = {}
|
||||
if (n.has) obj[''] = n.own
|
||||
for (const [k, v] of n.kids) obj[k] = build(v)
|
||||
return obj
|
||||
}
|
||||
return (root.kids ? build(root) : {}) as T
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up direct children by an index declared for this node, by exact value or range.
|
||||
* Each hit's value is re-read and re-checked; stale entries are skipped (and deleted when
|
||||
* `readRepair` is on). Multi-entry hits are de-duplicated.
|
||||
*/
|
||||
async *findBy(index: string, query: IndexQuery, opts: FindByOptions = {}): AsyncGenerator<StringEntryOf<S>> {
|
||||
const idx = this.#childIdx
|
||||
if (!idx || !idx.has(index)) throw new TypeError(`No index "${index}" on node "${this.selfKey}"`)
|
||||
const { driver, readRepair } = this.#ctx
|
||||
const { gte, lt } = idx.range(index, query)
|
||||
let remaining = opts.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
const seen = isIndexRange(query) ? new Set<string>() : undefined
|
||||
let chunk: string[] = []
|
||||
const it = driver.scan({ gte, lt, reverse: opts.reverse })[Symbol.asyncIterator]()
|
||||
|
||||
// Processes one chunk of entry keys; returns false when the limit is reached.
|
||||
const flush = async function* (this: KVNode<S>): AsyncGenerator<StringEntryOf<S>, boolean> {
|
||||
const entries = chunk
|
||||
chunk = []
|
||||
const escs = entries.map((e) => idx.childOf(index, e))
|
||||
const values = await driver.getMany(escs.map((e) => this.prefix + e))
|
||||
const stale: string[] = []
|
||||
try {
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const esc = escs[i]!
|
||||
if (!idx.matches(index, esc, values[i], entries[i]!)) {
|
||||
stale.push(entries[i]!)
|
||||
continue
|
||||
}
|
||||
if (seen) {
|
||||
if (seen.has(esc)) continue
|
||||
seen.add(esc)
|
||||
}
|
||||
yield [unescapeSegment(esc), values[i]] as StringEntryOf<S>
|
||||
if (--remaining <= 0) return false
|
||||
}
|
||||
return true
|
||||
} finally {
|
||||
if (readRepair && stale.length) await driver.deleteMany(stale)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const r = await it.next()
|
||||
if (r.done) break
|
||||
chunk.push(r.value[0])
|
||||
// Small first chunk so `findOne`-style lookups cost one probe.
|
||||
if (chunk.length >= (remaining < FIND_CHUNK ? remaining : FIND_CHUNK)) {
|
||||
if (!(yield* flush.call(this))) return
|
||||
}
|
||||
}
|
||||
if (chunk.length) yield* flush.call(this)
|
||||
} finally {
|
||||
await it.return?.()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- internals
|
||||
|
||||
#childKey(key: unknown): string {
|
||||
return this.prefix + escapeSegment(toSegment(key), this.#ctx.sep)
|
||||
}
|
||||
|
||||
/** Self vs child by argument count; `selfArity` is the self form's argument count. */
|
||||
#resolve(args: readonly unknown[], selfArity: number): { key: string; esc: string; idx: BoundIndexes | undefined } {
|
||||
if (args.length <= selfArity) {
|
||||
if (this.isRoot) throw new TypeError('The root node has no own value')
|
||||
return { key: this.selfKey, esc: this.#esc, idx: this.#selfIdx }
|
||||
}
|
||||
const esc = escapeSegment(toSegment(args[0]), this.#ctx.sep)
|
||||
return { key: this.prefix + esc, esc, idx: this.#childIdx }
|
||||
}
|
||||
|
||||
/** Writes slots (`value: undefined` = delete), maintaining indexes when present. */
|
||||
async #write(idx: BoundIndexes | undefined, slots: Slot[]): Promise<void> {
|
||||
const { driver } = this.#ctx
|
||||
if (!idx) {
|
||||
const sets: [string, unknown][] = []
|
||||
const dels: string[] = []
|
||||
for (const s of slots) {
|
||||
if (s.value === undefined) dels.push(s.key)
|
||||
else sets.push([s.key, s.value])
|
||||
}
|
||||
if (sets.length) await driver.setMany(sets)
|
||||
if (dels.length) await driver.deleteMany(dels)
|
||||
return
|
||||
}
|
||||
// Last write per key wins.
|
||||
const byKey = new Map<string, Slot>()
|
||||
for (const s of slots) byKey.set(s.key, s)
|
||||
const unique = [...byKey.values()]
|
||||
const old = await driver.getMany(unique.map((s) => s.key))
|
||||
const adds: string[] = []
|
||||
const removes: string[] = []
|
||||
for (let i = 0; i < unique.length; i++) {
|
||||
const s = unique[i]!
|
||||
const before = idx.entriesFor(s.esc, old[i])
|
||||
const after = idx.entriesFor(s.esc, s.value)
|
||||
for (const e of after) if (!before.has(e)) adds.push(e)
|
||||
for (const e of before) if (!after.has(e)) removes.push(e)
|
||||
}
|
||||
if (driver.batch) {
|
||||
const ops: BatchOp[] = []
|
||||
for (const k of adds) ops.push({ type: 'set', key: k, value: true })
|
||||
for (const s of unique) ops.push(s.value === undefined ? { type: 'delete', key: s.key } : { type: 'set', key: s.key, value: s.value })
|
||||
for (const k of removes) ops.push({ type: 'delete', key: k })
|
||||
return driver.batch(ops)
|
||||
}
|
||||
// Non-atomic: adds first, then values, then removes. Entries are hints; findBy re-checks.
|
||||
if (adds.length) await driver.setMany(adds.map((k) => [k, true]))
|
||||
const sets = unique.filter((s) => s.value !== undefined)
|
||||
const dels = unique.filter((s) => s.value === undefined)
|
||||
if (sets.length) await driver.setMany(sets.map((s) => [s.key, s.value]))
|
||||
if (dels.length) await driver.deleteMany(dels.map((s) => s.key))
|
||||
if (removes.length) await driver.deleteMany(removes)
|
||||
}
|
||||
|
||||
async #deleteRange(gte: string, lt: string | undefined): Promise<void> {
|
||||
const { driver } = this.#ctx
|
||||
if (lt !== undefined && driver.deleteRange) return driver.deleteRange(gte, lt)
|
||||
for (;;) {
|
||||
const keys: string[] = []
|
||||
for await (const [k] of driver.scan({ gte, lt, limit: DELETE_CHUNK })) keys.push(k)
|
||||
if (keys.length === 0) return
|
||||
await driver.deleteMany(keys)
|
||||
if (keys.length < DELETE_CHUNK) return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The root `KVNode` over a driver. */
|
||||
export class KVStorage<S extends Spec = AnySpec> extends KVNode<S> {
|
||||
constructor(driver: KVDriver, opts: KVStorageOptions = {}) {
|
||||
const sep = opts.separator ?? DEFAULT_SEPARATOR
|
||||
validateSeparator(sep)
|
||||
const registry = new IndexRegistry(opts.indexes ?? {}, sep)
|
||||
super({ driver, sep, registry, readRepair: opts.readRepair ?? true }, [], '', '', registry.trie.rootState, undefined)
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
/**
|
||||
* Ordered string-keyed map used by the in-memory driver. Kept behind an interface so the
|
||||
* implementation can be swapped (e.g. for a B-tree) without touching the driver.
|
||||
*/
|
||||
export interface SortedMap<V> {
|
||||
readonly size: number
|
||||
get(key: string): V | undefined
|
||||
has(key: string): boolean
|
||||
set(key: string, value: V): void
|
||||
/** Inserts many entries; may be faster than repeated `set`. */
|
||||
setMany(entries: Iterable<[string, V]>): void
|
||||
delete(key: string): boolean
|
||||
/**
|
||||
* Keys in `[gte, lt)` in order (descending when `reverse`). The iterator stays valid across
|
||||
* mutations between steps: it resumes after (before) the last key it returned.
|
||||
*/
|
||||
keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator<string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Sorted chunks of at most `maxChunk` keys (split in half on overflow) plus a `Map` for values.
|
||||
* Inserts and deletes cost O(log n + maxChunk); iteration is O(1) per step while the map
|
||||
* is unchanged and re-seeks in O(log n) after a mutation.
|
||||
*/
|
||||
export class ChunkedSortedMap<V> implements SortedMap<V> {
|
||||
constructor(readonly maxChunk = 1024) {}
|
||||
|
||||
#chunks: string[][] = [[]]
|
||||
readonly #values = new Map<string, V>()
|
||||
#version = 0
|
||||
|
||||
get size(): number {
|
||||
return this.#values.size
|
||||
}
|
||||
|
||||
get(key: string): V | undefined {
|
||||
return this.#values.get(key)
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.#values.has(key)
|
||||
}
|
||||
|
||||
set(key: string, value: V): void {
|
||||
if (!this.#values.has(key)) this.#insert(key)
|
||||
this.#values.set(key, value)
|
||||
}
|
||||
|
||||
setMany(entries: Iterable<[string, V]>): void {
|
||||
const fresh: string[] = []
|
||||
for (const [key, value] of entries) {
|
||||
if (!this.#values.has(key)) fresh.push(key)
|
||||
this.#values.set(key, value)
|
||||
}
|
||||
if (fresh.length === 0) return
|
||||
if (fresh.length < 64 || fresh.length * 8 < this.#values.size) {
|
||||
for (const key of fresh) this.#insert(key)
|
||||
return
|
||||
}
|
||||
// Bulk path: merge and re-chunk, O(n + k log k).
|
||||
fresh.sort(cmp)
|
||||
const old = this.#chunks.flat()
|
||||
const merged = new Array<string>(old.length + fresh.length)
|
||||
let i = 0
|
||||
let j = 0
|
||||
let k = 0
|
||||
while (i < old.length && j < fresh.length) merged[k++] = old[i]! < fresh[j]! ? old[i++]! : fresh[j++]!
|
||||
while (i < old.length) merged[k++] = old[i++]!
|
||||
while (j < fresh.length) merged[k++] = fresh[j++]!
|
||||
const half = Math.max(1, this.maxChunk >>> 1)
|
||||
this.#chunks = []
|
||||
for (let s = 0; s < merged.length; s += half) this.#chunks.push(merged.slice(s, s + half))
|
||||
this.#version++
|
||||
}
|
||||
|
||||
delete(key: string): boolean {
|
||||
if (!this.#values.delete(key)) return false
|
||||
const c = this.#chunkFor(key)
|
||||
const chunk = this.#chunks[c]!
|
||||
chunk.splice(lowerBound(chunk, key), 1)
|
||||
if (chunk.length === 0 && this.#chunks.length > 1) this.#chunks.splice(c, 1)
|
||||
this.#version++
|
||||
return true
|
||||
}
|
||||
|
||||
*keys(range: { gte?: string; lt?: string; reverse?: boolean }): IterableIterator<string> {
|
||||
const { gte, lt, reverse = false } = range
|
||||
let version = this.#version
|
||||
let c: number
|
||||
let i: number
|
||||
// Position of the first candidate.
|
||||
if (reverse) {
|
||||
;[c, i] = lt === undefined ? this.#end() : this.#seek(lt, false)
|
||||
;[c, i] = this.#prev(c, i)
|
||||
} else {
|
||||
;[c, i] = gte === undefined ? [0, 0] : this.#seek(gte, false)
|
||||
;[c, i] = this.#normalize(c, i)
|
||||
}
|
||||
for (;;) {
|
||||
const key = this.#chunks[c]?.[i]
|
||||
if (key === undefined || c < 0) return
|
||||
if (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt) return
|
||||
yield key
|
||||
if (this.#version !== version) {
|
||||
version = this.#version
|
||||
;[c, i] = this.#seek(key, !reverse)
|
||||
if (reverse) [c, i] = this.#prev(c, i)
|
||||
else [c, i] = this.#normalize(c, i)
|
||||
} else if (reverse) [c, i] = this.#prev(c, i)
|
||||
else [c, i] = this.#normalize(c, i + 1)
|
||||
}
|
||||
}
|
||||
|
||||
#insert(key: string): void {
|
||||
const c = this.#chunkFor(key)
|
||||
const chunk = this.#chunks[c]!
|
||||
chunk.splice(lowerBound(chunk, key), 0, key)
|
||||
if (chunk.length > this.maxChunk) this.#chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1))
|
||||
this.#version++
|
||||
}
|
||||
|
||||
/** Index of the last chunk whose first key is <= key (0 if none). */
|
||||
#chunkFor(key: string): number {
|
||||
const chunks = this.#chunks
|
||||
let lo = 0
|
||||
let hi = chunks.length - 1
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi + 1) >>> 1
|
||||
if (chunks[mid]![0]! <= key) lo = mid
|
||||
else hi = mid - 1
|
||||
}
|
||||
return lo
|
||||
}
|
||||
|
||||
/** Position of the first key `>= key` (`> key` when exclusive); may point one past a chunk's end. */
|
||||
#seek(key: string, exclusive: boolean): [number, number] {
|
||||
const c = this.#chunkFor(key)
|
||||
const chunk = this.#chunks[c]!
|
||||
let i = lowerBound(chunk, key)
|
||||
if (exclusive && chunk[i] === key) i++
|
||||
return [c, i]
|
||||
}
|
||||
|
||||
#end(): [number, number] {
|
||||
const c = this.#chunks.length - 1
|
||||
return [c, this.#chunks[c]!.length]
|
||||
}
|
||||
|
||||
/** Moves a position that is past its chunk's end to the start of the next chunk. */
|
||||
#normalize(c: number, i: number): [number, number] {
|
||||
while (c < this.#chunks.length && i >= this.#chunks[c]!.length) {
|
||||
c++
|
||||
i = 0
|
||||
}
|
||||
return [c, i]
|
||||
}
|
||||
|
||||
#prev(c: number, i: number): [number, number] {
|
||||
i--
|
||||
while (i < 0 && c > 0) {
|
||||
c--
|
||||
i = this.#chunks[c]!.length - 1
|
||||
}
|
||||
return i < 0 ? [-1, -1] : [c, i]
|
||||
}
|
||||
}
|
||||
|
||||
function lowerBound(arr: string[], key: string): number {
|
||||
let lo = 0
|
||||
let hi = arr.length
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1
|
||||
if (arr[mid]! < key) lo = mid + 1
|
||||
else hi = mid
|
||||
}
|
||||
return lo
|
||||
}
|
||||
|
||||
function cmp(a: string, b: string): number {
|
||||
return a < b ? -1 : a > b ? 1 : 0
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from './kv-conformance.js'
|
||||
@@ -1,171 +0,0 @@
|
||||
import { beforeEach, describe, expect, it } from 'vitest'
|
||||
import type { KVDriver, ScanOptions } from '../kv/driver.js'
|
||||
|
||||
/**
|
||||
* Reusable conformance suite for {@link KVDriver} implementations. Call it from a Vitest test file:
|
||||
*
|
||||
* ```ts
|
||||
* runKVDriverConformance('MyDriver', () => new MyDriver())
|
||||
* ```
|
||||
*
|
||||
* `makeDriver` must return an empty driver each time.
|
||||
*/
|
||||
export function runKVDriverConformance(name: string, makeDriver: () => KVDriver | Promise<KVDriver>): void {
|
||||
describe(`KVDriver conformance: ${name}`, () => {
|
||||
let d: KVDriver
|
||||
|
||||
beforeEach(async () => {
|
||||
d = await makeDriver()
|
||||
})
|
||||
|
||||
async function collect(opts: ScanOptions): Promise<[string, unknown][]> {
|
||||
const out: [string, unknown][] = []
|
||||
for await (const e of d.scan(opts)) out.push(e)
|
||||
return out
|
||||
}
|
||||
|
||||
const keysOf = async (opts: ScanOptions) => (await collect(opts)).map(([k]) => k)
|
||||
|
||||
async function seed(): Promise<void> {
|
||||
// Deliberately inserted out of order.
|
||||
await d.setMany([
|
||||
['b', 2],
|
||||
['a/2', { n: 'a2' }],
|
||||
['c', [3]],
|
||||
['a', 1],
|
||||
['a/1', 'a1'],
|
||||
['a0', null],
|
||||
['10', 10],
|
||||
['9', 9],
|
||||
])
|
||||
}
|
||||
|
||||
it('round-trips JSON values with getMany in key order, undefined for missing', async () => {
|
||||
await seed()
|
||||
expect(await d.getMany(['c', 'missing', 'a/2', 'a0', 'a'])).toEqual([[3], undefined, { n: 'a2' }, null, 1])
|
||||
expect(await d.getMany([])).toEqual([])
|
||||
})
|
||||
|
||||
it('overwrites on setMany and deletes with deleteMany', async () => {
|
||||
await seed()
|
||||
await d.setMany([['a', 'one']])
|
||||
await d.deleteMany(['b', 'missing'])
|
||||
expect(await d.getMany(['a', 'b'])).toEqual(['one', undefined])
|
||||
})
|
||||
|
||||
it('scans all keys in lexicographic order', async () => {
|
||||
await seed()
|
||||
expect(await keysOf({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c'])
|
||||
})
|
||||
|
||||
it('honours gte (inclusive) and lt (exclusive)', async () => {
|
||||
await seed()
|
||||
expect(await keysOf({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2'])
|
||||
expect(await keysOf({ gte: 'a', lt: 'b' })).toEqual(['a', 'a/1', 'a/2', 'a0'])
|
||||
expect(await keysOf({ gte: 'a0' })).toEqual(['a0', 'b', 'c'])
|
||||
expect(await keysOf({ lt: 'a' })).toEqual(['10', '9'])
|
||||
expect(await keysOf({ gte: 'zz' })).toEqual([])
|
||||
expect(await keysOf({ gte: 'b', lt: 'b' })).toEqual([])
|
||||
})
|
||||
|
||||
it('scans in reverse within bounds', async () => {
|
||||
await seed()
|
||||
expect(await keysOf({ reverse: true })).toEqual(['c', 'b', 'a0', 'a/2', 'a/1', 'a', '9', '10'])
|
||||
expect(await keysOf({ reverse: true, gte: 'a', lt: 'a0' })).toEqual(['a/2', 'a/1', 'a'])
|
||||
})
|
||||
|
||||
it('honours limit, forwards and in reverse', async () => {
|
||||
await seed()
|
||||
expect(await keysOf({ limit: 2 })).toEqual(['10', '9'])
|
||||
expect(await keysOf({ limit: 2, reverse: true, lt: 'b' })).toEqual(['a0', 'a/2'])
|
||||
expect(await keysOf({ limit: 0 })).toEqual([])
|
||||
})
|
||||
|
||||
it('yields values only when values: true', async () => {
|
||||
await seed()
|
||||
expect(await collect({ gte: 'a/', lt: 'a0' })).toEqual([
|
||||
['a/1', undefined],
|
||||
['a/2', undefined],
|
||||
])
|
||||
expect(await collect({ gte: 'a/', lt: 'a0', values: true })).toEqual([
|
||||
['a/1', 'a1'],
|
||||
['a/2', { n: 'a2' }],
|
||||
])
|
||||
})
|
||||
|
||||
it('tolerates writes while a scan is suspended', async () => {
|
||||
await d.setMany([
|
||||
['k1', 1],
|
||||
['k2', 2],
|
||||
['k3', 3],
|
||||
])
|
||||
const seen: string[] = []
|
||||
for await (const [key] of d.scan({})) {
|
||||
seen.push(key)
|
||||
if (key === 'k1') await d.deleteMany(['k1'])
|
||||
}
|
||||
expect(seen).toEqual(['k1', 'k2', 'k3'])
|
||||
})
|
||||
|
||||
it('does not let callers mutate stored values through returned references', async () => {
|
||||
await d.setMany([['o', { n: 1 }]])
|
||||
const [v] = (await d.getMany(['o'])) as [{ n: number }]
|
||||
v.n = 2
|
||||
expect(await d.getMany(['o'])).toEqual([{ n: 1 }])
|
||||
})
|
||||
|
||||
it('stores unicode keys and orders them by code unit', async () => {
|
||||
await d.setMany([
|
||||
['é', 1],
|
||||
['z', 2],
|
||||
['日本', 3],
|
||||
])
|
||||
expect(await keysOf({})).toEqual(['z', 'é', '日本'])
|
||||
})
|
||||
|
||||
describe('optional: setIfAbsent', () => {
|
||||
it('writes only absent keys and reports which were written', async (ctx) => {
|
||||
if (!d.setIfAbsent) return ctx.skip()
|
||||
await d.setMany([['x', 1]])
|
||||
expect(
|
||||
await d.setIfAbsent([
|
||||
['x', 2],
|
||||
['y', 3],
|
||||
]),
|
||||
).toEqual([false, true])
|
||||
expect(await d.getMany(['x', 'y'])).toEqual([1, 3])
|
||||
})
|
||||
|
||||
it('only the first of concurrent claims wins', async (ctx) => {
|
||||
if (!d.setIfAbsent) return ctx.skip()
|
||||
const results = await Promise.all([1, 2, 3, 4].map((n) => d.setIfAbsent!([['claim', n]])))
|
||||
expect(results.flat().filter(Boolean)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('optional: batch', () => {
|
||||
it('applies sets and deletes in order', async (ctx) => {
|
||||
if (!d.batch) return ctx.skip()
|
||||
await d.setMany([['gone', 1]])
|
||||
await d.batch([
|
||||
{ type: 'set', key: 'p', value: 1 },
|
||||
{ type: 'delete', key: 'gone' },
|
||||
{ type: 'set', key: 'q', value: { a: [1] } },
|
||||
{ type: 'set', key: 'p', value: 2 },
|
||||
])
|
||||
expect(await d.getMany(['p', 'q', 'gone'])).toEqual([2, { a: [1] }, undefined])
|
||||
})
|
||||
})
|
||||
|
||||
describe('optional: deleteRange', () => {
|
||||
it('deletes exactly [gte, lt)', async (ctx) => {
|
||||
if (!d.deleteRange) return ctx.skip()
|
||||
await seed()
|
||||
await d.deleteRange('a/', 'a0')
|
||||
expect(await keysOf({})).toEqual(['10', '9', 'a', 'a0', 'b', 'c'])
|
||||
await d.deleteRange('a', 'b')
|
||||
expect(await keysOf({})).toEqual(['10', '9', 'b', 'c'])
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { Json } from './json.js'
|
||||
|
||||
export type Doc = Record<string, unknown>
|
||||
|
||||
/** Dot-path access into plain JSON documents (`'address.city'`, `'tags.0'`). */
|
||||
export class DocumentPath {
|
||||
static get(doc: unknown, path: string): unknown {
|
||||
let value = doc
|
||||
for (const segment of path.split('.')) {
|
||||
if (Array.isArray(value) || Json.isObject(value)) value = (value as Record<string, unknown>)[segment]
|
||||
else return undefined
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/** Sets a path in place, creating intermediate objects. */
|
||||
static set(doc: Doc, path: string, value: unknown): void {
|
||||
const segments = path.split('.')
|
||||
let target = doc as Record<string, unknown>
|
||||
for (const segment of segments.slice(0, -1)) {
|
||||
if (typeof target[segment] !== 'object' || target[segment] === null) target[segment] = {}
|
||||
target = target[segment] as Record<string, unknown>
|
||||
}
|
||||
target[segments.at(-1)!] = value
|
||||
}
|
||||
|
||||
static unset(doc: Doc, path: string): void {
|
||||
const segments = path.split('.')
|
||||
const parent = segments.length === 1 ? doc : DocumentPath.get(doc, segments.slice(0, -1).join('.'))
|
||||
if (Json.isObject(parent)) delete parent[segments.at(-1)!]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { DocumentPath, type Doc } from './document-path.js'
|
||||
import { Json } from './json.js'
|
||||
|
||||
export type Comparison = { $eq?: unknown; $gt?: unknown; $gte?: unknown; $lt?: unknown; $lte?: unknown; $in?: unknown[] }
|
||||
|
||||
/** Field (dot path) → value (equality) or comparison operators. All fields must match. */
|
||||
export type Filter = Record<string, unknown>
|
||||
|
||||
export interface FindOptions {
|
||||
/** Field → 1 (ascending) or -1 (descending). */
|
||||
sort?: Record<string, 1 | -1>
|
||||
limit?: number
|
||||
skip?: number
|
||||
}
|
||||
|
||||
export interface FindResult<T = Doc> {
|
||||
key: string
|
||||
data: T
|
||||
}
|
||||
|
||||
const OPERATORS = new Set(['$eq', '$gt', '$gte', '$lt', '$lte', '$in'])
|
||||
|
||||
/**
|
||||
* Filter evaluation, sorting and paging for document stores that don't query natively.
|
||||
*
|
||||
* - `{ field: value }` matches equal values; `null` also matches a missing field.
|
||||
* - Comparisons only match values of the same type (`$gt: 5` never matches a string).
|
||||
* - On an array field, a condition matches if the array itself or any one element satisfies it.
|
||||
*/
|
||||
export class DocumentQuery {
|
||||
constructor(
|
||||
readonly filter: Filter = {},
|
||||
readonly options: FindOptions = {},
|
||||
) {}
|
||||
|
||||
matches(doc: Doc): boolean {
|
||||
return Object.entries(this.filter).every(([path, condition]) => DocumentQuery.matchesValue(DocumentPath.get(doc, path), condition))
|
||||
}
|
||||
|
||||
/** Whether one field value satisfies a condition. */
|
||||
static matchesValue(value: unknown, condition: unknown): boolean {
|
||||
const candidates = Array.isArray(value) ? [value, ...value] : [value]
|
||||
if (!DocumentQuery.isComparison(condition)) return candidates.some((c) => DocumentQuery.equal(c, condition))
|
||||
const ops = Object.entries(condition)
|
||||
return candidates.some((c) => ops.every(([op, arg]) => DocumentQuery.compare(op, c, arg)))
|
||||
}
|
||||
|
||||
/** Applies filter, sort, skip and limit to a stream of candidates. */
|
||||
async *run(source: AsyncIterable<FindResult>): AsyncGenerator<FindResult> {
|
||||
const { sort, skip = 0, limit = Infinity } = this.options
|
||||
if (limit <= 0) return
|
||||
if (sort && Object.keys(sort).length) {
|
||||
const all: FindResult[] = []
|
||||
for await (const result of source) if (this.matches(result.data)) all.push(result)
|
||||
all.sort((a, b) => this.compareResults(a, b))
|
||||
yield* all.slice(skip, skip + limit)
|
||||
return
|
||||
}
|
||||
let skipped = 0
|
||||
let emitted = 0
|
||||
for await (const result of source) {
|
||||
if (!this.matches(result.data)) continue
|
||||
if (skipped++ < skip) continue
|
||||
yield result
|
||||
if (++emitted >= limit) return
|
||||
}
|
||||
}
|
||||
|
||||
private compareResults(a: FindResult, b: FindResult): number {
|
||||
for (const [path, direction] of Object.entries(this.options.sort!)) {
|
||||
const order = DocumentQuery.order(DocumentPath.get(a.data, path), DocumentPath.get(b.data, path))
|
||||
if (order !== 0) return order * direction
|
||||
}
|
||||
return a.key < b.key ? -1 : a.key > b.key ? 1 : 0
|
||||
}
|
||||
|
||||
private static isComparison(condition: unknown): condition is Comparison {
|
||||
if (!Json.isObject(condition)) return false
|
||||
const keys = Object.keys(condition)
|
||||
if (keys.length === 0 || !keys.every((k) => k.startsWith('$'))) return false
|
||||
for (const k of keys) if (!OPERATORS.has(k)) throw new TypeError(`Unsupported operator ${k}`)
|
||||
return true
|
||||
}
|
||||
|
||||
private static equal(value: unknown, expected: unknown): boolean {
|
||||
return expected === null ? value === null || value === undefined : Json.equal(value, expected)
|
||||
}
|
||||
|
||||
private static compare(op: string, value: unknown, arg: unknown): boolean {
|
||||
if (op === '$eq') return DocumentQuery.equal(value, arg)
|
||||
if (op === '$in') return (arg as unknown[]).some((x) => DocumentQuery.equal(value, x))
|
||||
if (typeof value !== typeof arg || !['number', 'string', 'boolean'].includes(typeof value)) return false
|
||||
const order = DocumentQuery.order(value, arg)
|
||||
return op === '$gt' ? order > 0 : op === '$gte' ? order >= 0 : op === '$lt' ? order < 0 : order <= 0
|
||||
}
|
||||
|
||||
/** Total order: missing/null < booleans < numbers < strings < anything else. */
|
||||
static order(a: unknown, b: unknown): number {
|
||||
const rank = (v: unknown) => (v === undefined || v === null ? 0 : (({ boolean: 1, number: 2, string: 3 } as Record<string, number>)[typeof v] ?? 4))
|
||||
const ra = rank(a)
|
||||
const rb = rank(b)
|
||||
if (ra !== rb) return ra - rb
|
||||
if (ra === 4) [a, b] = [JSON.stringify(a), JSON.stringify(b)]
|
||||
return a === b ? 0 : (a as number) < (b as number) ? -1 : 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { DocumentPath, type Doc } from './document-path.js'
|
||||
import { DocumentQuery } from './document-query.js'
|
||||
import { Json } from './json.js'
|
||||
|
||||
/** Update operators. Paths may use dots. `$push`/`$addToSet` accept `{ $each: [...] }`. */
|
||||
export interface UpdateSpec {
|
||||
$set?: Record<string, unknown>
|
||||
$unset?: Record<string, unknown>
|
||||
$inc?: Record<string, number>
|
||||
$push?: Record<string, unknown>
|
||||
$pull?: Record<string, unknown>
|
||||
$addToSet?: Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* A resolved change to one document. Operators are resolved to final values (`$inc` becomes
|
||||
* `set: { n: 6 }`), so replaying changes is deterministic.
|
||||
*/
|
||||
export type DocChange = { op: 'set'; doc: Doc } | { op: 'patch'; set: Record<string, unknown>; unset: string[] } | { op: 'delete' }
|
||||
|
||||
export class DocumentUpdate {
|
||||
/** Resolves update operators against the current state into a patch. */
|
||||
static resolve(state: Doc, spec: UpdateSpec): DocChange {
|
||||
const set: Record<string, unknown> = {}
|
||||
const unset: string[] = []
|
||||
const array = (path: string) => {
|
||||
const current = DocumentPath.get(state, path) ?? []
|
||||
if (!Array.isArray(current)) throw new TypeError(`"${path}" is not an array`)
|
||||
return Json.clone(current)
|
||||
}
|
||||
const items = (value: unknown) => (Json.isObject(value) && Array.isArray(value.$each) ? value.$each : [value])
|
||||
|
||||
for (const [op, fields] of Object.entries(spec)) {
|
||||
for (const [path, value] of Object.entries(fields as Record<string, unknown>)) {
|
||||
if (path in set || unset.includes(path)) throw new TypeError(`"${path}" is updated twice`)
|
||||
switch (op) {
|
||||
case '$set':
|
||||
set[path] = Json.clone(value)
|
||||
break
|
||||
case '$unset':
|
||||
unset.push(path)
|
||||
break
|
||||
case '$inc': {
|
||||
const current = DocumentPath.get(state, path) ?? 0
|
||||
if (typeof current !== 'number' || typeof value !== 'number') throw new TypeError(`$inc needs numbers at "${path}"`)
|
||||
set[path] = current + value
|
||||
break
|
||||
}
|
||||
case '$push':
|
||||
set[path] = [...array(path), ...Json.clone(items(value))]
|
||||
break
|
||||
case '$addToSet':
|
||||
set[path] = items(value).reduce<unknown[]>((acc, item) => (acc.some((x) => Json.equal(x, item)) ? acc : [...acc, Json.clone(item)]), array(path))
|
||||
break
|
||||
case '$pull':
|
||||
set[path] = array(path).filter((item) => !DocumentQuery.matchesValue(item, value))
|
||||
break
|
||||
default:
|
||||
throw new TypeError(`Unsupported update operator ${op}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return { op: 'patch', set, unset }
|
||||
}
|
||||
|
||||
/** Applies a change to a state (`undefined` = no document). */
|
||||
static apply(state: Doc | undefined, change: DocChange): Doc | undefined {
|
||||
if (change.op === 'delete') return undefined
|
||||
if (change.op === 'set') return Json.clone(change.doc)
|
||||
const next = Json.clone(state ?? {})
|
||||
for (const [path, value] of Object.entries(change.set)) DocumentPath.set(next, path, Json.clone(value))
|
||||
for (const path of change.unset) DocumentPath.unset(next, path)
|
||||
return next
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/** A key argument was passed but is `undefined`/`null`, empty, or not a string/number. */
|
||||
export class InvalidKeyError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'InvalidKeyError'
|
||||
}
|
||||
}
|
||||
|
||||
/** A remote storage could not be reached (network failure or 502/503/504). Writes may be retried later. */
|
||||
export class StorageUnavailableError extends Error {
|
||||
constructor(message: string, options?: { cause?: unknown }) {
|
||||
super(message, options)
|
||||
this.name = 'StorageUnavailableError'
|
||||
}
|
||||
}
|
||||
|
||||
/** A request was not authenticated (401) or not allowed for this caller (403). */
|
||||
export class AuthError extends Error {
|
||||
constructor(
|
||||
readonly status: 401 | 403,
|
||||
message: string,
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'AuthError'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Adapted from xo-cash-utils `source/event-emitter.ts` (branch installable-sse): payloads are passed
|
||||
// through as-is (no cloning, freezing or DeeplyReadonly), and timeouts reject with a plain Error.
|
||||
|
||||
export type EventMap = Record<string, unknown>
|
||||
|
||||
type Listener<T> = (detail: T) => void
|
||||
|
||||
/** Lets listeners for individual payloads live in one collection typed with the union of payloads. */
|
||||
type StoredListener<T> = {
|
||||
bivarianceHack(detail: T): void
|
||||
}['bivarianceHack']
|
||||
|
||||
interface ListenerEntry<T> {
|
||||
listener: StoredListener<T>
|
||||
wrappedListener: StoredListener<T>
|
||||
cancel: () => void
|
||||
}
|
||||
|
||||
/** Returned by {@link EventEmitter.on} and {@link EventEmitter.once} to remove the listener. */
|
||||
export type OffCallback = () => void
|
||||
|
||||
/** A small typed event emitter. */
|
||||
export class EventEmitter<T extends EventMap> {
|
||||
private readonly listeners: Map<keyof T, Set<ListenerEntry<T[keyof T]>>> = new Map()
|
||||
|
||||
/**
|
||||
* Adds a listener.
|
||||
* @param debounceMilliseconds - When > 0, only the last event within this window is delivered.
|
||||
*/
|
||||
on<K extends keyof T>(type: K, listener: Listener<T[K]>, debounceMilliseconds: number = 0): OffCallback {
|
||||
const { cancel, listener: cancellableListener } = this.cancellable(listener)
|
||||
const wrappedListener = debounceMilliseconds > 0 ? this.debounce(cancellableListener, debounceMilliseconds) : cancellableListener
|
||||
this.entriesFor(type).add({ listener, wrappedListener, cancel } as ListenerEntry<T[keyof T]>)
|
||||
return () => this.off(type, listener)
|
||||
}
|
||||
|
||||
/** Adds a listener that is removed after its first call. */
|
||||
once<K extends keyof T>(type: K, listener: Listener<T[K]>, debounceMilliseconds: number = 0): OffCallback {
|
||||
const wrappedListener: Listener<T[K]> = (detail) => {
|
||||
this.off(type, listener)
|
||||
listener(detail)
|
||||
}
|
||||
const { cancel, listener: cancellableListener } = this.cancellable(wrappedListener)
|
||||
const debouncedListener = debounceMilliseconds > 0 ? this.debounce(cancellableListener, debounceMilliseconds) : cancellableListener
|
||||
this.entriesFor(type).add({ listener, wrappedListener: debouncedListener, cancel } as ListenerEntry<T[keyof T]>)
|
||||
return () => this.off(type, listener)
|
||||
}
|
||||
|
||||
/** Removes one listener, or all listeners for `type` when none is given. */
|
||||
off<K extends keyof T>(type: K, listener?: Listener<T[K]>): void {
|
||||
const listeners = this.listeners.get(type)
|
||||
if (!listeners) return
|
||||
for (const entry of Array.from(listeners)) {
|
||||
if (listener && entry.listener !== listener && entry.wrappedListener !== listener) continue
|
||||
// Cancel so a pending debounced call doesn't fire after removal.
|
||||
entry.cancel()
|
||||
listeners.delete(entry)
|
||||
}
|
||||
if (!listener || listeners.size === 0) this.listeners.delete(type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls every listener for `type` with `payload`. Listener errors are logged, not thrown.
|
||||
* @returns Whether any listener was registered.
|
||||
*/
|
||||
emit<K extends keyof T>(type: K, payload: T[K]): boolean {
|
||||
const listeners = this.listeners.get(type)
|
||||
if (!listeners) return false
|
||||
for (const entry of Array.from(listeners)) {
|
||||
try {
|
||||
entry.wrappedListener(payload)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
}
|
||||
}
|
||||
return listeners.size > 0
|
||||
}
|
||||
|
||||
removeAllListeners(): void {
|
||||
for (const type of Array.from(this.listeners.keys())) this.off(type)
|
||||
}
|
||||
|
||||
/** Resolves with the first payload of `type` matching `predicate`; rejects after `timeoutMs` if given. */
|
||||
waitFor<K extends keyof T>(type: K, predicate: (payload: T[K]) => boolean, timeoutMs?: number): Promise<T[K]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined
|
||||
const cleanup = (): void => {
|
||||
this.off(type, listener)
|
||||
if (timeoutId !== undefined) clearTimeout(timeoutId)
|
||||
}
|
||||
const listener = (payload: T[K]): void => {
|
||||
try {
|
||||
if (!predicate(payload)) return
|
||||
cleanup()
|
||||
resolve(payload)
|
||||
} catch (error) {
|
||||
cleanup()
|
||||
reject(error)
|
||||
}
|
||||
}
|
||||
if (timeoutMs !== undefined) {
|
||||
timeoutId = setTimeout(() => {
|
||||
this.off(type, listener)
|
||||
reject(new Error(`Timed out waiting for "${String(type)}"`))
|
||||
}, timeoutMs)
|
||||
}
|
||||
this.on(type, listener)
|
||||
})
|
||||
}
|
||||
|
||||
private entriesFor<K extends keyof T>(type: K): Set<ListenerEntry<T[keyof T]>> {
|
||||
let entries = this.listeners.get(type)
|
||||
if (!entries) this.listeners.set(type, (entries = new Set()))
|
||||
return entries
|
||||
}
|
||||
|
||||
private debounce<K extends keyof T>(func: Listener<T[K]>, wait: number): Listener<T[K]> {
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined
|
||||
return (detail) => {
|
||||
if (timeout !== undefined) clearTimeout(timeout)
|
||||
timeout = setTimeout(() => func(detail), wait)
|
||||
}
|
||||
}
|
||||
|
||||
private cancellable<K extends keyof T>(func: Listener<T[K]>): { cancel: () => void; listener: Listener<T[K]> } {
|
||||
let cancelled = false
|
||||
return {
|
||||
cancel: () => void (cancelled = true),
|
||||
listener: (detail) => {
|
||||
if (!cancelled) func(detail)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Adapted from xo-cash-utils `source/extended-json.ts` (branch installable-sse), with local hex
|
||||
// encoding instead of @bitauth/libauth.
|
||||
|
||||
import { Hex } from './hex.js'
|
||||
|
||||
/** Matches a bigint encoded as `<bigint: 123n>`. */
|
||||
const BIGINT_PATTERN = /^<bigint: (?<bigint>[+-]?[0-9]+)n>$/u
|
||||
|
||||
/** Matches a Uint8Array encoded as `<uint8array: abcd>`. */
|
||||
const UINT8ARRAY_PATTERN = /^<uint8array: (?<hex>[0-9a-f]*)>$/u
|
||||
|
||||
/**
|
||||
* JSON that also round-trips `bigint` (`<bigint: 123n>`) and `Uint8Array` (`<uint8array: abcd>`).
|
||||
* Used for every value that crosses the network.
|
||||
*/
|
||||
export class ExtendedJson {
|
||||
static stringify(value: unknown): string {
|
||||
return JSON.stringify(value, ExtendedJson.replacer)
|
||||
}
|
||||
|
||||
/** Like `stringify`, with object keys sorted, so equal values always give the same text (for hashing). */
|
||||
static canonical(value: unknown): string {
|
||||
return JSON.stringify(value, (key, v) => {
|
||||
const replaced = ExtendedJson.replacer(key, v)
|
||||
if (typeof replaced !== 'object' || replaced === null || Array.isArray(replaced)) return replaced
|
||||
return Object.fromEntries(
|
||||
Object.keys(replaced)
|
||||
.sort()
|
||||
.map((k) => [k, (replaced as Record<string, unknown>)[k]]),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
static parse(text: string): unknown {
|
||||
return JSON.parse(text, ExtendedJson.reviver)
|
||||
}
|
||||
|
||||
/** `JSON.stringify` replacer. */
|
||||
static replacer(this: void, _key: string, value: unknown): unknown {
|
||||
if (value instanceof Uint8Array) return `<uint8array: ${Hex.encode(value)}>`
|
||||
if (typeof value === 'bigint') return `<bigint: ${value.toString()}n>`
|
||||
return value
|
||||
}
|
||||
|
||||
/** `JSON.parse` reviver. */
|
||||
static reviver(this: void, _key: string, value: unknown): unknown {
|
||||
if (typeof value !== 'string') return value
|
||||
const bigint = value.match(BIGINT_PATTERN)
|
||||
if (bigint) return BigInt(bigint.groups!.bigint!)
|
||||
const bytes = value.match(UINT8ARRAY_PATTERN)
|
||||
if (bytes) return Hex.decode(bytes.groups!.hex!)
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/** Lowercase hex encoding of bytes. */
|
||||
export class Hex {
|
||||
static encode(bytes: Uint8Array): string {
|
||||
let hex = ''
|
||||
for (const byte of bytes) hex += byte.toString(16).padStart(2, '0')
|
||||
return hex
|
||||
}
|
||||
|
||||
/** Throws on odd length or non-hex characters. */
|
||||
static decode(hex: string): Uint8Array {
|
||||
if (hex.length % 2 !== 0 || !/^[0-9a-fA-F]*$/.test(hex)) throw new TypeError('Invalid hex string')
|
||||
const bytes = new Uint8Array(hex.length / 2)
|
||||
for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16)
|
||||
return bytes
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Helpers for stored values: JSON plus `bigint` and `Uint8Array` (see `ExtendedJson`).
|
||||
*/
|
||||
export class Json {
|
||||
/** Deep copy; much cheaper than `structuredClone` for plain data. */
|
||||
static clone<T>(value: T): T {
|
||||
if (value === null || typeof value !== 'object') return value
|
||||
if (value instanceof Uint8Array) return value.slice() as T
|
||||
if (Array.isArray(value)) return value.map((v) => Json.clone(v)) as T
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const k in value) out[k] = Json.clone((value as Record<string, unknown>)[k])
|
||||
return out as T
|
||||
}
|
||||
|
||||
static equal(a: unknown, b: unknown): boolean {
|
||||
if (a === b) return true
|
||||
if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false
|
||||
if (a instanceof Uint8Array || b instanceof Uint8Array) {
|
||||
return a instanceof Uint8Array && b instanceof Uint8Array && a.length === b.length && a.every((byte, i) => byte === b[i])
|
||||
}
|
||||
if (Array.isArray(a) !== Array.isArray(b)) return false
|
||||
const ka = Object.keys(a)
|
||||
const kb = Object.keys(b)
|
||||
return ka.length === kb.length && ka.every((k) => Json.equal((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]))
|
||||
}
|
||||
|
||||
static isObject(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { ScanOptions } from '../kv-storage/base-kv-storage.js'
|
||||
import { Keys } from './keys.js'
|
||||
|
||||
/**
|
||||
* A key and its whole subtree: `[prefix, prefix + '0')`. Escaping guarantees nothing else sorts
|
||||
* in that range (not even lookalikes such as `prefix-x`). The empty prefix is every key.
|
||||
*/
|
||||
export class KeyRange {
|
||||
/** Inclusive lower bound (`undefined` = unbounded). */
|
||||
readonly gte: string | undefined
|
||||
/** Exclusive upper bound (`undefined` = unbounded). */
|
||||
readonly lt: string | undefined
|
||||
|
||||
constructor(readonly prefix: string) {
|
||||
this.gte = prefix === '' ? undefined : prefix
|
||||
this.lt = prefix === '' ? undefined : prefix + Keys.afterSeparator
|
||||
}
|
||||
|
||||
contains(key: string): boolean {
|
||||
return this.prefix === '' || key === this.prefix || key.startsWith(this.prefix + Keys.separator)
|
||||
}
|
||||
|
||||
/** Whether a scan over `[gte, lt)` stays inside this range. */
|
||||
covers(gte: string | undefined, lt: string | undefined): boolean {
|
||||
if (this.prefix === '') return true
|
||||
return gte !== undefined && lt !== undefined && gte >= this.gte! && lt <= this.lt!
|
||||
}
|
||||
|
||||
/** Scan options narrowed to this range, or `undefined` if nothing is left. */
|
||||
clamp(options: ScanOptions): ScanOptions | undefined {
|
||||
const gte = this.gte !== undefined && (options.gte === undefined || options.gte < this.gte) ? this.gte : options.gte
|
||||
const lt = this.lt !== undefined && (options.lt === undefined || options.lt > this.lt) ? this.lt : options.lt
|
||||
if (gte !== undefined && lt !== undefined && gte >= lt) return undefined
|
||||
return { ...options, gte, lt }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { InvalidKeyError } from './errors.js'
|
||||
|
||||
/**
|
||||
* Key encoding. A node's key is its escaped path segments joined by `/`.
|
||||
*
|
||||
* Every character that sorts at or below `/` (and `~` itself) is escaped as `~XX` (hex). Escaped
|
||||
* segments therefore only contain characters above `/`, so all keys of one child sort together:
|
||||
* `p/a`, `p/a/...`, then siblings like `p/a~2D1` (`a-1`). That keeps child listing a simple skip-scan.
|
||||
*
|
||||
* Segments starting with `~` + a lowercase letter are reserved for internal use (e.g. `~e` for
|
||||
* event commits); escaping only ever produces `~` + uppercase hex, so user keys can't collide.
|
||||
*/
|
||||
export class Keys {
|
||||
static readonly separator = '/'
|
||||
/** The character right after `/`: `prefix + segment + '0'` is the first key past a subtree. */
|
||||
static readonly afterSeparator = '0'
|
||||
|
||||
/** Validates a user key and converts it to a segment. */
|
||||
static segment(key: unknown): string {
|
||||
if (key === undefined || key === null) throw new InvalidKeyError(`Key must not be ${key}`)
|
||||
if (typeof key === 'number' && Number.isFinite(key)) return String(key)
|
||||
if (typeof key !== 'string') throw new InvalidKeyError(`Key must be a string or finite number, got ${String(key)}`)
|
||||
if (key === '') throw new InvalidKeyError('Key must not be empty')
|
||||
return key
|
||||
}
|
||||
|
||||
static escape(segment: string): string {
|
||||
if (!/[\x00-\x2f~]/.test(segment)) return segment
|
||||
return segment.replace(/[\x00-\x2f~]/g, (c) => '~' + c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0'))
|
||||
}
|
||||
|
||||
static unescape(segment: string): string {
|
||||
return segment.includes('~') ? segment.replace(/~([0-9A-F]{2})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16))) : segment
|
||||
}
|
||||
|
||||
static join(path: readonly string[]): string {
|
||||
return path.map((s) => Keys.escape(s)).join(Keys.separator)
|
||||
}
|
||||
|
||||
static isReserved(escapedSegment: string): boolean {
|
||||
return /^~[a-z]/.test(escapedSegment)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { AuthError } from './errors.js'
|
||||
import { Hex } from './hex.js'
|
||||
|
||||
/** Signs requests. Implement this to use another algorithm (e.g. secp256k1 via libauth). */
|
||||
export interface RequestSigner {
|
||||
/** Algorithm name sent in `X-Signature-Algorithm`; the server must have a matching verifier. */
|
||||
readonly algorithm: string
|
||||
/** Public key as lowercase hex. Also the owner segment of the caller's keys (`/<publicKey>/...`). */
|
||||
readonly publicKey: string
|
||||
sign(payload: Uint8Array): Promise<Uint8Array>
|
||||
}
|
||||
|
||||
/** Verifies signatures for one algorithm. */
|
||||
export interface SignatureVerifier {
|
||||
readonly algorithm: string
|
||||
verify(publicKey: string, payload: Uint8Array, signature: Uint8Array): Promise<boolean>
|
||||
}
|
||||
|
||||
export const AUTH_HEADERS = {
|
||||
publicKey: 'X-Public-Key',
|
||||
timestamp: 'X-Timestamp',
|
||||
nonce: 'X-Nonce',
|
||||
signature: 'X-Signature',
|
||||
algorithm: 'X-Signature-Algorithm',
|
||||
} as const
|
||||
|
||||
export interface RequestAuthOptions {
|
||||
/** Accepted difference between the request timestamp and the server clock. Default 60 000 ms. */
|
||||
maxSkewMs?: number
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/**
|
||||
* Signed-request authentication. The client signs `${timestamp}:${nonce}:${path}:${body}` (nonce =
|
||||
* random per request, path = URL pathname, body = the exact request body) and sends the public
|
||||
* key, timestamp, nonce, signature (hex) and algorithm as headers. The server checks the
|
||||
* timestamp is recent, verifies the signature with the verifier for that algorithm, and rejects a
|
||||
* nonce the same key already used within the timestamp window (replay protection). The nonce also
|
||||
* keeps identical requests made at the same moment distinct (Ed25519 signatures are deterministic).
|
||||
*/
|
||||
export class RequestAuth {
|
||||
private readonly verifiers: Map<string, SignatureVerifier>
|
||||
private readonly maxSkewMs: number
|
||||
private readonly now: () => number
|
||||
/** `<publicKey>:<nonce>` accepted within the window → when it expires; insertion order ≈ expiry order. */
|
||||
private readonly seen = new Map<string, number>()
|
||||
|
||||
constructor(verifiers: SignatureVerifier[], options: RequestAuthOptions = {}) {
|
||||
if (verifiers.length === 0) throw new TypeError('RequestAuth needs at least one verifier')
|
||||
this.verifiers = new Map(verifiers.map((v) => [v.algorithm, v]))
|
||||
this.maxSkewMs = options.maxSkewMs ?? 60_000
|
||||
this.now = options.now ?? Date.now
|
||||
}
|
||||
|
||||
static payload(timestamp: string, nonce: string, path: string, body: string): Uint8Array {
|
||||
return new TextEncoder().encode(`${timestamp}:${nonce}:${path}:${body}`)
|
||||
}
|
||||
|
||||
/** Auth headers for a request to `path` with `body`. */
|
||||
static async headers(signer: RequestSigner, path: string, body: string, now: number = Date.now()): Promise<Record<string, string>> {
|
||||
const timestamp = String(now)
|
||||
const nonce = Hex.encode(crypto.getRandomValues(new Uint8Array(16)))
|
||||
const signature = await signer.sign(RequestAuth.payload(timestamp, nonce, path, body))
|
||||
return {
|
||||
[AUTH_HEADERS.publicKey]: signer.publicKey,
|
||||
[AUTH_HEADERS.timestamp]: timestamp,
|
||||
[AUTH_HEADERS.nonce]: nonce,
|
||||
[AUTH_HEADERS.signature]: Hex.encode(signature),
|
||||
[AUTH_HEADERS.algorithm]: signer.algorithm,
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the caller's public key, or throws `AuthError` (401). */
|
||||
async verify(headers: Headers, path: string, body: string): Promise<string> {
|
||||
const publicKey = headers.get(AUTH_HEADERS.publicKey)
|
||||
const timestamp = headers.get(AUTH_HEADERS.timestamp)
|
||||
const nonce = headers.get(AUTH_HEADERS.nonce)
|
||||
const signature = headers.get(AUTH_HEADERS.signature)
|
||||
const algorithm = headers.get(AUTH_HEADERS.algorithm)
|
||||
if (!publicKey || !timestamp || !nonce || !signature || !algorithm) throw new AuthError(401, 'Missing authentication headers')
|
||||
const verifier = this.verifiers.get(algorithm)
|
||||
if (!verifier) throw new AuthError(401, `Unsupported signature algorithm "${algorithm}"`)
|
||||
if (!/^[0-9a-f]+$/.test(publicKey)) throw new AuthError(401, 'Public key must be lowercase hex')
|
||||
if (!/^[0-9A-Za-z_-]{16,128}$/.test(nonce)) throw new AuthError(401, 'Nonce must be 16-128 characters of [0-9A-Za-z_-]')
|
||||
const now = this.now()
|
||||
const time = Number(timestamp)
|
||||
if (!Number.isInteger(time) || Math.abs(now - time) > this.maxSkewMs) throw new AuthError(401, 'Timestamp outside the accepted window')
|
||||
this.forgetExpired(now)
|
||||
const used = `${publicKey}:${nonce}`
|
||||
if (this.seen.has(used)) throw new AuthError(401, 'Replayed request')
|
||||
let valid = false
|
||||
try {
|
||||
valid = await verifier.verify(publicKey, RequestAuth.payload(timestamp, nonce, path, body), Hex.decode(signature))
|
||||
} catch {
|
||||
valid = false
|
||||
}
|
||||
if (!valid) throw new AuthError(401, 'Invalid signature')
|
||||
this.seen.set(used, time + this.maxSkewMs)
|
||||
return publicKey
|
||||
}
|
||||
|
||||
private forgetExpired(now: number): void {
|
||||
for (const [used, expires] of this.seen) {
|
||||
if (expires >= now) break
|
||||
this.seen.delete(used)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Ed25519 signer using WebCrypto (Node 22+ and current browsers; no dependencies). */
|
||||
export class Ed25519Signer implements RequestSigner {
|
||||
readonly algorithm = 'ed25519'
|
||||
|
||||
private constructor(
|
||||
private readonly privateKey: CryptoKey,
|
||||
readonly publicKey: string,
|
||||
) {}
|
||||
|
||||
/** A new random key pair. */
|
||||
static async generate(): Promise<Ed25519Signer> {
|
||||
const pair = (await crypto.subtle.generateKey({ name: 'Ed25519' }, true, ['sign', 'verify'])) as CryptoKeyPair
|
||||
return new Ed25519Signer(pair.privateKey, Hex.encode(new Uint8Array(await crypto.subtle.exportKey('raw', pair.publicKey))))
|
||||
}
|
||||
|
||||
/** Restores a key exported with `exportPrivateKey()` (PKCS #8). */
|
||||
static async fromPrivateKey(pkcs8: Uint8Array): Promise<Ed25519Signer> {
|
||||
const privateKey = await crypto.subtle.importKey('pkcs8', pkcs8 as BufferSource, { name: 'Ed25519' }, true, ['sign'])
|
||||
// WebCrypto can't derive the public key directly; the JWK export of a private key includes it (`x`).
|
||||
const jwk = await crypto.subtle.exportKey('jwk', privateKey)
|
||||
const publicKey = Uint8Array.from(atob(jwk.x!.replace(/-/g, '+').replace(/_/g, '/')), (c) => c.charCodeAt(0))
|
||||
return new Ed25519Signer(privateKey, Hex.encode(publicKey))
|
||||
}
|
||||
|
||||
async exportPrivateKey(): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.exportKey('pkcs8', this.privateKey))
|
||||
}
|
||||
|
||||
async sign(payload: Uint8Array): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.sign({ name: 'Ed25519' }, this.privateKey, payload as BufferSource))
|
||||
}
|
||||
}
|
||||
|
||||
/** Ed25519 verifier using WebCrypto. */
|
||||
export class Ed25519Verifier implements SignatureVerifier {
|
||||
readonly algorithm = 'ed25519'
|
||||
private readonly keys = new Map<string, Promise<CryptoKey>>()
|
||||
|
||||
async verify(publicKey: string, payload: Uint8Array, signature: Uint8Array): Promise<boolean> {
|
||||
let key = this.keys.get(publicKey)
|
||||
if (!key) {
|
||||
key = crypto.subtle.importKey('raw', Hex.decode(publicKey) as BufferSource, { name: 'Ed25519' }, false, ['verify'])
|
||||
this.keys.set(publicKey, key)
|
||||
if (this.keys.size > 10_000) this.keys.delete(this.keys.keys().next().value!)
|
||||
}
|
||||
return crypto.subtle.verify({ name: 'Ed25519' }, await key, signature as BufferSource, payload as BufferSource)
|
||||
}
|
||||
}
|
||||
@@ -15,13 +15,16 @@ export type ChildrenOf<S> = S extends { children: infer C } ? C : {}
|
||||
export type KeyOf<S> = Extract<keyof ChildrenOf<S>, string | number>
|
||||
export type ChildOf<S, K extends KeyOf<S>> = ChildrenOf<S>[K] extends infer C extends Spec ? C : never
|
||||
|
||||
/** Value of this node (`K = undefined`) or of child `K`. */
|
||||
export type ValueAt<S, K> = [K] extends [KeyOf<S>] ? ValueOf<ChildOf<S, K>> : ValueOf<S>
|
||||
|
||||
/** Child keys as yielded at runtime (always strings). */
|
||||
export type KeyString<S> = `${KeyOf<S>}`
|
||||
|
||||
/** `[key, value]` for each child key, as a union of precise pairs. */
|
||||
export type EntryOf<S> = { [K in KeyOf<S>]: [K, ValueOf<ChildOf<S, K>>] }[KeyOf<S>]
|
||||
|
||||
/** `[key, value]` as yielded at runtime (string keys). Dynamic levels collapse to `[string, V]`. */
|
||||
/** `[key, value]` as yielded at runtime (string keys). */
|
||||
export type StringEntryOf<S> =
|
||||
string extends KeyOf<S> ? [string, ValueOf<ChildOf<S, KeyOf<S> & string>>] : { [K in KeyOf<S>]: [`${K}`, ValueOf<ChildOf<S, K>>] }[KeyOf<S>]
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Ordered string-keyed map: sorted chunks of at most `maxChunk` keys (split in half when full)
|
||||
* plus a `Map` for values. Inserts and deletes cost O(log n + maxChunk).
|
||||
*/
|
||||
export class SortedMap<V> {
|
||||
private chunks: string[][] = [[]]
|
||||
private readonly values = new Map<string, V>()
|
||||
/** Bumped on every insert/delete so iterators know to re-seek. */
|
||||
private version = 0
|
||||
|
||||
constructor(private readonly maxChunk = 1024) {}
|
||||
|
||||
get size(): number {
|
||||
return this.values.size
|
||||
}
|
||||
|
||||
get(key: string): V | undefined {
|
||||
return this.values.get(key)
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.values.has(key)
|
||||
}
|
||||
|
||||
set(key: string, value: V): void {
|
||||
if (!this.values.has(key)) {
|
||||
const c = this.chunkFor(key)
|
||||
const chunk = this.chunks[c]!
|
||||
chunk.splice(SortedMap.lowerBound(chunk, key), 0, key)
|
||||
if (chunk.length > this.maxChunk) this.chunks.splice(c + 1, 0, chunk.splice(this.maxChunk >>> 1))
|
||||
this.version++
|
||||
}
|
||||
this.values.set(key, value)
|
||||
}
|
||||
|
||||
delete(key: string): boolean {
|
||||
if (!this.values.delete(key)) return false
|
||||
const c = this.chunkFor(key)
|
||||
const chunk = this.chunks[c]!
|
||||
chunk.splice(SortedMap.lowerBound(chunk, key), 1)
|
||||
if (chunk.length === 0 && this.chunks.length > 1) this.chunks.splice(c, 1)
|
||||
this.version++
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Keys in `[gte, lt)`, descending when `reverse`. Safe across mutations between steps:
|
||||
* after a change it re-seeks from the last key returned.
|
||||
*/
|
||||
*keys(gte?: string, lt?: string, reverse = false): IterableIterator<string> {
|
||||
let last: string | undefined
|
||||
let version = -1
|
||||
let c = 0
|
||||
let i = 0
|
||||
for (;;) {
|
||||
if (version !== this.version) {
|
||||
version = this.version
|
||||
const from = last ?? (reverse ? lt : gte)
|
||||
if (from === undefined) {
|
||||
c = reverse ? this.chunks.length - 1 : 0
|
||||
i = reverse ? this.chunks[c]!.length - 1 : 0
|
||||
} else {
|
||||
c = this.chunkFor(from)
|
||||
i = SortedMap.lowerBound(this.chunks[c]!, from)
|
||||
// Forward: skip `last` itself. Reverse: step to the key before `from`.
|
||||
if (reverse) i--
|
||||
else if (last !== undefined && this.chunks[c]![i] === last) i++
|
||||
}
|
||||
}
|
||||
// Move across chunk boundaries.
|
||||
while (!reverse && c < this.chunks.length - 1 && i >= this.chunks[c]!.length) (c++, (i = 0))
|
||||
while (reverse && c > 0 && i < 0) i = this.chunks[--c]!.length - 1
|
||||
const key = this.chunks[c]?.[i]
|
||||
if (key === undefined || (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt)) return
|
||||
last = key
|
||||
yield key
|
||||
i += reverse ? -1 : 1
|
||||
}
|
||||
}
|
||||
|
||||
/** Index of the last chunk whose first key is <= key (0 if none). */
|
||||
private chunkFor(key: string): number {
|
||||
let lo = 0
|
||||
let hi = this.chunks.length - 1
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi + 1) >>> 1
|
||||
if (this.chunks[mid]![0]! <= key) lo = mid
|
||||
else hi = mid - 1
|
||||
}
|
||||
return lo
|
||||
}
|
||||
|
||||
private static lowerBound(arr: string[], key: string): number {
|
||||
let lo = 0
|
||||
let hi = arr.length
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1
|
||||
if (arr[mid]! < key) lo = mid + 1
|
||||
else hi = mid
|
||||
}
|
||||
return lo
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c).
|
||||
// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession.
|
||||
|
||||
/**
|
||||
* An async iterable queue that bridges push-based producers and pull-based consumers.
|
||||
*
|
||||
* Composes an internal {@link ReadableStream} instead of extending it, so producers
|
||||
* call {@link push} while consumers use standard async iteration (`for await...of`).
|
||||
*
|
||||
* ```ts
|
||||
* const messages = new AsyncPushIterator<SSEvent>();
|
||||
*
|
||||
* // Producer (elsewhere)
|
||||
* messages.push(event);
|
||||
*
|
||||
* // Consumer
|
||||
* for await (const event of messages) {
|
||||
* handle(event);
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* {@link Symbol.asyncIterator} returns `stream.values({ preventCancel: true })` so
|
||||
* breaking out of `for await...of` does not cancel the underlying stream. That
|
||||
* matters for long-lived sessions where the producer keeps pushing after a consumer
|
||||
* stops reading early (for example, test helpers that only collect a fixed count).
|
||||
*/
|
||||
export class AsyncPushIterator<T> {
|
||||
/** ReadableStream backing the async iterator returned from {@link Symbol.asyncIterator}. */
|
||||
private readonly stream: ReadableStream<T>
|
||||
|
||||
/** Controller used to enqueue values and close the stream from {@link push} and {@link close}. */
|
||||
private controller: ReadableStreamDefaultController<T> | undefined
|
||||
|
||||
/** When true, no more values are accepted and iteration eventually completes. */
|
||||
public closed = false
|
||||
|
||||
public constructor() {
|
||||
// `start`'s `this` is the underlying source object when using a plain method.
|
||||
// An arrow function captures the class instance so the controller is stored here.
|
||||
this.stream = new ReadableStream({
|
||||
start: (controller: ReadableStreamDefaultController<T>): void => {
|
||||
this.controller = controller
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueues a value for the consumer.
|
||||
*
|
||||
* After {@link close}, pushes are silently dropped.
|
||||
*
|
||||
* @param value - The next value to yield from the iterator.
|
||||
*/
|
||||
push(value: T): void {
|
||||
if (this.closed) return
|
||||
|
||||
this.controller?.enqueue(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Ends the stream.
|
||||
*
|
||||
* Marks the iterator closed so future {@link push} calls are ignored.
|
||||
* Buffered values are still yielded before iteration completes.
|
||||
*/
|
||||
close(): void {
|
||||
this.closed = true
|
||||
|
||||
try {
|
||||
this.controller?.close()
|
||||
} catch {
|
||||
// The reader may already have released or cancelled the stream.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an async iterator over the composed stream.
|
||||
*
|
||||
* Uses `preventCancel: true` so early `break` from `for await...of` does not
|
||||
* close the stream and block later pushes.
|
||||
*/
|
||||
[Symbol.asyncIterator](): AsyncIterableIterator<T> {
|
||||
return this.stream.values({ preventCancel: true })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c).
|
||||
// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession.
|
||||
|
||||
/**
|
||||
* Exponential backoff is a technique used to retry a function after a delay.
|
||||
*
|
||||
* The delay increases exponentially with each attempt, up to a maximum delay.
|
||||
*
|
||||
* The jitter is a random amount of time added to the delay to prevent thundering herd problems.
|
||||
*
|
||||
* The growth rate is the factor by which the delay increases with each attempt.
|
||||
*/
|
||||
export class ExponentialBackoff {
|
||||
/**
|
||||
* Create a new ExponentialBackoff instance
|
||||
*
|
||||
* @param config - The configuration for the exponential backoff
|
||||
* @returns The ExponentialBackoff instance
|
||||
*/
|
||||
static from(config?: Partial<ExponentialBackoffOptions>): ExponentialBackoff {
|
||||
const backoff = new ExponentialBackoff(config)
|
||||
|
||||
return backoff
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the function with exponential backoff
|
||||
*
|
||||
* @param fn - The function to run
|
||||
* @param onError - The callback to call when an error occurs
|
||||
* @param options - The configuration for the exponential backoff
|
||||
*
|
||||
* @throws An AggregateError with all the errors that were thrown by the task function
|
||||
*
|
||||
* @returns The result of the function
|
||||
*/
|
||||
static run<T>(
|
||||
fn: (context: ExpontialBackoffCallbackContext) => Promise<T>,
|
||||
onError = (_error: Error, _context: ExpontialBackoffCallbackContext): void => {},
|
||||
options?: Partial<ExponentialBackoffOptions>,
|
||||
): Promise<T> {
|
||||
const backoff = ExponentialBackoff.from(options)
|
||||
|
||||
return backoff.run(fn, onError)
|
||||
}
|
||||
|
||||
private readonly options: ExponentialBackoffOptions
|
||||
|
||||
/**
|
||||
* Creates a new exponential-backoff instance.
|
||||
*
|
||||
* Unspecified options use the defaults listed below.
|
||||
*
|
||||
* @param options - Exponential-backoff configuration overrides.
|
||||
* @param options.maxDelay - Maximum delay between retries. Default: `10_000` ms.
|
||||
* @param options.maxAttempts - Maximum number of attempts; `0` retries indefinitely. Default: `10`.
|
||||
* @param options.baseDelay - Delay used as the basis for the first retry. Default: `1_000` ms.
|
||||
* @param options.growthRate - Multiplier applied to the delay after each attempt. Default: `2`.
|
||||
* @param options.jitter - Random proportional variation applied to each delay. Default: `0.1`.
|
||||
*/
|
||||
constructor(options: Partial<ExponentialBackoffOptions> = {}) {
|
||||
this.options = {
|
||||
maxDelay: 10_000,
|
||||
maxAttempts: 10,
|
||||
baseDelay: 1_000,
|
||||
growthRate: 2,
|
||||
jitter: 0.1,
|
||||
...options,
|
||||
}
|
||||
|
||||
ExponentialBackoff.validateOptions(this.options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the function with exponential backoff
|
||||
*
|
||||
* If the function fails but we have not hit the max attempts, the error will be passed to the onError callback
|
||||
* and the function will be retried with an exponential delay
|
||||
*
|
||||
* If the function fails and we have hit the max attempts, an AggregateError will be thrown with all the errors that were thrown by the task function
|
||||
*
|
||||
* @param fn - The function to run
|
||||
* @param onError - The callback to call when an error occurs
|
||||
*
|
||||
* @throws An AggregateError with all the errors that were thrown by the task function
|
||||
*
|
||||
* @returns The result of the function
|
||||
*/
|
||||
async run<T>(
|
||||
fn: (context: ExpontialBackoffCallbackContext) => Promise<T>,
|
||||
onError = (_error: Error, _context: ExpontialBackoffCallbackContext): void => {},
|
||||
): Promise<T> {
|
||||
const abortController = new AbortController()
|
||||
|
||||
const context: ExpontialBackoffCallbackContext = {
|
||||
errors: [],
|
||||
attempt: 1,
|
||||
options: { ...this.options },
|
||||
nextDelay: 0,
|
||||
signal: abortController.signal,
|
||||
abort: (reason?: unknown) => abortController.abort(reason),
|
||||
}
|
||||
|
||||
while (context.options.maxAttempts === 0 || context.attempt <= context.options.maxAttempts) {
|
||||
context.nextDelay = ExponentialBackoff.calculateDelay(context.options, context.attempt - 1)
|
||||
|
||||
try {
|
||||
// Await the promise before returning so its execution context remains in the try-catch
|
||||
// If we didn't await, this `run` function would successfully return and any errors would not be caught here.
|
||||
return await fn(context)
|
||||
} catch (error) {
|
||||
// Store the error in case we fail every attempt
|
||||
const errorInstance = error instanceof Error ? error : new Error(`${error}`)
|
||||
context.errors.push(errorInstance)
|
||||
await onError(errorInstance, context)
|
||||
}
|
||||
|
||||
// If the abort controller is aborted, throw an error
|
||||
if (abortController.signal.aborted) {
|
||||
const reasonError = abortController.signal.reason instanceof Error ? abortController.signal.reason : new Error(`${abortController.signal.reason}`)
|
||||
throw new Error('Exponential backoff aborted', { cause: reasonError })
|
||||
}
|
||||
|
||||
// If we have hit the max attempts, break out of the loop
|
||||
const unlimitedAttempts = context.options.maxAttempts === 0
|
||||
if (!unlimitedAttempts && context.attempt >= context.options.maxAttempts) {
|
||||
break
|
||||
}
|
||||
|
||||
// Wait before going to the next attempt
|
||||
await this.wait(context.nextDelay, abortController.signal)
|
||||
|
||||
context.attempt++
|
||||
}
|
||||
|
||||
// We completed the loop without ever succeeding. Throw an ExponentialBackoffMaxRetriesHitError with all the errors we got
|
||||
throw new AggregateError(context.errors, 'Exponential backoff max retries hit')
|
||||
}
|
||||
|
||||
private wait(delay: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Set a timeout to resolve the promise
|
||||
const timeout = setTimeout(() => {
|
||||
resolve()
|
||||
}, delay)
|
||||
|
||||
// If the signal is aborted, reject the promise
|
||||
signal.addEventListener('abort', () => {
|
||||
clearTimeout(timeout)
|
||||
reject(new Error('Exponential backoff aborted'))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the delay before we should attempt to retry
|
||||
*
|
||||
* NOTE: The maximum delay is (maxDelay * (1 + jitter))
|
||||
*
|
||||
* @param attempt
|
||||
* @returns The time in milliseconds before another attempt should be made
|
||||
*/
|
||||
public static calculateDelay(options: ExponentialBackoffOptions, attempt: number): number {
|
||||
// Get the power of the growth rate
|
||||
const power = options.growthRate ** attempt
|
||||
|
||||
// Get the delay before jitter or limit
|
||||
const rawDelay = options.baseDelay * power
|
||||
|
||||
// Cap the delay to the maximum. Do this before the jitter so jitter does not become larger than delay
|
||||
const cappedDelay = Math.min(rawDelay, options.maxDelay)
|
||||
|
||||
// Get a random number for the amount to "jitter" the delay by
|
||||
const jitterAmount = Math.random()
|
||||
|
||||
// Calculate the jitter
|
||||
const jitter = jitterAmount * options.jitter * cappedDelay
|
||||
|
||||
// Subtract the jitter from the delay
|
||||
return cappedDelay - jitter
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the options, ensure that no values are NaN
|
||||
* @param options
|
||||
*/
|
||||
public static validateOptions(options: ExponentialBackoffOptions): void {
|
||||
const keys = Object.keys(options) as (keyof ExponentialBackoffOptions)[]
|
||||
|
||||
for (const key of keys) {
|
||||
if (!isFinite(options[key])) {
|
||||
throw new Error(`Invalid option: ${key} is not finite`)
|
||||
}
|
||||
|
||||
if (key === 'jitter' && (options[key] < 0 || options[key] > 1)) {
|
||||
throw new Error(`Invalid option: jitter is not between 0 and 1`)
|
||||
}
|
||||
|
||||
if (options[key] < 0) {
|
||||
throw new Error(`Invalid option: ${key} is less than 0`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type ExponentialBackoffOptions = {
|
||||
/**
|
||||
* The maximum delay between attempts in milliseconds
|
||||
*/
|
||||
maxDelay: number
|
||||
|
||||
/**
|
||||
* The maximum number of attempts. Passing 0 will result in infinite attempts.
|
||||
*/
|
||||
maxAttempts: number
|
||||
|
||||
/**
|
||||
* The base delay between attempts in milliseconds
|
||||
*/
|
||||
baseDelay: number
|
||||
|
||||
/**
|
||||
* The growth rate of the delay
|
||||
*/
|
||||
growthRate: number
|
||||
|
||||
/**
|
||||
* The jitter of the delay as a percentage of growthRate
|
||||
*/
|
||||
jitter: number
|
||||
}
|
||||
|
||||
export type ExpontialBackoffCallbackContext = {
|
||||
/** The errors that have occurred */
|
||||
errors: Error[]
|
||||
|
||||
/** The current attempt */
|
||||
attempt: number
|
||||
|
||||
/** The options for the exponential backoff */
|
||||
options: ExponentialBackoffOptions
|
||||
|
||||
/** The next delay before the next attempt */
|
||||
nextDelay: number
|
||||
|
||||
/** The abort signal */
|
||||
signal: AbortSignal
|
||||
|
||||
/** The abort function */
|
||||
abort: (reason?: unknown) => void
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from './types.js'
|
||||
export * from './sse-session.js'
|
||||
export * from './sse-event-parser.js'
|
||||
export * from './async-push-iterator.js'
|
||||
export * from './exponential-backoff.js'
|
||||
@@ -0,0 +1,214 @@
|
||||
// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c).
|
||||
// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession.
|
||||
|
||||
import type { SSEvent } from './types.js'
|
||||
|
||||
/**
|
||||
* Optional encoders used when decoding incoming SSE bytes and re-encoding
|
||||
* any buffered remainder between chunks.
|
||||
*/
|
||||
export interface SSEEventParserOptions {
|
||||
/** Decodes raw stream bytes into text. Defaults to a new `TextDecoder`. */
|
||||
textDecoder: TextDecoder
|
||||
}
|
||||
|
||||
/**
|
||||
* Regex that splits decoded SSE text into lines.
|
||||
*
|
||||
* The SSE wire format is line-oriented (`field: value` per line). Servers may
|
||||
* send `\r\n` (HTTP default), `\n` (Unix), or `\r` (legacy Mac). Matching all
|
||||
* three keeps parsing correct regardless of platform or server implementation.
|
||||
*/
|
||||
export const SSE_LINE_ENDINGS = /\r\n|\r|\n/
|
||||
|
||||
/**
|
||||
* Regex that matches the single optional leading space in an SSE field value.
|
||||
*
|
||||
* Per the SSE spec, `field: value` may include one space immediately after the
|
||||
* colon; that space is not part of the value. Used with `.replace()` to strip
|
||||
* it when parsing lines such as `data: hello` → `hello`.
|
||||
*/
|
||||
export const SSE_FIELD_VALUE_REGEX = /^ /
|
||||
|
||||
/**
|
||||
* The newline character used when normalizing SSE text internally.
|
||||
*
|
||||
* Used to join consecutive `data:` lines into one payload and to reassemble
|
||||
* buffered partial lines between streamed chunks before the next parse call.
|
||||
*/
|
||||
export const NEW_LINE = '\n'
|
||||
|
||||
/**
|
||||
* Incrementally parses Server-Sent Events (SSE) from streamed byte chunks.
|
||||
*
|
||||
* SSE payloads are line-oriented: each event is a sequence of `field: value`
|
||||
* lines terminated by a blank line. This parser accepts arbitrary chunk
|
||||
* boundaries from a live HTTP response body and emits only complete events.
|
||||
*
|
||||
* Typical usage is one parser instance per connection, calling {@link parseEvents}
|
||||
* for each chunk received from the stream:
|
||||
*
|
||||
* ```ts
|
||||
* const parser = new SSEEventParser();
|
||||
*
|
||||
* for await (const chunk of response.body) {
|
||||
* for (const event of parser.parseEvents(chunk)) {
|
||||
* // handle event.data, event.event, event.id, event.retry
|
||||
* }
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* Supported fields follow the SSE spec: `data`, `event`, `id`, and `retry`.
|
||||
* Multiple `data:` lines in one event are joined with `\n`. An event is only
|
||||
* emitted once a blank line is seen and at least one `data` field was collected.
|
||||
*/
|
||||
export class SSEEventParser {
|
||||
private readonly textDecoder: TextDecoder
|
||||
|
||||
/** Bytes from a partial line or incomplete event, carried over to the next chunk. */
|
||||
private messageBuffer: string = ''
|
||||
|
||||
/**
|
||||
* Creates a parser for one SSE stream.
|
||||
*
|
||||
* Inject custom encoders in tests or when a non-default character encoding
|
||||
* is required; production callers can rely on the defaults.
|
||||
*
|
||||
* @param options - Optional text encoders for decode/encode of stream bytes.
|
||||
*/
|
||||
constructor(options: Partial<SSEEventParserOptions> = {}) {
|
||||
this.textDecoder = options.textDecoder ?? new TextDecoder()
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears any buffered bytes from a partial line or incomplete event.
|
||||
*
|
||||
* Call when abandoning a transport so the next connection does not prepend
|
||||
* stale bytes to incoming chunks.
|
||||
*/
|
||||
public reset(): void {
|
||||
// Clear the message buffer
|
||||
this.messageBuffer = ''
|
||||
|
||||
// Reset the decoder to clear any buffered bytes
|
||||
this.textDecoder.decode()
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses all complete SSE events contained in a newly received chunk.
|
||||
*
|
||||
* The chunk is appended to any bytes buffered from earlier calls. Complete
|
||||
* events (blank-line delimited blocks with at least one `data` field) are
|
||||
* returned immediately; any trailing partial line or in-progress event stays
|
||||
* in the internal buffer until a later chunk completes it.
|
||||
*
|
||||
* @param chunk - Newly received SSE stream bytes.
|
||||
* @returns Zero or more complete parsed SSE events from this chunk.
|
||||
*/
|
||||
public parseEvents(chunk: Uint8Array): SSEvent[] {
|
||||
const lines = this.getBufferedLines(chunk)
|
||||
|
||||
// The final split item is the incomplete remainder after the final line
|
||||
// ending. It must not be processed as a complete SSE line.
|
||||
const completeLines = lines.slice(0, -1)
|
||||
|
||||
const events: SSEvent[] = []
|
||||
let event: Partial<SSEvent> = {}
|
||||
let processedLineCount = 0
|
||||
|
||||
for (const [index, line] of completeLines.entries()) {
|
||||
// A blank line indicates the end of an event. If we have received data, we can complete the event
|
||||
if (line === '') {
|
||||
if (event.data !== undefined) {
|
||||
// Cast event when pushing to an SSE Event so it changes from Partial<SSEvent> to SSEvent
|
||||
// The "Partial" can safely be removed as we have guaranteed that `data` is defined
|
||||
events.push(event as SSEvent)
|
||||
}
|
||||
|
||||
event = {}
|
||||
processedLineCount = index + 1
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
this.parseLine(line, event)
|
||||
}
|
||||
|
||||
this.storeRemainingLines(lines, processedLineCount)
|
||||
|
||||
return events
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends a new chunk to the buffered bytes and splits the combined payload
|
||||
* into lines.
|
||||
*
|
||||
* Accepts `\r\n`, `\r`, and `\n` line endings so events parse correctly
|
||||
* regardless of server or platform conventions.
|
||||
*/
|
||||
private getBufferedLines(chunk: Uint8Array): string[] {
|
||||
this.messageBuffer += this.textDecoder.decode(chunk, { stream: true })
|
||||
|
||||
return this.messageBuffer.split(SSE_LINE_ENDINGS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses one SSE field line into an in-progress event.
|
||||
*
|
||||
* Lines starting with a colon are ignored as comments. A single optional space after the colon
|
||||
* is stripped from the field value, per the SSE spec.
|
||||
*/
|
||||
private parseLine(line: string, event: Partial<SSEvent>): void {
|
||||
// Split the line at the first colon. Before the first colon is the field, after the first colon is the value.
|
||||
const [field, ...valueArray] = line.split(':')
|
||||
|
||||
// Join the rest of the array back together to get the value as it was before the split.
|
||||
// If the value has a space at the start, remove it (according to the SSE spec)
|
||||
const value = valueArray.join(':').replace(SSE_FIELD_VALUE_REGEX, '')
|
||||
|
||||
switch (field) {
|
||||
case 'data':
|
||||
event.data = event.data !== undefined ? `${event.data}${NEW_LINE}${value}` : value
|
||||
|
||||
return
|
||||
|
||||
case 'event':
|
||||
event.event = value
|
||||
|
||||
return
|
||||
|
||||
case 'id':
|
||||
event.id = value
|
||||
|
||||
return
|
||||
|
||||
case 'retry':
|
||||
this.parseRetry(value, event)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a numeric `retry:` field to an in-progress event.
|
||||
*
|
||||
* Non-numeric values are ignored rather than failing the parse.
|
||||
*/
|
||||
private parseRetry(value: string, event: Partial<SSEvent>): void {
|
||||
const retry = parseInt(value, 10)
|
||||
|
||||
if (!isNaN(retry)) {
|
||||
event.retry = retry
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Preserves incomplete trailing lines for the next received chunk.
|
||||
*
|
||||
* Only lines that were fully processed (through a completed event boundary)
|
||||
* are discarded; the remainder is re-encoded into {@link messageBuffer}.
|
||||
*/
|
||||
private storeRemainingLines(lines: string[], processedLineCount: number): void {
|
||||
this.messageBuffer = lines.slice(processedLineCount).join(NEW_LINE)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c).
|
||||
// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession.
|
||||
|
||||
import type { SSESessionOptions, SSESessionEventMap, SSEvent } from './types.js'
|
||||
|
||||
import { EventEmitter } from '../event-emitter.js'
|
||||
import { AsyncPushIterator } from './async-push-iterator.js'
|
||||
import { ExponentialBackoff } from './exponential-backoff.js'
|
||||
|
||||
import { SSEEventParser } from './sse-event-parser.js'
|
||||
|
||||
/**
|
||||
* A fetch-based Server-Sent Events (SSE) client with reconnect and optional
|
||||
* browser tab visibility handling.
|
||||
*
|
||||
* Each session maintains one HTTP streaming connection at a time. Incoming
|
||||
* bytes are parsed into {@link SSEvent} objects and delivered through two
|
||||
* surfaces:
|
||||
*
|
||||
* - **Events** — `"connected"`, `"message"`, `"disconnected"`, `"error"`,
|
||||
* and `"closed"` on the session itself (extends {@link EventEmitter}).
|
||||
* - **Messages** — {@link messages}, an async iterable for `for await...of`
|
||||
* consumers.
|
||||
*
|
||||
* Typical usage:
|
||||
*
|
||||
* ```ts
|
||||
* const session = await SSESession.create("/events");
|
||||
*
|
||||
* session.on("message", (event) => console.log(event.data));
|
||||
*
|
||||
* for await (const event of session.messages) {
|
||||
* handle(event);
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* ## Lifecycle
|
||||
*
|
||||
* - {@link connect} opens (or reopens) the transport. It resolves once the
|
||||
* HTTP stream is established; reading continues in the background.
|
||||
* - {@link abort} stops the in-flight fetch without ending the session.
|
||||
* Used internally for tab visibility. The {@link messages} iterator stays
|
||||
* open so an existing consumer resumes when the tab becomes visible again.
|
||||
* - {@link disconnect} aborts the transport, closes {@link messages}, emits
|
||||
* `"closed"`, and disables attached visibility handlers until the next
|
||||
* manual {@link connect}.
|
||||
*
|
||||
* Automatic reconnect is controlled by {@link SSESessionOptions.persistent}
|
||||
* (server closed the stream) and
|
||||
* {@link SSESessionOptions.attemptReconnect} (transport error).
|
||||
*
|
||||
* ## Connection supersession
|
||||
*
|
||||
* Each {@link connect} or {@link abort} bumps an internal `connectionId`.
|
||||
* Background read loops capture their id at start and exit quietly when a
|
||||
* newer connection supersedes them, avoiding duplicate events or errors from
|
||||
* stale transports.
|
||||
*/
|
||||
export class SSESession extends EventEmitter<SSESessionEventMap> {
|
||||
/**
|
||||
* Creates a session and waits until the first connection is established.
|
||||
*
|
||||
* @param url - The SSE endpoint URL.
|
||||
* @param options - Configuration merged with instance defaults.
|
||||
* @returns A connected session.
|
||||
* @throws When the initial connection cannot be established.
|
||||
*/
|
||||
static async create(url: string, options: Partial<SSESessionOptions> = {}): Promise<SSESession> {
|
||||
const client = new SSESession(url, options)
|
||||
await client.connect()
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
/**
|
||||
* Enables SSE resume semantics by sending `Last-Event-ID` on reconnect.
|
||||
*
|
||||
* Listens for incoming `"message"` events and remembers the most recent
|
||||
* {@link SSEvent.id}. On every subsequent connect or reconnect, the session's
|
||||
* {@link onRequest} hook is wrapped so that header is attached when an id is
|
||||
* known, allowing the server to replay only events the client has not yet
|
||||
* received.
|
||||
*
|
||||
* The existing {@link onRequest} callback is preserved and runs after the
|
||||
* header is applied, so auth or other header mutations continue to work.
|
||||
*
|
||||
* Attach as early in the session lifetime as possible. When added after
|
||||
* {@link create}, the initial connection omits the header (no id yet);
|
||||
* all later reconnects include it. To instrument before the first connect,
|
||||
* call this on the session returned from {@link withBrowserVisibility}
|
||||
* before awaiting a separate {@link connect} when the tab starts hidden.
|
||||
*
|
||||
* ```ts
|
||||
* const session = await SSESession.create(url);
|
||||
* await SSESession.addLastEventIdReconnect(session);
|
||||
* // Reconnects send Last-Event-ID once an event with an id is received.
|
||||
* ```
|
||||
*
|
||||
* @param client - The session to instrument.
|
||||
* @returns The same session, for chaining.
|
||||
*/
|
||||
static async addLastEventIdReconnect(client: SSESession): Promise<SSESession> {
|
||||
let lastEventId: string | undefined
|
||||
|
||||
client.on('message', (event) => {
|
||||
// If the event has no id, we should not update the lastEventId and instead keep the old one.
|
||||
lastEventId = event.id ?? lastEventId
|
||||
})
|
||||
|
||||
const originalOnRequest = client.options.onRequest
|
||||
|
||||
client.options.onRequest = async (request: RequestInit): Promise<RequestInit> => {
|
||||
if (lastEventId) {
|
||||
request.headers = { ...request.headers, 'Last-Event-ID': lastEventId }
|
||||
}
|
||||
|
||||
return originalOnRequest(request)
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
/**
|
||||
* Pauses and resumes a session based on browser tab visibility.
|
||||
*
|
||||
* Uses the Page Visibility API (`document.visibilitychange`):
|
||||
*
|
||||
* - **hidden** — {@link abort} stops the active fetch. {@link messages}
|
||||
* stays open; `"disconnected"` fires but `"closed"` does not.
|
||||
* - **visible** — {@link connect} re-establishes the stream if needed.
|
||||
*
|
||||
* The listener is removed when {@link disconnect} emits `"closed"`, and
|
||||
* re-attached automatically on the next `"connected"` event.
|
||||
*
|
||||
* No-op in non-browser environments where `document` is undefined.
|
||||
*
|
||||
* @param client - The session to manage.
|
||||
*/
|
||||
static addBrowserVisibilityHandler(client: SSESession): SSESession {
|
||||
if (typeof document === 'undefined') return client
|
||||
|
||||
const handleVisibilityChange = (): void => {
|
||||
if (document.visibilityState === 'hidden') {
|
||||
void client.abort()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
client.connect().catch(() => {
|
||||
// connect() reports failures via onError and the "error" event.
|
||||
})
|
||||
}
|
||||
|
||||
document.addEventListener('visibilitychange', handleVisibilityChange)
|
||||
|
||||
// Stop managing visibility after an explicit disconnect; re-register
|
||||
// when the same instance is manually connected again.
|
||||
client.once('closed', () => {
|
||||
document.removeEventListener('visibilitychange', handleVisibilityChange)
|
||||
|
||||
client.once('connected', () => {
|
||||
SSESession.addBrowserVisibilityHandler(client)
|
||||
})
|
||||
})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
/** SSE endpoint URL for this session. */
|
||||
private readonly url: string
|
||||
|
||||
/**
|
||||
* Per-instance configuration.
|
||||
*
|
||||
* Defaults live on the instance field (not a shared static) so each session
|
||||
* gets its own {@link SSEEventParser} and {@link ExponentialBackoff}.
|
||||
*/
|
||||
public options: SSESessionOptions = {
|
||||
fetch: (...args) => fetch(...args),
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
body: new FormData(),
|
||||
|
||||
onRequest: (request) => Promise.resolve(request),
|
||||
onConnected: () => {},
|
||||
onDisconnected: () => {},
|
||||
onError: (error) => console.error('SSEClient error:', error),
|
||||
|
||||
// Retry the initial fetch until it succeeds (maxAttempts: 0 = unlimited).
|
||||
retry: new ExponentialBackoff({
|
||||
baseDelay: 1000,
|
||||
maxDelay: 10000,
|
||||
maxAttempts: 0,
|
||||
growthRate: 1.3,
|
||||
jitter: 0.3,
|
||||
}),
|
||||
|
||||
attemptReconnect: true,
|
||||
persistent: false,
|
||||
|
||||
eventParser: new SSEEventParser(),
|
||||
}
|
||||
|
||||
/** AbortController for the currently active fetch, if any. */
|
||||
private controller: AbortController | null = null
|
||||
|
||||
/**
|
||||
* Asynchronous stream of parsed SSE events for the active connection.
|
||||
*
|
||||
* Stays open across {@link abort} and automatic reconnects so an existing
|
||||
* `for await` consumer keeps receiving events after visibility resumes.
|
||||
*
|
||||
* Closes when:
|
||||
* - the server ends the stream and {@link SSESessionOptions.persistent}
|
||||
* is false,
|
||||
* - {@link disconnect} is called, or
|
||||
* - a transport error occurs with
|
||||
* {@link SSESessionOptions.attemptReconnect} disabled.
|
||||
*
|
||||
* A later {@link connect} replaces this with a new iterator when the
|
||||
* previous one was closed. Consumers should read from `session.messages`
|
||||
* rather than caching a reference across terminal disconnects.
|
||||
*/
|
||||
public messages: AsyncPushIterator<SSEvent> = new AsyncPushIterator<SSEvent>()
|
||||
|
||||
public constructor(url: string, options: Partial<SSESessionOptions> = {}) {
|
||||
super()
|
||||
|
||||
this.url = url
|
||||
this.options = {
|
||||
...this.options,
|
||||
...options,
|
||||
// Shallow merge would drop default headers when options.headers is set.
|
||||
headers: { ...this.options.headers, ...options.headers },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects or reconnects to the SSE endpoint.
|
||||
*
|
||||
* Resolves once the HTTP stream is established and `"connected"` has been
|
||||
* emitted. Body reading continues asynchronously in the background via
|
||||
* {@link readStream}.
|
||||
*
|
||||
* @throws When the fetch retry policy exhausts attempts or the connection
|
||||
* is superseded before the reader is handed off (in the latter case the
|
||||
* promise resolves without throwing).
|
||||
*/
|
||||
public async connect(): Promise<void> {
|
||||
// If there is already a controller present, we are already connected.
|
||||
if (this.controller) return
|
||||
|
||||
// Prepare for a fresh transport. Parser state from an abandoned connection
|
||||
// must not bleed into the next one; reopen messages if a prior terminal
|
||||
// close ended the consumer's iteration loop.
|
||||
// NOTE: The resetEventParser may be out of spec and should not be done here: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation
|
||||
this.options.eventParser.reset()
|
||||
|
||||
// If the last messages iterator wasnt closed, we keep it around as this is likely a reconnect.
|
||||
if (this.messages.closed) {
|
||||
this.messages = new AsyncPushIterator<SSEvent>()
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
this.controller = controller
|
||||
|
||||
const { method, headers, body } = this.options
|
||||
|
||||
const fetchBody = method === 'POST' ? body : null
|
||||
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers: headers || {},
|
||||
body: fetchBody ?? null,
|
||||
signal: controller.signal,
|
||||
cache: 'no-store',
|
||||
}
|
||||
|
||||
let reader: ReadableStreamDefaultReader<Uint8Array>
|
||||
|
||||
try {
|
||||
reader = await this.options.retry.run(() => this.createReader(fetchOptions))
|
||||
} catch (error) {
|
||||
// A newer abort/connect superseded this attempt — leave state to the winner.
|
||||
if (this.controller !== controller) return
|
||||
|
||||
this.controller = null
|
||||
|
||||
await this.notifyDisconnected()
|
||||
await this.notifyError(error)
|
||||
this.messages.close()
|
||||
|
||||
throw error
|
||||
}
|
||||
|
||||
// Connection succeeded but was already replaced (for example abort during fetch).
|
||||
if (this.controller !== controller) {
|
||||
await reader.cancel()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await SSESession.tryAsync(
|
||||
() => this.options.onConnected(),
|
||||
(error) => this.notifyError(error),
|
||||
)
|
||||
this.emit('connected', undefined)
|
||||
|
||||
// Fire-and-forget: connect() resolves while the stream is consumed.
|
||||
this.readStream(reader, controller).catch((error) => {
|
||||
this.options.onError(error)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Aborts only the currently active transport.
|
||||
*
|
||||
* The session remains reusable: {@link messages} stays open, visibility
|
||||
* handling stays attached, and {@link connect} can reopen the stream.
|
||||
* Partial parser state from the abandoned transport is discarded.
|
||||
*
|
||||
* Emits `"disconnected"` but not `"closed"`.
|
||||
*/
|
||||
public async abort(): Promise<void> {
|
||||
if (!this.controller) return
|
||||
|
||||
// Grab the current controller to ensure we are aborting the correct one.
|
||||
const controller = this.controller
|
||||
this.controller = null
|
||||
|
||||
// Invalidate any in-flight read loop and fetch for this transport.
|
||||
controller.abort()
|
||||
this.options.eventParser.reset()
|
||||
|
||||
await this.notifyDisconnected()
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminates the session and disables attached visibility handling until
|
||||
* the same instance is manually {@link connect connected} again.
|
||||
*
|
||||
* Closes {@link messages} and emits `"closed"`.
|
||||
*/
|
||||
public async disconnect(): Promise<void> {
|
||||
this.messages.close()
|
||||
this.emit('closed', undefined)
|
||||
|
||||
if (this.controller) {
|
||||
await this.abort()
|
||||
} else {
|
||||
this.options.eventParser.reset()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the HTTP request and returns a reader for the response body.
|
||||
*
|
||||
* {@link SSESessionOptions.onRequest} may mutate headers (for example auth
|
||||
* tokens or `Last-Event-ID`) before the fetch runs.
|
||||
*/
|
||||
private async createReader(fetchOptions: RequestInit): Promise<ReadableStreamDefaultReader<Uint8Array>> {
|
||||
const requestOptions = await this.options.onRequest(fetchOptions)
|
||||
const response = await this.options.fetch(this.url, requestOptions)
|
||||
|
||||
if (!response.ok) {
|
||||
const responseCode = response.status
|
||||
const responseText = await response.text()
|
||||
|
||||
const error = new Error(`HTTP error: ${responseCode} ${responseText}`)
|
||||
void this.notifyError(error)
|
||||
throw error
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
const error = new Error('HTTP error: Response body is null')
|
||||
void this.notifyError(error)
|
||||
throw error
|
||||
}
|
||||
|
||||
return response.body.getReader()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads bytes from an established stream until it ends, errors, or is
|
||||
* superseded by a newer connection.
|
||||
*/
|
||||
private async readStream(reader: ReadableStreamDefaultReader<Uint8Array>, controller: AbortController): Promise<void> {
|
||||
try {
|
||||
while (this.controller === controller) {
|
||||
const { done, value } = await reader.read()
|
||||
|
||||
// abort() or a newer connect() may have landed while we were awaiting.
|
||||
if (this.controller !== controller) return
|
||||
|
||||
if (done) {
|
||||
this.controller = null
|
||||
|
||||
await this.notifyDisconnected()
|
||||
|
||||
if (this.options.persistent) {
|
||||
// Server closed gracefully — reopen unless the consumer opted out.
|
||||
await this.connect()
|
||||
} else {
|
||||
this.messages.close()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Some environments yield `{ done: false, value: undefined }`.
|
||||
if (!value) continue
|
||||
|
||||
for (const event of this.options.eventParser.parseEvents(value)) {
|
||||
this.emit('message', event)
|
||||
this.messages.push(event)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If the controller is different, we already started a new connection and it would be confusing to handle this error.
|
||||
if (controller !== this.controller) return
|
||||
|
||||
// Invalidate the current controller to allow for reconnection if needed
|
||||
this.controller = null
|
||||
|
||||
await this.notifyDisconnected()
|
||||
|
||||
// Expected path for abort() — do not treat as an error or reconnect.
|
||||
if (controller.signal.aborted) return
|
||||
|
||||
await this.notifyError(error)
|
||||
|
||||
if (this.options.attemptReconnect) {
|
||||
await this.connect()
|
||||
} else {
|
||||
this.messages.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Invokes {@link SSESessionOptions.onDisconnected} and emits `"disconnected"`. */
|
||||
private async notifyDisconnected(): Promise<void> {
|
||||
await SSESession.tryAsync(
|
||||
() => this.options.onDisconnected(),
|
||||
(error) => this.options.onError(error),
|
||||
)
|
||||
this.emit('disconnected', undefined)
|
||||
}
|
||||
|
||||
/** Invokes {@link SSESessionOptions.onError} and emits `"error"`. */
|
||||
private async notifyError(error: unknown): Promise<void> {
|
||||
const errorInstance = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
await SSESession.tryAsync(
|
||||
() => this.options.onError(errorInstance),
|
||||
(callbackError) => console.error('SSESession error:', callbackError),
|
||||
)
|
||||
this.emit('error', errorInstance)
|
||||
}
|
||||
|
||||
/** Runs `fn`, passing any error to `onError` instead of throwing. */
|
||||
private static async tryAsync(fn: () => unknown, onError?: (error: Error) => void): Promise<void> {
|
||||
try {
|
||||
await fn()
|
||||
} catch (error) {
|
||||
onError?.(error instanceof Error ? error : new Error(`${error}`))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Vendored from https://git.harvmaster.com/Harvmaster/sse-session (commit c08267e85c).
|
||||
// Local changes: import paths, uses ../event-emitter.ts, tryAsync folded into SSESession.
|
||||
|
||||
export type SSERequestInit = {
|
||||
/**
|
||||
* The HTTP method to use.
|
||||
*/
|
||||
method: 'GET' | 'POST'
|
||||
|
||||
/**
|
||||
* Request headers sent on every connect and reconnect.
|
||||
*/
|
||||
headers?: Record<string, string>
|
||||
|
||||
/**
|
||||
* Request body for POST-based SSE endpoints.
|
||||
*/
|
||||
body?: string | FormData
|
||||
}
|
||||
|
||||
/**
|
||||
* The fetch function to use.
|
||||
*
|
||||
* NOTE: This is compatible with Browser/Node's native "fetch" function.
|
||||
* We use this in place of "typeof fetch" so that we can accept non-standard URLs ("url" is a "string" here).
|
||||
* For example, a LibP2P adapter might not use a standardized URL format (and might only include "path").
|
||||
* This would cause a type error as native fetch expects type "URL".
|
||||
*/
|
||||
export type SSERequestFunction = {
|
||||
fetch: (url: string, options: RequestInit) => Promise<Response>
|
||||
}
|
||||
|
||||
/**
|
||||
* Lifecycle hooks invoked by {@link SSESession} during connect, read, and teardown.
|
||||
*/
|
||||
export type SSESessionCallbacks = {
|
||||
/**
|
||||
* Called before each fetch so callers can attach auth headers, cookies, or
|
||||
* a `Last-Event-ID` for resume semantics.
|
||||
*/
|
||||
onRequest: (request: RequestInit) => Promise<RequestInit>
|
||||
|
||||
/**
|
||||
* Called after the HTTP stream is established and before body reading begins.
|
||||
*/
|
||||
onConnected: () => void
|
||||
|
||||
/**
|
||||
* Called when the active transport ends — including {@link SSESession.abort},
|
||||
* server stream completion, and errors. Not paired with {@link SSESessionCallbacks.onConnected}
|
||||
* when the initial connect never succeeds.
|
||||
*/
|
||||
onDisconnected: () => void
|
||||
|
||||
/**
|
||||
* Called on fetch or read failures. Not invoked for intentional
|
||||
* {@link SSESession.abort} aborts.
|
||||
*/
|
||||
onError: (error: Error) => void
|
||||
}
|
||||
|
||||
export type SSESessionRetryInterface = {
|
||||
/**
|
||||
* Retry policy used while establishing the HTTP connection in
|
||||
* {@link SSESession.connect}. Defaults to {@link ExponentialBackoff} with
|
||||
* unlimited attempts.
|
||||
*/
|
||||
retry: {
|
||||
run<T>(fn: () => Promise<T>, onError?: (error: Error) => void): Promise<T>
|
||||
}
|
||||
}
|
||||
|
||||
export interface SSEParser {
|
||||
/**
|
||||
* Incremental SSE frame parser for the response body.
|
||||
*
|
||||
* {@link SSEEventParser.reset} is called by the session when abandoning a
|
||||
* transport so partial frames do not carry over to the next connection.
|
||||
*/
|
||||
eventParser: {
|
||||
parseEvents(buffer: Uint8Array): SSEvent[]
|
||||
reset(): void
|
||||
}
|
||||
}
|
||||
|
||||
export type SSELifecycleOptions = {
|
||||
/**
|
||||
* When true, {@link SSESession} calls {@link SSESession.connect} again after
|
||||
* a transport **error** (not an intentional abort).
|
||||
*/
|
||||
attemptReconnect: boolean
|
||||
|
||||
/**
|
||||
* When true, {@link SSESession} calls {@link SSESession.connect} again after
|
||||
* the **server** closes the stream normally (`done`).
|
||||
*/
|
||||
persistent: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Events emitted by {@link SSESession}.
|
||||
*
|
||||
* - `"connected"` — HTTP stream established.
|
||||
* - `"message"` — A complete SSE event was parsed.
|
||||
* - `"disconnected"` — The active transport ended (including {@link SSESession.abort}).
|
||||
* - `"error"` — An unexpected fetch or read failure.
|
||||
* - `"closed"` — {@link SSESession.disconnect} was called; visibility handling is detached.
|
||||
*/
|
||||
export type SSESessionEventMap = {
|
||||
connected: void
|
||||
disconnected: void
|
||||
error: Error
|
||||
message: SSEvent
|
||||
closed: void
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for {@link SSESession}.
|
||||
*/
|
||||
export type SSESessionOptions = SSESessionCallbacks & SSERequestInit & SSERequestFunction & SSESessionRetryInterface & SSELifecycleOptions & SSEParser
|
||||
|
||||
/**
|
||||
* Represents a Server-Sent Event.
|
||||
*/
|
||||
export interface SSEvent {
|
||||
/**
|
||||
* Event data.
|
||||
*/
|
||||
data: string
|
||||
|
||||
/**
|
||||
* Event type.
|
||||
*/
|
||||
event?: string
|
||||
|
||||
/**
|
||||
* Event ID.
|
||||
*/
|
||||
id?: string
|
||||
|
||||
/**
|
||||
* Reconnection time in milliseconds.
|
||||
*/
|
||||
retry?: number
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
const ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' // Crockford base32
|
||||
|
||||
/**
|
||||
* ULID-like ids: 10 chars of millisecond time + 16 random chars, sortable by creation time.
|
||||
* Ids from one generator within the same millisecond increment the random part, so they stay
|
||||
* strictly increasing.
|
||||
*/
|
||||
export class Ulid {
|
||||
private lastTime = -1
|
||||
private readonly random = new Uint8Array(16) // one base32 digit (0–31) per byte
|
||||
|
||||
constructor(private readonly now: () => number = Date.now) {}
|
||||
|
||||
generate(): string {
|
||||
let time = this.now()
|
||||
if (time <= this.lastTime) {
|
||||
time = this.lastTime
|
||||
let i = 15
|
||||
while (i >= 0 && this.random[i] === 31) this.random[i--] = 0
|
||||
if (i < 0) throw new Error('ULID overflow within one millisecond')
|
||||
this.random[i]!++
|
||||
} else {
|
||||
crypto.getRandomValues(this.random)
|
||||
for (let i = 0; i < 16; i++) this.random[i]! &= 31
|
||||
this.random[0]! &= 15 // headroom for increments
|
||||
this.lastTime = time
|
||||
}
|
||||
let out = ''
|
||||
for (let i = 0; i < 10; i++) {
|
||||
out = ALPHABET[time % 32]! + out
|
||||
time = Math.floor(time / 32)
|
||||
}
|
||||
for (const digit of this.random) out += ALPHABET[digit]
|
||||
return out
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { TestProject } from 'vitest/node'
|
||||
import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { Ed25519Verifier } from '../../src/utils/request-auth.js'
|
||||
import { serve } from '../kv-storage/node-server.js'
|
||||
|
||||
declare module 'vitest' {
|
||||
interface ProvidedContext {
|
||||
/** Base URL of the test server. `<url>/open/<space>` and `<url>/signed/<space>` are fresh, isolated stores. */
|
||||
serverUrl: string
|
||||
}
|
||||
}
|
||||
|
||||
/** A real HTTP server on another origin than the test page, with one store per `/<auth>/<space>`. */
|
||||
export default async function setup(project: TestProject): Promise<() => Promise<void>> {
|
||||
const spaces = new Map<string, KVStorageHttpServer>()
|
||||
const server = await serve(async (request) => {
|
||||
const [, auth, space] = new URL(request.url).pathname.split('/')
|
||||
const basePath = `/${auth}/${space}`
|
||||
let handler = spaces.get(basePath)
|
||||
if (!handler) {
|
||||
handler = new KVStorageHttpServer(new KVStorageInMemory(), {
|
||||
auth: auth === 'signed' ? { verifiers: [new Ed25519Verifier()] } : 'none',
|
||||
basePath,
|
||||
allowOrigins: '*',
|
||||
})
|
||||
spaces.set(basePath, handler)
|
||||
}
|
||||
return handler.handle(request)
|
||||
})
|
||||
project.provide('serverUrl', server.url)
|
||||
return server.close
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { describe, expect, inject, it, vi } from 'vitest'
|
||||
import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
|
||||
import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
|
||||
import { KVStorageEncrypted } from '../../src/kv-storage/kv-storage-encrypted.js'
|
||||
import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { KVStorageIndexedDB } from '../../src/kv-storage/kv-storage-indexeddb.js'
|
||||
import { Ed25519Signer, type RequestSigner } from '../../src/utils/request-auth.js'
|
||||
import { collect } from '../helpers.js'
|
||||
import { runKVStorageConformance } from '../kv-storage/kv-storage.conformance.js'
|
||||
|
||||
const serverUrl = inject('serverUrl')
|
||||
const key = await KVStorageEncrypted.generateKey()
|
||||
let n = 0
|
||||
const unique = (name: string) => `${name}-${Date.now()}-${n++}`
|
||||
|
||||
/** A fresh store on the test server: `open/<id>` (no auth) or `signed/<id>` (Ed25519). */
|
||||
const space = (auth: 'open' | 'signed') => `${auth}/${unique('space')}`
|
||||
|
||||
/** A client for a store on the test server (another origin, so CORS applies). */
|
||||
function http(path = space('open'), options: { signer?: RequestSigner; network?: { online: boolean } } = {}) {
|
||||
return new KVStorageHttp({
|
||||
url: `${serverUrl}/${path}`,
|
||||
signer: options.signer,
|
||||
fetch: (url, init) => (options.network?.online === false ? Promise.reject(new TypeError('Failed to fetch')) : fetch(url, init)),
|
||||
})
|
||||
}
|
||||
|
||||
runKVStorageConformance('KVStorageIndexedDB (Chromium)', () => new KVStorageIndexedDB({ name: unique('idb') }))
|
||||
runKVStorageConformance('KVStorageEncrypted over IndexedDB (Chromium)', () => new KVStorageEncrypted(new KVStorageIndexedDB({ name: unique('idb') }), { key }))
|
||||
runKVStorageConformance('KVStorageHttp (Chromium, cross-origin)', () => http())
|
||||
runKVStorageConformance(
|
||||
'KVStorageComposite: encrypted HTTP + memory + IndexedDB (Chromium)',
|
||||
() =>
|
||||
new KVStorageComposite({
|
||||
source: new KVStorageEncrypted(http(), { key }),
|
||||
replicas: [new KVStorageInMemory(), new KVStorageIndexedDB({ name: unique('idb') })],
|
||||
}),
|
||||
)
|
||||
|
||||
describe('signed requests (Chromium)', () => {
|
||||
it('signs with Ed25519 and only allows the signer’s own keys', async () => {
|
||||
const path = space('signed')
|
||||
const alice = await Ed25519Signer.generate()
|
||||
const bob = await Ed25519Signer.generate()
|
||||
const client = http(path, { signer: alice })
|
||||
await client.child(alice.publicKey).set('profile', { name: 'Alice' })
|
||||
expect(await client.child(alice.publicKey).get('profile')).toEqual({ name: 'Alice' })
|
||||
await expect(client.child(bob.publicKey).set('profile', {})).rejects.toMatchObject({ status: 403 })
|
||||
await expect(http(path).child(alice.publicKey).get('profile')).rejects.toMatchObject({ status: 401 })
|
||||
})
|
||||
|
||||
it('streams changes over SSE across origins', async () => {
|
||||
const path = space('signed')
|
||||
const alice = await Ed25519Signer.generate()
|
||||
const watcher = http(path, { signer: alice })
|
||||
await watcher.child(alice.publicKey).subscribe()
|
||||
const pushed = watcher.waitFor('change', (e) => e.origin === 'remote')
|
||||
await http(path, { signer: alice }).child(alice.publicKey).set('x', 1n)
|
||||
expect((await pushed).changes).toEqual([{ key: `${alice.publicKey}/x`, value: 1n }])
|
||||
await watcher.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('two devices syncing through the server (Chromium)', () => {
|
||||
/** One device: an encrypted, signed source and a persistent IndexedDB replica and queue. */
|
||||
function device(path: string, signer: Ed25519Signer, dbName: string, network = { online: true }) {
|
||||
const indexedDb = new KVStorageIndexedDB({ name: dbName })
|
||||
const db = new KVStorageComposite({
|
||||
source: new KVStorageEncrypted(http(path, { signer, network }), { key }),
|
||||
replicas: [new KVStorageInMemory(), indexedDb],
|
||||
queue: indexedDb.child('~sync'),
|
||||
prefix: signer.publicKey,
|
||||
retryMs: 50,
|
||||
})
|
||||
return { db, user: db.child(signer.publicKey), network }
|
||||
}
|
||||
|
||||
it('pushes writes from one device to the other', async () => {
|
||||
const path = space('signed')
|
||||
const signer = await Ed25519Signer.generate()
|
||||
const a = device(path, signer, unique('a'))
|
||||
const b = device(path, signer, unique('b'))
|
||||
await Promise.all([a.db.sync(), b.db.sync()])
|
||||
const seen = b.db.waitFor('change', (e) => e.origin === 'remote')
|
||||
await a.user.child('notes').set('1', { text: 'from A' })
|
||||
expect((await seen).changes).toEqual([{ key: `${signer.publicKey}/notes/1`, value: { text: 'from A' } }])
|
||||
expect(await b.user.child('notes').get('1')).toEqual({ text: 'from A' })
|
||||
await Promise.all([a.db.close(), b.db.close()])
|
||||
})
|
||||
|
||||
it('works offline, survives a reload with queued writes, and syncs when back online', async () => {
|
||||
const path = space('signed')
|
||||
const signer = await Ed25519Signer.generate()
|
||||
const dbName = unique('a')
|
||||
const first = device(path, signer, dbName)
|
||||
const docs = new DocumentStorageKV(first.user).child('todos')
|
||||
await docs.set('1', { title: 'Synced', done: false })
|
||||
await first.db.sync()
|
||||
|
||||
first.network.online = false
|
||||
await docs.update('1', { $set: { done: true } })
|
||||
await docs.set('2', { title: 'Written offline', done: false })
|
||||
expect(first.db.queued).toBe(2)
|
||||
await first.db.close()
|
||||
|
||||
// "Reload": new instances over the same IndexedDB database, still offline.
|
||||
const network = { online: false }
|
||||
const reloaded = device(path, signer, dbName, network)
|
||||
const reloadedDocs = new DocumentStorageKV(reloaded.user).child('todos')
|
||||
expect(await collect(reloadedDocs.find({ done: false }))).toEqual([{ key: '2', data: { title: 'Written offline', done: false } }])
|
||||
|
||||
network.online = true
|
||||
await reloaded.db.sync()
|
||||
expect(reloaded.db.queued).toBe(0)
|
||||
|
||||
const other = device(path, signer, unique('b'))
|
||||
await other.db.sync()
|
||||
expect(await collect(new DocumentStorageKV(other.user).child('todos').find())).toEqual([
|
||||
{ key: '1', data: { title: 'Synced', done: true } },
|
||||
{ key: '2', data: { title: 'Written offline', done: false } },
|
||||
])
|
||||
await Promise.all([reloaded.db.close(), other.db.close()])
|
||||
})
|
||||
|
||||
it('stores only ciphertext on the server', async () => {
|
||||
const path = space('signed')
|
||||
const signer = await Ed25519Signer.generate()
|
||||
const a = device(path, signer, unique('a'))
|
||||
await a.user.set('secret', 'the plans')
|
||||
const raw: BaseKVStorage = http(path, { signer })
|
||||
const [stored] = await raw.readMany([`${signer.publicKey}/secret`])
|
||||
expect(stored).toBeInstanceOf(Uint8Array)
|
||||
await vi.waitFor(async () => expect(await a.user.get('secret')).toBe('the plans'))
|
||||
await a.db.close()
|
||||
})
|
||||
})
|
||||
@@ -1,73 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { decodeIndexValue, encodeIndexValue, type IndexValue } from '../../src/core/encoding.js'
|
||||
|
||||
const separators = fc.constantFrom('/', ':', '.', '|', '#', '!')
|
||||
|
||||
const indexValue: fc.Arbitrary<IndexValue> = fc.oneof(
|
||||
fc.constant(null),
|
||||
fc.boolean(),
|
||||
fc.double({ noNaN: true }),
|
||||
fc.integer(),
|
||||
fc.string({ unit: 'binary' }),
|
||||
fc.string({ unit: fc.constantFrom('/', ':', '0', '1', 'a', '%', '~', '\0', '!') }),
|
||||
)
|
||||
|
||||
const rank = (v: IndexValue) => (v === null ? 0 : v === false ? 1 : v === true ? 2 : typeof v === 'number' ? 3 : 4)
|
||||
|
||||
function compare(a: IndexValue, b: IndexValue): number {
|
||||
const ra = rank(a)
|
||||
const rb = rank(b)
|
||||
if (ra !== rb) return ra - rb
|
||||
if (typeof a === 'number' && typeof b === 'number') return a === b ? 0 : a < b ? -1 : 1
|
||||
if (typeof a === 'string' && typeof b === 'string') return a === b ? 0 : a < b ? -1 : 1
|
||||
return 0
|
||||
}
|
||||
|
||||
const sign = (n: number) => (n < 0 ? -1 : n > 0 ? 1 : 0)
|
||||
const cmpStr = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0)
|
||||
|
||||
describe('encodeIndexValue', () => {
|
||||
it('round-trips (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(indexValue, separators, (v, sep) => {
|
||||
const decoded = decodeIndexValue(encodeIndexValue(v, sep), sep)
|
||||
if (typeof v === 'number' && v === 0)
|
||||
expect(decoded).toBe(0) // -0 normalised
|
||||
else expect(decoded).toEqual(v)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('emits only characters above the separator (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(indexValue, separators, (v, sep) => {
|
||||
for (const c of encodeIndexValue(v, sep)) expect(c > sep).toBe(true)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves order, including when followed by a separator and suffix (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(indexValue, indexValue, fc.string(), fc.string(), separators, (a, b, sa, sb, sep) => {
|
||||
const ea = encodeIndexValue(a, sep)
|
||||
const eb = encodeIndexValue(b, sep)
|
||||
const expected = sign(compare(a, b))
|
||||
expect(cmpStr(ea, eb)).toBe(expected)
|
||||
if (expected !== 0) expect(cmpStr(ea + sep + sa, eb + sep + sb)).toBe(expected)
|
||||
}),
|
||||
{ numRuns: 2000 },
|
||||
)
|
||||
})
|
||||
|
||||
it('orders types null < false < true < numbers < strings', () => {
|
||||
const values: IndexValue[] = ['', 'a', -Infinity, -1, 0, 1e300, Infinity, true, false, null]
|
||||
const sorted = [...values].sort((a, b) => cmpStr(encodeIndexValue(a), encodeIndexValue(b)))
|
||||
expect(sorted).toEqual([null, false, true, -Infinity, -1, 0, 1e300, Infinity, '', 'a'])
|
||||
})
|
||||
|
||||
it('rejects NaN and non-index values', () => {
|
||||
expect(() => encodeIndexValue(NaN)).toThrow()
|
||||
expect(() => encodeIndexValue({} as never)).toThrow()
|
||||
})
|
||||
})
|
||||
@@ -1,73 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { escapeSegment, joinPath, pathKeys, splitPath, succ, toSegment, unescapeSegment, validateSeparator } from '../../src/core/path.js'
|
||||
import { InvalidKeyError } from '../../src/errors.js'
|
||||
|
||||
const separators = fc.constantFrom('/', ':', '.', '|', '#', '\\', '!')
|
||||
|
||||
describe('escapeSegment', () => {
|
||||
it('escapes %, the separator and a leading ~', () => {
|
||||
expect(escapeSegment('a/b')).toBe('a%2Fb')
|
||||
expect(escapeSegment('50%')).toBe('50%25')
|
||||
expect(escapeSegment('~x')).toBe('%7Ex')
|
||||
expect(escapeSegment('x~')).toBe('x~')
|
||||
expect(escapeSegment('a:b', ':')).toBe('a%3Ab')
|
||||
expect(escapeSegment('a/b', ':')).toBe('a/b')
|
||||
expect(escapeSegment('plain')).toBe('plain')
|
||||
})
|
||||
|
||||
it('round-trips (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => {
|
||||
expect(unescapeSegment(escapeSegment(s, sep))).toBe(s)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('never emits a raw separator or a leading ~ (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(fc.string({ unit: 'binary' }), separators, (s, sep) => {
|
||||
const e = escapeSegment(s, sep)
|
||||
expect(e.includes(sep)).toBe(false)
|
||||
expect(e.startsWith('~')).toBe(false)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('joins and splits paths (property)', () => {
|
||||
fc.assert(
|
||||
fc.property(fc.array(fc.string({ minLength: 1 }), { maxLength: 6 }), separators, (path, sep) => {
|
||||
expect(splitPath(joinPath(path, sep), sep)).toEqual(path)
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pathKeys', () => {
|
||||
it('precomputes selfKey, prefix and subtree end', () => {
|
||||
expect(pathKeys(['users', '123'], '/')).toMatchObject({ selfKey: 'users/123', prefix: 'users/123/', end: 'users/1230' })
|
||||
expect(pathKeys([], '/')).toMatchObject({ selfKey: '', prefix: '', end: undefined })
|
||||
expect(pathKeys(['a/b'], '/').selfKey).toBe('a%2Fb')
|
||||
})
|
||||
})
|
||||
|
||||
describe('misc', () => {
|
||||
it('succ returns the next char', () => {
|
||||
expect(succ('/')).toBe('0')
|
||||
})
|
||||
|
||||
it('toSegment rejects undefined, null, empty and non-finite', () => {
|
||||
expect(() => toSegment(undefined)).toThrow(InvalidKeyError)
|
||||
expect(() => toSegment(null)).toThrow(TypeError)
|
||||
expect(() => toSegment('')).toThrow(InvalidKeyError)
|
||||
expect(() => toSegment(NaN)).toThrow(InvalidKeyError)
|
||||
expect(toSegment(12)).toBe('12')
|
||||
})
|
||||
|
||||
it('validates separators', () => {
|
||||
expect(() => validateSeparator('//')).toThrow()
|
||||
expect(() => validateSeparator('%')).toThrow()
|
||||
expect(() => validateSeparator('~')).toThrow()
|
||||
expect(() => validateSeparator(':')).not.toThrow()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { BaseDocumentStorage } from '../../src/document-storage/base-document-storage.js'
|
||||
import { collect } from '../helpers.js'
|
||||
|
||||
type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } }
|
||||
|
||||
/** Behaviour every document storage must have. `make` returns an empty storage. */
|
||||
export function runDocumentStorageSuite(name: string, make: () => BaseDocumentStorage): void {
|
||||
describe(`document storage: ${name}`, () => {
|
||||
const users = () => make().child('users')
|
||||
|
||||
it('inserts with time-sortable ids, gets and lists', async () => {
|
||||
const u = users()
|
||||
const a = await u.insert({ name: 'Ada' })
|
||||
const b = await u.insert({ name: 'Bob' })
|
||||
expect(a < b).toBe(true)
|
||||
expect(await u.get(a)).toEqual({ name: 'Ada' })
|
||||
expect(await collect(u.keys())).toEqual([a, b])
|
||||
})
|
||||
|
||||
it('set / get / delete in self and child forms', async () => {
|
||||
const u = users()
|
||||
await u.set('1', { name: 'Ada' })
|
||||
expect(await u.child('1').get()).toEqual({ name: 'Ada' })
|
||||
await u.child('1').set({ name: 'Ada L.' })
|
||||
expect(await u.getMany(['1', '2'])).toEqual([{ name: 'Ada L.' }, undefined])
|
||||
expect(await u.delete('1')).toBe(true)
|
||||
expect(await u.child('1').delete()).toBe(false)
|
||||
expect(await u.get('1')).toBeUndefined()
|
||||
await u.set('1', { name: 'Again' })
|
||||
expect(await u.get('1')).toEqual({ name: 'Again' })
|
||||
})
|
||||
|
||||
it('updates with operators, returns the new state, and supports upsert', async () => {
|
||||
const u = users()
|
||||
await u.set('1', { name: 'Ada', tags: ['a'], logins: 1 })
|
||||
expect(await u.update('1', { $inc: { logins: 2 }, $push: { tags: 'b' }, $set: { 'address.city': 'London' } })).toEqual({
|
||||
name: 'Ada',
|
||||
tags: ['a', 'b'],
|
||||
logins: 3,
|
||||
address: { city: 'London' },
|
||||
})
|
||||
expect(await u.child('1').update({ $unset: { address: 1 } })).toEqual({ name: 'Ada', tags: ['a', 'b'], logins: 3 })
|
||||
expect(await u.update('missing', { $set: { a: 1 } })).toBeUndefined()
|
||||
expect(await u.update('missing', { $inc: { n: 1 } }, { upsert: true })).toEqual({ n: 1 })
|
||||
})
|
||||
|
||||
it('rejects bad keys and documents', async () => {
|
||||
const u = users()
|
||||
await expect(u.get(undefined as never)).rejects.toThrow(TypeError)
|
||||
await expect(u.update(undefined as never, { $set: {} })).rejects.toThrow(TypeError)
|
||||
await expect(make().get()).rejects.toThrow(/root/)
|
||||
await expect(u.set('x', [] as never)).rejects.toThrow(/plain object/)
|
||||
})
|
||||
|
||||
it('nests documents under documents', async () => {
|
||||
const u = users()
|
||||
await u.set('1', { name: 'Ada' })
|
||||
await u.child('1').child('posts').set('p1', { title: 'Hi' })
|
||||
expect(await u.child('1').child('posts').get('p1')).toEqual({ title: 'Hi' })
|
||||
expect((await collect(u.find())).map((r) => r.key)).toEqual(['1'])
|
||||
})
|
||||
|
||||
it('finds with equality, comparisons, $in, dot paths, sort, skip and limit', async () => {
|
||||
const u = users()
|
||||
const rows: [string, User][] = [
|
||||
['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }],
|
||||
['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'] }],
|
||||
['c', { name: 'Cy', age: 51, address: { city: 'London' } }],
|
||||
['d', { name: 'Di', age: 20 }],
|
||||
]
|
||||
for (const [k, v] of rows) await u.set(k, v)
|
||||
const keys = async (filter: Record<string, unknown>, options = {}) => (await collect(u.find(filter, options))).map((r) => r.key)
|
||||
expect(await keys({ age: 20 })).toEqual(['b', 'd'])
|
||||
expect(await keys({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd'])
|
||||
expect(await keys({ age: { $in: [36, 51] } })).toEqual(['a', 'c'])
|
||||
expect(await keys({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c'])
|
||||
expect(await keys({ tags: 'y' })).toEqual(['a', 'b'])
|
||||
expect(await keys({ email: null })).toEqual(['c', 'd'])
|
||||
expect(await keys({}, { sort: { age: -1 } })).toEqual(['c', 'a', 'b', 'd'])
|
||||
expect(await keys({}, { sort: { age: 1 }, skip: 1, limit: 2 })).toEqual(['d', 'a'])
|
||||
expect(await u.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: rows[1]![1] })
|
||||
expect(await u.findOne({ email: 'nobody' })).toBeUndefined()
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { DocumentStorageInMemory } from '../../src/document-storage/document-storage-in-memory.js'
|
||||
import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
|
||||
import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js'
|
||||
import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { collect, KVStorageCounting } from '../helpers.js'
|
||||
import { runDocumentStorageSuite } from './document-storage.suite.js'
|
||||
|
||||
const indexes = { users: ['email', 'age'] }
|
||||
|
||||
runDocumentStorageSuite('DocumentStorageInMemory', () => new DocumentStorageInMemory())
|
||||
runDocumentStorageSuite('DocumentStorageInMemory (indexed)', () => new DocumentStorageInMemory({ indexes }))
|
||||
runDocumentStorageSuite('DocumentStorageKV', () => new DocumentStorageKV(new KVStorageInMemory()))
|
||||
runDocumentStorageSuite('EventSourced (in-memory projections)', () => {
|
||||
const kv = new KVStorageInMemory()
|
||||
return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageInMemory({ indexes }))
|
||||
})
|
||||
runDocumentStorageSuite('EventSourced (KV projections)', () => {
|
||||
const kv = new KVStorageInMemory()
|
||||
return new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections')))
|
||||
})
|
||||
|
||||
describe('DocumentStorageInMemory indexes', () => {
|
||||
it('keeps the equality index in sync with updates and deletes', async () => {
|
||||
const users = new DocumentStorageInMemory({ indexes }).child('users')
|
||||
await users.set('1', { email: 'a', age: 1 })
|
||||
await users.set('2', { email: 'b', age: 1 })
|
||||
await users.update('1', { $set: { email: 'c' } })
|
||||
await users.delete('2')
|
||||
expect(await collect(users.find({ email: 'a' }))).toEqual([])
|
||||
expect(await collect(users.find({ email: 'c' }))).toEqual([{ key: '1', data: { email: 'c', age: 1 } }])
|
||||
expect(await collect(users.find({ age: { $eq: 1 } }))).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('EventSourcedDocumentStorage', () => {
|
||||
function setup() {
|
||||
const kv = new KVStorageCounting()
|
||||
const events = new EventStorageKV(kv)
|
||||
const projections = new DocumentStorageKV(kv.child('projections'))
|
||||
return { kv, events, projections, docs: new EventSourcedDocumentStorage(events, projections) }
|
||||
}
|
||||
|
||||
it('records resolved changes as history and replays any event', async () => {
|
||||
const users = setup().docs.child('users')
|
||||
await users.set('1', { n: 5 })
|
||||
await users.update('1', { $inc: { n: 1 } })
|
||||
await users.delete('1')
|
||||
await users.set('1', { n: 10 })
|
||||
const history = await collect(users.history('1'))
|
||||
expect(history.map((e) => e.data)).toEqual([
|
||||
{ op: 'set', doc: { n: 5 } },
|
||||
{ op: 'patch', set: { n: 6 }, unset: [] },
|
||||
{ op: 'delete' },
|
||||
{ op: 'set', doc: { n: 10 } },
|
||||
])
|
||||
expect(await users.getAt('1', history[1]!.id)).toEqual({ n: 6 })
|
||||
expect(await users.child('1').getAt(history[2]!.id)).toBeUndefined()
|
||||
expect(await users.get('1')).toEqual({ n: 10 })
|
||||
})
|
||||
|
||||
it('catchUp folds in events written elsewhere', async () => {
|
||||
const { docs, events } = setup()
|
||||
const users = docs.child('users')
|
||||
await users.set('1', { n: 1 })
|
||||
await events.child('users').append('1', [{ op: 'patch', set: { m: 2 }, unset: [] }]) // e.g. synced from another device
|
||||
expect(await users.get('1')).toEqual({ n: 1 })
|
||||
expect(await users.catchUp('1')).toEqual({ n: 1, m: 2 })
|
||||
expect(await users.get('1')).toEqual({ n: 1, m: 2 })
|
||||
await events.child('users').append('2', [{ op: 'set', doc: { n: 7 } }, { op: 'delete' }])
|
||||
expect(await users.catchUp('2')).toBeUndefined()
|
||||
expect(await users.get('2')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('two instances editing different fields both keep their edits; the same field ends with the later value', async () => {
|
||||
const kv = new KVStorageInMemory()
|
||||
let t = 1000
|
||||
const make = () =>
|
||||
new EventSourcedDocumentStorage(new EventStorageKV(kv, { now: () => t++ }), new DocumentStorageKV(kv.child('projections'))).child('users')
|
||||
const a = make()
|
||||
const b = make()
|
||||
await a.set('1', { name: 'x', age: 1 })
|
||||
await Promise.all([a.update('1', { $set: { name: 'Ada' } }), b.update('1', { $set: { age: 2 } })])
|
||||
await b.update('1', { $set: { name: 'Bob' } })
|
||||
expect(await a.catchUp('1')).toEqual({ name: 'Bob', age: 2 })
|
||||
})
|
||||
|
||||
it('a write is a projection read, an append and a projection write', async () => {
|
||||
const { docs, kv } = setup()
|
||||
await docs.child('users').set('1', { n: 1 })
|
||||
kv.reset()
|
||||
await docs.child('users').update('1', { $inc: { n: 1 } })
|
||||
expect(kv.calls).toEqual({ readMany: 2, writeMany: 2 })
|
||||
})
|
||||
})
|
||||
@@ -1,379 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { KVStorage } from '../../src/kv/node.js'
|
||||
import { InMemoryKVDriver } from '../../src/kv/memory.js'
|
||||
import { KVEventStorage } from '../../src/events/kv.js'
|
||||
import { EventSourcedDocumentStorage, type EventSourcedDocumentStorageOptions } from '../../src/documents/event-sourced.js'
|
||||
import type { DocIndexDeclarations } from '../../src/documents/indexes.js'
|
||||
import type { Filter } from '../../src/documents/filter.js'
|
||||
import { CapabilityError, UniqueConstraintError } from '../../src/errors.js'
|
||||
import { collect, CountingDriver, type Capability } from '../helpers.js'
|
||||
|
||||
type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } }
|
||||
|
||||
const userIndexes: DocIndexDeclarations = {
|
||||
users: [{ fields: ['email'], unique: true }, { fields: ['age'] }, { name: 'cityAge', fields: ['address.city', 'age'] }, { fields: ['tags'] }],
|
||||
'users/*/posts': [{ fields: ['createdAt'] }],
|
||||
}
|
||||
|
||||
function setup(opts: { hide?: Capability[]; inner?: InMemoryKVDriver } & EventSourcedDocumentStorageOptions = {}) {
|
||||
const { hide = [], inner = new InMemoryKVDriver(), ...docOpts } = opts
|
||||
const driver = new CountingDriver(inner, hide)
|
||||
const kv = new KVStorage(driver)
|
||||
const events = new KVEventStorage(kv)
|
||||
const docs = new EventSourcedDocumentStorage(events, { indexes: userIndexes, warnOnFullScan: false, ...docOpts })
|
||||
return { driver, kv, events, docs, users: docs.child('users') }
|
||||
}
|
||||
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
describe.each([
|
||||
['atomic batch', [] as Capability[]],
|
||||
['non-atomic', ['batch'] as Capability[]],
|
||||
])('EventSourcedDocumentStorage (%s)', (_, hide) => {
|
||||
describe('CRUD', () => {
|
||||
it('inserts with time-sortable ids and reads back', async () => {
|
||||
const { users } = setup({ hide })
|
||||
const a = await users.insert({ name: 'Ada', age: 36 })
|
||||
const b = await users.insert({ name: 'Bob', age: 20 })
|
||||
expect(a).toMatch(/^[0-9A-HJKMNP-TV-Z]{26}$/)
|
||||
expect(a < b).toBe(true)
|
||||
expect(await users.get(a)).toEqual({ name: 'Ada', age: 36 })
|
||||
expect(await collect(users.keys())).toEqual([a, b])
|
||||
})
|
||||
|
||||
it('set / get / delete in self and child forms', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('123', { name: 'Ada', age: 36 })
|
||||
expect(await users.child('123').get()).toEqual({ name: 'Ada', age: 36 })
|
||||
await users.child('123').set({ name: 'Ada L.' })
|
||||
expect(await users.get('123')).toEqual({ name: 'Ada L.' })
|
||||
expect(await users.delete('123')).toBe(true)
|
||||
expect(await users.delete('123')).toBe(false)
|
||||
expect(await users.get('123')).toBeUndefined()
|
||||
expect(await collect(users.keys())).toEqual([])
|
||||
await users.set('123', { name: 'Again' })
|
||||
expect(await users.get('123')).toEqual({ name: 'Again' })
|
||||
})
|
||||
|
||||
it('applies arity rules', async () => {
|
||||
const { users, docs } = setup({ hide })
|
||||
const id: string | undefined = undefined
|
||||
await expect(users.get(id as never)).rejects.toThrow(TypeError)
|
||||
await expect(users.update(id as never, { $set: { a: 1 } })).rejects.toThrow(TypeError)
|
||||
await expect(users.delete(id as never)).rejects.toThrow(TypeError)
|
||||
await expect(docs.get()).rejects.toThrow(/root/)
|
||||
await expect(users.set('x', [] as never)).rejects.toThrow(/plain object/)
|
||||
})
|
||||
|
||||
it('nests documents under documents', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('u1', { name: 'Ada' })
|
||||
const posts = users.child('u1').child('posts')
|
||||
await posts.set('p1', { title: 'Hello', createdAt: '2024-01' })
|
||||
expect(await posts.get('p1')).toEqual({ title: 'Hello', createdAt: '2024-01' })
|
||||
expect((await collect(users.find())).map((r) => r.key)).toEqual(['u1'])
|
||||
expect((await collect(posts.find({ createdAt: { $gte: '2024' } }))).map((r) => r.key)).toEqual(['p1'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('update', () => {
|
||||
it('supports $set $unset $inc $push $pull $addToSet with dot paths', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('1', { name: 'Ada', tags: ['a'], address: { city: 'X' }, nickname: 'A' } as User)
|
||||
const r = await users.update('1', {
|
||||
$set: { name: 'Ada L.', 'address.zip': '123' },
|
||||
$inc: { logins: 1 },
|
||||
$push: { tags: { $each: ['b', 'c'] } },
|
||||
$unset: { nickname: true },
|
||||
})
|
||||
expect(r).toEqual({ name: 'Ada L.', tags: ['a', 'b', 'c'], address: { city: 'X', zip: '123' }, logins: 1 })
|
||||
await users.update('1', { $pull: { tags: { $in: ['a', 'c'] } }, $addToSet: { 'address.list': { $each: [1, 1, 2] } } })
|
||||
await users.child('1').update({ $inc: { logins: 5 }, $addToSet: { tags: 'b' } })
|
||||
expect(await users.get('1')).toEqual({ name: 'Ada L.', tags: ['b'], address: { city: 'X', zip: '123', list: [1, 2] }, logins: 6 })
|
||||
})
|
||||
|
||||
it('returns undefined for missing docs unless upsert', async () => {
|
||||
const { users } = setup({ hide })
|
||||
expect(await users.update('nope', { $set: { a: 1 } })).toBeUndefined()
|
||||
expect(await users.get('nope')).toBeUndefined()
|
||||
expect(await users.update('nope', { $inc: { n: 2 } }, { upsert: true })).toEqual({ n: 2 })
|
||||
})
|
||||
|
||||
it('rejects conflicting paths and bad operators', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('1', { age: 1 })
|
||||
await expect(users.update('1', { $set: { a: 1 }, $unset: { 'a.b': 1 } })).rejects.toThrow(/conflict/)
|
||||
await expect(users.update('1', { $rename: {} } as never)).rejects.toThrow(/Unsupported/)
|
||||
await expect(users.update('1', { $inc: { name: 1 } }).then(() => users.update('1', { $push: { age: 1 } }))).rejects.toThrow(/non-array/)
|
||||
})
|
||||
|
||||
it('stores resolved deltas, not operators, and skips no-op updates', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('1', { logins: 5 })
|
||||
await users.update('1', { $inc: { logins: 1 } })
|
||||
await users.update('1', { $set: { logins: 6 } }) // no-op
|
||||
expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([
|
||||
{ op: 'set', doc: { logins: 5 } },
|
||||
{ op: 'patch', set: { logins: 6 } },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('history and getAt', () => {
|
||||
it('replays to any version', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('1', { n: 1 })
|
||||
await users.update('1', { $inc: { n: 1 } })
|
||||
await users.delete('1')
|
||||
await users.set('1', { n: 10 })
|
||||
expect(await users.getAt('1', 0)).toBeUndefined()
|
||||
expect(await users.getAt('1', 1)).toEqual({ n: 1 })
|
||||
expect(await users.getAt('1', 2)).toEqual({ n: 2 })
|
||||
expect(await users.getAt('1', 3)).toBeUndefined()
|
||||
expect(await users.child('1').getAt(4)).toEqual({ n: 10 })
|
||||
expect(await users.getAt('1', 99)).toEqual({ n: 10 })
|
||||
expect((await collect(users.child('1').history())).map((e) => e.version)).toEqual([1, 2, 3, 4])
|
||||
})
|
||||
})
|
||||
|
||||
describe('find', () => {
|
||||
async function seed() {
|
||||
const ctx = setup({ hide })
|
||||
const rows: [string, User][] = [
|
||||
['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }],
|
||||
['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'], address: { city: 'Paris' } }],
|
||||
['c', { name: 'Cy', age: 51, tags: [], address: { city: 'London' } }],
|
||||
['d', { name: 'Di', age: 20, email: 'di@x' }],
|
||||
['e', { name: 'Ed', email: 'ed@x', address: { city: 'London' } }],
|
||||
]
|
||||
for (const [k, v] of rows) await ctx.users.set(k, v)
|
||||
await ctx.users.set('gone', { name: 'Gone', age: 20 })
|
||||
await ctx.users.delete('gone')
|
||||
return ctx
|
||||
}
|
||||
const keysOf = async (it: AsyncIterable<{ key: string }>) => (await collect(it)).map((r) => r.key)
|
||||
|
||||
it('evaluates comparison, logical and element operators', async () => {
|
||||
const { users } = await seed()
|
||||
// Unsorted results come in index or key order depending on the plan; compare as sets.
|
||||
const q = async (f: Filter) => (await keysOf(users.find(f))).sort()
|
||||
expect(await q({})).toEqual(['a', 'b', 'c', 'd', 'e'])
|
||||
expect(await q({ age: 20 })).toEqual(['b', 'd'])
|
||||
expect(await q({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd'])
|
||||
expect(await q({ age: { $gt: 20 } })).toEqual(['a', 'c'])
|
||||
expect(await q({ age: { $ne: 20 } })).toEqual(['a', 'c', 'e'])
|
||||
expect(await q({ age: { $in: [36, 51] } })).toEqual(['a', 'c'])
|
||||
expect(await q({ age: { $nin: [36, 51] } })).toEqual(['b', 'd', 'e'])
|
||||
expect(await q({ age: { $exists: false } })).toEqual(['e'])
|
||||
expect(await q({ age: null })).toEqual(['e'])
|
||||
expect(await q({ email: { $exists: true }, age: { $lte: 20 } })).toEqual(['b', 'd'])
|
||||
expect(await q({ 'address.city': 'London' })).toEqual(['a', 'c', 'e'])
|
||||
expect(await q({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c'])
|
||||
expect(await q({ tags: 'y' })).toEqual(['a', 'b'])
|
||||
expect(await q({ tags: ['y'] })).toEqual(['b'])
|
||||
expect(await q({ $or: [{ age: 51 }, { name: 'Bob' }] })).toEqual(['b', 'c'])
|
||||
expect(await q({ $and: [{ age: { $gte: 20 } }, { age: { $lte: 20 } }] })).toEqual(['b', 'd'])
|
||||
expect(await q({ $not: { age: 20 } })).toEqual(['a', 'c', 'e'])
|
||||
expect(await q({ age: { $not: { $gte: 30 } } })).toEqual(['b', 'd', 'e'])
|
||||
expect(await q({ name: { $gt: 5 } })).toEqual([]) // type-bracketed
|
||||
})
|
||||
|
||||
it('sorts (index-backed and in-memory), skips, limits and projects', async () => {
|
||||
const { users } = await seed()
|
||||
expect(await keysOf(users.find({}, { sort: { age: 1 } }))).toEqual(['e', 'b', 'd', 'a', 'c'])
|
||||
expect(await keysOf(users.find({}, { sort: { age: -1 } }))).toEqual(['c', 'a', 'd', 'b', 'e'])
|
||||
expect(await keysOf(users.find({}, { sort: { name: -1 }, skip: 1, limit: 2 }))).toEqual(['d', 'c'])
|
||||
expect(await keysOf(users.find({ age: { $gte: 20 } }, { sort: { age: -1 }, limit: 2 }))).toEqual(['c', 'a'])
|
||||
expect(await keysOf(users.find({ 'address.city': 'London' }, { sort: { age: 1 } }))).toEqual(['e', 'a', 'c'])
|
||||
expect(await collect(users.find({ age: 36 }, { projection: { name: 1, 'address.city': 1 } }))).toEqual([
|
||||
{ key: 'a', data: { name: 'Ada', address: { city: 'London' } } },
|
||||
])
|
||||
expect(await collect(users.find({ age: 36 }, { projection: { tags: 0, address: 0, email: 0 } }))).toEqual([{ key: 'a', data: { name: 'Ada', age: 36 } }])
|
||||
expect(await users.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: expect.objectContaining({ name: 'Bob' }) })
|
||||
expect(await users.findOne({ email: 'nobody' })).toBeUndefined()
|
||||
})
|
||||
|
||||
it('uses an index: findOne costs one entry scan and one projection read', async () => {
|
||||
const { users, driver } = await seed()
|
||||
driver.reset()
|
||||
await users.findOne({ email: 'di@x' })
|
||||
expect(driver.calls).toEqual({ scan: 1, getMany: 1 })
|
||||
})
|
||||
|
||||
it('warns once per filter shape on full scans', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const { users } = setup({ hide, warnOnFullScan: true })
|
||||
await collect(users.find({ name: 'x' }))
|
||||
await collect(users.find({ name: 'y' }))
|
||||
await collect(users.find({ age: 1 }))
|
||||
expect(warn).toHaveBeenCalledTimes(1)
|
||||
expect(warn.mock.calls[0]![0]).toMatch(/full scan for filter \{"name":"\?"\}/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('unique indexes', () => {
|
||||
it('rejects duplicates and frees values on change and delete', async () => {
|
||||
const { users } = setup({ hide })
|
||||
await users.set('1', { email: 'a@x' })
|
||||
await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError)
|
||||
await expect(users.insert({ email: 'a@x' })).rejects.toMatchObject({ index: 'email', existingKey: '1' })
|
||||
expect(await collect(users.keys())).toEqual(['1'])
|
||||
await users.update('1', { $set: { email: 'b@x' } })
|
||||
await users.set('2', { email: 'a@x' })
|
||||
await users.delete('2')
|
||||
await users.set('3', { email: 'a@x' })
|
||||
await users.set('4', {}) // missing values are not claimed
|
||||
await users.set('5', {})
|
||||
await users.set('3', { email: 'a@x', name: 'same value, same doc' })
|
||||
})
|
||||
|
||||
it('exactly one of concurrent writers claiming the same value wins', async () => {
|
||||
const { users } = setup({ hide })
|
||||
const results = await Promise.allSettled(['1', '2', '3'].map((k) => users.set(k, { email: 'same' })))
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1)
|
||||
expect((await collect(users.find({ email: 'same' }))).length).toBe(1)
|
||||
})
|
||||
|
||||
it('takes over a stale claim only after the grace period', async () => {
|
||||
let t = 0
|
||||
const { users, driver } = setup({ hide, now: () => t, claimGraceMs: 1000 })
|
||||
// Leave an orphaned claim: doc 1's projection and events vanish but its claim remains.
|
||||
await users.set('1', { email: 'a@x' })
|
||||
await driver.inner.deleteMany(['~p/users/1'])
|
||||
await driver.inner.deleteMany((await collect(driver.inner.scan({ gte: 'users/1/', lt: 'users/10' }))).map(([k]) => k))
|
||||
await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError)
|
||||
t = 5000
|
||||
await users.set('2', { email: 'a@x' })
|
||||
expect(await users.get('2')).toEqual({ email: 'a@x' })
|
||||
})
|
||||
|
||||
it('requires setIfAbsent', () => {
|
||||
const kv = new KVStorage(new CountingDriver(new InMemoryKVDriver(), ['setIfAbsent']))
|
||||
expect(() => new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: userIndexes })).toThrow(CapabilityError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('projection lag (crash between append and projection write)', () => {
|
||||
async function lagged() {
|
||||
const ctx = setup({ hide })
|
||||
await ctx.users.set('1', { n: 1, email: 'a@x' })
|
||||
// Simulate a crash: the event is appended but the projection is never written.
|
||||
await ctx.events.child('users').append('1', [{ op: 'patch', set: { n: 2, email: 'b@x' } }])
|
||||
return ctx
|
||||
}
|
||||
|
||||
it('reads see the stale projection; consistent reads catch up', async () => {
|
||||
const { users } = await lagged()
|
||||
expect(await users.get('1')).toEqual({ n: 1, email: 'a@x' })
|
||||
expect(await users.get('1', { consistent: true })).toEqual({ n: 2, email: 'b@x' })
|
||||
expect(await users.findOne({ email: 'b@x' })).toMatchObject({ key: '1' })
|
||||
expect(await users.findOne({ email: 'a@x' })).toBeUndefined()
|
||||
})
|
||||
|
||||
it('the next write self-heals', async () => {
|
||||
const { users } = await lagged()
|
||||
expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3, email: 'b@x' })
|
||||
expect((await collect(users.history('1'))).map((e) => e.version)).toEqual([1, 2, 3])
|
||||
await users.set('2', { email: 'a@x' }) // old claim was released by the catch-up
|
||||
})
|
||||
|
||||
it('catchUp() repairs a subtree', async () => {
|
||||
const { users, docs, events } = await lagged()
|
||||
await events
|
||||
.child('users')
|
||||
.child('9')
|
||||
.child('posts')
|
||||
.append('p', [{ op: 'set', doc: { createdAt: 'x' } }])
|
||||
expect(await docs.catchUp()).toBe(2)
|
||||
expect(await users.get('1')).toEqual({ n: 2, email: 'b@x' })
|
||||
expect(await users.child('9').child('posts').get('p')).toEqual({ createdAt: 'x' })
|
||||
expect(await docs.catchUp()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('concurrency', () => {
|
||||
it('two storage instances incrementing the same doc converge', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
const a = setup({ hide, inner }).users
|
||||
const b = setup({ hide, inner }).users
|
||||
await a.set('1', { n: 0 })
|
||||
await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })]))
|
||||
expect(await a.get('1')).toEqual({ n: 10 })
|
||||
expect(await b.get('1', { consistent: true })).toEqual({ n: 10 })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('costs', () => {
|
||||
it('get is one driver call; update is projection read + append + projection write, even cold', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
await setup({ inner }).users.set('1', { name: 'a', age: 1 })
|
||||
const { users, driver } = setup({ inner }) // fresh instance: cold head cache
|
||||
driver.reset()
|
||||
await users.get('1')
|
||||
expect(driver.calls).toEqual({ getMany: 1 })
|
||||
driver.reset()
|
||||
await users.update('1', { $set: { age: 2 } })
|
||||
expect(driver.calls).toEqual({ getMany: 1, setIfAbsent: 1, batch: 1 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('property: indexed find equals unindexed find', () => {
|
||||
const doc = fc.record(
|
||||
{
|
||||
age: fc.option(fc.integer({ min: 0, max: 5 }), { nil: undefined }),
|
||||
email: fc.constantFrom('a', 'b', 'c', 'd', 'e', 'f'),
|
||||
tags: fc.array(fc.constantFrom('x', 'y', 'z'), { maxLength: 3 }),
|
||||
address: fc.option(fc.record({ city: fc.constantFrom('L', 'P') }), { nil: undefined }),
|
||||
},
|
||||
{ requiredKeys: [] },
|
||||
)
|
||||
const bound = fc.integer({ min: -1, max: 6 })
|
||||
const filter: fc.Arbitrary<Filter> = fc.oneof(
|
||||
fc.record({ age: bound }),
|
||||
fc.record({ age: fc.record({ $gte: bound, $lt: bound }, { requiredKeys: [] }) }),
|
||||
fc.record({ age: fc.record({ $gt: bound, $lte: bound }, { requiredKeys: [] }) }),
|
||||
fc.record({ tags: fc.constantFrom('x', 'y', 'z') }),
|
||||
fc.record({ tags: fc.record({ $gte: fc.constantFrom('x', 'y'), $lt: fc.constantFrom('y', 'zz') }) }),
|
||||
fc.record({ 'address.city': fc.constantFrom('L', 'P'), age: fc.record({ $gt: bound }) }),
|
||||
fc.record({ 'address.city': fc.constantFrom('L', 'P') }),
|
||||
fc.constant({ age: null }),
|
||||
fc.constant({}),
|
||||
)
|
||||
const sort = fc.option(fc.record({ age: fc.constantFrom(1 as const, -1 as const) }), { nil: undefined })
|
||||
|
||||
it('returns the same results in the same order', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(fc.tuple(fc.constantFrom('1', '2', '3', '4', '5'), doc), { maxLength: 12 }),
|
||||
filter,
|
||||
sort,
|
||||
fc.option(fc.integer({ min: 1, max: 4 }), { nil: undefined }),
|
||||
async (writes, f, s, limit) => {
|
||||
const plain = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { warnOnFullScan: false }).child('users')
|
||||
const indexed = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), {
|
||||
warnOnFullScan: false,
|
||||
indexes: { users: userIndexes.users!.map((i) => ({ ...i, unique: false })) },
|
||||
}).child('users')
|
||||
for (const [k, v] of writes) {
|
||||
await plain.set(k, v)
|
||||
await indexed.set(k, v)
|
||||
}
|
||||
const opts = { ...(s ? { sort: s } : {}), ...(limit ? { limit } : {}) }
|
||||
const a = await collect(plain.find(f, opts))
|
||||
const b = await collect(indexed.find(f, opts))
|
||||
if (s) {
|
||||
expect(b).toEqual(a)
|
||||
return
|
||||
}
|
||||
// Unsorted order depends on the plan: compare as sets (a limited result is any subset of that size).
|
||||
const all = new Set((await collect(plain.find(f))).map((r) => r.key))
|
||||
expect(b).toHaveLength(a.length)
|
||||
for (const r of b) expect(all.has(r.key)).toBe(true)
|
||||
},
|
||||
),
|
||||
{ numRuns: 300 },
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createIdGenerator } from '../../src/documents/ulid.js'
|
||||
import { matches } from '../../src/documents/filter.js'
|
||||
import { applyPatch, resolveUpdate } from '../../src/documents/update.js'
|
||||
|
||||
describe('ulid', () => {
|
||||
it('is strictly increasing within one millisecond and sorts by time', () => {
|
||||
let t = 1_700_000_000_000
|
||||
const gen = createIdGenerator(() => t)
|
||||
const ids = Array.from({ length: 1000 }, () => gen())
|
||||
expect([...ids].sort()).toEqual(ids)
|
||||
expect(new Set(ids).size).toBe(1000)
|
||||
t += 1
|
||||
const later = gen()
|
||||
expect(later > ids.at(-1)!).toBe(true)
|
||||
expect(later.slice(0, 10) > ids[0]!.slice(0, 10)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('filter semantics', () => {
|
||||
it('requires one array element to satisfy all positive operators of a condition', () => {
|
||||
expect(matches({ s: [1, 20] }, { s: { $gt: 5, $lt: 10 } })).toBe(false) // MongoDB would match
|
||||
expect(matches({ s: [1, 7] }, { s: { $gt: 5, $lt: 10 } })).toBe(true)
|
||||
expect(matches({ s: [1, 20] }, { $and: [{ s: { $gt: 5 } }, { s: { $lt: 10 } }] })).toBe(true)
|
||||
})
|
||||
|
||||
it('traverses arrays of sub-documents on dot paths', () => {
|
||||
expect(matches({ a: [{ b: 1 }, { b: 2 }] }, { 'a.b': 2 })).toBe(true)
|
||||
expect(matches({ a: [{ b: 1 }] }, { 'a.0.b': 1 })).toBe(true)
|
||||
expect(matches({ a: [{ b: 1 }] }, { 'a.b': { $exists: true } })).toBe(true)
|
||||
expect(matches({ a: [] }, { 'a.b': { $exists: true } })).toBe(false)
|
||||
})
|
||||
|
||||
it('treats null as matching missing', () => {
|
||||
expect(matches({}, { x: null })).toBe(true)
|
||||
expect(matches({ x: null }, { x: { $ne: null } })).toBe(false)
|
||||
expect(matches({ x: 0 }, { x: { $ne: null } })).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects unknown operators', () => {
|
||||
expect(() => matches({}, { x: { $regex: 'a' } })).toThrow(/Unknown operator/)
|
||||
expect(() => matches({}, { $nor: [] } as never)).toThrow(/Unknown top-level/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('update resolution', () => {
|
||||
it('$pull by sub-document filter and by condition', () => {
|
||||
const state = {
|
||||
items: [
|
||||
{ k: 1, v: 'a' },
|
||||
{ k: 2, v: 'b' },
|
||||
],
|
||||
n: [1, 5, 9],
|
||||
}
|
||||
const patch = resolveUpdate(state, { $pull: { items: { k: 2 }, n: { $gte: 5 } } })
|
||||
expect(applyPatch(state, patch)).toEqual({ items: [{ k: 1, v: 'a' }], n: [1] })
|
||||
})
|
||||
|
||||
it('$unset of an array position nulls it', () => {
|
||||
expect(applyPatch({ a: [1, 2] }, resolveUpdate({ a: [1, 2] }, { $unset: { 'a.0': true } }))).toEqual({ a: [null, 2] })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js'
|
||||
import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { collect, KVStorageCounting } from '../helpers.js'
|
||||
|
||||
function setup(kv: BaseKVStorage = new KVStorageCounting(), start = 1000) {
|
||||
let t = start
|
||||
return { kv, events: new EventStorageKV(kv, { now: () => t++ }) }
|
||||
}
|
||||
|
||||
describe('EventStorageKV', () => {
|
||||
it('appends and reads in self and child forms', async () => {
|
||||
const orders = setup().events.child('orders')
|
||||
const [created] = await orders.append('o1', [{ type: 'created' }])
|
||||
const ids = await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])
|
||||
expect(await collect(orders.read('o1'))).toEqual([
|
||||
{ id: created, at: 1000, data: { type: 'created' } },
|
||||
{ id: ids[0], at: 1001, data: { type: 'paid' } },
|
||||
{ id: ids[1], at: 1002, data: { type: 'shipped' } },
|
||||
])
|
||||
expect((await collect(orders.child('o1').read({ after: ids[0] }))).map((e) => e.data)).toEqual([{ type: 'shipped' }])
|
||||
expect((await collect(orders.read('o1', { reverse: true, limit: 2 }))).map((e) => e.id)).toEqual([ids[1], ids[0]])
|
||||
expect((await collect(orders.read('o1', { reverse: true, after: ids[0] }))).map((e) => e.id)).toEqual([created])
|
||||
})
|
||||
|
||||
it('ids are the time and a hash of the time and content', async () => {
|
||||
const { kv, events } = setup()
|
||||
const [id] = await events.child('s').append([{ b: 1, a: 2 }])
|
||||
expect(id).toMatch(/^000000000001000_[0-9a-f]{64}$/)
|
||||
expect(id).toBe(await EventStorageKV.id(1000, { a: 2, b: 1 }))
|
||||
expect((await collect(kv.scan())).map(([k]) => k)).toEqual([`s/~e/${id}`])
|
||||
})
|
||||
|
||||
it('one append is one write; one instance never reuses a time', async () => {
|
||||
const kv = new KVStorageCounting()
|
||||
const events = new EventStorageKV(kv, { now: () => 5 })
|
||||
const ids = await events.child('s').append(['a', 'a'])
|
||||
expect(kv.calls).toEqual({ writeMany: 1 })
|
||||
expect(ids.map((id) => id.slice(0, 15))).toEqual(['000000000000005', '000000000000006'])
|
||||
expect((await collect(events.child('s').read())).map((e) => e.data)).toEqual(['a', 'a'])
|
||||
})
|
||||
|
||||
it('keeps concurrent appends from different writers, ordered by time', async () => {
|
||||
const kv = new KVStorageInMemory()
|
||||
const a = setup(kv, 1000).events.child('s')
|
||||
const b = setup(kv, 1000).events.child('s')
|
||||
await Promise.all([a.append(['a1', 'a2']), b.append(['b1', 'b2'])])
|
||||
const read = await collect(a.read())
|
||||
expect(read).toHaveLength(4)
|
||||
expect(read.map((e) => e.at)).toEqual([1000, 1000, 1001, 1001])
|
||||
expect(read.map((e) => e.id)).toEqual([...read.map((e) => e.id)].sort())
|
||||
expect(await collect(b.read())).toEqual(read)
|
||||
})
|
||||
|
||||
it('writing the same event again does not duplicate it', async () => {
|
||||
const kv = new KVStorageInMemory()
|
||||
await setup(kv).events.child('s').append(['x'])
|
||||
await setup(kv).events.child('s').append(['x'])
|
||||
expect(await collect(new EventStorageKV(kv).child('s').read())).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('checks the arity rules', async () => {
|
||||
const { events } = setup()
|
||||
await expect(events.append(['x'])).rejects.toThrow(/root/)
|
||||
await expect(events.child('o').append(undefined as never, ['x'] as never)).rejects.toThrow(TypeError)
|
||||
})
|
||||
|
||||
it('lists child streams', async () => {
|
||||
const orders = setup().events.child('orders')
|
||||
await orders.append('o1', ['a'])
|
||||
await orders.append('o2', ['b'])
|
||||
await orders.child('o1').append('notes', ['n'])
|
||||
expect(await collect(orders.keys())).toEqual(['o1', 'o2'])
|
||||
expect(await collect(orders.child('o1').keys())).toEqual(['notes'])
|
||||
})
|
||||
|
||||
it('reads any window correctly (property)', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 20 }),
|
||||
fc.record({
|
||||
after: fc.option(fc.nat(60), { nil: undefined }),
|
||||
limit: fc.option(fc.nat(10), { nil: undefined }),
|
||||
reverse: fc.boolean(),
|
||||
}),
|
||||
async (sizes, { after, limit, reverse }) => {
|
||||
const s = setup().events.child('s')
|
||||
const ids: string[] = []
|
||||
for (const size of sizes) ids.push(...(await s.append(Array.from({ length: size }, () => ids.length))))
|
||||
const all = await collect(s.read())
|
||||
expect(all.map((e) => e.id)).toEqual(ids)
|
||||
const ordered = reverse ? [...ids].reverse() : ids
|
||||
const afterId = after === undefined ? undefined : ids[after]
|
||||
let expected = afterId === undefined ? ordered : ordered.slice(ordered.indexOf(afterId) + 1)
|
||||
if (limit !== undefined) expected = expected.slice(0, limit)
|
||||
expect((await collect(s.read({ after: afterId, limit, reverse }))).map((e) => e.id)).toEqual(expected)
|
||||
},
|
||||
),
|
||||
{ numRuns: 200 },
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,200 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { KVStorage } from '../../src/kv/node.js'
|
||||
import { InMemoryKVDriver } from '../../src/kv/memory.js'
|
||||
import { KVEventStorage } from '../../src/events/kv.js'
|
||||
import { ConcurrencyError } from '../../src/errors.js'
|
||||
import { collect, CountingDriver, type Capability } from '../helpers.js'
|
||||
|
||||
function setup(hide: Capability[] = [], inner = new InMemoryKVDriver()) {
|
||||
const driver = new CountingDriver(inner, hide)
|
||||
let t = 1000
|
||||
const events = new KVEventStorage(new KVStorage(driver), { now: () => t++ })
|
||||
return { driver, events }
|
||||
}
|
||||
|
||||
describe('KVEventStorage', () => {
|
||||
it('appends and reads with versions and metadata', async () => {
|
||||
const { events } = setup()
|
||||
const orders = events.child('orders')
|
||||
expect(await orders.append('o1', [{ type: 'created', total: 40 }], { expectedVersion: 0 })).toEqual({ version: 1, commit: 1 })
|
||||
expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3, commit: 2 })
|
||||
expect(await collect(orders.read('o1'))).toEqual([
|
||||
{ version: 1, commit: 1, at: 1000, data: { type: 'created', total: 40 } },
|
||||
{ version: 2, commit: 2, at: 1001, data: { type: 'paid' } },
|
||||
{ version: 3, commit: 2, at: 1001, data: { type: 'shipped' } },
|
||||
])
|
||||
expect(await orders.version('o1')).toBe(3)
|
||||
expect(await orders.child('o1').version()).toBe(3)
|
||||
expect(await orders.version('nope')).toBe(0)
|
||||
})
|
||||
|
||||
it('stores one key per commit under <stream>/~e/<seq:12>', async () => {
|
||||
const { events, driver } = setup()
|
||||
await events.child('orders').append('o1', ['a'])
|
||||
await events.child('orders').append('o1', ['b', 'c'])
|
||||
expect(await collect(driver.inner.scan({ values: true }))).toEqual([
|
||||
['orders/o1/~e/000000000001', { firstVersion: 1, at: 1000, events: ['a'] }],
|
||||
['orders/o1/~e/000000000002', { firstVersion: 2, at: 1001, events: ['b', 'c'] }],
|
||||
])
|
||||
})
|
||||
|
||||
it('enforces expectedVersion', async () => {
|
||||
const { events } = setup()
|
||||
const s = events.child('s')
|
||||
await s.append(['a'], { expectedVersion: 0 })
|
||||
await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError)
|
||||
await expect(s.append(['b'], { expectedVersion: 5 })).rejects.toMatchObject({ expectedVersion: 5, actualVersion: 1 })
|
||||
expect(await s.append(['b'], { expectedVersion: 1 })).toEqual({ version: 2, commit: 2 })
|
||||
await expect(s.append([], { expectedVersion: 1 })).rejects.toBeInstanceOf(ConcurrencyError)
|
||||
expect(await s.append([])).toEqual({ version: 2, commit: 2 })
|
||||
})
|
||||
|
||||
it('applies the arity rules', async () => {
|
||||
const { events } = setup()
|
||||
const orders = events.child('orders')
|
||||
const id: string | undefined = undefined
|
||||
await expect(orders.append(id as never, ['x'] as never)).rejects.toThrow(TypeError)
|
||||
await expect(orders.version(id as never)).rejects.toThrow(TypeError)
|
||||
await expect(collect(orders.read(id as never))).rejects.toThrow(TypeError)
|
||||
await expect(events.append(['x'])).rejects.toThrow(/root/)
|
||||
await expect(orders.append('o1', 'x' as never)).rejects.toThrow(/array/)
|
||||
})
|
||||
|
||||
it('lists child streams and keeps nested streams separate', async () => {
|
||||
const { events } = setup()
|
||||
const orders = events.child('orders')
|
||||
await orders.append('o1', ['a'])
|
||||
await orders.append('o2', ['b'])
|
||||
await orders.child('o1').child('shipments').append('s1', ['ship'])
|
||||
await orders.child('o1').append('notes', ['n'])
|
||||
expect(await collect(orders.keys())).toEqual(['o1', 'o2'])
|
||||
expect(await collect(orders.child('o1').keys())).toEqual(['notes', 'shipments'])
|
||||
expect((await collect(orders.read('o1'))).map((e) => e.data)).toEqual(['a'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('read windows (property)', () => {
|
||||
it('matches brute force for any commit layout', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 25 }),
|
||||
fc.record({
|
||||
from: fc.option(fc.integer({ min: -1, max: 70 }), { nil: undefined }),
|
||||
to: fc.option(fc.integer({ min: 0, max: 70 }), { nil: undefined }),
|
||||
reverse: fc.boolean(),
|
||||
limit: fc.option(fc.integer({ min: 0, max: 10 }), { nil: undefined }),
|
||||
}),
|
||||
async (sizes, opts) => {
|
||||
const { events } = setup()
|
||||
const s = events.child('s')
|
||||
const all: number[] = []
|
||||
for (const n of sizes) {
|
||||
const batch = Array.from({ length: n }, (_, i) => all.length + i + 1)
|
||||
await s.append(batch)
|
||||
all.push(...batch)
|
||||
}
|
||||
let exp = all.filter((v) => v >= Math.max(1, opts.from ?? 1) && v <= (opts.to ?? Infinity))
|
||||
if (opts.reverse) exp.reverse()
|
||||
if (opts.limit !== undefined) exp = exp.slice(0, opts.limit)
|
||||
const got = await collect(s.read(opts))
|
||||
expect(got.map((e) => e.data)).toEqual(exp)
|
||||
expect(got.map((e) => e.version)).toEqual(exp)
|
||||
},
|
||||
),
|
||||
{ numRuns: 300 },
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('costs', () => {
|
||||
it('cached head: a warm append is a single setIfAbsent', async () => {
|
||||
const { events, driver } = setup()
|
||||
const s = events.child('s')
|
||||
await s.append(['a'])
|
||||
driver.reset()
|
||||
await s.append(['b'])
|
||||
await s.append(['c'], { expectedVersion: 2 })
|
||||
expect(driver.calls).toEqual({ setIfAbsent: 2 })
|
||||
})
|
||||
|
||||
it('cold head lookup is one reverse scan with limit 1', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
await setup([], inner).events.child('s').append(['a', 'b'])
|
||||
const { events, driver } = setup([], inner)
|
||||
expect(await events.child('s').append(['c'])).toEqual({ version: 3, commit: 2 })
|
||||
expect(driver.calls).toEqual({ scan: 1, setIfAbsent: 1 })
|
||||
})
|
||||
|
||||
it('read({ from }) finds single-event commits in one probe', async () => {
|
||||
const { events, driver } = setup()
|
||||
const s = events.child('s')
|
||||
for (let i = 0; i < 200; i++) await s.append([i])
|
||||
driver.reset()
|
||||
expect((await collect(s.read({ from: 150, limit: 1 }))).map((e) => e.version)).toEqual([150])
|
||||
expect(driver.calls).toEqual({ getMany: 1, scan: 1 })
|
||||
})
|
||||
|
||||
it('read({ from }) uses O(log commits) probes for multi-event commits', async () => {
|
||||
const { events, driver } = setup()
|
||||
const s = events.child('s')
|
||||
for (let i = 0; i < 1000; i++) await s.append(i % 3 === 0 ? [i, i] : [i])
|
||||
driver.reset()
|
||||
const [first] = await collect(s.read({ from: 777, limit: 1 }))
|
||||
expect(first?.version).toBe(777)
|
||||
// 1 guess probe + ceil(log8(1000)) search rounds, then the read scan. The guess hit an
|
||||
// existing commit, so no head scan was needed.
|
||||
expect(driver.calls.getMany).toBeLessThanOrEqual(1 + 4)
|
||||
expect(driver.calls.scan).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('concurrency', () => {
|
||||
it('a stale writer collides with the existing commit key', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
const a = setup([], inner).events.child('s')
|
||||
const b = setup([], inner).events.child('s')
|
||||
await a.append(['a1'])
|
||||
await b.append(['b1']) // a's cached head is now stale
|
||||
await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 })
|
||||
// 'any' retries after refreshing the head
|
||||
expect(await a.append(['a3'])).toEqual({ version: 3, commit: 3 })
|
||||
expect((await collect(b.read())).map((e) => e.data)).toEqual(['a1', 'b1', 'a3'])
|
||||
})
|
||||
|
||||
it('a stale cached head is refreshed before rejecting expectedVersion', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
const a = setup([], inner).events.child('s')
|
||||
const b = setup([], inner).events.child('s')
|
||||
await a.append(['a1'])
|
||||
await b.append(['b1'])
|
||||
expect(await a.append(['a2'], { expectedVersion: 2 })).toEqual({ version: 3, commit: 3 })
|
||||
})
|
||||
|
||||
it('interleaved writers from two instances produce a dense, non-overlapping log', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
const a = setup([], inner).events.child('s')
|
||||
const b = setup([], inner).events.child('s')
|
||||
await Promise.all([a.append(['a1']), b.append(['b1']), a.append(['a2', 'a3']), b.append(['b2'])])
|
||||
const log = await collect(a.read())
|
||||
expect(log.map((e) => e.version)).toEqual([1, 2, 3, 4, 5])
|
||||
expect(new Set(log.map((e) => e.data))).toEqual(new Set(['a1', 'a2', 'a3', 'b1', 'b2']))
|
||||
})
|
||||
|
||||
it('exactly one of several optimistic writers at the same version wins', async () => {
|
||||
const inner = new InMemoryKVDriver()
|
||||
const writers = [0, 1, 2, 3].map(() => setup([], inner).events.child('s'))
|
||||
const results = await Promise.allSettled(writers.map((w, i) => w.append([i], { expectedVersion: 0 })))
|
||||
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1)
|
||||
expect(results.filter((r) => r.status === 'rejected').every((r) => (r as PromiseRejectedResult).reason instanceof ConcurrencyError)).toBe(true)
|
||||
})
|
||||
|
||||
it('without setIfAbsent, an in-process mutex serialises appends (single-writer mode)', async () => {
|
||||
const { events, driver } = setup(['setIfAbsent'])
|
||||
const s = events.child('s')
|
||||
await Promise.all(Array.from({ length: 20 }, (_, i) => s.append([i])))
|
||||
expect((await collect(s.read())).map((e) => e.version)).toEqual(Array.from({ length: 20 }, (_, i) => i + 1))
|
||||
expect(driver.calls.setIfAbsent).toBeUndefined()
|
||||
await expect(Promise.all([s.append(['x'], { expectedVersion: 20 }), s.append(['y'], { expectedVersion: 20 })])).rejects.toBeInstanceOf(ConcurrencyError)
|
||||
})
|
||||
})
|
||||
+23
-66
@@ -1,83 +1,40 @@
|
||||
import { BaseKVDriver, type BatchOp, type KVDriver, type ScanOptions } from '../src/kv/driver.js'
|
||||
import { InMemoryKVDriver } from '../src/kv/memory.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from '../src/kv-storage/base-kv-storage.js'
|
||||
import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js'
|
||||
|
||||
/** A minimal driver implementing only the four required methods, over a plain Map. */
|
||||
export class MinimalDriver extends BaseKVDriver {
|
||||
readonly map = new Map<string, string>()
|
||||
async getMany(keys: string[]) {
|
||||
return keys.map((k) => (this.map.has(k) ? JSON.parse(this.map.get(k)!) : undefined))
|
||||
}
|
||||
async setMany(entries: [string, unknown][]) {
|
||||
for (const [k, v] of entries) this.map.set(k, JSON.stringify(v))
|
||||
}
|
||||
async deleteMany(keys: string[]) {
|
||||
for (const k of keys) this.map.delete(k)
|
||||
}
|
||||
async *scan({ gte, lt, limit = Infinity, reverse, values }: ScanOptions): AsyncGenerator<[string, unknown]> {
|
||||
let keys = [...this.map.keys()].filter((k) => (gte === undefined || k >= gte) && (lt === undefined || k < lt)).sort()
|
||||
if (reverse) keys.reverse()
|
||||
keys = keys.slice(0, limit)
|
||||
for (const k of keys) {
|
||||
if (!this.map.has(k)) continue
|
||||
yield [k, values ? JSON.parse(this.map.get(k)!) : undefined]
|
||||
}
|
||||
}
|
||||
export async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
|
||||
const out: T[] = []
|
||||
for await (const item of iterable) out.push(item)
|
||||
return out
|
||||
}
|
||||
|
||||
export type Capability = 'setIfAbsent' | 'batch' | 'deleteRange'
|
||||
|
||||
/**
|
||||
* Wraps a driver, counting calls per method and optionally hiding optional capabilities.
|
||||
* `scan` counts each scan started (i.e. each seek).
|
||||
* Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern
|
||||
* (e.g. an encryption layer): extend BaseKVStorage and delegate the three primitives.
|
||||
* Writes are counted as `writeMany`.
|
||||
*/
|
||||
export class CountingDriver implements KVDriver {
|
||||
export class KVStorageCounting extends BaseKVStorage {
|
||||
calls: Record<string, number> = {}
|
||||
setIfAbsent?: KVDriver['setIfAbsent']
|
||||
batch?: KVDriver['batch']
|
||||
deleteRange?: KVDriver['deleteRange']
|
||||
|
||||
constructor(
|
||||
readonly inner: KVDriver = new InMemoryKVDriver(),
|
||||
hide: Capability[] = [],
|
||||
) {
|
||||
if (inner.setIfAbsent && !hide.includes('setIfAbsent')) this.setIfAbsent = (e: [string, unknown][]) => (this.#count('setIfAbsent'), inner.setIfAbsent!(e))
|
||||
if (inner.batch && !hide.includes('batch')) this.batch = (ops: BatchOp[]) => (this.#count('batch'), inner.batch!(ops))
|
||||
if (inner.deleteRange && !hide.includes('deleteRange'))
|
||||
this.deleteRange = (gte: string, lt: string) => (this.#count('deleteRange'), inner.deleteRange!(gte, lt))
|
||||
}
|
||||
|
||||
#count(m: string): void {
|
||||
this.calls[m] = (this.calls[m] ?? 0) + 1
|
||||
constructor(readonly inner: BaseKVStorage = new KVStorageInMemory()) {
|
||||
super()
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.calls = {}
|
||||
}
|
||||
|
||||
get total(): number {
|
||||
return Object.values(this.calls).reduce((a, b) => a + b, 0)
|
||||
readMany(keys: string[]) {
|
||||
return this.count('readMany', () => this.inner.readMany(keys))
|
||||
}
|
||||
protected write(changes: KVChange[]) {
|
||||
return this.count('writeMany', () => this.inner.writeMany(changes))
|
||||
}
|
||||
scan(options?: ScanOptions) {
|
||||
return this.count('scan', () => this.inner.scan(options))
|
||||
}
|
||||
|
||||
getMany(keys: string[]) {
|
||||
this.#count('getMany')
|
||||
return this.inner.getMany(keys)
|
||||
}
|
||||
setMany(entries: [string, unknown][]) {
|
||||
this.#count('setMany')
|
||||
return this.inner.setMany(entries)
|
||||
}
|
||||
deleteMany(keys: string[]) {
|
||||
this.#count('deleteMany')
|
||||
return this.inner.deleteMany(keys)
|
||||
}
|
||||
scan(opts: ScanOptions) {
|
||||
this.#count('scan')
|
||||
return this.inner.scan(opts)
|
||||
private count<T>(name: string, fn: () => T): T {
|
||||
this.calls[name] = (this.calls[name] ?? 0) + 1
|
||||
return fn()
|
||||
}
|
||||
}
|
||||
|
||||
export async function collect<T>(it: AsyncIterable<T>): Promise<T[]> {
|
||||
const out: T[] = []
|
||||
for await (const x of it) out.push(x)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { InMemoryKVDriver } from '../../src/kv/memory.js'
|
||||
import type { KVDriver } from '../../src/kv/driver.js'
|
||||
import { KVStorage } from '../../src/kv/node.js'
|
||||
import { KVEventStorage } from '../../src/events/kv.js'
|
||||
import { EventSourcedDocumentStorage } from '../../src/documents/event-sourced.js'
|
||||
import { createKVHttpHandler } from '../../src/http/handler.js'
|
||||
import { HttpKVDriver } from '../../src/http/client.js'
|
||||
import { runKVDriverConformance } from '../../src/testing/kv-conformance.js'
|
||||
import { collect, CountingDriver, MinimalDriver } from '../helpers.js'
|
||||
import { serve } from './node-server.js'
|
||||
|
||||
/** A client wired straight to a handler, no socket. */
|
||||
function inProcess(server: KVDriver = new InMemoryKVDriver(), caps?: ConstructorParameters<typeof HttpKVDriver>[0]['capabilities']) {
|
||||
const handler = createKVHttpHandler(server, { basePath: '/kv' })
|
||||
return new HttpKVDriver({ url: 'http://test/kv', capabilities: caps, fetch: async (input, init) => handler(new Request(input, init)) })
|
||||
}
|
||||
|
||||
runKVDriverConformance('HttpKVDriver (in-process)', () => inProcess())
|
||||
runKVDriverConformance('HttpKVDriver (minimal server driver, discovered capabilities)', () => {
|
||||
const handler = createKVHttpHandler(new MinimalDriver())
|
||||
return HttpKVDriver.connect({ url: 'http://test', fetch: async (input, init) => handler(new Request(input, init)) })
|
||||
})
|
||||
|
||||
describe('over a real socket', () => {
|
||||
let server: Awaited<ReturnType<typeof serve>>
|
||||
const backends: InMemoryKVDriver[] = []
|
||||
beforeAll(async () => {
|
||||
// One server; each driver gets a fresh backend selected by path.
|
||||
server = await serve(async (req) => {
|
||||
const m = /^\/db(\d+)/.exec(new URL(req.url).pathname)
|
||||
const backend = backends[Number(m?.[1])]
|
||||
if (!backend) return new Response('no db', { status: 404 })
|
||||
return createKVHttpHandler(backend, { basePath: `/db${m![1]}` })(req)
|
||||
})
|
||||
})
|
||||
afterAll(() => server.close())
|
||||
|
||||
const fresh = () => {
|
||||
backends.push(new InMemoryKVDriver())
|
||||
return HttpKVDriver.connect({ url: `${server.url}/db${backends.length - 1}` })
|
||||
}
|
||||
|
||||
runKVDriverConformance('HttpKVDriver (node:http)', fresh)
|
||||
|
||||
it('cancels a server-side scan when the client stops early', async () => {
|
||||
const d = await fresh()
|
||||
await d.setMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number]))
|
||||
const first: string[] = []
|
||||
for await (const [k] of d.scan({})) {
|
||||
first.push(k)
|
||||
if (first.length === 3) break
|
||||
}
|
||||
expect(first).toEqual(['k00000', 'k00001', 'k00002'])
|
||||
})
|
||||
|
||||
it('runs the whole stack (KV indexes, events, documents) over HTTP', async () => {
|
||||
const kv = new KVStorage(await fresh(), { indexes: { users: { byAge: (u: { age: number }) => u.age } } })
|
||||
await kv.child('users').set('1', { age: 30 })
|
||||
expect(await collect(kv.child('users').findBy('byAge', { gte: 18 }))).toEqual([['1', { age: 30 }]])
|
||||
const docs = new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: { people: [{ fields: ['email'], unique: true }] } })
|
||||
const id = await docs.child('people').insert({ email: 'a@x', n: 1 })
|
||||
await docs.child('people').update(id, { $inc: { n: 1 } })
|
||||
await expect(docs.child('people').insert({ email: 'a@x' })).rejects.toThrow(/Unique/)
|
||||
expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('handler', () => {
|
||||
it('reports capabilities and rejects unsupported operations with 501', async () => {
|
||||
const handler = createKVHttpHandler(new MinimalDriver())
|
||||
const caps = await (await handler(new Request('http://x/capabilities'))).json()
|
||||
expect(caps).toEqual({ setIfAbsent: false, batch: false, deleteRange: true })
|
||||
const res = await handler(new Request('http://x/batch', { method: 'POST', body: JSON.stringify({ ops: [] }) }))
|
||||
expect(res.status).toBe(501)
|
||||
})
|
||||
|
||||
it('maps driver errors to 500 and unknown routes to 404', async () => {
|
||||
const broken = new CountingDriver(new InMemoryKVDriver())
|
||||
broken.getMany = () => Promise.reject(new Error('boom'))
|
||||
const handler = createKVHttpHandler(broken)
|
||||
expect((await handler(new Request('http://x/getMany', { method: 'POST', body: '{"keys":["a"]}' }))).status).toBe(500)
|
||||
expect((await handler(new Request('http://x/nope'))).status).toBe(404)
|
||||
await expect(inProcess(new MinimalDriver(), { batch: true }).batch!([])).rejects.toThrow(/HTTP 501/)
|
||||
})
|
||||
|
||||
it('distinguishes null values from missing keys', async () => {
|
||||
const d = inProcess()
|
||||
await d.setMany([['n', null]])
|
||||
expect(await d.getMany(['n', 'missing'])).toEqual([null, undefined])
|
||||
})
|
||||
})
|
||||
@@ -1,36 +0,0 @@
|
||||
import { createServer, type IncomingHttpHeaders } from 'node:http'
|
||||
import type { AddressInfo } from 'node:net'
|
||||
|
||||
/** Serves a `Request → Response` handler over a real socket (test helper). */
|
||||
export async function serve(handler: (req: Request) => Promise<Response>): Promise<{ url: string; close: () => Promise<void> }> {
|
||||
const server = createServer(async (req, res) => {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const c of req) chunks.push(c as Buffer)
|
||||
const headers = Object.entries(req.headers as IncomingHttpHeaders).flatMap(([k, v]) => (v === undefined ? [] : [[k, String(v)] as [string, string]]))
|
||||
const request = new Request(`http://localhost${req.url}`, {
|
||||
method: req.method,
|
||||
headers,
|
||||
body: req.method === 'GET' || req.method === 'HEAD' ? undefined : Buffer.concat(chunks),
|
||||
})
|
||||
const response = await handler(request)
|
||||
res.writeHead(response.status, Object.fromEntries(response.headers))
|
||||
if (!response.body) return void res.end()
|
||||
const reader = response.body.getReader()
|
||||
res.on('close', () => void reader.cancel().catch(() => {}))
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
res.write(value)
|
||||
}
|
||||
} finally {
|
||||
res.end()
|
||||
}
|
||||
})
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const { port } = server.address() as AddressInfo
|
||||
return {
|
||||
url: `http://127.0.0.1:${port}`,
|
||||
close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import type { BaseKVStorage, KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
|
||||
import { collect, KVStorageCounting } from '../helpers.js'
|
||||
import { runKVStorageConformance } from './kv-storage.conformance.js'
|
||||
import { TestNetwork } from './sync-helpers.js'
|
||||
|
||||
runKVStorageConformance(
|
||||
'KVStorageComposite (in-memory source, 1 replica)',
|
||||
() => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [new KVStorageInMemory()] }),
|
||||
)
|
||||
runKVStorageConformance('KVStorageComposite (http source, 2 replicas)', () => {
|
||||
network = new TestNetwork()
|
||||
return new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), new KVStorageInMemory()] })
|
||||
})
|
||||
|
||||
let network: TestNetwork
|
||||
const composites: KVStorageComposite[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(composites.splice(0).map((c) => c.close()))
|
||||
await network?.close()
|
||||
})
|
||||
|
||||
function composite(persistent: BaseKVStorage = new KVStorageInMemory(), queue?: BaseKVStorage) {
|
||||
const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], queue, retryMs: 10 })
|
||||
composites.push(c)
|
||||
return c
|
||||
}
|
||||
|
||||
describe('KVStorageComposite sync', () => {
|
||||
it('loads existing data with one sync, then serves reads locally', async () => {
|
||||
network = new TestNetwork()
|
||||
await network.store.writeMany([
|
||||
{ key: 'users/1', value: { name: 'Ada' } },
|
||||
{ key: 'users/2', value: { name: 'Bob' } },
|
||||
])
|
||||
const db = composite()
|
||||
await db.sync()
|
||||
network.store.reset()
|
||||
expect(await db.child('users').get('1')).toEqual({ name: 'Ada' })
|
||||
expect(await collect(db.child('users').keys())).toEqual(['1', '2'])
|
||||
expect(network.store.calls.readMany).toBeUndefined() // (a follow-up sync started on subscribe may still scan)
|
||||
})
|
||||
|
||||
it('applies changes pushed from other clients to every replica', async () => {
|
||||
network = new TestNetwork()
|
||||
const memory = new KVStorageInMemory()
|
||||
const db = composite(memory)
|
||||
await db.sync()
|
||||
const seen = db.waitFor('change', (e) => e.origin === 'remote')
|
||||
await network.client().child('users').set('9', { name: 'Remote' })
|
||||
expect((await seen).changes).toEqual([{ key: 'users/9', value: { name: 'Remote' } }])
|
||||
expect(await db.child('users').get('9')).toEqual({ name: 'Remote' })
|
||||
expect(await memory.readMany(['users/9'])).toEqual([{ name: 'Remote' }])
|
||||
})
|
||||
|
||||
it('announces only remote changes that differ from local values', async () => {
|
||||
network = new TestNetwork()
|
||||
const db = composite()
|
||||
await db.sync()
|
||||
const events: KVChangeEvent[] = []
|
||||
db.on('change', (e) => events.push(e))
|
||||
await db.set('mine', 1) // relayed back by the server: same value, so no remote event
|
||||
const other = db.waitFor('change', (e) => e.origin === 'remote')
|
||||
await network.client().writeMany([
|
||||
{ key: 'mine', value: 1 },
|
||||
{ key: 'theirs', value: 2 },
|
||||
])
|
||||
await other
|
||||
expect(events).toEqual([
|
||||
{ changes: [{ key: 'mine', value: 1 }], origin: 'local' },
|
||||
{ changes: [{ key: 'theirs', value: 2 }], origin: 'remote' },
|
||||
])
|
||||
})
|
||||
|
||||
it('queues writes while offline and replays them in order when back online', async () => {
|
||||
network = new TestNetwork()
|
||||
const db = composite()
|
||||
await db.sync()
|
||||
network.online = false
|
||||
await db.child('users').set('1', { v: 1 })
|
||||
await db.child('users').set('1', { v: 2 })
|
||||
await db.child('users').delete('1')
|
||||
await db.child('users').set('2', { v: 3 })
|
||||
expect(db.queued).toBe(4)
|
||||
expect(await db.child('users').get('2')).toEqual({ v: 3 }) // served locally
|
||||
network.online = true
|
||||
await vi.waitFor(() => expect(db.queued).toBe(0))
|
||||
expect(await network.store.readMany(['users/1', 'users/2'])).toEqual([undefined, { v: 3 }])
|
||||
})
|
||||
|
||||
it('the last write wins when the server pushes a change while a local write is queued', async () => {
|
||||
network = new TestNetwork()
|
||||
const db = composite()
|
||||
await db.sync()
|
||||
network.online = false
|
||||
await db.set('k', { from: 'local' })
|
||||
await network.store.writeMany([{ key: 'k', value: { from: 'remote' } }]) // pushed over the still-open stream
|
||||
await vi.waitFor(async () => expect(await db.get('k')).toEqual({ from: 'remote' }))
|
||||
network.online = true
|
||||
await vi.waitFor(() => expect(db.queued).toBe(0)) // the queued write is sent last, so it wins
|
||||
expect(await network.store.readMany(['k'])).toEqual([{ from: 'local' }])
|
||||
expect(await db.get('k')).toEqual({ from: 'local' })
|
||||
})
|
||||
|
||||
it('restarts from the persistent replica, then catches up', async () => {
|
||||
network = new TestNetwork()
|
||||
const persistent = new KVStorageInMemory()
|
||||
const queue = new KVStorageInMemory()
|
||||
const first = composite(persistent, queue)
|
||||
await first.set('a', 1)
|
||||
await first.set('gone', 1)
|
||||
await first.sync()
|
||||
await first.close()
|
||||
await network.client().writeMany([{ key: 'b', value: 2 }, { key: 'gone' }]) // missed while closed
|
||||
|
||||
network.online = false
|
||||
const second = composite(persistent, queue)
|
||||
expect(await second.get('a')).toBe(1) // available without the server
|
||||
network.online = true
|
||||
await second.sync()
|
||||
expect(await collect(second.scan({ values: true }))).toEqual([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
])
|
||||
})
|
||||
|
||||
it('replays writes queued by a previous session from a persistent queue', async () => {
|
||||
network = new TestNetwork()
|
||||
const persistent = new KVStorageInMemory()
|
||||
const queue = persistent.child('~sync')
|
||||
const first = new KVStorageComposite({ source: network.client(), replicas: [persistent], queue, prefix: 'data' })
|
||||
composites.push(first)
|
||||
await first.sync()
|
||||
network.online = false
|
||||
await first.child('data').set('offline', 'queued')
|
||||
await first.close()
|
||||
network.online = true
|
||||
const second = new KVStorageComposite({ source: network.client(), replicas: [persistent], queue, prefix: 'data' })
|
||||
composites.push(second)
|
||||
expect(second.queued).toBe(0)
|
||||
await second.sync()
|
||||
expect(second.queued).toBe(0)
|
||||
expect(await network.store.readMany(['data/offline'])).toEqual(['queued'])
|
||||
})
|
||||
|
||||
it('catches up on changes missed while the stream was down', async () => {
|
||||
network = new TestNetwork()
|
||||
const db = composite()
|
||||
await db.sync()
|
||||
const resynced = vi.fn()
|
||||
db.on('change', resynced)
|
||||
network.online = false
|
||||
network.dropStreams()
|
||||
await network.store.writeMany([{ key: 'missed', value: 1 }])
|
||||
network.online = true
|
||||
await vi.waitFor(async () => expect(await db.get('missed')).toBe(1), { timeout: 5000 })
|
||||
expect(resynced).toHaveBeenCalledWith({ changes: [{ key: 'missed', value: 1 }], origin: 'remote' })
|
||||
})
|
||||
|
||||
it('syncs only its prefix and rejects keys outside it', async () => {
|
||||
network = new TestNetwork()
|
||||
await network.store.writeMany([
|
||||
{ key: 'alice/a', value: 1 },
|
||||
{ key: 'bob/b', value: 2 },
|
||||
])
|
||||
const db = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory()], prefix: 'alice' })
|
||||
composites.push(db)
|
||||
await db.sync()
|
||||
expect(await collect(db.scan({ values: true }))).toEqual([['alice/a', 1]])
|
||||
await expect(db.child('bob').get('b')).rejects.toBeInstanceOf(RangeError)
|
||||
})
|
||||
|
||||
it('refuses a queue inside the synced range of a replica', () => {
|
||||
const replica = new KVStorageInMemory()
|
||||
expect(() => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [replica], queue: replica.child('q') })).toThrow(/outside/)
|
||||
expect(() => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [replica], queue: replica.child('q'), prefix: 'data' })).not.toThrow()
|
||||
})
|
||||
|
||||
it('serves document queries from the local replica, offline too', async () => {
|
||||
network = new TestNetwork()
|
||||
const db = composite()
|
||||
const users = new DocumentStorageKV(db).child('users')
|
||||
await users.set('1', { age: 30 })
|
||||
await users.set('2', { age: 40 })
|
||||
await db.sync()
|
||||
network.online = false
|
||||
expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2'])
|
||||
})
|
||||
|
||||
it('does not undo writes made while a sync is scanning', async () => {
|
||||
// A source whose scan reads ahead (like the HTTP stream) and pauses after its first entry.
|
||||
let release!: () => void
|
||||
let reached!: () => void
|
||||
const gate = new Promise<void>((r) => (release = r))
|
||||
const paused = new Promise<void>((r) => (reached = r))
|
||||
const inner = new KVStorageInMemory()
|
||||
await inner.writeMany([
|
||||
{ key: 'a', value: 1 },
|
||||
{ key: 'c', value: 3 },
|
||||
])
|
||||
const source = new KVStorageCounting(inner)
|
||||
const scan = source.scan.bind(source)
|
||||
source.scan = async function* (options) {
|
||||
let first = true
|
||||
for (const entry of await collect(scan(options))) {
|
||||
yield entry
|
||||
if (first && options?.values) ((first = false), reached(), await gate)
|
||||
}
|
||||
}
|
||||
const persistent = new KVStorageInMemory()
|
||||
await persistent.writeMany([{ key: 'a0', value: 'stale' }])
|
||||
const db = new KVStorageComposite({ source, replicas: [persistent] })
|
||||
composites.push(db)
|
||||
void db.start()
|
||||
await paused // the diff has passed 'b' in the source but not in the replica
|
||||
await db.set('b', 2)
|
||||
release()
|
||||
await vi.waitFor(() => expect(db.queued).toBe(0))
|
||||
await db.sync()
|
||||
expect(await collect(db.scan({ values: true }))).toEqual([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
['c', 3],
|
||||
])
|
||||
})
|
||||
|
||||
it('sync() during a running sync runs one more, so changes after its scan are picked up', async () => {
|
||||
let release!: () => void
|
||||
let reached!: () => void
|
||||
const gate = new Promise<void>((r) => (release = r))
|
||||
const paused = new Promise<void>((r) => (reached = r))
|
||||
const inner = new KVStorageInMemory()
|
||||
await inner.writeMany([{ key: 'a', value: 1 }])
|
||||
const source = new KVStorageCounting(inner)
|
||||
const scan = source.scan.bind(source)
|
||||
let scans = 0
|
||||
source.scan = async function* (options) {
|
||||
const entries = await collect(scan(options)) // a snapshot, like a read-ahead stream
|
||||
if (options?.values && scans++ === 0) (reached(), await gate)
|
||||
yield* entries
|
||||
}
|
||||
const db = new KVStorageComposite({ source, replicas: [new KVStorageInMemory()] })
|
||||
composites.push(db)
|
||||
const first = db.sync()
|
||||
await paused
|
||||
await inner.writeMany([{ key: 'b', value: 2 }]) // after the running sync's snapshot
|
||||
const second = db.sync()
|
||||
expect(db.sync()).toBe(second) // later callers share the pending one
|
||||
release()
|
||||
await first
|
||||
await second
|
||||
expect(await collect(db.scan({ values: true }))).toEqual([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
])
|
||||
})
|
||||
|
||||
it('a sync started while a write is on its way to the source does not undo it', async () => {
|
||||
let release!: () => void
|
||||
const gate = new Promise<void>((r) => (release = r))
|
||||
const inner = new KVStorageInMemory()
|
||||
const source = new KVStorageCounting(inner)
|
||||
const writeMany = source.writeMany.bind(source)
|
||||
source.writeMany = async (changes) => {
|
||||
await gate // a slow request
|
||||
return writeMany(changes)
|
||||
}
|
||||
const replica = new KVStorageInMemory()
|
||||
const db = new KVStorageComposite({ source, replicas: [replica] })
|
||||
composites.push(db)
|
||||
await db.sync()
|
||||
const writing = db.set('a', 1)
|
||||
await vi.waitFor(async () => expect(await replica.readMany(['a'])).toEqual([1])) // applied locally, not yet on the source
|
||||
const syncing = db.sync()
|
||||
await new Promise((r) => setTimeout(r, 10))
|
||||
expect(await replica.readMany(['a'])).toEqual([1]) // the sync skips the unconfirmed key
|
||||
release()
|
||||
await Promise.all([writing, syncing])
|
||||
expect(await replica.readMany(['a'])).toEqual([1])
|
||||
expect(await inner.readMany(['a'])).toEqual([1])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
|
||||
import { KVStorageEncrypted } from '../../src/kv-storage/kv-storage-encrypted.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { collect } from '../helpers.js'
|
||||
import { runKVStorageConformance } from './kv-storage.conformance.js'
|
||||
import { TestNetwork } from './sync-helpers.js'
|
||||
|
||||
const key = await KVStorageEncrypted.generateKey()
|
||||
|
||||
runKVStorageConformance('KVStorageEncrypted (in memory)', () => new KVStorageEncrypted(new KVStorageInMemory(), { key }))
|
||||
|
||||
let network: TestNetwork | undefined
|
||||
afterEach(async () => {
|
||||
await network?.close()
|
||||
network = undefined
|
||||
})
|
||||
|
||||
describe('KVStorageEncrypted', () => {
|
||||
it('stores only ciphertext, with readable keys', async () => {
|
||||
const inner = new KVStorageInMemory()
|
||||
await new KVStorageEncrypted(inner, { key }).child('users').set('1', { name: 'Ada', n: 1n })
|
||||
const [[storedKey, stored]] = (await collect(inner.scan({ values: true }))) as [[string, Uint8Array]]
|
||||
expect(storedKey).toBe('users/1')
|
||||
expect(stored).toBeInstanceOf(Uint8Array)
|
||||
expect(new TextDecoder().decode(stored)).not.toContain('Ada')
|
||||
})
|
||||
|
||||
it('uses a fresh IV for every write', async () => {
|
||||
const inner = new KVStorageInMemory()
|
||||
const db = new KVStorageEncrypted(inner, { key })
|
||||
await db.set('a', 'same')
|
||||
await db.set('b', 'same')
|
||||
const [a, b] = (await inner.readMany(['a', 'b'])) as Uint8Array[]
|
||||
expect(a).not.toEqual(b)
|
||||
})
|
||||
|
||||
it('rejects a wrong key, a value moved to another key, tampering and plaintext', async () => {
|
||||
const inner = new KVStorageInMemory()
|
||||
await new KVStorageEncrypted(inner, { key }).set('a', 'secret')
|
||||
const other = new KVStorageEncrypted(inner, { key: await KVStorageEncrypted.generateKey() })
|
||||
await expect(other.get('a')).rejects.toThrow(/Cannot decrypt/)
|
||||
|
||||
const db = new KVStorageEncrypted(inner, { key })
|
||||
const [stored] = (await inner.readMany(['a'])) as [Uint8Array]
|
||||
await inner.writeMany([{ key: 'b', value: stored }])
|
||||
await expect(db.get('b')).rejects.toThrow(/Cannot decrypt/)
|
||||
|
||||
const tampered = stored.slice()
|
||||
tampered[tampered.length - 1]! ^= 1
|
||||
await inner.writeMany([{ key: 'c', value: tampered }])
|
||||
await expect(db.get('c')).rejects.toThrow(/Cannot decrypt/)
|
||||
|
||||
await inner.writeMany([{ key: 'd', value: 'plain' }])
|
||||
await expect(db.get('d')).rejects.toThrow(/not encrypted/)
|
||||
})
|
||||
|
||||
it('round-trips keys through export/import, and derives the same key from the same password and salt', async () => {
|
||||
const inner = new KVStorageInMemory()
|
||||
await new KVStorageEncrypted(inner, { key }).set('a', 1)
|
||||
const imported = await KVStorageEncrypted.importKey(await KVStorageEncrypted.exportKey(key))
|
||||
expect(await new KVStorageEncrypted(inner, { key: imported }).get('a')).toBe(1)
|
||||
|
||||
const salt = crypto.getRandomValues(new Uint8Array(16))
|
||||
await new KVStorageEncrypted(inner, { key: await KVStorageEncrypted.deriveKey('hunter2', salt, 1000) }).set('p', 'x')
|
||||
expect(await new KVStorageEncrypted(inner, { key: await KVStorageEncrypted.deriveKey('hunter2', salt, 1000) }).get('p')).toBe('x')
|
||||
await expect(new KVStorageEncrypted(inner, { key: await KVStorageEncrypted.deriveKey('wrong', salt, 1000) }).get('p')).rejects.toThrow(/Cannot decrypt/)
|
||||
})
|
||||
|
||||
it('decrypts pushed changes and passes subscriptions through', async () => {
|
||||
network = new TestNetwork()
|
||||
const watcher = new KVStorageEncrypted(network.client(), { key })
|
||||
await watcher.child('notes').subscribe()
|
||||
const pushed = watcher.waitFor('change', (e) => e.origin === 'remote')
|
||||
await new KVStorageEncrypted(network.client(), { key }).child('notes').set('1', { text: 'hi' })
|
||||
expect((await pushed).changes).toEqual([{ key: 'notes/1', value: { text: 'hi' } }])
|
||||
})
|
||||
|
||||
it('works as a composite source: the server stores ciphertext, replicas hold plaintext', async () => {
|
||||
network = new TestNetwork()
|
||||
const replica = new KVStorageInMemory()
|
||||
const a = new KVStorageComposite({ source: new KVStorageEncrypted(network.client(), { key }), replicas: [replica] })
|
||||
const b = new KVStorageComposite({ source: new KVStorageEncrypted(network.client(), { key }), replicas: [new KVStorageInMemory()] })
|
||||
await Promise.all([a.sync(), b.sync()])
|
||||
const seen = b.waitFor('change', (e) => e.origin === 'remote')
|
||||
await a.child('notes').set('1', { text: 'hi' })
|
||||
expect((await seen).changes).toEqual([{ key: 'notes/1', value: { text: 'hi' } }])
|
||||
expect(await replica.readMany(['notes/1'])).toEqual([{ text: 'hi' }])
|
||||
expect((await network.store.readMany(['notes/1']))[0]).toBeInstanceOf(Uint8Array)
|
||||
await Promise.all([a.close(), b.close()])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,126 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
|
||||
import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { AuthError } from '../../src/utils/errors.js'
|
||||
import { Ed25519Signer, Ed25519Verifier } from '../../src/utils/request-auth.js'
|
||||
import { collect, KVStorageCounting } from '../helpers.js'
|
||||
import { TestNetwork } from './sync-helpers.js'
|
||||
|
||||
let network: TestNetwork
|
||||
let alice: Ed25519Signer
|
||||
let bob: Ed25519Signer
|
||||
|
||||
beforeEach(async () => {
|
||||
network = new TestNetwork({ auth: { verifiers: [new Ed25519Verifier()] } })
|
||||
alice = await Ed25519Signer.generate()
|
||||
bob = await Ed25519Signer.generate()
|
||||
await network.store.writeMany([
|
||||
{ key: `${alice.publicKey}/profile`, value: { name: 'Alice' } },
|
||||
{ key: `${bob.publicKey}/profile`, value: { name: 'Bob' } },
|
||||
{ key: `${bob.publicKey}/posts/1`, value: { title: 'secret' } },
|
||||
])
|
||||
})
|
||||
afterEach(() => network.close())
|
||||
|
||||
/** Swaps in a server that rejects every signature (as if the user's access was revoked). */
|
||||
function refuseEverySignature() {
|
||||
network.server = new KVStorageHttpServer(network.store, { auth: { verifiers: [{ algorithm: 'ed25519', verify: async () => false }] } })
|
||||
}
|
||||
|
||||
describe('authenticated KVStorageHttpServer', () => {
|
||||
it('lets a user read, write and list their own keys', async () => {
|
||||
const mine = network.client(alice).child(alice.publicKey)
|
||||
expect(await mine.get('profile')).toEqual({ name: 'Alice' })
|
||||
await mine.child('posts').set('1', { title: 'hello' })
|
||||
expect(await collect(mine.keys())).toEqual(['posts', 'profile'])
|
||||
expect(await collect(mine.child('posts').entries())).toEqual([['1', { title: 'hello' }]])
|
||||
})
|
||||
|
||||
it("forbids reading, writing or scanning another user's keys", async () => {
|
||||
const client = network.client(alice)
|
||||
const theirs = client.child(bob.publicKey)
|
||||
await expect(theirs.get('profile')).rejects.toMatchObject({ status: 403 })
|
||||
await expect(theirs.set('profile', { name: 'hacked' })).rejects.toBeInstanceOf(AuthError)
|
||||
await expect(theirs.delete('profile')).rejects.toMatchObject({ status: 403 })
|
||||
await expect(collect(theirs.keys())).rejects.toMatchObject({ status: 403 })
|
||||
await expect(collect(client.keys())).rejects.toMatchObject({ status: 403 }) // root listing
|
||||
await expect(client.readMany([alice.publicKey + 'x/a'])).rejects.toMatchObject({ status: 403 }) // lookalike prefix
|
||||
expect(await network.store.readMany([`${bob.publicKey}/profile`])).toEqual([{ name: 'Bob' }])
|
||||
})
|
||||
|
||||
it('rejects unsigned requests', async () => {
|
||||
await expect(network.client().child(alice.publicKey).get('profile')).rejects.toMatchObject({ status: 401 })
|
||||
})
|
||||
|
||||
it('delivers only subscribed, authorized changes', async () => {
|
||||
const watcher = network.client(alice)
|
||||
const received: KVChangeEvent[] = []
|
||||
watcher.on('change', (e) => e.origin === 'remote' && received.push(e))
|
||||
await watcher.child(alice.publicKey).child('posts').subscribe()
|
||||
const writer = network.client(alice).child(alice.publicKey)
|
||||
await network.client(bob).child(bob.publicKey).child('posts').set('2', { title: 'bob only' })
|
||||
await writer.set('profile', { name: 'Alice 2' }) // not subscribed
|
||||
const arrived = watcher.waitFor('change', () => true)
|
||||
await writer.child('posts').set('1', { title: 'mine' })
|
||||
await arrived
|
||||
expect(received.flatMap((e) => e.changes)).toEqual([{ key: `${alice.publicKey}/posts/1`, value: { title: 'mine' } }])
|
||||
})
|
||||
|
||||
it("refuses subscriptions to another user's keys without retrying forever", async () => {
|
||||
await expect(network.client(alice).child(bob.publicKey).subscribe()).rejects.toMatchObject({ status: 403 })
|
||||
await expect(network.client(alice).subscribe('')).rejects.toMatchObject({ status: 403 })
|
||||
})
|
||||
|
||||
it('a composite scoped to the user syncs only their data', async () => {
|
||||
const db = new KVStorageComposite({ source: network.client(alice), prefix: alice.publicKey, replicas: [new KVStorageInMemory()] })
|
||||
await db.sync()
|
||||
expect(await collect(db.scan({ values: true }))).toEqual([[`${alice.publicKey}/profile`, { name: 'Alice' }]])
|
||||
expect(await collect(db.keys())).toEqual([alice.publicKey])
|
||||
await expect(db.child(bob.publicKey).get('profile')).rejects.toBeInstanceOf(RangeError)
|
||||
const pushed = db.waitFor('change', (e) => e.origin === 'remote')
|
||||
await network.client(bob).child(bob.publicKey).set('ignored', 1)
|
||||
await network.client(alice).child(alice.publicKey).set('fromElsewhere', 1)
|
||||
expect((await pushed).changes).toEqual([{ key: `${alice.publicKey}/fromElsewhere`, value: 1 }])
|
||||
await db.close()
|
||||
})
|
||||
|
||||
it('a composite undoes a local write the server refuses', async () => {
|
||||
// A source without subscriptions, so no follow-up sync is running when the write is made.
|
||||
const db = new KVStorageComposite({ source: new KVStorageCounting(network.client(alice)), prefix: alice.publicKey, replicas: [new KVStorageInMemory()] })
|
||||
await db.child(alice.publicKey).set('x', 'before')
|
||||
await db.sync()
|
||||
refuseEverySignature()
|
||||
await expect(db.child(alice.publicKey).set('x', 'after')).rejects.toMatchObject({ status: 401 })
|
||||
expect(await db.child(alice.publicKey).get('x')).toBe('before')
|
||||
await db.close()
|
||||
})
|
||||
|
||||
it('a composite drops a queued write the server refuses, restores the server value, and reports it', async () => {
|
||||
const errors: Error[] = []
|
||||
const db = new KVStorageComposite({
|
||||
source: network.client(alice),
|
||||
prefix: alice.publicKey,
|
||||
replicas: [new KVStorageInMemory()],
|
||||
onError: (e) => errors.push(e),
|
||||
})
|
||||
const mine = db.child(alice.publicKey)
|
||||
await mine.set('x', 'before')
|
||||
await db.sync()
|
||||
network.online = false
|
||||
await mine.set('x', 'queued')
|
||||
await mine.set('y', 'queued after')
|
||||
expect(db.queued).toBe(2)
|
||||
const server = network.server
|
||||
refuseEverySignature()
|
||||
network.online = true
|
||||
await db.sync()
|
||||
network.server = server
|
||||
expect(db.queued).toBe(0) // dropped, not retried forever
|
||||
expect(errors.filter((e) => e.message === 'A queued write was refused by the source and dropped')).toHaveLength(2) // the sync's own scan is reported too
|
||||
await db.sync()
|
||||
expect(await mine.getMany(['x', 'y'])).toEqual(['before', undefined])
|
||||
await db.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { StorageUnavailableError } from '../../src/utils/errors.js'
|
||||
import { TestNetwork } from './sync-helpers.js'
|
||||
|
||||
let network: TestNetwork
|
||||
afterEach(() => network?.close())
|
||||
|
||||
describe('KVStorageHttp.subscribe (SSE)', () => {
|
||||
it('relays every change under the path as remote change events, its own writes included', async () => {
|
||||
network = new TestNetwork()
|
||||
const watcher = network.client()
|
||||
await watcher.child('users').subscribe()
|
||||
const received: KVChangeEvent[] = []
|
||||
watcher.on('change', (e) => e.origin === 'remote' && received.push(e))
|
||||
const other = watcher.waitFor('change', (e) => e.origin === 'remote')
|
||||
await network.client().child('users').set('1', { name: 'Ada', n: 1n })
|
||||
await other
|
||||
const own = watcher.waitFor('change', (e) => e.origin === 'remote')
|
||||
await network.client().set('elsewhere', 1) // not subscribed
|
||||
await watcher.child('users').delete('1')
|
||||
await own
|
||||
expect(received.map((e) => e.changes)).toEqual([[{ key: 'users/1', value: { name: 'Ada', n: 1n } }], [{ key: 'users/1' }]])
|
||||
})
|
||||
|
||||
it('emits resync when a dropped stream reconnects', async () => {
|
||||
network = new TestNetwork()
|
||||
const watcher = network.client()
|
||||
await watcher.subscribe('')
|
||||
const resync = watcher.waitFor('resync', () => true)
|
||||
network.dropStreams()
|
||||
await resync
|
||||
const after = watcher.waitFor('change', () => true)
|
||||
await network.client().set('a', 1)
|
||||
expect((await after).changes).toEqual([{ key: 'a', value: 1 }])
|
||||
})
|
||||
|
||||
it('stops relaying after unsubscribe', async () => {
|
||||
network = new TestNetwork()
|
||||
const watcher = network.client()
|
||||
await watcher.subscribe('a')
|
||||
await watcher.subscribe('b')
|
||||
await watcher.unsubscribe('a')
|
||||
const received: KVChangeEvent[] = []
|
||||
watcher.on('change', (e) => received.push(e))
|
||||
const arrived = watcher.waitFor('change', () => true)
|
||||
await network.client().set('a', 1)
|
||||
await network.client().set('b', 2)
|
||||
await arrived
|
||||
expect(received.flatMap((e) => e.changes)).toEqual([{ key: 'b', value: 2 }])
|
||||
})
|
||||
|
||||
it('reports an unreachable server as StorageUnavailableError', async () => {
|
||||
network = new TestNetwork()
|
||||
const client = network.client()
|
||||
network.online = false
|
||||
await expect(client.readMany(['x'])).rejects.toBeInstanceOf(StorageUnavailableError)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,67 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js'
|
||||
import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js'
|
||||
import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
|
||||
import { EventSourcedDocumentStorage } from '../../src/document-storage/event-sourced-document-storage.js'
|
||||
import { runKVStorageConformance } from './kv-storage.conformance.js'
|
||||
import { serve } from './node-server.js'
|
||||
|
||||
runKVStorageConformance('KVStorageHttp (in-process)', () => {
|
||||
const server = new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none', basePath: '/kv' })
|
||||
return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) })
|
||||
})
|
||||
|
||||
describe('over a real socket', () => {
|
||||
let server: Awaited<ReturnType<typeof serve>>
|
||||
const stores: KVStorageHttpServer[] = []
|
||||
beforeAll(async () => {
|
||||
// One socket; each test storage gets its own backend under /<n>.
|
||||
server = await serve((request) => stores[Number(new URL(request.url).pathname.split('/')[1])]!.handle(request))
|
||||
})
|
||||
afterAll(() => server.close())
|
||||
const fresh = () => {
|
||||
stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none', basePath: `/${stores.length}` }))
|
||||
return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` })
|
||||
}
|
||||
|
||||
runKVStorageConformance('KVStorageHttp (node:http)', fresh)
|
||||
|
||||
it('stops a long scan early', async () => {
|
||||
const db = fresh()
|
||||
await db.writeMany(Array.from({ length: 5000 }, (_, i) => ({ key: `k${String(i).padStart(5, '0')}`, value: i })))
|
||||
const seen: string[] = []
|
||||
for await (const [key] of db.scan()) if (seen.push(key) === 3) break
|
||||
expect(seen).toEqual(['k00000', 'k00001', 'k00002'])
|
||||
})
|
||||
|
||||
it('runs events and event-sourced documents over HTTP', async () => {
|
||||
const kv = fresh()
|
||||
const docs = new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections')))
|
||||
const id = await docs.child('people').insert({ email: 'a@x', n: 1 })
|
||||
await docs.child('people').update(id, { $inc: { n: 1 } })
|
||||
expect(await docs.child('people').findOne({ email: 'a@x' })).toEqual({ key: id, data: { email: 'a@x', n: 2 } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('server', () => {
|
||||
it('maps errors to 404 / 500', async () => {
|
||||
const server = new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none' })
|
||||
expect((await server.handle(new Request('http://x/nope', { method: 'POST', body: '{}' }))).status).toBe(404)
|
||||
expect((await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys": 5}' }))).status).toBe(500)
|
||||
})
|
||||
|
||||
it('answers CORS preflights and allows only listed origins', async () => {
|
||||
const server = new KVStorageHttpServer(new KVStorageInMemory(), { auth: 'none', allowOrigins: ['https://app.example'] })
|
||||
const preflight = await server.handle(new Request('http://x/write', { method: 'OPTIONS', headers: { origin: 'https://app.example' } }))
|
||||
expect(preflight.status).toBe(204)
|
||||
expect(preflight.headers.get('access-control-allow-origin')).toBe('https://app.example')
|
||||
expect(preflight.headers.get('access-control-allow-headers')).toContain('X-Signature')
|
||||
expect(preflight.headers.get('access-control-allow-headers')).toContain('cache-control') // sent by SSESession
|
||||
const allowed = await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys":[]}', headers: { origin: 'https://app.example' } }))
|
||||
expect(allowed.headers.get('access-control-allow-origin')).toBe('https://app.example')
|
||||
const other = await server.handle(new Request('http://x/readMany', { method: 'POST', body: '{"keys":[]}', headers: { origin: 'https://evil.example' } }))
|
||||
expect(other.headers.get('access-control-allow-origin')).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fc from 'fast-check'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import type { BaseKVStorage, KVChange } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { collect, KVStorageCounting } from '../helpers.js'
|
||||
import { runKVStorageConformance } from './kv-storage.conformance.js'
|
||||
|
||||
runKVStorageConformance('KVStorageInMemory', () => new KVStorageInMemory())
|
||||
runKVStorageConformance('wrapper (KVStorageCounting)', () => new KVStorageCounting())
|
||||
|
||||
describe('costs', () => {
|
||||
it('single-key operations are one primitive call', async () => {
|
||||
const db = new KVStorageCounting()
|
||||
const users = db.child('users')
|
||||
await users.set('1', 'x')
|
||||
await users.get('1')
|
||||
await users.child('1').get()
|
||||
await users.delete('1')
|
||||
expect(db.calls).toEqual({ writeMany: 2, readMany: 2 })
|
||||
})
|
||||
|
||||
it('keys() seeks once per child with descendants, not once per key', async () => {
|
||||
const db = new KVStorageCounting()
|
||||
const writes: KVChange[] = []
|
||||
for (let c = 0; c < 50; c++) for (let d = 0; d < 20; d++) writes.push({ key: `n/c${c}/d${d}`, value: d })
|
||||
for (let c = 0; c < 50; c++) writes.push({ key: `n/leaf${c}`, value: c })
|
||||
await db.writeMany(writes)
|
||||
db.reset()
|
||||
expect(await collect(db.child('n').keys())).toHaveLength(100)
|
||||
expect(db.calls).toEqual({ scan: 51 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('model-based: random writes vs a reference map', () => {
|
||||
const segment = fc.constantFrom('a', 'a-1', 'a.b', 'b', 'x/y', '%', '~e', '1', '10', '9', 'ab', 'a!', 'é')
|
||||
const path = fc.array(segment, { minLength: 1, maxLength: 3 })
|
||||
const op = fc.oneof(
|
||||
fc.record({ t: fc.constant('set' as const), path, value: fc.integer() }),
|
||||
fc.record({ t: fc.constant('delete' as const), path }),
|
||||
fc.record({ t: fc.constant('deleteTree' as const), path }),
|
||||
)
|
||||
const at = (db: BaseKVStorage, p: string[]) => p.reduce<BaseKVStorage>((n, s) => n.child(s), db)
|
||||
|
||||
it('get, keys and entries agree with the model', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(fc.array(op, { maxLength: 30 }), fc.array(segment, { maxLength: 2 }), async (ops, parent) => {
|
||||
const db = new KVStorageInMemory()
|
||||
const model = new Map<string, number>()
|
||||
const isUnder = (p: string[], q: string[]) => q.length >= p.length && p.every((s, i) => q[i] === s)
|
||||
for (const o of ops) {
|
||||
if (o.t === 'set') (await at(db, o.path).set(o.value), model.set(JSON.stringify(o.path), o.value))
|
||||
else if (o.t === 'delete') (await at(db, o.path).delete(), model.delete(JSON.stringify(o.path)))
|
||||
else {
|
||||
await at(db, o.path).deleteTree()
|
||||
for (const k of [...model.keys()]) if (isUnder(o.path, JSON.parse(k))) model.delete(k)
|
||||
}
|
||||
}
|
||||
for (const [k, v] of model) expect(await at(db, JSON.parse(k)).get()).toBe(v)
|
||||
const children = new Map<string, number | undefined>()
|
||||
for (const [k, v] of model) {
|
||||
const p = JSON.parse(k) as string[]
|
||||
if (p.length <= parent.length || !isUnder(parent, p)) continue
|
||||
const own = p.length === parent.length + 1
|
||||
if (own || !children.has(p[parent.length]!)) children.set(p[parent.length]!, own ? v : children.get(p[parent.length]!))
|
||||
}
|
||||
const node = at(db, parent)
|
||||
const listed = await collect(node.keys())
|
||||
expect([...listed].sort()).toEqual([...children.keys()].sort())
|
||||
expect(await collect(node.keys({ reverse: true }))).toEqual([...listed].reverse())
|
||||
const withValues = listed.filter((k) => children.get(k) !== undefined)
|
||||
expect(await collect(node.entries())).toEqual(withValues.map((k) => [k, children.get(k)]))
|
||||
expect(await collect(node.entries({ reverse: true }))).toEqual(withValues.reverse().map((k) => [k, children.get(k)]))
|
||||
for (let i = 0; i < listed.length; i++) expect(await collect(node.keys({ after: listed[i]! }))).toEqual(listed.slice(i + 1))
|
||||
}),
|
||||
{ numRuns: 300 },
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('child nodes', () => {
|
||||
it('are frozen, so implementations cannot accidentally shadow shared state', () => {
|
||||
const child = new KVStorageInMemory().child('a').child('b')
|
||||
expect(Object.isFrozen(child)).toBe(true)
|
||||
expect(() => ((child as unknown as { state: number }).state = 1)).toThrow(TypeError)
|
||||
expect(child.key).toBe('a/b')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,8 @@
|
||||
import 'fake-indexeddb/auto'
|
||||
import { KVStorageIndexedDB } from '../../src/kv-storage/kv-storage-indexeddb.js'
|
||||
import { runKVStorageConformance } from './kv-storage.conformance.js'
|
||||
|
||||
let n = 0
|
||||
runKVStorageConformance('KVStorageIndexedDB', () => new KVStorageIndexedDB({ name: `db-${n++}` }))
|
||||
// Tiny batches exercise resuming scans across transactions.
|
||||
runKVStorageConformance('KVStorageIndexedDB (scan batches of 2)', () => new KVStorageIndexedDB({ name: `db-${n++}`, scanBatchSize: 2 }))
|
||||
@@ -0,0 +1,192 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import type { BaseKVStorage, ScanOptions } from '../../src/kv-storage/base-kv-storage.js'
|
||||
import { InvalidKeyError } from '../../src/utils/errors.js'
|
||||
import { collect } from '../helpers.js'
|
||||
|
||||
/** Behaviour every KV storage must have. `make` returns an empty storage. */
|
||||
export function runKVStorageConformance(name: string, make: () => BaseKVStorage | Promise<BaseKVStorage>): void {
|
||||
describe(`KV storage conformance: ${name}`, () => {
|
||||
let db: BaseKVStorage
|
||||
beforeEach(async () => {
|
||||
db = await make()
|
||||
})
|
||||
// Stores with open connections (e.g. a subscribed composite) release them; browsers allow
|
||||
// only 6 HTTP/1.1 connections per host.
|
||||
afterEach(async () => {
|
||||
await (db as Partial<{ close(): Promise<void> }>).close?.()
|
||||
})
|
||||
const scanKeys = async (options: ScanOptions) => (await collect(db.scan(options))).map(([k]) => k)
|
||||
|
||||
describe('primitives', () => {
|
||||
beforeEach(async () => {
|
||||
await db.writeMany([
|
||||
{ key: 'b', value: 2 },
|
||||
{ key: 'a/2', value: { n: 'a2' } },
|
||||
{ key: 'c', value: [3] },
|
||||
{ key: 'a', value: 1 },
|
||||
{ key: 'a/1', value: 'a1' },
|
||||
{ key: 'a0', value: null },
|
||||
{ key: '10', value: 10 },
|
||||
{ key: '9', value: 9 },
|
||||
])
|
||||
})
|
||||
|
||||
it('reads in request order with undefined for missing keys and null preserved', async () => {
|
||||
expect(await db.readMany(['c', 'missing', 'a/2', 'a0'])).toEqual([[3], undefined, { n: 'a2' }, null])
|
||||
})
|
||||
|
||||
it('scans in key order with bounds, reverse, limit and optional values', async () => {
|
||||
expect(await scanKeys({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c'])
|
||||
expect(await scanKeys({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2'])
|
||||
expect(await scanKeys({ reverse: true, gte: 'a', lt: 'b' })).toEqual(['a0', 'a/2', 'a/1', 'a'])
|
||||
expect(await scanKeys({ limit: 2, reverse: true })).toEqual(['c', 'b'])
|
||||
expect(await scanKeys({ limit: 0 })).toEqual([])
|
||||
expect(await collect(db.scan({ gte: 'a/', lt: 'a0', values: true }))).toEqual([
|
||||
['a/1', 'a1'],
|
||||
['a/2', { n: 'a2' }],
|
||||
])
|
||||
expect((await collect(db.scan({ gte: 'b' })))[0]).toEqual(['b', undefined])
|
||||
})
|
||||
|
||||
it('removes keys', async () => {
|
||||
await db.removeMany(['b', 'missing'])
|
||||
expect(await db.readMany(['b', 'c'])).toEqual([undefined, [3]])
|
||||
})
|
||||
|
||||
it('writes and removes in one batch', async () => {
|
||||
await db.writeMany([{ key: 'b' }, { key: 'new', value: 'y' }, { key: 'a', value: 'x' }])
|
||||
expect(await db.readMany(['a', 'b', 'new'])).toEqual(['x', undefined, 'y'])
|
||||
})
|
||||
|
||||
it('keeps scanning correctly when keys change mid-scan', async () => {
|
||||
// Keys removed mid-scan may or may not still appear (stores may read ahead), but every key
|
||||
// present throughout must appear exactly once, in order.
|
||||
const seen: string[] = []
|
||||
for await (const [key] of db.scan({})) {
|
||||
seen.push(key)
|
||||
if (key === 'a') await db.removeMany(['a', 'a/1'])
|
||||
}
|
||||
expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c'])
|
||||
})
|
||||
|
||||
it('emits change events for writes and removals', async () => {
|
||||
const events: unknown[] = []
|
||||
const off = db.on('change', (event) => events.push(event))
|
||||
await db.writeMany([{ key: 'x', value: 1 }])
|
||||
await db.removeMany(['x'])
|
||||
await db.writeMany([])
|
||||
off()
|
||||
expect(events).toEqual([
|
||||
{ changes: [{ key: 'x', value: 1 }], origin: 'local' },
|
||||
{ changes: [{ key: 'x' }], origin: 'local' },
|
||||
])
|
||||
})
|
||||
|
||||
it('stores bigint and Uint8Array values', async () => {
|
||||
await db.writeMany([{ key: 'bin', value: { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) } }])
|
||||
expect(await db.readMany(['bin'])).toEqual([{ n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }])
|
||||
})
|
||||
|
||||
it('does not share references with callers', async () => {
|
||||
const [value] = (await db.readMany(['a/2'])) as [{ n: string }]
|
||||
value.n = 'changed'
|
||||
expect(await db.readMany(['a/2'])).toEqual([{ n: 'a2' }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('nodes', () => {
|
||||
it('reads and writes own and child values at the same key', async () => {
|
||||
const users = db.child('users')
|
||||
await users.set('123', { name: 'Ada' })
|
||||
expect(await users.child('123').get()).toEqual({ name: 'Ada' })
|
||||
await users.child('123').set({ name: 'Bob' })
|
||||
expect(await users.get('123')).toEqual({ name: 'Bob' })
|
||||
expect(await db.readMany(['users/123'])).toEqual([{ name: 'Bob' }])
|
||||
await db.child('n').set(5, 'five')
|
||||
expect(await db.child('n').get('5')).toBe('five')
|
||||
})
|
||||
|
||||
it('batch reads, writes and deletes children', async () => {
|
||||
const n = db.child('n')
|
||||
await n.setMany([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
])
|
||||
expect(await n.getMany(['b', 'x', 'a'])).toEqual([2, undefined, 1])
|
||||
await n.deleteMany(['a'])
|
||||
expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2])
|
||||
})
|
||||
|
||||
it('sets and gets direct children as fields', async () => {
|
||||
const user = db.child('users').child('1')
|
||||
await user.setFields({ name: 'Ada', hobby: 'chess', skipped: undefined })
|
||||
await user.child('address').set('city', 'Paris')
|
||||
expect(await user.get('hobby')).toBe('chess')
|
||||
expect(await user.getFields()).toEqual({ hobby: 'chess', name: 'Ada' })
|
||||
})
|
||||
|
||||
it('keeps own values and children independent', async () => {
|
||||
const user = db.child('users').child('1')
|
||||
await user.set('own')
|
||||
await user.set('name', 'Ada')
|
||||
await user.delete()
|
||||
expect(await user.get()).toBeUndefined()
|
||||
expect(await user.get('name')).toBe('Ada')
|
||||
})
|
||||
|
||||
it('rejects passed-but-undefined keys, root values and undefined values', async () => {
|
||||
const users = db.child('users')
|
||||
const id: string | undefined = undefined
|
||||
await expect(users.get(id as never)).rejects.toThrow(InvalidKeyError)
|
||||
await expect(users.delete(null as never)).rejects.toThrow(TypeError)
|
||||
await expect(users.set(id as never, 1 as never)).rejects.toThrow(TypeError)
|
||||
await expect(users.set('x', undefined)).rejects.toThrow(/undefined/)
|
||||
await expect(db.get()).rejects.toThrow(/root/)
|
||||
expect(() => db.child('')).toThrow(InvalidKeyError)
|
||||
})
|
||||
|
||||
it('escapes keys so special characters round-trip', async () => {
|
||||
const n = db.child('a/b')
|
||||
for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) await n.set(key, key)
|
||||
for (const key of ['x/y', '50%', '~e', 'a-1', 'a.b', 'a b', 'é']) expect(await n.get(key)).toBe(key)
|
||||
expect((await collect(n.keys())).sort()).toEqual(['50%', 'a b', 'a-1', 'a.b', 'x/y', '~e', 'é'].sort())
|
||||
})
|
||||
|
||||
it('lists direct children (including ones with only descendants), skipping reserved keys', async () => {
|
||||
const n = db.child('n')
|
||||
await n.set('a', 1)
|
||||
await n.child('b').set('deep', 1)
|
||||
await n.set('c', 3)
|
||||
await n.child('c').child('x').set('y', 1)
|
||||
await n.set('a-1', 4)
|
||||
await db.writeMany([{ key: 'n/~e/000000000001', value: {} }])
|
||||
expect(await collect(n.keys())).toEqual(['a', 'a-1', 'b', 'c'])
|
||||
expect(await collect(n.keys({ reverse: true }))).toEqual(['c', 'b', 'a-1', 'a'])
|
||||
expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['a-1', 'b'])
|
||||
expect(await collect(n.keys({ after: 'c', reverse: true }))).toEqual(['b', 'a-1', 'a'])
|
||||
expect(await collect(n.entries())).toEqual([
|
||||
['a', 1],
|
||||
['a-1', 4],
|
||||
['c', 3],
|
||||
])
|
||||
expect(await collect(n.entries({ reverse: true, limit: 2 }))).toEqual([
|
||||
['c', 3],
|
||||
['a-1', 4],
|
||||
])
|
||||
expect(await collect(db.keys())).toEqual(['n'])
|
||||
})
|
||||
|
||||
it('deleteTree removes a subtree but not lookalike siblings', async () => {
|
||||
const n = db.child('n')
|
||||
await n.set('a', 1)
|
||||
await n.child('a').child('b').set('c', 3)
|
||||
await n.set('a!', 2)
|
||||
await n.set('a0', 3)
|
||||
await n.deleteTree('a')
|
||||
expect(await collect(n.keys())).toEqual(['a0', 'a!']) // 'a!' is stored as 'a~21'
|
||||
await db.deleteTree()
|
||||
expect(await collect(db.scan())).toEqual([])
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { createServer } from 'node:http'
|
||||
import type { AddressInfo } from 'node:net'
|
||||
|
||||
/** Serves a `Request → Response` handler over a real socket (test helper). */
|
||||
export async function serve(handle: (request: Request) => Promise<Response>): Promise<{ url: string; close: () => Promise<void> }> {
|
||||
const server = createServer(async (req, res) => {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of req) chunks.push(chunk as Buffer)
|
||||
const response = await handle(
|
||||
new Request(`http://localhost${req.url}`, {
|
||||
method: req.method,
|
||||
headers: Object.entries(req.headers).flatMap(([name, value]) => (typeof value === 'string' ? [[name, value] as [string, string]] : [])),
|
||||
body: req.method === 'POST' ? Buffer.concat(chunks) : undefined,
|
||||
}),
|
||||
)
|
||||
res.writeHead(response.status, Object.fromEntries(response.headers))
|
||||
if (!response.body) return void res.end()
|
||||
const reader = response.body.getReader()
|
||||
res.on('close', () => void reader.cancel().catch(() => {}))
|
||||
for (let next = await reader.read(); !next.done; next = await reader.read()) res.write(next.value)
|
||||
res.end()
|
||||
})
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
return {
|
||||
url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
|
||||
close: () => new Promise((resolve) => (server.closeAllConnections(), server.close(() => resolve()))),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js'
|
||||
import { KVStorageHttpServer, type KVStorageHttpServerOptions } from '../../src/kv-storage/kv-storage-http-server.js'
|
||||
import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
|
||||
import type { RequestSigner } from '../../src/utils/request-auth.js'
|
||||
import { KVStorageCounting } from '../helpers.js'
|
||||
|
||||
/**
|
||||
* An in-process server with a network switch. While `online` is false, new requests fail like a
|
||||
* dropped connection (open SSE streams stay up, so pushes can still be simulated).
|
||||
* `dropStreams()` ends every open SSE stream, as a proxy or server restart would.
|
||||
*/
|
||||
export class TestNetwork {
|
||||
online = true
|
||||
readonly store = new KVStorageCounting(new KVStorageInMemory())
|
||||
server: KVStorageHttpServer
|
||||
private readonly clients: KVStorageHttp[] = []
|
||||
private readonly streams = new Set<() => void>()
|
||||
|
||||
constructor(options: Partial<KVStorageHttpServerOptions> = {}) {
|
||||
this.server = new KVStorageHttpServer(this.store, { auth: 'none', ...options })
|
||||
}
|
||||
|
||||
client(signer?: RequestSigner): KVStorageHttp {
|
||||
const client = new KVStorageHttp({
|
||||
url: 'http://server',
|
||||
signer,
|
||||
fetch: async (url, init) => {
|
||||
if (!this.online) throw new TypeError('fetch failed')
|
||||
const response = await this.server.handle(new Request(url, init))
|
||||
return url.endsWith('/events') && response.body ? this.droppable(response) : response
|
||||
},
|
||||
})
|
||||
this.clients.push(client)
|
||||
return client
|
||||
}
|
||||
|
||||
dropStreams(): void {
|
||||
for (const drop of this.streams) drop()
|
||||
this.streams.clear()
|
||||
}
|
||||
|
||||
private droppable(response: Response): Response {
|
||||
const reader = response.body!.getReader()
|
||||
let drop!: () => void
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start: (controller) => {
|
||||
drop = () => {
|
||||
controller.close()
|
||||
void reader.cancel()
|
||||
}
|
||||
this.streams.add(drop)
|
||||
},
|
||||
pull: async (controller) => {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) controller.close()
|
||||
else controller.enqueue(value)
|
||||
},
|
||||
cancel: async () => {
|
||||
this.streams.delete(drop)
|
||||
await reader.cancel()
|
||||
},
|
||||
})
|
||||
return new Response(body, { status: response.status, headers: response.headers })
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await Promise.all(this.clients.map((c) => c.close()))
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user