M2: KV driver interface, BaseKVDriver, InMemoryKVDriver, conformance suite

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE
This commit is contained in:
Claude
2026-10-23 06:53:54 +00:00
parent 69108d1557
commit f1ef4a71e9
11 changed files with 599 additions and 0 deletions
+17
View File
@@ -380,6 +380,23 @@ Decisions taken where the brief was silent. Each is the simplest option consiste
- **Ordering** of strings is by UTF-16 code unit (JS `<`). Drivers that compare UTF-8 bytes agree
except for astral-plane characters vs. U+E000–U+FFFF; noted as a driver-author caveat.
### M2 — KV driver
- **Conformance suite location**: `src/testing/kv-conformance.ts`, exported as `potential-storage/testing`.
It imports `vitest`, which is an *optional peer dependency*; the core entry point never imports it.
- **`InMemoryKVDriver` clones values** on write and read (`structuredClone`, skipped for primitives) so
stored state can't be mutated through shared references, matching serialising drivers. Opt out
with `{ clone: false }`.
- **Scans tolerate concurrent writes**: the in-memory scan remembers the last key yielded and re-seeks
if the map changed while suspended. Read-repair and `deleteTree` rely on this. The conformance suite
requires it of every driver (deleting an already-yielded key mid-scan must not skip keys).
- **`setMany` bulk path**: ≥32 new keys are sorted and merged (O(n + k log k)); fewer are spliced.
- **`BaseKVDriver.deleteRange`** deletes in chunks of 1000 (scan with `limit`, then `deleteMany`,
repeat), so it never deletes from under its own live iterator. It also provides `get`/`set`/`delete`
single-key helpers.
- **No fake `batch`**: `BaseKVDriver` deliberately does not provide a non-atomic `batch`, since stores
use its presence to mean "atomic".
<!-- decisions appended per milestone -->
## 13. Benchmark baselines
+12
View File
@@ -28,6 +28,10 @@
"./http": {
"types": "./dist/http/index.d.ts",
"import": "./dist/http/index.js"
},
"./testing": {
"types": "./dist/testing/index.d.ts",
"import": "./dist/testing/index.js"
}
},
"files": [
@@ -45,5 +49,13 @@
"fast-check": "^4.10.2",
"typescript": "^5.9.3",
"vitest": "^3.2.7"
},
"peerDependencies": {
"vitest": ">=3"
},
"peerDependenciesMeta": {
"vitest": {
"optional": true
}
}
}
+1
View File
@@ -1,3 +1,4 @@
export * from './errors.js'
export * from './core/path.js'
export * from './core/encoding.js'
export * from './kv/index.js'
+77
View File
@@ -0,0 +1,77 @@
export interface ScanOptions {
gte?: string
lt?: string
limit?: number
reverse?: boolean
/** When false/omitted, yielded values are `undefined`. */
values?: boolean
}
export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string }
/**
* A flat, string-keyed backend. Values are JSON-serialisable; byte-oriented backends serialise
* internally with a {@link Codec}. All path, index and typing logic lives above this interface.
*/
export interface KVDriver {
/** Same order as `keys`; `undefined` for missing keys. */
getMany(keys: string[]): Promise<(unknown | undefined)[]>
setMany(entries: [key: string, value: unknown][]): Promise<void>
deleteMany(keys: string[]): Promise<void>
/** MUST yield in key order (descending when `reverse`). `gte` is inclusive, `lt` exclusive. */
scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]>
// Optional capabilities. Stores feature-detect them and fall back where possible.
/** Writes each entry only if its key is absent. `true` = written. */
setIfAbsent?(entries: [string, unknown][]): Promise<boolean[]>
/** Applies all ops atomically. */
batch?(ops: BatchOp[]): Promise<void>
deleteRange?(gte: string, lt: string): Promise<void>
}
/** Serialises values for byte/string-oriented backends. */
export interface Codec {
encode(value: unknown): string
decode(encoded: string): unknown
}
export const JsonCodec: Codec = {
encode: (value) => JSON.stringify(value),
decode: (encoded) => JSON.parse(encoded),
}
/**
* Base class for drivers: implement the four required methods and get `deleteRange`
* (scan + deleteMany in chunks) and single-key helpers for free.
*/
export abstract class BaseKVDriver implements KVDriver {
abstract getMany(keys: string[]): Promise<(unknown | undefined)[]>
abstract setMany(entries: [string, unknown][]): Promise<void>
abstract deleteMany(keys: string[]): Promise<void>
abstract scan(opts: ScanOptions): AsyncIterable<[string, unknown]>
/** Chunk size used by the default `deleteRange`. */
protected deleteRangeChunk = 1000
async deleteRange(gte: string, lt: string): Promise<void> {
for (;;) {
const keys: string[] = []
for await (const [key] of this.scan({ gte, lt, limit: this.deleteRangeChunk })) keys.push(key)
if (keys.length === 0) return
await this.deleteMany(keys)
if (keys.length < this.deleteRangeChunk) return
}
}
async get(key: string): Promise<unknown | undefined> {
return (await this.getMany([key]))[0]
}
set(key: string, value: unknown): Promise<void> {
return this.setMany([[key, value]])
}
delete(key: string): Promise<void> {
return this.deleteMany([key])
}
}
+3
View File
@@ -0,0 +1,3 @@
export * from './driver.js'
export * from './memory.js'
export * from './sorted.js'
+104
View File
@@ -0,0 +1,104 @@
import { BaseKVDriver, type BatchOp, type ScanOptions } from './driver.js'
import { SortedArrayMap, type SortedMap } from './sorted.js'
export interface InMemoryKVDriverOptions {
/**
* Copy values on write and read so callers can't mutate stored state through a shared
* reference (matching serialising drivers). Default `true`. Disable for speed when values
* are treated as immutable.
*/
clone?: boolean
}
/**
* In-memory driver over a sorted map. Implements every optional capability:
* `setIfAbsent`, atomic `batch` and `deleteRange`.
*/
export class InMemoryKVDriver extends BaseKVDriver {
readonly #data: SortedMap<unknown> = new SortedArrayMap()
readonly #clone: (v: unknown) => unknown
constructor(opts: InMemoryKVDriverOptions = {}) {
super()
this.#clone = opts.clone === false ? identity : cloneValue
}
get size(): number {
return this.#data.size
}
async getMany(keys: string[]): Promise<unknown[]> {
const out = new Array<unknown>(keys.length)
for (let i = 0; i < keys.length; i++) out[i] = this.#clone(this.#data.get(keys[i]!))
return out
}
async setMany(entries: [string, unknown][]): Promise<void> {
const clone = this.#clone
this.#data.setMany(entries.map(([k, v]) => [k, clone(v)]))
}
async deleteMany(keys: string[]): Promise<void> {
for (const key of keys) this.#data.delete(key)
}
async setIfAbsent(entries: [string, unknown][]): Promise<boolean[]> {
return entries.map(([key, value]) => {
if (this.#data.has(key)) return false
this.#data.set(key, this.#clone(value))
return true
})
}
async batch(ops: BatchOp[]): Promise<void> {
// Applied synchronously within one microtask, so no other operation can observe a partial batch.
for (const op of ops) {
if (op.type === 'set') this.#data.set(op.key, this.#clone(op.value))
else this.#data.delete(op.key)
}
}
override async deleteRange(gte: string, lt: string): Promise<void> {
const data = this.#data
const start = data.lowerBound(gte)
const doomed: string[] = []
for (let i = start; ; i++) {
const key = data.keyAt(i)
if (key === undefined || key >= lt) break
doomed.push(key)
}
for (const key of doomed) data.delete(key)
}
async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> {
const data = this.#data
const { gte, lt, reverse = false, values = false } = opts
let remaining = opts.limit ?? Infinity
if (remaining <= 0) return
// `last` is the last key yielded; if the map changes while we're suspended, re-seek from it.
let last: string | undefined
let version = data.version
let i = reverse ? (lt === undefined ? data.size : data.lowerBound(lt)) - 1 : gte === undefined ? 0 : data.lowerBound(gte)
for (;;) {
if (data.version !== version) {
version = data.version
if (last !== undefined) i = reverse ? data.lowerBound(last) - 1 : data.lowerBound(last, true)
}
const key = data.keyAt(i)
if (key === undefined || i < 0) return
if (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt) return
last = key
i += reverse ? -1 : 1
yield [key, values ? this.#clone(data.get(key)) : undefined]
if (--remaining <= 0) return
}
}
}
function identity(v: unknown): unknown {
return v
}
function cloneValue(v: unknown): unknown {
return v === null || typeof v !== 'object' ? v : structuredClone(v)
}
+101
View File
@@ -0,0 +1,101 @@
/**
* Ordered string-keyed map used by the in-memory driver. Kept behind an interface so the
* sorted-array implementation can be swapped for a B-tree without touching the driver.
*/
export interface SortedMap<V> {
readonly size: number
/** Incremented on every structural change (insert or delete), for iterator revalidation. */
readonly version: number
get(key: string): V | undefined
has(key: string): boolean
set(key: string, value: V): void
/** Inserts many entries; may be faster than repeated `set`. */
setMany(entries: Iterable<[string, V]>): void
delete(key: string): boolean
/** Index of the first key `>= key` (or `> key` when `exclusive`). */
lowerBound(key: string, exclusive?: boolean): number
keyAt(index: number): string | undefined
}
export class SortedArrayMap<V> implements SortedMap<V> {
#keys: string[] = []
#values = new Map<string, V>()
#version = 0
get size(): number {
return this.#keys.length
}
get version(): number {
return this.#version
}
get(key: string): V | undefined {
return this.#values.get(key)
}
has(key: string): boolean {
return this.#values.has(key)
}
set(key: string, value: V): void {
if (!this.#values.has(key)) {
this.#keys.splice(this.lowerBound(key), 0, key)
this.#version++
}
this.#values.set(key, value)
}
setMany(entries: Iterable<[string, V]>): void {
const fresh: string[] = []
for (const [key, value] of entries) {
if (!this.#values.has(key)) fresh.push(key)
this.#values.set(key, value)
}
if (fresh.length === 0) return
this.#version++
if (fresh.length < 32) {
for (const key of fresh) this.#keys.splice(this.lowerBound(key), 0, key)
return
}
// Bulk path: sort the new keys and merge, O(n + k log k) instead of O(n·k).
fresh.sort(cmp)
const old = this.#keys
const merged = new Array<string>(old.length + fresh.length)
let i = 0
let j = 0
let k = 0
while (i < old.length && j < fresh.length) merged[k++] = old[i]! < fresh[j]! ? old[i++]! : fresh[j++]!
while (i < old.length) merged[k++] = old[i++]!
while (j < fresh.length) merged[k++] = fresh[j++]!
this.#keys = merged
}
delete(key: string): boolean {
if (!this.#values.delete(key)) return false
this.#keys.splice(this.lowerBound(key), 1)
this.#version++
return true
}
lowerBound(key: string, exclusive = false): number {
const keys = this.#keys
let lo = 0
let hi = keys.length
while (lo < hi) {
const mid = (lo + hi) >>> 1
const k = keys[mid]!
if (k < key || (exclusive && k === key)) lo = mid + 1
else hi = mid
}
return lo
}
keyAt(index: number): string | undefined {
return this.#keys[index]
}
}
function cmp(a: string, b: string): number {
return a < b ? -1 : a > b ? 1 : 0
}
+1
View File
@@ -0,0 +1 @@
export * from './kv-conformance.js'
+171
View File
@@ -0,0 +1,171 @@
import { beforeEach, describe, expect, it } from 'vitest'
import type { KVDriver, ScanOptions } from '../kv/driver.js'
/**
* Reusable conformance suite for {@link KVDriver} implementations. Call it from a Vitest test file:
*
* ```ts
* runKVDriverConformance('MyDriver', () => new MyDriver())
* ```
*
* `makeDriver` must return an empty driver each time.
*/
export function runKVDriverConformance(name: string, makeDriver: () => KVDriver | Promise<KVDriver>): void {
describe(`KVDriver conformance: ${name}`, () => {
let d: KVDriver
beforeEach(async () => {
d = await makeDriver()
})
async function collect(opts: ScanOptions): Promise<[string, unknown][]> {
const out: [string, unknown][] = []
for await (const e of d.scan(opts)) out.push(e)
return out
}
const keysOf = async (opts: ScanOptions) => (await collect(opts)).map(([k]) => k)
async function seed(): Promise<void> {
// Deliberately inserted out of order.
await d.setMany([
['b', 2],
['a/2', { n: 'a2' }],
['c', [3]],
['a', 1],
['a/1', 'a1'],
['a0', null],
['10', 10],
['9', 9],
])
}
it('round-trips JSON values with getMany in key order, undefined for missing', async () => {
await seed()
expect(await d.getMany(['c', 'missing', 'a/2', 'a0', 'a'])).toEqual([[3], undefined, { n: 'a2' }, null, 1])
expect(await d.getMany([])).toEqual([])
})
it('overwrites on setMany and deletes with deleteMany', async () => {
await seed()
await d.setMany([['a', 'one']])
await d.deleteMany(['b', 'missing'])
expect(await d.getMany(['a', 'b'])).toEqual(['one', undefined])
})
it('scans all keys in lexicographic order', async () => {
await seed()
expect(await keysOf({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c'])
})
it('honours gte (inclusive) and lt (exclusive)', async () => {
await seed()
expect(await keysOf({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2'])
expect(await keysOf({ gte: 'a', lt: 'b' })).toEqual(['a', 'a/1', 'a/2', 'a0'])
expect(await keysOf({ gte: 'a0' })).toEqual(['a0', 'b', 'c'])
expect(await keysOf({ lt: 'a' })).toEqual(['10', '9'])
expect(await keysOf({ gte: 'zz' })).toEqual([])
expect(await keysOf({ gte: 'b', lt: 'b' })).toEqual([])
})
it('scans in reverse within bounds', async () => {
await seed()
expect(await keysOf({ reverse: true })).toEqual(['c', 'b', 'a0', 'a/2', 'a/1', 'a', '9', '10'])
expect(await keysOf({ reverse: true, gte: 'a', lt: 'a0' })).toEqual(['a/2', 'a/1', 'a'])
})
it('honours limit, forwards and in reverse', async () => {
await seed()
expect(await keysOf({ limit: 2 })).toEqual(['10', '9'])
expect(await keysOf({ limit: 2, reverse: true, lt: 'b' })).toEqual(['a0', 'a/2'])
expect(await keysOf({ limit: 0 })).toEqual([])
})
it('yields values only when values: true', async () => {
await seed()
expect(await collect({ gte: 'a/', lt: 'a0' })).toEqual([
['a/1', undefined],
['a/2', undefined],
])
expect(await collect({ gte: 'a/', lt: 'a0', values: true })).toEqual([
['a/1', 'a1'],
['a/2', { n: 'a2' }],
])
})
it('tolerates writes while a scan is suspended', async () => {
await d.setMany([
['k1', 1],
['k2', 2],
['k3', 3],
])
const seen: string[] = []
for await (const [key] of d.scan({})) {
seen.push(key)
if (key === 'k1') await d.deleteMany(['k1'])
}
expect(seen).toEqual(['k1', 'k2', 'k3'])
})
it('does not let callers mutate stored values through returned references', async () => {
await d.setMany([['o', { n: 1 }]])
const [v] = (await d.getMany(['o'])) as [{ n: number }]
v.n = 2
expect(await d.getMany(['o'])).toEqual([{ n: 1 }])
})
it('stores unicode keys and orders them by code unit', async () => {
await d.setMany([
['é', 1],
['z', 2],
['日本', 3],
])
expect(await keysOf({})).toEqual(['z', 'é', '日本'])
})
describe('optional: setIfAbsent', () => {
it('writes only absent keys and reports which were written', async (ctx) => {
if (!d.setIfAbsent) return ctx.skip()
await d.setMany([['x', 1]])
expect(
await d.setIfAbsent([
['x', 2],
['y', 3],
]),
).toEqual([false, true])
expect(await d.getMany(['x', 'y'])).toEqual([1, 3])
})
it('only the first of concurrent claims wins', async (ctx) => {
if (!d.setIfAbsent) return ctx.skip()
const results = await Promise.all([1, 2, 3, 4].map((n) => d.setIfAbsent!([['claim', n]])))
expect(results.flat().filter(Boolean)).toHaveLength(1)
})
})
describe('optional: batch', () => {
it('applies sets and deletes in order', async (ctx) => {
if (!d.batch) return ctx.skip()
await d.setMany([['gone', 1]])
await d.batch([
{ type: 'set', key: 'p', value: 1 },
{ type: 'delete', key: 'gone' },
{ type: 'set', key: 'q', value: { a: [1] } },
{ type: 'set', key: 'p', value: 2 },
])
expect(await d.getMany(['p', 'q', 'gone'])).toEqual([2, { a: [1] }, undefined])
})
})
describe('optional: deleteRange', () => {
it('deletes exactly [gte, lt)', async (ctx) => {
if (!d.deleteRange) return ctx.skip()
await seed()
await d.deleteRange('a/', 'a0')
expect(await keysOf({})).toEqual(['10', '9', 'a', 'a0', 'b', 'c'])
await d.deleteRange('a', 'b')
expect(await keysOf({})).toEqual(['10', '9', 'b', 'c'])
})
})
})
}
+84
View File
@@ -0,0 +1,84 @@
import { BaseKVDriver, type BatchOp, type KVDriver, type ScanOptions } from '../src/kv/driver.js'
import { InMemoryKVDriver } from '../src/kv/memory.js'
/** A minimal driver implementing only the four required methods, over a plain Map. */
export class MinimalDriver extends BaseKVDriver {
readonly map = new Map<string, string>()
async getMany(keys: string[]) {
return keys.map((k) => (this.map.has(k) ? JSON.parse(this.map.get(k)!) : undefined))
}
async setMany(entries: [string, unknown][]) {
for (const [k, v] of entries) this.map.set(k, JSON.stringify(v))
}
async deleteMany(keys: string[]) {
for (const k of keys) this.map.delete(k)
}
async *scan({ gte, lt, limit = Infinity, reverse, values }: ScanOptions): AsyncGenerator<[string, unknown]> {
let keys = [...this.map.keys()].filter((k) => (gte === undefined || k >= gte) && (lt === undefined || k < lt)).sort()
if (reverse) keys.reverse()
keys = keys.slice(0, limit)
for (const k of keys) {
if (!this.map.has(k)) continue
yield [k, values ? JSON.parse(this.map.get(k)!) : undefined]
}
}
}
export type Capability = 'setIfAbsent' | 'batch' | 'deleteRange'
/**
* Wraps a driver, counting calls per method and optionally hiding optional capabilities.
* `scan` counts each scan started (i.e. each seek).
*/
export class CountingDriver implements KVDriver {
calls: Record<string, number> = {}
setIfAbsent?: KVDriver['setIfAbsent']
batch?: KVDriver['batch']
deleteRange?: KVDriver['deleteRange']
constructor(
readonly inner: KVDriver = new InMemoryKVDriver(),
hide: Capability[] = [],
) {
if (inner.setIfAbsent && !hide.includes('setIfAbsent'))
this.setIfAbsent = (e: [string, unknown][]) => (this.#count('setIfAbsent'), inner.setIfAbsent!(e))
if (inner.batch && !hide.includes('batch')) this.batch = (ops: BatchOp[]) => (this.#count('batch'), inner.batch!(ops))
if (inner.deleteRange && !hide.includes('deleteRange'))
this.deleteRange = (gte: string, lt: string) => (this.#count('deleteRange'), inner.deleteRange!(gte, lt))
}
#count(m: string): void {
this.calls[m] = (this.calls[m] ?? 0) + 1
}
reset(): void {
this.calls = {}
}
get total(): number {
return Object.values(this.calls).reduce((a, b) => a + b, 0)
}
getMany(keys: string[]) {
this.#count('getMany')
return this.inner.getMany(keys)
}
setMany(entries: [string, unknown][]) {
this.#count('setMany')
return this.inner.setMany(entries)
}
deleteMany(keys: string[]) {
this.#count('deleteMany')
return this.inner.deleteMany(keys)
}
scan(opts: ScanOptions) {
this.#count('scan')
return this.inner.scan(opts)
}
}
export async function collect<T>(it: AsyncIterable<T>): Promise<T[]> {
const out: T[] = []
for await (const x of it) out.push(x)
return out
}
+28
View File
@@ -0,0 +1,28 @@
import { describe, expect, it } from 'vitest'
import { InMemoryKVDriver } from '../../src/kv/memory.js'
import { runKVDriverConformance } from '../../src/testing/kv-conformance.js'
import { CountingDriver, MinimalDriver } from '../helpers.js'
runKVDriverConformance('InMemoryKVDriver', () => new InMemoryKVDriver())
runKVDriverConformance('BaseKVDriver (minimal)', () => new MinimalDriver())
runKVDriverConformance('CountingDriver(InMemory)', () => new CountingDriver())
describe('InMemoryKVDriver', () => {
it('keeps order after large bulk merges', async () => {
const d = new InMemoryKVDriver()
const keys = Array.from({ length: 5000 }, (_, i) => `k${(i * 7919) % 5000}`)
await d.setMany(keys.slice(0, 2500).map((k) => [k, 1]))
await d.setMany(keys.slice(2500).map((k) => [k, 2]))
const seen: string[] = []
for await (const [k] of d.scan({})) seen.push(k)
expect(seen).toEqual([...new Set(keys)].sort())
expect(d.size).toBe(5000)
})
it('can skip cloning', async () => {
const d = new InMemoryKVDriver({ clone: false })
const v = { n: 1 }
await d.setMany([['k', v]])
expect((await d.getMany(['k']))[0]).toBe(v)
})
})