M2: KV driver interface, BaseKVDriver, InMemoryKVDriver, conformance suite

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE

src/index.ts

@@ -1,3 +1,4 @@
 export * from './errors.js'
 export * from './core/path.js'
 export * from './core/encoding.js'
+export * from './kv/index.js'

src/kv/driver.ts

@@ -0,0 +1,77 @@
+export interface ScanOptions {
+  gte?: string
+  lt?: string
+  limit?: number
+  reverse?: boolean
+  /** When false/omitted, yielded values are `undefined`. */
+  values?: boolean
+}
+
+export type BatchOp = { type: 'set'; key: string; value: unknown } | { type: 'delete'; key: string }
+
+/**
+ * A flat, string-keyed backend. Values are JSON-serialisable; byte-oriented backends serialise
+ * internally with a {@link Codec}. All path, index and typing logic lives above this interface.
+ */
+export interface KVDriver {
+  /** Same order as `keys`; `undefined` for missing keys. */
+  getMany(keys: string[]): Promise<(unknown | undefined)[]>
+  setMany(entries: [key: string, value: unknown][]): Promise<void>
+  deleteMany(keys: string[]): Promise<void>
+  /** MUST yield in key order (descending when `reverse`). `gte` is inclusive, `lt` exclusive. */
+  scan(opts: ScanOptions): AsyncIterable<[key: string, value: unknown]>
+
+  // Optional capabilities. Stores feature-detect them and fall back where possible.
+  /** Writes each entry only if its key is absent. `true` = written. */
+  setIfAbsent?(entries: [string, unknown][]): Promise<boolean[]>
+  /** Applies all ops atomically. */
+  batch?(ops: BatchOp[]): Promise<void>
+  deleteRange?(gte: string, lt: string): Promise<void>
+}
+
+/** Serialises values for byte/string-oriented backends. */
+export interface Codec {
+  encode(value: unknown): string
+  decode(encoded: string): unknown
+}
+
+export const JsonCodec: Codec = {
+  encode: (value) => JSON.stringify(value),
+  decode: (encoded) => JSON.parse(encoded),
+}
+
+/**
+ * Base class for drivers: implement the four required methods and get `deleteRange`
+ * (scan + deleteMany in chunks) and single-key helpers for free.
+ */
+export abstract class BaseKVDriver implements KVDriver {
+  abstract getMany(keys: string[]): Promise<(unknown | undefined)[]>
+  abstract setMany(entries: [string, unknown][]): Promise<void>
+  abstract deleteMany(keys: string[]): Promise<void>
+  abstract scan(opts: ScanOptions): AsyncIterable<[string, unknown]>
+
+  /** Chunk size used by the default `deleteRange`. */
+  protected deleteRangeChunk = 1000
+
+  async deleteRange(gte: string, lt: string): Promise<void> {
+    for (;;) {
+      const keys: string[] = []
+      for await (const [key] of this.scan({ gte, lt, limit: this.deleteRangeChunk })) keys.push(key)
+      if (keys.length === 0) return
+      await this.deleteMany(keys)
+      if (keys.length < this.deleteRangeChunk) return
+    }
+  }
+
+  async get(key: string): Promise<unknown | undefined> {
+    return (await this.getMany([key]))[0]
+  }
+
+  set(key: string, value: unknown): Promise<void> {
+    return this.setMany([[key, value]])
+  }
+
+  delete(key: string): Promise<void> {
+    return this.deleteMany([key])
+  }
+}

src/kv/index.ts

@@ -0,0 +1,3 @@
+export * from './driver.js'
+export * from './memory.js'
+export * from './sorted.js'

src/kv/memory.ts

@@ -0,0 +1,104 @@
+import { BaseKVDriver, type BatchOp, type ScanOptions } from './driver.js'
+import { SortedArrayMap, type SortedMap } from './sorted.js'
+
+export interface InMemoryKVDriverOptions {
+  /**
+   * Copy values on write and read so callers can't mutate stored state through a shared
+   * reference (matching serialising drivers). Default `true`. Disable for speed when values
+   * are treated as immutable.
+   */
+  clone?: boolean
+}
+
+/**
+ * In-memory driver over a sorted map. Implements every optional capability:
+ * `setIfAbsent`, atomic `batch` and `deleteRange`.
+ */
+export class InMemoryKVDriver extends BaseKVDriver {
+  readonly #data: SortedMap<unknown> = new SortedArrayMap()
+  readonly #clone: (v: unknown) => unknown
+
+  constructor(opts: InMemoryKVDriverOptions = {}) {
+    super()
+    this.#clone = opts.clone === false ? identity : cloneValue
+  }
+
+  get size(): number {
+    return this.#data.size
+  }
+
+  async getMany(keys: string[]): Promise<unknown[]> {
+    const out = new Array<unknown>(keys.length)
+    for (let i = 0; i < keys.length; i++) out[i] = this.#clone(this.#data.get(keys[i]!))
+    return out
+  }
+
+  async setMany(entries: [string, unknown][]): Promise<void> {
+    const clone = this.#clone
+    this.#data.setMany(entries.map(([k, v]) => [k, clone(v)]))
+  }
+
+  async deleteMany(keys: string[]): Promise<void> {
+    for (const key of keys) this.#data.delete(key)
+  }
+
+  async setIfAbsent(entries: [string, unknown][]): Promise<boolean[]> {
+    return entries.map(([key, value]) => {
+      if (this.#data.has(key)) return false
+      this.#data.set(key, this.#clone(value))
+      return true
+    })
+  }
+
+  async batch(ops: BatchOp[]): Promise<void> {
+    // Applied synchronously within one microtask, so no other operation can observe a partial batch.
+    for (const op of ops) {
+      if (op.type === 'set') this.#data.set(op.key, this.#clone(op.value))
+      else this.#data.delete(op.key)
+    }
+  }
+
+  override async deleteRange(gte: string, lt: string): Promise<void> {
+    const data = this.#data
+    const start = data.lowerBound(gte)
+    const doomed: string[] = []
+    for (let i = start; ; i++) {
+      const key = data.keyAt(i)
+      if (key === undefined || key >= lt) break
+      doomed.push(key)
+    }
+    for (const key of doomed) data.delete(key)
+  }
+
+  async *scan(opts: ScanOptions): AsyncGenerator<[string, unknown]> {
+    const data = this.#data
+    const { gte, lt, reverse = false, values = false } = opts
+    let remaining = opts.limit ?? Infinity
+    if (remaining <= 0) return
+    // `last` is the last key yielded; if the map changes while we're suspended, re-seek from it.
+    let last: string | undefined
+    let version = data.version
+    let i = reverse ? (lt === undefined ? data.size : data.lowerBound(lt)) - 1 : gte === undefined ? 0 : data.lowerBound(gte)
+    for (;;) {
+      if (data.version !== version) {
+        version = data.version
+        if (last !== undefined) i = reverse ? data.lowerBound(last) - 1 : data.lowerBound(last, true)
+      }
+      const key = data.keyAt(i)
+      if (key === undefined || i < 0) return
+      if (reverse ? gte !== undefined && key < gte : lt !== undefined && key >= lt) return
+      last = key
+      i += reverse ? -1 : 1
+      yield [key, values ? this.#clone(data.get(key)) : undefined]
+      if (--remaining <= 0) return
+    }
+  }
+}
+
+function identity(v: unknown): unknown {
+  return v
+}
+
+function cloneValue(v: unknown): unknown {
+  return v === null || typeof v !== 'object' ? v : structuredClone(v)
+}

src/kv/sorted.ts

@@ -0,0 +1,101 @@
+/**
+ * Ordered string-keyed map used by the in-memory driver. Kept behind an interface so the
+ * sorted-array implementation can be swapped for a B-tree without touching the driver.
+ */
+export interface SortedMap<V> {
+  readonly size: number
+  /** Incremented on every structural change (insert or delete), for iterator revalidation. */
+  readonly version: number
+  get(key: string): V | undefined
+  has(key: string): boolean
+  set(key: string, value: V): void
+  /** Inserts many entries; may be faster than repeated `set`. */
+  setMany(entries: Iterable<[string, V]>): void
+  delete(key: string): boolean
+  /** Index of the first key `>= key` (or `> key` when `exclusive`). */
+  lowerBound(key: string, exclusive?: boolean): number
+  keyAt(index: number): string | undefined
+}
+
+export class SortedArrayMap<V> implements SortedMap<V> {
+  #keys: string[] = []
+  #values = new Map<string, V>()
+  #version = 0
+
+  get size(): number {
+    return this.#keys.length
+  }
+
+  get version(): number {
+    return this.#version
+  }
+
+  get(key: string): V | undefined {
+    return this.#values.get(key)
+  }
+
+  has(key: string): boolean {
+    return this.#values.has(key)
+  }
+
+  set(key: string, value: V): void {
+    if (!this.#values.has(key)) {
+      this.#keys.splice(this.lowerBound(key), 0, key)
+      this.#version++
+    }
+    this.#values.set(key, value)
+  }
+
+  setMany(entries: Iterable<[string, V]>): void {
+    const fresh: string[] = []
+    for (const [key, value] of entries) {
+      if (!this.#values.has(key)) fresh.push(key)
+      this.#values.set(key, value)
+    }
+    if (fresh.length === 0) return
+    this.#version++
+    if (fresh.length < 32) {
+      for (const key of fresh) this.#keys.splice(this.lowerBound(key), 0, key)
+      return
+    }
+    // Bulk path: sort the new keys and merge, O(n + k log k) instead of O(n·k).
+    fresh.sort(cmp)
+    const old = this.#keys
+    const merged = new Array<string>(old.length + fresh.length)
+    let i = 0
+    let j = 0
+    let k = 0
+    while (i < old.length && j < fresh.length) merged[k++] = old[i]! < fresh[j]! ? old[i++]! : fresh[j++]!
+    while (i < old.length) merged[k++] = old[i++]!
+    while (j < fresh.length) merged[k++] = fresh[j++]!
+    this.#keys = merged
+  }
+
+  delete(key: string): boolean {
+    if (!this.#values.delete(key)) return false
+    this.#keys.splice(this.lowerBound(key), 1)
+    this.#version++
+    return true
+  }
+
+  lowerBound(key: string, exclusive = false): number {
+    const keys = this.#keys
+    let lo = 0
+    let hi = keys.length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      const k = keys[mid]!
+      if (k < key || (exclusive && k === key)) lo = mid + 1
+      else hi = mid
+    }
+    return lo
+  }
+
+  keyAt(index: number): string | undefined {
+    return this.#keys[index]
+  }
+}
+
+function cmp(a: string, b: string): number {
+  return a < b ? -1 : a > b ? 1 : 0
+}

src/testing/index.ts

@@ -0,0 +1 @@
+export * from './kv-conformance.js'

src/testing/kv-conformance.ts

@@ -0,0 +1,171 @@
+import { beforeEach, describe, expect, it } from 'vitest'
+import type { KVDriver, ScanOptions } from '../kv/driver.js'
+
+/**
+ * Reusable conformance suite for {@link KVDriver} implementations. Call it from a Vitest test file:
+ *
+ * ```ts
+ * runKVDriverConformance('MyDriver', () => new MyDriver())
+ * ```
+ *
+ * `makeDriver` must return an empty driver each time.
+ */
+export function runKVDriverConformance(name: string, makeDriver: () => KVDriver | Promise<KVDriver>): void {
+  describe(`KVDriver conformance: ${name}`, () => {
+    let d: KVDriver
+
+    beforeEach(async () => {
+      d = await makeDriver()
+    })
+
+    async function collect(opts: ScanOptions): Promise<[string, unknown][]> {
+      const out: [string, unknown][] = []
+      for await (const e of d.scan(opts)) out.push(e)
+      return out
+    }
+
+    const keysOf = async (opts: ScanOptions) => (await collect(opts)).map(([k]) => k)
+
+    async function seed(): Promise<void> {
+      // Deliberately inserted out of order.
+      await d.setMany([
+        ['b', 2],
+        ['a/2', { n: 'a2' }],
+        ['c', [3]],
+        ['a', 1],
+        ['a/1', 'a1'],
+        ['a0', null],
+        ['10', 10],
+        ['9', 9],
+      ])
+    }
+
+    it('round-trips JSON values with getMany in key order, undefined for missing', async () => {
+      await seed()
+      expect(await d.getMany(['c', 'missing', 'a/2', 'a0', 'a'])).toEqual([[3], undefined, { n: 'a2' }, null, 1])
+      expect(await d.getMany([])).toEqual([])
+    })
+
+    it('overwrites on setMany and deletes with deleteMany', async () => {
+      await seed()
+      await d.setMany([['a', 'one']])
+      await d.deleteMany(['b', 'missing'])
+      expect(await d.getMany(['a', 'b'])).toEqual(['one', undefined])
+    })
+
+    it('scans all keys in lexicographic order', async () => {
+      await seed()
+      expect(await keysOf({})).toEqual(['10', '9', 'a', 'a/1', 'a/2', 'a0', 'b', 'c'])
+    })
+
+    it('honours gte (inclusive) and lt (exclusive)', async () => {
+      await seed()
+      expect(await keysOf({ gte: 'a/', lt: 'a0' })).toEqual(['a/1', 'a/2'])
+      expect(await keysOf({ gte: 'a', lt: 'b' })).toEqual(['a', 'a/1', 'a/2', 'a0'])
+      expect(await keysOf({ gte: 'a0' })).toEqual(['a0', 'b', 'c'])
+      expect(await keysOf({ lt: 'a' })).toEqual(['10', '9'])
+      expect(await keysOf({ gte: 'zz' })).toEqual([])
+      expect(await keysOf({ gte: 'b', lt: 'b' })).toEqual([])
+    })
+
+    it('scans in reverse within bounds', async () => {
+      await seed()
+      expect(await keysOf({ reverse: true })).toEqual(['c', 'b', 'a0', 'a/2', 'a/1', 'a', '9', '10'])
+      expect(await keysOf({ reverse: true, gte: 'a', lt: 'a0' })).toEqual(['a/2', 'a/1', 'a'])
+    })
+
+    it('honours limit, forwards and in reverse', async () => {
+      await seed()
+      expect(await keysOf({ limit: 2 })).toEqual(['10', '9'])
+      expect(await keysOf({ limit: 2, reverse: true, lt: 'b' })).toEqual(['a0', 'a/2'])
+      expect(await keysOf({ limit: 0 })).toEqual([])
+    })
+
+    it('yields values only when values: true', async () => {
+      await seed()
+      expect(await collect({ gte: 'a/', lt: 'a0' })).toEqual([
+        ['a/1', undefined],
+        ['a/2', undefined],
+      ])
+      expect(await collect({ gte: 'a/', lt: 'a0', values: true })).toEqual([
+        ['a/1', 'a1'],
+        ['a/2', { n: 'a2' }],
+      ])
+    })
+
+    it('tolerates writes while a scan is suspended', async () => {
+      await d.setMany([
+        ['k1', 1],
+        ['k2', 2],
+        ['k3', 3],
+      ])
+      const seen: string[] = []
+      for await (const [key] of d.scan({})) {
+        seen.push(key)
+        if (key === 'k1') await d.deleteMany(['k1'])
+      }
+      expect(seen).toEqual(['k1', 'k2', 'k3'])
+    })
+
+    it('does not let callers mutate stored values through returned references', async () => {
+      await d.setMany([['o', { n: 1 }]])
+      const [v] = (await d.getMany(['o'])) as [{ n: number }]
+      v.n = 2
+      expect(await d.getMany(['o'])).toEqual([{ n: 1 }])
+    })
+
+    it('stores unicode keys and orders them by code unit', async () => {
+      await d.setMany([
+        ['é', 1],
+        ['z', 2],
+        ['日本', 3],
+      ])
+      expect(await keysOf({})).toEqual(['z', 'é', '日本'])
+    })
+
+    describe('optional: setIfAbsent', () => {
+      it('writes only absent keys and reports which were written', async (ctx) => {
+        if (!d.setIfAbsent) return ctx.skip()
+        await d.setMany([['x', 1]])
+        expect(
+          await d.setIfAbsent([
+            ['x', 2],
+            ['y', 3],
+          ]),
+        ).toEqual([false, true])
+        expect(await d.getMany(['x', 'y'])).toEqual([1, 3])
+      })
+
+      it('only the first of concurrent claims wins', async (ctx) => {
+        if (!d.setIfAbsent) return ctx.skip()
+        const results = await Promise.all([1, 2, 3, 4].map((n) => d.setIfAbsent!([['claim', n]])))
+        expect(results.flat().filter(Boolean)).toHaveLength(1)
+      })
+    })
+
+    describe('optional: batch', () => {
+      it('applies sets and deletes in order', async (ctx) => {
+        if (!d.batch) return ctx.skip()
+        await d.setMany([['gone', 1]])
+        await d.batch([
+          { type: 'set', key: 'p', value: 1 },
+          { type: 'delete', key: 'gone' },
+          { type: 'set', key: 'q', value: { a: [1] } },
+          { type: 'set', key: 'p', value: 2 },
+        ])
+        expect(await d.getMany(['p', 'q', 'gone'])).toEqual([2, { a: [1] }, undefined])
+      })
+    })
+
+    describe('optional: deleteRange', () => {
+      it('deletes exactly [gte, lt)', async (ctx) => {
+        if (!d.deleteRange) return ctx.skip()
+        await seed()
+        await d.deleteRange('a/', 'a0')
+        expect(await keysOf({})).toEqual(['10', '9', 'a', 'a0', 'b', 'c'])
+        await d.deleteRange('a', 'b')
+        expect(await keysOf({})).toEqual(['10', '9', 'b', 'c'])
+      })
+    })
+  })
+}