Add KVStorageComposite, IndexedDB storage and SSE change streams

- Every KV storage emits change events (base class owns writeMany/removeMany/
  writeIfAbsent; implementations provide protected apply* methods)
- KVStorageHttpServer streams changes over SSE (GET /events) with a resumable
  history; KVStorageHttp.watch() subscribes and resumes via Last-Event-ID
- KVStorageIndexedDB for local persistence
- KVStorageComposite keeps replicas of a source in sync: local reads, resume
  instead of full sync, offline outbox replayed in order, echo tracking
- Values may contain bigint/Uint8Array (Extended JSON over the network)
- Vendor sse-session, event-emitter and extended-json (no runtime deps)
- Freeze child nodes so shadowed state on children fails loudly
- CI on Node 22, 24 and 26

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE

test/helpers.ts

@@ -10,6 +10,7 @@ export async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
 /**
  * Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern
  * (e.g. an encryption layer): extend BaseKVStorage and delegate the five primitives.
+ * Counts are reported under the public method names (writeMany, removeMany, writeIfAbsent).
  */
 export class KVStorageCounting extends BaseKVStorage {
   calls: Record<string, number> = {}
@@ -25,13 +26,13 @@ export class KVStorageCounting extends BaseKVStorage {
   readMany(keys: string[]) {
     return this.count('readMany', () => this.inner.readMany(keys))
   }
-  writeMany(entries: [string, unknown][]) {
+  protected applyWrite(entries: [string, unknown][]) {
     return this.count('writeMany', () => this.inner.writeMany(entries))
   }
-  removeMany(keys: string[]) {
+  protected applyRemove(keys: string[]) {
     return this.count('removeMany', () => this.inner.removeMany(keys))
   }
-  writeIfAbsent(entries: [string, unknown][]) {
+  protected applyWriteIfAbsent(entries: [string, unknown][]) {
     return this.count('writeIfAbsent', () => this.inner.writeIfAbsent(entries))
   }
   scan(options?: ScanOptions) {

test/kv-storage/kv-storage-composite.test.ts

@@ -0,0 +1,144 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
+import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
+import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
+import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
+import { collect } from '../helpers.js'
+import { runKVStorageConformance } from './kv-storage.conformance.js'
+import { TestNetwork } from './sync-helpers.js'
+
+runKVStorageConformance(
+  'KVStorageComposite (in-memory source, 1 replica)',
+  () => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [new KVStorageInMemory()] }),
+)
+runKVStorageConformance('KVStorageComposite (http source, 2 replicas)', () => {
+  network = new TestNetwork()
+  return new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), new KVStorageInMemory()] })
+})
+
+let network: TestNetwork
+const composites: KVStorageComposite[] = []
+afterEach(async () => {
+  await Promise.all(composites.splice(0).map((c) => c.close()))
+  await network?.close()
+})
+
+function composite(persistent: BaseKVStorage = new KVStorageInMemory()) {
+  const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], retry: { baseDelay: 5, maxDelay: 20 } })
+  composites.push(c)
+  return c
+}
+
+describe('KVStorageComposite sync', () => {
+  it('loads existing data with one full sync, then serves reads locally', async () => {
+    network = new TestNetwork()
+    await network.store.writeMany([
+      ['users/1', { name: 'Ada' }],
+      ['users/2', { name: 'Bob' }],
+    ])
+    const db = composite()
+    await db.synced()
+    network.store.reset()
+    expect(await db.child('users').get('1')).toEqual({ name: 'Ada' })
+    expect(await collect(db.child('users').keys())).toEqual(['1', '2'])
+    expect(network.store.calls).toEqual({})
+  })
+
+  it('applies changes pushed from other clients to every replica', async () => {
+    network = new TestNetwork()
+    const memory = new KVStorageInMemory()
+    const db = composite(memory)
+    await db.synced()
+    const seen = db.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().child('users').set('9', { name: 'Remote' })
+    expect((await seen).changes).toEqual([{ key: 'users/9', value: { name: 'Remote' } }])
+    expect(await db.child('users').get('9')).toEqual({ name: 'Remote' })
+    expect(await memory.readMany(['users/9'])).toEqual([{ name: 'Remote' }])
+  })
+
+  it('queues writes while offline and replays them in order when back online', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.synced()
+    network.online = false
+    await db.child('users').set('1', { v: 1 })
+    await db.child('users').set('1', { v: 2 })
+    await db.child('users').delete('1')
+    await db.child('users').set('2', { v: 3 })
+    expect(db.queued).toBe(4)
+    expect(await db.child('users').get('2')).toEqual({ v: 3 }) // served locally
+    await expect(db.writeIfAbsent([['x', 1]])).rejects.toThrow(/Cannot reach/)
+    network.online = true
+    await vi.waitFor(() => expect(db.queued).toBe(0))
+    expect(await network.store.readMany(['users/1', 'users/2'])).toEqual([undefined, { v: 3 }])
+  })
+
+  it('keeps queued local values when the server pushes an older value for the same key', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.synced()
+    network.online = false
+    await db.child('k').set({ from: 'local' })
+    await network.store.writeMany([['k', { from: 'remote' }]]) // pushed over the still-open stream
+    await new Promise((r) => setTimeout(r, 20))
+    expect(await db.child('k').get()).toEqual({ from: 'local' })
+    network.online = true
+    await vi.waitFor(() => expect(db.queued).toBe(0))
+    expect(await network.store.readMany(['k'])).toEqual([{ from: 'local' }])
+  })
+
+  it('restarts from the persistent replica and resumes without a full sync', async () => {
+    network = new TestNetwork()
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.child('a').set(1)
+    await first.synced()
+    await first.close()
+    await network.client().writeMany([['b', 2]]) // missed while "closed"
+
+    network.store.reset()
+    const second = composite(persistent)
+    expect(await second.child('a').get()).toBe(1) // available before contacting the server
+    await vi.waitFor(async () => expect(await second.child('b').get()).toBe(2))
+    expect(network.store.calls.scan).toBeUndefined() // resumed via Last-Event-ID, no full scan
+  })
+
+  it('falls back to a full sync when the server cannot resume', async () => {
+    network = new TestNetwork({ historySize: 1 })
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.child('a').set(1)
+    await first.synced()
+    await first.close()
+    await network.client().writeMany([['b', 2]])
+    await network.client().writeMany([['c', 3]])
+    await network.client().removeMany(['a'])
+    const second = composite(persistent)
+    await vi.waitFor(async () => expect(await collect(second.keys())).toEqual(['b', 'c']))
+  })
+
+  it('replays an outbox left by a previous session', async () => {
+    network = new TestNetwork()
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.synced()
+    network.online = false
+    await first.child('offline').set('queued')
+    await first.close()
+    network.online = true
+    const second = composite(persistent)
+    await second.start()
+    await vi.waitFor(async () => expect(await network.store.readMany(['offline'])).toEqual(['queued']))
+  })
+
+  it('serves document queries from the local replica, offline too', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    const users = new DocumentStorageKV(db).child('users')
+    await users.set('1', { age: 30 })
+    await users.set('2', { age: 40 })
+    await db.synced()
+    network.online = false
+    expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2'])
+  })
+})

test/kv-storage/kv-storage-http-watch.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
+import { StorageUnavailableError } from '../../src/utils/errors.js'
+import { TestNetwork } from './sync-helpers.js'
+
+let network: TestNetwork
+afterEach(() => network?.close())
+
+describe('KVStorageHttp.watch (SSE)', () => {
+  it('receives changes made by other clients as remote change events', async () => {
+    network = new TestNetwork()
+    const watcher = network.client()
+    await watcher.watch()
+    const received = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().child('users').set('1', { name: 'Ada', n: 1n })
+    const event = await received
+    expect(event.changes).toEqual([{ key: 'users/1', value: { name: 'Ada', n: 1n } }])
+    expect(event.id).toMatch(/^\w+-1$/)
+    expect(watcher.lastEventId).toBe(event.id)
+  })
+
+  it('resumes from lastEventId and replays missed changes', async () => {
+    network = new TestNetwork()
+    const writer = network.client()
+    const watcher = network.client()
+    await watcher.watch()
+    const first = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await writer.writeMany([['a', 1]])
+    await first
+    await watcher.unwatch()
+    await writer.writeMany([['b', 2]])
+    await writer.writeMany([['c', 3]])
+    const replayed: KVChangeEvent[] = []
+    watcher.on('change', (e) => replayed.push(e))
+    await watcher.watch() // resolves once caught up, so the missed changes have been emitted
+    expect(replayed.map((e) => e.changes)).toEqual([[{ key: 'b', value: 2 }], [{ key: 'c', value: 3 }]])
+  })
+
+  it('asks for a resync when the missed changes are no longer in the history', async () => {
+    network = new TestNetwork({ historySize: 1 })
+    const writer = network.client()
+    const watcher = network.client()
+    await watcher.watch()
+    const seen = watcher.waitFor('change', () => true)
+    await writer.writeMany([['a', 1]])
+    await seen
+    await watcher.unwatch()
+    await writer.writeMany([['b', 2]])
+    await writer.writeMany([['c', 3]])
+    const resync = watcher.waitFor('resync', () => true)
+    await watcher.watch()
+    expect(await resync).toEqual({ id: expect.stringMatching(/-3$/) })
+  })
+
+  it('reports an unreachable server as StorageUnavailableError', async () => {
+    network = new TestNetwork()
+    const client = network.client()
+    network.online = false
+    await expect(client.readMany(['x'])).rejects.toBeInstanceOf(StorageUnavailableError)
+  })
+})

test/kv-storage/kv-storage-http.test.ts

@@ -9,7 +9,7 @@ import { runKVStorageConformance } from './kv-storage.conformance.js'
 import { serve } from './node-server.js'
 
 runKVStorageConformance('KVStorageHttp (in-process)', () => {
-  const server = new KVStorageHttpServer(new KVStorageInMemory(), '/kv')
+  const server = new KVStorageHttpServer(new KVStorageInMemory(), { basePath: '/kv' })
   return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) })
 })
 
@@ -22,7 +22,7 @@ describe('over a real socket', () => {
   })
   afterAll(() => server.close())
   const fresh = () => {
-    stores.push(new KVStorageHttpServer(new KVStorageInMemory(), `/${stores.length}`))
+    stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { basePath: `/${stores.length}` }))
     return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` })
   }
 

test/kv-storage/kv-storage-in-memory.test.ts

@@ -76,3 +76,12 @@ describe('model-based: random writes vs a reference map', () => {
     )
   })
 })
+
+describe('child nodes', () => {
+  it('are frozen, so implementations cannot accidentally shadow shared state', () => {
+    const child = new KVStorageInMemory().child('a').child('b')
+    expect(Object.isFrozen(child)).toBe(true)
+    expect(() => ((child as unknown as { state: number }).state = 1)).toThrow(TypeError)
+    expect(child.key).toBe('a/b')
+  })
+})

test/kv-storage/kv-storage-indexeddb.test.ts

@@ -0,0 +1,8 @@
+import 'fake-indexeddb/auto'
+import { KVStorageIndexedDB } from '../../src/kv-storage/kv-storage-indexeddb.js'
+import { runKVStorageConformance } from './kv-storage.conformance.js'
+
+let n = 0
+runKVStorageConformance('KVStorageIndexedDB', () => new KVStorageIndexedDB({ name: `db-${n++}` }))
+// Tiny batches exercise resuming scans across transactions.
+runKVStorageConformance('KVStorageIndexedDB (scan batches of 2)', () => new KVStorageIndexedDB({ name: `db-${n++}`, scanBatchSize: 2 }))

test/kv-storage/kv-storage.conformance.ts

@@ -71,6 +71,28 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
         expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c'])
       })
 
+      it('emits change events for writes and removals', async () => {
+        const events: unknown[] = []
+        const off = db.on('change', (event) => events.push(event))
+        await db.writeMany([['x', 1]])
+        await db.removeMany(['x'])
+        await db.writeIfAbsent([
+          ['a', 'no'],
+          ['y', 2],
+        ])
+        off()
+        expect(events).toEqual([
+          { changes: [{ key: 'x', value: 1 }], origin: 'local' },
+          { changes: [{ key: 'x' }], origin: 'local' },
+          { changes: [{ key: 'y', value: 2 }], origin: 'local' },
+        ])
+      })
+
+      it('stores bigint and Uint8Array values', async () => {
+        await db.writeMany([['bin', { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }]])
+        expect(await db.readMany(['bin'])).toEqual([{ n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }])
+      })
+
       it('does not share references with callers', async () => {
         const [value] = (await db.readMany(['a/2'])) as [{ n: string }]
         value.n = 'changed'

test/kv-storage/sync-helpers.ts

@@ -0,0 +1,35 @@
+import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js'
+import { KVStorageHttpServer, type KVStorageHttpServerOptions } from '../../src/kv-storage/kv-storage-http-server.js'
+import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
+import { KVStorageCounting } from '../helpers.js'
+
+/**
+ * An in-process server with a network switch. While `online` is false, new requests fail like a
+ * dropped connection (open SSE streams stay up, so pushes can still be simulated).
+ */
+export class TestNetwork {
+  online = true
+  readonly store = new KVStorageCounting(new KVStorageInMemory())
+  readonly server: KVStorageHttpServer
+  private readonly clients: KVStorageHttp[] = []
+
+  constructor(options: KVStorageHttpServerOptions = {}) {
+    this.server = new KVStorageHttpServer(this.store, options)
+  }
+
+  client(): KVStorageHttp {
+    const client = new KVStorageHttp({
+      url: 'http://server',
+      fetch: async (url, init) => {
+        if (!this.online) throw new TypeError('fetch failed')
+        return this.server.handle(new Request(url, init))
+      },
+    })
+    this.clients.push(client)
+    return client
+  }
+
+  async close(): Promise<void> {
+    await Promise.all(this.clients.map((c) => c.unwatch()))
+  }
+}

test/utils/event-emitter.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it, vi } from 'vitest'
+import { EventEmitter } from '../../src/utils/event-emitter.js'
+
+type Events = { tick: number; done: void }
+
+describe('EventEmitter', () => {
+  it('delivers payloads as-is (no cloning or freezing) and supports off', () => {
+    const emitter = new EventEmitter<Events & { obj: { n: number } }>()
+    const seen: unknown[] = []
+    const payload = { n: 1 }
+    const off = emitter.on('obj', (p) => seen.push(p))
+    expect(emitter.emit('obj', payload)).toBe(true)
+    off()
+    expect(emitter.emit('obj', payload)).toBe(false)
+    expect(seen[0]).toBe(payload)
+    expect(Object.isFrozen(payload)).toBe(false)
+  })
+
+  it('once fires a single time; listener errors are logged, not thrown', () => {
+    const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+    const emitter = new EventEmitter<Events>()
+    const once = vi.fn()
+    emitter.once('tick', once)
+    emitter.on('tick', () => {
+      throw new Error('boom')
+    })
+    emitter.emit('tick', 1)
+    emitter.emit('tick', 2)
+    expect(once).toHaveBeenCalledTimes(1)
+    expect(error).toHaveBeenCalledTimes(2)
+    error.mockRestore()
+  })
+
+  it('debounces and cancels pending debounced calls on off', async () => {
+    vi.useFakeTimers()
+    const emitter = new EventEmitter<Events>()
+    const listener = vi.fn()
+    const off = emitter.on('tick', listener, 50)
+    emitter.emit('tick', 1)
+    emitter.emit('tick', 2)
+    vi.advanceTimersByTime(60)
+    expect(listener).toHaveBeenCalledExactlyOnceWith(2)
+    emitter.emit('tick', 3)
+    off()
+    vi.advanceTimersByTime(60)
+    expect(listener).toHaveBeenCalledTimes(1)
+    vi.useRealTimers()
+  })
+
+  it('waitFor resolves on a matching payload and rejects on timeout', async () => {
+    const emitter = new EventEmitter<Events>()
+    const waiting = emitter.waitFor('tick', (n) => n > 1)
+    emitter.emit('tick', 1)
+    emitter.emit('tick', 2)
+    expect(await waiting).toBe(2)
+    await expect(emitter.waitFor('done', () => true, 5)).rejects.toThrow(/Timed out waiting for "done"/)
+  })
+})

test/utils/extended-json.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest'
+import fc from 'fast-check'
+import { ExtendedJson } from '../../src/utils/extended-json.js'
+
+describe('ExtendedJson', () => {
+  it('round-trips bigint and Uint8Array, nested anywhere', () => {
+    const value = { a: 1n, b: [new Uint8Array([0, 1, 255]), -(2n ** 70n)], c: { d: new Uint8Array(0), e: 'plain', f: null } }
+    expect(ExtendedJson.stringify({ n: 5n, bytes: new Uint8Array([171, 205]) })).toBe('{"n":"<bigint: 5n>","bytes":"<uint8array: abcd>"}')
+    expect(ExtendedJson.parse(ExtendedJson.stringify(value))).toEqual(value)
+  })
+
+  it('round-trips arbitrary data (property)', () => {
+    const data = fc.letrec((tie) => ({
+      value: fc.oneof(
+        { depthSize: 'small' },
+        fc.bigInt(),
+        fc.uint8Array(),
+        fc.string(),
+        fc.double({ noNaN: true, noDefaultInfinity: true }),
+        fc.boolean(),
+        fc.constant(null),
+        fc.array(tie('value')),
+        fc.dictionary(fc.string(), tie('value')),
+      ),
+    })).value
+    fc.assert(
+      fc.property(data, (v) => {
+        expect(ExtendedJson.parse(ExtendedJson.stringify(v))).toEqual(v)
+      }),
+    )
+  })
+})

test/utils/sse-session.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest'
+import { SSEEventParser } from '../../src/utils/sse-session/sse-event-parser.js'
+import { SSESession } from '../../src/utils/sse-session/sse-session.js'
+
+const encode = (text: string) => new TextEncoder().encode(text)
+
+describe('vendored sse-session', () => {
+  it('parses events across arbitrary chunk boundaries', () => {
+    const parser = new SSEEventParser()
+    const text = 'id: 1\nevent: change\ndata: {"a":1}\n\n: comment\ndata: line1\ndata: line2\n\n'
+    const events = []
+    for (let i = 0; i < text.length; i += 3) events.push(...parser.parseEvents(encode(text.slice(i, i + 3))))
+    expect(events).toEqual([{ id: '1', event: 'change', data: '{"a":1}' }, { data: 'line1\nline2' }])
+  })
+
+  it('connects, delivers messages and reconnects when persistent', async () => {
+    let connections = 0
+    const session = new SSESession('http://test/events', {
+      persistent: true,
+      fetch: async () => {
+        connections++
+        return new Response(encode(`data: hello ${connections}\n\n`))
+      },
+    })
+    const received: string[] = []
+    session.on('message', (m) => received.push(m.data))
+    await session.connect()
+    await session.waitFor('message', (m) => m.data === 'hello 2')
+    await session.disconnect()
+    expect(received.slice(0, 2)).toEqual(['hello 1', 'hello 2'])
+  })
+})