Add KVStorageComposite, IndexedDB storage and SSE change streams

- Every KV storage emits change events (base class owns writeMany/removeMany/
  writeIfAbsent; implementations provide protected apply* methods)
- KVStorageHttpServer streams changes over SSE (GET /events) with a resumable
  history; KVStorageHttp.watch() subscribes and resumes via Last-Event-ID
- KVStorageIndexedDB for local persistence
- KVStorageComposite keeps replicas of a source in sync: local reads, resume
  instead of full sync, offline outbox replayed in order, echo tracking
- Values may contain bigint/Uint8Array (Extended JSON over the network)
- Vendor sse-session, event-emitter and extended-json (no runtime deps)
- Freeze child nodes so shadowed state on children fails loudly
- CI on Node 22, 24 and 26

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE

test/kv-storage/kv-storage-composite.test.ts

@@ -0,0 +1,144 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
+import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
+import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
+import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
+import { collect } from '../helpers.js'
+import { runKVStorageConformance } from './kv-storage.conformance.js'
+import { TestNetwork } from './sync-helpers.js'
+
+runKVStorageConformance(
+  'KVStorageComposite (in-memory source, 1 replica)',
+  () => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [new KVStorageInMemory()] }),
+)
+runKVStorageConformance('KVStorageComposite (http source, 2 replicas)', () => {
+  network = new TestNetwork()
+  return new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), new KVStorageInMemory()] })
+})
+
+let network: TestNetwork
+const composites: KVStorageComposite[] = []
+afterEach(async () => {
+  await Promise.all(composites.splice(0).map((c) => c.close()))
+  await network?.close()
+})
+
+function composite(persistent: BaseKVStorage = new KVStorageInMemory()) {
+  const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], retry: { baseDelay: 5, maxDelay: 20 } })
+  composites.push(c)
+  return c
+}
+
+describe('KVStorageComposite sync', () => {
+  it('loads existing data with one full sync, then serves reads locally', async () => {
+    network = new TestNetwork()
+    await network.store.writeMany([
+      ['users/1', { name: 'Ada' }],
+      ['users/2', { name: 'Bob' }],
+    ])
+    const db = composite()
+    await db.synced()
+    network.store.reset()
+    expect(await db.child('users').get('1')).toEqual({ name: 'Ada' })
+    expect(await collect(db.child('users').keys())).toEqual(['1', '2'])
+    expect(network.store.calls).toEqual({})
+  })
+
+  it('applies changes pushed from other clients to every replica', async () => {
+    network = new TestNetwork()
+    const memory = new KVStorageInMemory()
+    const db = composite(memory)
+    await db.synced()
+    const seen = db.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().child('users').set('9', { name: 'Remote' })
+    expect((await seen).changes).toEqual([{ key: 'users/9', value: { name: 'Remote' } }])
+    expect(await db.child('users').get('9')).toEqual({ name: 'Remote' })
+    expect(await memory.readMany(['users/9'])).toEqual([{ name: 'Remote' }])
+  })
+
+  it('queues writes while offline and replays them in order when back online', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.synced()
+    network.online = false
+    await db.child('users').set('1', { v: 1 })
+    await db.child('users').set('1', { v: 2 })
+    await db.child('users').delete('1')
+    await db.child('users').set('2', { v: 3 })
+    expect(db.queued).toBe(4)
+    expect(await db.child('users').get('2')).toEqual({ v: 3 }) // served locally
+    await expect(db.writeIfAbsent([['x', 1]])).rejects.toThrow(/Cannot reach/)
+    network.online = true
+    await vi.waitFor(() => expect(db.queued).toBe(0))
+    expect(await network.store.readMany(['users/1', 'users/2'])).toEqual([undefined, { v: 3 }])
+  })
+
+  it('keeps queued local values when the server pushes an older value for the same key', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.synced()
+    network.online = false
+    await db.child('k').set({ from: 'local' })
+    await network.store.writeMany([['k', { from: 'remote' }]]) // pushed over the still-open stream
+    await new Promise((r) => setTimeout(r, 20))
+    expect(await db.child('k').get()).toEqual({ from: 'local' })
+    network.online = true
+    await vi.waitFor(() => expect(db.queued).toBe(0))
+    expect(await network.store.readMany(['k'])).toEqual([{ from: 'local' }])
+  })
+
+  it('restarts from the persistent replica and resumes without a full sync', async () => {
+    network = new TestNetwork()
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.child('a').set(1)
+    await first.synced()
+    await first.close()
+    await network.client().writeMany([['b', 2]]) // missed while "closed"
+
+    network.store.reset()
+    const second = composite(persistent)
+    expect(await second.child('a').get()).toBe(1) // available before contacting the server
+    await vi.waitFor(async () => expect(await second.child('b').get()).toBe(2))
+    expect(network.store.calls.scan).toBeUndefined() // resumed via Last-Event-ID, no full scan
+  })
+
+  it('falls back to a full sync when the server cannot resume', async () => {
+    network = new TestNetwork({ historySize: 1 })
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.child('a').set(1)
+    await first.synced()
+    await first.close()
+    await network.client().writeMany([['b', 2]])
+    await network.client().writeMany([['c', 3]])
+    await network.client().removeMany(['a'])
+    const second = composite(persistent)
+    await vi.waitFor(async () => expect(await collect(second.keys())).toEqual(['b', 'c']))
+  })
+
+  it('replays an outbox left by a previous session', async () => {
+    network = new TestNetwork()
+    const persistent = new KVStorageInMemory()
+    const first = composite(persistent)
+    await first.synced()
+    network.online = false
+    await first.child('offline').set('queued')
+    await first.close()
+    network.online = true
+    const second = composite(persistent)
+    await second.start()
+    await vi.waitFor(async () => expect(await network.store.readMany(['offline'])).toEqual(['queued']))
+  })
+
+  it('serves document queries from the local replica, offline too', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    const users = new DocumentStorageKV(db).child('users')
+    await users.set('1', { age: 30 })
+    await users.set('2', { age: 40 })
+    await db.synced()
+    network.online = false
+    expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2'])
+  })
+})

test/kv-storage/kv-storage-http-watch.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
+import { StorageUnavailableError } from '../../src/utils/errors.js'
+import { TestNetwork } from './sync-helpers.js'
+
+let network: TestNetwork
+afterEach(() => network?.close())
+
+describe('KVStorageHttp.watch (SSE)', () => {
+  it('receives changes made by other clients as remote change events', async () => {
+    network = new TestNetwork()
+    const watcher = network.client()
+    await watcher.watch()
+    const received = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().child('users').set('1', { name: 'Ada', n: 1n })
+    const event = await received
+    expect(event.changes).toEqual([{ key: 'users/1', value: { name: 'Ada', n: 1n } }])
+    expect(event.id).toMatch(/^\w+-1$/)
+    expect(watcher.lastEventId).toBe(event.id)
+  })
+
+  it('resumes from lastEventId and replays missed changes', async () => {
+    network = new TestNetwork()
+    const writer = network.client()
+    const watcher = network.client()
+    await watcher.watch()
+    const first = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await writer.writeMany([['a', 1]])
+    await first
+    await watcher.unwatch()
+    await writer.writeMany([['b', 2]])
+    await writer.writeMany([['c', 3]])
+    const replayed: KVChangeEvent[] = []
+    watcher.on('change', (e) => replayed.push(e))
+    await watcher.watch() // resolves once caught up, so the missed changes have been emitted
+    expect(replayed.map((e) => e.changes)).toEqual([[{ key: 'b', value: 2 }], [{ key: 'c', value: 3 }]])
+  })
+
+  it('asks for a resync when the missed changes are no longer in the history', async () => {
+    network = new TestNetwork({ historySize: 1 })
+    const writer = network.client()
+    const watcher = network.client()
+    await watcher.watch()
+    const seen = watcher.waitFor('change', () => true)
+    await writer.writeMany([['a', 1]])
+    await seen
+    await watcher.unwatch()
+    await writer.writeMany([['b', 2]])
+    await writer.writeMany([['c', 3]])
+    const resync = watcher.waitFor('resync', () => true)
+    await watcher.watch()
+    expect(await resync).toEqual({ id: expect.stringMatching(/-3$/) })
+  })
+
+  it('reports an unreachable server as StorageUnavailableError', async () => {
+    network = new TestNetwork()
+    const client = network.client()
+    network.online = false
+    await expect(client.readMany(['x'])).rejects.toBeInstanceOf(StorageUnavailableError)
+  })
+})

test/kv-storage/kv-storage-http.test.ts

@@ -9,7 +9,7 @@ import { runKVStorageConformance } from './kv-storage.conformance.js'
 import { serve } from './node-server.js'
 
 runKVStorageConformance('KVStorageHttp (in-process)', () => {
-  const server = new KVStorageHttpServer(new KVStorageInMemory(), '/kv')
+  const server = new KVStorageHttpServer(new KVStorageInMemory(), { basePath: '/kv' })
   return new KVStorageHttp({ url: 'http://test/kv', fetch: async (input, init) => server.handle(new Request(input, init)) })
 })
 
@@ -22,7 +22,7 @@ describe('over a real socket', () => {
   })
   afterAll(() => server.close())
   const fresh = () => {
-    stores.push(new KVStorageHttpServer(new KVStorageInMemory(), `/${stores.length}`))
+    stores.push(new KVStorageHttpServer(new KVStorageInMemory(), { basePath: `/${stores.length}` }))
     return new KVStorageHttp({ url: `${server.url}/${stores.length - 1}` })
   }
 

test/kv-storage/kv-storage-in-memory.test.ts

@@ -76,3 +76,12 @@ describe('model-based: random writes vs a reference map', () => {
     )
   })
 })
+
+describe('child nodes', () => {
+  it('are frozen, so implementations cannot accidentally shadow shared state', () => {
+    const child = new KVStorageInMemory().child('a').child('b')
+    expect(Object.isFrozen(child)).toBe(true)
+    expect(() => ((child as unknown as { state: number }).state = 1)).toThrow(TypeError)
+    expect(child.key).toBe('a/b')
+  })
+})

test/kv-storage/kv-storage-indexeddb.test.ts

@@ -0,0 +1,8 @@
+import 'fake-indexeddb/auto'
+import { KVStorageIndexedDB } from '../../src/kv-storage/kv-storage-indexeddb.js'
+import { runKVStorageConformance } from './kv-storage.conformance.js'
+
+let n = 0
+runKVStorageConformance('KVStorageIndexedDB', () => new KVStorageIndexedDB({ name: `db-${n++}` }))
+// Tiny batches exercise resuming scans across transactions.
+runKVStorageConformance('KVStorageIndexedDB (scan batches of 2)', () => new KVStorageIndexedDB({ name: `db-${n++}`, scanBatchSize: 2 }))

test/kv-storage/kv-storage.conformance.ts

@@ -71,6 +71,28 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
         expect(seen.filter((k) => k !== 'a/1')).toEqual(['10', '9', 'a', 'a/2', 'a0', 'b', 'c'])
       })
 
+      it('emits change events for writes and removals', async () => {
+        const events: unknown[] = []
+        const off = db.on('change', (event) => events.push(event))
+        await db.writeMany([['x', 1]])
+        await db.removeMany(['x'])
+        await db.writeIfAbsent([
+          ['a', 'no'],
+          ['y', 2],
+        ])
+        off()
+        expect(events).toEqual([
+          { changes: [{ key: 'x', value: 1 }], origin: 'local' },
+          { changes: [{ key: 'x' }], origin: 'local' },
+          { changes: [{ key: 'y', value: 2 }], origin: 'local' },
+        ])
+      })
+
+      it('stores bigint and Uint8Array values', async () => {
+        await db.writeMany([['bin', { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }]])
+        expect(await db.readMany(['bin'])).toEqual([{ n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }])
+      })
+
       it('does not share references with callers', async () => {
         const [value] = (await db.readMany(['a/2'])) as [{ n: string }]
         value.n = 'changed'

test/kv-storage/sync-helpers.ts

@@ -0,0 +1,35 @@
+import { KVStorageHttp } from '../../src/kv-storage/kv-storage-http.js'
+import { KVStorageHttpServer, type KVStorageHttpServerOptions } from '../../src/kv-storage/kv-storage-http-server.js'
+import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
+import { KVStorageCounting } from '../helpers.js'
+
+/**
+ * An in-process server with a network switch. While `online` is false, new requests fail like a
+ * dropped connection (open SSE streams stay up, so pushes can still be simulated).
+ */
+export class TestNetwork {
+  online = true
+  readonly store = new KVStorageCounting(new KVStorageInMemory())
+  readonly server: KVStorageHttpServer
+  private readonly clients: KVStorageHttp[] = []
+
+  constructor(options: KVStorageHttpServerOptions = {}) {
+    this.server = new KVStorageHttpServer(this.store, options)
+  }
+
+  client(): KVStorageHttp {
+    const client = new KVStorageHttp({
+      url: 'http://server',
+      fetch: async (url, init) => {
+        if (!this.online) throw new TypeError('fetch failed')
+        return this.server.handle(new Request(url, init))
+      },
+    })
+    this.clients.push(client)
+    return client
+  }
+
+  async close(): Promise<void> {
+    await Promise.all(this.clients.map((c) => c.unwatch()))
+  }
+}