Plain operations: last write wins, hash-keyed events, simpler composite

- KV: one write primitive, write(changes) with { key, value? }; writeIfAbsent removed.
  The server executes exactly what clients send and never reads values, so they can be
  encrypted end to end. Adds setFields/getFields on the base class.
- HTTP: no server history or event ids; SSE relays every change (own writes included) and
  the client emits resync when a dropped stream reconnects.
- Composite: writes apply locally, then go to the source (undone locally if refused);
  pushed changes are applied and announced only where they differ from local values;
  offline writes go to a queue store (any KV storage); sync() = replay, diff, replay.
  Removes KVReplicaSet, KVOutbox, KeyCounter, SerialQueue and RetryTimer.
- Events: id = <time>_<sha256(time + canonical data)>, so concurrent appends never
  collide and retries can't duplicate; no versions or ConcurrencyError.
- Event-sourced documents: read, resolve, append, apply; catchUp folds in foreign events;
  getAt takes an event id.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE

test/document-storage/document-storage.test.ts

@@ -42,43 +42,48 @@ describe('EventSourcedDocumentStorage', () => {
     return { kv, events, projections, docs: new EventSourcedDocumentStorage(events, projections) }
   }
 
-  it('records resolved changes as history and replays any version', async () => {
+  it('records resolved changes as history and replays any event', async () => {
     const users = setup().docs.child('users')
     await users.set('1', { n: 5 })
     await users.update('1', { $inc: { n: 1 } })
     await users.delete('1')
     await users.set('1', { n: 10 })
-    expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([
+    const history = await collect(users.history('1'))
+    expect(history.map((e) => e.data)).toEqual([
       { op: 'set', doc: { n: 5 } },
       { op: 'patch', set: { n: 6 }, unset: [] },
       { op: 'delete' },
       { op: 'set', doc: { n: 10 } },
     ])
-    expect(await users.getAt('1', 2)).toEqual({ n: 6 })
-    expect(await users.child('1').getAt(3)).toBeUndefined()
+    expect(await users.getAt('1', history[1]!.id)).toEqual({ n: 6 })
+    expect(await users.child('1').getAt(history[2]!.id)).toBeUndefined()
     expect(await users.get('1')).toEqual({ n: 10 })
   })
 
-  it('heals a projection left behind by a crash between append and projection write', async () => {
+  it('catchUp folds in events written elsewhere', async () => {
     const { docs, events } = setup()
     const users = docs.child('users')
     await users.set('1', { n: 1 })
-    await events.child('users').append('1', [{ op: 'patch', set: { n: 2 }, unset: [] }]) // projection never written
+    await events.child('users').append('1', [{ op: 'patch', set: { m: 2 }, unset: [] }]) // e.g. synced from another device
     expect(await users.get('1')).toEqual({ n: 1 })
-    expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3 })
-    await events.child('users').append('2', [{ op: 'set', doc: { n: 7 } }]) // first write crashed
-    expect(await users.catchUp('2')).toEqual({ n: 7 })
-    expect(await users.get('2')).toEqual({ n: 7 })
+    expect(await users.catchUp('1')).toEqual({ n: 1, m: 2 })
+    expect(await users.get('1')).toEqual({ n: 1, m: 2 })
+    await events.child('users').append('2', [{ op: 'set', doc: { n: 7 } }, { op: 'delete' }])
+    expect(await users.catchUp('2')).toBeUndefined()
+    expect(await users.get('2')).toBeUndefined()
   })
 
-  it('two instances incrementing the same document converge', async () => {
+  it('two instances editing different fields both keep their edits; the same field ends with the later value', async () => {
     const kv = new KVStorageInMemory()
-    const make = () => new EventSourcedDocumentStorage(new EventStorageKV(kv), new DocumentStorageKV(kv.child('projections'))).child('users')
+    let t = 1000
+    const make = () =>
+      new EventSourcedDocumentStorage(new EventStorageKV(kv, { now: () => t++ }), new DocumentStorageKV(kv.child('projections'))).child('users')
     const a = make()
     const b = make()
-    await a.set('1', { n: 0 })
-    await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })]))
-    expect(await a.get('1')).toEqual({ n: 10 })
+    await a.set('1', { name: 'x', age: 1 })
+    await Promise.all([a.update('1', { $set: { name: 'Ada' } }), b.update('1', { $set: { age: 2 } })])
+    await b.update('1', { $set: { name: 'Bob' } })
+    expect(await a.catchUp('1')).toEqual({ name: 'Bob', age: 2 })
   })
 
   it('a write is a projection read, an append and a projection write', async () => {
@@ -86,6 +91,6 @@ describe('EventSourcedDocumentStorage', () => {
     await docs.child('users').set('1', { n: 1 })
     kv.reset()
     await docs.child('users').update('1', { $inc: { n: 1 } })
-    expect(kv.calls).toEqual({ readMany: 2, writeIfAbsent: 1, writeMany: 1 })
+    expect(kv.calls).toEqual({ readMany: 2, writeMany: 2 })
   })
 })

test/event-storage/event-storage-kv.test.ts

@@ -1,45 +1,70 @@
 import { describe, expect, it } from 'vitest'
 import fc from 'fast-check'
 import { EventStorageKV } from '../../src/event-storage/event-storage-kv.js'
+import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
 import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
-import { ConcurrencyError } from '../../src/utils/errors.js'
 import { collect, KVStorageCounting } from '../helpers.js'
 
-function setup(kv = new KVStorageCounting()) {
-  let t = 1000
+function setup(kv: BaseKVStorage = new KVStorageCounting(), start = 1000) {
+  let t = start
   return { kv, events: new EventStorageKV(kv, { now: () => t++ }) }
 }
 
 describe('EventStorageKV', () => {
   it('appends and reads in self and child forms', async () => {
     const orders = setup().events.child('orders')
-    expect(await orders.append('o1', [{ type: 'created' }], { expectedVersion: 0 })).toEqual({ version: 1 })
-    expect(await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])).toEqual({ version: 3 })
+    const [created] = await orders.append('o1', [{ type: 'created' }])
+    const ids = await orders.child('o1').append([{ type: 'paid' }, { type: 'shipped' }])
     expect(await collect(orders.read('o1'))).toEqual([
-      { version: 1, at: 1000, data: { type: 'created' } },
-      { version: 2, at: 1001, data: { type: 'paid' } },
-      { version: 3, at: 1001, data: { type: 'shipped' } },
+      { id: created, at: 1000, data: { type: 'created' } },
+      { id: ids[0], at: 1001, data: { type: 'paid' } },
+      { id: ids[1], at: 1002, data: { type: 'shipped' } },
     ])
-    expect(await collect(orders.child('o1').read({ from: 3 }))).toHaveLength(1)
-    expect(await orders.version('o1')).toBe(3)
-    expect(await orders.version('none')).toBe(0)
+    expect((await collect(orders.child('o1').read({ after: ids[0] }))).map((e) => e.data)).toEqual([{ type: 'shipped' }])
+    expect((await collect(orders.read('o1', { reverse: true, limit: 2 }))).map((e) => e.id)).toEqual([ids[1], ids[0]])
+    expect((await collect(orders.read('o1', { reverse: true, after: ids[0] }))).map((e) => e.id)).toEqual([created])
   })
 
-  it('stores one key per commit', async () => {
+  it('ids are the time and a hash of the time and content', async () => {
     const { kv, events } = setup()
-    await events.child('s').append(['a'])
-    await events.child('s').append(['b', 'c'])
-    expect((await collect(kv.scan())).map(([k]) => k)).toEqual(['s/~e/000000000001', 's/~e/000000000002'])
+    const [id] = await events.child('s').append([{ b: 1, a: 2 }])
+    expect(id).toMatch(/^000000000001000_[0-9a-f]{64}$/)
+    expect(id).toBe(await EventStorageKV.id(1000, { a: 2, b: 1 }))
+    expect((await collect(kv.scan())).map(([k]) => k)).toEqual([`s/~e/${id}`])
   })
 
-  it('enforces expectedVersion and the arity rules', async () => {
+  it('one append is one write; one instance never reuses a time', async () => {
+    const kv = new KVStorageCounting()
+    const events = new EventStorageKV(kv, { now: () => 5 })
+    const ids = await events.child('s').append(['a', 'a'])
+    expect(kv.calls).toEqual({ writeMany: 1 })
+    expect(ids.map((id) => id.slice(0, 15))).toEqual(['000000000000005', '000000000000006'])
+    expect((await collect(events.child('s').read())).map((e) => e.data)).toEqual(['a', 'a'])
+  })
+
+  it('keeps concurrent appends from different writers, ordered by time', async () => {
+    const kv = new KVStorageInMemory()
+    const a = setup(kv, 1000).events.child('s')
+    const b = setup(kv, 1000).events.child('s')
+    await Promise.all([a.append(['a1', 'a2']), b.append(['b1', 'b2'])])
+    const read = await collect(a.read())
+    expect(read).toHaveLength(4)
+    expect(read.map((e) => e.at)).toEqual([1000, 1000, 1001, 1001])
+    expect(read.map((e) => e.id)).toEqual([...read.map((e) => e.id)].sort())
+    expect(await collect(b.read())).toEqual(read)
+  })
+
+  it('writing the same event again does not duplicate it', async () => {
+    const kv = new KVStorageInMemory()
+    await setup(kv).events.child('s').append(['x'])
+    await setup(kv).events.child('s').append(['x'])
+    expect(await collect(new EventStorageKV(kv).child('s').read())).toHaveLength(1)
+  })
+
+  it('checks the arity rules', async () => {
     const { events } = setup()
-    const s = events.child('s')
-    await s.append(['a'])
-    await expect(s.append(['b'], { expectedVersion: 0 })).rejects.toBeInstanceOf(ConcurrencyError)
     await expect(events.append(['x'])).rejects.toThrow(/root/)
     await expect(events.child('o').append(undefined as never, ['x'] as never)).rejects.toThrow(TypeError)
-    await expect(events.child('o').version(undefined as never)).rejects.toThrow(TypeError)
   })
 
   it('lists child streams', async () => {
@@ -51,46 +76,26 @@ describe('EventStorageKV', () => {
     expect(await collect(orders.child('o1').keys())).toEqual(['notes'])
   })
 
-  it('a warm append is one writeIfAbsent; a cold one adds one reverse scan', async () => {
-    const { kv, events } = setup()
-    await events.child('s').append(['a'])
-    kv.reset()
-    await events.child('s').append(['b'])
-    expect(kv.calls).toEqual({ writeIfAbsent: 1 })
-    const cold = setup(kv).events
-    kv.reset()
-    await cold.child('s').append(['c'])
-    expect(kv.calls).toEqual({ scan: 1, writeIfAbsent: 1 })
-  })
-
-  it('a writer with a stale cache collides; with "any" it retries', async () => {
-    const kv = new KVStorageInMemory()
-    const a = setup(new KVStorageCounting(kv)).events.child('s')
-    const b = setup(new KVStorageCounting(kv)).events.child('s')
-    await a.append(['a1'])
-    await b.append(['b1'])
-    await expect(a.append(['a2'], { expectedVersion: 1 })).rejects.toMatchObject({ actualVersion: 2 })
-    expect(await a.append(['a3'])).toEqual({ version: 3 })
-    const results = await Promise.allSettled([a.append(['x'], { expectedVersion: 3 }), b.append(['y'], { expectedVersion: 3 })])
-    expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1)
-  })
-
   it('reads any window correctly (property)', async () => {
     await fc.assert(
       fc.asyncProperty(
         fc.array(fc.integer({ min: 1, max: 4 }), { maxLength: 20 }),
         fc.record({
-          from: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }),
-          to: fc.option(fc.integer({ min: 0, max: 60 }), { nil: undefined }),
+          after: fc.option(fc.nat(60), { nil: undefined }),
           limit: fc.option(fc.nat(10), { nil: undefined }),
+          reverse: fc.boolean(),
         }),
-        async (sizes, options) => {
+        async (sizes, { after, limit, reverse }) => {
           const s = setup().events.child('s')
-          let n = 0
-          for (const size of sizes) await s.append(Array.from({ length: size }, () => ++n))
-          let expected = Array.from({ length: n }, (_, i) => i + 1).filter((v) => v >= (options.from ?? 1) && v <= (options.to ?? Infinity))
-          if (options.limit !== undefined) expected = expected.slice(0, options.limit)
-          expect((await collect(s.read(options))).map((e) => e.data)).toEqual(expected)
+          const ids: string[] = []
+          for (const size of sizes) ids.push(...(await s.append(Array.from({ length: size }, () => ids.length))))
+          const all = await collect(s.read())
+          expect(all.map((e) => e.id)).toEqual(ids)
+          const ordered = reverse ? [...ids].reverse() : ids
+          const afterId = after === undefined ? undefined : ids[after]
+          let expected = afterId === undefined ? ordered : ordered.slice(ordered.indexOf(afterId) + 1)
+          if (limit !== undefined) expected = expected.slice(0, limit)
+          expect((await collect(s.read({ after: afterId, limit, reverse }))).map((e) => e.id)).toEqual(expected)
         },
       ),
       { numRuns: 200 },

test/helpers.ts

@@ -1,4 +1,4 @@
-import { BaseKVStorage, type ScanOptions } from '../src/kv-storage/base-kv-storage.js'
+import { BaseKVStorage, type KVChange, type ScanOptions } from '../src/kv-storage/base-kv-storage.js'
 import { KVStorageInMemory } from '../src/kv-storage/kv-storage-in-memory.js'
 
 export async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
@@ -9,8 +9,8 @@ export async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
 
 /**
  * Wraps another KV storage and counts primitive calls. Also shows the wrapper pattern
- * (e.g. an encryption layer): extend BaseKVStorage and delegate the five primitives.
- * Counts are reported under the public method names (writeMany, removeMany, writeIfAbsent).
+ * (e.g. an encryption layer): extend BaseKVStorage and delegate the three primitives.
+ * Writes are counted as `writeMany`.
  */
 export class KVStorageCounting extends BaseKVStorage {
   calls: Record<string, number> = {}
@@ -26,14 +26,8 @@ export class KVStorageCounting extends BaseKVStorage {
   readMany(keys: string[]) {
     return this.count('readMany', () => this.inner.readMany(keys))
   }
-  protected applyWrite(entries: [string, unknown][]) {
-    return this.count('writeMany', () => this.inner.writeMany(entries))
-  }
-  protected applyRemove(keys: string[]) {
-    return this.count('removeMany', () => this.inner.removeMany(keys))
-  }
-  protected applyWriteIfAbsent(entries: [string, unknown][]) {
-    return this.count('writeIfAbsent', () => this.inner.writeIfAbsent(entries))
+  protected write(changes: KVChange[]) {
+    return this.count('writeMany', () => this.inner.writeMany(changes))
   }
   scan(options?: ScanOptions) {
     return this.count('scan', () => this.inner.scan(options))

test/kv-storage/kv-outbox.test.ts

@@ -1,36 +0,0 @@
-import { describe, expect, it, vi } from 'vitest'
-import { KVOutbox, type OutboxEntry } from '../../src/kv-storage/kv-outbox.js'
-import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
-import { RetryTimer } from '../../src/utils/retry-timer.js'
-import { StorageUnavailableError } from '../../src/utils/errors.js'
-
-describe('KVOutbox', () => {
-  it('replays entries in order once the source is reachable, and survives a restart', async () => {
-    const store = new KVStorageInMemory()
-    let online = false
-    const sent: OutboxEntry[] = []
-    const send = async (entry: OutboxEntry) => {
-      if (!online) throw new StorageUnavailableError('offline')
-      sent.push(entry)
-    }
-    const firstRetry = new RetryTimer({ baseDelay: 5, maxDelay: 5 })
-    const first = new KVOutbox(store, send, firstRetry)
-    await first.push({ type: 'write', entries: [['a', 1]] })
-    await first.push({ type: 'remove', keys: ['a'] })
-    expect(first.size).toBe(2)
-    expect(first.pending.has('a')).toBe(true)
-
-    firstRetry.stop() // the first session ends...
-    await first.replay() // ...once its in-flight replay attempt has failed
-    // A new session over the same store picks up the queue.
-    const second = new KVOutbox(store, send, new RetryTimer({ baseDelay: 5, maxDelay: 5 }))
-    online = true
-    await second.load()
-    await vi.waitFor(() => expect(second.size).toBe(0))
-    expect(sent).toEqual([
-      { type: 'write', entries: [['a', 1]] },
-      { type: 'remove', keys: ['a'] },
-    ])
-    expect(await store.readMany(['~q/o/000000000001', '~q/o/000000000002'])).toEqual([undefined, undefined])
-  })
-})

test/kv-storage/kv-storage-composite.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, describe, expect, it, vi } from 'vitest'
 import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
 import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
-import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
+import type { BaseKVStorage, KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
 import { DocumentStorageKV } from '../../src/document-storage/document-storage-kv.js'
 import { collect, KVStorageCounting } from '../helpers.js'
 import { runKVStorageConformance } from './kv-storage.conformance.js'
@@ -23,21 +23,21 @@ afterEach(async () => {
   await network?.close()
 })
 
-function composite(persistent: BaseKVStorage = new KVStorageInMemory()) {
-  const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], retry: { baseDelay: 5, maxDelay: 20 } })
+function composite(persistent: BaseKVStorage = new KVStorageInMemory(), queue?: BaseKVStorage) {
+  const c = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory(), persistent], queue, retryMs: 10 })
   composites.push(c)
   return c
 }
 
 describe('KVStorageComposite sync', () => {
-  it('loads existing data with one full sync, then serves reads locally', async () => {
+  it('loads existing data with one sync, then serves reads locally', async () => {
     network = new TestNetwork()
     await network.store.writeMany([
-      ['users/1', { name: 'Ada' }],
-      ['users/2', { name: 'Bob' }],
+      { key: 'users/1', value: { name: 'Ada' } },
+      { key: 'users/2', value: { name: 'Bob' } },
     ])
     const db = composite()
-    await db.synced()
+    await db.sync()
     network.store.reset()
     expect(await db.child('users').get('1')).toEqual({ name: 'Ada' })
     expect(await collect(db.child('users').keys())).toEqual(['1', '2'])
@@ -48,7 +48,7 @@ describe('KVStorageComposite sync', () => {
     network = new TestNetwork()
     const memory = new KVStorageInMemory()
     const db = composite(memory)
-    await db.synced()
+    await db.sync()
     const seen = db.waitFor('change', (e) => e.origin === 'remote')
     await network.client().child('users').set('9', { name: 'Remote' })
     expect((await seen).changes).toEqual([{ key: 'users/9', value: { name: 'Remote' } }])
@@ -56,10 +56,29 @@ describe('KVStorageComposite sync', () => {
     expect(await memory.readMany(['users/9'])).toEqual([{ name: 'Remote' }])
   })
 
+  it('announces only remote changes that differ from local values', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.sync()
+    const events: KVChangeEvent[] = []
+    db.on('change', (e) => events.push(e))
+    await db.set('mine', 1) // relayed back by the server: same value, so no remote event
+    const other = db.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().writeMany([
+      { key: 'mine', value: 1 },
+      { key: 'theirs', value: 2 },
+    ])
+    await other
+    expect(events).toEqual([
+      { changes: [{ key: 'mine', value: 1 }], origin: 'local' },
+      { changes: [{ key: 'theirs', value: 2 }], origin: 'remote' },
+    ])
+  })
+
   it('queues writes while offline and replays them in order when back online', async () => {
     network = new TestNetwork()
     const db = composite()
-    await db.synced()
+    await db.sync()
     network.online = false
     await db.child('users').set('1', { v: 1 })
     await db.child('users').set('1', { v: 2 })
@@ -67,68 +86,97 @@ describe('KVStorageComposite sync', () => {
     await db.child('users').set('2', { v: 3 })
     expect(db.queued).toBe(4)
     expect(await db.child('users').get('2')).toEqual({ v: 3 }) // served locally
-    await expect(db.writeIfAbsent([['x', 1]])).rejects.toThrow(/Cannot reach/)
     network.online = true
     await vi.waitFor(() => expect(db.queued).toBe(0))
     expect(await network.store.readMany(['users/1', 'users/2'])).toEqual([undefined, { v: 3 }])
   })
 
-  it('keeps queued local values when the server pushes an older value for the same key', async () => {
+  it('the last write wins when the server pushes a change while a local write is queued', async () => {
     network = new TestNetwork()
     const db = composite()
-    await db.synced()
+    await db.sync()
     network.online = false
-    await db.child('k').set({ from: 'local' })
-    await network.store.writeMany([['k', { from: 'remote' }]]) // pushed over the still-open stream
-    await new Promise((r) => setTimeout(r, 20))
-    expect(await db.child('k').get()).toEqual({ from: 'local' })
+    await db.set('k', { from: 'local' })
+    await network.store.writeMany([{ key: 'k', value: { from: 'remote' } }]) // pushed over the still-open stream
+    await vi.waitFor(async () => expect(await db.get('k')).toEqual({ from: 'remote' }))
     network.online = true
-    await vi.waitFor(() => expect(db.queued).toBe(0))
+    await vi.waitFor(() => expect(db.queued).toBe(0)) // the queued write is sent last, so it wins
     expect(await network.store.readMany(['k'])).toEqual([{ from: 'local' }])
+    expect(await db.get('k')).toEqual({ from: 'local' })
   })
 
-  it('restarts from the persistent replica and resumes without a full sync', async () => {
+  it('restarts from the persistent replica, then catches up', async () => {
     network = new TestNetwork()
     const persistent = new KVStorageInMemory()
-    const first = composite(persistent)
-    await first.child('a').set(1)
-    await first.synced()
+    const queue = new KVStorageInMemory()
+    const first = composite(persistent, queue)
+    await first.set('a', 1)
+    await first.set('gone', 1)
+    await first.sync()
     await first.close()
-    await network.client().writeMany([['b', 2]]) // missed while "closed"
+    await network.client().writeMany([{ key: 'b', value: 2 }, { key: 'gone' }]) // missed while closed
 
-    network.store.reset()
-    const second = composite(persistent)
-    expect(await second.child('a').get()).toBe(1) // available before contacting the server
-    await vi.waitFor(async () => expect(await second.child('b').get()).toBe(2))
-    expect(network.store.calls.scan).toBeUndefined() // resumed via Last-Event-ID, no full scan
-  })
-
-  it('falls back to a full sync when the server cannot resume', async () => {
-    network = new TestNetwork({ historySize: 1 })
-    const persistent = new KVStorageInMemory()
-    const first = composite(persistent)
-    await first.child('a').set(1)
-    await first.synced()
-    await first.close()
-    await network.client().writeMany([['b', 2]])
-    await network.client().writeMany([['c', 3]])
-    await network.client().removeMany(['a'])
-    const second = composite(persistent)
-    await vi.waitFor(async () => expect(await collect(second.keys())).toEqual(['b', 'c']))
-  })
-
-  it('replays an outbox left by a previous session', async () => {
-    network = new TestNetwork()
-    const persistent = new KVStorageInMemory()
-    const first = composite(persistent)
-    await first.synced()
     network.online = false
-    await first.child('offline').set('queued')
+    const second = composite(persistent, queue)
+    expect(await second.get('a')).toBe(1) // available without the server
+    network.online = true
+    await second.sync()
+    expect(await collect(second.scan({ values: true }))).toEqual([
+      ['a', 1],
+      ['b', 2],
+    ])
+  })
+
+  it('replays writes queued by a previous session from a persistent queue', async () => {
+    network = new TestNetwork()
+    const persistent = new KVStorageInMemory()
+    const queue = persistent.child('~sync')
+    const first = new KVStorageComposite({ source: network.client(), replicas: [persistent], queue, prefix: 'data' })
+    composites.push(first)
+    await first.sync()
+    network.online = false
+    await first.child('data').set('offline', 'queued')
     await first.close()
     network.online = true
-    const second = composite(persistent)
-    await second.start()
-    await vi.waitFor(async () => expect(await network.store.readMany(['offline'])).toEqual(['queued']))
+    const second = new KVStorageComposite({ source: network.client(), replicas: [persistent], queue, prefix: 'data' })
+    composites.push(second)
+    expect(second.queued).toBe(0)
+    await second.sync()
+    expect(second.queued).toBe(0)
+    expect(await network.store.readMany(['data/offline'])).toEqual(['queued'])
+  })
+
+  it('catches up on changes missed while the stream was down', async () => {
+    network = new TestNetwork()
+    const db = composite()
+    await db.sync()
+    const resynced = vi.fn()
+    db.on('change', resynced)
+    network.online = false
+    network.dropStreams()
+    await network.store.writeMany([{ key: 'missed', value: 1 }])
+    network.online = true
+    await vi.waitFor(async () => expect(await db.get('missed')).toBe(1), { timeout: 5000 })
+    expect(resynced).toHaveBeenCalledWith({ changes: [{ key: 'missed', value: 1 }], origin: 'remote' })
+  })
+
+  it('syncs only its prefix and rejects keys outside it', async () => {
+    network = new TestNetwork()
+    await network.store.writeMany([
+      { key: 'alice/a', value: 1 },
+      { key: 'bob/b', value: 2 },
+    ])
+    const db = new KVStorageComposite({ source: network.client(), replicas: [new KVStorageInMemory()], prefix: 'alice' })
+    composites.push(db)
+    await db.sync()
+    expect(await collect(db.scan({ values: true }))).toEqual([['alice/a', 1]])
+    await expect(db.child('bob').get('b')).rejects.toBeInstanceOf(RangeError)
+  })
+
+  it('refuses a queue inside the synced range of a replica', () => {
+    const replica = new KVStorageInMemory()
+    expect(() => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [replica], queue: replica.child('q') })).toThrow(/outside/)
+    expect(() => new KVStorageComposite({ source: new KVStorageInMemory(), replicas: [replica], queue: replica.child('q'), prefix: 'data' })).not.toThrow()
   })
 
   it('serves document queries from the local replica, offline too', async () => {
@@ -137,12 +185,12 @@ describe('KVStorageComposite sync', () => {
     const users = new DocumentStorageKV(db).child('users')
     await users.set('1', { age: 30 })
     await users.set('2', { age: 40 })
-    await db.synced()
+    await db.sync()
     network.online = false
     expect((await collect(users.find({ age: { $gt: 35 } }))).map((r) => r.key)).toEqual(['2'])
   })
 
-  it('does not undo writes made while a full sync is scanning', async () => {
+  it('does not undo writes made while a sync is scanning', async () => {
     // A source whose scan reads ahead (like the HTTP stream) and pauses after its first entry.
     let release!: () => void
     let reached!: () => void
@@ -150,8 +198,8 @@ describe('KVStorageComposite sync', () => {
     const paused = new Promise<void>((r) => (reached = r))
     const inner = new KVStorageInMemory()
     await inner.writeMany([
-      ['a', 1],
-      ['c', 3],
+      { key: 'a', value: 1 },
+      { key: 'c', value: 3 },
     ])
     const source = new KVStorageCounting(inner)
     const scan = source.scan.bind(source)
@@ -163,13 +211,15 @@ describe('KVStorageComposite sync', () => {
       }
     }
     const persistent = new KVStorageInMemory()
-    await persistent.writeMany([['a0', 'stale']]) // a replica copy from before, without sync metadata
+    await persistent.writeMany([{ key: 'a0', value: 'stale' }])
     const db = new KVStorageComposite({ source, replicas: [persistent] })
+    composites.push(db)
     void db.start()
     await paused // the diff has passed 'b' in the source but not in the replica
-    await db.writeMany([['b', 2]])
+    await db.set('b', 2)
     release()
-    await db.synced()
+    await vi.waitFor(() => expect(db.queued).toBe(0))
+    await db.sync()
     expect(await collect(db.scan({ values: true }))).toEqual([
       ['a', 1],
       ['b', 2],

test/kv-storage/kv-storage-http-auth.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import type { KVChangeEvent } from '../../src/kv-storage/base-kv-storage.js'
 import { KVStorageComposite } from '../../src/kv-storage/kv-storage-composite.js'
+import { KVStorageHttpServer } from '../../src/kv-storage/kv-storage-http-server.js'
 import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
 import { AuthError } from '../../src/utils/errors.js'
 import { Ed25519Signer, Ed25519Verifier } from '../../src/utils/request-auth.js'
@@ -16,9 +17,9 @@ beforeEach(async () => {
   alice = await Ed25519Signer.generate()
   bob = await Ed25519Signer.generate()
   await network.store.writeMany([
-    [`${alice.publicKey}/profile`, { name: 'Alice' }],
-    [`${bob.publicKey}/profile`, { name: 'Bob' }],
-    [`${bob.publicKey}/posts/1`, { title: 'secret' }],
+    { key: `${alice.publicKey}/profile`, value: { name: 'Alice' } },
+    { key: `${bob.publicKey}/profile`, value: { name: 'Bob' } },
+    { key: `${bob.publicKey}/posts/1`, value: { title: 'secret' } },
   ])
 })
 afterEach(() => network.close())
@@ -67,22 +68,9 @@ describe('authenticated KVStorageHttpServer', () => {
     await expect(network.client(alice).subscribe('')).rejects.toMatchObject({ status: 403 })
   })
 
-  it('replays only matching changes when resuming', async () => {
-    const watcher = network.client(alice)
-    const mine = watcher.child(alice.publicKey)
-    await mine.subscribe()
-    await mine.unsubscribe()
-    await network.client(bob).child(bob.publicKey).set('x', 1)
-    await network.client(alice).child(alice.publicKey).set('y', 2)
-    const replayed: KVChangeEvent[] = []
-    watcher.on('change', (e) => replayed.push(e))
-    await mine.subscribe()
-    expect(replayed.flatMap((e) => e.changes)).toEqual([{ key: `${alice.publicKey}/y`, value: 2 }])
-  })
-
   it('a composite scoped to the user syncs only their data', async () => {
     const db = new KVStorageComposite({ source: network.client(alice), prefix: alice.publicKey, replicas: [new KVStorageInMemory()] })
-    await db.synced()
+    await db.sync()
     expect(await collect(db.scan({ values: true }))).toEqual([[`${alice.publicKey}/profile`, { name: 'Alice' }]])
     expect(await collect(db.keys())).toEqual([alice.publicKey])
     await expect(db.child(bob.publicKey).get('profile')).rejects.toBeInstanceOf(RangeError)
@@ -92,4 +80,15 @@ describe('authenticated KVStorageHttpServer', () => {
     expect((await pushed).changes).toEqual([{ key: `${alice.publicKey}/fromElsewhere`, value: 1 }])
     await db.close()
   })
+
+  it('a composite undoes a local write the server refuses', async () => {
+    const replica = new KVStorageInMemory()
+    const db = new KVStorageComposite({ source: network.client(alice), prefix: alice.publicKey, replicas: [replica] })
+    await db.sync()
+    await db.child(alice.publicKey).set('x', 'before')
+    network.server = new KVStorageHttpServer(network.store, { auth: { verifiers: [{ algorithm: 'ed25519', verify: async () => false }] } }) // now rejects every signature
+    await expect(db.child(alice.publicKey).set('x', 'after')).rejects.toMatchObject({ status: 401 })
+    expect(await db.child(alice.publicKey).get('x')).toBe('before')
+    await db.close()
+  })
 })

test/kv-storage/kv-storage-http-watch.test.ts

@@ -7,49 +7,47 @@ let network: TestNetwork
 afterEach(() => network?.close())
 
 describe('KVStorageHttp.subscribe (SSE)', () => {
-  it('receives changes made by other clients as remote change events', async () => {
+  it('relays every change under the path as remote change events, its own writes included', async () => {
     network = new TestNetwork()
     const watcher = network.client()
-    await watcher.subscribe('')
-    const received = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await watcher.child('users').subscribe()
+    const received: KVChangeEvent[] = []
+    watcher.on('change', (e) => e.origin === 'remote' && received.push(e))
+    const other = watcher.waitFor('change', (e) => e.origin === 'remote')
     await network.client().child('users').set('1', { name: 'Ada', n: 1n })
-    const event = await received
-    expect(event.changes).toEqual([{ key: 'users/1', value: { name: 'Ada', n: 1n } }])
-    expect(event.id).toMatch(/^\w+-1$/)
-    expect(watcher.lastEventId).toBe(event.id)
+    await other
+    const own = watcher.waitFor('change', (e) => e.origin === 'remote')
+    await network.client().set('elsewhere', 1) // not subscribed
+    await watcher.child('users').delete('1')
+    await own
+    expect(received.map((e) => e.changes)).toEqual([[{ key: 'users/1', value: { name: 'Ada', n: 1n } }], [{ key: 'users/1' }]])
   })
 
-  it('resumes from lastEventId and replays missed changes', async () => {
+  it('emits resync when a dropped stream reconnects', async () => {
     network = new TestNetwork()
-    const writer = network.client()
     const watcher = network.client()
     await watcher.subscribe('')
-    const first = watcher.waitFor('change', (e) => e.origin === 'remote')
-    await writer.writeMany([['a', 1]])
-    await first
-    await watcher.unsubscribe('')
-    await writer.writeMany([['b', 2]])
-    await writer.writeMany([['c', 3]])
-    const replayed: KVChangeEvent[] = []
-    watcher.on('change', (e) => replayed.push(e))
-    await watcher.subscribe('') // resolves once caught up, so the missed changes have been emitted
-    expect(replayed.map((e) => e.changes)).toEqual([[{ key: 'b', value: 2 }], [{ key: 'c', value: 3 }]])
+    const resync = watcher.waitFor('resync', () => true)
+    network.dropStreams()
+    await resync
+    const after = watcher.waitFor('change', () => true)
+    await network.client().set('a', 1)
+    expect((await after).changes).toEqual([{ key: 'a', value: 1 }])
   })
 
-  it('asks for a resync when the missed changes are no longer in the history', async () => {
-    network = new TestNetwork({ historySize: 1 })
-    const writer = network.client()
+  it('stops relaying after unsubscribe', async () => {
+    network = new TestNetwork()
     const watcher = network.client()
-    await watcher.subscribe('')
-    const seen = watcher.waitFor('change', () => true)
-    await writer.writeMany([['a', 1]])
-    await seen
-    await watcher.unsubscribe('')
-    await writer.writeMany([['b', 2]])
-    await writer.writeMany([['c', 3]])
-    const resync = watcher.waitFor('resync', () => true)
-    await watcher.subscribe('')
-    expect(await resync).toEqual({ id: expect.stringMatching(/-3$/) })
+    await watcher.subscribe('a')
+    await watcher.subscribe('b')
+    await watcher.unsubscribe('a')
+    const received: KVChangeEvent[] = []
+    watcher.on('change', (e) => received.push(e))
+    const arrived = watcher.waitFor('change', () => true)
+    await network.client().set('a', 1)
+    await network.client().set('b', 2)
+    await arrived
+    expect(received.flatMap((e) => e.changes)).toEqual([{ key: 'b', value: 2 }])
   })
 
   it('reports an unreachable server as StorageUnavailableError', async () => {

test/kv-storage/kv-storage-http.test.ts

@@ -30,7 +30,7 @@ describe('over a real socket', () => {
 
   it('stops a long scan early', async () => {
     const db = fresh()
-    await db.writeMany(Array.from({ length: 5000 }, (_, i) => [`k${String(i).padStart(5, '0')}`, i] as [string, number]))
+    await db.writeMany(Array.from({ length: 5000 }, (_, i) => ({ key: `k${String(i).padStart(5, '0')}`, value: i })))
     const seen: string[] = []
     for await (const [key] of db.scan()) if (seen.push(key) === 3) break
     expect(seen).toEqual(['k00000', 'k00001', 'k00002'])

test/kv-storage/kv-storage-in-memory.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest'
 import fc from 'fast-check'
 import { KVStorageInMemory } from '../../src/kv-storage/kv-storage-in-memory.js'
-import type { BaseKVStorage } from '../../src/kv-storage/base-kv-storage.js'
+import type { BaseKVStorage, KVChange } from '../../src/kv-storage/base-kv-storage.js'
 import { collect, KVStorageCounting } from '../helpers.js'
 import { runKVStorageConformance } from './kv-storage.conformance.js'
 
@@ -16,14 +16,14 @@ describe('costs', () => {
     await users.get('1')
     await users.child('1').get()
     await users.delete('1')
-    expect(db.calls).toEqual({ writeMany: 1, readMany: 2, removeMany: 1 })
+    expect(db.calls).toEqual({ writeMany: 2, readMany: 2 })
   })
 
   it('keys() seeks once per child with descendants, not once per key', async () => {
     const db = new KVStorageCounting()
-    const writes: [string, number][] = []
-    for (let c = 0; c < 50; c++) for (let d = 0; d < 20; d++) writes.push([`n/c${c}/d${d}`, d])
-    for (let c = 0; c < 50; c++) writes.push([`n/leaf${c}`, c])
+    const writes: KVChange[] = []
+    for (let c = 0; c < 50; c++) for (let d = 0; d < 20; d++) writes.push({ key: `n/c${c}/d${d}`, value: d })
+    for (let c = 0; c < 50; c++) writes.push({ key: `n/leaf${c}`, value: c })
     await db.writeMany(writes)
     db.reset()
     expect(await collect(db.child('n').keys())).toHaveLength(100)

test/kv-storage/kv-storage.conformance.ts

@@ -15,14 +15,14 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
     describe('primitives', () => {
       beforeEach(async () => {
         await db.writeMany([
-          ['b', 2],
-          ['a/2', { n: 'a2' }],
-          ['c', [3]],
-          ['a', 1],
-          ['a/1', 'a1'],
-          ['a0', null],
-          ['10', 10],
-          ['9', 9],
+          { key: 'b', value: 2 },
+          { key: 'a/2', value: { n: 'a2' } },
+          { key: 'c', value: [3] },
+          { key: 'a', value: 1 },
+          { key: 'a/1', value: 'a1' },
+          { key: 'a0', value: null },
+          { key: '10', value: 10 },
+          { key: '9', value: 9 },
         ])
       })
 
@@ -48,16 +48,9 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
         expect(await db.readMany(['b', 'c'])).toEqual([undefined, [3]])
       })
 
-      it('writeIfAbsent writes only absent keys, and only one concurrent claim wins', async () => {
-        expect(
-          await db.writeIfAbsent([
-            ['a', 'x'],
-            ['new', 'y'],
-          ]),
-        ).toEqual([false, true])
-        expect(await db.readMany(['a', 'new'])).toEqual([1, 'y'])
-        const results = await Promise.all([1, 2, 3].map((n) => db.writeIfAbsent([['claim', n]])))
-        expect(results.flat().filter(Boolean)).toHaveLength(1)
+      it('writes and removes in one batch', async () => {
+        await db.writeMany([{ key: 'b' }, { key: 'new', value: 'y' }, { key: 'a', value: 'x' }])
+        expect(await db.readMany(['a', 'b', 'new'])).toEqual(['x', undefined, 'y'])
       })
 
       it('keeps scanning correctly when keys change mid-scan', async () => {
@@ -74,22 +67,18 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
       it('emits change events for writes and removals', async () => {
         const events: unknown[] = []
         const off = db.on('change', (event) => events.push(event))
-        await db.writeMany([['x', 1]])
+        await db.writeMany([{ key: 'x', value: 1 }])
         await db.removeMany(['x'])
-        await db.writeIfAbsent([
-          ['a', 'no'],
-          ['y', 2],
-        ])
+        await db.writeMany([])
         off()
         expect(events).toEqual([
           { changes: [{ key: 'x', value: 1 }], origin: 'local' },
           { changes: [{ key: 'x' }], origin: 'local' },
-          { changes: [{ key: 'y', value: 2 }], origin: 'local' },
         ])
       })
 
       it('stores bigint and Uint8Array values', async () => {
-        await db.writeMany([['bin', { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }]])
+        await db.writeMany([{ key: 'bin', value: { n: 10n ** 20n, bytes: new Uint8Array([0, 255]) } }])
         expect(await db.readMany(['bin'])).toEqual([{ n: 10n ** 20n, bytes: new Uint8Array([0, 255]) }])
       })
 
@@ -123,6 +112,14 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
         expect(await n.getMany(['a', 'b'])).toEqual([undefined, 2])
       })
 
+      it('sets and gets direct children as fields', async () => {
+        const user = db.child('users').child('1')
+        await user.setFields({ name: 'Ada', hobby: 'chess', skipped: undefined })
+        await user.child('address').set('city', 'Paris')
+        expect(await user.get('hobby')).toBe('chess')
+        expect(await user.getFields()).toEqual({ hobby: 'chess', name: 'Ada' })
+      })
+
       it('keeps own values and children independent', async () => {
         const user = db.child('users').child('1')
         await user.set('own')
@@ -157,7 +154,7 @@ export function runKVStorageConformance(name: string, make: () => BaseKVStorage
         await n.set('c', 3)
         await n.child('c').child('x').set('y', 1)
         await n.set('a-1', 4)
-        await db.writeMany([['n/~e/000000000001', {}]])
+        await db.writeMany([{ key: 'n/~e/000000000001', value: {} }])
         expect(await collect(n.keys())).toEqual(['a', 'a-1', 'b', 'c'])
         expect(await collect(n.keys({ reverse: true }))).toEqual(['c', 'b', 'a-1', 'a'])
         expect(await collect(n.keys({ after: 'a', limit: 2 }))).toEqual(['a-1', 'b'])

test/kv-storage/sync-helpers.ts

@@ -7,12 +7,14 @@ import { KVStorageCounting } from '../helpers.js'
 /**
  * An in-process server with a network switch. While `online` is false, new requests fail like a
  * dropped connection (open SSE streams stay up, so pushes can still be simulated).
+ * `dropStreams()` ends every open SSE stream, as a proxy or server restart would.
  */
 export class TestNetwork {
   online = true
   readonly store = new KVStorageCounting(new KVStorageInMemory())
-  readonly server: KVStorageHttpServer
+  server: KVStorageHttpServer
   private readonly clients: KVStorageHttp[] = []
+  private readonly streams = new Set<() => void>()
 
   constructor(options: Partial<KVStorageHttpServerOptions> = {}) {
     this.server = new KVStorageHttpServer(this.store, { auth: 'none', ...options })
@@ -24,13 +26,43 @@ export class TestNetwork {
       signer,
       fetch: async (url, init) => {
         if (!this.online) throw new TypeError('fetch failed')
-        return this.server.handle(new Request(url, init))
+        const response = await this.server.handle(new Request(url, init))
+        return url.endsWith('/events') && response.body ? this.droppable(response) : response
       },
     })
     this.clients.push(client)
     return client
   }
 
+  dropStreams(): void {
+    for (const drop of this.streams) drop()
+    this.streams.clear()
+  }
+
+  private droppable(response: Response): Response {
+    const reader = response.body!.getReader()
+    let drop!: () => void
+    const body = new ReadableStream<Uint8Array>({
+      start: (controller) => {
+        drop = () => {
+          controller.close()
+          void reader.cancel()
+        }
+        this.streams.add(drop)
+      },
+      pull: async (controller) => {
+        const { done, value } = await reader.read()
+        if (done) controller.close()
+        else controller.enqueue(value)
+      },
+      cancel: async () => {
+        this.streams.delete(drop)
+        await reader.cancel()
+      },
+    })
+    return new Response(body, { status: response.status, headers: response.headers })
+  }
+
   async close(): Promise<void> {
     await Promise.all(this.clients.map((c) => c.close()))
   }

test/types.test-d.ts

@@ -84,6 +84,6 @@ describe('event and document types', () => {
       new DocumentStorageInMemory(),
     )
     expectTypeOf(docs.child('users').history('1')).toEqualTypeOf<AsyncIterable<StoredEvent<DocChange>>>()
-    expectTypeOf(docs.child('users').getAt('1', 3)).toEqualTypeOf<Promise<User | undefined>>()
+    expectTypeOf(docs.child('users').getAt('1', 'id')).toEqualTypeOf<Promise<User | undefined>>()
   })
 })

test/utils/building-blocks.test.ts

@@ -1,9 +1,6 @@
-import { describe, expect, it, vi } from 'vitest'
+import { describe, expect, it } from 'vitest'
 import fc from 'fast-check'
 import { KeyRange } from '../../src/utils/key-range.js'
-import { KeyCounter } from '../../src/utils/key-counter.js'
-import { SerialQueue } from '../../src/utils/serial-queue.js'
-import { RetryTimer } from '../../src/utils/retry-timer.js'
 import { Keys } from '../../src/utils/keys.js'
 
 describe('KeyRange', () => {
@@ -36,54 +33,3 @@ describe('KeyRange', () => {
     expect(new KeyRange('').covers(undefined, undefined)).toBe(true)
   })
 })
-
-describe('KeyCounter', () => {
-  it('counts per key and in total', () => {
-    const counter = new KeyCounter()
-    counter.add(['a', 'a', 'b'])
-    expect(counter.size).toBe(3)
-    expect(counter.take('a')).toBe(true)
-    expect(counter.has('a')).toBe(true)
-    expect(counter.take('a')).toBe(true)
-    expect(counter.has('a')).toBe(false)
-    expect(counter.take('a')).toBe(false)
-    expect(counter.size).toBe(1)
-    counter.clear()
-    expect(counter.size).toBe(0)
-  })
-})
-
-describe('SerialQueue', () => {
-  it('runs tasks in order per key, keys independently, and survives failures', async () => {
-    const queue = new SerialQueue()
-    const log: string[] = []
-    const task = (name: string, ms: number) => () => new Promise<void>((r) => setTimeout(() => (log.push(name), r()), ms))
-    const a1 = queue.run(task('a1', 20), 'a')
-    const failed = queue.run(() => Promise.reject(new Error('boom')), 'a')
-    const a2 = queue.run(task('a2', 1), 'a')
-    const b1 = queue.run(task('b1', 1), 'b')
-    await expect(failed).rejects.toThrow('boom')
-    await Promise.all([a1, a2, b1])
-    expect(log).toEqual(['b1', 'a1', 'a2'])
-    await queue.idle('a')
-  })
-})
-
-describe('RetryTimer', () => {
-  it('replaces a pending retry and stops cleanly', () => {
-    vi.useFakeTimers()
-    const timer = new RetryTimer({ baseDelay: 10, maxDelay: 10 })
-    const first = vi.fn()
-    const second = vi.fn()
-    timer.schedule(first)
-    timer.schedule(second)
-    vi.advanceTimersByTime(20)
-    expect(first).not.toHaveBeenCalled()
-    expect(second).toHaveBeenCalledOnce()
-    timer.schedule(first)
-    timer.stop()
-    vi.advanceTimersByTime(20)
-    expect(first).not.toHaveBeenCalled()
-    vi.useRealTimers()
-  })
-})