M6: EventSourcedDocumentStorage with projections, update operators, queries, planner, unique indexes, catch-up

Writes append resolved deltas with optimistic concurrency and self-heal lagging
projections. Reads hit projections only. Index-backed find is property-tested
against unindexed find.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE
This commit is contained in:
Claude
2026-10-23 07:18:03 +00:00
parent 67f0ba11f7
commit 0c4e35b9b6
14 changed files with 1864 additions and 3 deletions
+367
View File
@@ -0,0 +1,367 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import fc from 'fast-check'
import { KVStorage } from '../../src/kv/node.js'
import { InMemoryKVDriver } from '../../src/kv/memory.js'
import { KVEventStorage } from '../../src/events/kv.js'
import { EventSourcedDocumentStorage, type EventSourcedDocumentStorageOptions } from '../../src/documents/event-sourced.js'
import type { DocIndexDeclarations } from '../../src/documents/indexes.js'
import type { Filter } from '../../src/documents/filter.js'
import { CapabilityError, UniqueConstraintError } from '../../src/errors.js'
import { collect, CountingDriver, type Capability } from '../helpers.js'
type User = { name?: string; email?: string; age?: number; tags?: string[]; logins?: number; address?: { city?: string } }
const userIndexes: DocIndexDeclarations = {
users: [{ fields: ['email'], unique: true }, { fields: ['age'] }, { name: 'cityAge', fields: ['address.city', 'age'] }, { fields: ['tags'] }],
'users/*/posts': [{ fields: ['createdAt'] }],
}
function setup(opts: { hide?: Capability[]; inner?: InMemoryKVDriver } & EventSourcedDocumentStorageOptions = {}) {
const { hide = [], inner = new InMemoryKVDriver(), ...docOpts } = opts
const driver = new CountingDriver(inner, hide)
const kv = new KVStorage(driver)
const events = new KVEventStorage(kv)
const docs = new EventSourcedDocumentStorage(events, { indexes: userIndexes, warnOnFullScan: false, ...docOpts })
return { driver, kv, events, docs, users: docs.child('users') }
}
afterEach(() => vi.restoreAllMocks())
describe.each([
['atomic batch', [] as Capability[]],
['non-atomic', ['batch'] as Capability[]],
])('EventSourcedDocumentStorage (%s)', (_, hide) => {
describe('CRUD', () => {
it('inserts with time-sortable ids and reads back', async () => {
const { users } = setup({ hide })
const a = await users.insert({ name: 'Ada', age: 36 })
const b = await users.insert({ name: 'Bob', age: 20 })
expect(a).toMatch(/^[0-9A-HJKMNP-TV-Z]{26}$/)
expect(a < b).toBe(true)
expect(await users.get(a)).toEqual({ name: 'Ada', age: 36 })
expect(await collect(users.keys())).toEqual([a, b])
})
it('set / get / delete in self and child forms', async () => {
const { users } = setup({ hide })
await users.set('123', { name: 'Ada', age: 36 })
expect(await users.child('123').get()).toEqual({ name: 'Ada', age: 36 })
await users.child('123').set({ name: 'Ada L.' })
expect(await users.get('123')).toEqual({ name: 'Ada L.' })
expect(await users.delete('123')).toBe(true)
expect(await users.delete('123')).toBe(false)
expect(await users.get('123')).toBeUndefined()
expect(await collect(users.keys())).toEqual([])
await users.set('123', { name: 'Again' })
expect(await users.get('123')).toEqual({ name: 'Again' })
})
it('applies arity rules', async () => {
const { users, docs } = setup({ hide })
const id: string | undefined = undefined
await expect(users.get(id as never)).rejects.toThrow(TypeError)
await expect(users.update(id as never, { $set: { a: 1 } })).rejects.toThrow(TypeError)
await expect(users.delete(id as never)).rejects.toThrow(TypeError)
await expect(docs.get()).rejects.toThrow(/root/)
await expect(users.set('x', [] as never)).rejects.toThrow(/plain object/)
})
it('nests documents under documents', async () => {
const { users } = setup({ hide })
await users.set('u1', { name: 'Ada' })
const posts = users.child('u1').child('posts')
await posts.set('p1', { title: 'Hello', createdAt: '2024-01' })
expect(await posts.get('p1')).toEqual({ title: 'Hello', createdAt: '2024-01' })
expect((await collect(users.find())).map((r) => r.key)).toEqual(['u1'])
expect((await collect(posts.find({ createdAt: { $gte: '2024' } }))).map((r) => r.key)).toEqual(['p1'])
})
})
describe('update', () => {
it('supports $set $unset $inc $push $pull $addToSet with dot paths', async () => {
const { users } = setup({ hide })
await users.set('1', { name: 'Ada', tags: ['a'], address: { city: 'X' }, nickname: 'A' } as User)
const r = await users.update('1', {
$set: { name: 'Ada L.', 'address.zip': '123' },
$inc: { logins: 1 },
$push: { tags: { $each: ['b', 'c'] } },
$unset: { nickname: true },
})
expect(r).toEqual({ name: 'Ada L.', tags: ['a', 'b', 'c'], address: { city: 'X', zip: '123' }, logins: 1 })
await users.update('1', { $pull: { tags: { $in: ['a', 'c'] } }, $addToSet: { 'address.list': { $each: [1, 1, 2] } } })
await users.child('1').update({ $inc: { logins: 5 }, $addToSet: { tags: 'b' } })
expect(await users.get('1')).toEqual({ name: 'Ada L.', tags: ['b'], address: { city: 'X', zip: '123', list: [1, 2] }, logins: 6 })
})
it('returns undefined for missing docs unless upsert', async () => {
const { users } = setup({ hide })
expect(await users.update('nope', { $set: { a: 1 } })).toBeUndefined()
expect(await users.get('nope')).toBeUndefined()
expect(await users.update('nope', { $inc: { n: 2 } }, { upsert: true })).toEqual({ n: 2 })
})
it('rejects conflicting paths and bad operators', async () => {
const { users } = setup({ hide })
await users.set('1', { age: 1 })
await expect(users.update('1', { $set: { a: 1 }, $unset: { 'a.b': 1 } })).rejects.toThrow(/conflict/)
await expect(users.update('1', { $rename: {} } as never)).rejects.toThrow(/Unsupported/)
await expect(users.update('1', { $inc: { name: 1 } }).then(() => users.update('1', { $push: { age: 1 } }))).rejects.toThrow(/non-array/)
})
it('stores resolved deltas, not operators, and skips no-op updates', async () => {
const { users } = setup({ hide })
await users.set('1', { logins: 5 })
await users.update('1', { $inc: { logins: 1 } })
await users.update('1', { $set: { logins: 6 } }) // no-op
expect((await collect(users.history('1'))).map((e) => e.data)).toEqual([
{ op: 'set', doc: { logins: 5 } },
{ op: 'patch', set: { logins: 6 } },
])
})
})
describe('history and getAt', () => {
it('replays to any version', async () => {
const { users } = setup({ hide })
await users.set('1', { n: 1 })
await users.update('1', { $inc: { n: 1 } })
await users.delete('1')
await users.set('1', { n: 10 })
expect(await users.getAt('1', 0)).toBeUndefined()
expect(await users.getAt('1', 1)).toEqual({ n: 1 })
expect(await users.getAt('1', 2)).toEqual({ n: 2 })
expect(await users.getAt('1', 3)).toBeUndefined()
expect(await users.child('1').getAt(4)).toEqual({ n: 10 })
expect(await users.getAt('1', 99)).toEqual({ n: 10 })
expect((await collect(users.child('1').history())).map((e) => e.version)).toEqual([1, 2, 3, 4])
})
})
describe('find', () => {
async function seed() {
const ctx = setup({ hide })
const rows: [string, User][] = [
['a', { name: 'Ada', age: 36, email: 'ada@x', tags: ['x', 'y'], address: { city: 'London' } }],
['b', { name: 'Bob', age: 20, email: 'bob@x', tags: ['y'], address: { city: 'Paris' } }],
['c', { name: 'Cy', age: 51, tags: [], address: { city: 'London' } }],
['d', { name: 'Di', age: 20, email: 'di@x' }],
['e', { name: 'Ed', email: 'ed@x', address: { city: 'London' } }],
]
for (const [k, v] of rows) await ctx.users.set(k, v)
await ctx.users.set('gone', { name: 'Gone', age: 20 })
await ctx.users.delete('gone')
return ctx
}
const keysOf = async (it: AsyncIterable<{ key: string }>) => (await collect(it)).map((r) => r.key)
it('evaluates comparison, logical and element operators', async () => {
const { users } = await seed()
// Unsorted results come in index or key order depending on the plan; compare as sets.
const q = async (f: Filter) => (await keysOf(users.find(f))).sort()
expect(await q({})).toEqual(['a', 'b', 'c', 'd', 'e'])
expect(await q({ age: 20 })).toEqual(['b', 'd'])
expect(await q({ age: { $gte: 20, $lt: 40 } })).toEqual(['a', 'b', 'd'])
expect(await q({ age: { $gt: 20 } })).toEqual(['a', 'c'])
expect(await q({ age: { $ne: 20 } })).toEqual(['a', 'c', 'e'])
expect(await q({ age: { $in: [36, 51] } })).toEqual(['a', 'c'])
expect(await q({ age: { $nin: [36, 51] } })).toEqual(['b', 'd', 'e'])
expect(await q({ age: { $exists: false } })).toEqual(['e'])
expect(await q({ age: null })).toEqual(['e'])
expect(await q({ email: { $exists: true }, age: { $lte: 20 } })).toEqual(['b', 'd'])
expect(await q({ 'address.city': 'London' })).toEqual(['a', 'c', 'e'])
expect(await q({ 'address.city': 'London', age: { $gt: 40 } })).toEqual(['c'])
expect(await q({ tags: 'y' })).toEqual(['a', 'b'])
expect(await q({ tags: ['y'] })).toEqual(['b'])
expect(await q({ $or: [{ age: 51 }, { name: 'Bob' }] })).toEqual(['b', 'c'])
expect(await q({ $and: [{ age: { $gte: 20 } }, { age: { $lte: 20 } }] })).toEqual(['b', 'd'])
expect(await q({ $not: { age: 20 } })).toEqual(['a', 'c', 'e'])
expect(await q({ age: { $not: { $gte: 30 } } })).toEqual(['b', 'd', 'e'])
expect(await q({ name: { $gt: 5 } })).toEqual([]) // type-bracketed
})
it('sorts (index-backed and in-memory), skips, limits and projects', async () => {
const { users } = await seed()
expect(await keysOf(users.find({}, { sort: { age: 1 } }))).toEqual(['e', 'b', 'd', 'a', 'c'])
expect(await keysOf(users.find({}, { sort: { age: -1 } }))).toEqual(['c', 'a', 'd', 'b', 'e'])
expect(await keysOf(users.find({}, { sort: { name: -1 }, skip: 1, limit: 2 }))).toEqual(['d', 'c'])
expect(await keysOf(users.find({ age: { $gte: 20 } }, { sort: { age: -1 }, limit: 2 }))).toEqual(['c', 'a'])
expect(await keysOf(users.find({ 'address.city': 'London' }, { sort: { age: 1 } }))).toEqual(['e', 'a', 'c'])
expect(await collect(users.find({ age: 36 }, { projection: { name: 1, 'address.city': 1 } }))).toEqual([{ key: 'a', data: { name: 'Ada', address: { city: 'London' } } }])
expect(await collect(users.find({ age: 36 }, { projection: { tags: 0, address: 0, email: 0 } }))).toEqual([{ key: 'a', data: { name: 'Ada', age: 36 } }])
expect(await users.findOne({ email: 'bob@x' })).toEqual({ key: 'b', data: expect.objectContaining({ name: 'Bob' }) })
expect(await users.findOne({ email: 'nobody' })).toBeUndefined()
})
it('uses an index: findOne costs one entry scan and one projection read', async () => {
const { users, driver } = await seed()
driver.reset()
await users.findOne({ email: 'di@x' })
expect(driver.calls).toEqual({ scan: 1, getMany: 1 })
})
it('warns once per filter shape on full scans', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { users } = setup({ hide, warnOnFullScan: true })
await collect(users.find({ name: 'x' }))
await collect(users.find({ name: 'y' }))
await collect(users.find({ age: 1 }))
expect(warn).toHaveBeenCalledTimes(1)
expect(warn.mock.calls[0]![0]).toMatch(/full scan for filter \{"name":"\?"\}/)
})
})
describe('unique indexes', () => {
it('rejects duplicates and frees values on change and delete', async () => {
const { users } = setup({ hide })
await users.set('1', { email: 'a@x' })
await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError)
await expect(users.insert({ email: 'a@x' })).rejects.toMatchObject({ index: 'email', existingKey: '1' })
expect(await collect(users.keys())).toEqual(['1'])
await users.update('1', { $set: { email: 'b@x' } })
await users.set('2', { email: 'a@x' })
await users.delete('2')
await users.set('3', { email: 'a@x' })
await users.set('4', {}) // missing values are not claimed
await users.set('5', {})
await users.set('3', { email: 'a@x', name: 'same value, same doc' })
})
it('exactly one of concurrent writers claiming the same value wins', async () => {
const { users } = setup({ hide })
const results = await Promise.allSettled(['1', '2', '3'].map((k) => users.set(k, { email: 'same' })))
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(1)
expect((await collect(users.find({ email: 'same' }))).length).toBe(1)
})
it('takes over a stale claim only after the grace period', async () => {
let t = 0
const { users, driver } = setup({ hide, now: () => t, claimGraceMs: 1000 })
// Leave an orphaned claim: doc 1's projection and events vanish but its claim remains.
await users.set('1', { email: 'a@x' })
await driver.inner.deleteMany(['~p/users/1'])
await driver.inner.deleteMany((await collect(driver.inner.scan({ gte: 'users/1/', lt: 'users/10' }))).map(([k]) => k))
await expect(users.set('2', { email: 'a@x' })).rejects.toBeInstanceOf(UniqueConstraintError)
t = 5000
await users.set('2', { email: 'a@x' })
expect(await users.get('2')).toEqual({ email: 'a@x' })
})
it('requires setIfAbsent', () => {
const kv = new KVStorage(new CountingDriver(new InMemoryKVDriver(), ['setIfAbsent']))
expect(() => new EventSourcedDocumentStorage(new KVEventStorage(kv), { indexes: userIndexes })).toThrow(CapabilityError)
})
})
describe('projection lag (crash between append and projection write)', () => {
async function lagged() {
const ctx = setup({ hide })
await ctx.users.set('1', { n: 1, email: 'a@x' })
// Simulate a crash: the event is appended but the projection is never written.
await ctx.events.child('users').append('1', [{ op: 'patch', set: { n: 2, email: 'b@x' } }])
return ctx
}
it('reads see the stale projection; consistent reads catch up', async () => {
const { users } = await lagged()
expect(await users.get('1')).toEqual({ n: 1, email: 'a@x' })
expect(await users.get('1', { consistent: true })).toEqual({ n: 2, email: 'b@x' })
expect(await users.findOne({ email: 'b@x' })).toMatchObject({ key: '1' })
expect(await users.findOne({ email: 'a@x' })).toBeUndefined()
})
it('the next write self-heals', async () => {
const { users } = await lagged()
expect(await users.update('1', { $inc: { n: 1 } })).toEqual({ n: 3, email: 'b@x' })
expect((await collect(users.history('1'))).map((e) => e.version)).toEqual([1, 2, 3])
await users.set('2', { email: 'a@x' }) // old claim was released by the catch-up
})
it('catchUp() repairs a subtree', async () => {
const { users, docs, events } = await lagged()
await events.child('users').child('9').child('posts').append('p', [{ op: 'set', doc: { createdAt: 'x' } }])
expect(await docs.catchUp()).toBe(2)
expect(await users.get('1')).toEqual({ n: 2, email: 'b@x' })
expect(await users.child('9').child('posts').get('p')).toEqual({ createdAt: 'x' })
expect(await docs.catchUp()).toBe(0)
})
})
describe('concurrency', () => {
it('two storage instances incrementing the same doc converge', async () => {
const inner = new InMemoryKVDriver()
const a = setup({ hide, inner }).users
const b = setup({ hide, inner }).users
await a.set('1', { n: 0 })
await Promise.all([...Array(5)].flatMap(() => [a.update('1', { $inc: { n: 1 } }), b.update('1', { $inc: { n: 1 } })]))
expect(await a.get('1')).toEqual({ n: 10 })
expect(await b.get('1', { consistent: true })).toEqual({ n: 10 })
})
})
})
describe('costs', () => {
it('get is one driver call; update is projection read + append + projection write, even cold', async () => {
const inner = new InMemoryKVDriver()
await setup({ inner }).users.set('1', { name: 'a', age: 1 })
const { users, driver } = setup({ inner }) // fresh instance: cold head cache
driver.reset()
await users.get('1')
expect(driver.calls).toEqual({ getMany: 1 })
driver.reset()
await users.update('1', { $set: { age: 2 } })
expect(driver.calls).toEqual({ getMany: 1, setIfAbsent: 1, batch: 1 })
})
})
describe('property: indexed find equals unindexed find', () => {
const doc = fc.record(
{
age: fc.option(fc.integer({ min: 0, max: 5 }), { nil: undefined }),
email: fc.constantFrom('a', 'b', 'c', 'd', 'e', 'f'),
tags: fc.array(fc.constantFrom('x', 'y', 'z'), { maxLength: 3 }),
address: fc.option(fc.record({ city: fc.constantFrom('L', 'P') }), { nil: undefined }),
},
{ requiredKeys: [] },
)
const bound = fc.integer({ min: -1, max: 6 })
const filter: fc.Arbitrary<Filter> = fc.oneof(
fc.record({ age: bound }),
fc.record({ age: fc.record({ $gte: bound, $lt: bound }, { requiredKeys: [] }) }),
fc.record({ age: fc.record({ $gt: bound, $lte: bound }, { requiredKeys: [] }) }),
fc.record({ tags: fc.constantFrom('x', 'y', 'z') }),
fc.record({ tags: fc.record({ $gte: fc.constantFrom('x', 'y'), $lt: fc.constantFrom('y', 'zz') }) }),
fc.record({ 'address.city': fc.constantFrom('L', 'P'), age: fc.record({ $gt: bound }) }),
fc.record({ 'address.city': fc.constantFrom('L', 'P') }),
fc.constant({ age: null }),
fc.constant({}),
)
const sort = fc.option(fc.record({ age: fc.constantFrom(1 as const, -1 as const) }), { nil: undefined })
it('returns the same results in the same order', async () => {
await fc.assert(
fc.asyncProperty(fc.array(fc.tuple(fc.constantFrom('1', '2', '3', '4', '5'), doc), { maxLength: 12 }), filter, sort, fc.option(fc.integer({ min: 1, max: 4 }), { nil: undefined }), async (writes, f, s, limit) => {
const plain = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), { warnOnFullScan: false }).child('users')
const indexed = new EventSourcedDocumentStorage(new KVEventStorage(new KVStorage(new InMemoryKVDriver())), {
warnOnFullScan: false,
indexes: { users: userIndexes.users!.map((i) => ({ ...i, unique: false })) },
}).child('users')
for (const [k, v] of writes) {
await plain.set(k, v)
await indexed.set(k, v)
}
const opts = { ...(s ? { sort: s } : {}), ...(limit ? { limit } : {}) }
const a = await collect(plain.find(f, opts))
const b = await collect(indexed.find(f, opts))
if (s) {
expect(b).toEqual(a)
return
}
// Unsorted order depends on the plan: compare as sets (a limited result is any subset of that size).
const all = new Set((await collect(plain.find(f))).map((r) => r.key))
expect(b).toHaveLength(a.length)
for (const r of b) expect(all.has(r.key)).toBe(true)
}),
{ numRuns: 300 },
)
})
})
+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest'
import { createIdGenerator } from '../../src/documents/ulid.js'
import { matches } from '../../src/documents/filter.js'
import { applyPatch, resolveUpdate } from '../../src/documents/update.js'
describe('ulid', () => {
it('is strictly increasing within one millisecond and sorts by time', () => {
let t = 1_700_000_000_000
const gen = createIdGenerator(() => t)
const ids = Array.from({ length: 1000 }, () => gen())
expect([...ids].sort()).toEqual(ids)
expect(new Set(ids).size).toBe(1000)
t += 1
const later = gen()
expect(later > ids.at(-1)!).toBe(true)
expect(later.slice(0, 10) > ids[0]!.slice(0, 10)).toBe(true)
})
})
describe('filter semantics', () => {
it('requires one array element to satisfy all positive operators of a condition', () => {
expect(matches({ s: [1, 20] }, { s: { $gt: 5, $lt: 10 } })).toBe(false) // MongoDB would match
expect(matches({ s: [1, 7] }, { s: { $gt: 5, $lt: 10 } })).toBe(true)
expect(matches({ s: [1, 20] }, { $and: [{ s: { $gt: 5 } }, { s: { $lt: 10 } }] })).toBe(true)
})
it('traverses arrays of sub-documents on dot paths', () => {
expect(matches({ a: [{ b: 1 }, { b: 2 }] }, { 'a.b': 2 })).toBe(true)
expect(matches({ a: [{ b: 1 }] }, { 'a.0.b': 1 })).toBe(true)
expect(matches({ a: [{ b: 1 }] }, { 'a.b': { $exists: true } })).toBe(true)
expect(matches({ a: [] }, { 'a.b': { $exists: true } })).toBe(false)
})
it('treats null as matching missing', () => {
expect(matches({}, { x: null })).toBe(true)
expect(matches({ x: null }, { x: { $ne: null } })).toBe(false)
expect(matches({ x: 0 }, { x: { $ne: null } })).toBe(true)
})
it('rejects unknown operators', () => {
expect(() => matches({}, { x: { $regex: 'a' } })).toThrow(/Unknown operator/)
expect(() => matches({}, { $nor: [] } as never)).toThrow(/Unknown top-level/)
})
})
describe('update resolution', () => {
it('$pull by sub-document filter and by condition', () => {
const state = { items: [{ k: 1, v: 'a' }, { k: 2, v: 'b' }], n: [1, 5, 9] }
const patch = resolveUpdate(state, { $pull: { items: { k: 2 }, n: { $gte: 5 } } })
expect(applyPatch(state, patch)).toEqual({ items: [{ k: 1, v: 'a' }], n: [1] })
})
it('$unset of an array position nulls it', () => {
expect(applyPatch({ a: [1, 2] }, resolveUpdate({ a: [1, 2] }, { $unset: { 'a.0': true } }))).toEqual({ a: [null, 2] })
})
})