WIP: plain operations, simpler composite (does not build or pass tests yet)
In progress, not for review: - KV adapters reduced to readMany/scan/write; writeIfAbsent removed - server executes operations verbatim; SSE relays changes, no history - composite rewritten as a combination of stores with a queue store - client-id echo tagging (to be removed per review) Events, event-sourced documents, tests and docs still to be updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE
This commit is contained in:
Claude
@@ -1,37 +1,33 @@
|
||||
import { StorageUnavailableError } from '../utils/errors.js'
|
||||
import type { OffCallback } from '../utils/event-emitter.js'
|
||||
import { KeyCounter } from '../utils/key-counter.js'
|
||||
import { Json } from '../utils/json.js'
|
||||
import { KeyRange } from '../utils/key-range.js'
|
||||
import { RetryTimer, type RetryTimerOptions } from '../utils/retry-timer.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { SerialQueue } from '../utils/serial-queue.js'
|
||||
import { BaseKVStorage, type KVChange, type KVChangeEvent, type ScanOptions } from './base-kv-storage.js'
|
||||
import { KVOutbox, type OutboxEntry } from './kv-outbox.js'
|
||||
import { KVReplicaSet, SYNC_INTERNAL } from './kv-replica-set.js'
|
||||
import { Ulid } from '../utils/ulid.js'
|
||||
import { BaseKVStorage, type KVChange, type ScanOptions } from './base-kv-storage.js'
|
||||
import { KVStorageInMemory } from './kv-storage-in-memory.js'
|
||||
|
||||
export interface KVStorageCompositeOptions {
|
||||
/** The source of truth (e.g. `KVStorageHttp`). */
|
||||
/** The source of truth, e.g. `KVStorageHttp`. */
|
||||
source: BaseKVStorage
|
||||
/**
|
||||
* Only this key and its subtree are synced, cached and subscribed to (e.g. the user's public
|
||||
* key). Keys outside it are rejected. Default `''`: the whole source (only for sources the
|
||||
* client may read entirely).
|
||||
*/
|
||||
prefix?: string
|
||||
/**
|
||||
* Local copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be persistent:
|
||||
* it seeds the others on start and holds the offline outbox and sync metadata.
|
||||
*/
|
||||
/** Local copies, fastest first, e.g. `[inMemory, indexedDb]`. The last one seeds the others on start. */
|
||||
replicas: BaseKVStorage[]
|
||||
/** Backoff for replaying queued writes and retrying a failed sync. */
|
||||
retry?: RetryTimerOptions
|
||||
/**
|
||||
* Holds writes made while the source is unreachable (under its `writes` child) and whether a
|
||||
* sync has completed (`synced`). Use a persistent node outside `prefix`, e.g.
|
||||
* `indexedDb.child('sync')`, so queued writes survive a reload. Default: in memory.
|
||||
*/
|
||||
queue?: BaseKVStorage
|
||||
/** Only this key and its subtree are synced (e.g. the user's public key). Default: everything. */
|
||||
prefix?: string
|
||||
/** Delay before retrying when the source is unreachable. Default 5 000 ms. */
|
||||
retryMs?: number
|
||||
}
|
||||
|
||||
/** Implemented by sources that push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface SubscribableSource {
|
||||
/** Sources that push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface Subscribable {
|
||||
subscribe(path: string): Promise<void>
|
||||
unsubscribe(path: string): Promise<void>
|
||||
lastEventId: string | undefined
|
||||
}
|
||||
|
||||
// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates
|
||||
@@ -41,239 +37,201 @@ export interface KVStorageComposite<S extends Spec = AnySpec> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps local replicas of (part of) a source in sync, so reads, including `keys()`, `entries()`
|
||||
* and document queries, are local, and the app keeps working offline.
|
||||
* Combines a source with local replicas: reads come from the first replica, writes go to the
|
||||
* source and then every replica, exactly as given (no merging: the last write wins).
|
||||
*
|
||||
* - **Start** (lazily, on first use): the persistent replica seeds the faster ones. If it holds a
|
||||
* previous sync, reads are served locally straight away (possibly stale until caught up).
|
||||
* - **Staying in sync:** remote changes pushed by the source are applied to every replica, in
|
||||
* order. On restart the source resumes from the saved `lastEventId`; a full sync (diff of the
|
||||
* source against the replicas) only runs the first time, when the source can't resume
|
||||
* (`resync`), or when it can't push changes at all.
|
||||
* - **Writes** go to the source, then the replicas. When the source is unreachable they are
|
||||
* applied locally and queued in the outbox (`KVOutbox`), which replays them in order. While
|
||||
* anything is queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source.
|
||||
*
|
||||
* Write through the composite, not the source: changes made directly through the same source
|
||||
* instance (`origin: 'local'`) are not mirrored.
|
||||
* - Changes others make arrive from the source (`subscribe`) and are applied to the replicas.
|
||||
* - A write the source can't take (`StorageUnavailableError`) is applied locally and stored in
|
||||
* the `queue` store; later writes queue behind it, so order is kept.
|
||||
* - `sync()` replays the queue, then copies what changed on the source into the replicas. It runs
|
||||
* on start, when the source reconnects (`resync`), and after a retry delay while offline.
|
||||
* - Before the first successful sync, reads go to the source.
|
||||
*/
|
||||
export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly source: BaseKVStorage
|
||||
private readonly replicas: BaseKVStorage[]
|
||||
private readonly queue: BaseKVStorage
|
||||
private readonly writes: BaseKVStorage
|
||||
private readonly range: KeyRange
|
||||
private readonly replicas: KVReplicaSet
|
||||
private readonly outbox: KVOutbox
|
||||
/** Remote changes and full syncs are applied strictly in order. */
|
||||
private readonly applying = new SerialQueue()
|
||||
private readonly syncRetry: RetryTimer
|
||||
private readonly outboxRetry: RetryTimer
|
||||
/**
|
||||
* Echoes still expected per key: one per write sent to a subscribable source. A remote change to
|
||||
* such a key is the echo or older than it (the stream is in server order), so it is consumed
|
||||
* instead of applied; this avoids briefly showing a stale value.
|
||||
*/
|
||||
private readonly echoes = new KeyCounter()
|
||||
private readonly offs: OffCallback[] = []
|
||||
private readonly retryMs: number
|
||||
private readonly ids = new Ulid()
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state: {
|
||||
started?: Promise<void>
|
||||
/** The replicas hold a complete (possibly stale) copy, so reads can be served locally. */
|
||||
hasSnapshot: boolean
|
||||
snapshotTaken: () => void
|
||||
snapshot: Promise<void>
|
||||
/**
|
||||
* Keys written while a full sync runs. Its diff may have scanned the source before such a
|
||||
* write and the replica after it, so these keys are left out of the diff.
|
||||
*/
|
||||
writtenDuringSync?: Set<string>
|
||||
}
|
||||
syncing?: Promise<void>
|
||||
/** A sync has completed at some point, so the replicas hold a full copy. */
|
||||
synced: boolean
|
||||
/** Writes waiting in the queue. */
|
||||
queued: number
|
||||
/** Remote changes are applied one after another. */
|
||||
applying: Promise<unknown>
|
||||
retry?: ReturnType<typeof setTimeout>
|
||||
offs: OffCallback[]
|
||||
} = { synced: false, queued: 0, applying: Promise.resolve(), offs: [] }
|
||||
|
||||
constructor(options: KVStorageCompositeOptions) {
|
||||
super()
|
||||
if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica')
|
||||
this.source = options.source
|
||||
this.replicas = options.replicas
|
||||
this.queue = options.queue ?? new KVStorageInMemory()
|
||||
this.writes = this.queue.child('writes')
|
||||
this.range = new KeyRange(options.prefix ?? '')
|
||||
this.replicas = new KVReplicaSet(options.replicas, this.range)
|
||||
this.syncRetry = new RetryTimer(options.retry)
|
||||
this.outboxRetry = new RetryTimer(options.retry)
|
||||
this.outbox = new KVOutbox(this.replicas.persistent, (entry) => this.send(entry), this.outboxRetry)
|
||||
let snapshotTaken!: () => void
|
||||
const snapshot = new Promise<void>((resolve) => (snapshotTaken = resolve))
|
||||
this.state = { hasSnapshot: false, snapshot, snapshotTaken }
|
||||
this.retryMs = options.retryMs ?? 5000
|
||||
}
|
||||
|
||||
/** Starts syncing (also happens on first use). Resolves once local data is loaded. */
|
||||
/** Loads local data and starts syncing (also happens on first use). */
|
||||
start(): Promise<void> {
|
||||
return (this.state.started ??= this.open())
|
||||
}
|
||||
|
||||
/** Resolves once the replicas hold a complete copy and all received remote changes are applied. */
|
||||
async synced(): Promise<void> {
|
||||
await this.start()
|
||||
await this.state.snapshot
|
||||
await this.applying.idle()
|
||||
/** Replays queued writes, then brings the replicas up to date with the source. */
|
||||
sync(): Promise<void> {
|
||||
return (this.state.syncing ??= this.runSync().finally(() => (this.state.syncing = undefined)))
|
||||
}
|
||||
|
||||
/** Number of writes waiting to be sent to the source. */
|
||||
/** Number of writes waiting for the source. */
|
||||
get queued(): number {
|
||||
return this.outbox.size
|
||||
return this.state.queued
|
||||
}
|
||||
|
||||
/** Stops following the source and cancels retries. */
|
||||
async close(): Promise<void> {
|
||||
this.syncRetry.stop()
|
||||
this.outboxRetry.stop()
|
||||
for (const off of this.offs.splice(0)) off()
|
||||
if (this.subscribable) await this.subscribable.unsubscribe(this.range.prefix)
|
||||
clearTimeout(this.state.retry)
|
||||
for (const off of this.state.offs.splice(0)) off()
|
||||
if (KVStorageComposite.isSubscribable(this.source)) await this.source.unsubscribe(this.range.prefix)
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
this.checkRange(keys)
|
||||
await this.start()
|
||||
return (this.state.hasSnapshot ? this.replicas.fastest : this.source).readMany(keys)
|
||||
return (this.state.synced ? this.replicas[0]! : this.source).readMany(keys)
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
await this.start()
|
||||
if (this.state.hasSnapshot) return yield* this.replicas.scan(options)
|
||||
const scoped = this.range.clamp(options)
|
||||
if (scoped) yield* this.source.scan(scoped)
|
||||
if (scoped) yield* (this.state.synced ? this.replicas[0]! : this.source).scan(scoped)
|
||||
}
|
||||
|
||||
protected async applyWrite(entries: [string, unknown][]): Promise<void> {
|
||||
await this.write({ type: 'write', entries })
|
||||
}
|
||||
|
||||
protected async applyRemove(keys: string[]): Promise<void> {
|
||||
await this.write({ type: 'remove', keys })
|
||||
}
|
||||
|
||||
protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise<boolean[]> {
|
||||
this.checkRange(entries.map(([key]) => key))
|
||||
protected async write(changes: KVChange[]): Promise<void> {
|
||||
this.checkRange(changes.map(({ key }) => key))
|
||||
await this.start()
|
||||
const written = await this.source.writeIfAbsent(entries)
|
||||
const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value }))
|
||||
for (const { key } of changes) this.state.writtenDuringSync?.add(key)
|
||||
await this.replicas.apply(changes)
|
||||
return written
|
||||
// While syncing or with writes queued, queue this one too so the source sees writes in order.
|
||||
if (this.state.queued === 0 && !this.state.syncing) {
|
||||
try {
|
||||
await this.source.writeMany(changes)
|
||||
return this.apply(changes)
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) throw error
|
||||
}
|
||||
}
|
||||
await this.writes.set(this.ids.generate(), changes)
|
||||
this.state.queued++
|
||||
await this.apply(changes)
|
||||
this.retryLater()
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ start
|
||||
|
||||
private async open(): Promise<void> {
|
||||
const meta = await this.replicas.readMeta()
|
||||
await this.outbox.load()
|
||||
await this.replicas.seed()
|
||||
if (meta) this.snapshotTaken()
|
||||
|
||||
this.offs.push(
|
||||
this.state.synced = (await this.queue.get('synced')) === true
|
||||
for await (const _ of this.writes.keys()) this.state.queued++
|
||||
await this.seed()
|
||||
this.state.offs.push(
|
||||
this.source.on('change', (event) => {
|
||||
if (event.origin === 'remote') this.enqueue(() => this.applyRemote(event))
|
||||
if (this.outbox.size > 0) void this.outbox.replay() // the source is reachable again
|
||||
if (event.origin !== 'remote') return
|
||||
const changes = event.changes.filter(({ key }) => this.range.contains(key))
|
||||
this.state.applying = this.state.applying.then(() => this.apply(changes, true)).catch((error) => console.error(error))
|
||||
}),
|
||||
this.source.on('resync', () => this.enqueue(() => this.fullSync())),
|
||||
this.source.on('resync', () => void this.sync()),
|
||||
)
|
||||
|
||||
const source = this.subscribable
|
||||
if (source) {
|
||||
source.lastEventId = meta?.lastEventId
|
||||
// Without a resume point, catch up on the stream first so no change after the snapshot is missed.
|
||||
const subscribed = source.subscribe(this.range.prefix)
|
||||
if (!meta?.lastEventId) void subscribed.then(() => this.enqueue(() => this.fullSync())).catch((error) => console.error(error))
|
||||
else void subscribed.catch((error) => console.error(error))
|
||||
if (KVStorageComposite.isSubscribable(this.source)) {
|
||||
void this.source
|
||||
.subscribe(this.range.prefix)
|
||||
.then(() => this.sync())
|
||||
.catch((error) => console.error(error))
|
||||
} else {
|
||||
this.enqueue(() => this.fullSync())
|
||||
void this.sync()
|
||||
}
|
||||
}
|
||||
|
||||
/** Queues a sync task; failures are logged (an unreachable source is retried elsewhere). */
|
||||
private enqueue(task: () => Promise<void>): void {
|
||||
this.applying.run(task).catch((error) => {
|
||||
private async runSync(): Promise<void> {
|
||||
try {
|
||||
await this.replay()
|
||||
let batch: KVChange[] = []
|
||||
for await (const change of this.diff()) if (batch.push(change) >= 500) await this.apply(batch.splice(0), true)
|
||||
await this.apply(batch, true)
|
||||
await this.replay() // writes made during the diff: send them, and re-apply them over it
|
||||
await this.queue.set('synced', true)
|
||||
this.state.synced = true
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) console.error(error)
|
||||
})
|
||||
this.retryLater()
|
||||
}
|
||||
}
|
||||
|
||||
private snapshotTaken(): void {
|
||||
this.state.hasSnapshot = true
|
||||
this.state.snapshotTaken()
|
||||
/** Sends queued writes to the source in order and re-applies them locally. */
|
||||
private async replay(): Promise<void> {
|
||||
for (;;) {
|
||||
let next: [string, unknown] | undefined
|
||||
for await (const entry of this.writes.entries({ limit: 1 })) next = entry
|
||||
if (!next) return
|
||||
const changes = next[1] as KVChange[]
|
||||
await this.source.writeMany(changes)
|
||||
await this.apply(changes)
|
||||
await this.writes.delete(next[0])
|
||||
this.state.queued--
|
||||
}
|
||||
}
|
||||
|
||||
private get subscribable(): SubscribableSource | undefined {
|
||||
const source = this.source as unknown as Partial<SubscribableSource>
|
||||
return typeof source.subscribe === 'function' ? (source as SubscribableSource) : undefined
|
||||
/** What the source has that differs from the last replica (both scanned in key order). */
|
||||
private async *diff(): AsyncGenerator<KVChange> {
|
||||
const options = { gte: this.range.gte, lt: this.range.lt, values: true }
|
||||
const remote = this.source.scan(options)[Symbol.asyncIterator]()
|
||||
const local = this.replicas.at(-1)!.scan(options)[Symbol.asyncIterator]()
|
||||
let r = await remote.next()
|
||||
let l = await local.next()
|
||||
while (!r.done || !l.done) {
|
||||
if (l.done || (!r.done && r.value[0] < l.value[0])) {
|
||||
yield { key: r.value![0], value: r.value![1] }
|
||||
r = await remote.next()
|
||||
} else if (r.done || l.value[0] < r.value[0]) {
|
||||
yield { key: l.value[0] }
|
||||
l = await local.next()
|
||||
} else {
|
||||
if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] }
|
||||
;[r, l] = await Promise.all([remote.next(), local.next()])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes changes to every replica; `remote` ones are also announced as `change` events. */
|
||||
private async apply(changes: KVChange[], remote = false): Promise<void> {
|
||||
if (changes.length === 0) return
|
||||
await Promise.all(this.replicas.map((replica) => replica.writeMany(changes)))
|
||||
if (remote) this.emit('change', { changes, origin: 'remote' })
|
||||
}
|
||||
|
||||
/** Copies the last (persistent) replica into the faster ones. */
|
||||
private async seed(): Promise<void> {
|
||||
if (this.replicas.length < 2) return
|
||||
let batch: KVChange[] = []
|
||||
const flush = () => Promise.all(this.replicas.slice(0, -1).map((replica) => replica.writeMany(batch.splice(0))))
|
||||
for await (const [key, value] of this.replicas.at(-1)!.scan({ gte: this.range.gte, lt: this.range.lt, values: true })) {
|
||||
if (batch.push({ key, value }) >= 500) await flush()
|
||||
}
|
||||
await flush()
|
||||
}
|
||||
|
||||
private retryLater(): void {
|
||||
clearTimeout(this.state.retry)
|
||||
this.state.retry = setTimeout(() => void this.sync(), this.retryMs)
|
||||
;(this.state.retry as { unref?: () => void }).unref?.()
|
||||
}
|
||||
|
||||
private checkRange(keys: string[]): void {
|
||||
for (const key of keys) if (!this.range.contains(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.range.prefix}"`)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ source → replicas
|
||||
|
||||
private async applyRemote(event: KVChangeEvent): Promise<void> {
|
||||
const relevant = event.changes.filter(
|
||||
({ key }) => this.range.contains(key) && !SYNC_INTERNAL.contains(key) && !this.echoes.take(key) && !this.outbox.pending.has(key),
|
||||
)
|
||||
const changes = await this.replicas.unapplied(relevant)
|
||||
await this.replicas.apply(changes)
|
||||
if (event.id) await this.replicas.writeMeta({ lastEventId: event.id })
|
||||
if (changes.length) this.emit('change', { changes, origin: 'remote', id: event.id })
|
||||
}
|
||||
|
||||
/** Makes the replicas match the source (within the range). */
|
||||
private async fullSync(): Promise<void> {
|
||||
// The diff re-reads the source, so outstanding echoes no longer matter (and may never arrive).
|
||||
this.echoes.clear()
|
||||
const writtenDuringSync = (this.state.writtenDuringSync = new Set())
|
||||
const skip = ({ key }: KVChange) => this.outbox.pending.has(key) || writtenDuringSync.has(key)
|
||||
try {
|
||||
const lastEventId = this.subscribable?.lastEventId
|
||||
let batch: KVChange[] = []
|
||||
const flush = async () => {
|
||||
const changes = batch.filter((change) => !skip(change))
|
||||
await this.replicas.apply(changes)
|
||||
if (changes.length) this.emit('change', { changes, origin: 'remote' })
|
||||
batch = []
|
||||
}
|
||||
for await (const change of this.replicas.diff(this.source)) if (batch.push(change) >= 500) await flush()
|
||||
await flush()
|
||||
await this.replicas.writeMeta({ lastEventId })
|
||||
this.snapshotTaken()
|
||||
this.syncRetry.succeeded()
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) throw error
|
||||
this.syncRetry.schedule(() => this.enqueue(() => this.fullSync()))
|
||||
} finally {
|
||||
this.state.writtenDuringSync = undefined
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ writes → source
|
||||
|
||||
private async write(entry: OutboxEntry): Promise<void> {
|
||||
const keys = KVOutbox.keysOf(entry)
|
||||
this.checkRange(keys)
|
||||
await this.start()
|
||||
for (const key of keys) this.state.writtenDuringSync?.add(key)
|
||||
if (this.outbox.size === 0) {
|
||||
try {
|
||||
await this.send(entry)
|
||||
return this.replicas.apply(KVOutbox.changesOf(entry))
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) throw error
|
||||
}
|
||||
}
|
||||
// Offline, or earlier writes are still queued: apply locally and queue for the source.
|
||||
await this.outbox.push(entry)
|
||||
await this.replicas.apply(KVOutbox.changesOf(entry))
|
||||
}
|
||||
|
||||
/** Sends a write to the source, expecting its echo when the source pushes changes. */
|
||||
private async send(entry: OutboxEntry): Promise<void> {
|
||||
const keys = this.subscribable ? KVOutbox.keysOf(entry) : []
|
||||
this.echoes.add(keys)
|
||||
try {
|
||||
await (entry.type === 'write' ? this.source.writeMany(entry.entries) : this.source.removeMany(entry.keys))
|
||||
} catch (error) {
|
||||
for (const key of keys) this.echoes.take(key)
|
||||
throw error
|
||||
}
|
||||
private static isSubscribable(source: BaseKVStorage): source is BaseKVStorage & Subscribable {
|
||||
return typeof (source as Partial<Subscribable>).subscribe === 'function'
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user