Break KVStorageComposite into focused classes
- KVReplicaSet: seed, apply, local scan, diff against the source, sync meta - KVOutbox: persisted offline queue with in-order replay (load() resumes it) - utils: KeyRange (also used for server authorization), KeyCounter, SerialQueue (also used by KVStorageHttp and the event-sourced store), RetryTimer (separate timers for sync and replay; previously one shared timer could cancel the other's retry) The composite now only orchestrates. Behaviour unchanged; new unit tests for the building blocks and the outbox. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE
This commit is contained in:
@@ -1,10 +1,13 @@
|
||||
import { StorageUnavailableError } from '../utils/errors.js'
|
||||
import type { OffCallback } from '../utils/event-emitter.js'
|
||||
import { Json } from '../utils/json.js'
|
||||
import { Keys } from '../utils/keys.js'
|
||||
import { KeyCounter } from '../utils/key-counter.js'
|
||||
import { KeyRange } from '../utils/key-range.js'
|
||||
import { RetryTimer, type RetryTimerOptions } from '../utils/retry-timer.js'
|
||||
import type { AnySpec, ChildOf, KeyOf, Spec } from '../utils/schema.js'
|
||||
import { ExponentialBackoff } from '../utils/sse-session/exponential-backoff.js'
|
||||
import { SerialQueue } from '../utils/serial-queue.js'
|
||||
import { BaseKVStorage, type KVChange, type KVChangeEvent, type ScanOptions } from './base-kv-storage.js'
|
||||
import { KVOutbox, type OutboxEntry } from './kv-outbox.js'
|
||||
import { KVReplicaSet, SYNC_INTERNAL } from './kv-replica-set.js'
|
||||
|
||||
export interface KVStorageCompositeOptions {
|
||||
/** The source of truth (e.g. `KVStorageHttp`). */
|
||||
@@ -16,115 +19,85 @@ export interface KVStorageCompositeOptions {
|
||||
*/
|
||||
prefix?: string
|
||||
/**
|
||||
* Local full copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be
|
||||
* persistent: it also stores the offline outbox and sync metadata, and seeds the others on start.
|
||||
* Local copies, fastest first (e.g. `[inMemory, indexedDb]`). The last one should be persistent:
|
||||
* it seeds the others on start and holds the offline outbox and sync metadata.
|
||||
*/
|
||||
replicas: BaseKVStorage[]
|
||||
/** Delays between attempts to replay queued writes or re-sync. */
|
||||
retry?: Partial<{ baseDelay: number; maxDelay: number }>
|
||||
/** Backoff for replaying queued writes and retrying a failed sync. */
|
||||
retry?: RetryTimerOptions
|
||||
}
|
||||
|
||||
/** Implemented by sources that can push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface WatchableSource {
|
||||
/** Implemented by sources that push remote changes (e.g. `KVStorageHttp`). */
|
||||
interface SubscribableSource {
|
||||
subscribe(path: string): Promise<void>
|
||||
unsubscribe(path: string): Promise<void>
|
||||
lastEventId: string | undefined
|
||||
}
|
||||
|
||||
type OutboxEntry = { type: 'write'; entries: [string, unknown][] } | { type: 'remove'; keys: string[] }
|
||||
|
||||
interface SyncMeta {
|
||||
lastEventId?: string
|
||||
}
|
||||
|
||||
/** Internal keys in the persistent replica; hidden from reads. */
|
||||
const INTERNAL_PREFIX = '~q/'
|
||||
const INTERNAL_END = '~q0'
|
||||
const META_KEY = '~q/meta'
|
||||
const OUTBOX_PREFIX = '~q/o/'
|
||||
const OUTBOX_END = '~q/o0'
|
||||
const SYNC_BATCH = 500
|
||||
|
||||
// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates them with
|
||||
// Object.create, so they share this class's prototype).
|
||||
// Types only: children of a KVStorageComposite are KVStorageComposite nodes too (child() creates
|
||||
// them with Object.create, so they share this class's prototype).
|
||||
export interface KVStorageComposite<S extends Spec = AnySpec> {
|
||||
child<K extends KeyOf<S>>(key: K): KVStorageComposite<ChildOf<S, K>>
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps local replicas of a source in sync, so reads (including `keys()`, `entries()` and document
|
||||
* queries) are served locally, and the app keeps working offline.
|
||||
* Keeps local replicas of (part of) a source in sync, so reads, including `keys()`, `entries()`
|
||||
* and document queries, are local, and the app keeps working offline.
|
||||
*
|
||||
* - **Start** (lazily, on first use): the persistent replica seeds the faster ones. If it holds a
|
||||
* previous sync, reads are served from it straight away (possibly stale until caught up).
|
||||
* - **Staying in sync:** remote changes pushed by the source (`subscribe(prefix)`, e.g. SSE) are applied to
|
||||
* every replica. On restart the source resumes from the saved `lastEventId`; a full sync (diffing
|
||||
* the source against the replicas) only runs the first time, or when the source says it can't
|
||||
* resume (`resync`). Without `subscribe()`, a full sync runs on start.
|
||||
* - **Writes** go to the source, then the replicas. If the source is unreachable
|
||||
* (`StorageUnavailableError`), they are applied locally and queued in a persisted outbox, then
|
||||
* replayed in order when it's reachable again (last writer wins on conflict). While any write is
|
||||
* queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source and
|
||||
* fails offline.
|
||||
* previous sync, reads are served locally straight away (possibly stale until caught up).
|
||||
* - **Staying in sync:** remote changes pushed by the source are applied to every replica, in
|
||||
* order. On restart the source resumes from the saved `lastEventId`; a full sync (diff of the
|
||||
* source against the replicas) only runs the first time, when the source can't resume
|
||||
* (`resync`), or when it can't push changes at all.
|
||||
* - **Writes** go to the source, then the replicas. When the source is unreachable they are
|
||||
* applied locally and queued in the outbox (`KVOutbox`), which replays them in order. While
|
||||
* anything is queued, later writes queue too, to keep their order. `writeIfAbsent` needs the source.
|
||||
*
|
||||
* Write through the composite, not the source: changes made directly through the same source
|
||||
* instance (`origin: 'local'`) are not mirrored.
|
||||
*/
|
||||
export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<S> {
|
||||
private readonly source: BaseKVStorage
|
||||
/** Synced key range: `[prefix, prefix + '0')` is the prefix's own key and its subtree. */
|
||||
private readonly scope: { prefix: string; gte?: string; lt?: string }
|
||||
private readonly replicas: BaseKVStorage[]
|
||||
/** Persistent replica: seeds the others and holds the outbox and metadata. */
|
||||
private readonly persistent: BaseKVStorage
|
||||
private readonly backoff: { baseDelay: number; maxDelay: number }
|
||||
/** Queued-write count per key; remote changes to these keys are ignored until replayed. */
|
||||
private readonly pending = new Map<string, number>()
|
||||
private readonly range: KeyRange
|
||||
private readonly replicas: KVReplicaSet
|
||||
private readonly outbox: KVOutbox
|
||||
/** Remote changes and full syncs are applied strictly in order. */
|
||||
private readonly applying = new SerialQueue()
|
||||
private readonly syncRetry: RetryTimer
|
||||
private readonly outboxRetry: RetryTimer
|
||||
/**
|
||||
* Echoes still expected per key: one per write this client sent to a watchable source. A remote
|
||||
* change to such a key is the echo or older than it (the stream is in server order), so it is
|
||||
* consumed instead of applied; this avoids briefly showing a stale value.
|
||||
* Echoes still expected per key: one per write sent to a subscribable source. A remote change to
|
||||
* such a key is the echo or older than it (the stream is in server order), so it is consumed
|
||||
* instead of applied; this avoids briefly showing a stale value.
|
||||
*/
|
||||
private readonly echoes = new Map<string, number>()
|
||||
private readonly echoes = new KeyCounter()
|
||||
private readonly offs: OffCallback[] = []
|
||||
/** Mutable state lives here: child nodes share this object (see `child()`). */
|
||||
private readonly state: {
|
||||
started?: Promise<void>
|
||||
/** Replicas hold a complete (possibly stale) copy, so reads can be served locally. */
|
||||
/** The replicas hold a complete (possibly stale) copy, so reads can be served locally. */
|
||||
hasSnapshot: boolean
|
||||
snapshotReady: Promise<void>
|
||||
markSnapshot: () => void
|
||||
snapshotTaken: () => void
|
||||
snapshot: Promise<void>
|
||||
/**
|
||||
* Remote changes and full syncs are applied strictly in order through this chain. Changes that
|
||||
* arrive during a full sync wait behind it, so the end state matches the source even if it
|
||||
* changed during the scan.
|
||||
*/
|
||||
applying: Promise<void>
|
||||
/**
|
||||
* Keys written through the composite while a full sync runs. The diff may have scanned the
|
||||
* source before such a write and the replica after it; these keys are left out of the diff
|
||||
* (the write already reached the source and the replicas).
|
||||
* Keys written while a full sync runs. Its diff may have scanned the source before such a
|
||||
* write and the replica after it, so these keys are left out of the diff.
|
||||
*/
|
||||
writtenDuringSync?: Set<string>
|
||||
replaying?: Promise<void>
|
||||
outboxSeq: number
|
||||
retryTimer?: ReturnType<typeof setTimeout>
|
||||
retryAttempt: number
|
||||
closed: boolean
|
||||
}
|
||||
|
||||
constructor(options: KVStorageCompositeOptions) {
|
||||
super()
|
||||
if (options.replicas.length === 0) throw new TypeError('KVStorageComposite needs at least one replica')
|
||||
this.source = options.source
|
||||
const prefix = options.prefix ?? ''
|
||||
this.scope = prefix === '' ? { prefix } : { prefix, gte: prefix, lt: prefix + Keys.afterSeparator }
|
||||
this.replicas = options.replicas
|
||||
this.persistent = options.replicas.at(-1)!
|
||||
this.backoff = { baseDelay: 1000, maxDelay: 30_000, ...options.retry }
|
||||
let markSnapshot!: () => void
|
||||
const snapshotReady = new Promise<void>((resolve) => (markSnapshot = resolve))
|
||||
this.state = { hasSnapshot: false, snapshotReady, markSnapshot, applying: Promise.resolve(), outboxSeq: 0, retryAttempt: 0, closed: false }
|
||||
this.range = new KeyRange(options.prefix ?? '')
|
||||
this.replicas = new KVReplicaSet(options.replicas, this.range)
|
||||
this.syncRetry = new RetryTimer(options.retry)
|
||||
this.outboxRetry = new RetryTimer(options.retry)
|
||||
this.outbox = new KVOutbox(this.replicas.persistent, (entry) => this.send(entry), this.outboxRetry)
|
||||
let snapshotTaken!: () => void
|
||||
const snapshot = new Promise<void>((resolve) => (snapshotTaken = resolve))
|
||||
this.state = { hasSnapshot: false, snapshot, snapshotTaken }
|
||||
}
|
||||
|
||||
/** Starts syncing (also happens on first use). Resolves once local data is loaded. */
|
||||
@@ -135,296 +108,172 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
|
||||
/** Resolves once the replicas hold a complete copy and all received remote changes are applied. */
|
||||
async synced(): Promise<void> {
|
||||
await this.start()
|
||||
await this.state.snapshotReady
|
||||
await this.state.applying
|
||||
await this.state.snapshot
|
||||
await this.applying.idle()
|
||||
}
|
||||
|
||||
/** Number of writes waiting to be replayed to the source. */
|
||||
/** Number of writes waiting to be sent to the source. */
|
||||
get queued(): number {
|
||||
let total = 0
|
||||
for (const count of this.pending.values()) total += count
|
||||
return total
|
||||
return this.outbox.size
|
||||
}
|
||||
|
||||
/** Stops watching the source and cancels retries. */
|
||||
/** Stops following the source and cancels retries. */
|
||||
async close(): Promise<void> {
|
||||
this.state.closed = true
|
||||
clearTimeout(this.state.retryTimer)
|
||||
this.syncRetry.stop()
|
||||
this.outboxRetry.stop()
|
||||
for (const off of this.offs.splice(0)) off()
|
||||
if (this.watchable) await (this.source as unknown as WatchableSource).unsubscribe(this.scope.prefix)
|
||||
if (this.subscribable) await this.subscribable.unsubscribe(this.range.prefix)
|
||||
}
|
||||
|
||||
async readMany(keys: string[]): Promise<unknown[]> {
|
||||
this.checkScope(keys)
|
||||
this.checkRange(keys)
|
||||
await this.start()
|
||||
return (this.state.hasSnapshot ? this.replicas[0]! : this.source).readMany(keys)
|
||||
return (this.state.hasSnapshot ? this.replicas.fastest : this.source).readMany(keys)
|
||||
}
|
||||
|
||||
async *scan(options: ScanOptions = {}): AsyncGenerator<[string, unknown]> {
|
||||
await this.start()
|
||||
// Narrow the range to the synced scope (and, against an authenticated server, to what it allows).
|
||||
const gte = this.scope.gte !== undefined && (options.gte === undefined || options.gte < this.scope.gte) ? this.scope.gte : options.gte
|
||||
const lt = this.scope.lt !== undefined && (options.lt === undefined || options.lt > this.scope.lt) ? this.scope.lt : options.lt
|
||||
if (gte !== undefined && lt !== undefined && gte >= lt) return
|
||||
const scoped = { ...options, gte, lt }
|
||||
if (!this.state.hasSnapshot) return yield* this.source.scan(scoped)
|
||||
// Skip internal keys before applying the limit.
|
||||
let remaining = options.limit ?? Infinity
|
||||
if (remaining <= 0) return
|
||||
for await (const entry of this.replicas[0]!.scan({ ...scoped, limit: undefined })) {
|
||||
if (entry[0] >= INTERNAL_PREFIX && entry[0] < INTERNAL_END) continue
|
||||
yield entry
|
||||
if (--remaining <= 0) return
|
||||
}
|
||||
if (this.state.hasSnapshot) return yield* this.replicas.scan(options)
|
||||
const scoped = this.range.clamp(options)
|
||||
if (scoped) yield* this.source.scan(scoped)
|
||||
}
|
||||
|
||||
protected async applyWrite(entries: [string, unknown][]): Promise<void> {
|
||||
this.checkScope(entries.map(([key]) => key))
|
||||
await this.start()
|
||||
await this.writeThrough({ type: 'write', entries })
|
||||
await this.write({ type: 'write', entries })
|
||||
}
|
||||
|
||||
protected async applyRemove(keys: string[]): Promise<void> {
|
||||
this.checkScope(keys)
|
||||
await this.start()
|
||||
await this.writeThrough({ type: 'remove', keys })
|
||||
await this.write({ type: 'remove', keys })
|
||||
}
|
||||
|
||||
protected async applyWriteIfAbsent(entries: [string, unknown][]): Promise<boolean[]> {
|
||||
this.checkScope(entries.map(([key]) => key))
|
||||
this.checkRange(entries.map(([key]) => key))
|
||||
await this.start()
|
||||
const written = await this.source.writeIfAbsent(entries)
|
||||
const changes = entries.filter((_, i) => written[i]).map(([key, value]) => ({ key, value }))
|
||||
for (const { key } of changes) this.state.writtenDuringSync?.add(key)
|
||||
await this.applyToReplicas(changes)
|
||||
await this.replicas.apply(changes)
|
||||
return written
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ start
|
||||
|
||||
private inScope(key: string): boolean {
|
||||
const { prefix } = this.scope
|
||||
return prefix === '' || key === prefix || key.startsWith(prefix + Keys.separator)
|
||||
}
|
||||
|
||||
private checkScope(keys: string[]): void {
|
||||
for (const key of keys) if (!this.inScope(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.scope.prefix}"`)
|
||||
}
|
||||
|
||||
private get watchable(): boolean {
|
||||
return typeof (this.source as unknown as Partial<WatchableSource>).subscribe === 'function'
|
||||
}
|
||||
|
||||
private async open(): Promise<void> {
|
||||
const [meta] = (await this.persistent.readMany([META_KEY])) as (SyncMeta | undefined)[]
|
||||
for await (const [key, value] of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, values: true })) {
|
||||
this.state.outboxSeq = Number(key.slice(OUTBOX_PREFIX.length))
|
||||
for (const k of KVStorageComposite.keysOf(value as OutboxEntry)) this.pending.set(k, (this.pending.get(k) ?? 0) + 1)
|
||||
}
|
||||
await this.seedFasterReplicas()
|
||||
const meta = await this.replicas.readMeta()
|
||||
await this.outbox.load()
|
||||
await this.replicas.seed()
|
||||
if (meta) this.snapshotTaken()
|
||||
|
||||
this.offs.push(
|
||||
this.source.on('change', (event) => {
|
||||
if (event.origin === 'remote') this.enqueueApply(() => this.applyRemote(event))
|
||||
if (this.queued > 0) void this.replay()
|
||||
if (event.origin === 'remote') this.enqueue(() => this.applyRemote(event))
|
||||
if (this.outbox.size > 0) void this.outbox.replay() // the source is reachable again
|
||||
}),
|
||||
this.source.on('resync', () => this.enqueueApply(() => this.fullSync())),
|
||||
this.source.on('resync', () => this.enqueue(() => this.fullSync())),
|
||||
)
|
||||
|
||||
if (this.watchable) {
|
||||
const source = this.source as unknown as WatchableSource
|
||||
const source = this.subscribable
|
||||
if (source) {
|
||||
source.lastEventId = meta?.lastEventId
|
||||
// Without a resume point, catch up on the stream first so no change after the snapshot is missed.
|
||||
const watching = source.subscribe(this.scope.prefix)
|
||||
if (!meta?.lastEventId) void watching.then(() => this.enqueueApply(() => this.fullSync())).catch((error) => console.error(error))
|
||||
else void watching.catch((error) => console.error(error))
|
||||
const subscribed = source.subscribe(this.range.prefix)
|
||||
if (!meta?.lastEventId) void subscribed.then(() => this.enqueue(() => this.fullSync())).catch((error) => console.error(error))
|
||||
else void subscribed.catch((error) => console.error(error))
|
||||
} else {
|
||||
this.enqueueApply(() => this.fullSync())
|
||||
this.enqueue(() => this.fullSync())
|
||||
}
|
||||
if (this.queued > 0) void this.replay()
|
||||
}
|
||||
|
||||
/** Copies the persistent replica into the faster ones. */
|
||||
private async seedFasterReplicas(): Promise<void> {
|
||||
const faster = this.replicas.slice(0, -1)
|
||||
if (faster.length === 0) return
|
||||
let batch: [string, unknown][] = []
|
||||
const flush = async () => {
|
||||
await Promise.all(faster.map((replica) => replica.writeMany(batch)))
|
||||
batch = []
|
||||
}
|
||||
for await (const entry of this.persistent.scan({ values: true })) {
|
||||
if (!entry[0].startsWith(INTERNAL_PREFIX) && batch.push(entry) >= SYNC_BATCH) await flush()
|
||||
}
|
||||
await flush()
|
||||
}
|
||||
|
||||
private snapshotTaken(): void {
|
||||
this.state.hasSnapshot = true
|
||||
this.state.markSnapshot()
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ remote → replicas
|
||||
|
||||
private enqueueApply(task: () => Promise<void>): void {
|
||||
this.state.applying = this.state.applying.then(task).catch((error) => {
|
||||
/** Queues a sync task; failures are logged (an unreachable source is retried elsewhere). */
|
||||
private enqueue(task: () => Promise<void>): void {
|
||||
this.applying.run(task).catch((error) => {
|
||||
if (!(error instanceof StorageUnavailableError)) console.error(error)
|
||||
})
|
||||
}
|
||||
|
||||
private snapshotTaken(): void {
|
||||
this.state.hasSnapshot = true
|
||||
this.state.snapshotTaken()
|
||||
}
|
||||
|
||||
private get subscribable(): SubscribableSource | undefined {
|
||||
const source = this.source as unknown as Partial<SubscribableSource>
|
||||
return typeof source.subscribe === 'function' ? (source as SubscribableSource) : undefined
|
||||
}
|
||||
|
||||
private checkRange(keys: string[]): void {
|
||||
for (const key of keys) if (!this.range.contains(key)) throw new RangeError(`"${key}" is outside this composite's prefix "${this.range.prefix}"`)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ source → replicas
|
||||
|
||||
private async applyRemote(event: KVChangeEvent): Promise<void> {
|
||||
const candidates = event.changes.filter(
|
||||
({ key }) => this.inScope(key) && !this.consumeEcho(key) && !this.pending.has(key) && !key.startsWith(INTERNAL_PREFIX),
|
||||
const relevant = event.changes.filter(
|
||||
({ key }) => this.range.contains(key) && !SYNC_INTERNAL.contains(key) && !this.echoes.take(key) && !this.outbox.pending.has(key),
|
||||
)
|
||||
// Skip changes the replicas already reflect.
|
||||
const current = await this.replicas[0]!.readMany(candidates.map(({ key }) => key))
|
||||
const changes = candidates.filter(({ value }, i) => !Json.equal(value, current[i]))
|
||||
await this.applyToReplicas(changes)
|
||||
if (event.id) await this.persistent.writeMany([[META_KEY, { lastEventId: event.id } satisfies SyncMeta]])
|
||||
const changes = await this.replicas.unapplied(relevant)
|
||||
await this.replicas.apply(changes)
|
||||
if (event.id) await this.replicas.writeMeta({ lastEventId: event.id })
|
||||
if (changes.length) this.emit('change', { changes, origin: 'remote', id: event.id })
|
||||
}
|
||||
|
||||
/** Diffs the source against the persistent replica and applies the differences to every replica. */
|
||||
/** Makes the replicas match the source (within the range). */
|
||||
private async fullSync(): Promise<void> {
|
||||
// The diff re-reads the source, so outstanding echoes no longer matter (and may never arrive).
|
||||
this.echoes.clear()
|
||||
const writtenDuringSync = (this.state.writtenDuringSync = new Set())
|
||||
const skip = ({ key }: KVChange) => this.outbox.pending.has(key) || writtenDuringSync.has(key)
|
||||
try {
|
||||
const id = this.watchable ? (this.source as unknown as WatchableSource).lastEventId : undefined
|
||||
let changes: KVChange[] = []
|
||||
const lastEventId = this.subscribable?.lastEventId
|
||||
let batch: KVChange[] = []
|
||||
const flush = async () => {
|
||||
const applied = changes.filter(({ key }) => !this.pending.has(key) && !writtenDuringSync.has(key))
|
||||
await this.applyToReplicas(applied)
|
||||
if (applied.length) this.emit('change', { changes: applied, origin: 'remote' })
|
||||
changes = []
|
||||
const changes = batch.filter((change) => !skip(change))
|
||||
await this.replicas.apply(changes)
|
||||
if (changes.length) this.emit('change', { changes, origin: 'remote' })
|
||||
batch = []
|
||||
}
|
||||
for await (const change of this.diff()) if (changes.push(change) >= SYNC_BATCH) await flush()
|
||||
for await (const change of this.replicas.diff(this.source)) if (batch.push(change) >= 500) await flush()
|
||||
await flush()
|
||||
await this.persistent.writeMany([[META_KEY, { lastEventId: id } satisfies SyncMeta]])
|
||||
await this.replicas.writeMeta({ lastEventId })
|
||||
this.snapshotTaken()
|
||||
this.state.retryAttempt = 0
|
||||
this.syncRetry.succeeded()
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) throw error
|
||||
this.scheduleRetry(() => this.enqueueApply(() => this.fullSync()))
|
||||
this.syncRetry.schedule(() => this.enqueue(() => this.fullSync()))
|
||||
} finally {
|
||||
this.state.writtenDuringSync = undefined
|
||||
}
|
||||
}
|
||||
|
||||
/** Changes that turn the persistent replica into the source (both scanned in key order). */
|
||||
private async *diff(): AsyncGenerator<KVChange> {
|
||||
const range = { gte: this.scope.gte, lt: this.scope.lt, values: true }
|
||||
const remote = this.source.scan(range)[Symbol.asyncIterator]()
|
||||
const local = this.persistent.scan(range)[Symbol.asyncIterator]()
|
||||
const nextLocal = async () => {
|
||||
for (let next = await local.next(); ; next = await local.next()) if (next.done || !next.value[0].startsWith(INTERNAL_PREFIX)) return next
|
||||
}
|
||||
let r = await remote.next()
|
||||
let l = await nextLocal()
|
||||
while (!r.done || !l.done) {
|
||||
if (l.done || (!r.done && r.value[0] < l.value[0])) {
|
||||
yield { key: r.value![0], value: r.value![1] }
|
||||
r = await remote.next()
|
||||
} else if (r.done || l.value[0] < r.value[0]) {
|
||||
yield { key: l.value[0] }
|
||||
l = await nextLocal()
|
||||
} else {
|
||||
if (!Json.equal(r.value[1], l.value[1])) yield { key: r.value[0], value: r.value[1] }
|
||||
r = await remote.next()
|
||||
l = await nextLocal()
|
||||
}
|
||||
}
|
||||
}
|
||||
// ------------------------------------------------------------------ writes → source
|
||||
|
||||
private async applyToReplicas(changes: KVChange[]): Promise<void> {
|
||||
const writes = changes.filter((c) => c.value !== undefined).map((c): [string, unknown] => [c.key, c.value])
|
||||
const removes = changes.filter((c) => c.value === undefined).map((c) => c.key)
|
||||
await Promise.all(this.replicas.map(async (replica) => (await replica.writeMany(writes), await replica.removeMany(removes))))
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ local writes → source
|
||||
|
||||
private async writeThrough(entry: OutboxEntry): Promise<void> {
|
||||
for (const key of KVStorageComposite.keysOf(entry)) this.state.writtenDuringSync?.add(key)
|
||||
if (this.queued === 0) {
|
||||
private async write(entry: OutboxEntry): Promise<void> {
|
||||
const keys = KVOutbox.keysOf(entry)
|
||||
this.checkRange(keys)
|
||||
await this.start()
|
||||
for (const key of keys) this.state.writtenDuringSync?.add(key)
|
||||
if (this.outbox.size === 0) {
|
||||
try {
|
||||
await this.send(entry)
|
||||
return this.applyToReplicas(KVStorageComposite.changesOf(entry))
|
||||
return this.replicas.apply(KVOutbox.changesOf(entry))
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) throw error
|
||||
}
|
||||
}
|
||||
// Offline, or earlier writes are still queued: apply locally and queue for the source.
|
||||
const seq = ++this.state.outboxSeq
|
||||
for (const key of KVStorageComposite.keysOf(entry)) this.pending.set(key, (this.pending.get(key) ?? 0) + 1)
|
||||
await this.persistent.writeMany([[OUTBOX_PREFIX + String(seq).padStart(12, '0'), entry]])
|
||||
await this.applyToReplicas(KVStorageComposite.changesOf(entry))
|
||||
void this.replay()
|
||||
await this.outbox.push(entry)
|
||||
await this.replicas.apply(KVOutbox.changesOf(entry))
|
||||
}
|
||||
|
||||
/** Sends queued writes to the source in order; stops (and retries later) if it's unreachable. */
|
||||
private replay(): Promise<void> {
|
||||
const run = async () => {
|
||||
try {
|
||||
for (;;) {
|
||||
let next: [string, unknown] | undefined
|
||||
for await (const entry of this.persistent.scan({ gte: OUTBOX_PREFIX, lt: OUTBOX_END, limit: 1, values: true })) next = entry
|
||||
if (!next) return
|
||||
const [key, entry] = next as [string, OutboxEntry]
|
||||
await this.send(entry)
|
||||
await this.persistent.removeMany([key])
|
||||
for (const k of KVStorageComposite.keysOf(entry)) {
|
||||
const count = this.pending.get(k)! - 1
|
||||
if (count > 0) this.pending.set(k, count)
|
||||
else this.pending.delete(k)
|
||||
}
|
||||
this.state.retryAttempt = 0
|
||||
}
|
||||
} catch (error) {
|
||||
if (!(error instanceof StorageUnavailableError)) console.error(error)
|
||||
this.scheduleRetry(() => void this.replay())
|
||||
} finally {
|
||||
this.state.replaying = undefined
|
||||
}
|
||||
}
|
||||
return (this.state.replaying ??= run())
|
||||
}
|
||||
|
||||
/** Sends a write to the source, expecting its echo when the source is watched. */
|
||||
/** Sends a write to the source, expecting its echo when the source pushes changes. */
|
||||
private async send(entry: OutboxEntry): Promise<void> {
|
||||
const keys = this.watchable ? KVStorageComposite.keysOf(entry) : []
|
||||
for (const key of keys) this.echoes.set(key, (this.echoes.get(key) ?? 0) + 1)
|
||||
const keys = this.subscribable ? KVOutbox.keysOf(entry) : []
|
||||
this.echoes.add(keys)
|
||||
try {
|
||||
await (entry.type === 'write' ? this.source.writeMany(entry.entries) : this.source.removeMany(entry.keys))
|
||||
} catch (error) {
|
||||
for (const key of keys) this.consumeEcho(key)
|
||||
for (const key of keys) this.echoes.take(key)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether `key` had an outstanding echo (now consumed). */
|
||||
private consumeEcho(key: string): boolean {
|
||||
const count = this.echoes.get(key)
|
||||
if (!count) return false
|
||||
if (count > 1) this.echoes.set(key, count - 1)
|
||||
else this.echoes.delete(key)
|
||||
return true
|
||||
}
|
||||
|
||||
private scheduleRetry(task: () => void): void {
|
||||
if (this.state.closed) return
|
||||
clearTimeout(this.state.retryTimer)
|
||||
const delay = ExponentialBackoff.calculateDelay({ ...this.backoff, growthRate: 2, jitter: 0.2, maxAttempts: 0 }, this.state.retryAttempt++)
|
||||
this.state.retryTimer = setTimeout(task, delay)
|
||||
;(this.state.retryTimer as { unref?: () => void }).unref?.()
|
||||
}
|
||||
|
||||
private static keysOf(entry: OutboxEntry): string[] {
|
||||
return entry.type === 'write' ? entry.entries.map(([key]) => key) : entry.keys
|
||||
}
|
||||
|
||||
private static changesOf(entry: OutboxEntry): KVChange[] {
|
||||
return entry.type === 'write' ? entry.entries.map(([key, value]) => ({ key, value })) : entry.keys.map((key) => ({ key }))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user