|
|
|
@@ -0,0 +1,288 @@
|
|
|
|
|
import { LRU } from '../core/lru.js'
|
|
|
|
|
import { KeyedMutex } from '../core/mutex.js'
|
|
|
|
|
import { escapeSegment, RESERVED, succ, toSegment } from '../core/path.js'
|
|
|
|
|
import { ConcurrencyError } from '../errors.js'
|
|
|
|
|
import type { KVDriver } from '../kv/driver.js'
|
|
|
|
|
import type { KVNode } from '../kv/node.js'
|
|
|
|
|
import type { AnySpec, ChildOf, KeyOf, KeyString, Spec, ValueOf } from '../types/schema.js'
|
|
|
|
|
import type { AppendOptions, AppendResult, EventNode, ReadOptions, StoredEvent, StreamKeysOptions } from './types.js'
|
|
|
|
|
|
|
|
|
|
export interface KVEventStorageOptions {
|
|
|
|
|
/** Max streams whose head is cached. Default 10 000. */
|
|
|
|
|
headCacheSize?: number
|
|
|
|
|
/** Retries after a commit collision when `expectedVersion` is `'any'`. Default 5. */
|
|
|
|
|
maxRetries?: number
|
|
|
|
|
/** Clock for commit timestamps. Default `Date.now`. */
|
|
|
|
|
now?: () => number
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Stored value of one commit key. */
|
|
|
|
|
export interface CommitRecord<E = unknown> {
|
|
|
|
|
firstVersion: number
|
|
|
|
|
at: number
|
|
|
|
|
events: E[]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface Head {
|
|
|
|
|
commit: number
|
|
|
|
|
version: number
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface Stream {
|
|
|
|
|
/** Stream's own key (escaped); identifies it in the cache and in errors. */
|
|
|
|
|
key: string
|
|
|
|
|
/** `<key>/~e/` — commit keys are `base + pad12(seq)`. */
|
|
|
|
|
base: string
|
|
|
|
|
/** Exclusive upper bound of commit keys. */
|
|
|
|
|
end: string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @internal */
|
|
|
|
|
interface EventContext {
|
|
|
|
|
readonly driver: KVDriver
|
|
|
|
|
readonly sep: string
|
|
|
|
|
readonly heads: LRU<string, Head>
|
|
|
|
|
readonly mutex: KeyedMutex
|
|
|
|
|
readonly maxRetries: number
|
|
|
|
|
readonly now: () => number
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const SEQ_WIDTH = 12
|
|
|
|
|
const SEARCH_FANOUT = 8
|
|
|
|
|
const EMPTY_HEAD: Head = { commit: 0, version: 0 }
|
|
|
|
|
|
|
|
|
|
const pad = (seq: number) => String(seq).padStart(SEQ_WIDTH, '0')
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Event node over a `KVNode`. Each append is one commit key, densely numbered:
|
|
|
|
|
* `<stream>/~e/<commitSeq:12>` → `{ firstVersion, at, events }`. Commits are claimed with
|
|
|
|
|
* `setIfAbsent`, so any stale writer collides with the existing key. Without `setIfAbsent`,
|
|
|
|
|
* appends are serialised by an in-process per-stream mutex (single-writer only).
|
|
|
|
|
*/
|
|
|
|
|
export class KVEventNode<S extends Spec = AnySpec> implements EventNode<S> {
|
|
|
|
|
readonly #ctx: EventContext
|
|
|
|
|
readonly #kv: KVNode<AnySpec>
|
|
|
|
|
|
|
|
|
|
/** @internal Use `KVEventStorage` and `child()`. */
|
|
|
|
|
constructor(ctx: EventContext, kv: KVNode<AnySpec>) {
|
|
|
|
|
this.#ctx = ctx
|
|
|
|
|
this.#kv = kv
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
get path(): readonly string[] {
|
|
|
|
|
return this.#kv.path
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** The underlying KV node. */
|
|
|
|
|
get kv(): KVNode<AnySpec> {
|
|
|
|
|
return this.#kv
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
child<K extends KeyOf<S>>(key: K): KVEventNode<ChildOf<S, K>> {
|
|
|
|
|
return new KVEventNode(this.#ctx, this.#kv.child(key as string).as<AnySpec>())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
as<S2 extends Spec>(): KVEventNode<S2> {
|
|
|
|
|
return this as unknown as KVEventNode<S2>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
append(events: readonly ValueOf<S>[], opts?: AppendOptions): Promise<AppendResult>
|
|
|
|
|
append<K extends KeyOf<S>>(key: K, events: readonly ValueOf<ChildOf<S, K>>[], opts?: AppendOptions): Promise<AppendResult>
|
|
|
|
|
async append(...args: unknown[]): Promise<AppendResult> {
|
|
|
|
|
const self = Array.isArray(args[0])
|
|
|
|
|
const stream = self ? this.#self() : this.#child(args[0])
|
|
|
|
|
const events = args[self ? 0 : 1]
|
|
|
|
|
const opts = (args[self ? 1 : 2] ?? {}) as AppendOptions
|
|
|
|
|
if (!Array.isArray(events)) throw new TypeError('append() expects an array of events')
|
|
|
|
|
const expected = opts.expectedVersion ?? 'any'
|
|
|
|
|
if (expected !== 'any' && !(Number.isInteger(expected) && expected >= 0)) throw new TypeError(`Invalid expectedVersion ${String(expected)}`)
|
|
|
|
|
return this.#append(stream, events, expected)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
read(opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<S>>>
|
|
|
|
|
read<K extends KeyOf<S>>(key: K, opts?: ReadOptions): AsyncIterable<StoredEvent<ValueOf<ChildOf<S, K>>>>
|
|
|
|
|
async *read(...args: unknown[]): AsyncGenerator<StoredEvent<any>> {
|
|
|
|
|
const self = args.length === 0 || (typeof args[0] === 'object' && args[0] !== null)
|
|
|
|
|
const stream = self ? this.#self() : this.#child(args[0])
|
|
|
|
|
yield* this.#read(stream, ((self ? args[0] : args[1]) ?? {}) as ReadOptions)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
version(): Promise<number>
|
|
|
|
|
version<K extends KeyOf<S>>(key: K): Promise<number>
|
|
|
|
|
async version(...args: unknown[]): Promise<number> {
|
|
|
|
|
const stream = args.length === 0 ? this.#self() : this.#child(args[0])
|
|
|
|
|
return (await this.#loadHead(stream)).version
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
keys(opts: StreamKeysOptions = {}): AsyncIterable<KeyString<S>> {
|
|
|
|
|
return this.#kv.keys(opts) as AsyncIterable<KeyString<S>>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- internals
|
|
|
|
|
|
|
|
|
|
#self(): Stream {
|
|
|
|
|
if (this.#kv.isRoot) throw new TypeError('The root node is not a stream')
|
|
|
|
|
return this.#streamAt(this.#kv.selfKey)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#child(key: unknown): Stream {
|
|
|
|
|
return this.#streamAt(this.#kv.prefix + escapeSegment(toSegment(key), this.#ctx.sep))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#streamAt(key: string): Stream {
|
|
|
|
|
const sep = this.#ctx.sep
|
|
|
|
|
const head = key + sep + RESERVED.events
|
|
|
|
|
return { key, base: head + sep, end: head + succ(sep) }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async #loadHead(s: Stream): Promise<Head> {
|
|
|
|
|
for await (const [key, value] of this.#ctx.driver.scan({ gte: s.base, lt: s.end, reverse: true, limit: 1, values: true })) {
|
|
|
|
|
const c = value as CommitRecord
|
|
|
|
|
const head = { commit: Number(key.slice(s.base.length)), version: c.firstVersion + c.events.length - 1 }
|
|
|
|
|
this.#ctx.heads.set(s.key, head)
|
|
|
|
|
return head
|
|
|
|
|
}
|
|
|
|
|
return EMPTY_HEAD
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async #append(s: Stream, events: readonly unknown[], expected: number | 'any'): Promise<AppendResult> {
|
|
|
|
|
const { driver, heads, maxRetries } = this.#ctx
|
|
|
|
|
if (events.length === 0) {
|
|
|
|
|
const head = await this.#loadHead(s)
|
|
|
|
|
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
|
|
|
|
return { version: head.version, commit: head.commit }
|
|
|
|
|
}
|
|
|
|
|
if (!driver.setIfAbsent) return this.#ctx.mutex.run(s.key, () => this.#appendSingleWriter(s, events, expected))
|
|
|
|
|
|
|
|
|
|
for (let attempt = 0; ; attempt++) {
|
|
|
|
|
let head = heads.get(s.key)
|
|
|
|
|
if (!head) head = await this.#loadHead(s)
|
|
|
|
|
else if (expected !== 'any' && head.version !== expected) head = await this.#loadHead(s) // cache may be stale
|
|
|
|
|
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
|
|
|
|
|
|
|
|
|
const seq = head.commit + 1
|
|
|
|
|
const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] }
|
|
|
|
|
const [written] = await driver.setIfAbsent([[s.base + pad(seq), record]])
|
|
|
|
|
if (written) {
|
|
|
|
|
const next = { commit: seq, version: head.version + events.length }
|
|
|
|
|
heads.set(s.key, next)
|
|
|
|
|
return next
|
|
|
|
|
}
|
|
|
|
|
// Lost the race for this commit number.
|
|
|
|
|
heads.delete(s.key)
|
|
|
|
|
if (expected !== 'any' || attempt >= maxRetries) {
|
|
|
|
|
throw new ConcurrencyError(s.key, expected, (await this.#loadHead(s)).version)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async #appendSingleWriter(s: Stream, events: readonly unknown[], expected: number | 'any'): Promise<AppendResult> {
|
|
|
|
|
const head = this.#ctx.heads.get(s.key) ?? (await this.#loadHead(s))
|
|
|
|
|
if (expected !== 'any' && head.version !== expected) throw new ConcurrencyError(s.key, expected, head.version)
|
|
|
|
|
const seq = head.commit + 1
|
|
|
|
|
const record: CommitRecord = { firstVersion: head.version + 1, at: this.#ctx.now(), events: [...events] }
|
|
|
|
|
await this.#ctx.driver.setMany([[s.base + pad(seq), record]])
|
|
|
|
|
const next = { commit: seq, version: head.version + events.length }
|
|
|
|
|
this.#ctx.heads.set(s.key, next)
|
|
|
|
|
return next
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Sequence number of the commit containing `version`, or 0 if the stream hasn't reached it.
|
|
|
|
|
* Commit `n` has `firstVersion >= n`, so the answer is at most `version`: probe that first
|
|
|
|
|
* (one round trip when commits hold single events), then k-ary search with `getMany` probes.
|
|
|
|
|
*/
|
|
|
|
|
async #locate(s: Stream, version: number): Promise<number> {
|
|
|
|
|
const { driver } = this.#ctx
|
|
|
|
|
const [guess] = (await driver.getMany([s.base + pad(version)])) as (CommitRecord | undefined)[]
|
|
|
|
|
if (guess && guess.firstVersion <= version) return version
|
|
|
|
|
let hi: number
|
|
|
|
|
if (guess) hi = version - 1
|
|
|
|
|
else {
|
|
|
|
|
const head = await this.#loadHead(s)
|
|
|
|
|
if (head.version < version) return 0
|
|
|
|
|
hi = head.commit
|
|
|
|
|
}
|
|
|
|
|
let lo = 1 // commit 1 starts at version 1
|
|
|
|
|
while (lo < hi) {
|
|
|
|
|
const probes: number[] = []
|
|
|
|
|
for (let i = 1; i <= SEARCH_FANOUT; i++) {
|
|
|
|
|
const p = lo + Math.ceil(((hi - lo) * i) / SEARCH_FANOUT)
|
|
|
|
|
if (p !== probes.at(-1)) probes.push(p)
|
|
|
|
|
}
|
|
|
|
|
const found = (await driver.getMany(probes.map((p) => s.base + pad(p)))) as (CommitRecord | undefined)[]
|
|
|
|
|
for (let i = 0; i < probes.length; i++) {
|
|
|
|
|
const c = found[i]
|
|
|
|
|
if (c && c.firstVersion <= version) lo = probes[i]!
|
|
|
|
|
else {
|
|
|
|
|
hi = probes[i]! - 1
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return lo
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async *#read(s: Stream, opts: ReadOptions): AsyncGenerator<StoredEvent<unknown>> {
|
|
|
|
|
const from = Math.max(1, opts.from ?? 1)
|
|
|
|
|
const to = opts.to ?? Infinity
|
|
|
|
|
let remaining = opts.limit ?? Infinity
|
|
|
|
|
if (to < from || remaining <= 0) return
|
|
|
|
|
const { driver } = this.#ctx
|
|
|
|
|
|
|
|
|
|
if (!opts.reverse) {
|
|
|
|
|
let gte = s.base
|
|
|
|
|
if (from > 1) {
|
|
|
|
|
const seq = await this.#locate(s, from)
|
|
|
|
|
if (seq === 0) return
|
|
|
|
|
gte = s.base + pad(seq)
|
|
|
|
|
}
|
|
|
|
|
for await (const [key, value] of driver.scan({ gte, lt: s.end, values: true })) {
|
|
|
|
|
const c = value as CommitRecord
|
|
|
|
|
const commit = Number(key.slice(s.base.length))
|
|
|
|
|
for (let i = 0; i < c.events.length; i++) {
|
|
|
|
|
const version = c.firstVersion + i
|
|
|
|
|
if (version < from) continue
|
|
|
|
|
if (version > to) return
|
|
|
|
|
yield { version, commit, at: c.at, data: c.events[i] }
|
|
|
|
|
if (--remaining <= 0) return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let lt = s.end
|
|
|
|
|
if (to !== Infinity) {
|
|
|
|
|
const seq = await this.#locate(s, to)
|
|
|
|
|
if (seq !== 0) lt = s.base + pad(seq + 1)
|
|
|
|
|
}
|
|
|
|
|
for await (const [key, value] of driver.scan({ gte: s.base, lt, reverse: true, values: true })) {
|
|
|
|
|
const c = value as CommitRecord
|
|
|
|
|
const commit = Number(key.slice(s.base.length))
|
|
|
|
|
for (let i = c.events.length - 1; i >= 0; i--) {
|
|
|
|
|
const version = c.firstVersion + i
|
|
|
|
|
if (version > to) continue
|
|
|
|
|
if (version < from) return
|
|
|
|
|
yield { version, commit, at: c.at, data: c.events[i] }
|
|
|
|
|
if (--remaining <= 0) return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Event storage over a `KVStorage` (or any `KVNode` as root). */
|
|
|
|
|
export class KVEventStorage<S extends Spec = AnySpec> extends KVEventNode<S> {
|
|
|
|
|
constructor(kv: KVNode<AnySpec>, opts: KVEventStorageOptions = {}) {
|
|
|
|
|
super(
|
|
|
|
|
{
|
|
|
|
|
driver: kv.driver,
|
|
|
|
|
sep: kv.separator,
|
|
|
|
|
heads: new LRU(opts.headCacheSize ?? 10_000),
|
|
|
|
|
mutex: new KeyedMutex(),
|
|
|
|
|
maxRetries: opts.maxRetries ?? 5,
|
|
|
|
|
now: opts.now ?? Date.now,
|
|
|
|
|
},
|
|
|
|
|
kv,
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
}
|
Claude