diff --git a/benchmarks/storage.ts b/benchmarks/storage.ts index 2460d90..3e75c7e 100644 --- a/benchmarks/storage.ts +++ b/benchmarks/storage.ts @@ -1,5 +1,6 @@ import { AESKey } from '../src/crypto/aes-key.js'; import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js'; +import { BTreeCache, KvCache } from '../src/cache/index.js'; // --------------------------------------------------------------------------- // Helpers @@ -161,9 +162,19 @@ const DOC_COUNTS = [1_000, 10_000, 50_000]; for (const count of DOC_COUNTS) { const docs = generateDocs(count); - // Indexes on id, name, AND age — range queries on age use B+ Tree. - const indexedWithAge = StorageMemory.from(['id', 'name', 'age']); - await benchmarkStorage('StorageMemory (indexed: id,name,age)', indexedWithAge, docs, { hasAgeIndex: true }); + // Indexes on id, name, AND age with explicit B+ Tree cache. + const indexedWithAgeBTree = StorageMemory.from( + ['id', 'name', 'age'], + new BTreeCache(), + ); + await benchmarkStorage('StorageMemory (B+Tree cache, indexed: id,name,age)', indexedWithAgeBTree, docs, { hasAgeIndex: true }); + + // Same indexes, but KV cache (no range support). + const indexedWithAgeKv = StorageMemory.from( + ['id', 'name', 'age'], + new KvCache(), + ); + await benchmarkStorage('StorageMemory (KV cache, indexed: id,name,age)', indexedWithAgeKv, docs); // Indexes on id, name only — range queries on age fall back to full scan. const indexed = StorageMemory.from(['id', 'name']); diff --git a/package.json b/package.json index ae99315..b680ce1 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "scripts": { "build": "tsc", "format": "prettier --write .", - "test": "echo \"Error: no test specified\" && exit 1", + + "test": "vitest --run", + "test:watch": "vitest", "benchmark:sha256": "tsx benchmarks/sha256.ts", "benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts", diff --git a/src/cache/b-tree-cache.ts b/src/cache/b-tree-cache.ts new file mode 100644 index 0000000..ec937fc --- /dev/null +++ b/src/cache/b-tree-cache.ts @@ -0,0 +1,79 @@ +import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js'; + +import { BaseCache, type CacheRangeOptions } from './base-cache.js'; + +function tupleCompare(a: any[], b: any[]): number { + const len = Math.min(a.length, b.length); + for (let i = 0; i < len; i++) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + return a.length - b.length; +} + +/** + * B+ tree-backed cache implementation. + * + * - Equality: O(log n) + * - Range: O(log n + k) + */ +export class BTreeCache extends BaseCache { + private trees = new Map>(); + + registerIndex(indexName: string, fields: string[]): void { + if (this.trees.has(indexName)) return; + const comparator = fields.length > 1 ? tupleCompare : undefined; + this.trees.set(indexName, new BPlusTree(32, comparator)); + } + + clearIndex(indexName: string): void { + const tree = this.trees.get(indexName); + if (!tree) return; + tree.clear(); + } + + insert(indexName: string, key: any, internalKey: number): void { + const tree = this.trees.get(indexName); + if (!tree) return; + tree.insert(key, internalKey); + } + + delete(indexName: string, key: any, internalKey: number): void { + const tree = this.trees.get(indexName); + if (!tree) return; + tree.delete(key, internalKey); + } + + get(indexName: string, key: any): Iterable { + const tree = this.trees.get(indexName); + if (!tree) return []; + return tree.get(key) ?? []; + } + + range( + indexName: string, + min?: any, + max?: any, + options: CacheRangeOptions = {}, + ): Iterable | null { + const tree = this.trees.get(indexName); + if (!tree) return []; + const entries = tree.range(min, max, options); + return this.flattenEntryKeys(entries); + } + + createChild(): BaseCache { + return new BTreeCache(); + } + + private flattenEntryKeys(entries: BPlusTreeEntry[]): number[] { + const result: number[] = []; + for (const entry of entries) { + for (const key of entry.values) { + result.push(key); + } + } + return result; + } +} + diff --git a/src/cache/base-cache.ts b/src/cache/base-cache.ts new file mode 100644 index 0000000..6f17d7e --- /dev/null +++ b/src/cache/base-cache.ts @@ -0,0 +1,37 @@ +export type CacheRangeOptions = { + lowerInclusive?: boolean; + upperInclusive?: boolean; +}; + +/** + * Shared cache abstraction used by storage adapters for secondary indexes. + * + * Implementations may support only equality (`get`) or both equality and + * range queries (`range`). Returning `null` from `range` indicates the cache + * cannot serve that range query efficiently. + */ +export abstract class BaseCache { + abstract registerIndex(indexName: string, fields: string[]): void; + + abstract clearIndex(indexName: string): void; + + abstract insert(indexName: string, key: any, internalKey: number): void; + + abstract delete(indexName: string, key: any, internalKey: number): void; + + abstract get(indexName: string, key: any): Iterable; + + abstract range( + indexName: string, + min?: any, + max?: any, + options?: CacheRangeOptions, + ): Iterable | null; + + /** + * Create a new empty cache instance of the same concrete type. + * Used by deriveChild() to preserve cache strategy across children. + */ + abstract createChild(): BaseCache; +} + diff --git a/src/cache/index.ts b/src/cache/index.ts new file mode 100644 index 0000000..8eff9b0 --- /dev/null +++ b/src/cache/index.ts @@ -0,0 +1,4 @@ +export * from './base-cache.js'; +export * from './b-tree-cache.js'; +export * from './kv-cache.js'; + diff --git a/src/cache/kv-cache.ts b/src/cache/kv-cache.ts new file mode 100644 index 0000000..059f067 --- /dev/null +++ b/src/cache/kv-cache.ts @@ -0,0 +1,69 @@ +import { BaseCache, type CacheRangeOptions } from './base-cache.js'; + +/** + * Hash-map cache implementation. + * + * - Equality: O(1) average + * - Range: unsupported (returns null -> caller falls back to scan) + */ +export class KvCache extends BaseCache { + private indexes = new Map>>(); + + registerIndex(indexName: string, _fields: string[]): void { + if (!this.indexes.has(indexName)) { + this.indexes.set(indexName, new Map()); + } + } + + clearIndex(indexName: string): void { + this.indexes.get(indexName)?.clear(); + } + + insert(indexName: string, key: any, internalKey: number): void { + const index = this.indexes.get(indexName); + if (!index) return; + + const keyStr = this.encodeKey(key); + const set = index.get(keyStr) ?? new Set(); + set.add(internalKey); + index.set(keyStr, set); + } + + delete(indexName: string, key: any, internalKey: number): void { + const index = this.indexes.get(indexName); + if (!index) return; + + const keyStr = this.encodeKey(key); + const set = index.get(keyStr); + if (!set) return; + + set.delete(internalKey); + if (set.size === 0) { + index.delete(keyStr); + } + } + + get(indexName: string, key: any): Iterable { + const index = this.indexes.get(indexName); + if (!index) return []; + return index.get(this.encodeKey(key)) ?? []; + } + + range( + _indexName: string, + _min?: any, + _max?: any, + _options?: CacheRangeOptions, + ): Iterable | null { + return null; + } + + createChild(): BaseCache { + return new KvCache(); + } + + private encodeKey(value: any): string { + return JSON.stringify(value); + } +} + diff --git a/src/storage/base-storage.ts b/src/storage/base-storage.ts index 909816b..26ddc3f 100644 --- a/src/storage/base-storage.ts +++ b/src/storage/base-storage.ts @@ -188,7 +188,7 @@ export abstract class BaseStorage< */ abstract deleteMany( filter: Filter, - options: Partial, + options?: Partial, ): Promise; /** diff --git a/src/storage/encrypted-storage.ts b/src/storage/encrypted-storage.ts index 6ac5b21..e85ce64 100644 --- a/src/storage/encrypted-storage.ts +++ b/src/storage/encrypted-storage.ts @@ -3,7 +3,14 @@ import { Packr } from 'msgpackr'; import { AESKey } from 'src/crypto/aes-key.js'; import { Bytes } from 'src/crypto/bytes.js'; -import { BaseStorage, type FindOptions, type Filter, isOperatorObject, isLogicalKey } from './base-storage.js'; +import { + BaseStorage, + type ComparisonOperators, + type FindOptions, + type Filter, + isOperatorObject, + isLogicalKey, +} from './base-storage.js'; import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; @@ -269,7 +276,7 @@ export class EncryptedStorage< if (this.plaintextKeys.has(key)) { // Plaintext field — pass through (including operator objects). result[key] = isOperatorObject(value) - ? value + ? this.formatOperatorValuesForStorage(value) : this.formatValueForEncryption(value); } else if (isOperatorObject(value)) { // Encrypted field with an operator — not supported. @@ -306,6 +313,24 @@ export class EncryptedStorage< return result; } + /** + * Normalize operator object values for plaintext fields so they use the + * same storage representation as document values (e.g. Date). + */ + private formatOperatorValuesForStorage( + ops: ComparisonOperators, + ): ComparisonOperators { + const result: Record = {}; + for (const [op, value] of Object.entries(ops)) { + if (op === '$not' && isOperatorObject(value)) { + result[op] = this.formatOperatorValuesForStorage(value); + continue; + } + result[op] = this.formatValueForEncryption(value); + } + return result as ComparisonOperators; + } + // --------------------------------------------------------------------------- // Value formatting // --------------------------------------------------------------------------- diff --git a/src/storage/storage-localstorage.ts b/src/storage/storage-localstorage.ts index 259c5c3..87d9eb3 100644 --- a/src/storage/storage-localstorage.ts +++ b/src/storage/storage-localstorage.ts @@ -7,7 +7,7 @@ import { isOperatorObject, isLogicalKey, } from './base-storage.js'; -import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js'; +import { BaseCache, BTreeCache } from 'src/cache/index.js'; /** * Key prefix separator used to namespace documents within localStorage. @@ -26,6 +26,13 @@ const MANIFEST_SUFFIX = '__keys__'; */ const COUNTER_SUFFIX = '__next__'; +/** + * Suffix for the mutation/version marker. This value increments on every + * write operation so other tabs can detect index-staleness even when the + * manifest key set itself is unchanged (e.g. updateMany). + */ +const VERSION_SUFFIX = '__version__'; + /** * Separator used when joining field names to create the index map key. */ @@ -43,18 +50,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] { return indexes as string[][]; } -/** - * Comparator for compound index keys (arrays of raw values). - */ -function tupleCompare(a: any[], b: any[]): number { - const len = Math.min(a.length, b.length); - for (let i = 0; i < len; i++) { - if (a[i] < b[i]) return -1; - if (a[i] > b[i]) return 1; - } - return a.length - b.length; -} - /** * Implementation of BaseStorage using the browser's localStorage API. * @@ -80,8 +75,9 @@ export class StorageLocalStorage< static from>( prefix = 'qs', indexes?: IndexDefinition, + cache?: BaseCache, ): StorageLocalStorage { - return new StorageLocalStorage(prefix, indexes); + return new StorageLocalStorage(prefix, indexes, cache); } /** @@ -100,16 +96,13 @@ export class StorageLocalStorage< * valid and we skip the expensive rebuild. */ private lastManifestRaw: string; + private lastVersionRaw: string; /** The normalized index definitions supplied at construction time. */ private indexDefs: string[][]; - /** - * Secondary indexes backed by B+ Trees. - * Map key = index name (joined field names). - * Map value = B+ Tree mapping index keys to sets of internal document keys. - */ - private indexes: Map>; + /** Secondary index cache (B-Tree or KV implementation). */ + private cache: BaseCache; /** Lazily-created child storage instances. */ private children: Map>; @@ -117,20 +110,21 @@ export class StorageLocalStorage< constructor( private readonly prefix: string = 'qs', indexes?: IndexDefinition, + cache?: BaseCache, ) { super(); this.children = new Map(); this.indexDefs = normalizeIndexes(indexes); - this.indexes = new Map(); + this.cache = cache ?? new BTreeCache(); for (const fields of this.indexDefs) { const name = fields.join(INDEX_KEY_SEP); - const comparator = fields.length > 1 ? tupleCompare : undefined; - this.indexes.set(name, new BPlusTree(32, comparator)); + this.cache.registerIndex(name, fields); } // Bootstrap from localStorage. this.lastManifestRaw = ''; + this.lastVersionRaw = ''; this.manifest = new Set(); this.nextKey = 0; this.refreshManifest(); @@ -162,10 +156,7 @@ export class StorageLocalStorage< if (resolution !== null) { const { keys, resolvedFields } = resolution; - const filterKeys = filter ? Object.keys(filter) : []; - const hasLogicalOps = filterKeys.some(isLogicalKey); - const needsVerification = hasLogicalOps - || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + const needsVerification = this.filterNeedsVerification(filter, resolvedFields); results = []; for (const key of keys) { @@ -265,7 +256,7 @@ export class StorageLocalStorage< const childPrefix = `${this.prefix}${KEY_SEP}${path}`; this.children.set( path, - new StorageLocalStorage(childPrefix, this.indexDefs), + new StorageLocalStorage(childPrefix, this.indexDefs, this.cache.createChild()), ); } return this.children.get(path) as StorageLocalStorage; @@ -328,6 +319,21 @@ export class StorageLocalStorage< return true; } + /** + * Determine whether candidate documents returned by index resolution still + * require full filter verification. + */ + private filterNeedsVerification( + filter: Filter | undefined, + resolvedFields: string[], + ): boolean { + if (!filter) return false; + const filterKeys = Object.keys(filter); + const hasLogicalOps = filterKeys.some(isLogicalKey); + return hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + } + /** * Collect all [internalKey, document] pairs that match a filter. * Uses an index when possible, otherwise falls back to a full scan. @@ -338,10 +344,7 @@ export class StorageLocalStorage< if (resolution !== null) { const { keys, resolvedFields } = resolution; - const filterKeys = filter ? Object.keys(filter) : []; - const hasLogicalOps = filterKeys.some(isLogicalKey); - const needsVerification = hasLogicalOps - || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + const needsVerification = this.filterNeedsVerification(filter, resolvedFields); for (const key of keys) { const raw = localStorage.getItem(this.docKey(key)); @@ -409,7 +412,7 @@ export class StorageLocalStorage< if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); - this.indexes.get(name)!.insert(indexKey, internalKey); + this.cache.insert(name, indexKey, internalKey); } } @@ -420,7 +423,7 @@ export class StorageLocalStorage< if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); - this.indexes.get(name)!.delete(indexKey, internalKey); + this.cache.delete(name, indexKey, internalKey); } } @@ -442,7 +445,6 @@ export class StorageLocalStorage< for (const fields of this.indexDefs) { const indexName = fields.join(INDEX_KEY_SEP); - const btree = this.indexes.get(indexName)!; if (fields.length === 1) { // --- Single-field index --- @@ -452,13 +454,13 @@ export class StorageLocalStorage< const filterValue = (filter as any)[field]; if (isOperatorObject(filterValue)) { - const keys = this.resolveOperatorViaTree(btree, filterValue); + const keys = this.resolveOperatorViaTree(indexName, filterValue); if (keys !== null) return { keys, resolvedFields: [field] }; continue; } // Plain equality. - return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] }; + return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] }; } else { // --- Compound index — all fields must be plain equality --- if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) { @@ -466,7 +468,7 @@ export class StorageLocalStorage< } const tupleKey = fields.map((f) => (filter as any)[f]); - return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] }; + return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] }; } } @@ -485,14 +487,25 @@ export class StorageLocalStorage< * - `$ne`, `$contains`, `$not` → cannot use index, returns null */ private resolveOperatorViaTree( - btree: BPlusTree, + indexName: string, ops: ComparisonOperators, ): Iterable | null { // Operators that prevent efficient index use. if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null; if (ops.$eq !== undefined) { - return btree.get(ops.$eq) ?? []; + // If $eq is combined with other operators, this path does not fully + // resolve the predicate. Let caller fall back to verification/full scan. + if ( + ops.$gt !== undefined + || ops.$gte !== undefined + || ops.$lt !== undefined + || ops.$lte !== undefined + || ops.$startsWith !== undefined + ) { + return null; + } + return this.cache.get(indexName, ops.$eq); } // $startsWith is converted to a range scan: "abc" → ["abc", "abd"). @@ -501,42 +514,65 @@ export class StorageLocalStorage< if (prefix.length === 0) return null; const upper = prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); - const entries = btree.range(prefix, upper, { + const entries = this.cache.range(indexName, prefix, upper, { lowerInclusive: true, upperInclusive: false, }); - return this.flattenEntryKeys(entries); + return entries; } + // Extract range bounds from the remaining operators. + // If strict/non-strict variants are both provided, use the stricter bound. let min: any = undefined; let max: any = undefined; let lowerInclusive = true; let upperInclusive = false; - if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; } - if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; } - if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; } - if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; } + if (ops.$gt !== undefined && ops.$gte !== undefined) { + if (ops.$gt > ops.$gte) { + min = ops.$gt; + lowerInclusive = false; + } else if (ops.$gt < ops.$gte) { + min = ops.$gte; + lowerInclusive = true; + } else { + min = ops.$gt; + lowerInclusive = false; + } + } else if (ops.$gt !== undefined) { + min = ops.$gt; + lowerInclusive = false; + } else if (ops.$gte !== undefined) { + min = ops.$gte; + lowerInclusive = true; + } + + if (ops.$lt !== undefined && ops.$lte !== undefined) { + if (ops.$lt < ops.$lte) { + max = ops.$lt; + upperInclusive = false; + } else if (ops.$lt > ops.$lte) { + max = ops.$lte; + upperInclusive = true; + } else { + max = ops.$lt; + upperInclusive = false; + } + } else if (ops.$lt !== undefined) { + max = ops.$lt; + upperInclusive = false; + } else if (ops.$lte !== undefined) { + max = ops.$lte; + upperInclusive = true; + } if (min === undefined && max === undefined) return null; - - const entries = btree.range(min, max, { lowerInclusive, upperInclusive }); - return this.flattenEntryKeys(entries); - } - - /** - * Flatten B+ Tree range results into a flat array of internal keys. - * Uses an array instead of a Set — no hash overhead, no deduplication - * needed because each internal key only appears under one index key. - */ - private flattenEntryKeys(entries: BPlusTreeEntry[]): number[] { - const result: number[] = []; - for (const entry of entries) { - for (const key of entry.values) { - result.push(key); - } + if (min !== undefined && max !== undefined) { + if (min > max) return []; + if (min === max && (!lowerInclusive || !upperInclusive)) return []; } - return result; + + return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive }); } /** @@ -544,8 +580,9 @@ export class StorageLocalStorage< * localStorage. Called only when a cross-tab manifest change is detected. */ private rebuildIndexes(): void { - for (const [, btree] of this.indexes) { - btree.clear(); + for (const fields of this.indexDefs) { + const name = fields.join(INDEX_KEY_SEP); + this.cache.clearIndex(name); } for (const key of this.manifest) { @@ -575,6 +612,11 @@ export class StorageLocalStorage< return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`; } + /** Build the localStorage key used to persist the mutation version. */ + private versionKey(): string { + return `${this.prefix}${KEY_SEP}${VERSION_SUFFIX}`; + } + /** * Re-read the manifest from localStorage into memory. * Called at the start of every public method so that cross-tab writes @@ -585,23 +627,41 @@ export class StorageLocalStorage< */ private refreshManifest(): void { const raw = localStorage.getItem(this.manifestKey()) ?? '[]'; + const versionRaw = localStorage.getItem(this.versionKey()) ?? '0'; - if (raw === this.lastManifestRaw) return; + if (raw === this.lastManifestRaw && versionRaw === this.lastVersionRaw) return; - this.lastManifestRaw = raw; + let parsedKeys: number[] = []; + let parsedOk = true; try { - const keys: number[] = JSON.parse(raw); - this.manifest = new Set(keys); + parsedKeys = JSON.parse(raw); } catch { + parsedOk = false; this.manifest = new Set(); } + if (parsedOk) { + this.manifest = new Set(parsedKeys); + this.lastManifestRaw = raw; + } + this.lastVersionRaw = versionRaw; + // Restore the counter from localStorage. const counterRaw = localStorage.getItem(this.counterKey()); - this.nextKey = counterRaw !== null ? Number(counterRaw) : 0; + if (counterRaw !== null) { + this.nextKey = Number(counterRaw); + } else if (this.manifest.size > 0) { + let max = -1; + for (const key of this.manifest) { + if (key > max) max = key; + } + this.nextKey = max + 1; + } else { + this.nextKey = 0; + } - // Manifest changed — indexes are potentially stale. + // Manifest or version changed — indexes are potentially stale. if (this.indexDefs.length > 0) { this.rebuildIndexes(); } @@ -612,11 +672,16 @@ export class StorageLocalStorage< */ private persistManifest(): void { const raw = JSON.stringify([...this.manifest]); + const nextVersion = Number(localStorage.getItem(this.versionKey()) ?? '0') + 1; + const versionRaw = String(nextVersion); + localStorage.setItem(this.manifestKey(), raw); localStorage.setItem(this.counterKey(), String(this.nextKey)); + localStorage.setItem(this.versionKey(), versionRaw); - // Keep the cached raw string in sync so the next refreshManifest() - // recognises this as "our own write" and skips the rebuild. + // Keep cached values in sync so the next refreshManifest() recognises + // this as our own write and skips unnecessary rebuild work. this.lastManifestRaw = raw; + this.lastVersionRaw = versionRaw; } } diff --git a/src/storage/storage-memory-synced.ts b/src/storage/storage-memory-synced.ts index 3f6b072..2172e2e 100644 --- a/src/storage/storage-memory-synced.ts +++ b/src/storage/storage-memory-synced.ts @@ -7,11 +7,17 @@ import { StorageMemory } from './storage-memory.js'; * All read operations will use system memory - all write operations will use the provided adapter. */ export class StorageMemorySynced = Record> extends BaseStorage { + private isPrimed: boolean; + private primePromise: Promise | null; + constructor( private inMemoryCache: StorageMemory, private store: BaseStorage, + isPrimed = false, ) { super(); + this.isPrimed = isPrimed; + this.primePromise = null; // Hook into all write operations so that we can sync the In-Memory cache. this.store.on('insert', async (payload) => { @@ -44,6 +50,23 @@ export class StorageMemorySynced = Record { + if (this.isPrimed) return; + if (!this.primePromise) { + this.primePromise = (async () => { + await this.inMemoryCache.deleteMany({}); + const allDocuments = await this.store.find(); + await this.inMemoryCache.insertMany(allDocuments); + this.isPrimed = true; + })(); + } + await this.primePromise; + } + static async create>(store: BaseStorage) { // Instantiate in-memory cache and the backing store. const inMemoryCache = new StorageMemory(); @@ -58,6 +81,7 @@ export class StorageMemorySynced = Record = Record, options?: FindOptions): Promise { + await this.ensurePrimed(); return await this.inMemoryCache.find(filter, options); } async updateMany( filter: Filter, update: Partial, - options: FindOptions = {} as FindOptions + options: Partial = {}, ): Promise { + await this.ensurePrimed(); return await this.store.updateMany(filter, update, options); } - async deleteMany(filter: Filter, options: FindOptions = {} as FindOptions): Promise { + async deleteMany(filter: Filter, options: Partial = {}): Promise { + await this.ensurePrimed(); return await this.store.deleteMany(filter, options); } deriveChild>(path: string): BaseStorage { const childStore = this.store.deriveChild(path); const childMemory = this.inMemoryCache.deriveChild(path); - + + if (!(childMemory instanceof StorageMemory)) { + throw new Error('Expected derived in-memory cache to be a StorageMemory instance'); + } + // Create a new synced storage for the child - return new StorageMemorySynced(childMemory as StorageMemory, childStore); + return new StorageMemorySynced(childMemory, childStore); } } diff --git a/src/storage/storage-memory.ts b/src/storage/storage-memory.ts index 413727a..46a7823 100644 --- a/src/storage/storage-memory.ts +++ b/src/storage/storage-memory.ts @@ -7,7 +7,7 @@ import { isOperatorObject, isLogicalKey, } from './base-storage.js'; -import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js'; +import { BaseCache, BTreeCache } from 'src/cache/index.js'; /** * Separator used when joining field names to create the index map key. @@ -30,19 +30,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] { return indexes as string[][]; } -/** - * Comparator for compound index keys (arrays of raw values). - * Compares element-by-element using native `<` / `>` operators. - */ -function tupleCompare(a: any[], b: any[]): number { - const len = Math.min(a.length, b.length); - for (let i = 0; i < len; i++) { - if (a[i] < b[i]) return -1; - if (a[i] > b[i]) return 1; - } - return a.length - b.length; -} - /** * Implementation of BaseStore using Memory as the storage backend. * @@ -56,8 +43,9 @@ export class StorageMemory< > extends BaseStorage { static from>( indexes?: IndexDefinition, + cache?: BaseCache, ): StorageMemory { - return new StorageMemory(indexes); + return new StorageMemory(indexes, cache); } /** Auto-incrementing counter used to generate internal keys. */ @@ -66,12 +54,8 @@ export class StorageMemory< /** Primary document store keyed by an opaque internal key. */ private store: Map; - /** - * Secondary indexes backed by B+ Trees. - * Map key = index name (joined field names). - * Map value = B+ Tree mapping index keys to sets of internal document keys. - */ - private indexes: Map>; + /** Secondary index cache (B-Tree or KV implementation). */ + private cache: BaseCache; /** The normalized index definitions supplied at construction time. */ private indexDefs: string[][]; @@ -79,19 +63,17 @@ export class StorageMemory< /** Lazily-created child storage instances. */ private children: Map>; - constructor(indexes?: IndexDefinition) { + constructor(indexes?: IndexDefinition, cache?: BaseCache) { super(); this.store = new Map(); this.children = new Map(); this.indexDefs = normalizeIndexes(indexes); - // Create a B+ Tree for each index definition. - this.indexes = new Map(); + this.cache = cache ?? new BTreeCache(); for (const fields of this.indexDefs) { const name = fields.join(INDEX_KEY_SEP); - const comparator = fields.length > 1 ? tupleCompare : undefined; - this.indexes.set(name, new BPlusTree(32, comparator)); + this.cache.registerIndex(name, fields); } } @@ -193,7 +175,7 @@ export class StorageMemory< deriveChild(path: string): BaseStorage { if (!this.children.has(path)) { - this.children.set(path, new StorageMemory(this.indexDefs)); + this.children.set(path, new StorageMemory(this.indexDefs, this.cache.createChild())); } return this.children.get(path) as StorageMemory; } @@ -257,6 +239,21 @@ export class StorageMemory< return true; } + /** + * Determine whether candidate documents returned by index resolution still + * require full filter verification. + */ + private filterNeedsVerification( + filter: Filter | undefined, + resolvedFields: string[], + ): boolean { + if (!filter) return false; + const filterKeys = Object.keys(filter); + const hasLogicalOps = filterKeys.some(isLogicalKey); + return hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + } + /** * Collect all [internalKey, document] pairs that match a filter. * Uses an index when possible, otherwise falls back to a full scan. @@ -266,10 +263,7 @@ export class StorageMemory< if (resolution !== null) { const { keys, resolvedFields } = resolution; - const filterKeys = filter ? Object.keys(filter) : []; - const hasLogicalOps = filterKeys.some(isLogicalKey); - const needsVerification = hasLogicalOps - || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + const needsVerification = this.filterNeedsVerification(filter, resolvedFields); const results: Array<[number, T]> = []; for (const key of keys) { @@ -337,7 +331,7 @@ export class StorageMemory< if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); - this.indexes.get(name)!.insert(indexKey, internalKey); + this.cache.insert(name, indexKey, internalKey); } } @@ -348,7 +342,7 @@ export class StorageMemory< if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); - this.indexes.get(name)!.delete(indexKey, internalKey); + this.cache.delete(name, indexKey, internalKey); } } @@ -367,7 +361,6 @@ export class StorageMemory< for (const fields of this.indexDefs) { const indexName = fields.join(INDEX_KEY_SEP); - const btree = this.indexes.get(indexName)!; if (fields.length === 1) { // --- Single-field index --- @@ -377,13 +370,13 @@ export class StorageMemory< const filterValue = (filter as any)[field]; if (isOperatorObject(filterValue)) { - const keys = this.resolveOperatorViaTree(btree, filterValue); + const keys = this.resolveOperatorViaTree(indexName, filterValue); if (keys !== null) return { keys, resolvedFields: [field] }; continue; } // Plain equality. - return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] }; + return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] }; } else { // --- Compound index — all fields must be plain equality --- if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) { @@ -391,7 +384,7 @@ export class StorageMemory< } const tupleKey = fields.map((f) => (filter as any)[f]); - return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] }; + return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] }; } } @@ -410,7 +403,7 @@ export class StorageMemory< * - `$ne`, `$contains`, `$not` → cannot use index, returns null */ private resolveOperatorViaTree( - btree: BPlusTree, + indexName: string, ops: ComparisonOperators, ): Iterable | null { // Operators that prevent efficient index use. @@ -418,7 +411,18 @@ export class StorageMemory< // $eq is a point lookup. if (ops.$eq !== undefined) { - return btree.get(ops.$eq) ?? []; + // If $eq is combined with other operators, this path does not fully + // resolve the predicate. Let caller fall back to verification/full scan. + if ( + ops.$gt !== undefined + || ops.$gte !== undefined + || ops.$lt !== undefined + || ops.$lte !== undefined + || ops.$startsWith !== undefined + ) { + return null; + } + return this.cache.get(indexName, ops.$eq); } // $startsWith is converted to a range scan: "abc" → ["abc", "abd"). @@ -427,43 +431,65 @@ export class StorageMemory< if (prefix.length === 0) return null; const upper = prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); - const entries = btree.range(prefix, upper, { + const entries = this.cache.range(indexName, prefix, upper, { lowerInclusive: true, upperInclusive: false, }); - return this.flattenEntryKeys(entries); + return entries; } // Extract range bounds from the remaining operators. + // If strict/non-strict variants are both provided, use the stricter bound. let min: any = undefined; let max: any = undefined; let lowerInclusive = true; let upperInclusive = false; - if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; } - if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; } - if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; } - if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; } + if (ops.$gt !== undefined && ops.$gte !== undefined) { + if (ops.$gt > ops.$gte) { + min = ops.$gt; + lowerInclusive = false; + } else if (ops.$gt < ops.$gte) { + min = ops.$gte; + lowerInclusive = true; + } else { + min = ops.$gt; + lowerInclusive = false; + } + } else if (ops.$gt !== undefined) { + min = ops.$gt; + lowerInclusive = false; + } else if (ops.$gte !== undefined) { + min = ops.$gte; + lowerInclusive = true; + } + + if (ops.$lt !== undefined && ops.$lte !== undefined) { + if (ops.$lt < ops.$lte) { + max = ops.$lt; + upperInclusive = false; + } else if (ops.$lt > ops.$lte) { + max = ops.$lte; + upperInclusive = true; + } else { + max = ops.$lt; + upperInclusive = false; + } + } else if (ops.$lt !== undefined) { + max = ops.$lt; + upperInclusive = false; + } else if (ops.$lte !== undefined) { + max = ops.$lte; + upperInclusive = true; + } if (min === undefined && max === undefined) return null; - - const entries = btree.range(min, max, { lowerInclusive, upperInclusive }); - return this.flattenEntryKeys(entries); - } - - /** - * Flatten B+ Tree range results into a flat array of internal keys. - * Uses an array instead of a Set — no hash overhead, no deduplication - * needed because each internal key only appears under one index key. - */ - private flattenEntryKeys(entries: BPlusTreeEntry[]): number[] { - const result: number[] = []; - for (const entry of entries) { - for (const key of entry.values) { - result.push(key); - } + if (min !== undefined && max !== undefined) { + if (min > max) return []; + if (min === max && (!lowerInclusive || !upperInclusive)) return []; } - return result; + + return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive }); } /** @@ -480,13 +506,7 @@ export class StorageMemory< if (resolution === null) return null; const { keys, resolvedFields } = resolution; - const filterKeys = filter ? Object.keys(filter) : []; - - // Logical operators ($and/$or/$not) are never resolved by the index, - // and any unresolved field conditions also require verification. - const hasLogicalOps = filterKeys.some(isLogicalKey); - const needsVerification = hasLogicalOps - || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); + const needsVerification = this.filterNeedsVerification(filter, resolvedFields); const results: T[] = []; for (const key of keys) { diff --git a/tests/storage/storage-regressions.test.ts b/tests/storage/storage-regressions.test.ts new file mode 100644 index 0000000..fa5a0f1 --- /dev/null +++ b/tests/storage/storage-regressions.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { AESKey } from 'src/crypto/aes-key.js'; + +import { EncryptedStorage } from 'src/storage/encrypted-storage.js'; +import { StorageMemory } from 'src/storage/storage-memory.js'; +import { StorageMemorySynced } from 'src/storage/storage-memory-synced.js'; + +describe('storage regressions', () => { + it('does not treat $eq as fully resolved when mixed with other operators', async () => { + const storage = StorageMemory.from<{ age: number }>(['age']); + await storage.insertMany([{ age: 25 }, { age: 35 }]); + + const results = await storage.find({ age: { $eq: 25, $gt: 30 } }); + expect(results).toEqual([]); + }); + + it('normalizes conflicting lower/upper bounds to strictest constraints', async () => { + const storage = StorageMemory.from<{ age: number }>(['age']); + await storage.insertMany([{ age: 30 }, { age: 31 }, { age: 32 }]); + + const lower = await storage.find({ age: { $gt: 30, $gte: 30 } }); + expect(lower.map((d) => d.age).sort((a, b) => a - b)).toEqual([31, 32]); + + const upper = await storage.find({ age: { $lt: 32, $lte: 32 } }); + expect(upper.map((d) => d.age).sort((a, b) => a - b)).toEqual([30, 31]); + }); + + it('formats plaintext operator values in encrypted filters', async () => { + type Doc = { createdAt: Date; name: string }; + const key = await AESKey.fromSeed('storage-regression-test-key'); + const base = StorageMemory.from>(['createdAt']); + const storage = EncryptedStorage.from(base, key, { plaintextKeys: ['createdAt'] }); + + await storage.insertOne({ + createdAt: new Date('2024-01-02T00:00:00.000Z'), + name: 'alice', + }); + + const results = await storage.find({ + createdAt: { $gte: new Date('2024-01-01T00:00:00.000Z') }, + }); + expect(results).toHaveLength(1); + expect(results[0].name).toBe('alice'); + }); + + it('primes derived child cache in StorageMemorySynced', async () => { + const store = StorageMemory.from>(); + const child = store.deriveChild<{ value: number }>('child'); + await child.insertOne({ value: 42 }); + + const synced = await StorageMemorySynced.create(store); + const syncedChild = synced.deriveChild<{ value: number }>('child'); + const results = await syncedChild.find(); + + expect(results).toEqual([{ value: 42 }]); + }); +});