Harvmaster/Hashpass-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Cache primtive

Browse Source
This commit is contained in:
Harvey Zuccon
2026-02-25 17:28:04 +11:00
parent f7c89046d1
commit 5c39d8add2
12 changed files with 554 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
benchmarks/storage.ts
View File

@@ -1,5 +1,6 @@
import { AESKey } from '../src/crypto/aes-key.js';
import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js';
import { BTreeCache, KvCache } from '../src/cache/index.js';
// ---------------------------------------------------------------------------
// Helpers
@@ -161,9 +162,19 @@ const DOC_COUNTS = [1_000, 10_000, 50_000];
for (const count of DOC_COUNTS) {
const docs = generateDocs(count);
// Indexes on id, name, AND age — range queries on age use B+ Tree.
const indexedWithAge = StorageMemory.from<Doc>(['id', 'name', 'age']);
await benchmarkStorage('StorageMemory (indexed: id,name,age)', indexedWithAge, docs, { hasAgeIndex: true });
// Indexes on id, name, AND age with explicit B+ Tree cache.
const indexedWithAgeBTree = StorageMemory.from<Doc>(
['id', 'name', 'age'],
new BTreeCache(),
);
await benchmarkStorage('StorageMemory (B+Tree cache, indexed: id,name,age)', indexedWithAgeBTree, docs, { hasAgeIndex: true });
// Same indexes, but KV cache (no range support).
const indexedWithAgeKv = StorageMemory.from<Doc>(
['id', 'name', 'age'],
new KvCache(),
);
await benchmarkStorage('StorageMemory (KV cache, indexed: id,name,age)', indexedWithAgeKv, docs);
// Indexes on id, name only — range queries on age fall back to full scan.
const indexed = StorageMemory.from<Doc>(['id', 'name']);

4
package.json
View File

@@ -6,7 +6,9 @@
"scripts": {
"build": "tsc",
"format": "prettier --write .",
"test": "echo \"Error: no test specified\" && exit 1",
"test": "vitest --run",
"test:watch": "vitest",
"benchmark:sha256": "tsx benchmarks/sha256.ts",
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",

79
src/cache/b-tree-cache.ts vendored Normal file
View File

@@ -0,0 +1,79 @@
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* B+ tree-backed cache implementation.
*
* - Equality: O(log n)
* - Range: O(log n + k)
*/
export class BTreeCache extends BaseCache {
private trees = new Map<string, BPlusTree<any, number>>();
registerIndex(indexName: string, fields: string[]): void {
if (this.trees.has(indexName)) return;
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.trees.set(indexName, new BPlusTree<any, number>(32, comparator));
}
clearIndex(indexName: string): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.clear();
}
insert(indexName: string, key: any, internalKey: number): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.insert(key, internalKey);
}
delete(indexName: string, key: any, internalKey: number): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.delete(key, internalKey);
}
get(indexName: string, key: any): Iterable<number> {
const tree = this.trees.get(indexName);
if (!tree) return [];
return tree.get(key) ?? [];
}
range(
indexName: string,
min?: any,
max?: any,
options: CacheRangeOptions = {},
): Iterable<number> | null {
const tree = this.trees.get(indexName);
if (!tree) return [];
const entries = tree.range(min, max, options);
return this.flattenEntryKeys(entries);
}
createChild(): BaseCache {
return new BTreeCache();
}
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
}
return result;
}
}

37
src/cache/base-cache.ts vendored Normal file
View File

@@ -0,0 +1,37 @@
export type CacheRangeOptions = {
lowerInclusive?: boolean;
upperInclusive?: boolean;
};
/**
* Shared cache abstraction used by storage adapters for secondary indexes.
*
* Implementations may support only equality (`get`) or both equality and
* range queries (`range`). Returning `null` from `range` indicates the cache
* cannot serve that range query efficiently.
*/
export abstract class BaseCache {
abstract registerIndex(indexName: string, fields: string[]): void;
abstract clearIndex(indexName: string): void;
abstract insert(indexName: string, key: any, internalKey: number): void;
abstract delete(indexName: string, key: any, internalKey: number): void;
abstract get(indexName: string, key: any): Iterable<number>;
abstract range(
indexName: string,
min?: any,
max?: any,
options?: CacheRangeOptions,
): Iterable<number> | null;
/**
* Create a new empty cache instance of the same concrete type.
* Used by deriveChild() to preserve cache strategy across children.
*/
abstract createChild(): BaseCache;
}

4
src/cache/index.ts vendored Normal file
View File

@@ -0,0 +1,4 @@
export * from './base-cache.js';
export * from './b-tree-cache.js';
export * from './kv-cache.js';

69
src/cache/kv-cache.ts vendored Normal file
View File

@@ -0,0 +1,69 @@
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
/**
* Hash-map cache implementation.
*
* - Equality: O(1) average
* - Range: unsupported (returns null -> caller falls back to scan)
*/
export class KvCache extends BaseCache {
private indexes = new Map<string, Map<string, Set<number>>>();
registerIndex(indexName: string, _fields: string[]): void {
if (!this.indexes.has(indexName)) {
this.indexes.set(indexName, new Map());
}
}
clearIndex(indexName: string): void {
this.indexes.get(indexName)?.clear();
}
insert(indexName: string, key: any, internalKey: number): void {
const index = this.indexes.get(indexName);
if (!index) return;
const keyStr = this.encodeKey(key);
const set = index.get(keyStr) ?? new Set<number>();
set.add(internalKey);
index.set(keyStr, set);
}
delete(indexName: string, key: any, internalKey: number): void {
const index = this.indexes.get(indexName);
if (!index) return;
const keyStr = this.encodeKey(key);
const set = index.get(keyStr);
if (!set) return;
set.delete(internalKey);
if (set.size === 0) {
index.delete(keyStr);
}
}
get(indexName: string, key: any): Iterable<number> {
const index = this.indexes.get(indexName);
if (!index) return [];
return index.get(this.encodeKey(key)) ?? [];
}
range(
_indexName: string,
_min?: any,
_max?: any,
_options?: CacheRangeOptions,
): Iterable<number> | null {
return null;
}
createChild(): BaseCache {
return new KvCache();
}
private encodeKey(value: any): string {
return JSON.stringify(value);
}
}

2
src/storage/base-storage.ts
View File

@@ -188,7 +188,7 @@ export abstract class BaseStorage<
*/
abstract deleteMany(
filter: Filter<T>,
options: Partial<FindOptions>,
options?: Partial<FindOptions>,
): Promise<number>;
/**

29
src/storage/encrypted-storage.ts
View File

@@ -3,7 +3,14 @@ import { Packr } from 'msgpackr';
import { AESKey } from 'src/crypto/aes-key.js';
import { Bytes } from 'src/crypto/bytes.js';
import { BaseStorage, type FindOptions, type Filter, isOperatorObject, isLogicalKey } from './base-storage.js';
import {
BaseStorage,
type ComparisonOperators,
type FindOptions,
type Filter,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
@@ -269,7 +276,7 @@ export class EncryptedStorage<
if (this.plaintextKeys.has(key)) {
// Plaintext field — pass through (including operator objects).
result[key] = isOperatorObject(value)
? value
? this.formatOperatorValuesForStorage(value)
: this.formatValueForEncryption(value);
} else if (isOperatorObject(value)) {
// Encrypted field with an operator — not supported.
@@ -306,6 +313,24 @@ export class EncryptedStorage<
return result;
}
/**
* Normalize operator object values for plaintext fields so they use the
* same storage representation as document values (e.g. Date).
*/
private formatOperatorValuesForStorage(
ops: ComparisonOperators<any>,
): ComparisonOperators<any> {
const result: Record<string, any> = {};
for (const [op, value] of Object.entries(ops)) {
if (op === '$not' && isOperatorObject(value)) {
result[op] = this.formatOperatorValuesForStorage(value);
continue;
}
result[op] = this.formatValueForEncryption(value);
}
return result as ComparisonOperators<any>;
}
// ---------------------------------------------------------------------------
// Value formatting
// ---------------------------------------------------------------------------

211
src/storage/storage-localstorage.ts
View File

@@ -7,7 +7,7 @@ import {
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, BTreeCache } from 'src/cache/index.js';
/**
* Key prefix separator used to namespace documents within localStorage.
@@ -26,6 +26,13 @@ const MANIFEST_SUFFIX = '__keys__';
*/
const COUNTER_SUFFIX = '__next__';
/**
* Suffix for the mutation/version marker. This value increments on every
* write operation so other tabs can detect index-staleness even when the
* manifest key set itself is unchanged (e.g. updateMany).
*/
const VERSION_SUFFIX = '__version__';
/**
* Separator used when joining field names to create the index map key.
*/
@@ -43,18 +50,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
return indexes as string[][];
}
/**
* Comparator for compound index keys (arrays of raw values).
*/
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* Implementation of BaseStorage using the browser's localStorage API.
*
@@ -80,8 +75,9 @@ export class StorageLocalStorage<
static from<T extends Record<string, any>>(
prefix = 'qs',
indexes?: IndexDefinition,
cache?: BaseCache,
): StorageLocalStorage<T> {
return new StorageLocalStorage<T>(prefix, indexes);
return new StorageLocalStorage<T>(prefix, indexes, cache);
}
/**
@@ -100,16 +96,13 @@ export class StorageLocalStorage<
* valid and we skip the expensive rebuild.
*/
private lastManifestRaw: string;
private lastVersionRaw: string;
/** The normalized index definitions supplied at construction time. */
private indexDefs: string[][];
/**
* Secondary indexes backed by B+ Trees.
* Map key = index name (joined field names).
* Map value = B+ Tree mapping index keys to sets of internal document keys.
*/
private indexes: Map<string, BPlusTree<any, number>>;
/** Secondary index cache (B-Tree or KV implementation). */
private cache: BaseCache;
/** Lazily-created child storage instances. */
private children: Map<string, StorageLocalStorage<any>>;
@@ -117,20 +110,21 @@ export class StorageLocalStorage<
constructor(
private readonly prefix: string = 'qs',
indexes?: IndexDefinition,
cache?: BaseCache,
) {
super();
this.children = new Map();
this.indexDefs = normalizeIndexes(indexes);
this.indexes = new Map();
this.cache = cache ?? new BTreeCache();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
this.cache.registerIndex(name, fields);
}
// Bootstrap from localStorage.
this.lastManifestRaw = '';
this.lastVersionRaw = '';
this.manifest = new Set();
this.nextKey = 0;
this.refreshManifest();
@@ -162,10 +156,7 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
results = [];
for (const key of keys) {
@@ -265,7 +256,7 @@ export class StorageLocalStorage<
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
this.children.set(
path,
new StorageLocalStorage<C>(childPrefix, this.indexDefs),
new StorageLocalStorage<C>(childPrefix, this.indexDefs, this.cache.createChild()),
);
}
return this.children.get(path) as StorageLocalStorage<C>;
@@ -328,6 +319,21 @@ export class StorageLocalStorage<
return true;
}
/**
* Determine whether candidate documents returned by index resolution still
* require full filter verification.
*/
private filterNeedsVerification(
filter: Filter<T> | undefined,
resolvedFields: string[],
): boolean {
if (!filter) return false;
const filterKeys = Object.keys(filter);
const hasLogicalOps = filterKeys.some(isLogicalKey);
return hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
@@ -338,10 +344,7 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
for (const key of keys) {
const raw = localStorage.getItem(this.docKey(key));
@@ -409,7 +412,7 @@ export class StorageLocalStorage<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.insert(indexKey, internalKey);
this.cache.insert(name, indexKey, internalKey);
}
}
@@ -420,7 +423,7 @@ export class StorageLocalStorage<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.delete(indexKey, internalKey);
this.cache.delete(name, indexKey, internalKey);
}
}
@@ -442,7 +445,6 @@ export class StorageLocalStorage<
for (const fields of this.indexDefs) {
const indexName = fields.join(INDEX_KEY_SEP);
const btree = this.indexes.get(indexName)!;
if (fields.length === 1) {
// --- Single-field index ---
@@ -452,13 +454,13 @@ export class StorageLocalStorage<
const filterValue = (filter as any)[field];
if (isOperatorObject(filterValue)) {
const keys = this.resolveOperatorViaTree(btree, filterValue);
const keys = this.resolveOperatorViaTree(indexName, filterValue);
if (keys !== null) return { keys, resolvedFields: [field] };
continue;
}
// Plain equality.
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
} else {
// --- Compound index — all fields must be plain equality ---
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
@@ -466,7 +468,7 @@ export class StorageLocalStorage<
}
const tupleKey = fields.map((f) => (filter as any)[f]);
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
}
}
@@ -485,14 +487,25 @@ export class StorageLocalStorage<
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
indexName: string,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
// Operators that prevent efficient index use.
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
// If $eq is combined with other operators, this path does not fully
// resolve the predicate. Let caller fall back to verification/full scan.
if (
ops.$gt !== undefined
|| ops.$gte !== undefined
|| ops.$lt !== undefined
|| ops.$lte !== undefined
|| ops.$startsWith !== undefined
) {
return null;
}
return this.cache.get(indexName, ops.$eq);
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
@@ -501,42 +514,65 @@ export class StorageLocalStorage<
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = btree.range(prefix, upper, {
const entries = this.cache.range(indexName, prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return this.flattenEntryKeys(entries);
return entries;
}
// Extract range bounds from the remaining operators.
// If strict/non-strict variants are both provided, use the stricter bound.
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;
let upperInclusive = false;
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
if (ops.$gt !== undefined && ops.$gte !== undefined) {
if (ops.$gt > ops.$gte) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gt < ops.$gte) {
min = ops.$gte;
lowerInclusive = true;
} else {
min = ops.$gt;
lowerInclusive = false;
}
} else if (ops.$gt !== undefined) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gte !== undefined) {
min = ops.$gte;
lowerInclusive = true;
}
if (ops.$lt !== undefined && ops.$lte !== undefined) {
if (ops.$lt < ops.$lte) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lt > ops.$lte) {
max = ops.$lte;
upperInclusive = true;
} else {
max = ops.$lt;
upperInclusive = false;
}
} else if (ops.$lt !== undefined) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lte !== undefined) {
max = ops.$lte;
upperInclusive = true;
}
if (min === undefined && max === undefined) return null;
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
return this.flattenEntryKeys(entries);
}
/**
* Flatten B+ Tree range results into a flat array of internal keys.
* Uses an array instead of a Set — no hash overhead, no deduplication
* needed because each internal key only appears under one index key.
*/
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
if (min !== undefined && max !== undefined) {
if (min > max) return [];
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
}
return result;
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
}
/**
@@ -544,8 +580,9 @@ export class StorageLocalStorage<
* localStorage. Called only when a cross-tab manifest change is detected.
*/
private rebuildIndexes(): void {
for (const [, btree] of this.indexes) {
btree.clear();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
this.cache.clearIndex(name);
}
for (const key of this.manifest) {
@@ -575,6 +612,11 @@ export class StorageLocalStorage<
return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`;
}
/** Build the localStorage key used to persist the mutation version. */
private versionKey(): string {
return `${this.prefix}${KEY_SEP}${VERSION_SUFFIX}`;
}
/**
* Re-read the manifest from localStorage into memory.
* Called at the start of every public method so that cross-tab writes
@@ -585,23 +627,41 @@ export class StorageLocalStorage<
*/
private refreshManifest(): void {
const raw = localStorage.getItem(this.manifestKey()) ?? '[]';
const versionRaw = localStorage.getItem(this.versionKey()) ?? '0';
if (raw === this.lastManifestRaw) return;
if (raw === this.lastManifestRaw && versionRaw === this.lastVersionRaw) return;
this.lastManifestRaw = raw;
let parsedKeys: number[] = [];
let parsedOk = true;
try {
const keys: number[] = JSON.parse(raw);
this.manifest = new Set(keys);
parsedKeys = JSON.parse(raw);
} catch {
parsedOk = false;
this.manifest = new Set();
}
if (parsedOk) {
this.manifest = new Set(parsedKeys);
this.lastManifestRaw = raw;
}
this.lastVersionRaw = versionRaw;
// Restore the counter from localStorage.
const counterRaw = localStorage.getItem(this.counterKey());
this.nextKey = counterRaw !== null ? Number(counterRaw) : 0;
if (counterRaw !== null) {
this.nextKey = Number(counterRaw);
} else if (this.manifest.size > 0) {
let max = -1;
for (const key of this.manifest) {
if (key > max) max = key;
}
this.nextKey = max + 1;
} else {
this.nextKey = 0;
}
// Manifest changed — indexes are potentially stale.
// Manifest or version changed — indexes are potentially stale.
if (this.indexDefs.length > 0) {
this.rebuildIndexes();
}
@@ -612,11 +672,16 @@ export class StorageLocalStorage<
*/
private persistManifest(): void {
const raw = JSON.stringify([...this.manifest]);
const nextVersion = Number(localStorage.getItem(this.versionKey()) ?? '0') + 1;
const versionRaw = String(nextVersion);
localStorage.setItem(this.manifestKey(), raw);
localStorage.setItem(this.counterKey(), String(this.nextKey));
localStorage.setItem(this.versionKey(), versionRaw);
// Keep the cached raw string in sync so the next refreshManifest()
// recognises this as "our own write" and skips the rebuild.
// Keep cached values in sync so the next refreshManifest() recognises
// this as our own write and skips unnecessary rebuild work.
this.lastManifestRaw = raw;
this.lastVersionRaw = versionRaw;
}
}

39
src/storage/storage-memory-synced.ts
View File

@@ -7,11 +7,17 @@ import { StorageMemory } from './storage-memory.js';
* All read operations will use system memory - all write operations will use the provided adapter.
*/
export class StorageMemorySynced<T extends Record<string, any> = Record<string, any>> extends BaseStorage<T> {
private isPrimed: boolean;
private primePromise: Promise<void> | null;
constructor(
private inMemoryCache: StorageMemory<T>,
private store: BaseStorage<T>,
isPrimed = false,
) {
super();
this.isPrimed = isPrimed;
this.primePromise = null;
// Hook into all write operations so that we can sync the In-Memory cache.
this.store.on('insert', async (payload) => {
@@ -44,6 +50,23 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
});
}
/**
* Ensure the in-memory cache has been initialized from the backing store.
* This is especially important for derived children, whose caches start empty.
*/
private async ensurePrimed(): Promise<void> {
if (this.isPrimed) return;
if (!this.primePromise) {
this.primePromise = (async () => {
await this.inMemoryCache.deleteMany({});
const allDocuments = await this.store.find();
await this.inMemoryCache.insertMany(allDocuments);
this.isPrimed = true;
})();
}
await this.primePromise;
}
static async create<T extends Record<string, any>>(store: BaseStorage<T>) {
// Instantiate in-memory cache and the backing store.
const inMemoryCache = new StorageMemory<T>();
@@ -58,6 +81,7 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
}
// Return our instance of this store.
memorySyncedStore.isPrimed = true;
return memorySyncedStore;
}
@@ -66,26 +90,33 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
}
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
await this.ensurePrimed();
return await this.inMemoryCache.find(filter, options);
}
async updateMany(
filter: Filter<T>,
update: Partial<T>,
options: FindOptions = {} as FindOptions
options: Partial<FindOptions> = {},
): Promise<number> {
await this.ensurePrimed();
return await this.store.updateMany(filter, update, options);
}
async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
async deleteMany(filter: Filter<T>, options: Partial<FindOptions> = {}): Promise<number> {
await this.ensurePrimed();
return await this.store.deleteMany(filter, options);
}
deriveChild<C extends Record<string, any>>(path: string): BaseStorage<C> {
const childStore = this.store.deriveChild<C>(path);
const childMemory = this.inMemoryCache.deriveChild<C>(path);
if (!(childMemory instanceof StorageMemory)) {
throw new Error('Expected derived in-memory cache to be a StorageMemory instance');
}
// Create a new synced storage for the child
return new StorageMemorySynced<C>(childMemory as StorageMemory<C>, childStore);
return new StorageMemorySynced<C>(childMemory, childStore);
}
}

158
src/storage/storage-memory.ts
View File

@@ -7,7 +7,7 @@ import {
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, BTreeCache } from 'src/cache/index.js';
/**
* Separator used when joining field names to create the index map key.
@@ -30,19 +30,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
return indexes as string[][];
}
/**
* Comparator for compound index keys (arrays of raw values).
* Compares element-by-element using native `<` / `>` operators.
*/
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* Implementation of BaseStore using Memory as the storage backend.
*
@@ -56,8 +43,9 @@ export class StorageMemory<
> extends BaseStorage<T> {
static from<T extends Record<string, any>>(
indexes?: IndexDefinition,
cache?: BaseCache,
): StorageMemory<T> {
return new StorageMemory<T>(indexes);
return new StorageMemory<T>(indexes, cache);
}
/** Auto-incrementing counter used to generate internal keys. */
@@ -66,12 +54,8 @@ export class StorageMemory<
/** Primary document store keyed by an opaque internal key. */
private store: Map<number, T>;
/**
* Secondary indexes backed by B+ Trees.
* Map key = index name (joined field names).
* Map value = B+ Tree mapping index keys to sets of internal document keys.
*/
private indexes: Map<string, BPlusTree<any, number>>;
/** Secondary index cache (B-Tree or KV implementation). */
private cache: BaseCache;
/** The normalized index definitions supplied at construction time. */
private indexDefs: string[][];
@@ -79,19 +63,17 @@ export class StorageMemory<
/** Lazily-created child storage instances. */
private children: Map<string, StorageMemory<any>>;
constructor(indexes?: IndexDefinition) {
constructor(indexes?: IndexDefinition, cache?: BaseCache) {
super();
this.store = new Map();
this.children = new Map();
this.indexDefs = normalizeIndexes(indexes);
// Create a B+ Tree for each index definition.
this.indexes = new Map();
this.cache = cache ?? new BTreeCache();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
this.cache.registerIndex(name, fields);
}
}
@@ -193,7 +175,7 @@ export class StorageMemory<
deriveChild<C>(path: string): BaseStorage<C> {
if (!this.children.has(path)) {
this.children.set(path, new StorageMemory<C>(this.indexDefs));
this.children.set(path, new StorageMemory<C>(this.indexDefs, this.cache.createChild()));
}
return this.children.get(path) as StorageMemory<C>;
}
@@ -257,6 +239,21 @@ export class StorageMemory<
return true;
}
/**
* Determine whether candidate documents returned by index resolution still
* require full filter verification.
*/
private filterNeedsVerification(
filter: Filter<T> | undefined,
resolvedFields: string[],
): boolean {
if (!filter) return false;
const filterKeys = Object.keys(filter);
const hasLogicalOps = filterKeys.some(isLogicalKey);
return hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
@@ -266,10 +263,7 @@ export class StorageMemory<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: Array<[number, T]> = [];
for (const key of keys) {
@@ -337,7 +331,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.insert(indexKey, internalKey);
this.cache.insert(name, indexKey, internalKey);
}
}
@@ -348,7 +342,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.delete(indexKey, internalKey);
this.cache.delete(name, indexKey, internalKey);
}
}
@@ -367,7 +361,6 @@ export class StorageMemory<
for (const fields of this.indexDefs) {
const indexName = fields.join(INDEX_KEY_SEP);
const btree = this.indexes.get(indexName)!;
if (fields.length === 1) {
// --- Single-field index ---
@@ -377,13 +370,13 @@ export class StorageMemory<
const filterValue = (filter as any)[field];
if (isOperatorObject(filterValue)) {
const keys = this.resolveOperatorViaTree(btree, filterValue);
const keys = this.resolveOperatorViaTree(indexName, filterValue);
if (keys !== null) return { keys, resolvedFields: [field] };
continue;
}
// Plain equality.
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
} else {
// --- Compound index — all fields must be plain equality ---
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
@@ -391,7 +384,7 @@ export class StorageMemory<
}
const tupleKey = fields.map((f) => (filter as any)[f]);
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
}
}
@@ -410,7 +403,7 @@ export class StorageMemory<
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
indexName: string,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
// Operators that prevent efficient index use.
@@ -418,7 +411,18 @@ export class StorageMemory<
// $eq is a point lookup.
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
// If $eq is combined with other operators, this path does not fully
// resolve the predicate. Let caller fall back to verification/full scan.
if (
ops.$gt !== undefined
|| ops.$gte !== undefined
|| ops.$lt !== undefined
|| ops.$lte !== undefined
|| ops.$startsWith !== undefined
) {
return null;
}
return this.cache.get(indexName, ops.$eq);
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
@@ -427,43 +431,65 @@ export class StorageMemory<
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = btree.range(prefix, upper, {
const entries = this.cache.range(indexName, prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return this.flattenEntryKeys(entries);
return entries;
}
// Extract range bounds from the remaining operators.
// If strict/non-strict variants are both provided, use the stricter bound.
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;
let upperInclusive = false;
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
if (ops.$gt !== undefined && ops.$gte !== undefined) {
if (ops.$gt > ops.$gte) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gt < ops.$gte) {
min = ops.$gte;
lowerInclusive = true;
} else {
min = ops.$gt;
lowerInclusive = false;
}
} else if (ops.$gt !== undefined) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gte !== undefined) {
min = ops.$gte;
lowerInclusive = true;
}
if (ops.$lt !== undefined && ops.$lte !== undefined) {
if (ops.$lt < ops.$lte) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lt > ops.$lte) {
max = ops.$lte;
upperInclusive = true;
} else {
max = ops.$lt;
upperInclusive = false;
}
} else if (ops.$lt !== undefined) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lte !== undefined) {
max = ops.$lte;
upperInclusive = true;
}
if (min === undefined && max === undefined) return null;
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
return this.flattenEntryKeys(entries);
}
/**
* Flatten B+ Tree range results into a flat array of internal keys.
* Uses an array instead of a Set — no hash overhead, no deduplication
* needed because each internal key only appears under one index key.
*/
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
if (min !== undefined && max !== undefined) {
if (min > max) return [];
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
}
return result;
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
}
/**
@@ -480,13 +506,7 @@ export class StorageMemory<
if (resolution === null) return null;
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
// Logical operators ($and/$or/$not) are never resolved by the index,
// and any unresolved field conditions also require verification.
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: T[] = [];
for (const key of keys) {

58
tests/storage/storage-regressions.test.ts Normal file
View File

@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';
import { AESKey } from 'src/crypto/aes-key.js';
import { EncryptedStorage } from 'src/storage/encrypted-storage.js';
import { StorageMemory } from 'src/storage/storage-memory.js';
import { StorageMemorySynced } from 'src/storage/storage-memory-synced.js';
describe('storage regressions', () => {
it('does not treat $eq as fully resolved when mixed with other operators', async () => {
const storage = StorageMemory.from<{ age: number }>(['age']);
await storage.insertMany([{ age: 25 }, { age: 35 }]);
const results = await storage.find({ age: { $eq: 25, $gt: 30 } });
expect(results).toEqual([]);
});
it('normalizes conflicting lower/upper bounds to strictest constraints', async () => {
const storage = StorageMemory.from<{ age: number }>(['age']);
await storage.insertMany([{ age: 30 }, { age: 31 }, { age: 32 }]);
const lower = await storage.find({ age: { $gt: 30, $gte: 30 } });
expect(lower.map((d) => d.age).sort((a, b) => a - b)).toEqual([31, 32]);
const upper = await storage.find({ age: { $lt: 32, $lte: 32 } });
expect(upper.map((d) => d.age).sort((a, b) => a - b)).toEqual([30, 31]);
});
it('formats plaintext operator values in encrypted filters', async () => {
type Doc = { createdAt: Date; name: string };
const key = await AESKey.fromSeed('storage-regression-test-key');
const base = StorageMemory.from<Record<string, any>>(['createdAt']);
const storage = EncryptedStorage.from<Doc>(base, key, { plaintextKeys: ['createdAt'] });
await storage.insertOne({
createdAt: new Date('2024-01-02T00:00:00.000Z'),
name: 'alice',
});
const results = await storage.find({
createdAt: { $gte: new Date('2024-01-01T00:00:00.000Z') },
});
expect(results).toHaveLength(1);
expect(results[0].name).toBe('alice');
});
it('primes derived child cache in StorageMemorySynced', async () => {
const store = StorageMemory.from<Record<string, any>>();
const child = store.deriveChild<{ value: number }>('child');
await child.insertOne({ value: 42 });
const synced = await StorageMemorySynced.create(store);
const syncedChild = synced.deriveChild<{ value: number }>('child');
const results = await syncedChild.find();
expect(results).toEqual([{ value: 42 }]);
});
});