Harvmaster/Hashpass-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Cache primtive

Browse Source
This commit is contained in:
Harvey Zuccon
2026-02-25 17:28:04 +11:00
parent f7c89046d1
commit 5c39d8add2
12 changed files with 554 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
src/storage/storage-memory.ts
View File

@@ -7,7 +7,7 @@ import {
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, BTreeCache } from 'src/cache/index.js';
/**
* Separator used when joining field names to create the index map key.
@@ -30,19 +30,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
return indexes as string[][];
}
/**
* Comparator for compound index keys (arrays of raw values).
* Compares element-by-element using native `<` / `>` operators.
*/
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* Implementation of BaseStore using Memory as the storage backend.
*
@@ -56,8 +43,9 @@ export class StorageMemory<
> extends BaseStorage<T> {
static from<T extends Record<string, any>>(
indexes?: IndexDefinition,
cache?: BaseCache,
): StorageMemory<T> {
return new StorageMemory<T>(indexes);
return new StorageMemory<T>(indexes, cache);
}
/** Auto-incrementing counter used to generate internal keys. */
@@ -66,12 +54,8 @@ export class StorageMemory<
/** Primary document store keyed by an opaque internal key. */
private store: Map<number, T>;
/**
* Secondary indexes backed by B+ Trees.
* Map key = index name (joined field names).
* Map value = B+ Tree mapping index keys to sets of internal document keys.
*/
private indexes: Map<string, BPlusTree<any, number>>;
/** Secondary index cache (B-Tree or KV implementation). */
private cache: BaseCache;
/** The normalized index definitions supplied at construction time. */
private indexDefs: string[][];
@@ -79,19 +63,17 @@ export class StorageMemory<
/** Lazily-created child storage instances. */
private children: Map<string, StorageMemory<any>>;
constructor(indexes?: IndexDefinition) {
constructor(indexes?: IndexDefinition, cache?: BaseCache) {
super();
this.store = new Map();
this.children = new Map();
this.indexDefs = normalizeIndexes(indexes);
// Create a B+ Tree for each index definition.
this.indexes = new Map();
this.cache = cache ?? new BTreeCache();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
this.cache.registerIndex(name, fields);
}
}
@@ -193,7 +175,7 @@ export class StorageMemory<
deriveChild<C>(path: string): BaseStorage<C> {
if (!this.children.has(path)) {
this.children.set(path, new StorageMemory<C>(this.indexDefs));
this.children.set(path, new StorageMemory<C>(this.indexDefs, this.cache.createChild()));
}
return this.children.get(path) as StorageMemory<C>;
}
@@ -257,6 +239,21 @@ export class StorageMemory<
return true;
}
/**
* Determine whether candidate documents returned by index resolution still
* require full filter verification.
*/
private filterNeedsVerification(
filter: Filter<T> | undefined,
resolvedFields: string[],
): boolean {
if (!filter) return false;
const filterKeys = Object.keys(filter);
const hasLogicalOps = filterKeys.some(isLogicalKey);
return hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
@@ -266,10 +263,7 @@ export class StorageMemory<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: Array<[number, T]> = [];
for (const key of keys) {
@@ -337,7 +331,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.insert(indexKey, internalKey);
this.cache.insert(name, indexKey, internalKey);
}
}
@@ -348,7 +342,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.delete(indexKey, internalKey);
this.cache.delete(name, indexKey, internalKey);
}
}
@@ -367,7 +361,6 @@ export class StorageMemory<
for (const fields of this.indexDefs) {
const indexName = fields.join(INDEX_KEY_SEP);
const btree = this.indexes.get(indexName)!;
if (fields.length === 1) {
// --- Single-field index ---
@@ -377,13 +370,13 @@ export class StorageMemory<
const filterValue = (filter as any)[field];
if (isOperatorObject(filterValue)) {
const keys = this.resolveOperatorViaTree(btree, filterValue);
const keys = this.resolveOperatorViaTree(indexName, filterValue);
if (keys !== null) return { keys, resolvedFields: [field] };
continue;
}
// Plain equality.
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
} else {
// --- Compound index — all fields must be plain equality ---
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
@@ -391,7 +384,7 @@ export class StorageMemory<
}
const tupleKey = fields.map((f) => (filter as any)[f]);
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
}
}
@@ -410,7 +403,7 @@ export class StorageMemory<
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
indexName: string,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
// Operators that prevent efficient index use.
@@ -418,7 +411,18 @@ export class StorageMemory<
// $eq is a point lookup.
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
// If $eq is combined with other operators, this path does not fully
// resolve the predicate. Let caller fall back to verification/full scan.
if (
ops.$gt !== undefined
|| ops.$gte !== undefined
|| ops.$lt !== undefined
|| ops.$lte !== undefined
|| ops.$startsWith !== undefined
) {
return null;
}
return this.cache.get(indexName, ops.$eq);
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
@@ -427,43 +431,65 @@ export class StorageMemory<
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = btree.range(prefix, upper, {
const entries = this.cache.range(indexName, prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return this.flattenEntryKeys(entries);
return entries;
}
// Extract range bounds from the remaining operators.
// If strict/non-strict variants are both provided, use the stricter bound.
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;
let upperInclusive = false;
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
if (ops.$gt !== undefined && ops.$gte !== undefined) {
if (ops.$gt > ops.$gte) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gt < ops.$gte) {
min = ops.$gte;
lowerInclusive = true;
} else {
min = ops.$gt;
lowerInclusive = false;
}
} else if (ops.$gt !== undefined) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gte !== undefined) {
min = ops.$gte;
lowerInclusive = true;
}
if (ops.$lt !== undefined && ops.$lte !== undefined) {
if (ops.$lt < ops.$lte) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lt > ops.$lte) {
max = ops.$lte;
upperInclusive = true;
} else {
max = ops.$lt;
upperInclusive = false;
}
} else if (ops.$lt !== undefined) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lte !== undefined) {
max = ops.$lte;
upperInclusive = true;
}
if (min === undefined && max === undefined) return null;
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
return this.flattenEntryKeys(entries);
}
/**
* Flatten B+ Tree range results into a flat array of internal keys.
* Uses an array instead of a Set — no hash overhead, no deduplication
* needed because each internal key only appears under one index key.
*/
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
if (min !== undefined && max !== undefined) {
if (min > max) return [];
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
}
return result;
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
}
/**
@@ -480,13 +506,7 @@ export class StorageMemory<
if (resolution === null) return null;
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
// Logical operators ($and/$or/$not) are never resolved by the index,
// and any unresolved field conditions also require verification.
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: T[] = [];
for (const key of keys) {