521 lines
16 KiB
TypeScript
521 lines
16 KiB
TypeScript
import {
|
|
BaseStorage,
|
|
FindOptions,
|
|
type IndexDefinition,
|
|
type Filter,
|
|
type ComparisonOperators,
|
|
isOperatorObject,
|
|
isLogicalKey,
|
|
} from './base-storage.js';
|
|
import { BaseCache, BTreeCache } from 'src/cache/index.js';
|
|
|
|
/**
|
|
* Separator used when joining field names to create the index map key.
|
|
*/
|
|
const INDEX_KEY_SEP = '\x00';
|
|
|
|
/**
|
|
* Normalize an IndexDefinition into a canonical `string[][]` form.
|
|
* A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`.
|
|
* An already-nested `string[][]` is returned as-is.
|
|
*/
|
|
function normalizeIndexes(indexes?: IndexDefinition): string[][] {
|
|
if (!indexes || indexes.length === 0) return [];
|
|
|
|
// If the first element is a string, treat the whole array as shorthand.
|
|
if (typeof indexes[0] === 'string') {
|
|
return (indexes as string[]).map((field) => [field]);
|
|
}
|
|
|
|
return indexes as string[][];
|
|
}
|
|
|
|
/**
|
|
* Implementation of BaseStore using Memory as the storage backend.
|
|
*
|
|
* @remarks
|
|
* Documents are keyed internally by an auto-incrementing numeric key.
|
|
* Optional indexes are backed by B+ Trees, providing O(log n) equality
|
|
* lookups and O(log n + k) range queries.
|
|
*/
|
|
export class StorageMemory<
|
|
T extends Record<string, any> = Record<string, any>,
|
|
> extends BaseStorage<T> {
|
|
static from<T extends Record<string, any>>(
|
|
indexes?: IndexDefinition,
|
|
cache?: BaseCache,
|
|
): StorageMemory<T> {
|
|
return new StorageMemory<T>(indexes, cache);
|
|
}
|
|
|
|
/** Auto-incrementing counter used to generate internal keys. */
|
|
private nextKey = 0;
|
|
|
|
/** Primary document store keyed by an opaque internal key. */
|
|
private store: Map<number, T>;
|
|
|
|
/** Secondary index cache (B-Tree or KV implementation). */
|
|
private cache: BaseCache;
|
|
|
|
/** The normalized index definitions supplied at construction time. */
|
|
private indexDefs: string[][];
|
|
|
|
/** Lazily-created child storage instances. */
|
|
private children: Map<string, StorageMemory<any>>;
|
|
|
|
constructor(indexes?: IndexDefinition, cache?: BaseCache) {
|
|
super();
|
|
|
|
this.store = new Map();
|
|
this.children = new Map();
|
|
this.indexDefs = normalizeIndexes(indexes);
|
|
|
|
this.cache = cache ?? new BTreeCache();
|
|
for (const fields of this.indexDefs) {
|
|
const name = fields.join(INDEX_KEY_SEP);
|
|
this.cache.registerIndex(name, fields);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Abstract method implementations
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async insertMany(documents: Array<T>): Promise<void> {
|
|
for (const document of documents) {
|
|
const key = this.nextKey++;
|
|
this.store.set(key, document);
|
|
this.addToIndexes(key, document);
|
|
this.emit('insert', { value: document });
|
|
}
|
|
}
|
|
|
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
|
let results: T[];
|
|
|
|
// Attempt to satisfy the query via an index.
|
|
const indexed = this.findViaIndex(filter);
|
|
|
|
if (indexed !== null) {
|
|
results = indexed;
|
|
} else {
|
|
// Fall back to a full scan.
|
|
results = [];
|
|
for (const [, value] of this.store) {
|
|
if (this.matchesFilter(value, filter)) {
|
|
results.push(value);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Apply sort before skip/limit so the window is deterministic.
|
|
if (options?.sort) {
|
|
results = this.applySorting(results, options.sort);
|
|
}
|
|
|
|
const startIndex = options?.skip ?? 0;
|
|
const endIndex = options?.limit
|
|
? startIndex + options.limit
|
|
: results.length;
|
|
|
|
return results.slice(startIndex, endIndex);
|
|
}
|
|
|
|
async updateMany(
|
|
filter: Filter<T>,
|
|
update: Partial<T>,
|
|
options: Partial<FindOptions> = {},
|
|
): Promise<number> {
|
|
const itemsToUpdate = this.collectMatches(filter);
|
|
|
|
const startIndex = options.skip ?? 0;
|
|
const endIndex = options.limit
|
|
? startIndex + options.limit
|
|
: itemsToUpdate.length;
|
|
const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex);
|
|
|
|
let updated = 0;
|
|
for (const [key, oldValue] of itemsToProcess) {
|
|
const updatedValue = { ...oldValue, ...update } as T;
|
|
|
|
// Re-index: remove old entries, store new doc, add new entries.
|
|
this.removeFromIndexes(key, oldValue);
|
|
this.store.set(key, updatedValue);
|
|
this.addToIndexes(key, updatedValue);
|
|
|
|
this.emit('update', { oldValue, value: updatedValue });
|
|
updated++;
|
|
}
|
|
|
|
return updated;
|
|
}
|
|
|
|
async deleteMany(
|
|
filter: Filter<T>,
|
|
options: Partial<FindOptions> = {},
|
|
): Promise<number> {
|
|
const rowsToDelete = this.collectMatches(filter);
|
|
|
|
const startIndex = options.skip ?? 0;
|
|
const endIndex = options.limit
|
|
? startIndex + options.limit
|
|
: rowsToDelete.length;
|
|
const rowsToProcess = rowsToDelete.slice(startIndex, endIndex);
|
|
|
|
let deleted = 0;
|
|
for (const [key, value] of rowsToProcess) {
|
|
this.removeFromIndexes(key, value);
|
|
this.store.delete(key);
|
|
this.emit('delete', { value });
|
|
deleted++;
|
|
}
|
|
|
|
return deleted;
|
|
}
|
|
|
|
deriveChild<C>(path: string): BaseStorage<C> {
|
|
if (!this.children.has(path)) {
|
|
this.children.set(path, new StorageMemory<C>(this.indexDefs, this.cache.createChild()));
|
|
}
|
|
return this.children.get(path) as StorageMemory<C>;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Private helpers — filtering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Checks whether a document satisfies a filter.
|
|
*
|
|
* Handles top-level logical operators ($and, $or, $not) first via
|
|
* recursion, then evaluates remaining field-level conditions.
|
|
*/
|
|
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
|
if (!filter || Object.keys(filter).length === 0) {
|
|
return true;
|
|
}
|
|
|
|
// Top-level logical operators.
|
|
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
|
|
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
|
|
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
|
|
|
|
// Field-level conditions (skip logical operator keys).
|
|
for (const [key, value] of Object.entries(filter)) {
|
|
if (isLogicalKey(key)) continue;
|
|
if (isOperatorObject(value)) {
|
|
if (!this.matchesOperators(item[key], value)) return false;
|
|
} else {
|
|
if (item[key] !== value) return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Evaluate a set of comparison / string operators against a single field value.
|
|
* All operators must pass for the field to match.
|
|
*/
|
|
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
|
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
|
|
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
|
|
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
|
|
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
|
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
|
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
|
|
|
if (ops.$startsWith !== undefined) {
|
|
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
|
|
}
|
|
if (ops.$contains !== undefined) {
|
|
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
|
|
}
|
|
|
|
// Field-level $not: invert the enclosed operator set.
|
|
if (ops.$not !== undefined) {
|
|
if (this.matchesOperators(fieldValue, ops.$not)) return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Determine whether candidate documents returned by index resolution still
|
|
* require full filter verification.
|
|
*/
|
|
private filterNeedsVerification(
|
|
filter: Filter<T> | undefined,
|
|
resolvedFields: string[],
|
|
): boolean {
|
|
if (!filter) return false;
|
|
const filterKeys = Object.keys(filter);
|
|
const hasLogicalOps = filterKeys.some(isLogicalKey);
|
|
return hasLogicalOps
|
|
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
|
|
}
|
|
|
|
/**
|
|
* Collect all [internalKey, document] pairs that match a filter.
|
|
* Uses an index when possible, otherwise falls back to a full scan.
|
|
*/
|
|
private collectMatches(filter?: Filter<T>): Array<[number, T]> {
|
|
const resolution = this.resolveIndexKeys(filter);
|
|
|
|
if (resolution !== null) {
|
|
const { keys, resolvedFields } = resolution;
|
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
|
|
|
const results: Array<[number, T]> = [];
|
|
for (const key of keys) {
|
|
const doc = this.store.get(key);
|
|
if (!doc) continue;
|
|
if (needsVerification && !this.matchesFilter(doc, filter)) continue;
|
|
results.push([key, doc]);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// Full scan.
|
|
const results: Array<[number, T]> = [];
|
|
for (const [key, value] of this.store) {
|
|
if (this.matchesFilter(value, filter)) {
|
|
results.push([key, value]);
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Sort an array of documents according to a sort specification.
|
|
* Keys map to `1` (ascending) or `-1` (descending).
|
|
*/
|
|
private applySorting(items: T[], sort: Record<string, 1 | -1>): T[] {
|
|
const sortEntries = Object.entries(sort);
|
|
return [...items].sort((a, b) => {
|
|
for (const [key, direction] of sortEntries) {
|
|
if (a[key] < b[key]) return -1 * direction;
|
|
if (a[key] > b[key]) return 1 * direction;
|
|
}
|
|
return 0;
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Private helpers — indexing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build the B+ Tree key for a document and a set of index fields.
|
|
* - Single-field indexes return the raw field value.
|
|
* - Compound indexes return an array of raw field values.
|
|
* Returns `null` if any required field is missing from the document.
|
|
*/
|
|
private buildIndexKey(doc: Record<string, any>, fields: string[]): any | null {
|
|
if (fields.length === 1) {
|
|
if (!(fields[0] in doc)) return null;
|
|
return doc[fields[0]];
|
|
}
|
|
|
|
const parts: any[] = [];
|
|
for (const field of fields) {
|
|
if (!(field in doc)) return null;
|
|
parts.push(doc[field]);
|
|
}
|
|
return parts;
|
|
}
|
|
|
|
/** Register a document in all applicable indexes. */
|
|
private addToIndexes(internalKey: number, doc: T): void {
|
|
for (const fields of this.indexDefs) {
|
|
const indexKey = this.buildIndexKey(doc, fields);
|
|
if (indexKey === null) continue;
|
|
|
|
const name = fields.join(INDEX_KEY_SEP);
|
|
this.cache.insert(name, indexKey, internalKey);
|
|
}
|
|
}
|
|
|
|
/** Remove a document from all applicable indexes. */
|
|
private removeFromIndexes(internalKey: number, doc: T): void {
|
|
for (const fields of this.indexDefs) {
|
|
const indexKey = this.buildIndexKey(doc, fields);
|
|
if (indexKey === null) continue;
|
|
|
|
const name = fields.join(INDEX_KEY_SEP);
|
|
this.cache.delete(name, indexKey, internalKey);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Result of an index resolution attempt.
|
|
* `keys` is an iterable of candidate internal keys.
|
|
* `resolvedFields` lists the filter fields fully satisfied by the index,
|
|
* so callers can skip re-verifying those conditions in matchesFilter.
|
|
*/
|
|
private resolveIndexKeys(
|
|
filter?: Filter<T>,
|
|
): { keys: Iterable<number>; resolvedFields: string[] } | null {
|
|
if (!filter) return null;
|
|
const filterKeys = Object.keys(filter);
|
|
if (filterKeys.length === 0) return null;
|
|
|
|
for (const fields of this.indexDefs) {
|
|
const indexName = fields.join(INDEX_KEY_SEP);
|
|
|
|
if (fields.length === 1) {
|
|
// --- Single-field index ---
|
|
const field = fields[0];
|
|
if (!(field in filter)) continue;
|
|
|
|
const filterValue = (filter as any)[field];
|
|
|
|
if (isOperatorObject(filterValue)) {
|
|
const keys = this.resolveOperatorViaTree(indexName, filterValue);
|
|
if (keys !== null) return { keys, resolvedFields: [field] };
|
|
continue;
|
|
}
|
|
|
|
// Plain equality.
|
|
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
|
|
} else {
|
|
// --- Compound index — all fields must be plain equality ---
|
|
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
|
|
continue;
|
|
}
|
|
|
|
const tupleKey = fields.map((f) => (filter as any)[f]);
|
|
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Try to resolve an operator filter against a single-field B+ Tree index.
|
|
* Returns a flat array of matching internal keys, or null if the
|
|
* operators can't be efficiently served by the tree.
|
|
*
|
|
* Supported acceleration:
|
|
* - `$eq` → point lookup via `.get()`
|
|
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
|
|
* - `$startsWith` → converted to a range scan on the prefix
|
|
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
|
|
*/
|
|
private resolveOperatorViaTree(
|
|
indexName: string,
|
|
ops: ComparisonOperators<any>,
|
|
): Iterable<number> | null {
|
|
// Operators that prevent efficient index use.
|
|
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
|
|
|
|
// $eq is a point lookup.
|
|
if (ops.$eq !== undefined) {
|
|
// If $eq is combined with other operators, this path does not fully
|
|
// resolve the predicate. Let caller fall back to verification/full scan.
|
|
if (
|
|
ops.$gt !== undefined
|
|
|| ops.$gte !== undefined
|
|
|| ops.$lt !== undefined
|
|
|| ops.$lte !== undefined
|
|
|| ops.$startsWith !== undefined
|
|
) {
|
|
return null;
|
|
}
|
|
return this.cache.get(indexName, ops.$eq);
|
|
}
|
|
|
|
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
|
|
if (ops.$startsWith !== undefined) {
|
|
const prefix = ops.$startsWith;
|
|
if (prefix.length === 0) return null;
|
|
const upper = prefix.slice(0, -1)
|
|
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
|
|
const entries = this.cache.range(indexName, prefix, upper, {
|
|
lowerInclusive: true,
|
|
upperInclusive: false,
|
|
});
|
|
return entries;
|
|
}
|
|
|
|
// Extract range bounds from the remaining operators.
|
|
// If strict/non-strict variants are both provided, use the stricter bound.
|
|
let min: any = undefined;
|
|
let max: any = undefined;
|
|
let lowerInclusive = true;
|
|
let upperInclusive = false;
|
|
|
|
if (ops.$gt !== undefined && ops.$gte !== undefined) {
|
|
if (ops.$gt > ops.$gte) {
|
|
min = ops.$gt;
|
|
lowerInclusive = false;
|
|
} else if (ops.$gt < ops.$gte) {
|
|
min = ops.$gte;
|
|
lowerInclusive = true;
|
|
} else {
|
|
min = ops.$gt;
|
|
lowerInclusive = false;
|
|
}
|
|
} else if (ops.$gt !== undefined) {
|
|
min = ops.$gt;
|
|
lowerInclusive = false;
|
|
} else if (ops.$gte !== undefined) {
|
|
min = ops.$gte;
|
|
lowerInclusive = true;
|
|
}
|
|
|
|
if (ops.$lt !== undefined && ops.$lte !== undefined) {
|
|
if (ops.$lt < ops.$lte) {
|
|
max = ops.$lt;
|
|
upperInclusive = false;
|
|
} else if (ops.$lt > ops.$lte) {
|
|
max = ops.$lte;
|
|
upperInclusive = true;
|
|
} else {
|
|
max = ops.$lt;
|
|
upperInclusive = false;
|
|
}
|
|
} else if (ops.$lt !== undefined) {
|
|
max = ops.$lt;
|
|
upperInclusive = false;
|
|
} else if (ops.$lte !== undefined) {
|
|
max = ops.$lte;
|
|
upperInclusive = true;
|
|
}
|
|
|
|
if (min === undefined && max === undefined) return null;
|
|
if (min !== undefined && max !== undefined) {
|
|
if (min > max) return [];
|
|
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
|
|
}
|
|
|
|
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
|
|
}
|
|
|
|
/**
|
|
* Try to answer a `find` query entirely through an index.
|
|
* Returns `null` when no index can serve the filter, signalling
|
|
* the caller to fall back to a full scan.
|
|
*
|
|
* When the index covers every field in the filter, matchesFilter
|
|
* is skipped entirely — the B+ Tree has already ensured the
|
|
* conditions are met.
|
|
*/
|
|
private findViaIndex(filter?: Filter<T>): T[] | null {
|
|
const resolution = this.resolveIndexKeys(filter);
|
|
if (resolution === null) return null;
|
|
|
|
const { keys, resolvedFields } = resolution;
|
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
|
|
|
const results: T[] = [];
|
|
for (const key of keys) {
|
|
const doc = this.store.get(key);
|
|
if (!doc) continue;
|
|
if (needsVerification && !this.matchesFilter(doc, filter)) continue;
|
|
results.push(doc);
|
|
}
|
|
return results;
|
|
}
|
|
}
|