import { BaseStorage, FindOptions, type IndexDefinition, type Filter, type ComparisonOperators, isOperatorObject, isLogicalKey, } from './base-storage.js'; import { BaseCache, BTreeCache } from 'src/cache/index.js'; /** * Separator used when joining field names to create the index map key. */ const INDEX_KEY_SEP = '\x00'; /** * Normalize an IndexDefinition into a canonical `string[][]` form. * A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`. * An already-nested `string[][]` is returned as-is. */ function normalizeIndexes(indexes?: IndexDefinition): string[][] { if (!indexes || indexes.length === 0) return []; // If the first element is a string, treat the whole array as shorthand. if (typeof indexes[0] === 'string') { return (indexes as string[]).map((field) => [field]); } return indexes as string[][]; } /** * Implementation of BaseStore using Memory as the storage backend. * * @remarks * Documents are keyed internally by an auto-incrementing numeric key. * Optional indexes are backed by B+ Trees, providing O(log n) equality * lookups and O(log n + k) range queries. */ export class StorageMemory< T extends Record = Record, > extends BaseStorage { static from>( indexes?: IndexDefinition, cache?: BaseCache, ): StorageMemory { return new StorageMemory(indexes, cache); } /** Auto-incrementing counter used to generate internal keys. */ private nextKey = 0; /** Primary document store keyed by an opaque internal key. */ private store: Map; /** Secondary index cache (B-Tree or KV implementation). */ private cache: BaseCache; /** The normalized index definitions supplied at construction time. */ private indexDefs: string[][]; /** Lazily-created child storage instances. */ private children: Map>; constructor(indexes?: IndexDefinition, cache?: BaseCache) { super(); this.store = new Map(); this.children = new Map(); this.indexDefs = normalizeIndexes(indexes); this.cache = cache ?? new BTreeCache(); for (const fields of this.indexDefs) { const name = fields.join(INDEX_KEY_SEP); this.cache.registerIndex(name, fields); } } // --------------------------------------------------------------------------- // Abstract method implementations // --------------------------------------------------------------------------- async insertMany(documents: Array): Promise { for (const document of documents) { const key = this.nextKey++; this.store.set(key, document); this.addToIndexes(key, document); this.emit('insert', { value: document }); } } async find(filter?: Filter, options?: FindOptions): Promise { let results: T[]; // Attempt to satisfy the query via an index. const indexed = this.findViaIndex(filter); if (indexed !== null) { results = indexed; } else { // Fall back to a full scan. results = []; for (const [, value] of this.store) { if (this.matchesFilter(value, filter)) { results.push(value); } } } // Apply sort before skip/limit so the window is deterministic. if (options?.sort) { results = this.applySorting(results, options.sort); } const startIndex = options?.skip ?? 0; const endIndex = options?.limit ? startIndex + options.limit : results.length; return results.slice(startIndex, endIndex); } async updateMany( filter: Filter, update: Partial, options: Partial = {}, ): Promise { const itemsToUpdate = this.collectMatches(filter); const startIndex = options.skip ?? 0; const endIndex = options.limit ? startIndex + options.limit : itemsToUpdate.length; const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex); let updated = 0; for (const [key, oldValue] of itemsToProcess) { const updatedValue = { ...oldValue, ...update } as T; // Re-index: remove old entries, store new doc, add new entries. this.removeFromIndexes(key, oldValue); this.store.set(key, updatedValue); this.addToIndexes(key, updatedValue); this.emit('update', { oldValue, value: updatedValue }); updated++; } return updated; } async deleteMany( filter: Filter, options: Partial = {}, ): Promise { const rowsToDelete = this.collectMatches(filter); const startIndex = options.skip ?? 0; const endIndex = options.limit ? startIndex + options.limit : rowsToDelete.length; const rowsToProcess = rowsToDelete.slice(startIndex, endIndex); let deleted = 0; for (const [key, value] of rowsToProcess) { this.removeFromIndexes(key, value); this.store.delete(key); this.emit('delete', { value }); deleted++; } return deleted; } deriveChild(path: string): BaseStorage { if (!this.children.has(path)) { this.children.set(path, new StorageMemory(this.indexDefs, this.cache.createChild())); } return this.children.get(path) as StorageMemory; } // --------------------------------------------------------------------------- // Private helpers — filtering // --------------------------------------------------------------------------- /** * Checks whether a document satisfies a filter. * * Handles top-level logical operators ($and, $or, $not) first via * recursion, then evaluates remaining field-level conditions. */ private matchesFilter(item: T, filter?: Filter): boolean { if (!filter || Object.keys(filter).length === 0) { return true; } // Top-level logical operators. if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false; if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false; if (filter.$not && this.matchesFilter(item, filter.$not)) return false; // Field-level conditions (skip logical operator keys). for (const [key, value] of Object.entries(filter)) { if (isLogicalKey(key)) continue; if (isOperatorObject(value)) { if (!this.matchesOperators(item[key], value)) return false; } else { if (item[key] !== value) return false; } } return true; } /** * Evaluate a set of comparison / string operators against a single field value. * All operators must pass for the field to match. */ private matchesOperators(fieldValue: any, ops: ComparisonOperators): boolean { if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false; if (ops.$ne !== undefined && fieldValue === ops.$ne) return false; if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false; if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false; if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false; if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false; if (ops.$startsWith !== undefined) { if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false; } if (ops.$contains !== undefined) { if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false; } // Field-level $not: invert the enclosed operator set. if (ops.$not !== undefined) { if (this.matchesOperators(fieldValue, ops.$not)) return false; } return true; } /** * Determine whether candidate documents returned by index resolution still * require full filter verification. */ private filterNeedsVerification( filter: Filter | undefined, resolvedFields: string[], ): boolean { if (!filter) return false; const filterKeys = Object.keys(filter); const hasLogicalOps = filterKeys.some(isLogicalKey); return hasLogicalOps || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); } /** * Collect all [internalKey, document] pairs that match a filter. * Uses an index when possible, otherwise falls back to a full scan. */ private collectMatches(filter?: Filter): Array<[number, T]> { const resolution = this.resolveIndexKeys(filter); if (resolution !== null) { const { keys, resolvedFields } = resolution; const needsVerification = this.filterNeedsVerification(filter, resolvedFields); const results: Array<[number, T]> = []; for (const key of keys) { const doc = this.store.get(key); if (!doc) continue; if (needsVerification && !this.matchesFilter(doc, filter)) continue; results.push([key, doc]); } return results; } // Full scan. const results: Array<[number, T]> = []; for (const [key, value] of this.store) { if (this.matchesFilter(value, filter)) { results.push([key, value]); } } return results; } /** * Sort an array of documents according to a sort specification. * Keys map to `1` (ascending) or `-1` (descending). */ private applySorting(items: T[], sort: Record): T[] { const sortEntries = Object.entries(sort); return [...items].sort((a, b) => { for (const [key, direction] of sortEntries) { if (a[key] < b[key]) return -1 * direction; if (a[key] > b[key]) return 1 * direction; } return 0; }); } // --------------------------------------------------------------------------- // Private helpers — indexing // --------------------------------------------------------------------------- /** * Build the B+ Tree key for a document and a set of index fields. * - Single-field indexes return the raw field value. * - Compound indexes return an array of raw field values. * Returns `null` if any required field is missing from the document. */ private buildIndexKey(doc: Record, fields: string[]): any | null { if (fields.length === 1) { if (!(fields[0] in doc)) return null; return doc[fields[0]]; } const parts: any[] = []; for (const field of fields) { if (!(field in doc)) return null; parts.push(doc[field]); } return parts; } /** Register a document in all applicable indexes. */ private addToIndexes(internalKey: number, doc: T): void { for (const fields of this.indexDefs) { const indexKey = this.buildIndexKey(doc, fields); if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); this.cache.insert(name, indexKey, internalKey); } } /** Remove a document from all applicable indexes. */ private removeFromIndexes(internalKey: number, doc: T): void { for (const fields of this.indexDefs) { const indexKey = this.buildIndexKey(doc, fields); if (indexKey === null) continue; const name = fields.join(INDEX_KEY_SEP); this.cache.delete(name, indexKey, internalKey); } } /** * Result of an index resolution attempt. * `keys` is an iterable of candidate internal keys. * `resolvedFields` lists the filter fields fully satisfied by the index, * so callers can skip re-verifying those conditions in matchesFilter. */ private resolveIndexKeys( filter?: Filter, ): { keys: Iterable; resolvedFields: string[] } | null { if (!filter) return null; const filterKeys = Object.keys(filter); if (filterKeys.length === 0) return null; for (const fields of this.indexDefs) { const indexName = fields.join(INDEX_KEY_SEP); if (fields.length === 1) { // --- Single-field index --- const field = fields[0]; if (!(field in filter)) continue; const filterValue = (filter as any)[field]; if (isOperatorObject(filterValue)) { const keys = this.resolveOperatorViaTree(indexName, filterValue); if (keys !== null) return { keys, resolvedFields: [field] }; continue; } // Plain equality. return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] }; } else { // --- Compound index — all fields must be plain equality --- if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) { continue; } const tupleKey = fields.map((f) => (filter as any)[f]); return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] }; } } return null; } /** * Try to resolve an operator filter against a single-field B+ Tree index. * Returns a flat array of matching internal keys, or null if the * operators can't be efficiently served by the tree. * * Supported acceleration: * - `$eq` → point lookup via `.get()` * - `$gt/$gte/$lt/$lte` → range scan via `.range()` * - `$startsWith` → converted to a range scan on the prefix * - `$ne`, `$contains`, `$not` → cannot use index, returns null */ private resolveOperatorViaTree( indexName: string, ops: ComparisonOperators, ): Iterable | null { // Operators that prevent efficient index use. if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null; // $eq is a point lookup. if (ops.$eq !== undefined) { // If $eq is combined with other operators, this path does not fully // resolve the predicate. Let caller fall back to verification/full scan. if ( ops.$gt !== undefined || ops.$gte !== undefined || ops.$lt !== undefined || ops.$lte !== undefined || ops.$startsWith !== undefined ) { return null; } return this.cache.get(indexName, ops.$eq); } // $startsWith is converted to a range scan: "abc" → ["abc", "abd"). if (ops.$startsWith !== undefined) { const prefix = ops.$startsWith; if (prefix.length === 0) return null; const upper = prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); const entries = this.cache.range(indexName, prefix, upper, { lowerInclusive: true, upperInclusive: false, }); return entries; } // Extract range bounds from the remaining operators. // If strict/non-strict variants are both provided, use the stricter bound. let min: any = undefined; let max: any = undefined; let lowerInclusive = true; let upperInclusive = false; if (ops.$gt !== undefined && ops.$gte !== undefined) { if (ops.$gt > ops.$gte) { min = ops.$gt; lowerInclusive = false; } else if (ops.$gt < ops.$gte) { min = ops.$gte; lowerInclusive = true; } else { min = ops.$gt; lowerInclusive = false; } } else if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; } else if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; } if (ops.$lt !== undefined && ops.$lte !== undefined) { if (ops.$lt < ops.$lte) { max = ops.$lt; upperInclusive = false; } else if (ops.$lt > ops.$lte) { max = ops.$lte; upperInclusive = true; } else { max = ops.$lt; upperInclusive = false; } } else if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; } else if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; } if (min === undefined && max === undefined) return null; if (min !== undefined && max !== undefined) { if (min > max) return []; if (min === max && (!lowerInclusive || !upperInclusive)) return []; } return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive }); } /** * Try to answer a `find` query entirely through an index. * Returns `null` when no index can serve the filter, signalling * the caller to fall back to a full scan. * * When the index covers every field in the filter, matchesFilter * is skipped entirely — the B+ Tree has already ensured the * conditions are met. */ private findViaIndex(filter?: Filter): T[] | null { const resolution = this.resolveIndexKeys(filter); if (resolution === null) return null; const { keys, resolvedFields } = resolution; const needsVerification = this.filterNeedsVerification(filter, resolvedFields); const results: T[] = []; for (const key of keys) { const doc = this.store.get(key); if (!doc) continue; if (needsVerification && !this.matchesFilter(doc, filter)) continue; results.push(doc); } return results; } }