From 77593fe3b47e131645568485bbb9cd1308b8b686 Mon Sep 17 00:00:00 2001 From: Harvey Zuccon Date: Wed, 25 Feb 2026 15:35:04 +1100 Subject: [PATCH] Plaintext values on encryption --- benchmarks/sekp256k1.ts | 1 + benchmarks/storage.ts | 85 ++++++--- package.json | 3 + src/storage/base-storage.ts | 60 +++++- src/storage/encrypted-storage.ts | 269 +++++++++++++++++++-------- src/storage/storage-localstorage.ts | 57 +++++- src/storage/storage-memory-synced.ts | 8 +- src/storage/storage-memory.ts | 67 +++++-- 8 files changed, 407 insertions(+), 143 deletions(-) diff --git a/benchmarks/sekp256k1.ts b/benchmarks/sekp256k1.ts index 411397b..3abbd7c 100644 --- a/benchmarks/sekp256k1.ts +++ b/benchmarks/sekp256k1.ts @@ -7,6 +7,7 @@ const alicePublicKey = await alice.getPublicKey(); const bobPublicKey = await bob.getPublicKey(); const keyStart = performance.now(); + const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey); const bobSharedSecret = await bob.getSharedSecret(alicePublicKey); diff --git a/benchmarks/storage.ts b/benchmarks/storage.ts index 4194e9e..9bb350d 100644 --- a/benchmarks/storage.ts +++ b/benchmarks/storage.ts @@ -49,7 +49,7 @@ function fmtOps(ops: number): string { /** * Run a full suite of benchmarks against a given storage instance. */ -async function benchmarkStorage(label: string, storage: BaseStorage, docs: Doc[]) { +async function benchmarkStorage(label: string, storage: BaseStorage, docs: Doc[], supportsRangeOps = true) { const count = docs.length; console.log(`\n${'='.repeat(60)}`); console.log(` ${label} (${count.toLocaleString()} documents)`); @@ -59,56 +59,75 @@ async function benchmarkStorage(label: string, storage: BaseStorage, docs: const insertMs = await time(async () => { await storage.insertMany(docs); }); - console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`); + console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`); // --- Find all (no filter) --- const findAllMs = await time(async () => { await storage.find(); }); - console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`); + console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`); - // --- Find by indexed field (single-key lookup, repeated) --- + // --- Find by indexed field (equality) --- const lookupCount = Math.min(count, 1_000); const findIndexedMs = await time(async () => { for (let i = 0; i < lookupCount; i++) { - await storage.findOne({ id: `id-${i}` } as Partial); + await storage.findOne({ id: `id-${i}` }); } }); - console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`); + console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`); - // --- Find by non-indexed field (full scan, repeated) --- + // --- Find by non-indexed field (full scan) --- const scanCount = Math.min(count, 1_000); const findScanMs = await time(async () => { for (let i = 0; i < scanCount; i++) { - await storage.findOne({ email: `user-${i}@test.com` } as Partial); + await storage.findOne({ email: `user-${i}@test.com` }); } }); - console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`); + console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`); + + // --- Find with $gte / $lt range (full scan) --- + if (supportsRangeOps) { + const rangeCount = Math.min(count, 100); + let rangeTotal = 0; + const findRangeMs = await time(async () => { + for (let i = 0; i < rangeCount; i++) { + const results = await storage.find({ age: { $gte: 30, $lt: 40 } }); + rangeTotal += results.length; + } + }); + console.log(` find $gte/$lt ${findRangeMs.toFixed(2)}ms (${fmtOps((rangeCount / findRangeMs) * 1000)} ops/sec) [${rangeCount} queries, ~${Math.round(rangeTotal / rangeCount)} hits/query]`); + + // --- Find with combined equality + operator (index narrows, operator verifies) --- + const comboCount = Math.min(count, 1_000); + const findComboMs = await time(async () => { + for (let i = 0; i < comboCount; i++) { + await storage.find({ id: `id-${i}`, age: { $gte: 20 } }); + } + }); + console.log(` find idx+operator ${findComboMs.toFixed(2)}ms (${fmtOps((comboCount / findComboMs) * 1000)} ops/sec) [${comboCount} queries]`); + } // --- Update by indexed field --- const updateCount = Math.min(count, 1_000); const updateMs = await time(async () => { for (let i = 0; i < updateCount; i++) { - await storage.updateOne( - { id: `id-${i}` } as Partial, - { age: 99 } as Partial, - ); + await storage.updateOne({ id: `id-${i}` }, { name: `updated-${i}` }); } }); - console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`); + console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`); // --- Delete by indexed field --- const deleteCount = Math.min(count, 1_000); const deleteMs = await time(async () => { for (let i = 0; i < deleteCount; i++) { - await storage.deleteOne({ id: `id-${i}` } as Partial); + await storage.deleteOne({ id: `id-${i}` }); } }); - console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`); + console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`); // --- Verify remaining count --- const remaining = await storage.find(); - console.log(` remaining docs: ${remaining.length.toLocaleString()}`); + console.log(` remaining docs: ${remaining.length.toLocaleString()}`); } // --------------------------------------------------------------------------- @@ -132,7 +151,7 @@ for (const count of DOC_COUNTS) { } // --------------------------------------------------------------------------- -// EncryptedStorage — crypto overhead dominates, so use smaller counts +// EncryptedStorage — with plaintextKeys for range queries // --------------------------------------------------------------------------- const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000]; @@ -141,15 +160,29 @@ const encryptionKey = await AESKey.fromSeed('benchmark-key'); for (const count of ENCRYPTED_DOC_COUNTS) { const docs = generateDocs(count); - // Encrypted + indexed backing store. - const encBase = StorageMemory.from>(['id', 'name']); - const encrypted = EncryptedStorage.from(encBase, encryptionKey); - await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs); + // No indexes + plaintextKeys — range queries on age work. + const encBaseNoIdx = StorageMemory.from>(); + const encNoIdx = EncryptedStorage.from(encBaseNoIdx, encryptionKey, { + plaintextKeys: ['age'], + }); + await benchmarkStorage('Encrypted (no indexes, plaintextKeys: age)', encNoIdx, docs); - // Encrypted + no-index backing store. - const encBaseNoIdx = StorageMemory.from>(); - const encryptedNoIdx = EncryptedStorage.from(encBaseNoIdx, encryptionKey); - await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs); + // Indexed + plaintextKeys — range queries on age work. + const encBaseA = StorageMemory.from>(['id', 'name']); + const encA = EncryptedStorage.from(encBaseA, encryptionKey, { + plaintextKeys: ['age'], + }); + await benchmarkStorage('Encrypted (indexed, plaintextKeys: age)', encA, docs); + + // Indexed, fully encrypted — same indexes but no plaintext keys. + const encBaseB = StorageMemory.from>(['id', 'name']); + const encB = EncryptedStorage.from(encBaseB, encryptionKey); + await benchmarkStorage('Encrypted (indexed, fully encrypted)', encB, docs, false); + + // No indexes, fully encrypted — worst case. + const encBaseC = StorageMemory.from>(); + const encC = EncryptedStorage.from(encBaseC, encryptionKey); + await benchmarkStorage('Encrypted (no indexes, fully encrypted)', encC, docs, false); } console.log('\nDone.\n'); diff --git a/package.json b/package.json index 8d2ee27..ae99315 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,9 @@ "format": "prettier --write .", "test": "echo \"Error: no test specified\" && exit 1", + "benchmark:sha256": "tsx benchmarks/sha256.ts", + "benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts", + "benchmark:encryption": "tsx benchmarks/sekp256k1.ts", "benchmark:storage": "tsx benchmarks/storage.ts" }, "keywords": [], diff --git a/src/storage/base-storage.ts b/src/storage/base-storage.ts index fbea6b9..e85a095 100644 --- a/src/storage/base-storage.ts +++ b/src/storage/base-storage.ts @@ -19,6 +19,54 @@ export type FindOptions = { */ export type IndexDefinition = string[] | string[][]; +/** + * MongoDB-style comparison operators for a single field value. + */ +export type ComparisonOperators = { + $eq?: V; + $ne?: V; + $lt?: V; + $lte?: V; + $gt?: V; + $gte?: V; +}; + +/** + * A filter value for a single field — either a plain value (equality shorthand) + * or an object of comparison operators. + */ +export type FieldFilter = V | ComparisonOperators; + +/** + * Query filter that supports both equality shorthand and comparison operators. + * + * @example + * // Equality shorthand + * { name: 'foo' } + * + * // Comparison operators + * { age: { $gte: 18, $lt: 65 } } + * + * // Combined + * { name: 'foo', age: { $gte: 18 } } + */ +export type Filter> = { + [K in keyof T]?: FieldFilter; +}; + +/** + * Detect whether a filter value is an operator object (e.g. `{ $lt: 50 }`) + * rather than a plain value. Guards against Date and Array which are objects + * but represent document values, not operators. + */ +export function isOperatorObject(value: unknown): value is ComparisonOperators { + return value !== null + && typeof value === 'object' + && !Array.isArray(value) + && !(value instanceof Date) + && Object.keys(value).some((k) => k.startsWith('$')); +} + export type StorageEvent> = { insert: { value: T; @@ -54,7 +102,7 @@ export abstract class BaseStorage< * Find a single document that matches the filter * @param filter MongoDB-like query filter */ - async findOne(filter?: Partial): Promise { + async findOne(filter?: Filter): Promise { const results = await this.find(filter); return results.length > 0 ? results[0] : null; } @@ -64,7 +112,7 @@ export abstract class BaseStorage< * @param filter MongoDB-like query filter * @param options Query options (limit, skip, sort) */ - abstract find(filter?: Partial, options?: FindOptions): Promise; + abstract find(filter?: Filter, options?: FindOptions): Promise; /** * Update a document that matches the filter @@ -72,7 +120,7 @@ export abstract class BaseStorage< * @param update Document or fields to update * @returns True if a document was updated, false otherwise */ - async updateOne(filter: Partial, update?: Partial): Promise { + async updateOne(filter: Filter, update?: Partial): Promise { const results = await this.updateMany(filter, update, { limit: 1 }); return results > 0; } @@ -85,7 +133,7 @@ export abstract class BaseStorage< * @returns Number of documents updated */ abstract updateMany( - filter: Partial, + filter: Filter, update: Partial, options?: Partial, ): Promise; @@ -95,7 +143,7 @@ export abstract class BaseStorage< * @param filter Query to match the document to delete * @returns True if a document was deleted, false otherwise */ - async deleteOne(filter: Partial): Promise { + async deleteOne(filter: Filter): Promise { const results = await this.deleteMany(filter, { limit: 1 }); return results > 0; } @@ -107,7 +155,7 @@ export abstract class BaseStorage< * @returns Number of documents deleted */ abstract deleteMany( - filter: Partial, + filter: Filter, options: Partial, ): Promise; diff --git a/src/storage/encrypted-storage.ts b/src/storage/encrypted-storage.ts index 7bb9bcc..032c1df 100644 --- a/src/storage/encrypted-storage.ts +++ b/src/storage/encrypted-storage.ts @@ -3,15 +3,29 @@ import { Packr } from 'msgpackr'; import { AESKey } from 'src/crypto/aes-key.js'; import { Bytes } from 'src/crypto/bytes.js'; -import { BaseStorage, type FindOptions } from './base-storage.js'; +import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js'; -import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; +import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; + +export type EncryptedStorageOptions = { + /** + * Fields that should be stored in plaintext (not encrypted). + * These fields retain their original types in the backing store, which + * allows comparison operators ($lt, $gt, etc.) to work on them. + * All other fields are encrypted. + */ + plaintextKeys?: string[]; +}; export class EncryptedStorage< T extends Record = Record, > extends BaseStorage { - static from(storage: BaseStorage>, key: AESKey) { - return new EncryptedStorage(storage, key); + static from( + storage: BaseStorage>, + key: AESKey, + options?: EncryptedStorageOptions, + ) { + return new EncryptedStorage(storage, key, options); } private readonly msgpackr = new Packr({ @@ -33,34 +47,32 @@ export class EncryptedStorage< */ private readonly decryptCache = new Map(); + /** Set of field names that are stored in plaintext (not encrypted). */ + private readonly plaintextKeys: Set; + constructor( - private readonly storage: BaseStorage>, + private readonly storage: BaseStorage>, private readonly key: AESKey, + options?: EncryptedStorageOptions, ) { super(); - - // Forward events from the underlying storage, decrypting the data - this.storage.on('insert', async (event) => { - // De-crypt the value before emitting the event. - const decryptedValue = await this.convertToDecrypted(event.value as Record); - // Re-emit the insert event with the original payload. + this.plaintextKeys = new Set(options?.plaintextKeys ?? []); + + // Forward events from the underlying storage, converting data back. + this.storage.on('insert', async (event) => { + const decryptedValue = await this.convertFromStorage(event.value); this.emit('insert', { value: decryptedValue }); }); this.storage.on('update', async (event) => { - // Decrypt both old and new values before re-emitting. - const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record); - const decryptedValue = await this.convertToDecrypted(event.value as Record); - + const decryptedOldValue = await this.convertFromStorage(event.oldValue); + const decryptedValue = await this.convertFromStorage(event.value); this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue }); }); this.storage.on('delete', async (event) => { - // De-crypt the value before emitting the event. - const decryptedValue = await this.convertToDecrypted(event.value as Record); - - // Re-emit the delete event with the original payload. + const decryptedValue = await this.convertFromStorage(event.value); this.emit('delete', { value: decryptedValue }); }); @@ -70,53 +82,83 @@ export class EncryptedStorage< } async insertMany(documents: Array): Promise { - // Encrypt all documents in parallel. - const encrypted = await Promise.all( - documents.map((doc) => this.convertToEncrypted(doc)), + const converted = await Promise.all( + documents.map((doc) => this.convertForStorage(doc)), ); - await this.storage.insertMany(encrypted); + await this.storage.insertMany(converted); } - async find(filter?: Partial, options?: FindOptions): Promise { - const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined; - const documents = await this.storage.find(encryptedFilter, options); + async find(filter?: Filter, options?: FindOptions): Promise { + const convertedFilter = filter + ? await this.convertFilterForStorage(filter) + : undefined; + const documents = await this.storage.find(convertedFilter, options); return Promise.all( - documents.map(async (document) => this.convertToDecrypted(document)), + documents.map((doc) => this.convertFromStorage(doc)), ); } async updateMany( - filter: Partial, + filter: Filter, update: Partial, options: Partial = {}, ): Promise { - const encryptedFilter = await this.convertToEncrypted(filter); - const encryptedUpdate = await this.convertToEncrypted(update); - return this.storage.updateMany(encryptedFilter, encryptedUpdate, options); + const convertedFilter = await this.convertFilterForStorage(filter); + const convertedUpdate = await this.convertForStorage(update); + return this.storage.updateMany(convertedFilter, convertedUpdate, options); } async deleteMany( - filter: Partial, + filter: Filter, options: Partial = {}, ): Promise { - const encryptedFilter = await this.convertToEncrypted(filter); - return this.storage.deleteMany(encryptedFilter, options); + const convertedFilter = await this.convertFilterForStorage(filter); + return this.storage.deleteMany(convertedFilter, options); } deriveChild(path: string): BaseStorage { - return EncryptedStorage.from(this.storage.deriveChild(path), this.key); + return EncryptedStorage.from( + this.storage.deriveChild(path), + this.key, + { plaintextKeys: [...this.plaintextKeys] }, + ); } - private async convertToEncrypted( + // --------------------------------------------------------------------------- + // Storage conversion — documents (insert/update values) + // --------------------------------------------------------------------------- + + /** + * Convert a document for storage. Encrypted fields are encrypted; + * plaintext fields are passed through with value formatting only. + */ + private async convertForStorage( document: Partial, - ): Promise> { - const encrypted: Record = {}; + ): Promise> { + const result: Record = {}; const formattedDocument = this.formatDocumentForEncryption(document); const entries = Object.entries(formattedDocument); - // Encrypt all fields in parallel, using the cache when possible. - const results = await Promise.all( - entries.map(async ([key, value]) => { + // Split into plaintext and encrypted fields. + const plaintextEntries: Array<[string, any]> = []; + const encryptedEntries: Array<[string, any]> = []; + + for (const entry of entries) { + if (this.plaintextKeys.has(entry[0])) { + plaintextEntries.push(entry); + } else { + encryptedEntries.push(entry); + } + } + + // Plaintext fields pass through directly. + for (const [key, value] of plaintextEntries) { + result[key] = value; + } + + // Encrypt fields in parallel, with memoization. + const encrypted = await Promise.all( + encryptedEntries.map(async ([key, value]) => { const bin = this.msgpackr.pack(value); const cacheKey = Bytes.from(bin).toBase64(); @@ -131,21 +173,36 @@ export class EncryptedStorage< }), ); - for (const [key, ciphertext] of results) { - encrypted[key] = ciphertext; + for (const [key, ciphertext] of encrypted) { + result[key] = ciphertext; } - return encrypted; + return result; } - private async convertToDecrypted( - document: Record, + /** + * Convert a stored document back to its original form. Encrypted fields + * are decrypted; plaintext fields are passed through with value formatting. + */ + private async convertFromStorage( + document: Record, ): Promise { const entries = Object.entries(document); - // Decrypt all fields in parallel, using the cache when possible. - const results = await Promise.all( - entries.map(async ([key, ciphertext]) => { + const plaintextEntries: Array<[string, any]> = []; + const encryptedEntries: Array<[string, any]> = []; + + for (const entry of entries) { + if (this.plaintextKeys.has(entry[0])) { + plaintextEntries.push(entry); + } else { + encryptedEntries.push(entry); + } + } + + // Decrypt encrypted fields in parallel, with memoization. + const decrypted = await Promise.all( + encryptedEntries.map(async ([key, ciphertext]) => { let value = this.decryptCache.get(ciphertext); if (value === undefined) { const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext)); @@ -156,70 +213,122 @@ export class EncryptedStorage< }), ); - const decrypted: Record = {}; - for (const [key, value] of results) { - decrypted[key] = value; + const result: Record = {}; + + for (const [key, value] of plaintextEntries) { + result[key] = value; + } + for (const [key, value] of decrypted) { + result[key] = value; } - const decodedDocument = this.formatDocumentFromDecryption(decrypted); + const decodedDocument = this.formatDocumentFromDecryption(result); return decodedDocument as T; } - private formatDocumentForEncryption(document: any): any { - // First, iterate through each key and value in the document and format the value for encryption. - const formattedDocument: any = {}; + // --------------------------------------------------------------------------- + // Storage conversion — filters (may contain operator objects) + // --------------------------------------------------------------------------- + /** + * Convert a query filter for storage. Handles both plain equality values + * and operator objects. + * + * - Plaintext fields: values and operator objects pass through as-is. + * - Encrypted fields: plain values are encrypted. Operator objects throw + * because range comparisons are meaningless on ciphertext. + */ + private async convertFilterForStorage( + filter: Filter, + ): Promise>> { + const result: Record = {}; + const entries = Object.entries(filter); + + const encryptionTasks: Array> = []; + + for (const [key, value] of entries) { + if (this.plaintextKeys.has(key)) { + // Plaintext field — pass through (including operator objects). + result[key] = isOperatorObject(value) + ? value + : this.formatValueForEncryption(value); + } else if (isOperatorObject(value)) { + // Encrypted field with an operator — not supported. + throw new Error( + `Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` + + `Add '${key}' to plaintextKeys if you need range queries on this field.`, + ); + } else { + // Encrypted field with a plain equality value — encrypt it. + const formatted = this.formatValueForEncryption(value); + encryptionTasks.push( + (async () => { + const bin = this.msgpackr.pack(formatted); + const cacheKey = Bytes.from(bin).toBase64(); + + let ciphertext = this.encryptCache.get(cacheKey); + if (ciphertext === undefined) { + const encryptedValue = await this.key.encrypt(bin, true); + ciphertext = encryptedValue.toBase64(); + this.encryptCache.set(cacheKey, ciphertext); + } + + return [key, ciphertext] as const; + })(), + ); + } + } + + const encryptedResults = await Promise.all(encryptionTasks); + for (const [key, ciphertext] of encryptedResults) { + result[key] = ciphertext; + } + + return result; + } + + // --------------------------------------------------------------------------- + // Value formatting + // --------------------------------------------------------------------------- + + private formatDocumentForEncryption(document: any): any { + const formattedDocument: any = {}; for (const [key, value] of Object.entries(document)) { formattedDocument[key] = this.formatValueForEncryption(value); } - - // Then, encode the document to extended JSON. - const encodedDocument = encodeExtendedJsonObject(formattedDocument); - - return encodedDocument; + return encodeExtendedJsonObject(formattedDocument); } private formatDocumentFromDecryption(document: any): any { - // First, decode the document from extended JSON. const decodedDocument = decodeExtendedJsonObject(document); - - // Then, iterate through each key and value in the document and format the value from decryption. for (const [key, value] of Object.entries(decodedDocument)) { decodedDocument[key] = this.formatValueFromDecryption(value); } - return decodedDocument; } + /** + * Format a value before encryption. Converts types that msgpackr + * doesn't natively support (e.g. Date) into serialisable forms. + */ private formatValueForEncryption(value: any): any { - // msgpackr doesnt support Date, so we need to convert it to a string. if (value instanceof Date) { return ``; } - return value; } + /** + * Restore a value after decryption. Reverses the transformations + * applied by `formatValueForEncryption`. + */ private formatValueFromDecryption(value: any): any { - // msgpackr doesnt support Date, so we need to convert it to a Date. if (typeof value === 'string') { - // Check if this value matches an Extended JSON encoded date. - // TODO: Do this without a regex for performance reasons. - // const dateMatch = value.match(/[0-9]+)>/); - - // Without regex if (value.startsWith('')) { const time = value.slice(7, -1); return new Date(parseInt(time)); } - - // If it does, convert it to a Date. - // if (dateMatch) { - // const { time } = dateMatch.groups!; - // return new Date(parseInt(time)); - // } } - return value; } } diff --git a/src/storage/storage-localstorage.ts b/src/storage/storage-localstorage.ts index 5fd6649..c70dc31 100644 --- a/src/storage/storage-localstorage.ts +++ b/src/storage/storage-localstorage.ts @@ -1,4 +1,11 @@ -import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; +import { + BaseStorage, + FindOptions, + type IndexDefinition, + type Filter, + type ComparisonOperators, + isOperatorObject, +} from './base-storage.js'; /** * Key prefix separator used to namespace documents within localStorage. @@ -132,7 +139,7 @@ export class StorageLocalStorage< this.persistManifest(); } - async find(filter?: Partial, options?: FindOptions): Promise { + async find(filter?: Filter, options?: FindOptions): Promise { this.refreshManifest(); let results: T[]; @@ -175,7 +182,7 @@ export class StorageLocalStorage< } async updateMany( - filter: Partial, + filter: Filter, update: Partial, options: Partial = {}, ): Promise { @@ -207,7 +214,7 @@ export class StorageLocalStorage< } async deleteMany( - filter: Partial, + filter: Filter, options: Partial = {}, ): Promise { this.refreshManifest(); @@ -251,20 +258,38 @@ export class StorageLocalStorage< /** * Checks whether a document satisfies every field in the filter. + * Supports both plain equality values and comparison operator objects. */ - private matchesFilter(item: T, filter?: Partial): boolean { + private matchesFilter(item: T, filter?: Filter): boolean { if (!filter || Object.keys(filter).length === 0) return true; for (const [key, value] of Object.entries(filter)) { - if (item[key] !== value) return false; + if (isOperatorObject(value)) { + if (!this.matchesOperators(item[key], value)) return false; + } else { + if (item[key] !== value) return false; + } } return true; } + /** + * Evaluate a set of comparison operators against a single field value. + */ + private matchesOperators(fieldValue: any, ops: ComparisonOperators): boolean { + if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false; + if (ops.$ne !== undefined && fieldValue === ops.$ne) return false; + if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false; + if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false; + if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false; + if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false; + return true; + } + /** * Collect all [internalKey, document] pairs that match a filter. * Uses an index when possible, otherwise falls back to a full scan. */ - private collectMatches(filter?: Partial): Array<[number, T]> { + private collectMatches(filter?: Filter): Array<[number, T]> { const indexKeys = this.resolveIndexKeys(filter); const keysToScan = indexKeys ?? this.manifest; const results: Array<[number, T]> = []; @@ -348,17 +373,29 @@ export class StorageLocalStorage< /** * Attempt to resolve candidate internal keys from the indexes. * Returns `null` if no index can serve the query. + * + * Only plain equality values are used for index resolution — operator + * objects are excluded since hash-based indexes only support equality. */ - private resolveIndexKeys(filter?: Partial): Set | null { + private resolveIndexKeys(filter?: Filter): Set | null { if (!filter) return null; const filterKeys = Object.keys(filter); if (filterKeys.length === 0) return null; + const equalityFilter: Record = {}; + for (const [key, value] of Object.entries(filter)) { + if (!isOperatorObject(value)) { + equalityFilter[key] = value; + } + } + + if (Object.keys(equalityFilter).length === 0) return null; + for (const fields of this.indexDefs) { - if (!fields.every((f) => f in filter)) continue; + if (!fields.every((f) => f in equalityFilter)) continue; const indexName = fields.join(INDEX_KEY_SEP); - const indexValue = this.buildIndexValue(filter, fields); + const indexValue = this.buildIndexValue(equalityFilter, fields); if (indexValue === null) continue; const indexMap = this.indexes.get(indexName)!; diff --git a/src/storage/storage-memory-synced.ts b/src/storage/storage-memory-synced.ts index a04cd00..3f6b072 100644 --- a/src/storage/storage-memory-synced.ts +++ b/src/storage/storage-memory-synced.ts @@ -1,4 +1,4 @@ -import { BaseStorage, FindOptions } from './base-storage.js'; +import { BaseStorage, FindOptions, type Filter } from './base-storage.js'; import { StorageMemory } from './storage-memory.js'; /** @@ -65,19 +65,19 @@ export class StorageMemorySynced = Record, options?: FindOptions): Promise { + async find(filter?: Filter, options?: FindOptions): Promise { return await this.inMemoryCache.find(filter, options); } async updateMany( - filter: Partial, + filter: Filter, update: Partial, options: FindOptions = {} as FindOptions ): Promise { return await this.store.updateMany(filter, update, options); } - async deleteMany(filter: Partial, options: FindOptions = {} as FindOptions): Promise { + async deleteMany(filter: Filter, options: FindOptions = {} as FindOptions): Promise { return await this.store.deleteMany(filter, options); } diff --git a/src/storage/storage-memory.ts b/src/storage/storage-memory.ts index ad64d7e..30ec34d 100644 --- a/src/storage/storage-memory.ts +++ b/src/storage/storage-memory.ts @@ -1,4 +1,11 @@ -import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; +import { + BaseStorage, + FindOptions, + type IndexDefinition, + type Filter, + type ComparisonOperators, + isOperatorObject, +} from './base-storage.js'; /** * Separator used when joining multiple field values into a single index key. @@ -86,7 +93,7 @@ export class StorageMemory< } } - async find(filter?: Partial, options?: FindOptions): Promise { + async find(filter?: Filter, options?: FindOptions): Promise { let results: T[]; // Attempt to satisfy the query via an index. @@ -118,7 +125,7 @@ export class StorageMemory< } async updateMany( - filter: Partial, + filter: Filter, update: Partial, options: Partial = {}, ): Promise { @@ -147,7 +154,7 @@ export class StorageMemory< } async deleteMany( - filter: Partial, + filter: Filter, options: Partial = {}, ): Promise { const rowsToDelete = this.collectMatches(filter); @@ -182,26 +189,42 @@ export class StorageMemory< /** * Checks whether a document satisfies every field in the filter. - * An empty or undefined filter matches everything. + * Supports both plain equality values and comparison operator objects. */ - private matchesFilter(item: T, filter?: Partial): boolean { + private matchesFilter(item: T, filter?: Filter): boolean { if (!filter || Object.keys(filter).length === 0) { return true; } for (const [key, value] of Object.entries(filter)) { - if (item[key] !== value) { - return false; + if (isOperatorObject(value)) { + if (!this.matchesOperators(item[key], value)) return false; + } else { + if (item[key] !== value) return false; } } return true; } + /** + * Evaluate a set of comparison operators against a single field value. + * All operators must pass for the field to match. + */ + private matchesOperators(fieldValue: any, ops: ComparisonOperators): boolean { + if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false; + if (ops.$ne !== undefined && fieldValue === ops.$ne) return false; + if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false; + if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false; + if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false; + if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false; + return true; + } + /** * Collect all [internalKey, document] pairs that match a filter. * Uses an index when possible, otherwise falls back to a full scan. */ - private collectMatches(filter?: Partial): Array<[number, T]> { + private collectMatches(filter?: Filter): Array<[number, T]> { const indexKeys = this.resolveIndexKeys(filter); if (indexKeys !== null) { @@ -296,21 +319,31 @@ export class StorageMemory< * Attempt to resolve a set of candidate internal keys from the indexes. * Returns `null` if no index can serve the query. * - * An index is used when the filter fields are a superset of (or equal to) - * an index's fields — meaning the index value can be fully constructed - * from the filter. + * An index is used when the filter contains plain equality values for every + * field in the index. Operator objects (e.g. `{ $lt: 50 }`) are excluded + * from index resolution since hash-based indexes only support equality. */ - private resolveIndexKeys(filter?: Partial): Set | null { + private resolveIndexKeys(filter?: Filter): Set | null { if (!filter) return null; const filterKeys = Object.keys(filter); if (filterKeys.length === 0) return null; + // Extract only the equality fields from the filter (skip operator objects). + const equalityFilter: Record = {}; + for (const [key, value] of Object.entries(filter)) { + if (!isOperatorObject(value)) { + equalityFilter[key] = value; + } + } + + if (Object.keys(equalityFilter).length === 0) return null; + for (const fields of this.indexDefs) { - // Every field in the index must be present in the filter. - if (!fields.every((f) => f in filter)) continue; + // Every field in the index must be present as an equality value. + if (!fields.every((f) => f in equalityFilter)) continue; const indexName = fields.join(INDEX_KEY_SEP); - const indexValue = this.buildIndexValue(filter, fields); + const indexValue = this.buildIndexValue(equalityFilter, fields); if (indexValue === null) continue; const indexMap = this.indexes.get(indexName)!; @@ -326,7 +359,7 @@ export class StorageMemory< * Returns `null` when no index can serve the filter, signalling * the caller to fall back to a full scan. */ - private findViaIndex(filter?: Partial): T[] | null { + private findViaIndex(filter?: Filter): T[] | null { const keys = this.resolveIndexKeys(filter); if (keys === null) return null;