From f1fa2acf170295d0ead3bd8adb59d702d5fe6be1 Mon Sep 17 00:00:00 2001 From: Harvey Zuccon Date: Wed, 25 Feb 2026 14:06:34 +1100 Subject: [PATCH] Dont assume id on storage objects --- benchmarks/storage.ts | 179 ++++++++++--- src/storage/base-storage.ts | 11 + src/storage/encrypted-storage.ts | 90 ++++++- src/storage/index.ts | 1 + src/storage/storage-localstorage.ts | 364 ++++++++++++++++++++------- src/storage/storage-memory-synced.ts | 13 +- src/storage/storage-memory.ts | 361 ++++++++++++++++++++------ src/utils/ext-json.ts | 124 +++++++++ 8 files changed, 922 insertions(+), 221 deletions(-) create mode 100644 src/utils/ext-json.ts diff --git a/benchmarks/storage.ts b/benchmarks/storage.ts index 7e6250d..4194e9e 100644 --- a/benchmarks/storage.ts +++ b/benchmarks/storage.ts @@ -1,52 +1,155 @@ import { AESKey } from '../src/crypto/aes-key.js'; -import { StorageMemory, EncryptedStorage } from '../src/storage/index.js'; +import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js'; -const storage = StorageMemory.from(); -// const storageSynced = StorageMemorySynced.from(storage); +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- -const currentDate = new Date(); +type Doc = { + id: string; + name: string; + age: number; + email: string; + createdAt: Date; +}; -const data = { - id: 'test', - name: 'test', - age: 20, - email: 'test@test.com', - password: 'test', - createdAt: currentDate, - updatedAt: new Date(currentDate.getTime() + 1000), +/** + * Generate a batch of unique documents. + */ +function generateDocs(count: number): Doc[] { + const docs: Doc[] = []; + for (let i = 0; i < count; i++) { + docs.push({ + id: `id-${i}`, + name: `user-${i}`, + age: 20 + (i % 50), + email: `user-${i}@test.com`, + createdAt: new Date(Date.now() + i), + }); + } + return docs; } -const storageEncryptedBase = StorageMemory.from() -const storageEncrypted = EncryptedStorage.from(storageEncryptedBase, await AESKey.fromSeed('test')); +/** + * Time an async operation and return elapsed milliseconds. + */ +async function time(fn: () => Promise): Promise { + const start = performance.now(); + await fn(); + return performance.now() - start; +} -storageEncryptedBase.on('insert', (event) => { - console.log('insert', event); -}); +/** + * Format ops/sec with thousands separators. + */ +function fmtOps(ops: number): string { + return Math.round(ops).toLocaleString('en-US'); +} -// Store data in storage -await storage.insertOne(data); -// storageSynced.insertOne('test', data); -await storageEncrypted.insertOne(data); +/** + * Run a full suite of benchmarks against a given storage instance. + */ +async function benchmarkStorage(label: string, storage: BaseStorage, docs: Doc[]) { + const count = docs.length; + console.log(`\n${'='.repeat(60)}`); + console.log(` ${label} (${count.toLocaleString()} documents)`); + console.log('='.repeat(60)); -// Retrieve data from storage -const retrievedData = await storage.findOne({ name: 'test' }); -// const retrievedDataSynced = await storageSynced.findOne('test'); -const retrievedDataEncrypted = await storageEncrypted.findOne({ name: 'test' }); + // --- Insert --- + const insertMs = await time(async () => { + await storage.insertMany(docs); + }); + console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`); -console.log(retrievedData); -// console.log(retrievedDataSynced); -console.log(retrievedDataEncrypted); + // --- Find all (no filter) --- + const findAllMs = await time(async () => { + await storage.find(); + }); + console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`); -// Update data in storage -await storage.updateOne({ id: 'test' }, { name: 'test2' }); -await storageEncrypted.updateOne({ id: 'test' }, { name: 'test2' }); + // --- Find by indexed field (single-key lookup, repeated) --- + const lookupCount = Math.min(count, 1_000); + const findIndexedMs = await time(async () => { + for (let i = 0; i < lookupCount; i++) { + await storage.findOne({ id: `id-${i}` } as Partial); + } + }); + console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`); -// Retrieve data from storage -const retrievedDataUpdated = await storage.findOne({ name: 'test2' }); -// const retrievedDataSynced = await storageSynced.findOne('test'); -const retrievedDataEncryptedUpdated = await storageEncrypted.findOne({ name: 'test2' }); + // --- Find by non-indexed field (full scan, repeated) --- + const scanCount = Math.min(count, 1_000); + const findScanMs = await time(async () => { + for (let i = 0; i < scanCount; i++) { + await storage.findOne({ email: `user-${i}@test.com` } as Partial); + } + }); + console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`); -console.log(retrievedDataUpdated); -// console.log(retrievedDataSynced); -console.log(retrievedDataEncryptedUpdated); + // --- Update by indexed field --- + const updateCount = Math.min(count, 1_000); + const updateMs = await time(async () => { + for (let i = 0; i < updateCount; i++) { + await storage.updateOne( + { id: `id-${i}` } as Partial, + { age: 99 } as Partial, + ); + } + }); + console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`); + // --- Delete by indexed field --- + const deleteCount = Math.min(count, 1_000); + const deleteMs = await time(async () => { + for (let i = 0; i < deleteCount; i++) { + await storage.deleteOne({ id: `id-${i}` } as Partial); + } + }); + console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`); + + // --- Verify remaining count --- + const remaining = await storage.find(); + console.log(` remaining docs: ${remaining.length.toLocaleString()}`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// StorageMemory — indexed vs non-indexed +// --------------------------------------------------------------------------- + +const DOC_COUNTS = [1_000, 10_000, 50_000]; + +for (const count of DOC_COUNTS) { + const docs = generateDocs(count); + + const indexed = StorageMemory.from(['id', 'name']); + await benchmarkStorage('StorageMemory (indexed: id, name)', indexed, docs); + + const noIndex = StorageMemory.from(); + await benchmarkStorage('StorageMemory (no indexes)', noIndex, docs); +} + +// --------------------------------------------------------------------------- +// EncryptedStorage — crypto overhead dominates, so use smaller counts +// --------------------------------------------------------------------------- + +const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000]; +const encryptionKey = await AESKey.fromSeed('benchmark-key'); + +for (const count of ENCRYPTED_DOC_COUNTS) { + const docs = generateDocs(count); + + // Encrypted + indexed backing store. + const encBase = StorageMemory.from>(['id', 'name']); + const encrypted = EncryptedStorage.from(encBase, encryptionKey); + await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs); + + // Encrypted + no-index backing store. + const encBaseNoIdx = StorageMemory.from>(); + const encryptedNoIdx = EncryptedStorage.from(encBaseNoIdx, encryptionKey); + await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs); +} + +console.log('\nDone.\n'); diff --git a/src/storage/base-storage.ts b/src/storage/base-storage.ts index 5fbcd4d..fbea6b9 100644 --- a/src/storage/base-storage.ts +++ b/src/storage/base-storage.ts @@ -9,11 +9,22 @@ export type FindOptions = { sort: { [key: string]: 1 | -1 }; }; +/** + * Defines which document fields should be indexed for fast lookups. + * + * - `string[]` is shorthand for multiple single-field indexes, + * e.g. `['id', 'name']` normalizes to `[['id'], ['name']]`. + * - `string[][]` defines explicit (possibly compound) indexes, + * e.g. `[['createdAt', 'name'], ['id']]`. + */ +export type IndexDefinition = string[] | string[][]; + export type StorageEvent> = { insert: { value: T; }; update: { + oldValue: T; value: T; }; delete: { diff --git a/src/storage/encrypted-storage.ts b/src/storage/encrypted-storage.ts index 9e2a9d4..9d50944 100644 --- a/src/storage/encrypted-storage.ts +++ b/src/storage/encrypted-storage.ts @@ -5,6 +5,8 @@ import { Bytes } from 'src/crypto/bytes.js'; import { BaseStorage, type FindOptions } from './base-storage.js'; +import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; + export class EncryptedStorage< T extends Record = Record, > extends BaseStorage { @@ -33,11 +35,11 @@ export class EncryptedStorage< }); this.storage.on('update', async (event) => { - // De-crypt the value before emitting the event. + // Decrypt both old and new values before re-emitting. + const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record); const decryptedValue = await this.convertToDecrypted(event.value as Record); - // Re-emit the update event with the original payload. - this.emit('update', { value: decryptedValue }); + this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue }); }); this.storage.on('delete', async (event) => { @@ -62,7 +64,7 @@ export class EncryptedStorage< } async find(filter?: Partial, options?: FindOptions): Promise { - const encryptedFilter = await this.convertToEncrypted(filter); + const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined; const documents = await this.storage.find(encryptedFilter, options); return Promise.all( documents.map(async (document) => this.convertToDecrypted(document)), @@ -97,14 +99,21 @@ export class EncryptedStorage< // For each key in the document, encrypt the value. This requires us to know the type of each value, so we must include it after converting it. Maybe this can be done by converting it to an object and json stringifying it. // Example: { a: 1, b: 'hello' } -> { a: { type: 'number', value: 1 }, b: { type: 'string', value: 'hello' } } const encrypted: Record = {}; - for (const [key, value] of Object.entries(document)) { + + const formattedDocument = this.formatDocumentForEncryption(document); + + for (const [key, value] of Object.entries(formattedDocument)) { // Create our object to encrypt const bin = this.msgpackr.pack(value); + // Encrypt it const encryptedValue = await this.key.encrypt(bin, true); + + // Store the encrypted value in the encrypted object. encrypted[key] = encryptedValue.toBase64(); } + // Return the encrypted object. return encrypted; } @@ -112,12 +121,81 @@ export class EncryptedStorage< document: Record, ): Promise { const decrypted: Record = {}; + + // Iterate through each key and value in the document and decrypt it. for (const [key, value] of Object.entries(document)) { + // Decrypt the value. const binaryString = await this.key.decrypt(Bytes.fromBase64(value)); + + // Unpack the value. const object = this.msgpackr.unpack(binaryString); + + // Decode the value. decrypted[key] = object; } - return decrypted as T; + // Format the document from decryption. + const decodedDocument = this.formatDocumentFromDecryption(decrypted); + + // Return the document as the original type. + return decodedDocument as T; + } + + private formatDocumentForEncryption(document: any): any { + // First, iterate through each key and value in the document and format the value for encryption. + const formattedDocument: any = {}; + + for (const [key, value] of Object.entries(document)) { + formattedDocument[key] = this.formatValueForEncryption(value); + } + + // Then, encode the document to extended JSON. + const encodedDocument = encodeExtendedJsonObject(formattedDocument); + + return encodedDocument; + } + + private formatDocumentFromDecryption(document: any): any { + // First, decode the document from extended JSON. + const decodedDocument = decodeExtendedJsonObject(document); + + // Then, iterate through each key and value in the document and format the value from decryption. + for (const [key, value] of Object.entries(decodedDocument)) { + decodedDocument[key] = this.formatValueFromDecryption(value); + } + + return decodedDocument; + } + + private formatValueForEncryption(value: any): any { + // msgpackr doesnt support Date, so we need to convert it to a string. + if (value instanceof Date) { + return ``; + } + + return value; + } + + private formatValueFromDecryption(value: any): any { + // msgpackr doesnt support Date, so we need to convert it to a Date. + if (typeof value === 'string') { + // Check if this value matches an Extended JSON encoded date. + // TODO: Do this without a regex for performance reasons. + // const dateMatch = value.match(/[0-9]+)>/); + + // Without regex + if (value.startsWith('')) { + const time = value.slice(7, -1); + return new Date(parseInt(time)); + } + + // If it does, convert it to a Date. + // if (dateMatch) { + // const { time } = dateMatch.groups!; + // return new Date(parseInt(time)); + // } + } + + return value; } } diff --git a/src/storage/index.ts b/src/storage/index.ts index d8a660e..532179a 100644 --- a/src/storage/index.ts +++ b/src/storage/index.ts @@ -1,3 +1,4 @@ +export * from './base-storage.js'; export * from './storage-memory.js'; export * from './storage-memory-synced.js'; export * from './storage-localstorage.js'; diff --git a/src/storage/storage-localstorage.ts b/src/storage/storage-localstorage.ts index 373a9c5..5fd6649 100644 --- a/src/storage/storage-localstorage.ts +++ b/src/storage/storage-localstorage.ts @@ -1,4 +1,4 @@ -import { BaseStorage, FindOptions } from './base-storage.js'; +import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; /** * Key prefix separator used to namespace documents within localStorage. @@ -7,20 +7,46 @@ const KEY_SEP = ':'; /** * Suffix appended to the namespace prefix to form the manifest key. - * The manifest tracks all document IDs belonging to this storage instance, + * The manifest tracks all internal keys belonging to this storage instance, * avoiding expensive full-scan iterations over the entire localStorage. */ const MANIFEST_SUFFIX = '__keys__'; +/** + * Suffix for the auto-incrementing counter persisted alongside the manifest. + */ +const COUNTER_SUFFIX = '__next__'; + +/** + * Separator used when joining multiple field values into a single index key. + */ +const INDEX_KEY_SEP = '\x00'; + +/** + * Normalize an IndexDefinition into a canonical `string[][]` form. + * A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`. + */ +function normalizeIndexes(indexes?: IndexDefinition): string[][] { + if (!indexes || indexes.length === 0) return []; + if (typeof indexes[0] === 'string') { + return (indexes as string[]).map((field) => [field]); + } + return indexes as string[][]; +} + /** * Implementation of BaseStorage using the browser's localStorage API. * * @remarks * Documents are stored individually in localStorage, keyed by a namespaced - * prefix combined with the document's `id` field. A separate manifest key - * tracks all known document IDs so that read operations avoid scanning every + * prefix combined with an auto-generated numeric key. A separate manifest + * tracks all internal keys so that read operations avoid scanning every * key in localStorage. * + * Optional indexes provide fast lookups when a query filter matches + * an index exactly. Indexes are held in memory and rebuilt only when a + * cross-tab manifest change is detected. + * * Because localStorage is synchronous and string-only, all values are * JSON-serialised on write and parsed on read. */ @@ -32,25 +58,60 @@ export class StorageLocalStorage< */ static from>( prefix = 'qs', + indexes?: IndexDefinition, ): StorageLocalStorage { - return new StorageLocalStorage(prefix); + return new StorageLocalStorage(prefix, indexes); } /** - * In-memory copy of the document ID set. + * In-memory set of internal numeric keys. * Re-read from localStorage at the start of every public method so that * cross-tab mutations are always visible. */ - private manifest: Set; + private manifest: Set; - /** Map of derived child instances, lazily created and cached. */ + /** Auto-incrementing counter for generating internal keys. */ + private nextKey: number; + + /** + * Raw JSON string of the manifest as last seen. Used to detect cross-tab + * changes cheaply — if the raw string hasn't changed, indexes are still + * valid and we skip the expensive rebuild. + */ + private lastManifestRaw: string; + + /** The normalized index definitions supplied at construction time. */ + private indexDefs: string[][]; + + /** + * Secondary index maps (same structure as StorageMemory). + * Outer key = index name (joined field names). + * Inner key = index value (joined field values from a document). + * Inner value = set of internal numeric keys. + */ + private indexes: Map>>; + + /** Lazily-created child storage instances. */ private children: Map>; - constructor(private readonly prefix: string = 'qs') { + constructor( + private readonly prefix: string = 'qs', + indexes?: IndexDefinition, + ) { super(); this.children = new Map(); - this.manifest = this.loadManifest(); + this.indexDefs = normalizeIndexes(indexes); + this.indexes = new Map(); + for (const fields of this.indexDefs) { + this.indexes.set(fields.join(INDEX_KEY_SEP), new Map()); + } + + // Bootstrap from localStorage. + this.lastManifestRaw = ''; + this.manifest = new Set(); + this.nextKey = 0; + this.refreshManifest(); } // --------------------------------------------------------------------------- @@ -61,9 +122,10 @@ export class StorageLocalStorage< this.refreshManifest(); for (const document of documents) { - const key = this.docKey(document.id); - localStorage.setItem(key, JSON.stringify(document)); - this.manifest.add(document.id); + const key = this.nextKey++; + localStorage.setItem(this.docKey(key), JSON.stringify(document)); + this.manifest.add(key); + this.addToIndexes(key, document); this.emit('insert', { value: document }); } @@ -73,19 +135,33 @@ export class StorageLocalStorage< async find(filter?: Partial, options?: FindOptions): Promise { this.refreshManifest(); - let results: T[] = []; + let results: T[]; + const indexedKeys = this.resolveIndexKeys(filter); - for (const id of this.manifest) { - const raw = localStorage.getItem(this.docKey(id)); - if (raw === null) continue; - - const doc = JSON.parse(raw) as T; - if (this.matchesFilter(doc, filter)) { - results.push(doc); + if (indexedKeys !== null) { + // Use the index to narrow which documents we read from localStorage. + results = []; + for (const key of indexedKeys) { + const raw = localStorage.getItem(this.docKey(key)); + if (raw === null) continue; + const doc = JSON.parse(raw) as T; + if (this.matchesFilter(doc, filter)) { + results.push(doc); + } + } + } else { + // Full scan over all documents in the manifest. + results = []; + for (const key of this.manifest) { + const raw = localStorage.getItem(this.docKey(key)); + if (raw === null) continue; + const doc = JSON.parse(raw) as T; + if (this.matchesFilter(doc, filter)) { + results.push(doc); + } } } - // Apply sort before skip/limit so the window is deterministic. if (options?.sort) { results = this.applySorting(results, options.sort); } @@ -105,17 +181,7 @@ export class StorageLocalStorage< ): Promise { this.refreshManifest(); - const candidates: T[] = []; - - for (const id of this.manifest) { - const raw = localStorage.getItem(this.docKey(id)); - if (raw === null) continue; - - const doc = JSON.parse(raw) as T; - if (this.matchesFilter(doc, filter)) { - candidates.push(doc); - } - } + const candidates = this.collectMatches(filter); const startIndex = options.skip ?? 0; const endIndex = options.limit @@ -123,18 +189,14 @@ export class StorageLocalStorage< : candidates.length; const toProcess = candidates.slice(startIndex, endIndex); - for (const doc of toProcess) { - const updatedDoc = { ...doc, ...update } as T; - localStorage.setItem(this.docKey(doc.id), JSON.stringify(updatedDoc)); + for (const [key, oldValue] of toProcess) { + const updatedDoc = { ...oldValue, ...update } as T; + localStorage.setItem(this.docKey(key), JSON.stringify(updatedDoc)); - // If the id itself changed, update the manifest accordingly. - if (update.id !== undefined && update.id !== doc.id) { - localStorage.removeItem(this.docKey(doc.id)); - this.manifest.delete(doc.id); - this.manifest.add(updatedDoc.id); - } + this.removeFromIndexes(key, oldValue); + this.addToIndexes(key, updatedDoc); - this.emit('update', { value: updatedDoc }); + this.emit('update', { oldValue, value: updatedDoc }); } if (toProcess.length > 0) { @@ -150,17 +212,7 @@ export class StorageLocalStorage< ): Promise { this.refreshManifest(); - const candidates: T[] = []; - - for (const id of this.manifest) { - const raw = localStorage.getItem(this.docKey(id)); - if (raw === null) continue; - - const doc = JSON.parse(raw) as T; - if (this.matchesFilter(doc, filter)) { - candidates.push(doc); - } - } + const candidates = this.collectMatches(filter); const startIndex = options.skip ?? 0; const endIndex = options.limit @@ -168,10 +220,11 @@ export class StorageLocalStorage< : candidates.length; const toProcess = candidates.slice(startIndex, endIndex); - for (const doc of toProcess) { - localStorage.removeItem(this.docKey(doc.id)); - this.manifest.delete(doc.id); - this.emit('delete', { value: doc }); + for (const [key, value] of toProcess) { + localStorage.removeItem(this.docKey(key)); + this.manifest.delete(key); + this.removeFromIndexes(key, value); + this.emit('delete', { value }); } if (toProcess.length > 0) { @@ -184,42 +237,55 @@ export class StorageLocalStorage< deriveChild(path: string): BaseStorage { if (!this.children.has(path)) { const childPrefix = `${this.prefix}${KEY_SEP}${path}`; - this.children.set(path, new StorageLocalStorage(childPrefix)); + this.children.set( + path, + new StorageLocalStorage(childPrefix, this.indexDefs), + ); } return this.children.get(path) as StorageLocalStorage; } // --------------------------------------------------------------------------- - // Private helpers + // Private helpers — filtering // --------------------------------------------------------------------------- /** * Checks whether a document satisfies every field in the filter. - * An empty or undefined filter matches everything. */ private matchesFilter(item: T, filter?: Partial): boolean { - if (!filter || Object.keys(filter).length === 0) { - return true; - } - + if (!filter || Object.keys(filter).length === 0) return true; for (const [key, value] of Object.entries(filter)) { - if (item[key] !== value) { - return false; - } + if (item[key] !== value) return false; } return true; } /** - * Sort an array of documents according to a sort specification. - * Keys map to `1` (ascending) or `-1` (descending). + * Collect all [internalKey, document] pairs that match a filter. + * Uses an index when possible, otherwise falls back to a full scan. */ - private applySorting( - items: T[], - sort: Record, - ): T[] { - const sortEntries = Object.entries(sort); + private collectMatches(filter?: Partial): Array<[number, T]> { + const indexKeys = this.resolveIndexKeys(filter); + const keysToScan = indexKeys ?? this.manifest; + const results: Array<[number, T]> = []; + for (const key of keysToScan) { + const raw = localStorage.getItem(this.docKey(key)); + if (raw === null) continue; + const doc = JSON.parse(raw) as T; + if (this.matchesFilter(doc, filter)) { + results.push([key, doc]); + } + } + + return results; + } + + /** + * Sort documents according to a sort specification. + */ + private applySorting(items: T[], sort: Record): T[] { + const sortEntries = Object.entries(sort); return [...items].sort((a, b) => { for (const [key, direction] of sortEntries) { if (a[key] < b[key]) return -1 * direction; @@ -229,52 +295,158 @@ export class StorageLocalStorage< }); } + // --------------------------------------------------------------------------- + // Private helpers — indexing + // --------------------------------------------------------------------------- + /** - * Build the full localStorage key for a given document ID. + * Build the index value string for a given document and set of fields. + * Returns `null` if any field is missing from the document. */ - private docKey(id: string): string { - return `${this.prefix}${KEY_SEP}${id}`; + private buildIndexValue(doc: Record, fields: string[]): string | null { + const parts: string[] = []; + for (const field of fields) { + if (!(field in doc)) return null; + parts.push(String(doc[field])); + } + return parts.join(INDEX_KEY_SEP); + } + + /** Register a document in all applicable indexes. */ + private addToIndexes(internalKey: number, doc: T): void { + for (const fields of this.indexDefs) { + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(doc, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + let bucket = indexMap.get(indexValue); + if (!bucket) { + bucket = new Set(); + indexMap.set(indexValue, bucket); + } + bucket.add(internalKey); + } + } + + /** Remove a document from all applicable indexes. */ + private removeFromIndexes(internalKey: number, doc: T): void { + for (const fields of this.indexDefs) { + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(doc, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + const bucket = indexMap.get(indexValue); + if (bucket) { + bucket.delete(internalKey); + if (bucket.size === 0) indexMap.delete(indexValue); + } + } } /** - * Build the localStorage key used to persist the manifest. + * Attempt to resolve candidate internal keys from the indexes. + * Returns `null` if no index can serve the query. */ + private resolveIndexKeys(filter?: Partial): Set | null { + if (!filter) return null; + const filterKeys = Object.keys(filter); + if (filterKeys.length === 0) return null; + + for (const fields of this.indexDefs) { + if (!fields.every((f) => f in filter)) continue; + + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(filter, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + const bucket = indexMap.get(indexValue); + return bucket ?? new Set(); + } + + return null; + } + + /** + * Rebuild all in-memory index maps by reading every document from + * localStorage. Called only when a cross-tab manifest change is detected. + */ + private rebuildIndexes(): void { + for (const [, indexMap] of this.indexes) { + indexMap.clear(); + } + + for (const key of this.manifest) { + const raw = localStorage.getItem(this.docKey(key)); + if (raw === null) continue; + const doc = JSON.parse(raw) as T; + this.addToIndexes(key, doc); + } + } + + // --------------------------------------------------------------------------- + // Private helpers — manifest & keys + // --------------------------------------------------------------------------- + + /** Build the full localStorage key for a given internal numeric key. */ + private docKey(internalKey: number): string { + return `${this.prefix}${KEY_SEP}${internalKey}`; + } + + /** Build the localStorage key used to persist the manifest. */ private manifestKey(): string { return `${this.prefix}${KEY_SEP}${MANIFEST_SUFFIX}`; } + /** Build the localStorage key used to persist the counter. */ + private counterKey(): string { + return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`; + } + /** * Re-read the manifest from localStorage into memory. * Called at the start of every public method so that cross-tab writes * are always reflected before we operate. + * + * Indexes are only rebuilt when the raw manifest string has actually + * changed, avoiding unnecessary full-document reads for same-tab calls. */ private refreshManifest(): void { - this.manifest = this.loadManifest(); - } + const raw = localStorage.getItem(this.manifestKey()) ?? '[]'; - /** - * Load the manifest (set of document IDs) from localStorage. - * Falls back to an empty set if nothing is stored yet. - */ - private loadManifest(): Set { - const raw = localStorage.getItem(this.manifestKey()); - if (raw === null) return new Set(); + if (raw === this.lastManifestRaw) return; + + this.lastManifestRaw = raw; try { - const ids: string[] = JSON.parse(raw); - return new Set(ids); + const keys: number[] = JSON.parse(raw); + this.manifest = new Set(keys); } catch { - return new Set(); + this.manifest = new Set(); + } + + // Restore the counter from localStorage. + const counterRaw = localStorage.getItem(this.counterKey()); + this.nextKey = counterRaw !== null ? Number(counterRaw) : 0; + + // Manifest changed — indexes are potentially stale. + if (this.indexDefs.length > 0) { + this.rebuildIndexes(); } } /** - * Persist the current in-memory manifest to localStorage. + * Persist the current in-memory manifest and counter to localStorage. */ private persistManifest(): void { - localStorage.setItem( - this.manifestKey(), - JSON.stringify([...this.manifest]), - ); + const raw = JSON.stringify([...this.manifest]); + localStorage.setItem(this.manifestKey(), raw); + localStorage.setItem(this.counterKey(), String(this.nextKey)); + + // Keep the cached raw string in sync so the next refreshManifest() + // recognises this as "our own write" and skips the rebuild. + this.lastManifestRaw = raw; } } diff --git a/src/storage/storage-memory-synced.ts b/src/storage/storage-memory-synced.ts index 006cee4..a04cd00 100644 --- a/src/storage/storage-memory-synced.ts +++ b/src/storage/storage-memory-synced.ts @@ -20,14 +20,11 @@ export class StorageMemorySynced = Record { - // For update events, we need to find and update the document in memory - // Since we don't have the filter, we'll update by key - const filter = { - id: payload.value.id, - } as unknown as Partial - - // Update the document in memory by ID. - await this.inMemoryCache.updateOne(filter, payload.value); + // Remove the old version and insert the new one. We use delete + insert + // rather than updateOne because the oldValue is the complete document, + // guaranteeing an exact match without assuming any particular key field. + await this.inMemoryCache.deleteOne(payload.oldValue); + await this.inMemoryCache.insertOne(payload.value); this.emit('update', payload); }); diff --git a/src/storage/storage-memory.ts b/src/storage/storage-memory.ts index e410b7b..ad64d7e 100644 --- a/src/storage/storage-memory.ts +++ b/src/storage/storage-memory.ts @@ -1,36 +1,189 @@ -import { BaseStorage, FindOptions } from './base-storage.js'; +import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; + +/** + * Separator used when joining multiple field values into a single index key. + * Chosen to be unlikely to appear in real field values. + */ +const INDEX_KEY_SEP = '\x00'; + +/** + * Normalize an IndexDefinition into a canonical `string[][]` form. + * A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`. + * An already-nested `string[][]` is returned as-is. + */ +function normalizeIndexes(indexes?: IndexDefinition): string[][] { + if (!indexes || indexes.length === 0) return []; + + // If the first element is a string, treat the whole array as shorthand. + if (typeof indexes[0] === 'string') { + return (indexes as string[]).map((field) => [field]); + } + + return indexes as string[][]; +} /** * Implementation of BaseStore using Memory as the storage backend. * * @remarks - * This implementation can be used testing and caching of expensive operations. + * Documents are keyed internally by an auto-incrementing numeric key. + * Optional indexes provide O(1) lookups when a query filter matches + * an index exactly. */ export class StorageMemory< T extends Record = Record, > extends BaseStorage { - // TODO: Eventually this may accept indexes as an argument. - static from>(): StorageMemory { - return new StorageMemory(); + static from>( + indexes?: IndexDefinition, + ): StorageMemory { + return new StorageMemory(indexes); } - private store: Map; + /** Auto-incrementing counter used to generate internal keys. */ + private nextKey = 0; + + /** Primary document store keyed by an opaque internal key. */ + private store: Map; + + /** + * Secondary index maps. + * Outer key = index name (joined field names). + * Inner key = index value (joined field values from a document). + * Inner value = set of internal keys that share this index value. + */ + private indexes: Map>>; + + /** The normalized index definitions supplied at construction time. */ + private indexDefs: string[][]; + + /** Lazily-created child storage instances. */ private children: Map>; - constructor() { + constructor(indexes?: IndexDefinition) { super(); this.store = new Map(); this.children = new Map(); + this.indexDefs = normalizeIndexes(indexes); + + // Initialise an empty map for each index definition. + this.indexes = new Map(); + for (const fields of this.indexDefs) { + this.indexes.set(fields.join(INDEX_KEY_SEP), new Map()); + } } + // --------------------------------------------------------------------------- + // Abstract method implementations + // --------------------------------------------------------------------------- + async insertMany(documents: Array): Promise { for (const document of documents) { - this.store.set(document.id, document); + const key = this.nextKey++; + this.store.set(key, document); + this.addToIndexes(key, document); this.emit('insert', { value: document }); } } + async find(filter?: Partial, options?: FindOptions): Promise { + let results: T[]; + + // Attempt to satisfy the query via an index. + const indexed = this.findViaIndex(filter); + + if (indexed !== null) { + results = indexed; + } else { + // Fall back to a full scan. + results = []; + for (const [, value] of this.store) { + if (this.matchesFilter(value, filter)) { + results.push(value); + } + } + } + + // Apply sort before skip/limit so the window is deterministic. + if (options?.sort) { + results = this.applySorting(results, options.sort); + } + + const startIndex = options?.skip ?? 0; + const endIndex = options?.limit + ? startIndex + options.limit + : results.length; + + return results.slice(startIndex, endIndex); + } + + async updateMany( + filter: Partial, + update: Partial, + options: Partial = {}, + ): Promise { + const itemsToUpdate = this.collectMatches(filter); + + const startIndex = options.skip ?? 0; + const endIndex = options.limit + ? startIndex + options.limit + : itemsToUpdate.length; + const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex); + + let updated = 0; + for (const [key, oldValue] of itemsToProcess) { + const updatedValue = { ...oldValue, ...update } as T; + + // Re-index: remove old entries, store new doc, add new entries. + this.removeFromIndexes(key, oldValue); + this.store.set(key, updatedValue); + this.addToIndexes(key, updatedValue); + + this.emit('update', { oldValue, value: updatedValue }); + updated++; + } + + return updated; + } + + async deleteMany( + filter: Partial, + options: Partial = {}, + ): Promise { + const rowsToDelete = this.collectMatches(filter); + + const startIndex = options.skip ?? 0; + const endIndex = options.limit + ? startIndex + options.limit + : rowsToDelete.length; + const rowsToProcess = rowsToDelete.slice(startIndex, endIndex); + + let deleted = 0; + for (const [key, value] of rowsToProcess) { + this.removeFromIndexes(key, value); + this.store.delete(key); + this.emit('delete', { value }); + deleted++; + } + + return deleted; + } + + deriveChild(path: string): BaseStorage { + if (!this.children.has(path)) { + this.children.set(path, new StorageMemory(this.indexDefs)); + } + return this.children.get(path) as StorageMemory; + } + + // --------------------------------------------------------------------------- + // Private helpers — filtering + // --------------------------------------------------------------------------- + + /** + * Checks whether a document satisfies every field in the filter. + * An empty or undefined filter matches everything. + */ private matchesFilter(item: T, filter?: Partial): boolean { if (!filter || Object.keys(filter).length === 0) { return true; @@ -44,84 +197,146 @@ export class StorageMemory< return true; } - async find(filter?: Partial, options?: FindOptions): Promise { - const results: T[] = []; - for (const [, value] of this.store) { + /** + * Collect all [internalKey, document] pairs that match a filter. + * Uses an index when possible, otherwise falls back to a full scan. + */ + private collectMatches(filter?: Partial): Array<[number, T]> { + const indexKeys = this.resolveIndexKeys(filter); + + if (indexKeys !== null) { + // We have candidate internal keys from the index — fetch and verify. + const results: Array<[number, T]> = []; + for (const key of indexKeys) { + const doc = this.store.get(key); + if (doc && this.matchesFilter(doc, filter)) { + results.push([key, doc]); + } + } + return results; + } + + // Full scan. + const results: Array<[number, T]> = []; + for (const [key, value] of this.store) { if (this.matchesFilter(value, filter)) { - results.push(value); + results.push([key, value]); } } return results; } - async updateMany( - filter: Partial, - update: Partial, - options: Partial = {}, - ): Promise { - let updated = 0; - const itemsToUpdate: Array<[string, T]> = []; - - // Collect all matching items - for (const [key, value] of this.store) { - if (this.matchesFilter(value, filter)) { - itemsToUpdate.push([key, value]); + /** + * Sort an array of documents according to a sort specification. + * Keys map to `1` (ascending) or `-1` (descending). + */ + private applySorting(items: T[], sort: Record): T[] { + const sortEntries = Object.entries(sort); + return [...items].sort((a, b) => { + for (const [key, direction] of sortEntries) { + if (a[key] < b[key]) return -1 * direction; + if (a[key] > b[key]) return 1 * direction; } - } - - // Apply skip and limit - const startIndex = options.skip || 0; - const endIndex = options.limit - ? startIndex + options.limit - : itemsToUpdate.length; - const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex); - - // Update items - for (const [key, oldValue] of itemsToProcess) { - const updatedValue = { ...oldValue, ...update }; - this.store.set(key, updatedValue); - this.emit('update', { value: updatedValue }); - updated++; - } - - return updated; + return 0; + }); } - async deleteMany( - filter: Partial, - options: Partial = {}, - ): Promise { - let deleted = 0; - const rowsToDelete: Array = []; + // --------------------------------------------------------------------------- + // Private helpers — indexing + // --------------------------------------------------------------------------- - // Collect all matching keys - for (const [key, value] of this.store) { - if (this.matchesFilter(value, filter)) { - rowsToDelete.push(value); - } + /** + * Build the index value string for a given document and set of fields. + * Returns `null` if any of the fields are missing from the document, + * since we can't meaningfully index a partial key. + */ + private buildIndexValue(doc: Record, fields: string[]): string | null { + const parts: string[] = []; + for (const field of fields) { + if (!(field in doc)) return null; + parts.push(String(doc[field])); } - - // Apply skip and limit - const startIndex = options.skip || 0; - const endIndex = options.limit - ? startIndex + options.limit - : rowsToDelete.length; - const rowsToProcess = rowsToDelete.slice(startIndex, endIndex); - - // Delete items - for (const row of rowsToProcess) { - this.store.delete(row.id); - this.emit('delete', { value: row }); - deleted++; - } - - return deleted; + return parts.join(INDEX_KEY_SEP); } - deriveChild(path: string): BaseStorage { - if (!this.children.has(path)) { - this.children.set(path, new StorageMemory()); + /** Register a document in all applicable indexes. */ + private addToIndexes(internalKey: number, doc: T): void { + for (const fields of this.indexDefs) { + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(doc, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + let bucket = indexMap.get(indexValue); + if (!bucket) { + bucket = new Set(); + indexMap.set(indexValue, bucket); + } + bucket.add(internalKey); } - return this.children.get(path) as StorageMemory; + } + + /** Remove a document from all applicable indexes. */ + private removeFromIndexes(internalKey: number, doc: T): void { + for (const fields of this.indexDefs) { + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(doc, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + const bucket = indexMap.get(indexValue); + if (bucket) { + bucket.delete(internalKey); + if (bucket.size === 0) indexMap.delete(indexValue); + } + } + } + + /** + * Attempt to resolve a set of candidate internal keys from the indexes. + * Returns `null` if no index can serve the query. + * + * An index is used when the filter fields are a superset of (or equal to) + * an index's fields — meaning the index value can be fully constructed + * from the filter. + */ + private resolveIndexKeys(filter?: Partial): Set | null { + if (!filter) return null; + const filterKeys = Object.keys(filter); + if (filterKeys.length === 0) return null; + + for (const fields of this.indexDefs) { + // Every field in the index must be present in the filter. + if (!fields.every((f) => f in filter)) continue; + + const indexName = fields.join(INDEX_KEY_SEP); + const indexValue = this.buildIndexValue(filter, fields); + if (indexValue === null) continue; + + const indexMap = this.indexes.get(indexName)!; + const bucket = indexMap.get(indexValue); + return bucket ?? new Set(); + } + + return null; + } + + /** + * Try to answer a `find` query entirely through an index. + * Returns `null` when no index can serve the filter, signalling + * the caller to fall back to a full scan. + */ + private findViaIndex(filter?: Partial): T[] | null { + const keys = this.resolveIndexKeys(filter); + if (keys === null) return null; + + const results: T[] = []; + for (const key of keys) { + const doc = this.store.get(key); + if (doc && this.matchesFilter(doc, filter)) { + results.push(doc); + } + } + return results; } } diff --git a/src/utils/ext-json.ts b/src/utils/ext-json.ts new file mode 100644 index 0000000..7170460 --- /dev/null +++ b/src/utils/ext-json.ts @@ -0,0 +1,124 @@ +/** + * TODO: These are intended as temporary stand-ins until this functionality has been implemented directly in LibAuth. + * We are doing this so that we may better standardize with the rest of the BCH eco-system in future. + * See: https://github.com/bitauth/libauth/pull/108 + */ + +import { binToHex, hexToBin } from '@bitauth/libauth'; + +export const extendedJsonReplacer = function (value: any): any { + if (typeof value === 'bigint') { + return ``; + } else if (value instanceof Uint8Array) { + return ``; + } + + return value; +}; + +export const extendedJsonReviver = function (value: any): any { + // Define RegEx that matches our Extended JSON fields. + const bigIntRegex = /^[+-]?[0-9]*)n>$/; + const uint8ArrayRegex = /^[a-f0-9]*)>$/; + + // Only perform a check if the value is a string. + // NOTE: We can skip all other values as all Extended JSON encoded fields WILL be a string. + if (typeof value === 'string') { + // Check if this value matches an Extended JSON encoded bigint. + const bigintMatch = value.match(bigIntRegex); + if (bigintMatch) { + // Access the named group directly instead of using array indices + const { bigint } = bigintMatch.groups!; + + // Return the value casted to bigint. + return BigInt(bigint); + } + + const uint8ArrayMatch = value.match(uint8ArrayRegex); + if (uint8ArrayMatch) { + // Access the named group directly instead of using array indices + const { hex } = uint8ArrayMatch.groups!; + + // Return the value casted to bigint. + return hexToBin(hex); + } + } + + // Return the original value. + return value; +}; + +export const encodeExtendedJsonObject = function (value: any): any { + // If this is an object type (and it is not null - which is technically an "object")... + // ... and it is not an ArrayBuffer (e.g. Uint8Array) which is also technically an "object... + if ( + typeof value === 'object' && + value !== null && + !ArrayBuffer.isView(value) + ) { + // If this is an array, recursively call this function on each value. + if (Array.isArray(value)) { + return value.map(encodeExtendedJsonObject); + } + + // Declare object to store extended JSON entries. + const encodedObject: any = {}; + + // Iterate through each entry and encode it to extended JSON. + for (const [key, valueToEncode] of Object.entries(value)) { + encodedObject[key] = encodeExtendedJsonObject(valueToEncode); + } + + // Return the extended JSON encoded object. + return encodedObject; + } + + // Return the replaced value. + return extendedJsonReplacer(value); +}; + +export const decodeExtendedJsonObject = function (value: any): any { + // If this is an object type (and it is not null - which is technically an "object")... + // ... and it is not an ArrayBuffer (e.g. Uint8Array) which is also technically an "object... + if ( + typeof value === 'object' && + value !== null && + !ArrayBuffer.isView(value) + ) { + // If this is an array, recursively call this function on each value. + if (Array.isArray(value)) { + return value.map(decodeExtendedJsonObject); + } + + // Declare object to store decoded JSON entries. + const decodedObject: any = {}; + + // Iterate through each entry and decode it from extended JSON. + for (const [key, valueToEncode] of Object.entries(value)) { + decodedObject[key] = decodeExtendedJsonObject(valueToEncode); + } + + // Return the extended JSON encoded object. + return decodedObject; + } + + // Return the revived value. + return extendedJsonReviver(value); +}; + +export const encodeExtendedJson = function ( + value: any, + space: number | undefined = undefined, +): string { + const replacedObject = encodeExtendedJsonObject(value); + const stringifiedObject = JSON.stringify(replacedObject, null, space); + + return stringifiedObject; +}; + +export const decodeExtendedJson = function (json: string): any { + const parsedObject = JSON.parse(json); + const revivedObject = decodeExtendedJsonObject(parsedObject); + + return revivedObject; +};