Dont assume id on storage objects
This commit is contained in:
@@ -1,52 +1,155 @@
|
|||||||
import { AESKey } from '../src/crypto/aes-key.js';
|
import { AESKey } from '../src/crypto/aes-key.js';
|
||||||
import { StorageMemory, EncryptedStorage } from '../src/storage/index.js';
|
import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js';
|
||||||
|
|
||||||
const storage = StorageMemory.from();
|
// ---------------------------------------------------------------------------
|
||||||
// const storageSynced = StorageMemorySynced.from(storage);
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const currentDate = new Date();
|
type Doc = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
age: number;
|
||||||
|
email: string;
|
||||||
|
createdAt: Date;
|
||||||
|
};
|
||||||
|
|
||||||
const data = {
|
/**
|
||||||
id: 'test',
|
* Generate a batch of unique documents.
|
||||||
name: 'test',
|
*/
|
||||||
age: 20,
|
function generateDocs(count: number): Doc[] {
|
||||||
email: 'test@test.com',
|
const docs: Doc[] = [];
|
||||||
password: 'test',
|
for (let i = 0; i < count; i++) {
|
||||||
createdAt: currentDate,
|
docs.push({
|
||||||
updatedAt: new Date(currentDate.getTime() + 1000),
|
id: `id-${i}`,
|
||||||
|
name: `user-${i}`,
|
||||||
|
age: 20 + (i % 50),
|
||||||
|
email: `user-${i}@test.com`,
|
||||||
|
createdAt: new Date(Date.now() + i),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return docs;
|
||||||
}
|
}
|
||||||
|
|
||||||
const storageEncryptedBase = StorageMemory.from()
|
/**
|
||||||
const storageEncrypted = EncryptedStorage.from(storageEncryptedBase, await AESKey.fromSeed('test'));
|
* Time an async operation and return elapsed milliseconds.
|
||||||
|
*/
|
||||||
|
async function time(fn: () => Promise<void>): Promise<number> {
|
||||||
|
const start = performance.now();
|
||||||
|
await fn();
|
||||||
|
return performance.now() - start;
|
||||||
|
}
|
||||||
|
|
||||||
storageEncryptedBase.on('insert', (event) => {
|
/**
|
||||||
console.log('insert', event);
|
* Format ops/sec with thousands separators.
|
||||||
});
|
*/
|
||||||
|
function fmtOps(ops: number): string {
|
||||||
|
return Math.round(ops).toLocaleString('en-US');
|
||||||
|
}
|
||||||
|
|
||||||
// Store data in storage
|
/**
|
||||||
await storage.insertOne(data);
|
* Run a full suite of benchmarks against a given storage instance.
|
||||||
// storageSynced.insertOne('test', data);
|
*/
|
||||||
await storageEncrypted.insertOne(data);
|
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[]) {
|
||||||
|
const count = docs.length;
|
||||||
|
console.log(`\n${'='.repeat(60)}`);
|
||||||
|
console.log(` ${label} (${count.toLocaleString()} documents)`);
|
||||||
|
console.log('='.repeat(60));
|
||||||
|
|
||||||
// Retrieve data from storage
|
// --- Insert ---
|
||||||
const retrievedData = await storage.findOne({ name: 'test' });
|
const insertMs = await time(async () => {
|
||||||
// const retrievedDataSynced = await storageSynced.findOne('test');
|
await storage.insertMany(docs);
|
||||||
const retrievedDataEncrypted = await storageEncrypted.findOne({ name: 'test' });
|
});
|
||||||
|
console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`);
|
||||||
|
|
||||||
console.log(retrievedData);
|
// --- Find all (no filter) ---
|
||||||
// console.log(retrievedDataSynced);
|
const findAllMs = await time(async () => {
|
||||||
console.log(retrievedDataEncrypted);
|
await storage.find();
|
||||||
|
});
|
||||||
|
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
|
||||||
|
|
||||||
// Update data in storage
|
// --- Find by indexed field (single-key lookup, repeated) ---
|
||||||
await storage.updateOne({ id: 'test' }, { name: 'test2' });
|
const lookupCount = Math.min(count, 1_000);
|
||||||
await storageEncrypted.updateOne({ id: 'test' }, { name: 'test2' });
|
const findIndexedMs = await time(async () => {
|
||||||
|
for (let i = 0; i < lookupCount; i++) {
|
||||||
|
await storage.findOne({ id: `id-${i}` } as Partial<Doc>);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
|
||||||
|
|
||||||
// Retrieve data from storage
|
// --- Find by non-indexed field (full scan, repeated) ---
|
||||||
const retrievedDataUpdated = await storage.findOne({ name: 'test2' });
|
const scanCount = Math.min(count, 1_000);
|
||||||
// const retrievedDataSynced = await storageSynced.findOne('test');
|
const findScanMs = await time(async () => {
|
||||||
const retrievedDataEncryptedUpdated = await storageEncrypted.findOne({ name: 'test2' });
|
for (let i = 0; i < scanCount; i++) {
|
||||||
|
await storage.findOne({ email: `user-${i}@test.com` } as Partial<Doc>);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
|
||||||
|
|
||||||
console.log(retrievedDataUpdated);
|
// --- Update by indexed field ---
|
||||||
// console.log(retrievedDataSynced);
|
const updateCount = Math.min(count, 1_000);
|
||||||
console.log(retrievedDataEncryptedUpdated);
|
const updateMs = await time(async () => {
|
||||||
|
for (let i = 0; i < updateCount; i++) {
|
||||||
|
await storage.updateOne(
|
||||||
|
{ id: `id-${i}` } as Partial<Doc>,
|
||||||
|
{ age: 99 } as Partial<Doc>,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
|
||||||
|
|
||||||
|
// --- Delete by indexed field ---
|
||||||
|
const deleteCount = Math.min(count, 1_000);
|
||||||
|
const deleteMs = await time(async () => {
|
||||||
|
for (let i = 0; i < deleteCount; i++) {
|
||||||
|
await storage.deleteOne({ id: `id-${i}` } as Partial<Doc>);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
|
||||||
|
|
||||||
|
// --- Verify remaining count ---
|
||||||
|
const remaining = await storage.find();
|
||||||
|
console.log(` remaining docs: ${remaining.length.toLocaleString()}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Main
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// StorageMemory — indexed vs non-indexed
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const DOC_COUNTS = [1_000, 10_000, 50_000];
|
||||||
|
|
||||||
|
for (const count of DOC_COUNTS) {
|
||||||
|
const docs = generateDocs(count);
|
||||||
|
|
||||||
|
const indexed = StorageMemory.from<Doc>(['id', 'name']);
|
||||||
|
await benchmarkStorage('StorageMemory (indexed: id, name)', indexed, docs);
|
||||||
|
|
||||||
|
const noIndex = StorageMemory.from<Doc>();
|
||||||
|
await benchmarkStorage('StorageMemory (no indexes)', noIndex, docs);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// EncryptedStorage — crypto overhead dominates, so use smaller counts
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000];
|
||||||
|
const encryptionKey = await AESKey.fromSeed('benchmark-key');
|
||||||
|
|
||||||
|
for (const count of ENCRYPTED_DOC_COUNTS) {
|
||||||
|
const docs = generateDocs(count);
|
||||||
|
|
||||||
|
// Encrypted + indexed backing store.
|
||||||
|
const encBase = StorageMemory.from<Record<string, string>>(['id', 'name']);
|
||||||
|
const encrypted = EncryptedStorage.from<Doc>(encBase, encryptionKey);
|
||||||
|
await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs);
|
||||||
|
|
||||||
|
// Encrypted + no-index backing store.
|
||||||
|
const encBaseNoIdx = StorageMemory.from<Record<string, string>>();
|
||||||
|
const encryptedNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey);
|
||||||
|
await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\nDone.\n');
|
||||||
|
|||||||
@@ -9,11 +9,22 @@ export type FindOptions = {
|
|||||||
sort: { [key: string]: 1 | -1 };
|
sort: { [key: string]: 1 | -1 };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Defines which document fields should be indexed for fast lookups.
|
||||||
|
*
|
||||||
|
* - `string[]` is shorthand for multiple single-field indexes,
|
||||||
|
* e.g. `['id', 'name']` normalizes to `[['id'], ['name']]`.
|
||||||
|
* - `string[][]` defines explicit (possibly compound) indexes,
|
||||||
|
* e.g. `[['createdAt', 'name'], ['id']]`.
|
||||||
|
*/
|
||||||
|
export type IndexDefinition = string[] | string[][];
|
||||||
|
|
||||||
export type StorageEvent<T = Record<string, any>> = {
|
export type StorageEvent<T = Record<string, any>> = {
|
||||||
insert: {
|
insert: {
|
||||||
value: T;
|
value: T;
|
||||||
};
|
};
|
||||||
update: {
|
update: {
|
||||||
|
oldValue: T;
|
||||||
value: T;
|
value: T;
|
||||||
};
|
};
|
||||||
delete: {
|
delete: {
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import { Bytes } from 'src/crypto/bytes.js';
|
|||||||
|
|
||||||
import { BaseStorage, type FindOptions } from './base-storage.js';
|
import { BaseStorage, type FindOptions } from './base-storage.js';
|
||||||
|
|
||||||
|
import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
|
||||||
|
|
||||||
export class EncryptedStorage<
|
export class EncryptedStorage<
|
||||||
T extends Record<string, any> = Record<string, any>,
|
T extends Record<string, any> = Record<string, any>,
|
||||||
> extends BaseStorage<T> {
|
> extends BaseStorage<T> {
|
||||||
@@ -33,11 +35,11 @@ export class EncryptedStorage<
|
|||||||
});
|
});
|
||||||
|
|
||||||
this.storage.on('update', async (event) => {
|
this.storage.on('update', async (event) => {
|
||||||
// De-crypt the value before emitting the event.
|
// Decrypt both old and new values before re-emitting.
|
||||||
|
const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record<string, string>);
|
||||||
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
|
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
|
||||||
|
|
||||||
// Re-emit the update event with the original payload.
|
this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue });
|
||||||
this.emit('update', { value: decryptedValue });
|
|
||||||
});
|
});
|
||||||
|
|
||||||
this.storage.on('delete', async (event) => {
|
this.storage.on('delete', async (event) => {
|
||||||
@@ -62,7 +64,7 @@ export class EncryptedStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
||||||
const encryptedFilter = await this.convertToEncrypted(filter);
|
const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined;
|
||||||
const documents = await this.storage.find(encryptedFilter, options);
|
const documents = await this.storage.find(encryptedFilter, options);
|
||||||
return Promise.all(
|
return Promise.all(
|
||||||
documents.map(async (document) => this.convertToDecrypted(document)),
|
documents.map(async (document) => this.convertToDecrypted(document)),
|
||||||
@@ -97,14 +99,21 @@ export class EncryptedStorage<
|
|||||||
// For each key in the document, encrypt the value. This requires us to know the type of each value, so we must include it after converting it. Maybe this can be done by converting it to an object and json stringifying it.
|
// For each key in the document, encrypt the value. This requires us to know the type of each value, so we must include it after converting it. Maybe this can be done by converting it to an object and json stringifying it.
|
||||||
// Example: { a: 1, b: 'hello' } -> { a: { type: 'number', value: 1 }, b: { type: 'string', value: 'hello' } }
|
// Example: { a: 1, b: 'hello' } -> { a: { type: 'number', value: 1 }, b: { type: 'string', value: 'hello' } }
|
||||||
const encrypted: Record<string, string> = {};
|
const encrypted: Record<string, string> = {};
|
||||||
for (const [key, value] of Object.entries(document)) {
|
|
||||||
|
const formattedDocument = this.formatDocumentForEncryption(document);
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(formattedDocument)) {
|
||||||
// Create our object to encrypt
|
// Create our object to encrypt
|
||||||
const bin = this.msgpackr.pack(value);
|
const bin = this.msgpackr.pack(value);
|
||||||
|
|
||||||
// Encrypt it
|
// Encrypt it
|
||||||
const encryptedValue = await this.key.encrypt(bin, true);
|
const encryptedValue = await this.key.encrypt(bin, true);
|
||||||
|
|
||||||
|
// Store the encrypted value in the encrypted object.
|
||||||
encrypted[key] = encryptedValue.toBase64();
|
encrypted[key] = encryptedValue.toBase64();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Return the encrypted object.
|
||||||
return encrypted;
|
return encrypted;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,12 +121,81 @@ export class EncryptedStorage<
|
|||||||
document: Record<string, string>,
|
document: Record<string, string>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
const decrypted: Record<string, any> = {};
|
const decrypted: Record<string, any> = {};
|
||||||
|
|
||||||
|
// Iterate through each key and value in the document and decrypt it.
|
||||||
for (const [key, value] of Object.entries(document)) {
|
for (const [key, value] of Object.entries(document)) {
|
||||||
|
// Decrypt the value.
|
||||||
const binaryString = await this.key.decrypt(Bytes.fromBase64(value));
|
const binaryString = await this.key.decrypt(Bytes.fromBase64(value));
|
||||||
|
|
||||||
|
// Unpack the value.
|
||||||
const object = this.msgpackr.unpack(binaryString);
|
const object = this.msgpackr.unpack(binaryString);
|
||||||
|
|
||||||
|
// Decode the value.
|
||||||
decrypted[key] = object;
|
decrypted[key] = object;
|
||||||
}
|
}
|
||||||
|
|
||||||
return decrypted as T;
|
// Format the document from decryption.
|
||||||
|
const decodedDocument = this.formatDocumentFromDecryption(decrypted);
|
||||||
|
|
||||||
|
// Return the document as the original type.
|
||||||
|
return decodedDocument as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatDocumentForEncryption(document: any): any {
|
||||||
|
// First, iterate through each key and value in the document and format the value for encryption.
|
||||||
|
const formattedDocument: any = {};
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(document)) {
|
||||||
|
formattedDocument[key] = this.formatValueForEncryption(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then, encode the document to extended JSON.
|
||||||
|
const encodedDocument = encodeExtendedJsonObject(formattedDocument);
|
||||||
|
|
||||||
|
return encodedDocument;
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatDocumentFromDecryption(document: any): any {
|
||||||
|
// First, decode the document from extended JSON.
|
||||||
|
const decodedDocument = decodeExtendedJsonObject(document);
|
||||||
|
|
||||||
|
// Then, iterate through each key and value in the document and format the value from decryption.
|
||||||
|
for (const [key, value] of Object.entries(decodedDocument)) {
|
||||||
|
decodedDocument[key] = this.formatValueFromDecryption(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
return decodedDocument;
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatValueForEncryption(value: any): any {
|
||||||
|
// msgpackr doesnt support Date, so we need to convert it to a string.
|
||||||
|
if (value instanceof Date) {
|
||||||
|
return `<Date: ${value.getTime()}>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatValueFromDecryption(value: any): any {
|
||||||
|
// msgpackr doesnt support Date, so we need to convert it to a Date.
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
// Check if this value matches an Extended JSON encoded date.
|
||||||
|
// TODO: Do this without a regex for performance reasons.
|
||||||
|
// const dateMatch = value.match(/<Date: (?<time>[0-9]+)>/);
|
||||||
|
|
||||||
|
// Without regex
|
||||||
|
if (value.startsWith('<Date: ') && value.endsWith('>')) {
|
||||||
|
const time = value.slice(7, -1);
|
||||||
|
return new Date(parseInt(time));
|
||||||
|
}
|
||||||
|
|
||||||
|
// If it does, convert it to a Date.
|
||||||
|
// if (dateMatch) {
|
||||||
|
// const { time } = dateMatch.groups!;
|
||||||
|
// return new Date(parseInt(time));
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
export * from './base-storage.js';
|
||||||
export * from './storage-memory.js';
|
export * from './storage-memory.js';
|
||||||
export * from './storage-memory-synced.js';
|
export * from './storage-memory-synced.js';
|
||||||
export * from './storage-localstorage.js';
|
export * from './storage-localstorage.js';
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { BaseStorage, FindOptions } from './base-storage.js';
|
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Key prefix separator used to namespace documents within localStorage.
|
* Key prefix separator used to namespace documents within localStorage.
|
||||||
@@ -7,20 +7,46 @@ const KEY_SEP = ':';
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Suffix appended to the namespace prefix to form the manifest key.
|
* Suffix appended to the namespace prefix to form the manifest key.
|
||||||
* The manifest tracks all document IDs belonging to this storage instance,
|
* The manifest tracks all internal keys belonging to this storage instance,
|
||||||
* avoiding expensive full-scan iterations over the entire localStorage.
|
* avoiding expensive full-scan iterations over the entire localStorage.
|
||||||
*/
|
*/
|
||||||
const MANIFEST_SUFFIX = '__keys__';
|
const MANIFEST_SUFFIX = '__keys__';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Suffix for the auto-incrementing counter persisted alongside the manifest.
|
||||||
|
*/
|
||||||
|
const COUNTER_SUFFIX = '__next__';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Separator used when joining multiple field values into a single index key.
|
||||||
|
*/
|
||||||
|
const INDEX_KEY_SEP = '\x00';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize an IndexDefinition into a canonical `string[][]` form.
|
||||||
|
* A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`.
|
||||||
|
*/
|
||||||
|
function normalizeIndexes(indexes?: IndexDefinition): string[][] {
|
||||||
|
if (!indexes || indexes.length === 0) return [];
|
||||||
|
if (typeof indexes[0] === 'string') {
|
||||||
|
return (indexes as string[]).map((field) => [field]);
|
||||||
|
}
|
||||||
|
return indexes as string[][];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implementation of BaseStorage using the browser's localStorage API.
|
* Implementation of BaseStorage using the browser's localStorage API.
|
||||||
*
|
*
|
||||||
* @remarks
|
* @remarks
|
||||||
* Documents are stored individually in localStorage, keyed by a namespaced
|
* Documents are stored individually in localStorage, keyed by a namespaced
|
||||||
* prefix combined with the document's `id` field. A separate manifest key
|
* prefix combined with an auto-generated numeric key. A separate manifest
|
||||||
* tracks all known document IDs so that read operations avoid scanning every
|
* tracks all internal keys so that read operations avoid scanning every
|
||||||
* key in localStorage.
|
* key in localStorage.
|
||||||
*
|
*
|
||||||
|
* Optional indexes provide fast lookups when a query filter matches
|
||||||
|
* an index exactly. Indexes are held in memory and rebuilt only when a
|
||||||
|
* cross-tab manifest change is detected.
|
||||||
|
*
|
||||||
* Because localStorage is synchronous and string-only, all values are
|
* Because localStorage is synchronous and string-only, all values are
|
||||||
* JSON-serialised on write and parsed on read.
|
* JSON-serialised on write and parsed on read.
|
||||||
*/
|
*/
|
||||||
@@ -32,25 +58,60 @@ export class StorageLocalStorage<
|
|||||||
*/
|
*/
|
||||||
static from<T extends Record<string, any>>(
|
static from<T extends Record<string, any>>(
|
||||||
prefix = 'qs',
|
prefix = 'qs',
|
||||||
|
indexes?: IndexDefinition,
|
||||||
): StorageLocalStorage<T> {
|
): StorageLocalStorage<T> {
|
||||||
return new StorageLocalStorage<T>(prefix);
|
return new StorageLocalStorage<T>(prefix, indexes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* In-memory copy of the document ID set.
|
* In-memory set of internal numeric keys.
|
||||||
* Re-read from localStorage at the start of every public method so that
|
* Re-read from localStorage at the start of every public method so that
|
||||||
* cross-tab mutations are always visible.
|
* cross-tab mutations are always visible.
|
||||||
*/
|
*/
|
||||||
private manifest: Set<string>;
|
private manifest: Set<number>;
|
||||||
|
|
||||||
/** Map of derived child instances, lazily created and cached. */
|
/** Auto-incrementing counter for generating internal keys. */
|
||||||
|
private nextKey: number;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Raw JSON string of the manifest as last seen. Used to detect cross-tab
|
||||||
|
* changes cheaply — if the raw string hasn't changed, indexes are still
|
||||||
|
* valid and we skip the expensive rebuild.
|
||||||
|
*/
|
||||||
|
private lastManifestRaw: string;
|
||||||
|
|
||||||
|
/** The normalized index definitions supplied at construction time. */
|
||||||
|
private indexDefs: string[][];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Secondary index maps (same structure as StorageMemory).
|
||||||
|
* Outer key = index name (joined field names).
|
||||||
|
* Inner key = index value (joined field values from a document).
|
||||||
|
* Inner value = set of internal numeric keys.
|
||||||
|
*/
|
||||||
|
private indexes: Map<string, Map<string, Set<number>>>;
|
||||||
|
|
||||||
|
/** Lazily-created child storage instances. */
|
||||||
private children: Map<string, StorageLocalStorage<any>>;
|
private children: Map<string, StorageLocalStorage<any>>;
|
||||||
|
|
||||||
constructor(private readonly prefix: string = 'qs') {
|
constructor(
|
||||||
|
private readonly prefix: string = 'qs',
|
||||||
|
indexes?: IndexDefinition,
|
||||||
|
) {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
this.children = new Map();
|
this.children = new Map();
|
||||||
this.manifest = this.loadManifest();
|
this.indexDefs = normalizeIndexes(indexes);
|
||||||
|
this.indexes = new Map();
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
this.indexes.set(fields.join(INDEX_KEY_SEP), new Map());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bootstrap from localStorage.
|
||||||
|
this.lastManifestRaw = '';
|
||||||
|
this.manifest = new Set();
|
||||||
|
this.nextKey = 0;
|
||||||
|
this.refreshManifest();
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -61,9 +122,10 @@ export class StorageLocalStorage<
|
|||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
|
|
||||||
for (const document of documents) {
|
for (const document of documents) {
|
||||||
const key = this.docKey(document.id);
|
const key = this.nextKey++;
|
||||||
localStorage.setItem(key, JSON.stringify(document));
|
localStorage.setItem(this.docKey(key), JSON.stringify(document));
|
||||||
this.manifest.add(document.id);
|
this.manifest.add(key);
|
||||||
|
this.addToIndexes(key, document);
|
||||||
this.emit('insert', { value: document });
|
this.emit('insert', { value: document });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,19 +135,33 @@ export class StorageLocalStorage<
|
|||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
|
|
||||||
let results: T[] = [];
|
let results: T[];
|
||||||
|
const indexedKeys = this.resolveIndexKeys(filter);
|
||||||
|
|
||||||
for (const id of this.manifest) {
|
if (indexedKeys !== null) {
|
||||||
const raw = localStorage.getItem(this.docKey(id));
|
// Use the index to narrow which documents we read from localStorage.
|
||||||
if (raw === null) continue;
|
results = [];
|
||||||
|
for (const key of indexedKeys) {
|
||||||
const doc = JSON.parse(raw) as T;
|
const raw = localStorage.getItem(this.docKey(key));
|
||||||
if (this.matchesFilter(doc, filter)) {
|
if (raw === null) continue;
|
||||||
results.push(doc);
|
const doc = JSON.parse(raw) as T;
|
||||||
|
if (this.matchesFilter(doc, filter)) {
|
||||||
|
results.push(doc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Full scan over all documents in the manifest.
|
||||||
|
results = [];
|
||||||
|
for (const key of this.manifest) {
|
||||||
|
const raw = localStorage.getItem(this.docKey(key));
|
||||||
|
if (raw === null) continue;
|
||||||
|
const doc = JSON.parse(raw) as T;
|
||||||
|
if (this.matchesFilter(doc, filter)) {
|
||||||
|
results.push(doc);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply sort before skip/limit so the window is deterministic.
|
|
||||||
if (options?.sort) {
|
if (options?.sort) {
|
||||||
results = this.applySorting(results, options.sort);
|
results = this.applySorting(results, options.sort);
|
||||||
}
|
}
|
||||||
@@ -105,17 +181,7 @@ export class StorageLocalStorage<
|
|||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
|
|
||||||
const candidates: T[] = [];
|
const candidates = this.collectMatches(filter);
|
||||||
|
|
||||||
for (const id of this.manifest) {
|
|
||||||
const raw = localStorage.getItem(this.docKey(id));
|
|
||||||
if (raw === null) continue;
|
|
||||||
|
|
||||||
const doc = JSON.parse(raw) as T;
|
|
||||||
if (this.matchesFilter(doc, filter)) {
|
|
||||||
candidates.push(doc);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const startIndex = options.skip ?? 0;
|
const startIndex = options.skip ?? 0;
|
||||||
const endIndex = options.limit
|
const endIndex = options.limit
|
||||||
@@ -123,18 +189,14 @@ export class StorageLocalStorage<
|
|||||||
: candidates.length;
|
: candidates.length;
|
||||||
const toProcess = candidates.slice(startIndex, endIndex);
|
const toProcess = candidates.slice(startIndex, endIndex);
|
||||||
|
|
||||||
for (const doc of toProcess) {
|
for (const [key, oldValue] of toProcess) {
|
||||||
const updatedDoc = { ...doc, ...update } as T;
|
const updatedDoc = { ...oldValue, ...update } as T;
|
||||||
localStorage.setItem(this.docKey(doc.id), JSON.stringify(updatedDoc));
|
localStorage.setItem(this.docKey(key), JSON.stringify(updatedDoc));
|
||||||
|
|
||||||
// If the id itself changed, update the manifest accordingly.
|
this.removeFromIndexes(key, oldValue);
|
||||||
if (update.id !== undefined && update.id !== doc.id) {
|
this.addToIndexes(key, updatedDoc);
|
||||||
localStorage.removeItem(this.docKey(doc.id));
|
|
||||||
this.manifest.delete(doc.id);
|
|
||||||
this.manifest.add(updatedDoc.id);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.emit('update', { value: updatedDoc });
|
this.emit('update', { oldValue, value: updatedDoc });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (toProcess.length > 0) {
|
if (toProcess.length > 0) {
|
||||||
@@ -150,17 +212,7 @@ export class StorageLocalStorage<
|
|||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
|
|
||||||
const candidates: T[] = [];
|
const candidates = this.collectMatches(filter);
|
||||||
|
|
||||||
for (const id of this.manifest) {
|
|
||||||
const raw = localStorage.getItem(this.docKey(id));
|
|
||||||
if (raw === null) continue;
|
|
||||||
|
|
||||||
const doc = JSON.parse(raw) as T;
|
|
||||||
if (this.matchesFilter(doc, filter)) {
|
|
||||||
candidates.push(doc);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const startIndex = options.skip ?? 0;
|
const startIndex = options.skip ?? 0;
|
||||||
const endIndex = options.limit
|
const endIndex = options.limit
|
||||||
@@ -168,10 +220,11 @@ export class StorageLocalStorage<
|
|||||||
: candidates.length;
|
: candidates.length;
|
||||||
const toProcess = candidates.slice(startIndex, endIndex);
|
const toProcess = candidates.slice(startIndex, endIndex);
|
||||||
|
|
||||||
for (const doc of toProcess) {
|
for (const [key, value] of toProcess) {
|
||||||
localStorage.removeItem(this.docKey(doc.id));
|
localStorage.removeItem(this.docKey(key));
|
||||||
this.manifest.delete(doc.id);
|
this.manifest.delete(key);
|
||||||
this.emit('delete', { value: doc });
|
this.removeFromIndexes(key, value);
|
||||||
|
this.emit('delete', { value });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (toProcess.length > 0) {
|
if (toProcess.length > 0) {
|
||||||
@@ -184,42 +237,55 @@ export class StorageLocalStorage<
|
|||||||
deriveChild<C>(path: string): BaseStorage<C> {
|
deriveChild<C>(path: string): BaseStorage<C> {
|
||||||
if (!this.children.has(path)) {
|
if (!this.children.has(path)) {
|
||||||
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
|
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
|
||||||
this.children.set(path, new StorageLocalStorage<C>(childPrefix));
|
this.children.set(
|
||||||
|
path,
|
||||||
|
new StorageLocalStorage<C>(childPrefix, this.indexDefs),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return this.children.get(path) as StorageLocalStorage<C>;
|
return this.children.get(path) as StorageLocalStorage<C>;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Private helpers
|
// Private helpers — filtering
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks whether a document satisfies every field in the filter.
|
* Checks whether a document satisfies every field in the filter.
|
||||||
* An empty or undefined filter matches everything.
|
|
||||||
*/
|
*/
|
||||||
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) {
|
if (!filter || Object.keys(filter).length === 0) return true;
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(filter)) {
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
if (item[key] !== value) {
|
if (item[key] !== value) return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sort an array of documents according to a sort specification.
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
* Keys map to `1` (ascending) or `-1` (descending).
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
*/
|
*/
|
||||||
private applySorting(
|
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
|
||||||
items: T[],
|
const indexKeys = this.resolveIndexKeys(filter);
|
||||||
sort: Record<string, 1 | -1>,
|
const keysToScan = indexKeys ?? this.manifest;
|
||||||
): T[] {
|
const results: Array<[number, T]> = [];
|
||||||
const sortEntries = Object.entries(sort);
|
|
||||||
|
|
||||||
|
for (const key of keysToScan) {
|
||||||
|
const raw = localStorage.getItem(this.docKey(key));
|
||||||
|
if (raw === null) continue;
|
||||||
|
const doc = JSON.parse(raw) as T;
|
||||||
|
if (this.matchesFilter(doc, filter)) {
|
||||||
|
results.push([key, doc]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sort documents according to a sort specification.
|
||||||
|
*/
|
||||||
|
private applySorting(items: T[], sort: Record<string, 1 | -1>): T[] {
|
||||||
|
const sortEntries = Object.entries(sort);
|
||||||
return [...items].sort((a, b) => {
|
return [...items].sort((a, b) => {
|
||||||
for (const [key, direction] of sortEntries) {
|
for (const [key, direction] of sortEntries) {
|
||||||
if (a[key] < b[key]) return -1 * direction;
|
if (a[key] < b[key]) return -1 * direction;
|
||||||
@@ -229,52 +295,158 @@ export class StorageLocalStorage<
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Private helpers — indexing
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build the full localStorage key for a given document ID.
|
* Build the index value string for a given document and set of fields.
|
||||||
|
* Returns `null` if any field is missing from the document.
|
||||||
*/
|
*/
|
||||||
private docKey(id: string): string {
|
private buildIndexValue(doc: Record<string, any>, fields: string[]): string | null {
|
||||||
return `${this.prefix}${KEY_SEP}${id}`;
|
const parts: string[] = [];
|
||||||
|
for (const field of fields) {
|
||||||
|
if (!(field in doc)) return null;
|
||||||
|
parts.push(String(doc[field]));
|
||||||
|
}
|
||||||
|
return parts.join(INDEX_KEY_SEP);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Register a document in all applicable indexes. */
|
||||||
|
private addToIndexes(internalKey: number, doc: T): void {
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(doc, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
let bucket = indexMap.get(indexValue);
|
||||||
|
if (!bucket) {
|
||||||
|
bucket = new Set();
|
||||||
|
indexMap.set(indexValue, bucket);
|
||||||
|
}
|
||||||
|
bucket.add(internalKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Remove a document from all applicable indexes. */
|
||||||
|
private removeFromIndexes(internalKey: number, doc: T): void {
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(doc, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
const bucket = indexMap.get(indexValue);
|
||||||
|
if (bucket) {
|
||||||
|
bucket.delete(internalKey);
|
||||||
|
if (bucket.size === 0) indexMap.delete(indexValue);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build the localStorage key used to persist the manifest.
|
* Attempt to resolve candidate internal keys from the indexes.
|
||||||
|
* Returns `null` if no index can serve the query.
|
||||||
*/
|
*/
|
||||||
|
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
|
||||||
|
if (!filter) return null;
|
||||||
|
const filterKeys = Object.keys(filter);
|
||||||
|
if (filterKeys.length === 0) return null;
|
||||||
|
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
if (!fields.every((f) => f in filter)) continue;
|
||||||
|
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(filter, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
const bucket = indexMap.get(indexValue);
|
||||||
|
return bucket ?? new Set();
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rebuild all in-memory index maps by reading every document from
|
||||||
|
* localStorage. Called only when a cross-tab manifest change is detected.
|
||||||
|
*/
|
||||||
|
private rebuildIndexes(): void {
|
||||||
|
for (const [, indexMap] of this.indexes) {
|
||||||
|
indexMap.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of this.manifest) {
|
||||||
|
const raw = localStorage.getItem(this.docKey(key));
|
||||||
|
if (raw === null) continue;
|
||||||
|
const doc = JSON.parse(raw) as T;
|
||||||
|
this.addToIndexes(key, doc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Private helpers — manifest & keys
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Build the full localStorage key for a given internal numeric key. */
|
||||||
|
private docKey(internalKey: number): string {
|
||||||
|
return `${this.prefix}${KEY_SEP}${internalKey}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build the localStorage key used to persist the manifest. */
|
||||||
private manifestKey(): string {
|
private manifestKey(): string {
|
||||||
return `${this.prefix}${KEY_SEP}${MANIFEST_SUFFIX}`;
|
return `${this.prefix}${KEY_SEP}${MANIFEST_SUFFIX}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Build the localStorage key used to persist the counter. */
|
||||||
|
private counterKey(): string {
|
||||||
|
return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Re-read the manifest from localStorage into memory.
|
* Re-read the manifest from localStorage into memory.
|
||||||
* Called at the start of every public method so that cross-tab writes
|
* Called at the start of every public method so that cross-tab writes
|
||||||
* are always reflected before we operate.
|
* are always reflected before we operate.
|
||||||
|
*
|
||||||
|
* Indexes are only rebuilt when the raw manifest string has actually
|
||||||
|
* changed, avoiding unnecessary full-document reads for same-tab calls.
|
||||||
*/
|
*/
|
||||||
private refreshManifest(): void {
|
private refreshManifest(): void {
|
||||||
this.manifest = this.loadManifest();
|
const raw = localStorage.getItem(this.manifestKey()) ?? '[]';
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
if (raw === this.lastManifestRaw) return;
|
||||||
* Load the manifest (set of document IDs) from localStorage.
|
|
||||||
* Falls back to an empty set if nothing is stored yet.
|
this.lastManifestRaw = raw;
|
||||||
*/
|
|
||||||
private loadManifest(): Set<string> {
|
|
||||||
const raw = localStorage.getItem(this.manifestKey());
|
|
||||||
if (raw === null) return new Set();
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const ids: string[] = JSON.parse(raw);
|
const keys: number[] = JSON.parse(raw);
|
||||||
return new Set(ids);
|
this.manifest = new Set(keys);
|
||||||
} catch {
|
} catch {
|
||||||
return new Set();
|
this.manifest = new Set();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore the counter from localStorage.
|
||||||
|
const counterRaw = localStorage.getItem(this.counterKey());
|
||||||
|
this.nextKey = counterRaw !== null ? Number(counterRaw) : 0;
|
||||||
|
|
||||||
|
// Manifest changed — indexes are potentially stale.
|
||||||
|
if (this.indexDefs.length > 0) {
|
||||||
|
this.rebuildIndexes();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Persist the current in-memory manifest to localStorage.
|
* Persist the current in-memory manifest and counter to localStorage.
|
||||||
*/
|
*/
|
||||||
private persistManifest(): void {
|
private persistManifest(): void {
|
||||||
localStorage.setItem(
|
const raw = JSON.stringify([...this.manifest]);
|
||||||
this.manifestKey(),
|
localStorage.setItem(this.manifestKey(), raw);
|
||||||
JSON.stringify([...this.manifest]),
|
localStorage.setItem(this.counterKey(), String(this.nextKey));
|
||||||
);
|
|
||||||
|
// Keep the cached raw string in sync so the next refreshManifest()
|
||||||
|
// recognises this as "our own write" and skips the rebuild.
|
||||||
|
this.lastManifestRaw = raw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,14 +20,11 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
});
|
});
|
||||||
|
|
||||||
this.store.on('update', async (payload) => {
|
this.store.on('update', async (payload) => {
|
||||||
// For update events, we need to find and update the document in memory
|
// Remove the old version and insert the new one. We use delete + insert
|
||||||
// Since we don't have the filter, we'll update by key
|
// rather than updateOne because the oldValue is the complete document,
|
||||||
const filter = {
|
// guaranteeing an exact match without assuming any particular key field.
|
||||||
id: payload.value.id,
|
await this.inMemoryCache.deleteOne(payload.oldValue);
|
||||||
} as unknown as Partial<T>
|
await this.inMemoryCache.insertOne(payload.value);
|
||||||
|
|
||||||
// Update the document in memory by ID.
|
|
||||||
await this.inMemoryCache.updateOne(filter, payload.value);
|
|
||||||
this.emit('update', payload);
|
this.emit('update', payload);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,36 +1,189 @@
|
|||||||
import { BaseStorage, FindOptions } from './base-storage.js';
|
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Separator used when joining multiple field values into a single index key.
|
||||||
|
* Chosen to be unlikely to appear in real field values.
|
||||||
|
*/
|
||||||
|
const INDEX_KEY_SEP = '\x00';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize an IndexDefinition into a canonical `string[][]` form.
|
||||||
|
* A flat `string[]` like `['id', 'name']` becomes `[['id'], ['name']]`.
|
||||||
|
* An already-nested `string[][]` is returned as-is.
|
||||||
|
*/
|
||||||
|
function normalizeIndexes(indexes?: IndexDefinition): string[][] {
|
||||||
|
if (!indexes || indexes.length === 0) return [];
|
||||||
|
|
||||||
|
// If the first element is a string, treat the whole array as shorthand.
|
||||||
|
if (typeof indexes[0] === 'string') {
|
||||||
|
return (indexes as string[]).map((field) => [field]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return indexes as string[][];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implementation of BaseStore using Memory as the storage backend.
|
* Implementation of BaseStore using Memory as the storage backend.
|
||||||
*
|
*
|
||||||
* @remarks
|
* @remarks
|
||||||
* This implementation can be used testing and caching of expensive operations.
|
* Documents are keyed internally by an auto-incrementing numeric key.
|
||||||
|
* Optional indexes provide O(1) lookups when a query filter matches
|
||||||
|
* an index exactly.
|
||||||
*/
|
*/
|
||||||
export class StorageMemory<
|
export class StorageMemory<
|
||||||
T extends Record<string, any> = Record<string, any>,
|
T extends Record<string, any> = Record<string, any>,
|
||||||
> extends BaseStorage<T> {
|
> extends BaseStorage<T> {
|
||||||
// TODO: Eventually this may accept indexes as an argument.
|
static from<T extends Record<string, any>>(
|
||||||
static from<T extends Record<string, any>>(): StorageMemory<T> {
|
indexes?: IndexDefinition,
|
||||||
return new StorageMemory<T>();
|
): StorageMemory<T> {
|
||||||
|
return new StorageMemory<T>(indexes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private store: Map<string, T>;
|
/** Auto-incrementing counter used to generate internal keys. */
|
||||||
|
private nextKey = 0;
|
||||||
|
|
||||||
|
/** Primary document store keyed by an opaque internal key. */
|
||||||
|
private store: Map<number, T>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Secondary index maps.
|
||||||
|
* Outer key = index name (joined field names).
|
||||||
|
* Inner key = index value (joined field values from a document).
|
||||||
|
* Inner value = set of internal keys that share this index value.
|
||||||
|
*/
|
||||||
|
private indexes: Map<string, Map<string, Set<number>>>;
|
||||||
|
|
||||||
|
/** The normalized index definitions supplied at construction time. */
|
||||||
|
private indexDefs: string[][];
|
||||||
|
|
||||||
|
/** Lazily-created child storage instances. */
|
||||||
private children: Map<string, StorageMemory<any>>;
|
private children: Map<string, StorageMemory<any>>;
|
||||||
|
|
||||||
constructor() {
|
constructor(indexes?: IndexDefinition) {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
this.store = new Map();
|
this.store = new Map();
|
||||||
this.children = new Map();
|
this.children = new Map();
|
||||||
|
this.indexDefs = normalizeIndexes(indexes);
|
||||||
|
|
||||||
|
// Initialise an empty map for each index definition.
|
||||||
|
this.indexes = new Map();
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
this.indexes.set(fields.join(INDEX_KEY_SEP), new Map());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Abstract method implementations
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async insertMany(documents: Array<T>): Promise<void> {
|
async insertMany(documents: Array<T>): Promise<void> {
|
||||||
for (const document of documents) {
|
for (const document of documents) {
|
||||||
this.store.set(document.id, document);
|
const key = this.nextKey++;
|
||||||
|
this.store.set(key, document);
|
||||||
|
this.addToIndexes(key, document);
|
||||||
this.emit('insert', { value: document });
|
this.emit('insert', { value: document });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
||||||
|
let results: T[];
|
||||||
|
|
||||||
|
// Attempt to satisfy the query via an index.
|
||||||
|
const indexed = this.findViaIndex(filter);
|
||||||
|
|
||||||
|
if (indexed !== null) {
|
||||||
|
results = indexed;
|
||||||
|
} else {
|
||||||
|
// Fall back to a full scan.
|
||||||
|
results = [];
|
||||||
|
for (const [, value] of this.store) {
|
||||||
|
if (this.matchesFilter(value, filter)) {
|
||||||
|
results.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply sort before skip/limit so the window is deterministic.
|
||||||
|
if (options?.sort) {
|
||||||
|
results = this.applySorting(results, options.sort);
|
||||||
|
}
|
||||||
|
|
||||||
|
const startIndex = options?.skip ?? 0;
|
||||||
|
const endIndex = options?.limit
|
||||||
|
? startIndex + options.limit
|
||||||
|
: results.length;
|
||||||
|
|
||||||
|
return results.slice(startIndex, endIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateMany(
|
||||||
|
filter: Partial<T>,
|
||||||
|
update: Partial<T>,
|
||||||
|
options: Partial<FindOptions> = {},
|
||||||
|
): Promise<number> {
|
||||||
|
const itemsToUpdate = this.collectMatches(filter);
|
||||||
|
|
||||||
|
const startIndex = options.skip ?? 0;
|
||||||
|
const endIndex = options.limit
|
||||||
|
? startIndex + options.limit
|
||||||
|
: itemsToUpdate.length;
|
||||||
|
const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex);
|
||||||
|
|
||||||
|
let updated = 0;
|
||||||
|
for (const [key, oldValue] of itemsToProcess) {
|
||||||
|
const updatedValue = { ...oldValue, ...update } as T;
|
||||||
|
|
||||||
|
// Re-index: remove old entries, store new doc, add new entries.
|
||||||
|
this.removeFromIndexes(key, oldValue);
|
||||||
|
this.store.set(key, updatedValue);
|
||||||
|
this.addToIndexes(key, updatedValue);
|
||||||
|
|
||||||
|
this.emit('update', { oldValue, value: updatedValue });
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteMany(
|
||||||
|
filter: Partial<T>,
|
||||||
|
options: Partial<FindOptions> = {},
|
||||||
|
): Promise<number> {
|
||||||
|
const rowsToDelete = this.collectMatches(filter);
|
||||||
|
|
||||||
|
const startIndex = options.skip ?? 0;
|
||||||
|
const endIndex = options.limit
|
||||||
|
? startIndex + options.limit
|
||||||
|
: rowsToDelete.length;
|
||||||
|
const rowsToProcess = rowsToDelete.slice(startIndex, endIndex);
|
||||||
|
|
||||||
|
let deleted = 0;
|
||||||
|
for (const [key, value] of rowsToProcess) {
|
||||||
|
this.removeFromIndexes(key, value);
|
||||||
|
this.store.delete(key);
|
||||||
|
this.emit('delete', { value });
|
||||||
|
deleted++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
deriveChild<C>(path: string): BaseStorage<C> {
|
||||||
|
if (!this.children.has(path)) {
|
||||||
|
this.children.set(path, new StorageMemory<C>(this.indexDefs));
|
||||||
|
}
|
||||||
|
return this.children.get(path) as StorageMemory<C>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Private helpers — filtering
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks whether a document satisfies every field in the filter.
|
||||||
|
* An empty or undefined filter matches everything.
|
||||||
|
*/
|
||||||
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) {
|
if (!filter || Object.keys(filter).length === 0) {
|
||||||
return true;
|
return true;
|
||||||
@@ -44,84 +197,146 @@ export class StorageMemory<
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
/**
|
||||||
const results: T[] = [];
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
for (const [, value] of this.store) {
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
|
*/
|
||||||
|
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
|
||||||
|
const indexKeys = this.resolveIndexKeys(filter);
|
||||||
|
|
||||||
|
if (indexKeys !== null) {
|
||||||
|
// We have candidate internal keys from the index — fetch and verify.
|
||||||
|
const results: Array<[number, T]> = [];
|
||||||
|
for (const key of indexKeys) {
|
||||||
|
const doc = this.store.get(key);
|
||||||
|
if (doc && this.matchesFilter(doc, filter)) {
|
||||||
|
results.push([key, doc]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Full scan.
|
||||||
|
const results: Array<[number, T]> = [];
|
||||||
|
for (const [key, value] of this.store) {
|
||||||
if (this.matchesFilter(value, filter)) {
|
if (this.matchesFilter(value, filter)) {
|
||||||
results.push(value);
|
results.push([key, value]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
/**
|
||||||
filter: Partial<T>,
|
* Sort an array of documents according to a sort specification.
|
||||||
update: Partial<T>,
|
* Keys map to `1` (ascending) or `-1` (descending).
|
||||||
options: Partial<FindOptions> = {},
|
*/
|
||||||
): Promise<number> {
|
private applySorting(items: T[], sort: Record<string, 1 | -1>): T[] {
|
||||||
let updated = 0;
|
const sortEntries = Object.entries(sort);
|
||||||
const itemsToUpdate: Array<[string, T]> = [];
|
return [...items].sort((a, b) => {
|
||||||
|
for (const [key, direction] of sortEntries) {
|
||||||
// Collect all matching items
|
if (a[key] < b[key]) return -1 * direction;
|
||||||
for (const [key, value] of this.store) {
|
if (a[key] > b[key]) return 1 * direction;
|
||||||
if (this.matchesFilter(value, filter)) {
|
|
||||||
itemsToUpdate.push([key, value]);
|
|
||||||
}
|
}
|
||||||
}
|
return 0;
|
||||||
|
});
|
||||||
// Apply skip and limit
|
|
||||||
const startIndex = options.skip || 0;
|
|
||||||
const endIndex = options.limit
|
|
||||||
? startIndex + options.limit
|
|
||||||
: itemsToUpdate.length;
|
|
||||||
const itemsToProcess = itemsToUpdate.slice(startIndex, endIndex);
|
|
||||||
|
|
||||||
// Update items
|
|
||||||
for (const [key, oldValue] of itemsToProcess) {
|
|
||||||
const updatedValue = { ...oldValue, ...update };
|
|
||||||
this.store.set(key, updatedValue);
|
|
||||||
this.emit('update', { value: updatedValue });
|
|
||||||
updated++;
|
|
||||||
}
|
|
||||||
|
|
||||||
return updated;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(
|
// ---------------------------------------------------------------------------
|
||||||
filter: Partial<T>,
|
// Private helpers — indexing
|
||||||
options: Partial<FindOptions> = {},
|
// ---------------------------------------------------------------------------
|
||||||
): Promise<number> {
|
|
||||||
let deleted = 0;
|
|
||||||
const rowsToDelete: Array<T> = [];
|
|
||||||
|
|
||||||
// Collect all matching keys
|
/**
|
||||||
for (const [key, value] of this.store) {
|
* Build the index value string for a given document and set of fields.
|
||||||
if (this.matchesFilter(value, filter)) {
|
* Returns `null` if any of the fields are missing from the document,
|
||||||
rowsToDelete.push(value);
|
* since we can't meaningfully index a partial key.
|
||||||
}
|
*/
|
||||||
|
private buildIndexValue(doc: Record<string, any>, fields: string[]): string | null {
|
||||||
|
const parts: string[] = [];
|
||||||
|
for (const field of fields) {
|
||||||
|
if (!(field in doc)) return null;
|
||||||
|
parts.push(String(doc[field]));
|
||||||
}
|
}
|
||||||
|
return parts.join(INDEX_KEY_SEP);
|
||||||
// Apply skip and limit
|
|
||||||
const startIndex = options.skip || 0;
|
|
||||||
const endIndex = options.limit
|
|
||||||
? startIndex + options.limit
|
|
||||||
: rowsToDelete.length;
|
|
||||||
const rowsToProcess = rowsToDelete.slice(startIndex, endIndex);
|
|
||||||
|
|
||||||
// Delete items
|
|
||||||
for (const row of rowsToProcess) {
|
|
||||||
this.store.delete(row.id);
|
|
||||||
this.emit('delete', { value: row });
|
|
||||||
deleted++;
|
|
||||||
}
|
|
||||||
|
|
||||||
return deleted;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
deriveChild<C>(path: string): BaseStorage<C> {
|
/** Register a document in all applicable indexes. */
|
||||||
if (!this.children.has(path)) {
|
private addToIndexes(internalKey: number, doc: T): void {
|
||||||
this.children.set(path, new StorageMemory<C>());
|
for (const fields of this.indexDefs) {
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(doc, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
let bucket = indexMap.get(indexValue);
|
||||||
|
if (!bucket) {
|
||||||
|
bucket = new Set();
|
||||||
|
indexMap.set(indexValue, bucket);
|
||||||
|
}
|
||||||
|
bucket.add(internalKey);
|
||||||
}
|
}
|
||||||
return this.children.get(path) as StorageMemory<C>;
|
}
|
||||||
|
|
||||||
|
/** Remove a document from all applicable indexes. */
|
||||||
|
private removeFromIndexes(internalKey: number, doc: T): void {
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(doc, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
const bucket = indexMap.get(indexValue);
|
||||||
|
if (bucket) {
|
||||||
|
bucket.delete(internalKey);
|
||||||
|
if (bucket.size === 0) indexMap.delete(indexValue);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to resolve a set of candidate internal keys from the indexes.
|
||||||
|
* Returns `null` if no index can serve the query.
|
||||||
|
*
|
||||||
|
* An index is used when the filter fields are a superset of (or equal to)
|
||||||
|
* an index's fields — meaning the index value can be fully constructed
|
||||||
|
* from the filter.
|
||||||
|
*/
|
||||||
|
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
|
||||||
|
if (!filter) return null;
|
||||||
|
const filterKeys = Object.keys(filter);
|
||||||
|
if (filterKeys.length === 0) return null;
|
||||||
|
|
||||||
|
for (const fields of this.indexDefs) {
|
||||||
|
// Every field in the index must be present in the filter.
|
||||||
|
if (!fields.every((f) => f in filter)) continue;
|
||||||
|
|
||||||
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
|
const indexValue = this.buildIndexValue(filter, fields);
|
||||||
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
const bucket = indexMap.get(indexValue);
|
||||||
|
return bucket ?? new Set();
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to answer a `find` query entirely through an index.
|
||||||
|
* Returns `null` when no index can serve the filter, signalling
|
||||||
|
* the caller to fall back to a full scan.
|
||||||
|
*/
|
||||||
|
private findViaIndex(filter?: Partial<T>): T[] | null {
|
||||||
|
const keys = this.resolveIndexKeys(filter);
|
||||||
|
if (keys === null) return null;
|
||||||
|
|
||||||
|
const results: T[] = [];
|
||||||
|
for (const key of keys) {
|
||||||
|
const doc = this.store.get(key);
|
||||||
|
if (doc && this.matchesFilter(doc, filter)) {
|
||||||
|
results.push(doc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
124
src/utils/ext-json.ts
Normal file
124
src/utils/ext-json.ts
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
/**
|
||||||
|
* TODO: These are intended as temporary stand-ins until this functionality has been implemented directly in LibAuth.
|
||||||
|
* We are doing this so that we may better standardize with the rest of the BCH eco-system in future.
|
||||||
|
* See: https://github.com/bitauth/libauth/pull/108
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { binToHex, hexToBin } from '@bitauth/libauth';
|
||||||
|
|
||||||
|
export const extendedJsonReplacer = function (value: any): any {
|
||||||
|
if (typeof value === 'bigint') {
|
||||||
|
return `<bigint: ${value.toString()}n>`;
|
||||||
|
} else if (value instanceof Uint8Array) {
|
||||||
|
return `<Uint8Array: ${binToHex(value)}>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const extendedJsonReviver = function (value: any): any {
|
||||||
|
// Define RegEx that matches our Extended JSON fields.
|
||||||
|
const bigIntRegex = /^<bigint: (?<bigint>[+-]?[0-9]*)n>$/;
|
||||||
|
const uint8ArrayRegex = /^<Uint8Array: (?<hex>[a-f0-9]*)>$/;
|
||||||
|
|
||||||
|
// Only perform a check if the value is a string.
|
||||||
|
// NOTE: We can skip all other values as all Extended JSON encoded fields WILL be a string.
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
// Check if this value matches an Extended JSON encoded bigint.
|
||||||
|
const bigintMatch = value.match(bigIntRegex);
|
||||||
|
if (bigintMatch) {
|
||||||
|
// Access the named group directly instead of using array indices
|
||||||
|
const { bigint } = bigintMatch.groups!;
|
||||||
|
|
||||||
|
// Return the value casted to bigint.
|
||||||
|
return BigInt(bigint);
|
||||||
|
}
|
||||||
|
|
||||||
|
const uint8ArrayMatch = value.match(uint8ArrayRegex);
|
||||||
|
if (uint8ArrayMatch) {
|
||||||
|
// Access the named group directly instead of using array indices
|
||||||
|
const { hex } = uint8ArrayMatch.groups!;
|
||||||
|
|
||||||
|
// Return the value casted to bigint.
|
||||||
|
return hexToBin(hex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the original value.
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const encodeExtendedJsonObject = function (value: any): any {
|
||||||
|
// If this is an object type (and it is not null - which is technically an "object")...
|
||||||
|
// ... and it is not an ArrayBuffer (e.g. Uint8Array) which is also technically an "object...
|
||||||
|
if (
|
||||||
|
typeof value === 'object' &&
|
||||||
|
value !== null &&
|
||||||
|
!ArrayBuffer.isView(value)
|
||||||
|
) {
|
||||||
|
// If this is an array, recursively call this function on each value.
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return value.map(encodeExtendedJsonObject);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Declare object to store extended JSON entries.
|
||||||
|
const encodedObject: any = {};
|
||||||
|
|
||||||
|
// Iterate through each entry and encode it to extended JSON.
|
||||||
|
for (const [key, valueToEncode] of Object.entries(value)) {
|
||||||
|
encodedObject[key] = encodeExtendedJsonObject(valueToEncode);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the extended JSON encoded object.
|
||||||
|
return encodedObject;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the replaced value.
|
||||||
|
return extendedJsonReplacer(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const decodeExtendedJsonObject = function (value: any): any {
|
||||||
|
// If this is an object type (and it is not null - which is technically an "object")...
|
||||||
|
// ... and it is not an ArrayBuffer (e.g. Uint8Array) which is also technically an "object...
|
||||||
|
if (
|
||||||
|
typeof value === 'object' &&
|
||||||
|
value !== null &&
|
||||||
|
!ArrayBuffer.isView(value)
|
||||||
|
) {
|
||||||
|
// If this is an array, recursively call this function on each value.
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return value.map(decodeExtendedJsonObject);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Declare object to store decoded JSON entries.
|
||||||
|
const decodedObject: any = {};
|
||||||
|
|
||||||
|
// Iterate through each entry and decode it from extended JSON.
|
||||||
|
for (const [key, valueToEncode] of Object.entries(value)) {
|
||||||
|
decodedObject[key] = decodeExtendedJsonObject(valueToEncode);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the extended JSON encoded object.
|
||||||
|
return decodedObject;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the revived value.
|
||||||
|
return extendedJsonReviver(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const encodeExtendedJson = function (
|
||||||
|
value: any,
|
||||||
|
space: number | undefined = undefined,
|
||||||
|
): string {
|
||||||
|
const replacedObject = encodeExtendedJsonObject(value);
|
||||||
|
const stringifiedObject = JSON.stringify(replacedObject, null, space);
|
||||||
|
|
||||||
|
return stringifiedObject;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const decodeExtendedJson = function (json: string): any {
|
||||||
|
const parsedObject = JSON.parse(json);
|
||||||
|
const revivedObject = decodeExtendedJsonObject(parsedObject);
|
||||||
|
|
||||||
|
return revivedObject;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user