Plaintext values on encryption
This commit is contained in:
@@ -7,6 +7,7 @@ const alicePublicKey = await alice.getPublicKey();
|
|||||||
const bobPublicKey = await bob.getPublicKey();
|
const bobPublicKey = await bob.getPublicKey();
|
||||||
|
|
||||||
const keyStart = performance.now();
|
const keyStart = performance.now();
|
||||||
|
|
||||||
const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey);
|
const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey);
|
||||||
const bobSharedSecret = await bob.getSharedSecret(alicePublicKey);
|
const bobSharedSecret = await bob.getSharedSecret(alicePublicKey);
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ function fmtOps(ops: number): string {
|
|||||||
/**
|
/**
|
||||||
* Run a full suite of benchmarks against a given storage instance.
|
* Run a full suite of benchmarks against a given storage instance.
|
||||||
*/
|
*/
|
||||||
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[]) {
|
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[], supportsRangeOps = true) {
|
||||||
const count = docs.length;
|
const count = docs.length;
|
||||||
console.log(`\n${'='.repeat(60)}`);
|
console.log(`\n${'='.repeat(60)}`);
|
||||||
console.log(` ${label} (${count.toLocaleString()} documents)`);
|
console.log(` ${label} (${count.toLocaleString()} documents)`);
|
||||||
@@ -67,32 +67,51 @@ async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs:
|
|||||||
});
|
});
|
||||||
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
|
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
|
||||||
|
|
||||||
// --- Find by indexed field (single-key lookup, repeated) ---
|
// --- Find by indexed field (equality) ---
|
||||||
const lookupCount = Math.min(count, 1_000);
|
const lookupCount = Math.min(count, 1_000);
|
||||||
const findIndexedMs = await time(async () => {
|
const findIndexedMs = await time(async () => {
|
||||||
for (let i = 0; i < lookupCount; i++) {
|
for (let i = 0; i < lookupCount; i++) {
|
||||||
await storage.findOne({ id: `id-${i}` } as Partial<Doc>);
|
await storage.findOne({ id: `id-${i}` });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
|
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
|
||||||
|
|
||||||
// --- Find by non-indexed field (full scan, repeated) ---
|
// --- Find by non-indexed field (full scan) ---
|
||||||
const scanCount = Math.min(count, 1_000);
|
const scanCount = Math.min(count, 1_000);
|
||||||
const findScanMs = await time(async () => {
|
const findScanMs = await time(async () => {
|
||||||
for (let i = 0; i < scanCount; i++) {
|
for (let i = 0; i < scanCount; i++) {
|
||||||
await storage.findOne({ email: `user-${i}@test.com` } as Partial<Doc>);
|
await storage.findOne({ email: `user-${i}@test.com` });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
|
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
|
||||||
|
|
||||||
|
// --- Find with $gte / $lt range (full scan) ---
|
||||||
|
if (supportsRangeOps) {
|
||||||
|
const rangeCount = Math.min(count, 100);
|
||||||
|
let rangeTotal = 0;
|
||||||
|
const findRangeMs = await time(async () => {
|
||||||
|
for (let i = 0; i < rangeCount; i++) {
|
||||||
|
const results = await storage.find({ age: { $gte: 30, $lt: 40 } });
|
||||||
|
rangeTotal += results.length;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` find $gte/$lt ${findRangeMs.toFixed(2)}ms (${fmtOps((rangeCount / findRangeMs) * 1000)} ops/sec) [${rangeCount} queries, ~${Math.round(rangeTotal / rangeCount)} hits/query]`);
|
||||||
|
|
||||||
|
// --- Find with combined equality + operator (index narrows, operator verifies) ---
|
||||||
|
const comboCount = Math.min(count, 1_000);
|
||||||
|
const findComboMs = await time(async () => {
|
||||||
|
for (let i = 0; i < comboCount; i++) {
|
||||||
|
await storage.find({ id: `id-${i}`, age: { $gte: 20 } });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(` find idx+operator ${findComboMs.toFixed(2)}ms (${fmtOps((comboCount / findComboMs) * 1000)} ops/sec) [${comboCount} queries]`);
|
||||||
|
}
|
||||||
|
|
||||||
// --- Update by indexed field ---
|
// --- Update by indexed field ---
|
||||||
const updateCount = Math.min(count, 1_000);
|
const updateCount = Math.min(count, 1_000);
|
||||||
const updateMs = await time(async () => {
|
const updateMs = await time(async () => {
|
||||||
for (let i = 0; i < updateCount; i++) {
|
for (let i = 0; i < updateCount; i++) {
|
||||||
await storage.updateOne(
|
await storage.updateOne({ id: `id-${i}` }, { name: `updated-${i}` });
|
||||||
{ id: `id-${i}` } as Partial<Doc>,
|
|
||||||
{ age: 99 } as Partial<Doc>,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
|
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
|
||||||
@@ -101,7 +120,7 @@ async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs:
|
|||||||
const deleteCount = Math.min(count, 1_000);
|
const deleteCount = Math.min(count, 1_000);
|
||||||
const deleteMs = await time(async () => {
|
const deleteMs = await time(async () => {
|
||||||
for (let i = 0; i < deleteCount; i++) {
|
for (let i = 0; i < deleteCount; i++) {
|
||||||
await storage.deleteOne({ id: `id-${i}` } as Partial<Doc>);
|
await storage.deleteOne({ id: `id-${i}` });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
|
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
|
||||||
@@ -132,7 +151,7 @@ for (const count of DOC_COUNTS) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// EncryptedStorage — crypto overhead dominates, so use smaller counts
|
// EncryptedStorage — with plaintextKeys for range queries
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000];
|
const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000];
|
||||||
@@ -141,15 +160,29 @@ const encryptionKey = await AESKey.fromSeed('benchmark-key');
|
|||||||
for (const count of ENCRYPTED_DOC_COUNTS) {
|
for (const count of ENCRYPTED_DOC_COUNTS) {
|
||||||
const docs = generateDocs(count);
|
const docs = generateDocs(count);
|
||||||
|
|
||||||
// Encrypted + indexed backing store.
|
// No indexes + plaintextKeys — range queries on age work.
|
||||||
const encBase = StorageMemory.from<Record<string, string>>(['id', 'name']);
|
const encBaseNoIdx = StorageMemory.from<Record<string, any>>();
|
||||||
const encrypted = EncryptedStorage.from<Doc>(encBase, encryptionKey);
|
const encNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey, {
|
||||||
await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs);
|
plaintextKeys: ['age'],
|
||||||
|
});
|
||||||
|
await benchmarkStorage('Encrypted (no indexes, plaintextKeys: age)', encNoIdx, docs);
|
||||||
|
|
||||||
// Encrypted + no-index backing store.
|
// Indexed + plaintextKeys — range queries on age work.
|
||||||
const encBaseNoIdx = StorageMemory.from<Record<string, string>>();
|
const encBaseA = StorageMemory.from<Record<string, any>>(['id', 'name']);
|
||||||
const encryptedNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey);
|
const encA = EncryptedStorage.from<Doc>(encBaseA, encryptionKey, {
|
||||||
await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs);
|
plaintextKeys: ['age'],
|
||||||
|
});
|
||||||
|
await benchmarkStorage('Encrypted (indexed, plaintextKeys: age)', encA, docs);
|
||||||
|
|
||||||
|
// Indexed, fully encrypted — same indexes but no plaintext keys.
|
||||||
|
const encBaseB = StorageMemory.from<Record<string, any>>(['id', 'name']);
|
||||||
|
const encB = EncryptedStorage.from<Doc>(encBaseB, encryptionKey);
|
||||||
|
await benchmarkStorage('Encrypted (indexed, fully encrypted)', encB, docs, false);
|
||||||
|
|
||||||
|
// No indexes, fully encrypted — worst case.
|
||||||
|
const encBaseC = StorageMemory.from<Record<string, any>>();
|
||||||
|
const encC = EncryptedStorage.from<Doc>(encBaseC, encryptionKey);
|
||||||
|
await benchmarkStorage('Encrypted (no indexes, fully encrypted)', encC, docs, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('\nDone.\n');
|
console.log('\nDone.\n');
|
||||||
|
|||||||
@@ -8,6 +8,9 @@
|
|||||||
"format": "prettier --write .",
|
"format": "prettier --write .",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1",
|
"test": "echo \"Error: no test specified\" && exit 1",
|
||||||
|
|
||||||
|
"benchmark:sha256": "tsx benchmarks/sha256.ts",
|
||||||
|
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",
|
||||||
|
"benchmark:encryption": "tsx benchmarks/sekp256k1.ts",
|
||||||
"benchmark:storage": "tsx benchmarks/storage.ts"
|
"benchmark:storage": "tsx benchmarks/storage.ts"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
|
|||||||
@@ -19,6 +19,54 @@ export type FindOptions = {
|
|||||||
*/
|
*/
|
||||||
export type IndexDefinition = string[] | string[][];
|
export type IndexDefinition = string[] | string[][];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB-style comparison operators for a single field value.
|
||||||
|
*/
|
||||||
|
export type ComparisonOperators<V> = {
|
||||||
|
$eq?: V;
|
||||||
|
$ne?: V;
|
||||||
|
$lt?: V;
|
||||||
|
$lte?: V;
|
||||||
|
$gt?: V;
|
||||||
|
$gte?: V;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A filter value for a single field — either a plain value (equality shorthand)
|
||||||
|
* or an object of comparison operators.
|
||||||
|
*/
|
||||||
|
export type FieldFilter<V> = V | ComparisonOperators<V>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query filter that supports both equality shorthand and comparison operators.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* // Equality shorthand
|
||||||
|
* { name: 'foo' }
|
||||||
|
*
|
||||||
|
* // Comparison operators
|
||||||
|
* { age: { $gte: 18, $lt: 65 } }
|
||||||
|
*
|
||||||
|
* // Combined
|
||||||
|
* { name: 'foo', age: { $gte: 18 } }
|
||||||
|
*/
|
||||||
|
export type Filter<T extends Record<string, any>> = {
|
||||||
|
[K in keyof T]?: FieldFilter<T[K]>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect whether a filter value is an operator object (e.g. `{ $lt: 50 }`)
|
||||||
|
* rather than a plain value. Guards against Date and Array which are objects
|
||||||
|
* but represent document values, not operators.
|
||||||
|
*/
|
||||||
|
export function isOperatorObject(value: unknown): value is ComparisonOperators<any> {
|
||||||
|
return value !== null
|
||||||
|
&& typeof value === 'object'
|
||||||
|
&& !Array.isArray(value)
|
||||||
|
&& !(value instanceof Date)
|
||||||
|
&& Object.keys(value).some((k) => k.startsWith('$'));
|
||||||
|
}
|
||||||
|
|
||||||
export type StorageEvent<T = Record<string, any>> = {
|
export type StorageEvent<T = Record<string, any>> = {
|
||||||
insert: {
|
insert: {
|
||||||
value: T;
|
value: T;
|
||||||
@@ -54,7 +102,7 @@ export abstract class BaseStorage<
|
|||||||
* Find a single document that matches the filter
|
* Find a single document that matches the filter
|
||||||
* @param filter MongoDB-like query filter
|
* @param filter MongoDB-like query filter
|
||||||
*/
|
*/
|
||||||
async findOne(filter?: Partial<T>): Promise<T | null> {
|
async findOne(filter?: Filter<T>): Promise<T | null> {
|
||||||
const results = await this.find(filter);
|
const results = await this.find(filter);
|
||||||
return results.length > 0 ? results[0] : null;
|
return results.length > 0 ? results[0] : null;
|
||||||
}
|
}
|
||||||
@@ -64,7 +112,7 @@ export abstract class BaseStorage<
|
|||||||
* @param filter MongoDB-like query filter
|
* @param filter MongoDB-like query filter
|
||||||
* @param options Query options (limit, skip, sort)
|
* @param options Query options (limit, skip, sort)
|
||||||
*/
|
*/
|
||||||
abstract find(filter?: Partial<T>, options?: FindOptions): Promise<T[]>;
|
abstract find(filter?: Filter<T>, options?: FindOptions): Promise<T[]>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update a document that matches the filter
|
* Update a document that matches the filter
|
||||||
@@ -72,7 +120,7 @@ export abstract class BaseStorage<
|
|||||||
* @param update Document or fields to update
|
* @param update Document or fields to update
|
||||||
* @returns True if a document was updated, false otherwise
|
* @returns True if a document was updated, false otherwise
|
||||||
*/
|
*/
|
||||||
async updateOne(filter: Partial<T>, update?: Partial<T>): Promise<boolean> {
|
async updateOne(filter: Filter<T>, update?: Partial<T>): Promise<boolean> {
|
||||||
const results = await this.updateMany(filter, update, { limit: 1 });
|
const results = await this.updateMany(filter, update, { limit: 1 });
|
||||||
return results > 0;
|
return results > 0;
|
||||||
}
|
}
|
||||||
@@ -85,7 +133,7 @@ export abstract class BaseStorage<
|
|||||||
* @returns Number of documents updated
|
* @returns Number of documents updated
|
||||||
*/
|
*/
|
||||||
abstract updateMany(
|
abstract updateMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options?: Partial<FindOptions>,
|
options?: Partial<FindOptions>,
|
||||||
): Promise<number>;
|
): Promise<number>;
|
||||||
@@ -95,7 +143,7 @@ export abstract class BaseStorage<
|
|||||||
* @param filter Query to match the document to delete
|
* @param filter Query to match the document to delete
|
||||||
* @returns True if a document was deleted, false otherwise
|
* @returns True if a document was deleted, false otherwise
|
||||||
*/
|
*/
|
||||||
async deleteOne(filter: Partial<T>): Promise<boolean> {
|
async deleteOne(filter: Filter<T>): Promise<boolean> {
|
||||||
const results = await this.deleteMany(filter, { limit: 1 });
|
const results = await this.deleteMany(filter, { limit: 1 });
|
||||||
return results > 0;
|
return results > 0;
|
||||||
}
|
}
|
||||||
@@ -107,7 +155,7 @@ export abstract class BaseStorage<
|
|||||||
* @returns Number of documents deleted
|
* @returns Number of documents deleted
|
||||||
*/
|
*/
|
||||||
abstract deleteMany(
|
abstract deleteMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
options: Partial<FindOptions>,
|
options: Partial<FindOptions>,
|
||||||
): Promise<number>;
|
): Promise<number>;
|
||||||
|
|
||||||
|
|||||||
@@ -3,15 +3,29 @@ import { Packr } from 'msgpackr';
|
|||||||
import { AESKey } from 'src/crypto/aes-key.js';
|
import { AESKey } from 'src/crypto/aes-key.js';
|
||||||
import { Bytes } from 'src/crypto/bytes.js';
|
import { Bytes } from 'src/crypto/bytes.js';
|
||||||
|
|
||||||
import { BaseStorage, type FindOptions } from './base-storage.js';
|
import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
|
||||||
|
|
||||||
import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
|
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
|
||||||
|
|
||||||
|
export type EncryptedStorageOptions = {
|
||||||
|
/**
|
||||||
|
* Fields that should be stored in plaintext (not encrypted).
|
||||||
|
* These fields retain their original types in the backing store, which
|
||||||
|
* allows comparison operators ($lt, $gt, etc.) to work on them.
|
||||||
|
* All other fields are encrypted.
|
||||||
|
*/
|
||||||
|
plaintextKeys?: string[];
|
||||||
|
};
|
||||||
|
|
||||||
export class EncryptedStorage<
|
export class EncryptedStorage<
|
||||||
T extends Record<string, any> = Record<string, any>,
|
T extends Record<string, any> = Record<string, any>,
|
||||||
> extends BaseStorage<T> {
|
> extends BaseStorage<T> {
|
||||||
static from<T>(storage: BaseStorage<Record<string, string>>, key: AESKey) {
|
static from<T>(
|
||||||
return new EncryptedStorage<T>(storage, key);
|
storage: BaseStorage<Record<string, any>>,
|
||||||
|
key: AESKey,
|
||||||
|
options?: EncryptedStorageOptions,
|
||||||
|
) {
|
||||||
|
return new EncryptedStorage<T>(storage, key, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly msgpackr = new Packr({
|
private readonly msgpackr = new Packr({
|
||||||
@@ -33,34 +47,32 @@ export class EncryptedStorage<
|
|||||||
*/
|
*/
|
||||||
private readonly decryptCache = new Map<string, any>();
|
private readonly decryptCache = new Map<string, any>();
|
||||||
|
|
||||||
|
/** Set of field names that are stored in plaintext (not encrypted). */
|
||||||
|
private readonly plaintextKeys: Set<string>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly storage: BaseStorage<Record<string, string>>,
|
private readonly storage: BaseStorage<Record<string, any>>,
|
||||||
private readonly key: AESKey,
|
private readonly key: AESKey,
|
||||||
|
options?: EncryptedStorageOptions,
|
||||||
) {
|
) {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
// Forward events from the underlying storage, decrypting the data
|
this.plaintextKeys = new Set(options?.plaintextKeys ?? []);
|
||||||
this.storage.on('insert', async (event) => {
|
|
||||||
// De-crypt the value before emitting the event.
|
|
||||||
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
|
|
||||||
|
|
||||||
// Re-emit the insert event with the original payload.
|
// Forward events from the underlying storage, converting data back.
|
||||||
|
this.storage.on('insert', async (event) => {
|
||||||
|
const decryptedValue = await this.convertFromStorage(event.value);
|
||||||
this.emit('insert', { value: decryptedValue });
|
this.emit('insert', { value: decryptedValue });
|
||||||
});
|
});
|
||||||
|
|
||||||
this.storage.on('update', async (event) => {
|
this.storage.on('update', async (event) => {
|
||||||
// Decrypt both old and new values before re-emitting.
|
const decryptedOldValue = await this.convertFromStorage(event.oldValue);
|
||||||
const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record<string, string>);
|
const decryptedValue = await this.convertFromStorage(event.value);
|
||||||
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
|
|
||||||
|
|
||||||
this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue });
|
this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue });
|
||||||
});
|
});
|
||||||
|
|
||||||
this.storage.on('delete', async (event) => {
|
this.storage.on('delete', async (event) => {
|
||||||
// De-crypt the value before emitting the event.
|
const decryptedValue = await this.convertFromStorage(event.value);
|
||||||
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
|
|
||||||
|
|
||||||
// Re-emit the delete event with the original payload.
|
|
||||||
this.emit('delete', { value: decryptedValue });
|
this.emit('delete', { value: decryptedValue });
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -70,53 +82,83 @@ export class EncryptedStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async insertMany(documents: Array<T>): Promise<void> {
|
async insertMany(documents: Array<T>): Promise<void> {
|
||||||
// Encrypt all documents in parallel.
|
const converted = await Promise.all(
|
||||||
const encrypted = await Promise.all(
|
documents.map((doc) => this.convertForStorage(doc)),
|
||||||
documents.map((doc) => this.convertToEncrypted(doc)),
|
|
||||||
);
|
);
|
||||||
await this.storage.insertMany(encrypted);
|
await this.storage.insertMany(converted);
|
||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
||||||
const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined;
|
const convertedFilter = filter
|
||||||
const documents = await this.storage.find(encryptedFilter, options);
|
? await this.convertFilterForStorage(filter)
|
||||||
|
: undefined;
|
||||||
|
const documents = await this.storage.find(convertedFilter, options);
|
||||||
return Promise.all(
|
return Promise.all(
|
||||||
documents.map(async (document) => this.convertToDecrypted(document)),
|
documents.map((doc) => this.convertFromStorage(doc)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
async updateMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const encryptedFilter = await this.convertToEncrypted(filter);
|
const convertedFilter = await this.convertFilterForStorage(filter);
|
||||||
const encryptedUpdate = await this.convertToEncrypted(update);
|
const convertedUpdate = await this.convertForStorage(update);
|
||||||
return this.storage.updateMany(encryptedFilter, encryptedUpdate, options);
|
return this.storage.updateMany(convertedFilter, convertedUpdate, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(
|
async deleteMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const encryptedFilter = await this.convertToEncrypted(filter);
|
const convertedFilter = await this.convertFilterForStorage(filter);
|
||||||
return this.storage.deleteMany(encryptedFilter, options);
|
return this.storage.deleteMany(convertedFilter, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
deriveChild<C>(path: string): BaseStorage<C> {
|
deriveChild<C>(path: string): BaseStorage<C> {
|
||||||
return EncryptedStorage.from(this.storage.deriveChild(path), this.key);
|
return EncryptedStorage.from<C>(
|
||||||
|
this.storage.deriveChild(path),
|
||||||
|
this.key,
|
||||||
|
{ plaintextKeys: [...this.plaintextKeys] },
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async convertToEncrypted(
|
// ---------------------------------------------------------------------------
|
||||||
|
// Storage conversion — documents (insert/update values)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a document for storage. Encrypted fields are encrypted;
|
||||||
|
* plaintext fields are passed through with value formatting only.
|
||||||
|
*/
|
||||||
|
private async convertForStorage(
|
||||||
document: Partial<T>,
|
document: Partial<T>,
|
||||||
): Promise<Record<string, string>> {
|
): Promise<Record<string, any>> {
|
||||||
const encrypted: Record<string, string> = {};
|
const result: Record<string, any> = {};
|
||||||
const formattedDocument = this.formatDocumentForEncryption(document);
|
const formattedDocument = this.formatDocumentForEncryption(document);
|
||||||
const entries = Object.entries(formattedDocument);
|
const entries = Object.entries(formattedDocument);
|
||||||
|
|
||||||
// Encrypt all fields in parallel, using the cache when possible.
|
// Split into plaintext and encrypted fields.
|
||||||
const results = await Promise.all(
|
const plaintextEntries: Array<[string, any]> = [];
|
||||||
entries.map(async ([key, value]) => {
|
const encryptedEntries: Array<[string, any]> = [];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (this.plaintextKeys.has(entry[0])) {
|
||||||
|
plaintextEntries.push(entry);
|
||||||
|
} else {
|
||||||
|
encryptedEntries.push(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plaintext fields pass through directly.
|
||||||
|
for (const [key, value] of plaintextEntries) {
|
||||||
|
result[key] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encrypt fields in parallel, with memoization.
|
||||||
|
const encrypted = await Promise.all(
|
||||||
|
encryptedEntries.map(async ([key, value]) => {
|
||||||
const bin = this.msgpackr.pack(value);
|
const bin = this.msgpackr.pack(value);
|
||||||
const cacheKey = Bytes.from(bin).toBase64();
|
const cacheKey = Bytes.from(bin).toBase64();
|
||||||
|
|
||||||
@@ -131,21 +173,36 @@ export class EncryptedStorage<
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
for (const [key, ciphertext] of results) {
|
for (const [key, ciphertext] of encrypted) {
|
||||||
encrypted[key] = ciphertext;
|
result[key] = ciphertext;
|
||||||
}
|
}
|
||||||
|
|
||||||
return encrypted;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async convertToDecrypted(
|
/**
|
||||||
document: Record<string, string>,
|
* Convert a stored document back to its original form. Encrypted fields
|
||||||
|
* are decrypted; plaintext fields are passed through with value formatting.
|
||||||
|
*/
|
||||||
|
private async convertFromStorage(
|
||||||
|
document: Record<string, any>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
const entries = Object.entries(document);
|
const entries = Object.entries(document);
|
||||||
|
|
||||||
// Decrypt all fields in parallel, using the cache when possible.
|
const plaintextEntries: Array<[string, any]> = [];
|
||||||
const results = await Promise.all(
|
const encryptedEntries: Array<[string, any]> = [];
|
||||||
entries.map(async ([key, ciphertext]) => {
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (this.plaintextKeys.has(entry[0])) {
|
||||||
|
plaintextEntries.push(entry);
|
||||||
|
} else {
|
||||||
|
encryptedEntries.push(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrypt encrypted fields in parallel, with memoization.
|
||||||
|
const decrypted = await Promise.all(
|
||||||
|
encryptedEntries.map(async ([key, ciphertext]) => {
|
||||||
let value = this.decryptCache.get(ciphertext);
|
let value = this.decryptCache.get(ciphertext);
|
||||||
if (value === undefined) {
|
if (value === undefined) {
|
||||||
const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext));
|
const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext));
|
||||||
@@ -156,70 +213,122 @@ export class EncryptedStorage<
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const decrypted: Record<string, any> = {};
|
const result: Record<string, any> = {};
|
||||||
for (const [key, value] of results) {
|
|
||||||
decrypted[key] = value;
|
for (const [key, value] of plaintextEntries) {
|
||||||
|
result[key] = value;
|
||||||
|
}
|
||||||
|
for (const [key, value] of decrypted) {
|
||||||
|
result[key] = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
const decodedDocument = this.formatDocumentFromDecryption(decrypted);
|
const decodedDocument = this.formatDocumentFromDecryption(result);
|
||||||
return decodedDocument as T;
|
return decodedDocument as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
private formatDocumentForEncryption(document: any): any {
|
// ---------------------------------------------------------------------------
|
||||||
// First, iterate through each key and value in the document and format the value for encryption.
|
// Storage conversion — filters (may contain operator objects)
|
||||||
const formattedDocument: any = {};
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a query filter for storage. Handles both plain equality values
|
||||||
|
* and operator objects.
|
||||||
|
*
|
||||||
|
* - Plaintext fields: values and operator objects pass through as-is.
|
||||||
|
* - Encrypted fields: plain values are encrypted. Operator objects throw
|
||||||
|
* because range comparisons are meaningless on ciphertext.
|
||||||
|
*/
|
||||||
|
private async convertFilterForStorage(
|
||||||
|
filter: Filter<T>,
|
||||||
|
): Promise<Filter<Record<string, any>>> {
|
||||||
|
const result: Record<string, any> = {};
|
||||||
|
const entries = Object.entries(filter);
|
||||||
|
|
||||||
|
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
|
||||||
|
|
||||||
|
for (const [key, value] of entries) {
|
||||||
|
if (this.plaintextKeys.has(key)) {
|
||||||
|
// Plaintext field — pass through (including operator objects).
|
||||||
|
result[key] = isOperatorObject(value)
|
||||||
|
? value
|
||||||
|
: this.formatValueForEncryption(value);
|
||||||
|
} else if (isOperatorObject(value)) {
|
||||||
|
// Encrypted field with an operator — not supported.
|
||||||
|
throw new Error(
|
||||||
|
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
|
||||||
|
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// Encrypted field with a plain equality value — encrypt it.
|
||||||
|
const formatted = this.formatValueForEncryption(value);
|
||||||
|
encryptionTasks.push(
|
||||||
|
(async () => {
|
||||||
|
const bin = this.msgpackr.pack(formatted);
|
||||||
|
const cacheKey = Bytes.from(bin).toBase64();
|
||||||
|
|
||||||
|
let ciphertext = this.encryptCache.get(cacheKey);
|
||||||
|
if (ciphertext === undefined) {
|
||||||
|
const encryptedValue = await this.key.encrypt(bin, true);
|
||||||
|
ciphertext = encryptedValue.toBase64();
|
||||||
|
this.encryptCache.set(cacheKey, ciphertext);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [key, ciphertext] as const;
|
||||||
|
})(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const encryptedResults = await Promise.all(encryptionTasks);
|
||||||
|
for (const [key, ciphertext] of encryptedResults) {
|
||||||
|
result[key] = ciphertext;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Value formatting
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
private formatDocumentForEncryption(document: any): any {
|
||||||
|
const formattedDocument: any = {};
|
||||||
for (const [key, value] of Object.entries(document)) {
|
for (const [key, value] of Object.entries(document)) {
|
||||||
formattedDocument[key] = this.formatValueForEncryption(value);
|
formattedDocument[key] = this.formatValueForEncryption(value);
|
||||||
}
|
}
|
||||||
|
return encodeExtendedJsonObject(formattedDocument);
|
||||||
// Then, encode the document to extended JSON.
|
|
||||||
const encodedDocument = encodeExtendedJsonObject(formattedDocument);
|
|
||||||
|
|
||||||
return encodedDocument;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private formatDocumentFromDecryption(document: any): any {
|
private formatDocumentFromDecryption(document: any): any {
|
||||||
// First, decode the document from extended JSON.
|
|
||||||
const decodedDocument = decodeExtendedJsonObject(document);
|
const decodedDocument = decodeExtendedJsonObject(document);
|
||||||
|
|
||||||
// Then, iterate through each key and value in the document and format the value from decryption.
|
|
||||||
for (const [key, value] of Object.entries(decodedDocument)) {
|
for (const [key, value] of Object.entries(decodedDocument)) {
|
||||||
decodedDocument[key] = this.formatValueFromDecryption(value);
|
decodedDocument[key] = this.formatValueFromDecryption(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
return decodedDocument;
|
return decodedDocument;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format a value before encryption. Converts types that msgpackr
|
||||||
|
* doesn't natively support (e.g. Date) into serialisable forms.
|
||||||
|
*/
|
||||||
private formatValueForEncryption(value: any): any {
|
private formatValueForEncryption(value: any): any {
|
||||||
// msgpackr doesnt support Date, so we need to convert it to a string.
|
|
||||||
if (value instanceof Date) {
|
if (value instanceof Date) {
|
||||||
return `<Date: ${value.getTime()}>`;
|
return `<Date: ${value.getTime()}>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restore a value after decryption. Reverses the transformations
|
||||||
|
* applied by `formatValueForEncryption`.
|
||||||
|
*/
|
||||||
private formatValueFromDecryption(value: any): any {
|
private formatValueFromDecryption(value: any): any {
|
||||||
// msgpackr doesnt support Date, so we need to convert it to a Date.
|
|
||||||
if (typeof value === 'string') {
|
if (typeof value === 'string') {
|
||||||
// Check if this value matches an Extended JSON encoded date.
|
|
||||||
// TODO: Do this without a regex for performance reasons.
|
|
||||||
// const dateMatch = value.match(/<Date: (?<time>[0-9]+)>/);
|
|
||||||
|
|
||||||
// Without regex
|
|
||||||
if (value.startsWith('<Date: ') && value.endsWith('>')) {
|
if (value.startsWith('<Date: ') && value.endsWith('>')) {
|
||||||
const time = value.slice(7, -1);
|
const time = value.slice(7, -1);
|
||||||
return new Date(parseInt(time));
|
return new Date(parseInt(time));
|
||||||
}
|
}
|
||||||
|
|
||||||
// If it does, convert it to a Date.
|
|
||||||
// if (dateMatch) {
|
|
||||||
// const { time } = dateMatch.groups!;
|
|
||||||
// return new Date(parseInt(time));
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,11 @@
|
|||||||
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
|
import {
|
||||||
|
BaseStorage,
|
||||||
|
FindOptions,
|
||||||
|
type IndexDefinition,
|
||||||
|
type Filter,
|
||||||
|
type ComparisonOperators,
|
||||||
|
isOperatorObject,
|
||||||
|
} from './base-storage.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Key prefix separator used to namespace documents within localStorage.
|
* Key prefix separator used to namespace documents within localStorage.
|
||||||
@@ -132,7 +139,7 @@ export class StorageLocalStorage<
|
|||||||
this.persistManifest();
|
this.persistManifest();
|
||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
|
|
||||||
let results: T[];
|
let results: T[];
|
||||||
@@ -175,7 +182,7 @@ export class StorageLocalStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
async updateMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
@@ -207,7 +214,7 @@ export class StorageLocalStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(
|
async deleteMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
@@ -251,12 +258,30 @@ export class StorageLocalStorage<
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks whether a document satisfies every field in the filter.
|
* Checks whether a document satisfies every field in the filter.
|
||||||
|
* Supports both plain equality values and comparison operator objects.
|
||||||
*/
|
*/
|
||||||
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) return true;
|
if (!filter || Object.keys(filter).length === 0) return true;
|
||||||
for (const [key, value] of Object.entries(filter)) {
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
|
if (isOperatorObject(value)) {
|
||||||
|
if (!this.matchesOperators(item[key], value)) return false;
|
||||||
|
} else {
|
||||||
if (item[key] !== value) return false;
|
if (item[key] !== value) return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Evaluate a set of comparison operators against a single field value.
|
||||||
|
*/
|
||||||
|
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
||||||
|
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
|
||||||
|
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
|
||||||
|
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
|
||||||
|
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
||||||
|
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
||||||
|
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,7 +289,7 @@ export class StorageLocalStorage<
|
|||||||
* Collect all [internalKey, document] pairs that match a filter.
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
* Uses an index when possible, otherwise falls back to a full scan.
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
*/
|
*/
|
||||||
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
|
private collectMatches(filter?: Filter<T>): Array<[number, T]> {
|
||||||
const indexKeys = this.resolveIndexKeys(filter);
|
const indexKeys = this.resolveIndexKeys(filter);
|
||||||
const keysToScan = indexKeys ?? this.manifest;
|
const keysToScan = indexKeys ?? this.manifest;
|
||||||
const results: Array<[number, T]> = [];
|
const results: Array<[number, T]> = [];
|
||||||
@@ -348,17 +373,29 @@ export class StorageLocalStorage<
|
|||||||
/**
|
/**
|
||||||
* Attempt to resolve candidate internal keys from the indexes.
|
* Attempt to resolve candidate internal keys from the indexes.
|
||||||
* Returns `null` if no index can serve the query.
|
* Returns `null` if no index can serve the query.
|
||||||
|
*
|
||||||
|
* Only plain equality values are used for index resolution — operator
|
||||||
|
* objects are excluded since hash-based indexes only support equality.
|
||||||
*/
|
*/
|
||||||
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
|
private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
|
||||||
if (!filter) return null;
|
if (!filter) return null;
|
||||||
const filterKeys = Object.keys(filter);
|
const filterKeys = Object.keys(filter);
|
||||||
if (filterKeys.length === 0) return null;
|
if (filterKeys.length === 0) return null;
|
||||||
|
|
||||||
|
const equalityFilter: Record<string, any> = {};
|
||||||
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
|
if (!isOperatorObject(value)) {
|
||||||
|
equalityFilter[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Object.keys(equalityFilter).length === 0) return null;
|
||||||
|
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
if (!fields.every((f) => f in filter)) continue;
|
if (!fields.every((f) => f in equalityFilter)) continue;
|
||||||
|
|
||||||
const indexName = fields.join(INDEX_KEY_SEP);
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
const indexValue = this.buildIndexValue(filter, fields);
|
const indexValue = this.buildIndexValue(equalityFilter, fields);
|
||||||
if (indexValue === null) continue;
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
const indexMap = this.indexes.get(indexName)!;
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { BaseStorage, FindOptions } from './base-storage.js';
|
import { BaseStorage, FindOptions, type Filter } from './base-storage.js';
|
||||||
import { StorageMemory } from './storage-memory.js';
|
import { StorageMemory } from './storage-memory.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -65,19 +65,19 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
await this.store.insertMany(documents);
|
await this.store.insertMany(documents);
|
||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
||||||
return await this.inMemoryCache.find(filter, options);
|
return await this.inMemoryCache.find(filter, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
async updateMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options: FindOptions = {} as FindOptions
|
options: FindOptions = {} as FindOptions
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
return await this.store.updateMany(filter, update, options);
|
return await this.store.updateMany(filter, update, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(filter: Partial<T>, options: FindOptions = {} as FindOptions): Promise<number> {
|
async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
|
||||||
return await this.store.deleteMany(filter, options);
|
return await this.store.deleteMany(filter, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,11 @@
|
|||||||
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
|
import {
|
||||||
|
BaseStorage,
|
||||||
|
FindOptions,
|
||||||
|
type IndexDefinition,
|
||||||
|
type Filter,
|
||||||
|
type ComparisonOperators,
|
||||||
|
isOperatorObject,
|
||||||
|
} from './base-storage.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Separator used when joining multiple field values into a single index key.
|
* Separator used when joining multiple field values into a single index key.
|
||||||
@@ -86,7 +93,7 @@ export class StorageMemory<
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
||||||
let results: T[];
|
let results: T[];
|
||||||
|
|
||||||
// Attempt to satisfy the query via an index.
|
// Attempt to satisfy the query via an index.
|
||||||
@@ -118,7 +125,7 @@ export class StorageMemory<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
async updateMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
@@ -147,7 +154,7 @@ export class StorageMemory<
|
|||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(
|
async deleteMany(
|
||||||
filter: Partial<T>,
|
filter: Filter<T>,
|
||||||
options: Partial<FindOptions> = {},
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const rowsToDelete = this.collectMatches(filter);
|
const rowsToDelete = this.collectMatches(filter);
|
||||||
@@ -182,26 +189,42 @@ export class StorageMemory<
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks whether a document satisfies every field in the filter.
|
* Checks whether a document satisfies every field in the filter.
|
||||||
* An empty or undefined filter matches everything.
|
* Supports both plain equality values and comparison operator objects.
|
||||||
*/
|
*/
|
||||||
private matchesFilter(item: T, filter?: Partial<T>): boolean {
|
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) {
|
if (!filter || Object.keys(filter).length === 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(filter)) {
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
if (item[key] !== value) {
|
if (isOperatorObject(value)) {
|
||||||
return false;
|
if (!this.matchesOperators(item[key], value)) return false;
|
||||||
|
} else {
|
||||||
|
if (item[key] !== value) return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Evaluate a set of comparison operators against a single field value.
|
||||||
|
* All operators must pass for the field to match.
|
||||||
|
*/
|
||||||
|
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
||||||
|
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
|
||||||
|
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
|
||||||
|
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
|
||||||
|
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
||||||
|
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
||||||
|
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collect all [internalKey, document] pairs that match a filter.
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
* Uses an index when possible, otherwise falls back to a full scan.
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
*/
|
*/
|
||||||
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
|
private collectMatches(filter?: Filter<T>): Array<[number, T]> {
|
||||||
const indexKeys = this.resolveIndexKeys(filter);
|
const indexKeys = this.resolveIndexKeys(filter);
|
||||||
|
|
||||||
if (indexKeys !== null) {
|
if (indexKeys !== null) {
|
||||||
@@ -296,21 +319,31 @@ export class StorageMemory<
|
|||||||
* Attempt to resolve a set of candidate internal keys from the indexes.
|
* Attempt to resolve a set of candidate internal keys from the indexes.
|
||||||
* Returns `null` if no index can serve the query.
|
* Returns `null` if no index can serve the query.
|
||||||
*
|
*
|
||||||
* An index is used when the filter fields are a superset of (or equal to)
|
* An index is used when the filter contains plain equality values for every
|
||||||
* an index's fields — meaning the index value can be fully constructed
|
* field in the index. Operator objects (e.g. `{ $lt: 50 }`) are excluded
|
||||||
* from the filter.
|
* from index resolution since hash-based indexes only support equality.
|
||||||
*/
|
*/
|
||||||
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
|
private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
|
||||||
if (!filter) return null;
|
if (!filter) return null;
|
||||||
const filterKeys = Object.keys(filter);
|
const filterKeys = Object.keys(filter);
|
||||||
if (filterKeys.length === 0) return null;
|
if (filterKeys.length === 0) return null;
|
||||||
|
|
||||||
|
// Extract only the equality fields from the filter (skip operator objects).
|
||||||
|
const equalityFilter: Record<string, any> = {};
|
||||||
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
|
if (!isOperatorObject(value)) {
|
||||||
|
equalityFilter[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Object.keys(equalityFilter).length === 0) return null;
|
||||||
|
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
// Every field in the index must be present in the filter.
|
// Every field in the index must be present as an equality value.
|
||||||
if (!fields.every((f) => f in filter)) continue;
|
if (!fields.every((f) => f in equalityFilter)) continue;
|
||||||
|
|
||||||
const indexName = fields.join(INDEX_KEY_SEP);
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
const indexValue = this.buildIndexValue(filter, fields);
|
const indexValue = this.buildIndexValue(equalityFilter, fields);
|
||||||
if (indexValue === null) continue;
|
if (indexValue === null) continue;
|
||||||
|
|
||||||
const indexMap = this.indexes.get(indexName)!;
|
const indexMap = this.indexes.get(indexName)!;
|
||||||
@@ -326,7 +359,7 @@ export class StorageMemory<
|
|||||||
* Returns `null` when no index can serve the filter, signalling
|
* Returns `null` when no index can serve the filter, signalling
|
||||||
* the caller to fall back to a full scan.
|
* the caller to fall back to a full scan.
|
||||||
*/
|
*/
|
||||||
private findViaIndex(filter?: Partial<T>): T[] | null {
|
private findViaIndex(filter?: Filter<T>): T[] | null {
|
||||||
const keys = this.resolveIndexKeys(filter);
|
const keys = this.resolveIndexKeys(filter);
|
||||||
if (keys === null) return null;
|
if (keys === null) return null;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user