Plaintext values on encryption

This commit is contained in:
2026-02-25 15:35:04 +11:00
parent f80aa2dcfc
commit 77593fe3b4
8 changed files with 407 additions and 143 deletions

View File

@@ -7,6 +7,7 @@ const alicePublicKey = await alice.getPublicKey();
const bobPublicKey = await bob.getPublicKey(); const bobPublicKey = await bob.getPublicKey();
const keyStart = performance.now(); const keyStart = performance.now();
const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey); const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey);
const bobSharedSecret = await bob.getSharedSecret(alicePublicKey); const bobSharedSecret = await bob.getSharedSecret(alicePublicKey);

View File

@@ -49,7 +49,7 @@ function fmtOps(ops: number): string {
/** /**
* Run a full suite of benchmarks against a given storage instance. * Run a full suite of benchmarks against a given storage instance.
*/ */
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[]) { async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[], supportsRangeOps = true) {
const count = docs.length; const count = docs.length;
console.log(`\n${'='.repeat(60)}`); console.log(`\n${'='.repeat(60)}`);
console.log(` ${label} (${count.toLocaleString()} documents)`); console.log(` ${label} (${count.toLocaleString()} documents)`);
@@ -59,56 +59,75 @@ async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs:
const insertMs = await time(async () => { const insertMs = await time(async () => {
await storage.insertMany(docs); await storage.insertMany(docs);
}); });
console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`); console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`);
// --- Find all (no filter) --- // --- Find all (no filter) ---
const findAllMs = await time(async () => { const findAllMs = await time(async () => {
await storage.find(); await storage.find();
}); });
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`); console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
// --- Find by indexed field (single-key lookup, repeated) --- // --- Find by indexed field (equality) ---
const lookupCount = Math.min(count, 1_000); const lookupCount = Math.min(count, 1_000);
const findIndexedMs = await time(async () => { const findIndexedMs = await time(async () => {
for (let i = 0; i < lookupCount; i++) { for (let i = 0; i < lookupCount; i++) {
await storage.findOne({ id: `id-${i}` } as Partial<Doc>); await storage.findOne({ id: `id-${i}` });
} }
}); });
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`); console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
// --- Find by non-indexed field (full scan, repeated) --- // --- Find by non-indexed field (full scan) ---
const scanCount = Math.min(count, 1_000); const scanCount = Math.min(count, 1_000);
const findScanMs = await time(async () => { const findScanMs = await time(async () => {
for (let i = 0; i < scanCount; i++) { for (let i = 0; i < scanCount; i++) {
await storage.findOne({ email: `user-${i}@test.com` } as Partial<Doc>); await storage.findOne({ email: `user-${i}@test.com` });
} }
}); });
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`); console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
// --- Find with $gte / $lt range (full scan) ---
if (supportsRangeOps) {
const rangeCount = Math.min(count, 100);
let rangeTotal = 0;
const findRangeMs = await time(async () => {
for (let i = 0; i < rangeCount; i++) {
const results = await storage.find({ age: { $gte: 30, $lt: 40 } });
rangeTotal += results.length;
}
});
console.log(` find $gte/$lt ${findRangeMs.toFixed(2)}ms (${fmtOps((rangeCount / findRangeMs) * 1000)} ops/sec) [${rangeCount} queries, ~${Math.round(rangeTotal / rangeCount)} hits/query]`);
// --- Find with combined equality + operator (index narrows, operator verifies) ---
const comboCount = Math.min(count, 1_000);
const findComboMs = await time(async () => {
for (let i = 0; i < comboCount; i++) {
await storage.find({ id: `id-${i}`, age: { $gte: 20 } });
}
});
console.log(` find idx+operator ${findComboMs.toFixed(2)}ms (${fmtOps((comboCount / findComboMs) * 1000)} ops/sec) [${comboCount} queries]`);
}
// --- Update by indexed field --- // --- Update by indexed field ---
const updateCount = Math.min(count, 1_000); const updateCount = Math.min(count, 1_000);
const updateMs = await time(async () => { const updateMs = await time(async () => {
for (let i = 0; i < updateCount; i++) { for (let i = 0; i < updateCount; i++) {
await storage.updateOne( await storage.updateOne({ id: `id-${i}` }, { name: `updated-${i}` });
{ id: `id-${i}` } as Partial<Doc>,
{ age: 99 } as Partial<Doc>,
);
} }
}); });
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`); console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
// --- Delete by indexed field --- // --- Delete by indexed field ---
const deleteCount = Math.min(count, 1_000); const deleteCount = Math.min(count, 1_000);
const deleteMs = await time(async () => { const deleteMs = await time(async () => {
for (let i = 0; i < deleteCount; i++) { for (let i = 0; i < deleteCount; i++) {
await storage.deleteOne({ id: `id-${i}` } as Partial<Doc>); await storage.deleteOne({ id: `id-${i}` });
} }
}); });
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`); console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
// --- Verify remaining count --- // --- Verify remaining count ---
const remaining = await storage.find(); const remaining = await storage.find();
console.log(` remaining docs: ${remaining.length.toLocaleString()}`); console.log(` remaining docs: ${remaining.length.toLocaleString()}`);
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -132,7 +151,7 @@ for (const count of DOC_COUNTS) {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// EncryptedStorage — crypto overhead dominates, so use smaller counts // EncryptedStorage — with plaintextKeys for range queries
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000]; const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000];
@@ -141,15 +160,29 @@ const encryptionKey = await AESKey.fromSeed('benchmark-key');
for (const count of ENCRYPTED_DOC_COUNTS) { for (const count of ENCRYPTED_DOC_COUNTS) {
const docs = generateDocs(count); const docs = generateDocs(count);
// Encrypted + indexed backing store. // No indexes + plaintextKeys — range queries on age work.
const encBase = StorageMemory.from<Record<string, string>>(['id', 'name']); const encBaseNoIdx = StorageMemory.from<Record<string, any>>();
const encrypted = EncryptedStorage.from<Doc>(encBase, encryptionKey); const encNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey, {
await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs); plaintextKeys: ['age'],
});
await benchmarkStorage('Encrypted (no indexes, plaintextKeys: age)', encNoIdx, docs);
// Encrypted + no-index backing store. // Indexed + plaintextKeys — range queries on age work.
const encBaseNoIdx = StorageMemory.from<Record<string, string>>(); const encBaseA = StorageMemory.from<Record<string, any>>(['id', 'name']);
const encryptedNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey); const encA = EncryptedStorage.from<Doc>(encBaseA, encryptionKey, {
await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs); plaintextKeys: ['age'],
});
await benchmarkStorage('Encrypted (indexed, plaintextKeys: age)', encA, docs);
// Indexed, fully encrypted — same indexes but no plaintext keys.
const encBaseB = StorageMemory.from<Record<string, any>>(['id', 'name']);
const encB = EncryptedStorage.from<Doc>(encBaseB, encryptionKey);
await benchmarkStorage('Encrypted (indexed, fully encrypted)', encB, docs, false);
// No indexes, fully encrypted — worst case.
const encBaseC = StorageMemory.from<Record<string, any>>();
const encC = EncryptedStorage.from<Doc>(encBaseC, encryptionKey);
await benchmarkStorage('Encrypted (no indexes, fully encrypted)', encC, docs, false);
} }
console.log('\nDone.\n'); console.log('\nDone.\n');

View File

@@ -8,6 +8,9 @@
"format": "prettier --write .", "format": "prettier --write .",
"test": "echo \"Error: no test specified\" && exit 1", "test": "echo \"Error: no test specified\" && exit 1",
"benchmark:sha256": "tsx benchmarks/sha256.ts",
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",
"benchmark:encryption": "tsx benchmarks/sekp256k1.ts",
"benchmark:storage": "tsx benchmarks/storage.ts" "benchmark:storage": "tsx benchmarks/storage.ts"
}, },
"keywords": [], "keywords": [],

View File

@@ -19,6 +19,54 @@ export type FindOptions = {
*/ */
export type IndexDefinition = string[] | string[][]; export type IndexDefinition = string[] | string[][];
/**
* MongoDB-style comparison operators for a single field value.
*/
export type ComparisonOperators<V> = {
$eq?: V;
$ne?: V;
$lt?: V;
$lte?: V;
$gt?: V;
$gte?: V;
};
/**
* A filter value for a single field — either a plain value (equality shorthand)
* or an object of comparison operators.
*/
export type FieldFilter<V> = V | ComparisonOperators<V>;
/**
* Query filter that supports both equality shorthand and comparison operators.
*
* @example
* // Equality shorthand
* { name: 'foo' }
*
* // Comparison operators
* { age: { $gte: 18, $lt: 65 } }
*
* // Combined
* { name: 'foo', age: { $gte: 18 } }
*/
export type Filter<T extends Record<string, any>> = {
[K in keyof T]?: FieldFilter<T[K]>;
};
/**
* Detect whether a filter value is an operator object (e.g. `{ $lt: 50 }`)
* rather than a plain value. Guards against Date and Array which are objects
* but represent document values, not operators.
*/
export function isOperatorObject(value: unknown): value is ComparisonOperators<any> {
return value !== null
&& typeof value === 'object'
&& !Array.isArray(value)
&& !(value instanceof Date)
&& Object.keys(value).some((k) => k.startsWith('$'));
}
export type StorageEvent<T = Record<string, any>> = { export type StorageEvent<T = Record<string, any>> = {
insert: { insert: {
value: T; value: T;
@@ -54,7 +102,7 @@ export abstract class BaseStorage<
* Find a single document that matches the filter * Find a single document that matches the filter
* @param filter MongoDB-like query filter * @param filter MongoDB-like query filter
*/ */
async findOne(filter?: Partial<T>): Promise<T | null> { async findOne(filter?: Filter<T>): Promise<T | null> {
const results = await this.find(filter); const results = await this.find(filter);
return results.length > 0 ? results[0] : null; return results.length > 0 ? results[0] : null;
} }
@@ -64,7 +112,7 @@ export abstract class BaseStorage<
* @param filter MongoDB-like query filter * @param filter MongoDB-like query filter
* @param options Query options (limit, skip, sort) * @param options Query options (limit, skip, sort)
*/ */
abstract find(filter?: Partial<T>, options?: FindOptions): Promise<T[]>; abstract find(filter?: Filter<T>, options?: FindOptions): Promise<T[]>;
/** /**
* Update a document that matches the filter * Update a document that matches the filter
@@ -72,7 +120,7 @@ export abstract class BaseStorage<
* @param update Document or fields to update * @param update Document or fields to update
* @returns True if a document was updated, false otherwise * @returns True if a document was updated, false otherwise
*/ */
async updateOne(filter: Partial<T>, update?: Partial<T>): Promise<boolean> { async updateOne(filter: Filter<T>, update?: Partial<T>): Promise<boolean> {
const results = await this.updateMany(filter, update, { limit: 1 }); const results = await this.updateMany(filter, update, { limit: 1 });
return results > 0; return results > 0;
} }
@@ -85,7 +133,7 @@ export abstract class BaseStorage<
* @returns Number of documents updated * @returns Number of documents updated
*/ */
abstract updateMany( abstract updateMany(
filter: Partial<T>, filter: Filter<T>,
update: Partial<T>, update: Partial<T>,
options?: Partial<FindOptions>, options?: Partial<FindOptions>,
): Promise<number>; ): Promise<number>;
@@ -95,7 +143,7 @@ export abstract class BaseStorage<
* @param filter Query to match the document to delete * @param filter Query to match the document to delete
* @returns True if a document was deleted, false otherwise * @returns True if a document was deleted, false otherwise
*/ */
async deleteOne(filter: Partial<T>): Promise<boolean> { async deleteOne(filter: Filter<T>): Promise<boolean> {
const results = await this.deleteMany(filter, { limit: 1 }); const results = await this.deleteMany(filter, { limit: 1 });
return results > 0; return results > 0;
} }
@@ -107,7 +155,7 @@ export abstract class BaseStorage<
* @returns Number of documents deleted * @returns Number of documents deleted
*/ */
abstract deleteMany( abstract deleteMany(
filter: Partial<T>, filter: Filter<T>,
options: Partial<FindOptions>, options: Partial<FindOptions>,
): Promise<number>; ): Promise<number>;

View File

@@ -3,15 +3,29 @@ import { Packr } from 'msgpackr';
import { AESKey } from 'src/crypto/aes-key.js'; import { AESKey } from 'src/crypto/aes-key.js';
import { Bytes } from 'src/crypto/bytes.js'; import { Bytes } from 'src/crypto/bytes.js';
import { BaseStorage, type FindOptions } from './base-storage.js'; import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
export type EncryptedStorageOptions = {
/**
* Fields that should be stored in plaintext (not encrypted).
* These fields retain their original types in the backing store, which
* allows comparison operators ($lt, $gt, etc.) to work on them.
* All other fields are encrypted.
*/
plaintextKeys?: string[];
};
export class EncryptedStorage< export class EncryptedStorage<
T extends Record<string, any> = Record<string, any>, T extends Record<string, any> = Record<string, any>,
> extends BaseStorage<T> { > extends BaseStorage<T> {
static from<T>(storage: BaseStorage<Record<string, string>>, key: AESKey) { static from<T>(
return new EncryptedStorage<T>(storage, key); storage: BaseStorage<Record<string, any>>,
key: AESKey,
options?: EncryptedStorageOptions,
) {
return new EncryptedStorage<T>(storage, key, options);
} }
private readonly msgpackr = new Packr({ private readonly msgpackr = new Packr({
@@ -33,34 +47,32 @@ export class EncryptedStorage<
*/ */
private readonly decryptCache = new Map<string, any>(); private readonly decryptCache = new Map<string, any>();
/** Set of field names that are stored in plaintext (not encrypted). */
private readonly plaintextKeys: Set<string>;
constructor( constructor(
private readonly storage: BaseStorage<Record<string, string>>, private readonly storage: BaseStorage<Record<string, any>>,
private readonly key: AESKey, private readonly key: AESKey,
options?: EncryptedStorageOptions,
) { ) {
super(); super();
// Forward events from the underlying storage, decrypting the data this.plaintextKeys = new Set(options?.plaintextKeys ?? []);
this.storage.on('insert', async (event) => {
// De-crypt the value before emitting the event.
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
// Re-emit the insert event with the original payload. // Forward events from the underlying storage, converting data back.
this.storage.on('insert', async (event) => {
const decryptedValue = await this.convertFromStorage(event.value);
this.emit('insert', { value: decryptedValue }); this.emit('insert', { value: decryptedValue });
}); });
this.storage.on('update', async (event) => { this.storage.on('update', async (event) => {
// Decrypt both old and new values before re-emitting. const decryptedOldValue = await this.convertFromStorage(event.oldValue);
const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record<string, string>); const decryptedValue = await this.convertFromStorage(event.value);
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue }); this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue });
}); });
this.storage.on('delete', async (event) => { this.storage.on('delete', async (event) => {
// De-crypt the value before emitting the event. const decryptedValue = await this.convertFromStorage(event.value);
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
// Re-emit the delete event with the original payload.
this.emit('delete', { value: decryptedValue }); this.emit('delete', { value: decryptedValue });
}); });
@@ -70,53 +82,83 @@ export class EncryptedStorage<
} }
async insertMany(documents: Array<T>): Promise<void> { async insertMany(documents: Array<T>): Promise<void> {
// Encrypt all documents in parallel. const converted = await Promise.all(
const encrypted = await Promise.all( documents.map((doc) => this.convertForStorage(doc)),
documents.map((doc) => this.convertToEncrypted(doc)),
); );
await this.storage.insertMany(encrypted); await this.storage.insertMany(converted);
} }
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> { async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined; const convertedFilter = filter
const documents = await this.storage.find(encryptedFilter, options); ? await this.convertFilterForStorage(filter)
: undefined;
const documents = await this.storage.find(convertedFilter, options);
return Promise.all( return Promise.all(
documents.map(async (document) => this.convertToDecrypted(document)), documents.map((doc) => this.convertFromStorage(doc)),
); );
} }
async updateMany( async updateMany(
filter: Partial<T>, filter: Filter<T>,
update: Partial<T>, update: Partial<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
const encryptedFilter = await this.convertToEncrypted(filter); const convertedFilter = await this.convertFilterForStorage(filter);
const encryptedUpdate = await this.convertToEncrypted(update); const convertedUpdate = await this.convertForStorage(update);
return this.storage.updateMany(encryptedFilter, encryptedUpdate, options); return this.storage.updateMany(convertedFilter, convertedUpdate, options);
} }
async deleteMany( async deleteMany(
filter: Partial<T>, filter: Filter<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
const encryptedFilter = await this.convertToEncrypted(filter); const convertedFilter = await this.convertFilterForStorage(filter);
return this.storage.deleteMany(encryptedFilter, options); return this.storage.deleteMany(convertedFilter, options);
} }
deriveChild<C>(path: string): BaseStorage<C> { deriveChild<C>(path: string): BaseStorage<C> {
return EncryptedStorage.from(this.storage.deriveChild(path), this.key); return EncryptedStorage.from<C>(
this.storage.deriveChild(path),
this.key,
{ plaintextKeys: [...this.plaintextKeys] },
);
} }
private async convertToEncrypted( // ---------------------------------------------------------------------------
// Storage conversion — documents (insert/update values)
// ---------------------------------------------------------------------------
/**
* Convert a document for storage. Encrypted fields are encrypted;
* plaintext fields are passed through with value formatting only.
*/
private async convertForStorage(
document: Partial<T>, document: Partial<T>,
): Promise<Record<string, string>> { ): Promise<Record<string, any>> {
const encrypted: Record<string, string> = {}; const result: Record<string, any> = {};
const formattedDocument = this.formatDocumentForEncryption(document); const formattedDocument = this.formatDocumentForEncryption(document);
const entries = Object.entries(formattedDocument); const entries = Object.entries(formattedDocument);
// Encrypt all fields in parallel, using the cache when possible. // Split into plaintext and encrypted fields.
const results = await Promise.all( const plaintextEntries: Array<[string, any]> = [];
entries.map(async ([key, value]) => { const encryptedEntries: Array<[string, any]> = [];
for (const entry of entries) {
if (this.plaintextKeys.has(entry[0])) {
plaintextEntries.push(entry);
} else {
encryptedEntries.push(entry);
}
}
// Plaintext fields pass through directly.
for (const [key, value] of plaintextEntries) {
result[key] = value;
}
// Encrypt fields in parallel, with memoization.
const encrypted = await Promise.all(
encryptedEntries.map(async ([key, value]) => {
const bin = this.msgpackr.pack(value); const bin = this.msgpackr.pack(value);
const cacheKey = Bytes.from(bin).toBase64(); const cacheKey = Bytes.from(bin).toBase64();
@@ -131,21 +173,36 @@ export class EncryptedStorage<
}), }),
); );
for (const [key, ciphertext] of results) { for (const [key, ciphertext] of encrypted) {
encrypted[key] = ciphertext; result[key] = ciphertext;
} }
return encrypted; return result;
} }
private async convertToDecrypted( /**
document: Record<string, string>, * Convert a stored document back to its original form. Encrypted fields
* are decrypted; plaintext fields are passed through with value formatting.
*/
private async convertFromStorage(
document: Record<string, any>,
): Promise<T> { ): Promise<T> {
const entries = Object.entries(document); const entries = Object.entries(document);
// Decrypt all fields in parallel, using the cache when possible. const plaintextEntries: Array<[string, any]> = [];
const results = await Promise.all( const encryptedEntries: Array<[string, any]> = [];
entries.map(async ([key, ciphertext]) => {
for (const entry of entries) {
if (this.plaintextKeys.has(entry[0])) {
plaintextEntries.push(entry);
} else {
encryptedEntries.push(entry);
}
}
// Decrypt encrypted fields in parallel, with memoization.
const decrypted = await Promise.all(
encryptedEntries.map(async ([key, ciphertext]) => {
let value = this.decryptCache.get(ciphertext); let value = this.decryptCache.get(ciphertext);
if (value === undefined) { if (value === undefined) {
const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext)); const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext));
@@ -156,70 +213,122 @@ export class EncryptedStorage<
}), }),
); );
const decrypted: Record<string, any> = {}; const result: Record<string, any> = {};
for (const [key, value] of results) {
decrypted[key] = value; for (const [key, value] of plaintextEntries) {
result[key] = value;
}
for (const [key, value] of decrypted) {
result[key] = value;
} }
const decodedDocument = this.formatDocumentFromDecryption(decrypted); const decodedDocument = this.formatDocumentFromDecryption(result);
return decodedDocument as T; return decodedDocument as T;
} }
private formatDocumentForEncryption(document: any): any { // ---------------------------------------------------------------------------
// First, iterate through each key and value in the document and format the value for encryption. // Storage conversion — filters (may contain operator objects)
const formattedDocument: any = {}; // ---------------------------------------------------------------------------
/**
* Convert a query filter for storage. Handles both plain equality values
* and operator objects.
*
* - Plaintext fields: values and operator objects pass through as-is.
* - Encrypted fields: plain values are encrypted. Operator objects throw
* because range comparisons are meaningless on ciphertext.
*/
private async convertFilterForStorage(
filter: Filter<T>,
): Promise<Filter<Record<string, any>>> {
const result: Record<string, any> = {};
const entries = Object.entries(filter);
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
for (const [key, value] of entries) {
if (this.plaintextKeys.has(key)) {
// Plaintext field — pass through (including operator objects).
result[key] = isOperatorObject(value)
? value
: this.formatValueForEncryption(value);
} else if (isOperatorObject(value)) {
// Encrypted field with an operator — not supported.
throw new Error(
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
);
} else {
// Encrypted field with a plain equality value — encrypt it.
const formatted = this.formatValueForEncryption(value);
encryptionTasks.push(
(async () => {
const bin = this.msgpackr.pack(formatted);
const cacheKey = Bytes.from(bin).toBase64();
let ciphertext = this.encryptCache.get(cacheKey);
if (ciphertext === undefined) {
const encryptedValue = await this.key.encrypt(bin, true);
ciphertext = encryptedValue.toBase64();
this.encryptCache.set(cacheKey, ciphertext);
}
return [key, ciphertext] as const;
})(),
);
}
}
const encryptedResults = await Promise.all(encryptionTasks);
for (const [key, ciphertext] of encryptedResults) {
result[key] = ciphertext;
}
return result;
}
// ---------------------------------------------------------------------------
// Value formatting
// ---------------------------------------------------------------------------
private formatDocumentForEncryption(document: any): any {
const formattedDocument: any = {};
for (const [key, value] of Object.entries(document)) { for (const [key, value] of Object.entries(document)) {
formattedDocument[key] = this.formatValueForEncryption(value); formattedDocument[key] = this.formatValueForEncryption(value);
} }
return encodeExtendedJsonObject(formattedDocument);
// Then, encode the document to extended JSON.
const encodedDocument = encodeExtendedJsonObject(formattedDocument);
return encodedDocument;
} }
private formatDocumentFromDecryption(document: any): any { private formatDocumentFromDecryption(document: any): any {
// First, decode the document from extended JSON.
const decodedDocument = decodeExtendedJsonObject(document); const decodedDocument = decodeExtendedJsonObject(document);
// Then, iterate through each key and value in the document and format the value from decryption.
for (const [key, value] of Object.entries(decodedDocument)) { for (const [key, value] of Object.entries(decodedDocument)) {
decodedDocument[key] = this.formatValueFromDecryption(value); decodedDocument[key] = this.formatValueFromDecryption(value);
} }
return decodedDocument; return decodedDocument;
} }
/**
* Format a value before encryption. Converts types that msgpackr
* doesn't natively support (e.g. Date) into serialisable forms.
*/
private formatValueForEncryption(value: any): any { private formatValueForEncryption(value: any): any {
// msgpackr doesnt support Date, so we need to convert it to a string.
if (value instanceof Date) { if (value instanceof Date) {
return `<Date: ${value.getTime()}>`; return `<Date: ${value.getTime()}>`;
} }
return value; return value;
} }
/**
* Restore a value after decryption. Reverses the transformations
* applied by `formatValueForEncryption`.
*/
private formatValueFromDecryption(value: any): any { private formatValueFromDecryption(value: any): any {
// msgpackr doesnt support Date, so we need to convert it to a Date.
if (typeof value === 'string') { if (typeof value === 'string') {
// Check if this value matches an Extended JSON encoded date.
// TODO: Do this without a regex for performance reasons.
// const dateMatch = value.match(/<Date: (?<time>[0-9]+)>/);
// Without regex
if (value.startsWith('<Date: ') && value.endsWith('>')) { if (value.startsWith('<Date: ') && value.endsWith('>')) {
const time = value.slice(7, -1); const time = value.slice(7, -1);
return new Date(parseInt(time)); return new Date(parseInt(time));
} }
// If it does, convert it to a Date.
// if (dateMatch) {
// const { time } = dateMatch.groups!;
// return new Date(parseInt(time));
// }
} }
return value; return value;
} }
} }

View File

@@ -1,4 +1,11 @@
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; import {
BaseStorage,
FindOptions,
type IndexDefinition,
type Filter,
type ComparisonOperators,
isOperatorObject,
} from './base-storage.js';
/** /**
* Key prefix separator used to namespace documents within localStorage. * Key prefix separator used to namespace documents within localStorage.
@@ -132,7 +139,7 @@ export class StorageLocalStorage<
this.persistManifest(); this.persistManifest();
} }
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> { async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
this.refreshManifest(); this.refreshManifest();
let results: T[]; let results: T[];
@@ -175,7 +182,7 @@ export class StorageLocalStorage<
} }
async updateMany( async updateMany(
filter: Partial<T>, filter: Filter<T>,
update: Partial<T>, update: Partial<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
@@ -207,7 +214,7 @@ export class StorageLocalStorage<
} }
async deleteMany( async deleteMany(
filter: Partial<T>, filter: Filter<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
this.refreshManifest(); this.refreshManifest();
@@ -251,20 +258,38 @@ export class StorageLocalStorage<
/** /**
* Checks whether a document satisfies every field in the filter. * Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
*/ */
private matchesFilter(item: T, filter?: Partial<T>): boolean { private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) return true; if (!filter || Object.keys(filter).length === 0) return true;
for (const [key, value] of Object.entries(filter)) { for (const [key, value] of Object.entries(filter)) {
if (item[key] !== value) return false; if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
if (item[key] !== value) return false;
}
} }
return true; return true;
} }
/**
* Evaluate a set of comparison operators against a single field value.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
return true;
}
/** /**
* Collect all [internalKey, document] pairs that match a filter. * Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan. * Uses an index when possible, otherwise falls back to a full scan.
*/ */
private collectMatches(filter?: Partial<T>): Array<[number, T]> { private collectMatches(filter?: Filter<T>): Array<[number, T]> {
const indexKeys = this.resolveIndexKeys(filter); const indexKeys = this.resolveIndexKeys(filter);
const keysToScan = indexKeys ?? this.manifest; const keysToScan = indexKeys ?? this.manifest;
const results: Array<[number, T]> = []; const results: Array<[number, T]> = [];
@@ -348,17 +373,29 @@ export class StorageLocalStorage<
/** /**
* Attempt to resolve candidate internal keys from the indexes. * Attempt to resolve candidate internal keys from the indexes.
* Returns `null` if no index can serve the query. * Returns `null` if no index can serve the query.
*
* Only plain equality values are used for index resolution — operator
* objects are excluded since hash-based indexes only support equality.
*/ */
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null { private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
if (!filter) return null; if (!filter) return null;
const filterKeys = Object.keys(filter); const filterKeys = Object.keys(filter);
if (filterKeys.length === 0) return null; if (filterKeys.length === 0) return null;
const equalityFilter: Record<string, any> = {};
for (const [key, value] of Object.entries(filter)) {
if (!isOperatorObject(value)) {
equalityFilter[key] = value;
}
}
if (Object.keys(equalityFilter).length === 0) return null;
for (const fields of this.indexDefs) { for (const fields of this.indexDefs) {
if (!fields.every((f) => f in filter)) continue; if (!fields.every((f) => f in equalityFilter)) continue;
const indexName = fields.join(INDEX_KEY_SEP); const indexName = fields.join(INDEX_KEY_SEP);
const indexValue = this.buildIndexValue(filter, fields); const indexValue = this.buildIndexValue(equalityFilter, fields);
if (indexValue === null) continue; if (indexValue === null) continue;
const indexMap = this.indexes.get(indexName)!; const indexMap = this.indexes.get(indexName)!;

View File

@@ -1,4 +1,4 @@
import { BaseStorage, FindOptions } from './base-storage.js'; import { BaseStorage, FindOptions, type Filter } from './base-storage.js';
import { StorageMemory } from './storage-memory.js'; import { StorageMemory } from './storage-memory.js';
/** /**
@@ -65,19 +65,19 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
await this.store.insertMany(documents); await this.store.insertMany(documents);
} }
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> { async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
return await this.inMemoryCache.find(filter, options); return await this.inMemoryCache.find(filter, options);
} }
async updateMany( async updateMany(
filter: Partial<T>, filter: Filter<T>,
update: Partial<T>, update: Partial<T>,
options: FindOptions = {} as FindOptions options: FindOptions = {} as FindOptions
): Promise<number> { ): Promise<number> {
return await this.store.updateMany(filter, update, options); return await this.store.updateMany(filter, update, options);
} }
async deleteMany(filter: Partial<T>, options: FindOptions = {} as FindOptions): Promise<number> { async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
return await this.store.deleteMany(filter, options); return await this.store.deleteMany(filter, options);
} }

View File

@@ -1,4 +1,11 @@
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js'; import {
BaseStorage,
FindOptions,
type IndexDefinition,
type Filter,
type ComparisonOperators,
isOperatorObject,
} from './base-storage.js';
/** /**
* Separator used when joining multiple field values into a single index key. * Separator used when joining multiple field values into a single index key.
@@ -86,7 +93,7 @@ export class StorageMemory<
} }
} }
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> { async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
let results: T[]; let results: T[];
// Attempt to satisfy the query via an index. // Attempt to satisfy the query via an index.
@@ -118,7 +125,7 @@ export class StorageMemory<
} }
async updateMany( async updateMany(
filter: Partial<T>, filter: Filter<T>,
update: Partial<T>, update: Partial<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
@@ -147,7 +154,7 @@ export class StorageMemory<
} }
async deleteMany( async deleteMany(
filter: Partial<T>, filter: Filter<T>,
options: Partial<FindOptions> = {}, options: Partial<FindOptions> = {},
): Promise<number> { ): Promise<number> {
const rowsToDelete = this.collectMatches(filter); const rowsToDelete = this.collectMatches(filter);
@@ -182,26 +189,42 @@ export class StorageMemory<
/** /**
* Checks whether a document satisfies every field in the filter. * Checks whether a document satisfies every field in the filter.
* An empty or undefined filter matches everything. * Supports both plain equality values and comparison operator objects.
*/ */
private matchesFilter(item: T, filter?: Partial<T>): boolean { private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) { if (!filter || Object.keys(filter).length === 0) {
return true; return true;
} }
for (const [key, value] of Object.entries(filter)) { for (const [key, value] of Object.entries(filter)) {
if (item[key] !== value) { if (isOperatorObject(value)) {
return false; if (!this.matchesOperators(item[key], value)) return false;
} else {
if (item[key] !== value) return false;
} }
} }
return true; return true;
} }
/**
* Evaluate a set of comparison operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
return true;
}
/** /**
* Collect all [internalKey, document] pairs that match a filter. * Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan. * Uses an index when possible, otherwise falls back to a full scan.
*/ */
private collectMatches(filter?: Partial<T>): Array<[number, T]> { private collectMatches(filter?: Filter<T>): Array<[number, T]> {
const indexKeys = this.resolveIndexKeys(filter); const indexKeys = this.resolveIndexKeys(filter);
if (indexKeys !== null) { if (indexKeys !== null) {
@@ -296,21 +319,31 @@ export class StorageMemory<
* Attempt to resolve a set of candidate internal keys from the indexes. * Attempt to resolve a set of candidate internal keys from the indexes.
* Returns `null` if no index can serve the query. * Returns `null` if no index can serve the query.
* *
* An index is used when the filter fields are a superset of (or equal to) * An index is used when the filter contains plain equality values for every
* an index's fields — meaning the index value can be fully constructed * field in the index. Operator objects (e.g. `{ $lt: 50 }`) are excluded
* from the filter. * from index resolution since hash-based indexes only support equality.
*/ */
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null { private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
if (!filter) return null; if (!filter) return null;
const filterKeys = Object.keys(filter); const filterKeys = Object.keys(filter);
if (filterKeys.length === 0) return null; if (filterKeys.length === 0) return null;
// Extract only the equality fields from the filter (skip operator objects).
const equalityFilter: Record<string, any> = {};
for (const [key, value] of Object.entries(filter)) {
if (!isOperatorObject(value)) {
equalityFilter[key] = value;
}
}
if (Object.keys(equalityFilter).length === 0) return null;
for (const fields of this.indexDefs) { for (const fields of this.indexDefs) {
// Every field in the index must be present in the filter. // Every field in the index must be present as an equality value.
if (!fields.every((f) => f in filter)) continue; if (!fields.every((f) => f in equalityFilter)) continue;
const indexName = fields.join(INDEX_KEY_SEP); const indexName = fields.join(INDEX_KEY_SEP);
const indexValue = this.buildIndexValue(filter, fields); const indexValue = this.buildIndexValue(equalityFilter, fields);
if (indexValue === null) continue; if (indexValue === null) continue;
const indexMap = this.indexes.get(indexName)!; const indexMap = this.indexes.get(indexName)!;
@@ -326,7 +359,7 @@ export class StorageMemory<
* Returns `null` when no index can serve the filter, signalling * Returns `null` when no index can serve the filter, signalling
* the caller to fall back to a full scan. * the caller to fall back to a full scan.
*/ */
private findViaIndex(filter?: Partial<T>): T[] | null { private findViaIndex(filter?: Filter<T>): T[] | null {
const keys = this.resolveIndexKeys(filter); const keys = this.resolveIndexKeys(filter);
if (keys === null) return null; if (keys === null) return null;