Harvmaster/Hashpass-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plaintext values on encryption

Browse Source
This commit is contained in:
Harvey Zuccon
2026-02-25 15:35:04 +11:00
parent f80aa2dcfc
commit 77593fe3b4
8 changed files with 407 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
benchmarks/sekp256k1.ts
View File

@@ -7,6 +7,7 @@ const alicePublicKey = await alice.getPublicKey();
const bobPublicKey = await bob.getPublicKey();
const keyStart = performance.now();
const aliceSharedSecret = await alice.getSharedSecret(bobPublicKey);
const bobSharedSecret = await bob.getSharedSecret(alicePublicKey);

85
benchmarks/storage.ts
View File

@@ -49,7 +49,7 @@ function fmtOps(ops: number): string {
/**
* Run a full suite of benchmarks against a given storage instance.
*/
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[]) {
async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs: Doc[], supportsRangeOps = true) {
const count = docs.length;
console.log(`\n${'='.repeat(60)}`);
console.log(` ${label} (${count.toLocaleString()} documents)`);
@@ -59,56 +59,75 @@ async function benchmarkStorage(label: string, storage: BaseStorage<Doc>, docs:
const insertMs = await time(async () => {
await storage.insertMany(docs);
});
console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`);
console.log(` insertMany ${insertMs.toFixed(2)}ms (${fmtOps((count / insertMs) * 1000)} ops/sec)`);
// --- Find all (no filter) ---
const findAllMs = await time(async () => {
await storage.find();
});
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
console.log(` find() ${findAllMs.toFixed(2)}ms (${fmtOps((count / findAllMs) * 1000)} docs/sec)`);
// --- Find by indexed field (single-key lookup, repeated) ---
// --- Find by indexed field (equality) ---
const lookupCount = Math.min(count, 1_000);
const findIndexedMs = await time(async () => {
for (let i = 0; i < lookupCount; i++) {
await storage.findOne({ id: `id-${i}` } as Partial<Doc>);
await storage.findOne({ id: `id-${i}` });
}
});
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
console.log(` findOne indexed ${findIndexedMs.toFixed(2)}ms (${fmtOps((lookupCount / findIndexedMs) * 1000)} ops/sec) [${lookupCount} lookups]`);
// --- Find by non-indexed field (full scan, repeated) ---
// --- Find by non-indexed field (full scan) ---
const scanCount = Math.min(count, 1_000);
const findScanMs = await time(async () => {
for (let i = 0; i < scanCount; i++) {
await storage.findOne({ email: `user-${i}@test.com` } as Partial<Doc>);
await storage.findOne({ email: `user-${i}@test.com` });
}
});
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
console.log(` findOne scan ${findScanMs.toFixed(2)}ms (${fmtOps((scanCount / findScanMs) * 1000)} ops/sec) [${scanCount} lookups]`);
// --- Find with $gte / $lt range (full scan) ---
if (supportsRangeOps) {
const rangeCount = Math.min(count, 100);
let rangeTotal = 0;
const findRangeMs = await time(async () => {
for (let i = 0; i < rangeCount; i++) {
const results = await storage.find({ age: { $gte: 30, $lt: 40 } });
rangeTotal += results.length;
}
});
console.log(` find $gte/$lt ${findRangeMs.toFixed(2)}ms (${fmtOps((rangeCount / findRangeMs) * 1000)} ops/sec) [${rangeCount} queries, ~${Math.round(rangeTotal / rangeCount)} hits/query]`);
// --- Find with combined equality + operator (index narrows, operator verifies) ---
const comboCount = Math.min(count, 1_000);
const findComboMs = await time(async () => {
for (let i = 0; i < comboCount; i++) {
await storage.find({ id: `id-${i}`, age: { $gte: 20 } });
}
});
console.log(` find idx+operator ${findComboMs.toFixed(2)}ms (${fmtOps((comboCount / findComboMs) * 1000)} ops/sec) [${comboCount} queries]`);
}
// --- Update by indexed field ---
const updateCount = Math.min(count, 1_000);
const updateMs = await time(async () => {
for (let i = 0; i < updateCount; i++) {
await storage.updateOne(
{ id: `id-${i}` } as Partial<Doc>,
{ age: 99 } as Partial<Doc>,
);
await storage.updateOne({ id: `id-${i}` }, { name: `updated-${i}` });
}
});
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
console.log(` updateOne indexed ${updateMs.toFixed(2)}ms (${fmtOps((updateCount / updateMs) * 1000)} ops/sec) [${updateCount} updates]`);
// --- Delete by indexed field ---
const deleteCount = Math.min(count, 1_000);
const deleteMs = await time(async () => {
for (let i = 0; i < deleteCount; i++) {
await storage.deleteOne({ id: `id-${i}` } as Partial<Doc>);
await storage.deleteOne({ id: `id-${i}` });
}
});
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
console.log(` deleteOne indexed ${deleteMs.toFixed(2)}ms (${fmtOps((deleteCount / deleteMs) * 1000)} ops/sec) [${deleteCount} deletes]`);
// --- Verify remaining count ---
const remaining = await storage.find();
console.log(` remaining docs: ${remaining.length.toLocaleString()}`);
console.log(` remaining docs: ${remaining.length.toLocaleString()}`);
}
// ---------------------------------------------------------------------------
@@ -132,7 +151,7 @@ for (const count of DOC_COUNTS) {
}
// ---------------------------------------------------------------------------
// EncryptedStorage — crypto overhead dominates, so use smaller counts
// EncryptedStorage — with plaintextKeys for range queries
// ---------------------------------------------------------------------------
const ENCRYPTED_DOC_COUNTS = [100, 1_000, 10_000];
@@ -141,15 +160,29 @@ const encryptionKey = await AESKey.fromSeed('benchmark-key');
for (const count of ENCRYPTED_DOC_COUNTS) {
const docs = generateDocs(count);
// Encrypted + indexed backing store.
const encBase = StorageMemory.from<Record<string, string>>(['id', 'name']);
const encrypted = EncryptedStorage.from<Doc>(encBase, encryptionKey);
await benchmarkStorage('EncryptedStorage (indexed backing store)', encrypted, docs);
// No indexes + plaintextKeys — range queries on age work.
const encBaseNoIdx = StorageMemory.from<Record<string, any>>();
const encNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey, {
plaintextKeys: ['age'],
});
await benchmarkStorage('Encrypted (no indexes, plaintextKeys: age)', encNoIdx, docs);
// Encrypted + no-index backing store.
const encBaseNoIdx = StorageMemory.from<Record<string, string>>();
const encryptedNoIdx = EncryptedStorage.from<Doc>(encBaseNoIdx, encryptionKey);
await benchmarkStorage('EncryptedStorage (no indexes)', encryptedNoIdx, docs);
// Indexed + plaintextKeys — range queries on age work.
const encBaseA = StorageMemory.from<Record<string, any>>(['id', 'name']);
const encA = EncryptedStorage.from<Doc>(encBaseA, encryptionKey, {
plaintextKeys: ['age'],
});
await benchmarkStorage('Encrypted (indexed, plaintextKeys: age)', encA, docs);
// Indexed, fully encrypted — same indexes but no plaintext keys.
const encBaseB = StorageMemory.from<Record<string, any>>(['id', 'name']);
const encB = EncryptedStorage.from<Doc>(encBaseB, encryptionKey);
await benchmarkStorage('Encrypted (indexed, fully encrypted)', encB, docs, false);
// No indexes, fully encrypted — worst case.
const encBaseC = StorageMemory.from<Record<string, any>>();
const encC = EncryptedStorage.from<Doc>(encBaseC, encryptionKey);
await benchmarkStorage('Encrypted (no indexes, fully encrypted)', encC, docs, false);
}
console.log('\nDone.\n');

3
package.json
View File

@@ -8,6 +8,9 @@
"format": "prettier --write .",
"test": "echo \"Error: no test specified\" && exit 1",
"benchmark:sha256": "tsx benchmarks/sha256.ts",
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",
"benchmark:encryption": "tsx benchmarks/sekp256k1.ts",
"benchmark:storage": "tsx benchmarks/storage.ts"
},
"keywords": [],

60
src/storage/base-storage.ts
View File

@@ -19,6 +19,54 @@ export type FindOptions = {
*/
export type IndexDefinition = string[] | string[][];
/**
* MongoDB-style comparison operators for a single field value.
*/
export type ComparisonOperators<V> = {
$eq?: V;
$ne?: V;
$lt?: V;
$lte?: V;
$gt?: V;
$gte?: V;
};
/**
* A filter value for a single field — either a plain value (equality shorthand)
* or an object of comparison operators.
*/
export type FieldFilter<V> = V | ComparisonOperators<V>;
/**
* Query filter that supports both equality shorthand and comparison operators.
*
* @example
* // Equality shorthand
* { name: 'foo' }
*
* // Comparison operators
* { age: { $gte: 18, $lt: 65 } }
*
* // Combined
* { name: 'foo', age: { $gte: 18 } }
*/
export type Filter<T extends Record<string, any>> = {
[K in keyof T]?: FieldFilter<T[K]>;
};
/**
* Detect whether a filter value is an operator object (e.g. `{ $lt: 50 }`)
* rather than a plain value. Guards against Date and Array which are objects
* but represent document values, not operators.
*/
export function isOperatorObject(value: unknown): value is ComparisonOperators<any> {
return value !== null
&& typeof value === 'object'
&& !Array.isArray(value)
&& !(value instanceof Date)
&& Object.keys(value).some((k) => k.startsWith('$'));
}
export type StorageEvent<T = Record<string, any>> = {
insert: {
value: T;
@@ -54,7 +102,7 @@ export abstract class BaseStorage<
* Find a single document that matches the filter
* @param filter MongoDB-like query filter
*/
async findOne(filter?: Partial<T>): Promise<T | null> {
async findOne(filter?: Filter<T>): Promise<T | null> {
const results = await this.find(filter);
return results.length > 0 ? results[0] : null;
}
@@ -64,7 +112,7 @@ export abstract class BaseStorage<
* @param filter MongoDB-like query filter
* @param options Query options (limit, skip, sort)
*/
abstract find(filter?: Partial<T>, options?: FindOptions): Promise<T[]>;
abstract find(filter?: Filter<T>, options?: FindOptions): Promise<T[]>;
/**
* Update a document that matches the filter
@@ -72,7 +120,7 @@ export abstract class BaseStorage<
* @param update Document or fields to update
* @returns True if a document was updated, false otherwise
*/
async updateOne(filter: Partial<T>, update?: Partial<T>): Promise<boolean> {
async updateOne(filter: Filter<T>, update?: Partial<T>): Promise<boolean> {
const results = await this.updateMany(filter, update, { limit: 1 });
return results > 0;
}
@@ -85,7 +133,7 @@ export abstract class BaseStorage<
* @returns Number of documents updated
*/
abstract updateMany(
filter: Partial<T>,
filter: Filter<T>,
update: Partial<T>,
options?: Partial<FindOptions>,
): Promise<number>;
@@ -95,7 +143,7 @@ export abstract class BaseStorage<
* @param filter Query to match the document to delete
* @returns True if a document was deleted, false otherwise
*/
async deleteOne(filter: Partial<T>): Promise<boolean> {
async deleteOne(filter: Filter<T>): Promise<boolean> {
const results = await this.deleteMany(filter, { limit: 1 });
return results > 0;
}
@@ -107,7 +155,7 @@ export abstract class BaseStorage<
* @returns Number of documents deleted
*/
abstract deleteMany(
filter: Partial<T>,
filter: Filter<T>,
options: Partial<FindOptions>,
): Promise<number>;

267
src/storage/encrypted-storage.ts
View File

@@ -3,15 +3,29 @@ import { Packr } from 'msgpackr';
import { AESKey } from 'src/crypto/aes-key.js';
import { Bytes } from 'src/crypto/bytes.js';
import { BaseStorage, type FindOptions } from './base-storage.js';
import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
import { encodeExtendedJson, decodeExtendedJson, encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
export type EncryptedStorageOptions = {
/**
* Fields that should be stored in plaintext (not encrypted).
* These fields retain their original types in the backing store, which
* allows comparison operators ($lt, $gt, etc.) to work on them.
* All other fields are encrypted.
*/
plaintextKeys?: string[];
};
export class EncryptedStorage<
T extends Record<string, any> = Record<string, any>,
> extends BaseStorage<T> {
static from<T>(storage: BaseStorage<Record<string, string>>, key: AESKey) {
return new EncryptedStorage<T>(storage, key);
static from<T>(
storage: BaseStorage<Record<string, any>>,
key: AESKey,
options?: EncryptedStorageOptions,
) {
return new EncryptedStorage<T>(storage, key, options);
}
private readonly msgpackr = new Packr({
@@ -33,34 +47,32 @@ export class EncryptedStorage<
*/
private readonly decryptCache = new Map<string, any>();
/** Set of field names that are stored in plaintext (not encrypted). */
private readonly plaintextKeys: Set<string>;
constructor(
private readonly storage: BaseStorage<Record<string, string>>,
private readonly storage: BaseStorage<Record<string, any>>,
private readonly key: AESKey,
options?: EncryptedStorageOptions,
) {
super();
// Forward events from the underlying storage, decrypting the data
this.storage.on('insert', async (event) => {
// De-crypt the value before emitting the event.
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
this.plaintextKeys = new Set(options?.plaintextKeys ?? []);
// Re-emit the insert event with the original payload.
// Forward events from the underlying storage, converting data back.
this.storage.on('insert', async (event) => {
const decryptedValue = await this.convertFromStorage(event.value);
this.emit('insert', { value: decryptedValue });
});
this.storage.on('update', async (event) => {
// Decrypt both old and new values before re-emitting.
const decryptedOldValue = await this.convertToDecrypted(event.oldValue as Record<string, string>);
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
const decryptedOldValue = await this.convertFromStorage(event.oldValue);
const decryptedValue = await this.convertFromStorage(event.value);
this.emit('update', { oldValue: decryptedOldValue, value: decryptedValue });
});
this.storage.on('delete', async (event) => {
// De-crypt the value before emitting the event.
const decryptedValue = await this.convertToDecrypted(event.value as Record<string, string>);
// Re-emit the delete event with the original payload.
const decryptedValue = await this.convertFromStorage(event.value);
this.emit('delete', { value: decryptedValue });
});
@@ -70,53 +82,83 @@ export class EncryptedStorage<
}
async insertMany(documents: Array<T>): Promise<void> {
// Encrypt all documents in parallel.
const encrypted = await Promise.all(
documents.map((doc) => this.convertToEncrypted(doc)),
const converted = await Promise.all(
documents.map((doc) => this.convertForStorage(doc)),
);
await this.storage.insertMany(encrypted);
await this.storage.insertMany(converted);
}
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
const encryptedFilter = filter ? await this.convertToEncrypted(filter) : undefined;
const documents = await this.storage.find(encryptedFilter, options);
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
const convertedFilter = filter
? await this.convertFilterForStorage(filter)
: undefined;
const documents = await this.storage.find(convertedFilter, options);
return Promise.all(
documents.map(async (document) => this.convertToDecrypted(document)),
documents.map((doc) => this.convertFromStorage(doc)),
);
}
async updateMany(
filter: Partial<T>,
filter: Filter<T>,
update: Partial<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
const encryptedFilter = await this.convertToEncrypted(filter);
const encryptedUpdate = await this.convertToEncrypted(update);
return this.storage.updateMany(encryptedFilter, encryptedUpdate, options);
const convertedFilter = await this.convertFilterForStorage(filter);
const convertedUpdate = await this.convertForStorage(update);
return this.storage.updateMany(convertedFilter, convertedUpdate, options);
}
async deleteMany(
filter: Partial<T>,
filter: Filter<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
const encryptedFilter = await this.convertToEncrypted(filter);
return this.storage.deleteMany(encryptedFilter, options);
const convertedFilter = await this.convertFilterForStorage(filter);
return this.storage.deleteMany(convertedFilter, options);
}
deriveChild<C>(path: string): BaseStorage<C> {
return EncryptedStorage.from(this.storage.deriveChild(path), this.key);
return EncryptedStorage.from<C>(
this.storage.deriveChild(path),
this.key,
{ plaintextKeys: [...this.plaintextKeys] },
);
}
private async convertToEncrypted(
// ---------------------------------------------------------------------------
// Storage conversion — documents (insert/update values)
// ---------------------------------------------------------------------------
/**
* Convert a document for storage. Encrypted fields are encrypted;
* plaintext fields are passed through with value formatting only.
*/
private async convertForStorage(
document: Partial<T>,
): Promise<Record<string, string>> {
const encrypted: Record<string, string> = {};
): Promise<Record<string, any>> {
const result: Record<string, any> = {};
const formattedDocument = this.formatDocumentForEncryption(document);
const entries = Object.entries(formattedDocument);
// Encrypt all fields in parallel, using the cache when possible.
const results = await Promise.all(
entries.map(async ([key, value]) => {
// Split into plaintext and encrypted fields.
const plaintextEntries: Array<[string, any]> = [];
const encryptedEntries: Array<[string, any]> = [];
for (const entry of entries) {
if (this.plaintextKeys.has(entry[0])) {
plaintextEntries.push(entry);
} else {
encryptedEntries.push(entry);
}
}
// Plaintext fields pass through directly.
for (const [key, value] of plaintextEntries) {
result[key] = value;
}
// Encrypt fields in parallel, with memoization.
const encrypted = await Promise.all(
encryptedEntries.map(async ([key, value]) => {
const bin = this.msgpackr.pack(value);
const cacheKey = Bytes.from(bin).toBase64();
@@ -131,21 +173,36 @@ export class EncryptedStorage<
}),
);
for (const [key, ciphertext] of results) {
encrypted[key] = ciphertext;
for (const [key, ciphertext] of encrypted) {
result[key] = ciphertext;
}
return encrypted;
return result;
}
private async convertToDecrypted(
document: Record<string, string>,
/**
* Convert a stored document back to its original form. Encrypted fields
* are decrypted; plaintext fields are passed through with value formatting.
*/
private async convertFromStorage(
document: Record<string, any>,
): Promise<T> {
const entries = Object.entries(document);
// Decrypt all fields in parallel, using the cache when possible.
const results = await Promise.all(
entries.map(async ([key, ciphertext]) => {
const plaintextEntries: Array<[string, any]> = [];
const encryptedEntries: Array<[string, any]> = [];
for (const entry of entries) {
if (this.plaintextKeys.has(entry[0])) {
plaintextEntries.push(entry);
} else {
encryptedEntries.push(entry);
}
}
// Decrypt encrypted fields in parallel, with memoization.
const decrypted = await Promise.all(
encryptedEntries.map(async ([key, ciphertext]) => {
let value = this.decryptCache.get(ciphertext);
if (value === undefined) {
const bin = await this.key.decrypt(Bytes.fromBase64(ciphertext));
@@ -156,70 +213,122 @@ export class EncryptedStorage<
}),
);
const decrypted: Record<string, any> = {};
for (const [key, value] of results) {
decrypted[key] = value;
const result: Record<string, any> = {};
for (const [key, value] of plaintextEntries) {
result[key] = value;
}
for (const [key, value] of decrypted) {
result[key] = value;
}
const decodedDocument = this.formatDocumentFromDecryption(decrypted);
const decodedDocument = this.formatDocumentFromDecryption(result);
return decodedDocument as T;
}
private formatDocumentForEncryption(document: any): any {
// First, iterate through each key and value in the document and format the value for encryption.
const formattedDocument: any = {};
// ---------------------------------------------------------------------------
// Storage conversion — filters (may contain operator objects)
// ---------------------------------------------------------------------------
/**
* Convert a query filter for storage. Handles both plain equality values
* and operator objects.
*
* - Plaintext fields: values and operator objects pass through as-is.
* - Encrypted fields: plain values are encrypted. Operator objects throw
* because range comparisons are meaningless on ciphertext.
*/
private async convertFilterForStorage(
filter: Filter<T>,
): Promise<Filter<Record<string, any>>> {
const result: Record<string, any> = {};
const entries = Object.entries(filter);
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
for (const [key, value] of entries) {
if (this.plaintextKeys.has(key)) {
// Plaintext field — pass through (including operator objects).
result[key] = isOperatorObject(value)
? value
: this.formatValueForEncryption(value);
} else if (isOperatorObject(value)) {
// Encrypted field with an operator — not supported.
throw new Error(
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
);
} else {
// Encrypted field with a plain equality value — encrypt it.
const formatted = this.formatValueForEncryption(value);
encryptionTasks.push(
(async () => {
const bin = this.msgpackr.pack(formatted);
const cacheKey = Bytes.from(bin).toBase64();
let ciphertext = this.encryptCache.get(cacheKey);
if (ciphertext === undefined) {
const encryptedValue = await this.key.encrypt(bin, true);
ciphertext = encryptedValue.toBase64();
this.encryptCache.set(cacheKey, ciphertext);
}
return [key, ciphertext] as const;
})(),
);
}
}
const encryptedResults = await Promise.all(encryptionTasks);
for (const [key, ciphertext] of encryptedResults) {
result[key] = ciphertext;
}
return result;
}
// ---------------------------------------------------------------------------
// Value formatting
// ---------------------------------------------------------------------------
private formatDocumentForEncryption(document: any): any {
const formattedDocument: any = {};
for (const [key, value] of Object.entries(document)) {
formattedDocument[key] = this.formatValueForEncryption(value);
}
// Then, encode the document to extended JSON.
const encodedDocument = encodeExtendedJsonObject(formattedDocument);
return encodedDocument;
return encodeExtendedJsonObject(formattedDocument);
}
private formatDocumentFromDecryption(document: any): any {
// First, decode the document from extended JSON.
const decodedDocument = decodeExtendedJsonObject(document);
// Then, iterate through each key and value in the document and format the value from decryption.
for (const [key, value] of Object.entries(decodedDocument)) {
decodedDocument[key] = this.formatValueFromDecryption(value);
}
return decodedDocument;
}
/**
* Format a value before encryption. Converts types that msgpackr
* doesn't natively support (e.g. Date) into serialisable forms.
*/
private formatValueForEncryption(value: any): any {
// msgpackr doesnt support Date, so we need to convert it to a string.
if (value instanceof Date) {
return `<Date: ${value.getTime()}>`;
}
return value;
}
/**
* Restore a value after decryption. Reverses the transformations
* applied by `formatValueForEncryption`.
*/
private formatValueFromDecryption(value: any): any {
// msgpackr doesnt support Date, so we need to convert it to a Date.
if (typeof value === 'string') {
// Check if this value matches an Extended JSON encoded date.
// TODO: Do this without a regex for performance reasons.
// const dateMatch = value.match(/<Date: (?<time>[0-9]+)>/);
// Without regex
if (value.startsWith('<Date: ') && value.endsWith('>')) {
const time = value.slice(7, -1);
return new Date(parseInt(time));
}
// If it does, convert it to a Date.
// if (dateMatch) {
// const { time } = dateMatch.groups!;
// return new Date(parseInt(time));
// }
}
return value;
}
}

57
src/storage/storage-localstorage.ts
View File

@@ -1,4 +1,11 @@
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
import {
BaseStorage,
FindOptions,
type IndexDefinition,
type Filter,
type ComparisonOperators,
isOperatorObject,
} from './base-storage.js';
/**
* Key prefix separator used to namespace documents within localStorage.
@@ -132,7 +139,7 @@ export class StorageLocalStorage<
this.persistManifest();
}
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
this.refreshManifest();
let results: T[];
@@ -175,7 +182,7 @@ export class StorageLocalStorage<
}
async updateMany(
filter: Partial<T>,
filter: Filter<T>,
update: Partial<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
@@ -207,7 +214,7 @@ export class StorageLocalStorage<
}
async deleteMany(
filter: Partial<T>,
filter: Filter<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
this.refreshManifest();
@@ -251,20 +258,38 @@ export class StorageLocalStorage<
/**
* Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
*/
private matchesFilter(item: T, filter?: Partial<T>): boolean {
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) return true;
for (const [key, value] of Object.entries(filter)) {
if (item[key] !== value) return false;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
if (item[key] !== value) return false;
}
}
return true;
}
/**
* Evaluate a set of comparison operators against a single field value.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
return true;
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
*/
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
private collectMatches(filter?: Filter<T>): Array<[number, T]> {
const indexKeys = this.resolveIndexKeys(filter);
const keysToScan = indexKeys ?? this.manifest;
const results: Array<[number, T]> = [];
@@ -348,17 +373,29 @@ export class StorageLocalStorage<
/**
* Attempt to resolve candidate internal keys from the indexes.
* Returns `null` if no index can serve the query.
*
* Only plain equality values are used for index resolution — operator
* objects are excluded since hash-based indexes only support equality.
*/
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
if (!filter) return null;
const filterKeys = Object.keys(filter);
if (filterKeys.length === 0) return null;
const equalityFilter: Record<string, any> = {};
for (const [key, value] of Object.entries(filter)) {
if (!isOperatorObject(value)) {
equalityFilter[key] = value;
}
}
if (Object.keys(equalityFilter).length === 0) return null;
for (const fields of this.indexDefs) {
if (!fields.every((f) => f in filter)) continue;
if (!fields.every((f) => f in equalityFilter)) continue;
const indexName = fields.join(INDEX_KEY_SEP);
const indexValue = this.buildIndexValue(filter, fields);
const indexValue = this.buildIndexValue(equalityFilter, fields);
if (indexValue === null) continue;
const indexMap = this.indexes.get(indexName)!;

8
src/storage/storage-memory-synced.ts
View File

@@ -1,4 +1,4 @@
import { BaseStorage, FindOptions } from './base-storage.js';
import { BaseStorage, FindOptions, type Filter } from './base-storage.js';
import { StorageMemory } from './storage-memory.js';
/**
@@ -65,19 +65,19 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
await this.store.insertMany(documents);
}
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
return await this.inMemoryCache.find(filter, options);
}
async updateMany(
filter: Partial<T>,
filter: Filter<T>,
update: Partial<T>,
options: FindOptions = {} as FindOptions
): Promise<number> {
return await this.store.updateMany(filter, update, options);
}
async deleteMany(filter: Partial<T>, options: FindOptions = {} as FindOptions): Promise<number> {
async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
return await this.store.deleteMany(filter, options);
}

67
src/storage/storage-memory.ts
View File

@@ -1,4 +1,11 @@
import { BaseStorage, FindOptions, type IndexDefinition } from './base-storage.js';
import {
BaseStorage,
FindOptions,
type IndexDefinition,
type Filter,
type ComparisonOperators,
isOperatorObject,
} from './base-storage.js';
/**
* Separator used when joining multiple field values into a single index key.
@@ -86,7 +93,7 @@ export class StorageMemory<
}
}
async find(filter?: Partial<T>, options?: FindOptions): Promise<T[]> {
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
let results: T[];
// Attempt to satisfy the query via an index.
@@ -118,7 +125,7 @@ export class StorageMemory<
}
async updateMany(
filter: Partial<T>,
filter: Filter<T>,
update: Partial<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
@@ -147,7 +154,7 @@ export class StorageMemory<
}
async deleteMany(
filter: Partial<T>,
filter: Filter<T>,
options: Partial<FindOptions> = {},
): Promise<number> {
const rowsToDelete = this.collectMatches(filter);
@@ -182,26 +189,42 @@ export class StorageMemory<
/**
* Checks whether a document satisfies every field in the filter.
* An empty or undefined filter matches everything.
* Supports both plain equality values and comparison operator objects.
*/
private matchesFilter(item: T, filter?: Partial<T>): boolean {
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) {
return true;
}
for (const [key, value] of Object.entries(filter)) {
if (item[key] !== value) {
return false;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
if (item[key] !== value) return false;
}
}
return true;
}
/**
* Evaluate a set of comparison operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
if (ops.$ne !== undefined && fieldValue === ops.$ne) return false;
if (ops.$lt !== undefined && !(fieldValue < ops.$lt)) return false;
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
return true;
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
*/
private collectMatches(filter?: Partial<T>): Array<[number, T]> {
private collectMatches(filter?: Filter<T>): Array<[number, T]> {
const indexKeys = this.resolveIndexKeys(filter);
if (indexKeys !== null) {
@@ -296,21 +319,31 @@ export class StorageMemory<
* Attempt to resolve a set of candidate internal keys from the indexes.
* Returns `null` if no index can serve the query.
*
* An index is used when the filter fields are a superset of (or equal to)
* an index's fields — meaning the index value can be fully constructed
* from the filter.
* An index is used when the filter contains plain equality values for every
* field in the index. Operator objects (e.g. `{ $lt: 50 }`) are excluded
* from index resolution since hash-based indexes only support equality.
*/
private resolveIndexKeys(filter?: Partial<T>): Set<number> | null {
private resolveIndexKeys(filter?: Filter<T>): Set<number> | null {
if (!filter) return null;
const filterKeys = Object.keys(filter);
if (filterKeys.length === 0) return null;
// Extract only the equality fields from the filter (skip operator objects).
const equalityFilter: Record<string, any> = {};
for (const [key, value] of Object.entries(filter)) {
if (!isOperatorObject(value)) {
equalityFilter[key] = value;
}
}
if (Object.keys(equalityFilter).length === 0) return null;
for (const fields of this.indexDefs) {
// Every field in the index must be present in the filter.
if (!fields.every((f) => f in filter)) continue;
// Every field in the index must be present as an equality value.
if (!fields.every((f) => f in equalityFilter)) continue;
const indexName = fields.join(INDEX_KEY_SEP);
const indexValue = this.buildIndexValue(filter, fields);
const indexValue = this.buildIndexValue(equalityFilter, fields);
if (indexValue === null) continue;
const indexMap = this.indexes.get(indexName)!;
@@ -326,7 +359,7 @@ export class StorageMemory<
* Returns `null` when no index can serve the filter, signalling
* the caller to fall back to a full scan.
*/
private findViaIndex(filter?: Partial<T>): T[] | null {
private findViaIndex(filter?: Filter<T>): T[] | null {
const keys = this.resolveIndexKeys(filter);
if (keys === null) return null;