Compare commits

...

5 Commits

15 changed files with 734 additions and 165 deletions

View File

@@ -1,5 +1,6 @@
import { AESKey } from '../src/crypto/aes-key.js';
import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js';
import { BTreeCache, KvCache } from '../src/cache/index.js';
// ---------------------------------------------------------------------------
// Helpers
@@ -161,9 +162,19 @@ const DOC_COUNTS = [1_000, 10_000, 50_000];
for (const count of DOC_COUNTS) {
const docs = generateDocs(count);
// Indexes on id, name, AND age — range queries on age use B+ Tree.
const indexedWithAge = StorageMemory.from<Doc>(['id', 'name', 'age']);
await benchmarkStorage('StorageMemory (indexed: id,name,age)', indexedWithAge, docs, { hasAgeIndex: true });
// Indexes on id, name, AND age with explicit B+ Tree cache.
const indexedWithAgeBTree = StorageMemory.from<Doc>(
['id', 'name', 'age'],
new BTreeCache(),
);
await benchmarkStorage('StorageMemory (B+Tree cache, indexed: id,name,age)', indexedWithAgeBTree, docs, { hasAgeIndex: true });
// Same indexes, but KV cache (no range support).
const indexedWithAgeKv = StorageMemory.from<Doc>(
['id', 'name', 'age'],
new KvCache(),
);
await benchmarkStorage('StorageMemory (KV cache, indexed: id,name,age)', indexedWithAgeKv, docs);
// Indexes on id, name only — range queries on age fall back to full scan.
const indexed = StorageMemory.from<Doc>(['id', 'name']);

View File

@@ -6,7 +6,9 @@
"scripts": {
"build": "tsc",
"format": "prettier --write .",
"test": "echo \"Error: no test specified\" && exit 1",
"test": "vitest --run",
"test:watch": "vitest",
"benchmark:sha256": "tsx benchmarks/sha256.ts",
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",

79
src/cache/b-tree-cache.ts vendored Normal file
View File

@@ -0,0 +1,79 @@
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* B+ tree-backed cache implementation.
*
* - Equality: O(log n)
* - Range: O(log n + k)
*/
export class BTreeCache extends BaseCache {
private trees = new Map<string, BPlusTree<any, number>>();
registerIndex(indexName: string, fields: string[]): void {
if (this.trees.has(indexName)) return;
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.trees.set(indexName, new BPlusTree<any, number>(32, comparator));
}
clearIndex(indexName: string): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.clear();
}
insert(indexName: string, key: any, internalKey: number): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.insert(key, internalKey);
}
delete(indexName: string, key: any, internalKey: number): void {
const tree = this.trees.get(indexName);
if (!tree) return;
tree.delete(key, internalKey);
}
get(indexName: string, key: any): Iterable<number> {
const tree = this.trees.get(indexName);
if (!tree) return [];
return tree.get(key) ?? [];
}
range(
indexName: string,
min?: any,
max?: any,
options: CacheRangeOptions = {},
): Iterable<number> | null {
const tree = this.trees.get(indexName);
if (!tree) return [];
const entries = tree.range(min, max, options);
return this.flattenEntryKeys(entries);
}
createChild(): BaseCache {
return new BTreeCache();
}
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
}
return result;
}
}

37
src/cache/base-cache.ts vendored Normal file
View File

@@ -0,0 +1,37 @@
export type CacheRangeOptions = {
lowerInclusive?: boolean;
upperInclusive?: boolean;
};
/**
* Shared cache abstraction used by storage adapters for secondary indexes.
*
* Implementations may support only equality (`get`) or both equality and
* range queries (`range`). Returning `null` from `range` indicates the cache
* cannot serve that range query efficiently.
*/
export abstract class BaseCache {
abstract registerIndex(indexName: string, fields: string[]): void;
abstract clearIndex(indexName: string): void;
abstract insert(indexName: string, key: any, internalKey: number): void;
abstract delete(indexName: string, key: any, internalKey: number): void;
abstract get(indexName: string, key: any): Iterable<number>;
abstract range(
indexName: string,
min?: any,
max?: any,
options?: CacheRangeOptions,
): Iterable<number> | null;
/**
* Create a new empty cache instance of the same concrete type.
* Used by deriveChild() to preserve cache strategy across children.
*/
abstract createChild(): BaseCache;
}

4
src/cache/index.ts vendored Normal file
View File

@@ -0,0 +1,4 @@
export * from './base-cache.js';
export * from './b-tree-cache.js';
export * from './kv-cache.js';

69
src/cache/kv-cache.ts vendored Normal file
View File

@@ -0,0 +1,69 @@
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
/**
* Hash-map cache implementation.
*
* - Equality: O(1) average
* - Range: unsupported (returns null -> caller falls back to scan)
*/
export class KvCache extends BaseCache {
private indexes = new Map<string, Map<string, Set<number>>>();
registerIndex(indexName: string, _fields: string[]): void {
if (!this.indexes.has(indexName)) {
this.indexes.set(indexName, new Map());
}
}
clearIndex(indexName: string): void {
this.indexes.get(indexName)?.clear();
}
insert(indexName: string, key: any, internalKey: number): void {
const index = this.indexes.get(indexName);
if (!index) return;
const keyStr = this.encodeKey(key);
const set = index.get(keyStr) ?? new Set<number>();
set.add(internalKey);
index.set(keyStr, set);
}
delete(indexName: string, key: any, internalKey: number): void {
const index = this.indexes.get(indexName);
if (!index) return;
const keyStr = this.encodeKey(key);
const set = index.get(keyStr);
if (!set) return;
set.delete(internalKey);
if (set.size === 0) {
index.delete(keyStr);
}
}
get(indexName: string, key: any): Iterable<number> {
const index = this.indexes.get(indexName);
if (!index) return [];
return index.get(this.encodeKey(key)) ?? [];
}
range(
_indexName: string,
_min?: any,
_max?: any,
_options?: CacheRangeOptions,
): Iterable<number> | null {
return null;
}
createChild(): BaseCache {
return new KvCache();
}
private encodeKey(value: any): string {
return JSON.stringify(value);
}
}

View File

@@ -19,6 +19,24 @@
* Only allow specific parameters, unless prefixed with 'x-'
*/
/**
* What I *think* this is:
* Deps: List of transports
*
* Recieving:
* 1. Create from a list of tranpsports and a lsit of items.
* 2. Starts listening on all of the transports and returns a URL (maybe a promise too? Not sure whether it should be a promise or event emitter. Maybe both?)
* 3. Once it recieves a message, decrypt it and resolve the promise with the message.
*
* Sending:
* 1. Create from a list of transports and the URL.
* 2. Encrypt the message
* 3. Send the encrypted message to all of the transports that are in both the requestURL and the list of transports.
*
* Possibilities:
* - Non-ephemeral. Keep listeners alive to allow for multiple requests
*/
import { z } from 'zod/v4';
import { PublicKey } from 'src/crypto/index.js';

View File

@@ -20,7 +20,14 @@ export type FindOptions = {
export type IndexDefinition = string[] | string[][];
/**
* MongoDB-style comparison operators for a single field value.
* MongoDB-style comparison and string operators for a single field value.
*
* - Comparison operators ($eq, $ne, $lt, $lte, $gt, $gte) are available for
* all value types.
* - String operators ($startsWith, $contains) are only available when the
* field value type extends `string`.
* - Field-level $not wraps another operator set, inverting its result:
* `{ age: { $not: { $gte: 18 } } }` matches documents where age < 18.
*/
export type ComparisonOperators<V> = {
$eq?: V;
@@ -29,7 +36,8 @@ export type ComparisonOperators<V> = {
$lte?: V;
$gt?: V;
$gte?: V;
};
$not?: ComparisonOperators<V>;
} & ([V] extends [string] ? { $startsWith?: string; $contains?: string } : {});
/**
* A filter value for a single field — either a plain value (equality shorthand)
@@ -38,7 +46,22 @@ export type ComparisonOperators<V> = {
export type FieldFilter<V> = V | ComparisonOperators<V>;
/**
* Query filter that supports both equality shorthand and comparison operators.
* Keys that represent top-level logical operators in a filter,
* as opposed to document field names.
*/
const LOGICAL_KEYS = new Set(['$and', '$or', '$not']);
/**
* Returns true when `key` is a top-level logical operator ($and, $or, $not)
* rather than a document field name.
*/
export function isLogicalKey(key: string): boolean {
return LOGICAL_KEYS.has(key);
}
/**
* Query filter that supports equality shorthand, comparison operators,
* and top-level logical operators ($and, $or, $not).
*
* @example
* // Equality shorthand
@@ -47,11 +70,20 @@ export type FieldFilter<V> = V | ComparisonOperators<V>;
* // Comparison operators
* { age: { $gte: 18, $lt: 65 } }
*
* // Combined
* { name: 'foo', age: { $gte: 18 } }
* // String operators (type-safe — only on string fields)
* { name: { $startsWith: 'A' } }
*
* // Logical operators
* { $or: [{ name: 'foo' }, { age: { $gte: 18 } }] }
* { $not: { name: 'bar' } }
* { $and: [{ age: { $gte: 18 } }, { name: { $startsWith: 'A' } }] }
*/
export type Filter<T extends Record<string, any>> = {
[K in keyof T]?: FieldFilter<T[K]>;
} & {
$and?: Filter<T>[];
$or?: Filter<T>[];
$not?: Filter<T>;
};
/**
@@ -156,7 +188,7 @@ export abstract class BaseStorage<
*/
abstract deleteMany(
filter: Filter<T>,
options: Partial<FindOptions>,
options?: Partial<FindOptions>,
): Promise<number>;
/**

View File

@@ -3,7 +3,14 @@ import { Packr } from 'msgpackr';
import { AESKey } from 'src/crypto/aes-key.js';
import { Bytes } from 'src/crypto/bytes.js';
import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
import {
BaseStorage,
type ComparisonOperators,
type FindOptions,
type Filter,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
@@ -231,12 +238,13 @@ export class EncryptedStorage<
// ---------------------------------------------------------------------------
/**
* Convert a query filter for storage. Handles both plain equality values
* and operator objects.
* Convert a query filter for storage. Handles plain equality values,
* operator objects, and top-level logical operators ($and, $or, $not).
*
* - Plaintext fields: values and operator objects pass through as-is.
* - Encrypted fields: plain values are encrypted. Operator objects throw
* because range comparisons are meaningless on ciphertext.
* because range/string comparisons are meaningless on ciphertext.
* - Logical operators: sub-filters are recursively converted.
*/
private async convertFilterForStorage(
filter: Filter<T>,
@@ -244,19 +252,37 @@ export class EncryptedStorage<
const result: Record<string, any> = {};
const entries = Object.entries(filter);
// Recursively convert logical operator sub-filters.
if (filter.$and) {
result.$and = await Promise.all(
filter.$and.map((f) => this.convertFilterForStorage(f)),
);
}
if (filter.$or) {
result.$or = await Promise.all(
filter.$or.map((f) => this.convertFilterForStorage(f)),
);
}
if (filter.$not) {
result.$not = await this.convertFilterForStorage(filter.$not);
}
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
for (const [key, value] of entries) {
// Logical operator keys are already handled above.
if (isLogicalKey(key)) continue;
if (this.plaintextKeys.has(key)) {
// Plaintext field — pass through (including operator objects).
result[key] = isOperatorObject(value)
? value
? this.formatOperatorValuesForStorage(value)
: this.formatValueForEncryption(value);
} else if (isOperatorObject(value)) {
// Encrypted field with an operator — not supported.
throw new Error(
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
`Operators ($lt, $gt, $startsWith, $contains, $not, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need operator queries on this field.`,
);
} else {
// Encrypted field with a plain equality value — encrypt it.
@@ -287,6 +313,24 @@ export class EncryptedStorage<
return result;
}
/**
* Normalize operator object values for plaintext fields so they use the
* same storage representation as document values (e.g. Date).
*/
private formatOperatorValuesForStorage(
ops: ComparisonOperators<any>,
): ComparisonOperators<any> {
const result: Record<string, any> = {};
for (const [op, value] of Object.entries(ops)) {
if (op === '$not' && isOperatorObject(value)) {
result[op] = this.formatOperatorValuesForStorage(value);
continue;
}
result[op] = this.formatValueForEncryption(value);
}
return result as ComparisonOperators<any>;
}
// ---------------------------------------------------------------------------
// Value formatting
// ---------------------------------------------------------------------------

View File

@@ -5,8 +5,9 @@ import {
type Filter,
type ComparisonOperators,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, BTreeCache } from 'src/cache/index.js';
/**
* Key prefix separator used to namespace documents within localStorage.
@@ -25,6 +26,13 @@ const MANIFEST_SUFFIX = '__keys__';
*/
const COUNTER_SUFFIX = '__next__';
/**
* Suffix for the mutation/version marker. This value increments on every
* write operation so other tabs can detect index-staleness even when the
* manifest key set itself is unchanged (e.g. updateMany).
*/
const VERSION_SUFFIX = '__version__';
/**
* Separator used when joining field names to create the index map key.
*/
@@ -42,18 +50,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
return indexes as string[][];
}
/**
* Comparator for compound index keys (arrays of raw values).
*/
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* Implementation of BaseStorage using the browser's localStorage API.
*
@@ -79,8 +75,9 @@ export class StorageLocalStorage<
static from<T extends Record<string, any>>(
prefix = 'qs',
indexes?: IndexDefinition,
cache?: BaseCache,
): StorageLocalStorage<T> {
return new StorageLocalStorage<T>(prefix, indexes);
return new StorageLocalStorage<T>(prefix, indexes, cache);
}
/**
@@ -99,16 +96,13 @@ export class StorageLocalStorage<
* valid and we skip the expensive rebuild.
*/
private lastManifestRaw: string;
private lastVersionRaw: string;
/** The normalized index definitions supplied at construction time. */
private indexDefs: string[][];
/**
* Secondary indexes backed by B+ Trees.
* Map key = index name (joined field names).
* Map value = B+ Tree mapping index keys to sets of internal document keys.
*/
private indexes: Map<string, BPlusTree<any, number>>;
/** Secondary index cache (B-Tree or KV implementation). */
private cache: BaseCache;
/** Lazily-created child storage instances. */
private children: Map<string, StorageLocalStorage<any>>;
@@ -116,20 +110,21 @@ export class StorageLocalStorage<
constructor(
private readonly prefix: string = 'qs',
indexes?: IndexDefinition,
cache?: BaseCache,
) {
super();
this.children = new Map();
this.indexDefs = normalizeIndexes(indexes);
this.indexes = new Map();
this.cache = cache ?? new BTreeCache();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
this.cache.registerIndex(name, fields);
}
// Bootstrap from localStorage.
this.lastManifestRaw = '';
this.lastVersionRaw = '';
this.manifest = new Set();
this.nextKey = 0;
this.refreshManifest();
@@ -161,8 +156,7 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
results = [];
for (const key of keys) {
@@ -262,7 +256,7 @@ export class StorageLocalStorage<
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
this.children.set(
path,
new StorageLocalStorage<C>(childPrefix, this.indexDefs),
new StorageLocalStorage<C>(childPrefix, this.indexDefs, this.cache.createChild()),
);
}
return this.children.get(path) as StorageLocalStorage<C>;
@@ -273,12 +267,22 @@ export class StorageLocalStorage<
// ---------------------------------------------------------------------------
/**
* Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
* Checks whether a document satisfies a filter.
*
* Handles top-level logical operators ($and, $or, $not) first via
* recursion, then evaluates remaining field-level conditions.
*/
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) return true;
// Top-level logical operators.
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
// Field-level conditions (skip logical operator keys).
for (const [key, value] of Object.entries(filter)) {
if (isLogicalKey(key)) continue;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
@@ -289,7 +293,8 @@ export class StorageLocalStorage<
}
/**
* Evaluate a set of comparison operators against a single field value.
* Evaluate a set of comparison / string operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
@@ -298,9 +303,37 @@ export class StorageLocalStorage<
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
if (ops.$startsWith !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
}
if (ops.$contains !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
}
// Field-level $not: invert the enclosed operator set.
if (ops.$not !== undefined) {
if (this.matchesOperators(fieldValue, ops.$not)) return false;
}
return true;
}
/**
* Determine whether candidate documents returned by index resolution still
* require full filter verification.
*/
private filterNeedsVerification(
filter: Filter<T> | undefined,
resolvedFields: string[],
): boolean {
if (!filter) return false;
const filterKeys = Object.keys(filter);
const hasLogicalOps = filterKeys.some(isLogicalKey);
return hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
@@ -311,8 +344,7 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
for (const key of keys) {
const raw = localStorage.getItem(this.docKey(key));
@@ -380,7 +412,7 @@ export class StorageLocalStorage<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.insert(indexKey, internalKey);
this.cache.insert(name, indexKey, internalKey);
}
}
@@ -391,7 +423,7 @@ export class StorageLocalStorage<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.delete(indexKey, internalKey);
this.cache.delete(name, indexKey, internalKey);
}
}
@@ -413,7 +445,6 @@ export class StorageLocalStorage<
for (const fields of this.indexDefs) {
const indexName = fields.join(INDEX_KEY_SEP);
const btree = this.indexes.get(indexName)!;
if (fields.length === 1) {
// --- Single-field index ---
@@ -423,13 +454,13 @@ export class StorageLocalStorage<
const filterValue = (filter as any)[field];
if (isOperatorObject(filterValue)) {
const keys = this.resolveOperatorViaTree(btree, filterValue);
const keys = this.resolveOperatorViaTree(indexName, filterValue);
if (keys !== null) return { keys, resolvedFields: [field] };
continue;
}
// Plain equality.
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
} else {
// --- Compound index — all fields must be plain equality ---
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
@@ -437,7 +468,7 @@ export class StorageLocalStorage<
}
const tupleKey = fields.map((f) => (filter as any)[f]);
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
}
}
@@ -447,47 +478,101 @@ export class StorageLocalStorage<
/**
* Try to resolve an operator filter against a single-field B+ Tree index.
* Returns a flat array of matching internal keys, or null if the
* operators can't be served by the tree ($ne).
* operators can't be efficiently served by the tree.
*
* Supported acceleration:
* - `$eq` → point lookup via `.get()`
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
* - `$startsWith` → converted to a range scan on the prefix
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
indexName: string,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
if (ops.$ne !== undefined) return null;
// Operators that prevent efficient index use.
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
// If $eq is combined with other operators, this path does not fully
// resolve the predicate. Let caller fall back to verification/full scan.
if (
ops.$gt !== undefined
|| ops.$gte !== undefined
|| ops.$lt !== undefined
|| ops.$lte !== undefined
|| ops.$startsWith !== undefined
) {
return null;
}
return this.cache.get(indexName, ops.$eq);
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
if (ops.$startsWith !== undefined) {
const prefix = ops.$startsWith;
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = this.cache.range(indexName, prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return entries;
}
// Extract range bounds from the remaining operators.
// If strict/non-strict variants are both provided, use the stricter bound.
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;
let upperInclusive = false;
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
if (ops.$gt !== undefined && ops.$gte !== undefined) {
if (ops.$gt > ops.$gte) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gt < ops.$gte) {
min = ops.$gte;
lowerInclusive = true;
} else {
min = ops.$gt;
lowerInclusive = false;
}
} else if (ops.$gt !== undefined) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gte !== undefined) {
min = ops.$gte;
lowerInclusive = true;
}
if (ops.$lt !== undefined && ops.$lte !== undefined) {
if (ops.$lt < ops.$lte) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lt > ops.$lte) {
max = ops.$lte;
upperInclusive = true;
} else {
max = ops.$lt;
upperInclusive = false;
}
} else if (ops.$lt !== undefined) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lte !== undefined) {
max = ops.$lte;
upperInclusive = true;
}
if (min === undefined && max === undefined) return null;
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
return this.flattenEntryKeys(entries);
}
/**
* Flatten B+ Tree range results into a flat array of internal keys.
* Uses an array instead of a Set — no hash overhead, no deduplication
* needed because each internal key only appears under one index key.
*/
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
if (min !== undefined && max !== undefined) {
if (min > max) return [];
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
}
return result;
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
}
/**
@@ -495,8 +580,9 @@ export class StorageLocalStorage<
* localStorage. Called only when a cross-tab manifest change is detected.
*/
private rebuildIndexes(): void {
for (const [, btree] of this.indexes) {
btree.clear();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
this.cache.clearIndex(name);
}
for (const key of this.manifest) {
@@ -526,6 +612,11 @@ export class StorageLocalStorage<
return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`;
}
/** Build the localStorage key used to persist the mutation version. */
private versionKey(): string {
return `${this.prefix}${KEY_SEP}${VERSION_SUFFIX}`;
}
/**
* Re-read the manifest from localStorage into memory.
* Called at the start of every public method so that cross-tab writes
@@ -536,23 +627,41 @@ export class StorageLocalStorage<
*/
private refreshManifest(): void {
const raw = localStorage.getItem(this.manifestKey()) ?? '[]';
const versionRaw = localStorage.getItem(this.versionKey()) ?? '0';
if (raw === this.lastManifestRaw) return;
if (raw === this.lastManifestRaw && versionRaw === this.lastVersionRaw) return;
this.lastManifestRaw = raw;
let parsedKeys: number[] = [];
let parsedOk = true;
try {
const keys: number[] = JSON.parse(raw);
this.manifest = new Set(keys);
parsedKeys = JSON.parse(raw);
} catch {
parsedOk = false;
this.manifest = new Set();
}
if (parsedOk) {
this.manifest = new Set(parsedKeys);
this.lastManifestRaw = raw;
}
this.lastVersionRaw = versionRaw;
// Restore the counter from localStorage.
const counterRaw = localStorage.getItem(this.counterKey());
this.nextKey = counterRaw !== null ? Number(counterRaw) : 0;
if (counterRaw !== null) {
this.nextKey = Number(counterRaw);
} else if (this.manifest.size > 0) {
let max = -1;
for (const key of this.manifest) {
if (key > max) max = key;
}
this.nextKey = max + 1;
} else {
this.nextKey = 0;
}
// Manifest changed — indexes are potentially stale.
// Manifest or version changed — indexes are potentially stale.
if (this.indexDefs.length > 0) {
this.rebuildIndexes();
}
@@ -563,11 +672,16 @@ export class StorageLocalStorage<
*/
private persistManifest(): void {
const raw = JSON.stringify([...this.manifest]);
const nextVersion = Number(localStorage.getItem(this.versionKey()) ?? '0') + 1;
const versionRaw = String(nextVersion);
localStorage.setItem(this.manifestKey(), raw);
localStorage.setItem(this.counterKey(), String(this.nextKey));
localStorage.setItem(this.versionKey(), versionRaw);
// Keep the cached raw string in sync so the next refreshManifest()
// recognises this as "our own write" and skips the rebuild.
// Keep cached values in sync so the next refreshManifest() recognises
// this as our own write and skips unnecessary rebuild work.
this.lastManifestRaw = raw;
this.lastVersionRaw = versionRaw;
}
}

View File

@@ -7,11 +7,17 @@ import { StorageMemory } from './storage-memory.js';
* All read operations will use system memory - all write operations will use the provided adapter.
*/
export class StorageMemorySynced<T extends Record<string, any> = Record<string, any>> extends BaseStorage<T> {
private isPrimed: boolean;
private primePromise: Promise<void> | null;
constructor(
private inMemoryCache: StorageMemory<T>,
private store: BaseStorage<T>,
isPrimed = false,
) {
super();
this.isPrimed = isPrimed;
this.primePromise = null;
// Hook into all write operations so that we can sync the In-Memory cache.
this.store.on('insert', async (payload) => {
@@ -44,6 +50,23 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
});
}
/**
* Ensure the in-memory cache has been initialized from the backing store.
* This is especially important for derived children, whose caches start empty.
*/
private async ensurePrimed(): Promise<void> {
if (this.isPrimed) return;
if (!this.primePromise) {
this.primePromise = (async () => {
await this.inMemoryCache.deleteMany({});
const allDocuments = await this.store.find();
await this.inMemoryCache.insertMany(allDocuments);
this.isPrimed = true;
})();
}
await this.primePromise;
}
static async create<T extends Record<string, any>>(store: BaseStorage<T>) {
// Instantiate in-memory cache and the backing store.
const inMemoryCache = new StorageMemory<T>();
@@ -58,6 +81,7 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
}
// Return our instance of this store.
memorySyncedStore.isPrimed = true;
return memorySyncedStore;
}
@@ -66,26 +90,33 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
}
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
await this.ensurePrimed();
return await this.inMemoryCache.find(filter, options);
}
async updateMany(
filter: Filter<T>,
update: Partial<T>,
options: FindOptions = {} as FindOptions
options: Partial<FindOptions> = {},
): Promise<number> {
await this.ensurePrimed();
return await this.store.updateMany(filter, update, options);
}
async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
async deleteMany(filter: Filter<T>, options: Partial<FindOptions> = {}): Promise<number> {
await this.ensurePrimed();
return await this.store.deleteMany(filter, options);
}
deriveChild<C extends Record<string, any>>(path: string): BaseStorage<C> {
const childStore = this.store.deriveChild<C>(path);
const childMemory = this.inMemoryCache.deriveChild<C>(path);
if (!(childMemory instanceof StorageMemory)) {
throw new Error('Expected derived in-memory cache to be a StorageMemory instance');
}
// Create a new synced storage for the child
return new StorageMemorySynced<C>(childMemory as StorageMemory<C>, childStore);
return new StorageMemorySynced<C>(childMemory, childStore);
}
}

View File

@@ -5,8 +5,9 @@ import {
type Filter,
type ComparisonOperators,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
import { BaseCache, BTreeCache } from 'src/cache/index.js';
/**
* Separator used when joining field names to create the index map key.
@@ -29,19 +30,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
return indexes as string[][];
}
/**
* Comparator for compound index keys (arrays of raw values).
* Compares element-by-element using native `<` / `>` operators.
*/
function tupleCompare(a: any[], b: any[]): number {
const len = Math.min(a.length, b.length);
for (let i = 0; i < len; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
return a.length - b.length;
}
/**
* Implementation of BaseStore using Memory as the storage backend.
*
@@ -55,8 +43,9 @@ export class StorageMemory<
> extends BaseStorage<T> {
static from<T extends Record<string, any>>(
indexes?: IndexDefinition,
cache?: BaseCache,
): StorageMemory<T> {
return new StorageMemory<T>(indexes);
return new StorageMemory<T>(indexes, cache);
}
/** Auto-incrementing counter used to generate internal keys. */
@@ -65,12 +54,8 @@ export class StorageMemory<
/** Primary document store keyed by an opaque internal key. */
private store: Map<number, T>;
/**
* Secondary indexes backed by B+ Trees.
* Map key = index name (joined field names).
* Map value = B+ Tree mapping index keys to sets of internal document keys.
*/
private indexes: Map<string, BPlusTree<any, number>>;
/** Secondary index cache (B-Tree or KV implementation). */
private cache: BaseCache;
/** The normalized index definitions supplied at construction time. */
private indexDefs: string[][];
@@ -78,19 +63,17 @@ export class StorageMemory<
/** Lazily-created child storage instances. */
private children: Map<string, StorageMemory<any>>;
constructor(indexes?: IndexDefinition) {
constructor(indexes?: IndexDefinition, cache?: BaseCache) {
super();
this.store = new Map();
this.children = new Map();
this.indexDefs = normalizeIndexes(indexes);
// Create a B+ Tree for each index definition.
this.indexes = new Map();
this.cache = cache ?? new BTreeCache();
for (const fields of this.indexDefs) {
const name = fields.join(INDEX_KEY_SEP);
const comparator = fields.length > 1 ? tupleCompare : undefined;
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
this.cache.registerIndex(name, fields);
}
}
@@ -192,7 +175,7 @@ export class StorageMemory<
deriveChild<C>(path: string): BaseStorage<C> {
if (!this.children.has(path)) {
this.children.set(path, new StorageMemory<C>(this.indexDefs));
this.children.set(path, new StorageMemory<C>(this.indexDefs, this.cache.createChild()));
}
return this.children.get(path) as StorageMemory<C>;
}
@@ -202,15 +185,24 @@ export class StorageMemory<
// ---------------------------------------------------------------------------
/**
* Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
* Checks whether a document satisfies a filter.
*
* Handles top-level logical operators ($and, $or, $not) first via
* recursion, then evaluates remaining field-level conditions.
*/
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) {
return true;
}
// Top-level logical operators.
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
// Field-level conditions (skip logical operator keys).
for (const [key, value] of Object.entries(filter)) {
if (isLogicalKey(key)) continue;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
@@ -221,7 +213,7 @@ export class StorageMemory<
}
/**
* Evaluate a set of comparison operators against a single field value.
* Evaluate a set of comparison / string operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
@@ -231,9 +223,37 @@ export class StorageMemory<
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
if (ops.$startsWith !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
}
if (ops.$contains !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
}
// Field-level $not: invert the enclosed operator set.
if (ops.$not !== undefined) {
if (this.matchesOperators(fieldValue, ops.$not)) return false;
}
return true;
}
/**
* Determine whether candidate documents returned by index resolution still
* require full filter verification.
*/
private filterNeedsVerification(
filter: Filter<T> | undefined,
resolvedFields: string[],
): boolean {
if (!filter) return false;
const filterKeys = Object.keys(filter);
const hasLogicalOps = filterKeys.some(isLogicalKey);
return hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
}
/**
* Collect all [internalKey, document] pairs that match a filter.
* Uses an index when possible, otherwise falls back to a full scan.
@@ -243,8 +263,7 @@ export class StorageMemory<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: Array<[number, T]> = [];
for (const key of keys) {
@@ -312,7 +331,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.insert(indexKey, internalKey);
this.cache.insert(name, indexKey, internalKey);
}
}
@@ -323,7 +342,7 @@ export class StorageMemory<
if (indexKey === null) continue;
const name = fields.join(INDEX_KEY_SEP);
this.indexes.get(name)!.delete(indexKey, internalKey);
this.cache.delete(name, indexKey, internalKey);
}
}
@@ -342,7 +361,6 @@ export class StorageMemory<
for (const fields of this.indexDefs) {
const indexName = fields.join(INDEX_KEY_SEP);
const btree = this.indexes.get(indexName)!;
if (fields.length === 1) {
// --- Single-field index ---
@@ -352,13 +370,13 @@ export class StorageMemory<
const filterValue = (filter as any)[field];
if (isOperatorObject(filterValue)) {
const keys = this.resolveOperatorViaTree(btree, filterValue);
const keys = this.resolveOperatorViaTree(indexName, filterValue);
if (keys !== null) return { keys, resolvedFields: [field] };
continue;
}
// Plain equality.
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
} else {
// --- Compound index — all fields must be plain equality ---
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
@@ -366,7 +384,7 @@ export class StorageMemory<
}
const tupleKey = fields.map((f) => (filter as any)[f]);
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
}
}
@@ -376,50 +394,102 @@ export class StorageMemory<
/**
* Try to resolve an operator filter against a single-field B+ Tree index.
* Returns a flat array of matching internal keys, or null if the
* operators can't be served by the tree ($ne).
* operators can't be efficiently served by the tree.
*
* Supported acceleration:
* - `$eq` → point lookup via `.get()`
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
* - `$startsWith` → converted to a range scan on the prefix
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
indexName: string,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
// $ne prevents efficient index use.
if (ops.$ne !== undefined) return null;
// Operators that prevent efficient index use.
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
// $eq is a point lookup.
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
// If $eq is combined with other operators, this path does not fully
// resolve the predicate. Let caller fall back to verification/full scan.
if (
ops.$gt !== undefined
|| ops.$gte !== undefined
|| ops.$lt !== undefined
|| ops.$lte !== undefined
|| ops.$startsWith !== undefined
) {
return null;
}
return this.cache.get(indexName, ops.$eq);
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
if (ops.$startsWith !== undefined) {
const prefix = ops.$startsWith;
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = this.cache.range(indexName, prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return entries;
}
// Extract range bounds from the remaining operators.
// If strict/non-strict variants are both provided, use the stricter bound.
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;
let upperInclusive = false;
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
if (ops.$gt !== undefined && ops.$gte !== undefined) {
if (ops.$gt > ops.$gte) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gt < ops.$gte) {
min = ops.$gte;
lowerInclusive = true;
} else {
min = ops.$gt;
lowerInclusive = false;
}
} else if (ops.$gt !== undefined) {
min = ops.$gt;
lowerInclusive = false;
} else if (ops.$gte !== undefined) {
min = ops.$gte;
lowerInclusive = true;
}
if (ops.$lt !== undefined && ops.$lte !== undefined) {
if (ops.$lt < ops.$lte) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lt > ops.$lte) {
max = ops.$lte;
upperInclusive = true;
} else {
max = ops.$lt;
upperInclusive = false;
}
} else if (ops.$lt !== undefined) {
max = ops.$lt;
upperInclusive = false;
} else if (ops.$lte !== undefined) {
max = ops.$lte;
upperInclusive = true;
}
if (min === undefined && max === undefined) return null;
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
return this.flattenEntryKeys(entries);
}
/**
* Flatten B+ Tree range results into a flat array of internal keys.
* Uses an array instead of a Set — no hash overhead, no deduplication
* needed because each internal key only appears under one index key.
*/
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
const result: number[] = [];
for (const entry of entries) {
for (const key of entry.values) {
result.push(key);
}
if (min !== undefined && max !== undefined) {
if (min > max) return [];
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
}
return result;
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
}
/**
@@ -436,8 +506,7 @@ export class StorageMemory<
if (resolution === null) return null;
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
const results: T[] = [];
for (const key of keys) {

View File

@@ -17,17 +17,17 @@ export class EphemeralTransport {
return response;
}
async receive<T extends ZodType>(url: string, schema: T): Promise<output<T>>;
async receive(url: string): Promise<unknown>;
async receive(url: string, schema?: ZodType): Promise<unknown> {
async receive<T extends ZodType | undefined>(url: string, schema?: T): Promise<T extends undefined ? unknown : output<T>> {
const transport = await this.transports(url);
const message = await transport.waitFor('message', () => true);
transport.disconnect();
if (schema) {
return schema.parse(message);
// TODO: Figure out how the hell to fix this assertion. It shouldnt be needed, but TS is being a bitch.
return schema.parse(message) as T extends undefined ? unknown : output<T>;
}
return message;
// TODO: Figure out how the hell to fix this assertion. It shouldnt be needed, but TS is being a bitch.
return message as T extends undefined ? unknown : output<T>;
}
}

View File

@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';
import { AESKey } from 'src/crypto/aes-key.js';
import { EncryptedStorage } from 'src/storage/encrypted-storage.js';
import { StorageMemory } from 'src/storage/storage-memory.js';
import { StorageMemorySynced } from 'src/storage/storage-memory-synced.js';
describe('storage regressions', () => {
it('does not treat $eq as fully resolved when mixed with other operators', async () => {
const storage = StorageMemory.from<{ age: number }>(['age']);
await storage.insertMany([{ age: 25 }, { age: 35 }]);
const results = await storage.find({ age: { $eq: 25, $gt: 30 } });
expect(results).toEqual([]);
});
it('normalizes conflicting lower/upper bounds to strictest constraints', async () => {
const storage = StorageMemory.from<{ age: number }>(['age']);
await storage.insertMany([{ age: 30 }, { age: 31 }, { age: 32 }]);
const lower = await storage.find({ age: { $gt: 30, $gte: 30 } });
expect(lower.map((d) => d.age).sort((a, b) => a - b)).toEqual([31, 32]);
const upper = await storage.find({ age: { $lt: 32, $lte: 32 } });
expect(upper.map((d) => d.age).sort((a, b) => a - b)).toEqual([30, 31]);
});
it('formats plaintext operator values in encrypted filters', async () => {
type Doc = { createdAt: Date; name: string };
const key = await AESKey.fromSeed('storage-regression-test-key');
const base = StorageMemory.from<Record<string, any>>(['createdAt']);
const storage = EncryptedStorage.from<Doc>(base, key, { plaintextKeys: ['createdAt'] });
await storage.insertOne({
createdAt: new Date('2024-01-02T00:00:00.000Z'),
name: 'alice',
});
const results = await storage.find({
createdAt: { $gte: new Date('2024-01-01T00:00:00.000Z') },
});
expect(results).toHaveLength(1);
expect(results[0].name).toBe('alice');
});
it('primes derived child cache in StorageMemorySynced', async () => {
const store = StorageMemory.from<Record<string, any>>();
const child = store.deriveChild<{ value: number }>('child');
await child.insertOne({ value: 42 });
const synced = await StorageMemorySynced.create(store);
const syncedChild = synced.deriveChild<{ value: number }>('child');
const results = await syncedChild.find();
expect(results).toEqual([{ value: 42 }]);
});
});

View File

@@ -1,5 +1,5 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { BPlusTree } from './btree.js';
import { BPlusTree } from '../../src/utils/btree.js';
describe('BPlusTree', () => {
let tree: BPlusTree<number, string>;
@@ -181,6 +181,7 @@ describe('BPlusTree', () => {
it('should handle range on empty tree', () => {
expect(tree.range()).toEqual([]);
expect(tree.range(1, 2)).toEqual([]);
});
it('should handle delete on empty tree', () => {