Compare commits
5 Commits
64b811f330
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
01bc9a8d15
|
|||
|
ddcd638937
|
|||
|
044e516ed3
|
|||
|
5c39d8add2
|
|||
|
f7c89046d1
|
@@ -1,5 +1,6 @@
|
|||||||
import { AESKey } from '../src/crypto/aes-key.js';
|
import { AESKey } from '../src/crypto/aes-key.js';
|
||||||
import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js';
|
import { StorageMemory, EncryptedStorage, type BaseStorage } from '../src/storage/index.js';
|
||||||
|
import { BTreeCache, KvCache } from '../src/cache/index.js';
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Helpers
|
// Helpers
|
||||||
@@ -161,9 +162,19 @@ const DOC_COUNTS = [1_000, 10_000, 50_000];
|
|||||||
for (const count of DOC_COUNTS) {
|
for (const count of DOC_COUNTS) {
|
||||||
const docs = generateDocs(count);
|
const docs = generateDocs(count);
|
||||||
|
|
||||||
// Indexes on id, name, AND age — range queries on age use B+ Tree.
|
// Indexes on id, name, AND age with explicit B+ Tree cache.
|
||||||
const indexedWithAge = StorageMemory.from<Doc>(['id', 'name', 'age']);
|
const indexedWithAgeBTree = StorageMemory.from<Doc>(
|
||||||
await benchmarkStorage('StorageMemory (indexed: id,name,age)', indexedWithAge, docs, { hasAgeIndex: true });
|
['id', 'name', 'age'],
|
||||||
|
new BTreeCache(),
|
||||||
|
);
|
||||||
|
await benchmarkStorage('StorageMemory (B+Tree cache, indexed: id,name,age)', indexedWithAgeBTree, docs, { hasAgeIndex: true });
|
||||||
|
|
||||||
|
// Same indexes, but KV cache (no range support).
|
||||||
|
const indexedWithAgeKv = StorageMemory.from<Doc>(
|
||||||
|
['id', 'name', 'age'],
|
||||||
|
new KvCache(),
|
||||||
|
);
|
||||||
|
await benchmarkStorage('StorageMemory (KV cache, indexed: id,name,age)', indexedWithAgeKv, docs);
|
||||||
|
|
||||||
// Indexes on id, name only — range queries on age fall back to full scan.
|
// Indexes on id, name only — range queries on age fall back to full scan.
|
||||||
const indexed = StorageMemory.from<Doc>(['id', 'name']);
|
const indexed = StorageMemory.from<Doc>(['id', 'name']);
|
||||||
|
|||||||
@@ -6,7 +6,9 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"format": "prettier --write .",
|
"format": "prettier --write .",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1",
|
|
||||||
|
"test": "vitest --run",
|
||||||
|
"test:watch": "vitest",
|
||||||
|
|
||||||
"benchmark:sha256": "tsx benchmarks/sha256.ts",
|
"benchmark:sha256": "tsx benchmarks/sha256.ts",
|
||||||
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",
|
"benchmark:diffie-helman": "tsx benchmarks/diffie-helman.ts",
|
||||||
|
|||||||
79
src/cache/b-tree-cache.ts
vendored
Normal file
79
src/cache/b-tree-cache.ts
vendored
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
|
||||||
|
|
||||||
|
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
|
||||||
|
|
||||||
|
function tupleCompare(a: any[], b: any[]): number {
|
||||||
|
const len = Math.min(a.length, b.length);
|
||||||
|
for (let i = 0; i < len; i++) {
|
||||||
|
if (a[i] < b[i]) return -1;
|
||||||
|
if (a[i] > b[i]) return 1;
|
||||||
|
}
|
||||||
|
return a.length - b.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* B+ tree-backed cache implementation.
|
||||||
|
*
|
||||||
|
* - Equality: O(log n)
|
||||||
|
* - Range: O(log n + k)
|
||||||
|
*/
|
||||||
|
export class BTreeCache extends BaseCache {
|
||||||
|
private trees = new Map<string, BPlusTree<any, number>>();
|
||||||
|
|
||||||
|
registerIndex(indexName: string, fields: string[]): void {
|
||||||
|
if (this.trees.has(indexName)) return;
|
||||||
|
const comparator = fields.length > 1 ? tupleCompare : undefined;
|
||||||
|
this.trees.set(indexName, new BPlusTree<any, number>(32, comparator));
|
||||||
|
}
|
||||||
|
|
||||||
|
clearIndex(indexName: string): void {
|
||||||
|
const tree = this.trees.get(indexName);
|
||||||
|
if (!tree) return;
|
||||||
|
tree.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
insert(indexName: string, key: any, internalKey: number): void {
|
||||||
|
const tree = this.trees.get(indexName);
|
||||||
|
if (!tree) return;
|
||||||
|
tree.insert(key, internalKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(indexName: string, key: any, internalKey: number): void {
|
||||||
|
const tree = this.trees.get(indexName);
|
||||||
|
if (!tree) return;
|
||||||
|
tree.delete(key, internalKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
get(indexName: string, key: any): Iterable<number> {
|
||||||
|
const tree = this.trees.get(indexName);
|
||||||
|
if (!tree) return [];
|
||||||
|
return tree.get(key) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
range(
|
||||||
|
indexName: string,
|
||||||
|
min?: any,
|
||||||
|
max?: any,
|
||||||
|
options: CacheRangeOptions = {},
|
||||||
|
): Iterable<number> | null {
|
||||||
|
const tree = this.trees.get(indexName);
|
||||||
|
if (!tree) return [];
|
||||||
|
const entries = tree.range(min, max, options);
|
||||||
|
return this.flattenEntryKeys(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
createChild(): BaseCache {
|
||||||
|
return new BTreeCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
|
||||||
|
const result: number[] = [];
|
||||||
|
for (const entry of entries) {
|
||||||
|
for (const key of entry.values) {
|
||||||
|
result.push(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
37
src/cache/base-cache.ts
vendored
Normal file
37
src/cache/base-cache.ts
vendored
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
export type CacheRangeOptions = {
|
||||||
|
lowerInclusive?: boolean;
|
||||||
|
upperInclusive?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared cache abstraction used by storage adapters for secondary indexes.
|
||||||
|
*
|
||||||
|
* Implementations may support only equality (`get`) or both equality and
|
||||||
|
* range queries (`range`). Returning `null` from `range` indicates the cache
|
||||||
|
* cannot serve that range query efficiently.
|
||||||
|
*/
|
||||||
|
export abstract class BaseCache {
|
||||||
|
abstract registerIndex(indexName: string, fields: string[]): void;
|
||||||
|
|
||||||
|
abstract clearIndex(indexName: string): void;
|
||||||
|
|
||||||
|
abstract insert(indexName: string, key: any, internalKey: number): void;
|
||||||
|
|
||||||
|
abstract delete(indexName: string, key: any, internalKey: number): void;
|
||||||
|
|
||||||
|
abstract get(indexName: string, key: any): Iterable<number>;
|
||||||
|
|
||||||
|
abstract range(
|
||||||
|
indexName: string,
|
||||||
|
min?: any,
|
||||||
|
max?: any,
|
||||||
|
options?: CacheRangeOptions,
|
||||||
|
): Iterable<number> | null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new empty cache instance of the same concrete type.
|
||||||
|
* Used by deriveChild() to preserve cache strategy across children.
|
||||||
|
*/
|
||||||
|
abstract createChild(): BaseCache;
|
||||||
|
}
|
||||||
|
|
||||||
4
src/cache/index.ts
vendored
Normal file
4
src/cache/index.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
export * from './base-cache.js';
|
||||||
|
export * from './b-tree-cache.js';
|
||||||
|
export * from './kv-cache.js';
|
||||||
|
|
||||||
69
src/cache/kv-cache.ts
vendored
Normal file
69
src/cache/kv-cache.ts
vendored
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
import { BaseCache, type CacheRangeOptions } from './base-cache.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hash-map cache implementation.
|
||||||
|
*
|
||||||
|
* - Equality: O(1) average
|
||||||
|
* - Range: unsupported (returns null -> caller falls back to scan)
|
||||||
|
*/
|
||||||
|
export class KvCache extends BaseCache {
|
||||||
|
private indexes = new Map<string, Map<string, Set<number>>>();
|
||||||
|
|
||||||
|
registerIndex(indexName: string, _fields: string[]): void {
|
||||||
|
if (!this.indexes.has(indexName)) {
|
||||||
|
this.indexes.set(indexName, new Map());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
clearIndex(indexName: string): void {
|
||||||
|
this.indexes.get(indexName)?.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
insert(indexName: string, key: any, internalKey: number): void {
|
||||||
|
const index = this.indexes.get(indexName);
|
||||||
|
if (!index) return;
|
||||||
|
|
||||||
|
const keyStr = this.encodeKey(key);
|
||||||
|
const set = index.get(keyStr) ?? new Set<number>();
|
||||||
|
set.add(internalKey);
|
||||||
|
index.set(keyStr, set);
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(indexName: string, key: any, internalKey: number): void {
|
||||||
|
const index = this.indexes.get(indexName);
|
||||||
|
if (!index) return;
|
||||||
|
|
||||||
|
const keyStr = this.encodeKey(key);
|
||||||
|
const set = index.get(keyStr);
|
||||||
|
if (!set) return;
|
||||||
|
|
||||||
|
set.delete(internalKey);
|
||||||
|
if (set.size === 0) {
|
||||||
|
index.delete(keyStr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
get(indexName: string, key: any): Iterable<number> {
|
||||||
|
const index = this.indexes.get(indexName);
|
||||||
|
if (!index) return [];
|
||||||
|
return index.get(this.encodeKey(key)) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
range(
|
||||||
|
_indexName: string,
|
||||||
|
_min?: any,
|
||||||
|
_max?: any,
|
||||||
|
_options?: CacheRangeOptions,
|
||||||
|
): Iterable<number> | null {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
createChild(): BaseCache {
|
||||||
|
return new KvCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
private encodeKey(value: any): string {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -19,6 +19,24 @@
|
|||||||
* Only allow specific parameters, unless prefixed with 'x-'
|
* Only allow specific parameters, unless prefixed with 'x-'
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What I *think* this is:
|
||||||
|
* Deps: List of transports
|
||||||
|
*
|
||||||
|
* Recieving:
|
||||||
|
* 1. Create from a list of tranpsports and a lsit of items.
|
||||||
|
* 2. Starts listening on all of the transports and returns a URL (maybe a promise too? Not sure whether it should be a promise or event emitter. Maybe both?)
|
||||||
|
* 3. Once it recieves a message, decrypt it and resolve the promise with the message.
|
||||||
|
*
|
||||||
|
* Sending:
|
||||||
|
* 1. Create from a list of transports and the URL.
|
||||||
|
* 2. Encrypt the message
|
||||||
|
* 3. Send the encrypted message to all of the transports that are in both the requestURL and the list of transports.
|
||||||
|
*
|
||||||
|
* Possibilities:
|
||||||
|
* - Non-ephemeral. Keep listeners alive to allow for multiple requests
|
||||||
|
*/
|
||||||
|
|
||||||
import { z } from 'zod/v4';
|
import { z } from 'zod/v4';
|
||||||
|
|
||||||
import { PublicKey } from 'src/crypto/index.js';
|
import { PublicKey } from 'src/crypto/index.js';
|
||||||
|
|||||||
@@ -20,7 +20,14 @@ export type FindOptions = {
|
|||||||
export type IndexDefinition = string[] | string[][];
|
export type IndexDefinition = string[] | string[][];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* MongoDB-style comparison operators for a single field value.
|
* MongoDB-style comparison and string operators for a single field value.
|
||||||
|
*
|
||||||
|
* - Comparison operators ($eq, $ne, $lt, $lte, $gt, $gte) are available for
|
||||||
|
* all value types.
|
||||||
|
* - String operators ($startsWith, $contains) are only available when the
|
||||||
|
* field value type extends `string`.
|
||||||
|
* - Field-level $not wraps another operator set, inverting its result:
|
||||||
|
* `{ age: { $not: { $gte: 18 } } }` matches documents where age < 18.
|
||||||
*/
|
*/
|
||||||
export type ComparisonOperators<V> = {
|
export type ComparisonOperators<V> = {
|
||||||
$eq?: V;
|
$eq?: V;
|
||||||
@@ -29,7 +36,8 @@ export type ComparisonOperators<V> = {
|
|||||||
$lte?: V;
|
$lte?: V;
|
||||||
$gt?: V;
|
$gt?: V;
|
||||||
$gte?: V;
|
$gte?: V;
|
||||||
};
|
$not?: ComparisonOperators<V>;
|
||||||
|
} & ([V] extends [string] ? { $startsWith?: string; $contains?: string } : {});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A filter value for a single field — either a plain value (equality shorthand)
|
* A filter value for a single field — either a plain value (equality shorthand)
|
||||||
@@ -38,7 +46,22 @@ export type ComparisonOperators<V> = {
|
|||||||
export type FieldFilter<V> = V | ComparisonOperators<V>;
|
export type FieldFilter<V> = V | ComparisonOperators<V>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Query filter that supports both equality shorthand and comparison operators.
|
* Keys that represent top-level logical operators in a filter,
|
||||||
|
* as opposed to document field names.
|
||||||
|
*/
|
||||||
|
const LOGICAL_KEYS = new Set(['$and', '$or', '$not']);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true when `key` is a top-level logical operator ($and, $or, $not)
|
||||||
|
* rather than a document field name.
|
||||||
|
*/
|
||||||
|
export function isLogicalKey(key: string): boolean {
|
||||||
|
return LOGICAL_KEYS.has(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query filter that supports equality shorthand, comparison operators,
|
||||||
|
* and top-level logical operators ($and, $or, $not).
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* // Equality shorthand
|
* // Equality shorthand
|
||||||
@@ -47,11 +70,20 @@ export type FieldFilter<V> = V | ComparisonOperators<V>;
|
|||||||
* // Comparison operators
|
* // Comparison operators
|
||||||
* { age: { $gte: 18, $lt: 65 } }
|
* { age: { $gte: 18, $lt: 65 } }
|
||||||
*
|
*
|
||||||
* // Combined
|
* // String operators (type-safe — only on string fields)
|
||||||
* { name: 'foo', age: { $gte: 18 } }
|
* { name: { $startsWith: 'A' } }
|
||||||
|
*
|
||||||
|
* // Logical operators
|
||||||
|
* { $or: [{ name: 'foo' }, { age: { $gte: 18 } }] }
|
||||||
|
* { $not: { name: 'bar' } }
|
||||||
|
* { $and: [{ age: { $gte: 18 } }, { name: { $startsWith: 'A' } }] }
|
||||||
*/
|
*/
|
||||||
export type Filter<T extends Record<string, any>> = {
|
export type Filter<T extends Record<string, any>> = {
|
||||||
[K in keyof T]?: FieldFilter<T[K]>;
|
[K in keyof T]?: FieldFilter<T[K]>;
|
||||||
|
} & {
|
||||||
|
$and?: Filter<T>[];
|
||||||
|
$or?: Filter<T>[];
|
||||||
|
$not?: Filter<T>;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -156,7 +188,7 @@ export abstract class BaseStorage<
|
|||||||
*/
|
*/
|
||||||
abstract deleteMany(
|
abstract deleteMany(
|
||||||
filter: Filter<T>,
|
filter: Filter<T>,
|
||||||
options: Partial<FindOptions>,
|
options?: Partial<FindOptions>,
|
||||||
): Promise<number>;
|
): Promise<number>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -3,7 +3,14 @@ import { Packr } from 'msgpackr';
|
|||||||
import { AESKey } from 'src/crypto/aes-key.js';
|
import { AESKey } from 'src/crypto/aes-key.js';
|
||||||
import { Bytes } from 'src/crypto/bytes.js';
|
import { Bytes } from 'src/crypto/bytes.js';
|
||||||
|
|
||||||
import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
|
import {
|
||||||
|
BaseStorage,
|
||||||
|
type ComparisonOperators,
|
||||||
|
type FindOptions,
|
||||||
|
type Filter,
|
||||||
|
isOperatorObject,
|
||||||
|
isLogicalKey,
|
||||||
|
} from './base-storage.js';
|
||||||
|
|
||||||
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
|
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
|
||||||
|
|
||||||
@@ -231,12 +238,13 @@ export class EncryptedStorage<
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Convert a query filter for storage. Handles both plain equality values
|
* Convert a query filter for storage. Handles plain equality values,
|
||||||
* and operator objects.
|
* operator objects, and top-level logical operators ($and, $or, $not).
|
||||||
*
|
*
|
||||||
* - Plaintext fields: values and operator objects pass through as-is.
|
* - Plaintext fields: values and operator objects pass through as-is.
|
||||||
* - Encrypted fields: plain values are encrypted. Operator objects throw
|
* - Encrypted fields: plain values are encrypted. Operator objects throw
|
||||||
* because range comparisons are meaningless on ciphertext.
|
* because range/string comparisons are meaningless on ciphertext.
|
||||||
|
* - Logical operators: sub-filters are recursively converted.
|
||||||
*/
|
*/
|
||||||
private async convertFilterForStorage(
|
private async convertFilterForStorage(
|
||||||
filter: Filter<T>,
|
filter: Filter<T>,
|
||||||
@@ -244,19 +252,37 @@ export class EncryptedStorage<
|
|||||||
const result: Record<string, any> = {};
|
const result: Record<string, any> = {};
|
||||||
const entries = Object.entries(filter);
|
const entries = Object.entries(filter);
|
||||||
|
|
||||||
|
// Recursively convert logical operator sub-filters.
|
||||||
|
if (filter.$and) {
|
||||||
|
result.$and = await Promise.all(
|
||||||
|
filter.$and.map((f) => this.convertFilterForStorage(f)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (filter.$or) {
|
||||||
|
result.$or = await Promise.all(
|
||||||
|
filter.$or.map((f) => this.convertFilterForStorage(f)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (filter.$not) {
|
||||||
|
result.$not = await this.convertFilterForStorage(filter.$not);
|
||||||
|
}
|
||||||
|
|
||||||
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
|
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
|
||||||
|
|
||||||
for (const [key, value] of entries) {
|
for (const [key, value] of entries) {
|
||||||
|
// Logical operator keys are already handled above.
|
||||||
|
if (isLogicalKey(key)) continue;
|
||||||
|
|
||||||
if (this.plaintextKeys.has(key)) {
|
if (this.plaintextKeys.has(key)) {
|
||||||
// Plaintext field — pass through (including operator objects).
|
// Plaintext field — pass through (including operator objects).
|
||||||
result[key] = isOperatorObject(value)
|
result[key] = isOperatorObject(value)
|
||||||
? value
|
? this.formatOperatorValuesForStorage(value)
|
||||||
: this.formatValueForEncryption(value);
|
: this.formatValueForEncryption(value);
|
||||||
} else if (isOperatorObject(value)) {
|
} else if (isOperatorObject(value)) {
|
||||||
// Encrypted field with an operator — not supported.
|
// Encrypted field with an operator — not supported.
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
|
`Operators ($lt, $gt, $startsWith, $contains, $not, etc.) cannot be used on encrypted field '${key}'. ` +
|
||||||
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
|
`Add '${key}' to plaintextKeys if you need operator queries on this field.`,
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
// Encrypted field with a plain equality value — encrypt it.
|
// Encrypted field with a plain equality value — encrypt it.
|
||||||
@@ -287,6 +313,24 @@ export class EncryptedStorage<
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize operator object values for plaintext fields so they use the
|
||||||
|
* same storage representation as document values (e.g. Date).
|
||||||
|
*/
|
||||||
|
private formatOperatorValuesForStorage(
|
||||||
|
ops: ComparisonOperators<any>,
|
||||||
|
): ComparisonOperators<any> {
|
||||||
|
const result: Record<string, any> = {};
|
||||||
|
for (const [op, value] of Object.entries(ops)) {
|
||||||
|
if (op === '$not' && isOperatorObject(value)) {
|
||||||
|
result[op] = this.formatOperatorValuesForStorage(value);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result[op] = this.formatValueForEncryption(value);
|
||||||
|
}
|
||||||
|
return result as ComparisonOperators<any>;
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Value formatting
|
// Value formatting
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -5,8 +5,9 @@ import {
|
|||||||
type Filter,
|
type Filter,
|
||||||
type ComparisonOperators,
|
type ComparisonOperators,
|
||||||
isOperatorObject,
|
isOperatorObject,
|
||||||
|
isLogicalKey,
|
||||||
} from './base-storage.js';
|
} from './base-storage.js';
|
||||||
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
|
import { BaseCache, BTreeCache } from 'src/cache/index.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Key prefix separator used to namespace documents within localStorage.
|
* Key prefix separator used to namespace documents within localStorage.
|
||||||
@@ -25,6 +26,13 @@ const MANIFEST_SUFFIX = '__keys__';
|
|||||||
*/
|
*/
|
||||||
const COUNTER_SUFFIX = '__next__';
|
const COUNTER_SUFFIX = '__next__';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Suffix for the mutation/version marker. This value increments on every
|
||||||
|
* write operation so other tabs can detect index-staleness even when the
|
||||||
|
* manifest key set itself is unchanged (e.g. updateMany).
|
||||||
|
*/
|
||||||
|
const VERSION_SUFFIX = '__version__';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Separator used when joining field names to create the index map key.
|
* Separator used when joining field names to create the index map key.
|
||||||
*/
|
*/
|
||||||
@@ -42,18 +50,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
|
|||||||
return indexes as string[][];
|
return indexes as string[][];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Comparator for compound index keys (arrays of raw values).
|
|
||||||
*/
|
|
||||||
function tupleCompare(a: any[], b: any[]): number {
|
|
||||||
const len = Math.min(a.length, b.length);
|
|
||||||
for (let i = 0; i < len; i++) {
|
|
||||||
if (a[i] < b[i]) return -1;
|
|
||||||
if (a[i] > b[i]) return 1;
|
|
||||||
}
|
|
||||||
return a.length - b.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implementation of BaseStorage using the browser's localStorage API.
|
* Implementation of BaseStorage using the browser's localStorage API.
|
||||||
*
|
*
|
||||||
@@ -79,8 +75,9 @@ export class StorageLocalStorage<
|
|||||||
static from<T extends Record<string, any>>(
|
static from<T extends Record<string, any>>(
|
||||||
prefix = 'qs',
|
prefix = 'qs',
|
||||||
indexes?: IndexDefinition,
|
indexes?: IndexDefinition,
|
||||||
|
cache?: BaseCache,
|
||||||
): StorageLocalStorage<T> {
|
): StorageLocalStorage<T> {
|
||||||
return new StorageLocalStorage<T>(prefix, indexes);
|
return new StorageLocalStorage<T>(prefix, indexes, cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -99,16 +96,13 @@ export class StorageLocalStorage<
|
|||||||
* valid and we skip the expensive rebuild.
|
* valid and we skip the expensive rebuild.
|
||||||
*/
|
*/
|
||||||
private lastManifestRaw: string;
|
private lastManifestRaw: string;
|
||||||
|
private lastVersionRaw: string;
|
||||||
|
|
||||||
/** The normalized index definitions supplied at construction time. */
|
/** The normalized index definitions supplied at construction time. */
|
||||||
private indexDefs: string[][];
|
private indexDefs: string[][];
|
||||||
|
|
||||||
/**
|
/** Secondary index cache (B-Tree or KV implementation). */
|
||||||
* Secondary indexes backed by B+ Trees.
|
private cache: BaseCache;
|
||||||
* Map key = index name (joined field names).
|
|
||||||
* Map value = B+ Tree mapping index keys to sets of internal document keys.
|
|
||||||
*/
|
|
||||||
private indexes: Map<string, BPlusTree<any, number>>;
|
|
||||||
|
|
||||||
/** Lazily-created child storage instances. */
|
/** Lazily-created child storage instances. */
|
||||||
private children: Map<string, StorageLocalStorage<any>>;
|
private children: Map<string, StorageLocalStorage<any>>;
|
||||||
@@ -116,20 +110,21 @@ export class StorageLocalStorage<
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly prefix: string = 'qs',
|
private readonly prefix: string = 'qs',
|
||||||
indexes?: IndexDefinition,
|
indexes?: IndexDefinition,
|
||||||
|
cache?: BaseCache,
|
||||||
) {
|
) {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
this.children = new Map();
|
this.children = new Map();
|
||||||
this.indexDefs = normalizeIndexes(indexes);
|
this.indexDefs = normalizeIndexes(indexes);
|
||||||
this.indexes = new Map();
|
this.cache = cache ?? new BTreeCache();
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
const comparator = fields.length > 1 ? tupleCompare : undefined;
|
this.cache.registerIndex(name, fields);
|
||||||
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bootstrap from localStorage.
|
// Bootstrap from localStorage.
|
||||||
this.lastManifestRaw = '';
|
this.lastManifestRaw = '';
|
||||||
|
this.lastVersionRaw = '';
|
||||||
this.manifest = new Set();
|
this.manifest = new Set();
|
||||||
this.nextKey = 0;
|
this.nextKey = 0;
|
||||||
this.refreshManifest();
|
this.refreshManifest();
|
||||||
@@ -161,8 +156,7 @@ export class StorageLocalStorage<
|
|||||||
|
|
||||||
if (resolution !== null) {
|
if (resolution !== null) {
|
||||||
const { keys, resolvedFields } = resolution;
|
const { keys, resolvedFields } = resolution;
|
||||||
const filterKeys = filter ? Object.keys(filter) : [];
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
||||||
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
|
|
||||||
|
|
||||||
results = [];
|
results = [];
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
@@ -262,7 +256,7 @@ export class StorageLocalStorage<
|
|||||||
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
|
const childPrefix = `${this.prefix}${KEY_SEP}${path}`;
|
||||||
this.children.set(
|
this.children.set(
|
||||||
path,
|
path,
|
||||||
new StorageLocalStorage<C>(childPrefix, this.indexDefs),
|
new StorageLocalStorage<C>(childPrefix, this.indexDefs, this.cache.createChild()),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return this.children.get(path) as StorageLocalStorage<C>;
|
return this.children.get(path) as StorageLocalStorage<C>;
|
||||||
@@ -273,12 +267,22 @@ export class StorageLocalStorage<
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks whether a document satisfies every field in the filter.
|
* Checks whether a document satisfies a filter.
|
||||||
* Supports both plain equality values and comparison operator objects.
|
*
|
||||||
|
* Handles top-level logical operators ($and, $or, $not) first via
|
||||||
|
* recursion, then evaluates remaining field-level conditions.
|
||||||
*/
|
*/
|
||||||
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) return true;
|
if (!filter || Object.keys(filter).length === 0) return true;
|
||||||
|
|
||||||
|
// Top-level logical operators.
|
||||||
|
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
|
||||||
|
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
|
||||||
|
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
|
||||||
|
|
||||||
|
// Field-level conditions (skip logical operator keys).
|
||||||
for (const [key, value] of Object.entries(filter)) {
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
|
if (isLogicalKey(key)) continue;
|
||||||
if (isOperatorObject(value)) {
|
if (isOperatorObject(value)) {
|
||||||
if (!this.matchesOperators(item[key], value)) return false;
|
if (!this.matchesOperators(item[key], value)) return false;
|
||||||
} else {
|
} else {
|
||||||
@@ -289,7 +293,8 @@ export class StorageLocalStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Evaluate a set of comparison operators against a single field value.
|
* Evaluate a set of comparison / string operators against a single field value.
|
||||||
|
* All operators must pass for the field to match.
|
||||||
*/
|
*/
|
||||||
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
||||||
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
|
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
|
||||||
@@ -298,9 +303,37 @@ export class StorageLocalStorage<
|
|||||||
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
||||||
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
||||||
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
||||||
|
|
||||||
|
if (ops.$startsWith !== undefined) {
|
||||||
|
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
|
||||||
|
}
|
||||||
|
if (ops.$contains !== undefined) {
|
||||||
|
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Field-level $not: invert the enclosed operator set.
|
||||||
|
if (ops.$not !== undefined) {
|
||||||
|
if (this.matchesOperators(fieldValue, ops.$not)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether candidate documents returned by index resolution still
|
||||||
|
* require full filter verification.
|
||||||
|
*/
|
||||||
|
private filterNeedsVerification(
|
||||||
|
filter: Filter<T> | undefined,
|
||||||
|
resolvedFields: string[],
|
||||||
|
): boolean {
|
||||||
|
if (!filter) return false;
|
||||||
|
const filterKeys = Object.keys(filter);
|
||||||
|
const hasLogicalOps = filterKeys.some(isLogicalKey);
|
||||||
|
return hasLogicalOps
|
||||||
|
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collect all [internalKey, document] pairs that match a filter.
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
* Uses an index when possible, otherwise falls back to a full scan.
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
@@ -311,8 +344,7 @@ export class StorageLocalStorage<
|
|||||||
|
|
||||||
if (resolution !== null) {
|
if (resolution !== null) {
|
||||||
const { keys, resolvedFields } = resolution;
|
const { keys, resolvedFields } = resolution;
|
||||||
const filterKeys = filter ? Object.keys(filter) : [];
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
||||||
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
|
|
||||||
|
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
const raw = localStorage.getItem(this.docKey(key));
|
const raw = localStorage.getItem(this.docKey(key));
|
||||||
@@ -380,7 +412,7 @@ export class StorageLocalStorage<
|
|||||||
if (indexKey === null) continue;
|
if (indexKey === null) continue;
|
||||||
|
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
this.indexes.get(name)!.insert(indexKey, internalKey);
|
this.cache.insert(name, indexKey, internalKey);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -391,7 +423,7 @@ export class StorageLocalStorage<
|
|||||||
if (indexKey === null) continue;
|
if (indexKey === null) continue;
|
||||||
|
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
this.indexes.get(name)!.delete(indexKey, internalKey);
|
this.cache.delete(name, indexKey, internalKey);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,7 +445,6 @@ export class StorageLocalStorage<
|
|||||||
|
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
const indexName = fields.join(INDEX_KEY_SEP);
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
const btree = this.indexes.get(indexName)!;
|
|
||||||
|
|
||||||
if (fields.length === 1) {
|
if (fields.length === 1) {
|
||||||
// --- Single-field index ---
|
// --- Single-field index ---
|
||||||
@@ -423,13 +454,13 @@ export class StorageLocalStorage<
|
|||||||
const filterValue = (filter as any)[field];
|
const filterValue = (filter as any)[field];
|
||||||
|
|
||||||
if (isOperatorObject(filterValue)) {
|
if (isOperatorObject(filterValue)) {
|
||||||
const keys = this.resolveOperatorViaTree(btree, filterValue);
|
const keys = this.resolveOperatorViaTree(indexName, filterValue);
|
||||||
if (keys !== null) return { keys, resolvedFields: [field] };
|
if (keys !== null) return { keys, resolvedFields: [field] };
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Plain equality.
|
// Plain equality.
|
||||||
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
|
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
|
||||||
} else {
|
} else {
|
||||||
// --- Compound index — all fields must be plain equality ---
|
// --- Compound index — all fields must be plain equality ---
|
||||||
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
|
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
|
||||||
@@ -437,7 +468,7 @@ export class StorageLocalStorage<
|
|||||||
}
|
}
|
||||||
|
|
||||||
const tupleKey = fields.map((f) => (filter as any)[f]);
|
const tupleKey = fields.map((f) => (filter as any)[f]);
|
||||||
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
|
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -447,47 +478,101 @@ export class StorageLocalStorage<
|
|||||||
/**
|
/**
|
||||||
* Try to resolve an operator filter against a single-field B+ Tree index.
|
* Try to resolve an operator filter against a single-field B+ Tree index.
|
||||||
* Returns a flat array of matching internal keys, or null if the
|
* Returns a flat array of matching internal keys, or null if the
|
||||||
* operators can't be served by the tree ($ne).
|
* operators can't be efficiently served by the tree.
|
||||||
|
*
|
||||||
|
* Supported acceleration:
|
||||||
|
* - `$eq` → point lookup via `.get()`
|
||||||
|
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
|
||||||
|
* - `$startsWith` → converted to a range scan on the prefix
|
||||||
|
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
|
||||||
*/
|
*/
|
||||||
private resolveOperatorViaTree(
|
private resolveOperatorViaTree(
|
||||||
btree: BPlusTree<any, number>,
|
indexName: string,
|
||||||
ops: ComparisonOperators<any>,
|
ops: ComparisonOperators<any>,
|
||||||
): Iterable<number> | null {
|
): Iterable<number> | null {
|
||||||
if (ops.$ne !== undefined) return null;
|
// Operators that prevent efficient index use.
|
||||||
|
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
|
||||||
|
|
||||||
if (ops.$eq !== undefined) {
|
if (ops.$eq !== undefined) {
|
||||||
return btree.get(ops.$eq) ?? [];
|
// If $eq is combined with other operators, this path does not fully
|
||||||
|
// resolve the predicate. Let caller fall back to verification/full scan.
|
||||||
|
if (
|
||||||
|
ops.$gt !== undefined
|
||||||
|
|| ops.$gte !== undefined
|
||||||
|
|| ops.$lt !== undefined
|
||||||
|
|| ops.$lte !== undefined
|
||||||
|
|| ops.$startsWith !== undefined
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return this.cache.get(indexName, ops.$eq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
|
||||||
|
if (ops.$startsWith !== undefined) {
|
||||||
|
const prefix = ops.$startsWith;
|
||||||
|
if (prefix.length === 0) return null;
|
||||||
|
const upper = prefix.slice(0, -1)
|
||||||
|
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
|
||||||
|
const entries = this.cache.range(indexName, prefix, upper, {
|
||||||
|
lowerInclusive: true,
|
||||||
|
upperInclusive: false,
|
||||||
|
});
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract range bounds from the remaining operators.
|
||||||
|
// If strict/non-strict variants are both provided, use the stricter bound.
|
||||||
let min: any = undefined;
|
let min: any = undefined;
|
||||||
let max: any = undefined;
|
let max: any = undefined;
|
||||||
let lowerInclusive = true;
|
let lowerInclusive = true;
|
||||||
let upperInclusive = false;
|
let upperInclusive = false;
|
||||||
|
|
||||||
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
|
if (ops.$gt !== undefined && ops.$gte !== undefined) {
|
||||||
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
|
if (ops.$gt > ops.$gte) {
|
||||||
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
|
min = ops.$gt;
|
||||||
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
|
lowerInclusive = false;
|
||||||
|
} else if (ops.$gt < ops.$gte) {
|
||||||
|
min = ops.$gte;
|
||||||
|
lowerInclusive = true;
|
||||||
|
} else {
|
||||||
|
min = ops.$gt;
|
||||||
|
lowerInclusive = false;
|
||||||
|
}
|
||||||
|
} else if (ops.$gt !== undefined) {
|
||||||
|
min = ops.$gt;
|
||||||
|
lowerInclusive = false;
|
||||||
|
} else if (ops.$gte !== undefined) {
|
||||||
|
min = ops.$gte;
|
||||||
|
lowerInclusive = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ops.$lt !== undefined && ops.$lte !== undefined) {
|
||||||
|
if (ops.$lt < ops.$lte) {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
} else if (ops.$lt > ops.$lte) {
|
||||||
|
max = ops.$lte;
|
||||||
|
upperInclusive = true;
|
||||||
|
} else {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
}
|
||||||
|
} else if (ops.$lt !== undefined) {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
} else if (ops.$lte !== undefined) {
|
||||||
|
max = ops.$lte;
|
||||||
|
upperInclusive = true;
|
||||||
|
}
|
||||||
|
|
||||||
if (min === undefined && max === undefined) return null;
|
if (min === undefined && max === undefined) return null;
|
||||||
|
if (min !== undefined && max !== undefined) {
|
||||||
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
|
if (min > max) return [];
|
||||||
return this.flattenEntryKeys(entries);
|
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flatten B+ Tree range results into a flat array of internal keys.
|
|
||||||
* Uses an array instead of a Set — no hash overhead, no deduplication
|
|
||||||
* needed because each internal key only appears under one index key.
|
|
||||||
*/
|
|
||||||
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
|
|
||||||
const result: number[] = [];
|
|
||||||
for (const entry of entries) {
|
|
||||||
for (const key of entry.values) {
|
|
||||||
result.push(key);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return result;
|
|
||||||
|
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -495,8 +580,9 @@ export class StorageLocalStorage<
|
|||||||
* localStorage. Called only when a cross-tab manifest change is detected.
|
* localStorage. Called only when a cross-tab manifest change is detected.
|
||||||
*/
|
*/
|
||||||
private rebuildIndexes(): void {
|
private rebuildIndexes(): void {
|
||||||
for (const [, btree] of this.indexes) {
|
for (const fields of this.indexDefs) {
|
||||||
btree.clear();
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
|
this.cache.clearIndex(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const key of this.manifest) {
|
for (const key of this.manifest) {
|
||||||
@@ -526,6 +612,11 @@ export class StorageLocalStorage<
|
|||||||
return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`;
|
return `${this.prefix}${KEY_SEP}${COUNTER_SUFFIX}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Build the localStorage key used to persist the mutation version. */
|
||||||
|
private versionKey(): string {
|
||||||
|
return `${this.prefix}${KEY_SEP}${VERSION_SUFFIX}`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Re-read the manifest from localStorage into memory.
|
* Re-read the manifest from localStorage into memory.
|
||||||
* Called at the start of every public method so that cross-tab writes
|
* Called at the start of every public method so that cross-tab writes
|
||||||
@@ -536,23 +627,41 @@ export class StorageLocalStorage<
|
|||||||
*/
|
*/
|
||||||
private refreshManifest(): void {
|
private refreshManifest(): void {
|
||||||
const raw = localStorage.getItem(this.manifestKey()) ?? '[]';
|
const raw = localStorage.getItem(this.manifestKey()) ?? '[]';
|
||||||
|
const versionRaw = localStorage.getItem(this.versionKey()) ?? '0';
|
||||||
|
|
||||||
if (raw === this.lastManifestRaw) return;
|
if (raw === this.lastManifestRaw && versionRaw === this.lastVersionRaw) return;
|
||||||
|
|
||||||
this.lastManifestRaw = raw;
|
let parsedKeys: number[] = [];
|
||||||
|
let parsedOk = true;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const keys: number[] = JSON.parse(raw);
|
parsedKeys = JSON.parse(raw);
|
||||||
this.manifest = new Set(keys);
|
|
||||||
} catch {
|
} catch {
|
||||||
|
parsedOk = false;
|
||||||
this.manifest = new Set();
|
this.manifest = new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (parsedOk) {
|
||||||
|
this.manifest = new Set(parsedKeys);
|
||||||
|
this.lastManifestRaw = raw;
|
||||||
|
}
|
||||||
|
this.lastVersionRaw = versionRaw;
|
||||||
|
|
||||||
// Restore the counter from localStorage.
|
// Restore the counter from localStorage.
|
||||||
const counterRaw = localStorage.getItem(this.counterKey());
|
const counterRaw = localStorage.getItem(this.counterKey());
|
||||||
this.nextKey = counterRaw !== null ? Number(counterRaw) : 0;
|
if (counterRaw !== null) {
|
||||||
|
this.nextKey = Number(counterRaw);
|
||||||
|
} else if (this.manifest.size > 0) {
|
||||||
|
let max = -1;
|
||||||
|
for (const key of this.manifest) {
|
||||||
|
if (key > max) max = key;
|
||||||
|
}
|
||||||
|
this.nextKey = max + 1;
|
||||||
|
} else {
|
||||||
|
this.nextKey = 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Manifest changed — indexes are potentially stale.
|
// Manifest or version changed — indexes are potentially stale.
|
||||||
if (this.indexDefs.length > 0) {
|
if (this.indexDefs.length > 0) {
|
||||||
this.rebuildIndexes();
|
this.rebuildIndexes();
|
||||||
}
|
}
|
||||||
@@ -563,11 +672,16 @@ export class StorageLocalStorage<
|
|||||||
*/
|
*/
|
||||||
private persistManifest(): void {
|
private persistManifest(): void {
|
||||||
const raw = JSON.stringify([...this.manifest]);
|
const raw = JSON.stringify([...this.manifest]);
|
||||||
|
const nextVersion = Number(localStorage.getItem(this.versionKey()) ?? '0') + 1;
|
||||||
|
const versionRaw = String(nextVersion);
|
||||||
|
|
||||||
localStorage.setItem(this.manifestKey(), raw);
|
localStorage.setItem(this.manifestKey(), raw);
|
||||||
localStorage.setItem(this.counterKey(), String(this.nextKey));
|
localStorage.setItem(this.counterKey(), String(this.nextKey));
|
||||||
|
localStorage.setItem(this.versionKey(), versionRaw);
|
||||||
|
|
||||||
// Keep the cached raw string in sync so the next refreshManifest()
|
// Keep cached values in sync so the next refreshManifest() recognises
|
||||||
// recognises this as "our own write" and skips the rebuild.
|
// this as our own write and skips unnecessary rebuild work.
|
||||||
this.lastManifestRaw = raw;
|
this.lastManifestRaw = raw;
|
||||||
|
this.lastVersionRaw = versionRaw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,11 +7,17 @@ import { StorageMemory } from './storage-memory.js';
|
|||||||
* All read operations will use system memory - all write operations will use the provided adapter.
|
* All read operations will use system memory - all write operations will use the provided adapter.
|
||||||
*/
|
*/
|
||||||
export class StorageMemorySynced<T extends Record<string, any> = Record<string, any>> extends BaseStorage<T> {
|
export class StorageMemorySynced<T extends Record<string, any> = Record<string, any>> extends BaseStorage<T> {
|
||||||
|
private isPrimed: boolean;
|
||||||
|
private primePromise: Promise<void> | null;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private inMemoryCache: StorageMemory<T>,
|
private inMemoryCache: StorageMemory<T>,
|
||||||
private store: BaseStorage<T>,
|
private store: BaseStorage<T>,
|
||||||
|
isPrimed = false,
|
||||||
) {
|
) {
|
||||||
super();
|
super();
|
||||||
|
this.isPrimed = isPrimed;
|
||||||
|
this.primePromise = null;
|
||||||
|
|
||||||
// Hook into all write operations so that we can sync the In-Memory cache.
|
// Hook into all write operations so that we can sync the In-Memory cache.
|
||||||
this.store.on('insert', async (payload) => {
|
this.store.on('insert', async (payload) => {
|
||||||
@@ -44,6 +50,23 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure the in-memory cache has been initialized from the backing store.
|
||||||
|
* This is especially important for derived children, whose caches start empty.
|
||||||
|
*/
|
||||||
|
private async ensurePrimed(): Promise<void> {
|
||||||
|
if (this.isPrimed) return;
|
||||||
|
if (!this.primePromise) {
|
||||||
|
this.primePromise = (async () => {
|
||||||
|
await this.inMemoryCache.deleteMany({});
|
||||||
|
const allDocuments = await this.store.find();
|
||||||
|
await this.inMemoryCache.insertMany(allDocuments);
|
||||||
|
this.isPrimed = true;
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
await this.primePromise;
|
||||||
|
}
|
||||||
|
|
||||||
static async create<T extends Record<string, any>>(store: BaseStorage<T>) {
|
static async create<T extends Record<string, any>>(store: BaseStorage<T>) {
|
||||||
// Instantiate in-memory cache and the backing store.
|
// Instantiate in-memory cache and the backing store.
|
||||||
const inMemoryCache = new StorageMemory<T>();
|
const inMemoryCache = new StorageMemory<T>();
|
||||||
@@ -58,6 +81,7 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Return our instance of this store.
|
// Return our instance of this store.
|
||||||
|
memorySyncedStore.isPrimed = true;
|
||||||
return memorySyncedStore;
|
return memorySyncedStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,18 +90,21 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
async find(filter?: Filter<T>, options?: FindOptions): Promise<T[]> {
|
||||||
|
await this.ensurePrimed();
|
||||||
return await this.inMemoryCache.find(filter, options);
|
return await this.inMemoryCache.find(filter, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateMany(
|
async updateMany(
|
||||||
filter: Filter<T>,
|
filter: Filter<T>,
|
||||||
update: Partial<T>,
|
update: Partial<T>,
|
||||||
options: FindOptions = {} as FindOptions
|
options: Partial<FindOptions> = {},
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
|
await this.ensurePrimed();
|
||||||
return await this.store.updateMany(filter, update, options);
|
return await this.store.updateMany(filter, update, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(filter: Filter<T>, options: FindOptions = {} as FindOptions): Promise<number> {
|
async deleteMany(filter: Filter<T>, options: Partial<FindOptions> = {}): Promise<number> {
|
||||||
|
await this.ensurePrimed();
|
||||||
return await this.store.deleteMany(filter, options);
|
return await this.store.deleteMany(filter, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,7 +112,11 @@ export class StorageMemorySynced<T extends Record<string, any> = Record<string,
|
|||||||
const childStore = this.store.deriveChild<C>(path);
|
const childStore = this.store.deriveChild<C>(path);
|
||||||
const childMemory = this.inMemoryCache.deriveChild<C>(path);
|
const childMemory = this.inMemoryCache.deriveChild<C>(path);
|
||||||
|
|
||||||
|
if (!(childMemory instanceof StorageMemory)) {
|
||||||
|
throw new Error('Expected derived in-memory cache to be a StorageMemory instance');
|
||||||
|
}
|
||||||
|
|
||||||
// Create a new synced storage for the child
|
// Create a new synced storage for the child
|
||||||
return new StorageMemorySynced<C>(childMemory as StorageMemory<C>, childStore);
|
return new StorageMemorySynced<C>(childMemory, childStore);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,8 +5,9 @@ import {
|
|||||||
type Filter,
|
type Filter,
|
||||||
type ComparisonOperators,
|
type ComparisonOperators,
|
||||||
isOperatorObject,
|
isOperatorObject,
|
||||||
|
isLogicalKey,
|
||||||
} from './base-storage.js';
|
} from './base-storage.js';
|
||||||
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
|
import { BaseCache, BTreeCache } from 'src/cache/index.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Separator used when joining field names to create the index map key.
|
* Separator used when joining field names to create the index map key.
|
||||||
@@ -29,19 +30,6 @@ function normalizeIndexes(indexes?: IndexDefinition): string[][] {
|
|||||||
return indexes as string[][];
|
return indexes as string[][];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Comparator for compound index keys (arrays of raw values).
|
|
||||||
* Compares element-by-element using native `<` / `>` operators.
|
|
||||||
*/
|
|
||||||
function tupleCompare(a: any[], b: any[]): number {
|
|
||||||
const len = Math.min(a.length, b.length);
|
|
||||||
for (let i = 0; i < len; i++) {
|
|
||||||
if (a[i] < b[i]) return -1;
|
|
||||||
if (a[i] > b[i]) return 1;
|
|
||||||
}
|
|
||||||
return a.length - b.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implementation of BaseStore using Memory as the storage backend.
|
* Implementation of BaseStore using Memory as the storage backend.
|
||||||
*
|
*
|
||||||
@@ -55,8 +43,9 @@ export class StorageMemory<
|
|||||||
> extends BaseStorage<T> {
|
> extends BaseStorage<T> {
|
||||||
static from<T extends Record<string, any>>(
|
static from<T extends Record<string, any>>(
|
||||||
indexes?: IndexDefinition,
|
indexes?: IndexDefinition,
|
||||||
|
cache?: BaseCache,
|
||||||
): StorageMemory<T> {
|
): StorageMemory<T> {
|
||||||
return new StorageMemory<T>(indexes);
|
return new StorageMemory<T>(indexes, cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Auto-incrementing counter used to generate internal keys. */
|
/** Auto-incrementing counter used to generate internal keys. */
|
||||||
@@ -65,12 +54,8 @@ export class StorageMemory<
|
|||||||
/** Primary document store keyed by an opaque internal key. */
|
/** Primary document store keyed by an opaque internal key. */
|
||||||
private store: Map<number, T>;
|
private store: Map<number, T>;
|
||||||
|
|
||||||
/**
|
/** Secondary index cache (B-Tree or KV implementation). */
|
||||||
* Secondary indexes backed by B+ Trees.
|
private cache: BaseCache;
|
||||||
* Map key = index name (joined field names).
|
|
||||||
* Map value = B+ Tree mapping index keys to sets of internal document keys.
|
|
||||||
*/
|
|
||||||
private indexes: Map<string, BPlusTree<any, number>>;
|
|
||||||
|
|
||||||
/** The normalized index definitions supplied at construction time. */
|
/** The normalized index definitions supplied at construction time. */
|
||||||
private indexDefs: string[][];
|
private indexDefs: string[][];
|
||||||
@@ -78,19 +63,17 @@ export class StorageMemory<
|
|||||||
/** Lazily-created child storage instances. */
|
/** Lazily-created child storage instances. */
|
||||||
private children: Map<string, StorageMemory<any>>;
|
private children: Map<string, StorageMemory<any>>;
|
||||||
|
|
||||||
constructor(indexes?: IndexDefinition) {
|
constructor(indexes?: IndexDefinition, cache?: BaseCache) {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
this.store = new Map();
|
this.store = new Map();
|
||||||
this.children = new Map();
|
this.children = new Map();
|
||||||
this.indexDefs = normalizeIndexes(indexes);
|
this.indexDefs = normalizeIndexes(indexes);
|
||||||
|
|
||||||
// Create a B+ Tree for each index definition.
|
this.cache = cache ?? new BTreeCache();
|
||||||
this.indexes = new Map();
|
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
const comparator = fields.length > 1 ? tupleCompare : undefined;
|
this.cache.registerIndex(name, fields);
|
||||||
this.indexes.set(name, new BPlusTree<any, number>(32, comparator));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,7 +175,7 @@ export class StorageMemory<
|
|||||||
|
|
||||||
deriveChild<C>(path: string): BaseStorage<C> {
|
deriveChild<C>(path: string): BaseStorage<C> {
|
||||||
if (!this.children.has(path)) {
|
if (!this.children.has(path)) {
|
||||||
this.children.set(path, new StorageMemory<C>(this.indexDefs));
|
this.children.set(path, new StorageMemory<C>(this.indexDefs, this.cache.createChild()));
|
||||||
}
|
}
|
||||||
return this.children.get(path) as StorageMemory<C>;
|
return this.children.get(path) as StorageMemory<C>;
|
||||||
}
|
}
|
||||||
@@ -202,15 +185,24 @@ export class StorageMemory<
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks whether a document satisfies every field in the filter.
|
* Checks whether a document satisfies a filter.
|
||||||
* Supports both plain equality values and comparison operator objects.
|
*
|
||||||
|
* Handles top-level logical operators ($and, $or, $not) first via
|
||||||
|
* recursion, then evaluates remaining field-level conditions.
|
||||||
*/
|
*/
|
||||||
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
private matchesFilter(item: T, filter?: Filter<T>): boolean {
|
||||||
if (!filter || Object.keys(filter).length === 0) {
|
if (!filter || Object.keys(filter).length === 0) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Top-level logical operators.
|
||||||
|
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
|
||||||
|
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
|
||||||
|
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
|
||||||
|
|
||||||
|
// Field-level conditions (skip logical operator keys).
|
||||||
for (const [key, value] of Object.entries(filter)) {
|
for (const [key, value] of Object.entries(filter)) {
|
||||||
|
if (isLogicalKey(key)) continue;
|
||||||
if (isOperatorObject(value)) {
|
if (isOperatorObject(value)) {
|
||||||
if (!this.matchesOperators(item[key], value)) return false;
|
if (!this.matchesOperators(item[key], value)) return false;
|
||||||
} else {
|
} else {
|
||||||
@@ -221,7 +213,7 @@ export class StorageMemory<
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Evaluate a set of comparison operators against a single field value.
|
* Evaluate a set of comparison / string operators against a single field value.
|
||||||
* All operators must pass for the field to match.
|
* All operators must pass for the field to match.
|
||||||
*/
|
*/
|
||||||
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
|
||||||
@@ -231,9 +223,37 @@ export class StorageMemory<
|
|||||||
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
|
||||||
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
|
||||||
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
|
||||||
|
|
||||||
|
if (ops.$startsWith !== undefined) {
|
||||||
|
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
|
||||||
|
}
|
||||||
|
if (ops.$contains !== undefined) {
|
||||||
|
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Field-level $not: invert the enclosed operator set.
|
||||||
|
if (ops.$not !== undefined) {
|
||||||
|
if (this.matchesOperators(fieldValue, ops.$not)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine whether candidate documents returned by index resolution still
|
||||||
|
* require full filter verification.
|
||||||
|
*/
|
||||||
|
private filterNeedsVerification(
|
||||||
|
filter: Filter<T> | undefined,
|
||||||
|
resolvedFields: string[],
|
||||||
|
): boolean {
|
||||||
|
if (!filter) return false;
|
||||||
|
const filterKeys = Object.keys(filter);
|
||||||
|
const hasLogicalOps = filterKeys.some(isLogicalKey);
|
||||||
|
return hasLogicalOps
|
||||||
|
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collect all [internalKey, document] pairs that match a filter.
|
* Collect all [internalKey, document] pairs that match a filter.
|
||||||
* Uses an index when possible, otherwise falls back to a full scan.
|
* Uses an index when possible, otherwise falls back to a full scan.
|
||||||
@@ -243,8 +263,7 @@ export class StorageMemory<
|
|||||||
|
|
||||||
if (resolution !== null) {
|
if (resolution !== null) {
|
||||||
const { keys, resolvedFields } = resolution;
|
const { keys, resolvedFields } = resolution;
|
||||||
const filterKeys = filter ? Object.keys(filter) : [];
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
||||||
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
|
|
||||||
|
|
||||||
const results: Array<[number, T]> = [];
|
const results: Array<[number, T]> = [];
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
@@ -312,7 +331,7 @@ export class StorageMemory<
|
|||||||
if (indexKey === null) continue;
|
if (indexKey === null) continue;
|
||||||
|
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
this.indexes.get(name)!.insert(indexKey, internalKey);
|
this.cache.insert(name, indexKey, internalKey);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -323,7 +342,7 @@ export class StorageMemory<
|
|||||||
if (indexKey === null) continue;
|
if (indexKey === null) continue;
|
||||||
|
|
||||||
const name = fields.join(INDEX_KEY_SEP);
|
const name = fields.join(INDEX_KEY_SEP);
|
||||||
this.indexes.get(name)!.delete(indexKey, internalKey);
|
this.cache.delete(name, indexKey, internalKey);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +361,6 @@ export class StorageMemory<
|
|||||||
|
|
||||||
for (const fields of this.indexDefs) {
|
for (const fields of this.indexDefs) {
|
||||||
const indexName = fields.join(INDEX_KEY_SEP);
|
const indexName = fields.join(INDEX_KEY_SEP);
|
||||||
const btree = this.indexes.get(indexName)!;
|
|
||||||
|
|
||||||
if (fields.length === 1) {
|
if (fields.length === 1) {
|
||||||
// --- Single-field index ---
|
// --- Single-field index ---
|
||||||
@@ -352,13 +370,13 @@ export class StorageMemory<
|
|||||||
const filterValue = (filter as any)[field];
|
const filterValue = (filter as any)[field];
|
||||||
|
|
||||||
if (isOperatorObject(filterValue)) {
|
if (isOperatorObject(filterValue)) {
|
||||||
const keys = this.resolveOperatorViaTree(btree, filterValue);
|
const keys = this.resolveOperatorViaTree(indexName, filterValue);
|
||||||
if (keys !== null) return { keys, resolvedFields: [field] };
|
if (keys !== null) return { keys, resolvedFields: [field] };
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Plain equality.
|
// Plain equality.
|
||||||
return { keys: btree.get(filterValue) ?? [], resolvedFields: [field] };
|
return { keys: this.cache.get(indexName, filterValue), resolvedFields: [field] };
|
||||||
} else {
|
} else {
|
||||||
// --- Compound index — all fields must be plain equality ---
|
// --- Compound index — all fields must be plain equality ---
|
||||||
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
|
if (!fields.every((f) => f in filter && !isOperatorObject((filter as any)[f]))) {
|
||||||
@@ -366,7 +384,7 @@ export class StorageMemory<
|
|||||||
}
|
}
|
||||||
|
|
||||||
const tupleKey = fields.map((f) => (filter as any)[f]);
|
const tupleKey = fields.map((f) => (filter as any)[f]);
|
||||||
return { keys: btree.get(tupleKey) ?? [], resolvedFields: [...fields] };
|
return { keys: this.cache.get(indexName, tupleKey), resolvedFields: [...fields] };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -376,50 +394,102 @@ export class StorageMemory<
|
|||||||
/**
|
/**
|
||||||
* Try to resolve an operator filter against a single-field B+ Tree index.
|
* Try to resolve an operator filter against a single-field B+ Tree index.
|
||||||
* Returns a flat array of matching internal keys, or null if the
|
* Returns a flat array of matching internal keys, or null if the
|
||||||
* operators can't be served by the tree ($ne).
|
* operators can't be efficiently served by the tree.
|
||||||
|
*
|
||||||
|
* Supported acceleration:
|
||||||
|
* - `$eq` → point lookup via `.get()`
|
||||||
|
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
|
||||||
|
* - `$startsWith` → converted to a range scan on the prefix
|
||||||
|
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
|
||||||
*/
|
*/
|
||||||
private resolveOperatorViaTree(
|
private resolveOperatorViaTree(
|
||||||
btree: BPlusTree<any, number>,
|
indexName: string,
|
||||||
ops: ComparisonOperators<any>,
|
ops: ComparisonOperators<any>,
|
||||||
): Iterable<number> | null {
|
): Iterable<number> | null {
|
||||||
// $ne prevents efficient index use.
|
// Operators that prevent efficient index use.
|
||||||
if (ops.$ne !== undefined) return null;
|
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
|
||||||
|
|
||||||
// $eq is a point lookup.
|
// $eq is a point lookup.
|
||||||
if (ops.$eq !== undefined) {
|
if (ops.$eq !== undefined) {
|
||||||
return btree.get(ops.$eq) ?? [];
|
// If $eq is combined with other operators, this path does not fully
|
||||||
|
// resolve the predicate. Let caller fall back to verification/full scan.
|
||||||
|
if (
|
||||||
|
ops.$gt !== undefined
|
||||||
|
|| ops.$gte !== undefined
|
||||||
|
|| ops.$lt !== undefined
|
||||||
|
|| ops.$lte !== undefined
|
||||||
|
|| ops.$startsWith !== undefined
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return this.cache.get(indexName, ops.$eq);
|
||||||
|
}
|
||||||
|
|
||||||
|
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
|
||||||
|
if (ops.$startsWith !== undefined) {
|
||||||
|
const prefix = ops.$startsWith;
|
||||||
|
if (prefix.length === 0) return null;
|
||||||
|
const upper = prefix.slice(0, -1)
|
||||||
|
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
|
||||||
|
const entries = this.cache.range(indexName, prefix, upper, {
|
||||||
|
lowerInclusive: true,
|
||||||
|
upperInclusive: false,
|
||||||
|
});
|
||||||
|
return entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract range bounds from the remaining operators.
|
// Extract range bounds from the remaining operators.
|
||||||
|
// If strict/non-strict variants are both provided, use the stricter bound.
|
||||||
let min: any = undefined;
|
let min: any = undefined;
|
||||||
let max: any = undefined;
|
let max: any = undefined;
|
||||||
let lowerInclusive = true;
|
let lowerInclusive = true;
|
||||||
let upperInclusive = false;
|
let upperInclusive = false;
|
||||||
|
|
||||||
if (ops.$gt !== undefined) { min = ops.$gt; lowerInclusive = false; }
|
if (ops.$gt !== undefined && ops.$gte !== undefined) {
|
||||||
if (ops.$gte !== undefined) { min = ops.$gte; lowerInclusive = true; }
|
if (ops.$gt > ops.$gte) {
|
||||||
if (ops.$lt !== undefined) { max = ops.$lt; upperInclusive = false; }
|
min = ops.$gt;
|
||||||
if (ops.$lte !== undefined) { max = ops.$lte; upperInclusive = true; }
|
lowerInclusive = false;
|
||||||
|
} else if (ops.$gt < ops.$gte) {
|
||||||
|
min = ops.$gte;
|
||||||
|
lowerInclusive = true;
|
||||||
|
} else {
|
||||||
|
min = ops.$gt;
|
||||||
|
lowerInclusive = false;
|
||||||
|
}
|
||||||
|
} else if (ops.$gt !== undefined) {
|
||||||
|
min = ops.$gt;
|
||||||
|
lowerInclusive = false;
|
||||||
|
} else if (ops.$gte !== undefined) {
|
||||||
|
min = ops.$gte;
|
||||||
|
lowerInclusive = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ops.$lt !== undefined && ops.$lte !== undefined) {
|
||||||
|
if (ops.$lt < ops.$lte) {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
} else if (ops.$lt > ops.$lte) {
|
||||||
|
max = ops.$lte;
|
||||||
|
upperInclusive = true;
|
||||||
|
} else {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
}
|
||||||
|
} else if (ops.$lt !== undefined) {
|
||||||
|
max = ops.$lt;
|
||||||
|
upperInclusive = false;
|
||||||
|
} else if (ops.$lte !== undefined) {
|
||||||
|
max = ops.$lte;
|
||||||
|
upperInclusive = true;
|
||||||
|
}
|
||||||
|
|
||||||
if (min === undefined && max === undefined) return null;
|
if (min === undefined && max === undefined) return null;
|
||||||
|
if (min !== undefined && max !== undefined) {
|
||||||
const entries = btree.range(min, max, { lowerInclusive, upperInclusive });
|
if (min > max) return [];
|
||||||
return this.flattenEntryKeys(entries);
|
if (min === max && (!lowerInclusive || !upperInclusive)) return [];
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flatten B+ Tree range results into a flat array of internal keys.
|
|
||||||
* Uses an array instead of a Set — no hash overhead, no deduplication
|
|
||||||
* needed because each internal key only appears under one index key.
|
|
||||||
*/
|
|
||||||
private flattenEntryKeys(entries: BPlusTreeEntry<any, number>[]): number[] {
|
|
||||||
const result: number[] = [];
|
|
||||||
for (const entry of entries) {
|
|
||||||
for (const key of entry.values) {
|
|
||||||
result.push(key);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return result;
|
|
||||||
|
return this.cache.range(indexName, min, max, { lowerInclusive, upperInclusive });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -436,8 +506,7 @@ export class StorageMemory<
|
|||||||
if (resolution === null) return null;
|
if (resolution === null) return null;
|
||||||
|
|
||||||
const { keys, resolvedFields } = resolution;
|
const { keys, resolvedFields } = resolution;
|
||||||
const filterKeys = filter ? Object.keys(filter) : [];
|
const needsVerification = this.filterNeedsVerification(filter, resolvedFields);
|
||||||
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
|
|
||||||
|
|
||||||
const results: T[] = [];
|
const results: T[] = [];
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
|
|||||||
@@ -17,17 +17,17 @@ export class EphemeralTransport {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
async receive<T extends ZodType>(url: string, schema: T): Promise<output<T>>;
|
async receive<T extends ZodType | undefined>(url: string, schema?: T): Promise<T extends undefined ? unknown : output<T>> {
|
||||||
async receive(url: string): Promise<unknown>;
|
|
||||||
async receive(url: string, schema?: ZodType): Promise<unknown> {
|
|
||||||
const transport = await this.transports(url);
|
const transport = await this.transports(url);
|
||||||
const message = await transport.waitFor('message', () => true);
|
const message = await transport.waitFor('message', () => true);
|
||||||
transport.disconnect();
|
transport.disconnect();
|
||||||
|
|
||||||
if (schema) {
|
if (schema) {
|
||||||
return schema.parse(message);
|
// TODO: Figure out how the hell to fix this assertion. It shouldnt be needed, but TS is being a bitch.
|
||||||
|
return schema.parse(message) as T extends undefined ? unknown : output<T>;
|
||||||
}
|
}
|
||||||
|
|
||||||
return message;
|
// TODO: Figure out how the hell to fix this assertion. It shouldnt be needed, but TS is being a bitch.
|
||||||
|
return message as T extends undefined ? unknown : output<T>;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
58
tests/storage/storage-regressions.test.ts
Normal file
58
tests/storage/storage-regressions.test.ts
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { AESKey } from 'src/crypto/aes-key.js';
|
||||||
|
|
||||||
|
import { EncryptedStorage } from 'src/storage/encrypted-storage.js';
|
||||||
|
import { StorageMemory } from 'src/storage/storage-memory.js';
|
||||||
|
import { StorageMemorySynced } from 'src/storage/storage-memory-synced.js';
|
||||||
|
|
||||||
|
describe('storage regressions', () => {
|
||||||
|
it('does not treat $eq as fully resolved when mixed with other operators', async () => {
|
||||||
|
const storage = StorageMemory.from<{ age: number }>(['age']);
|
||||||
|
await storage.insertMany([{ age: 25 }, { age: 35 }]);
|
||||||
|
|
||||||
|
const results = await storage.find({ age: { $eq: 25, $gt: 30 } });
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('normalizes conflicting lower/upper bounds to strictest constraints', async () => {
|
||||||
|
const storage = StorageMemory.from<{ age: number }>(['age']);
|
||||||
|
await storage.insertMany([{ age: 30 }, { age: 31 }, { age: 32 }]);
|
||||||
|
|
||||||
|
const lower = await storage.find({ age: { $gt: 30, $gte: 30 } });
|
||||||
|
expect(lower.map((d) => d.age).sort((a, b) => a - b)).toEqual([31, 32]);
|
||||||
|
|
||||||
|
const upper = await storage.find({ age: { $lt: 32, $lte: 32 } });
|
||||||
|
expect(upper.map((d) => d.age).sort((a, b) => a - b)).toEqual([30, 31]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('formats plaintext operator values in encrypted filters', async () => {
|
||||||
|
type Doc = { createdAt: Date; name: string };
|
||||||
|
const key = await AESKey.fromSeed('storage-regression-test-key');
|
||||||
|
const base = StorageMemory.from<Record<string, any>>(['createdAt']);
|
||||||
|
const storage = EncryptedStorage.from<Doc>(base, key, { plaintextKeys: ['createdAt'] });
|
||||||
|
|
||||||
|
await storage.insertOne({
|
||||||
|
createdAt: new Date('2024-01-02T00:00:00.000Z'),
|
||||||
|
name: 'alice',
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await storage.find({
|
||||||
|
createdAt: { $gte: new Date('2024-01-01T00:00:00.000Z') },
|
||||||
|
});
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].name).toBe('alice');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('primes derived child cache in StorageMemorySynced', async () => {
|
||||||
|
const store = StorageMemory.from<Record<string, any>>();
|
||||||
|
const child = store.deriveChild<{ value: number }>('child');
|
||||||
|
await child.insertOne({ value: 42 });
|
||||||
|
|
||||||
|
const synced = await StorageMemorySynced.create(store);
|
||||||
|
const syncedChild = synced.deriveChild<{ value: number }>('child');
|
||||||
|
const results = await syncedChild.find();
|
||||||
|
|
||||||
|
expect(results).toEqual([{ value: 42 }]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, it, expect, beforeEach } from 'vitest';
|
import { describe, it, expect, beforeEach } from 'vitest';
|
||||||
import { BPlusTree } from './btree.js';
|
import { BPlusTree } from '../../src/utils/btree.js';
|
||||||
|
|
||||||
describe('BPlusTree', () => {
|
describe('BPlusTree', () => {
|
||||||
let tree: BPlusTree<number, string>;
|
let tree: BPlusTree<number, string>;
|
||||||
@@ -181,6 +181,7 @@ describe('BPlusTree', () => {
|
|||||||
|
|
||||||
it('should handle range on empty tree', () => {
|
it('should handle range on empty tree', () => {
|
||||||
expect(tree.range()).toEqual([]);
|
expect(tree.range()).toEqual([]);
|
||||||
|
expect(tree.range(1, 2)).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should handle delete on empty tree', () => {
|
it('should handle delete on empty tree', () => {
|
||||||
Reference in New Issue
Block a user