diff --git a/src/storage/base-storage.ts b/src/storage/base-storage.ts index e85a095..909816b 100644 --- a/src/storage/base-storage.ts +++ b/src/storage/base-storage.ts @@ -20,7 +20,14 @@ export type FindOptions = { export type IndexDefinition = string[] | string[][]; /** - * MongoDB-style comparison operators for a single field value. + * MongoDB-style comparison and string operators for a single field value. + * + * - Comparison operators ($eq, $ne, $lt, $lte, $gt, $gte) are available for + * all value types. + * - String operators ($startsWith, $contains) are only available when the + * field value type extends `string`. + * - Field-level $not wraps another operator set, inverting its result: + * `{ age: { $not: { $gte: 18 } } }` matches documents where age < 18. */ export type ComparisonOperators = { $eq?: V; @@ -29,7 +36,8 @@ export type ComparisonOperators = { $lte?: V; $gt?: V; $gte?: V; -}; + $not?: ComparisonOperators; +} & ([V] extends [string] ? { $startsWith?: string; $contains?: string } : {}); /** * A filter value for a single field — either a plain value (equality shorthand) @@ -38,7 +46,22 @@ export type ComparisonOperators = { export type FieldFilter = V | ComparisonOperators; /** - * Query filter that supports both equality shorthand and comparison operators. + * Keys that represent top-level logical operators in a filter, + * as opposed to document field names. + */ +const LOGICAL_KEYS = new Set(['$and', '$or', '$not']); + +/** + * Returns true when `key` is a top-level logical operator ($and, $or, $not) + * rather than a document field name. + */ +export function isLogicalKey(key: string): boolean { + return LOGICAL_KEYS.has(key); +} + +/** + * Query filter that supports equality shorthand, comparison operators, + * and top-level logical operators ($and, $or, $not). * * @example * // Equality shorthand @@ -47,11 +70,20 @@ export type FieldFilter = V | ComparisonOperators; * // Comparison operators * { age: { $gte: 18, $lt: 65 } } * - * // Combined - * { name: 'foo', age: { $gte: 18 } } + * // String operators (type-safe — only on string fields) + * { name: { $startsWith: 'A' } } + * + * // Logical operators + * { $or: [{ name: 'foo' }, { age: { $gte: 18 } }] } + * { $not: { name: 'bar' } } + * { $and: [{ age: { $gte: 18 } }, { name: { $startsWith: 'A' } }] } */ export type Filter> = { [K in keyof T]?: FieldFilter; +} & { + $and?: Filter[]; + $or?: Filter[]; + $not?: Filter; }; /** diff --git a/src/storage/encrypted-storage.ts b/src/storage/encrypted-storage.ts index 032c1df..6ac5b21 100644 --- a/src/storage/encrypted-storage.ts +++ b/src/storage/encrypted-storage.ts @@ -3,7 +3,7 @@ import { Packr } from 'msgpackr'; import { AESKey } from 'src/crypto/aes-key.js'; import { Bytes } from 'src/crypto/bytes.js'; -import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js'; +import { BaseStorage, type FindOptions, type Filter, isOperatorObject, isLogicalKey } from './base-storage.js'; import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js'; @@ -231,12 +231,13 @@ export class EncryptedStorage< // --------------------------------------------------------------------------- /** - * Convert a query filter for storage. Handles both plain equality values - * and operator objects. + * Convert a query filter for storage. Handles plain equality values, + * operator objects, and top-level logical operators ($and, $or, $not). * * - Plaintext fields: values and operator objects pass through as-is. * - Encrypted fields: plain values are encrypted. Operator objects throw - * because range comparisons are meaningless on ciphertext. + * because range/string comparisons are meaningless on ciphertext. + * - Logical operators: sub-filters are recursively converted. */ private async convertFilterForStorage( filter: Filter, @@ -244,9 +245,27 @@ export class EncryptedStorage< const result: Record = {}; const entries = Object.entries(filter); + // Recursively convert logical operator sub-filters. + if (filter.$and) { + result.$and = await Promise.all( + filter.$and.map((f) => this.convertFilterForStorage(f)), + ); + } + if (filter.$or) { + result.$or = await Promise.all( + filter.$or.map((f) => this.convertFilterForStorage(f)), + ); + } + if (filter.$not) { + result.$not = await this.convertFilterForStorage(filter.$not); + } + const encryptionTasks: Array> = []; for (const [key, value] of entries) { + // Logical operator keys are already handled above. + if (isLogicalKey(key)) continue; + if (this.plaintextKeys.has(key)) { // Plaintext field — pass through (including operator objects). result[key] = isOperatorObject(value) @@ -255,8 +274,8 @@ export class EncryptedStorage< } else if (isOperatorObject(value)) { // Encrypted field with an operator — not supported. throw new Error( - `Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` + - `Add '${key}' to plaintextKeys if you need range queries on this field.`, + `Operators ($lt, $gt, $startsWith, $contains, $not, etc.) cannot be used on encrypted field '${key}'. ` + + `Add '${key}' to plaintextKeys if you need operator queries on this field.`, ); } else { // Encrypted field with a plain equality value — encrypt it. diff --git a/src/storage/storage-localstorage.ts b/src/storage/storage-localstorage.ts index fac54ea..259c5c3 100644 --- a/src/storage/storage-localstorage.ts +++ b/src/storage/storage-localstorage.ts @@ -5,6 +5,7 @@ import { type Filter, type ComparisonOperators, isOperatorObject, + isLogicalKey, } from './base-storage.js'; import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js'; @@ -162,7 +163,9 @@ export class StorageLocalStorage< if (resolution !== null) { const { keys, resolvedFields } = resolution; const filterKeys = filter ? Object.keys(filter) : []; - const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k)); + const hasLogicalOps = filterKeys.some(isLogicalKey); + const needsVerification = hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); results = []; for (const key of keys) { @@ -273,12 +276,22 @@ export class StorageLocalStorage< // --------------------------------------------------------------------------- /** - * Checks whether a document satisfies every field in the filter. - * Supports both plain equality values and comparison operator objects. + * Checks whether a document satisfies a filter. + * + * Handles top-level logical operators ($and, $or, $not) first via + * recursion, then evaluates remaining field-level conditions. */ private matchesFilter(item: T, filter?: Filter): boolean { if (!filter || Object.keys(filter).length === 0) return true; + + // Top-level logical operators. + if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false; + if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false; + if (filter.$not && this.matchesFilter(item, filter.$not)) return false; + + // Field-level conditions (skip logical operator keys). for (const [key, value] of Object.entries(filter)) { + if (isLogicalKey(key)) continue; if (isOperatorObject(value)) { if (!this.matchesOperators(item[key], value)) return false; } else { @@ -289,7 +302,8 @@ export class StorageLocalStorage< } /** - * Evaluate a set of comparison operators against a single field value. + * Evaluate a set of comparison / string operators against a single field value. + * All operators must pass for the field to match. */ private matchesOperators(fieldValue: any, ops: ComparisonOperators): boolean { if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false; @@ -298,6 +312,19 @@ export class StorageLocalStorage< if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false; if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false; if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false; + + if (ops.$startsWith !== undefined) { + if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false; + } + if (ops.$contains !== undefined) { + if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false; + } + + // Field-level $not: invert the enclosed operator set. + if (ops.$not !== undefined) { + if (this.matchesOperators(fieldValue, ops.$not)) return false; + } + return true; } @@ -312,7 +339,9 @@ export class StorageLocalStorage< if (resolution !== null) { const { keys, resolvedFields } = resolution; const filterKeys = filter ? Object.keys(filter) : []; - const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k)); + const hasLogicalOps = filterKeys.some(isLogicalKey); + const needsVerification = hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); for (const key of keys) { const raw = localStorage.getItem(this.docKey(key)); @@ -447,18 +476,38 @@ export class StorageLocalStorage< /** * Try to resolve an operator filter against a single-field B+ Tree index. * Returns a flat array of matching internal keys, or null if the - * operators can't be served by the tree ($ne). + * operators can't be efficiently served by the tree. + * + * Supported acceleration: + * - `$eq` → point lookup via `.get()` + * - `$gt/$gte/$lt/$lte` → range scan via `.range()` + * - `$startsWith` → converted to a range scan on the prefix + * - `$ne`, `$contains`, `$not` → cannot use index, returns null */ private resolveOperatorViaTree( btree: BPlusTree, ops: ComparisonOperators, ): Iterable | null { - if (ops.$ne !== undefined) return null; + // Operators that prevent efficient index use. + if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null; if (ops.$eq !== undefined) { return btree.get(ops.$eq) ?? []; } + // $startsWith is converted to a range scan: "abc" → ["abc", "abd"). + if (ops.$startsWith !== undefined) { + const prefix = ops.$startsWith; + if (prefix.length === 0) return null; + const upper = prefix.slice(0, -1) + + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); + const entries = btree.range(prefix, upper, { + lowerInclusive: true, + upperInclusive: false, + }); + return this.flattenEntryKeys(entries); + } + let min: any = undefined; let max: any = undefined; let lowerInclusive = true; diff --git a/src/storage/storage-memory.ts b/src/storage/storage-memory.ts index 9a0d064..413727a 100644 --- a/src/storage/storage-memory.ts +++ b/src/storage/storage-memory.ts @@ -5,6 +5,7 @@ import { type Filter, type ComparisonOperators, isOperatorObject, + isLogicalKey, } from './base-storage.js'; import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js'; @@ -202,15 +203,24 @@ export class StorageMemory< // --------------------------------------------------------------------------- /** - * Checks whether a document satisfies every field in the filter. - * Supports both plain equality values and comparison operator objects. + * Checks whether a document satisfies a filter. + * + * Handles top-level logical operators ($and, $or, $not) first via + * recursion, then evaluates remaining field-level conditions. */ private matchesFilter(item: T, filter?: Filter): boolean { if (!filter || Object.keys(filter).length === 0) { return true; } + // Top-level logical operators. + if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false; + if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false; + if (filter.$not && this.matchesFilter(item, filter.$not)) return false; + + // Field-level conditions (skip logical operator keys). for (const [key, value] of Object.entries(filter)) { + if (isLogicalKey(key)) continue; if (isOperatorObject(value)) { if (!this.matchesOperators(item[key], value)) return false; } else { @@ -221,7 +231,7 @@ export class StorageMemory< } /** - * Evaluate a set of comparison operators against a single field value. + * Evaluate a set of comparison / string operators against a single field value. * All operators must pass for the field to match. */ private matchesOperators(fieldValue: any, ops: ComparisonOperators): boolean { @@ -231,6 +241,19 @@ export class StorageMemory< if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false; if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false; if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false; + + if (ops.$startsWith !== undefined) { + if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false; + } + if (ops.$contains !== undefined) { + if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false; + } + + // Field-level $not: invert the enclosed operator set. + if (ops.$not !== undefined) { + if (this.matchesOperators(fieldValue, ops.$not)) return false; + } + return true; } @@ -244,7 +267,9 @@ export class StorageMemory< if (resolution !== null) { const { keys, resolvedFields } = resolution; const filterKeys = filter ? Object.keys(filter) : []; - const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k)); + const hasLogicalOps = filterKeys.some(isLogicalKey); + const needsVerification = hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); const results: Array<[number, T]> = []; for (const key of keys) { @@ -376,20 +401,39 @@ export class StorageMemory< /** * Try to resolve an operator filter against a single-field B+ Tree index. * Returns a flat array of matching internal keys, or null if the - * operators can't be served by the tree ($ne). + * operators can't be efficiently served by the tree. + * + * Supported acceleration: + * - `$eq` → point lookup via `.get()` + * - `$gt/$gte/$lt/$lte` → range scan via `.range()` + * - `$startsWith` → converted to a range scan on the prefix + * - `$ne`, `$contains`, `$not` → cannot use index, returns null */ private resolveOperatorViaTree( btree: BPlusTree, ops: ComparisonOperators, ): Iterable | null { - // $ne prevents efficient index use. - if (ops.$ne !== undefined) return null; + // Operators that prevent efficient index use. + if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null; // $eq is a point lookup. if (ops.$eq !== undefined) { return btree.get(ops.$eq) ?? []; } + // $startsWith is converted to a range scan: "abc" → ["abc", "abd"). + if (ops.$startsWith !== undefined) { + const prefix = ops.$startsWith; + if (prefix.length === 0) return null; + const upper = prefix.slice(0, -1) + + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1); + const entries = btree.range(prefix, upper, { + lowerInclusive: true, + upperInclusive: false, + }); + return this.flattenEntryKeys(entries); + } + // Extract range bounds from the remaining operators. let min: any = undefined; let max: any = undefined; @@ -437,7 +481,12 @@ export class StorageMemory< const { keys, resolvedFields } = resolution; const filterKeys = filter ? Object.keys(filter) : []; - const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k)); + + // Logical operators ($and/$or/$not) are never resolved by the index, + // and any unresolved field conditions also require verification. + const hasLogicalOps = filterKeys.some(isLogicalKey); + const needsVerification = hasLogicalOps + || filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k)); const results: T[] = []; for (const key of keys) {