Harvmaster/Hashpass-lib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

String filters. Logic operators in filters

Browse Source
This commit is contained in:
Harvey Zuccon
2026-02-25 16:27:01 +11:00
parent 64b811f330
commit f7c89046d1
4 changed files with 175 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/storage/base-storage.ts
View File

@@ -20,7 +20,14 @@ export type FindOptions = {
export type IndexDefinition = string[] | string[][];
/**
* MongoDB-style comparison operators for a single field value.
* MongoDB-style comparison and string operators for a single field value.
*
* - Comparison operators ($eq, $ne, $lt, $lte, $gt, $gte) are available for
* all value types.
* - String operators ($startsWith, $contains) are only available when the
* field value type extends `string`.
* - Field-level $not wraps another operator set, inverting its result:
* `{ age: { $not: { $gte: 18 } } }` matches documents where age < 18.
*/
export type ComparisonOperators<V> = {
$eq?: V;
@@ -29,7 +36,8 @@ export type ComparisonOperators<V> = {
$lte?: V;
$gt?: V;
$gte?: V;
};
$not?: ComparisonOperators<V>;
} & ([V] extends [string] ? { $startsWith?: string; $contains?: string } : {});
/**
* A filter value for a single field — either a plain value (equality shorthand)
@@ -38,7 +46,22 @@ export type ComparisonOperators<V> = {
export type FieldFilter<V> = V | ComparisonOperators<V>;
/**
* Query filter that supports both equality shorthand and comparison operators.
* Keys that represent top-level logical operators in a filter,
* as opposed to document field names.
*/
const LOGICAL_KEYS = new Set(['$and', '$or', '$not']);
/**
* Returns true when `key` is a top-level logical operator ($and, $or, $not)
* rather than a document field name.
*/
export function isLogicalKey(key: string): boolean {
return LOGICAL_KEYS.has(key);
}
/**
* Query filter that supports equality shorthand, comparison operators,
* and top-level logical operators ($and, $or, $not).
*
* @example
* // Equality shorthand
@@ -47,11 +70,20 @@ export type FieldFilter<V> = V | ComparisonOperators<V>;
* // Comparison operators
* { age: { $gte: 18, $lt: 65 } }
*
* // Combined
* { name: 'foo', age: { $gte: 18 } }
* // String operators (type-safe — only on string fields)
* { name: { $startsWith: 'A' } }
*
* // Logical operators
* { $or: [{ name: 'foo' }, { age: { $gte: 18 } }] }
* { $not: { name: 'bar' } }
* { $and: [{ age: { $gte: 18 } }, { name: { $startsWith: 'A' } }] }
*/
export type Filter<T extends Record<string, any>> = {
[K in keyof T]?: FieldFilter<T[K]>;
} & {
$and?: Filter<T>[];
$or?: Filter<T>[];
$not?: Filter<T>;
};
/**

31
src/storage/encrypted-storage.ts
View File

@@ -3,7 +3,7 @@ import { Packr } from 'msgpackr';
import { AESKey } from 'src/crypto/aes-key.js';
import { Bytes } from 'src/crypto/bytes.js';
import { BaseStorage, type FindOptions, type Filter, isOperatorObject } from './base-storage.js';
import { BaseStorage, type FindOptions, type Filter, isOperatorObject, isLogicalKey } from './base-storage.js';
import { encodeExtendedJsonObject, decodeExtendedJsonObject } from 'src/utils/ext-json.js';
@@ -231,12 +231,13 @@ export class EncryptedStorage<
// ---------------------------------------------------------------------------
/**
* Convert a query filter for storage. Handles both plain equality values
* and operator objects.
* Convert a query filter for storage. Handles plain equality values,
* operator objects, and top-level logical operators ($and, $or, $not).
*
* - Plaintext fields: values and operator objects pass through as-is.
* - Encrypted fields: plain values are encrypted. Operator objects throw
* because range comparisons are meaningless on ciphertext.
* because range/string comparisons are meaningless on ciphertext.
* - Logical operators: sub-filters are recursively converted.
*/
private async convertFilterForStorage(
filter: Filter<T>,
@@ -244,9 +245,27 @@ export class EncryptedStorage<
const result: Record<string, any> = {};
const entries = Object.entries(filter);
// Recursively convert logical operator sub-filters.
if (filter.$and) {
result.$and = await Promise.all(
filter.$and.map((f) => this.convertFilterForStorage(f)),
);
}
if (filter.$or) {
result.$or = await Promise.all(
filter.$or.map((f) => this.convertFilterForStorage(f)),
);
}
if (filter.$not) {
result.$not = await this.convertFilterForStorage(filter.$not);
}
const encryptionTasks: Array<Promise<readonly [string, any]>> = [];
for (const [key, value] of entries) {
// Logical operator keys are already handled above.
if (isLogicalKey(key)) continue;
if (this.plaintextKeys.has(key)) {
// Plaintext field — pass through (including operator objects).
result[key] = isOperatorObject(value)
@@ -255,8 +274,8 @@ export class EncryptedStorage<
} else if (isOperatorObject(value)) {
// Encrypted field with an operator — not supported.
throw new Error(
`Range operators ($lt, $gt, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need range queries on this field.`,
`Operators ($lt, $gt, $startsWith, $contains, $not, etc.) cannot be used on encrypted field '${key}'. ` +
`Add '${key}' to plaintextKeys if you need operator queries on this field.`,
);
} else {
// Encrypted field with a plain equality value — encrypt it.

63
src/storage/storage-localstorage.ts
View File

@@ -5,6 +5,7 @@ import {
type Filter,
type ComparisonOperators,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
@@ -162,7 +163,9 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
results = [];
for (const key of keys) {
@@ -273,12 +276,22 @@ export class StorageLocalStorage<
// ---------------------------------------------------------------------------
/**
* Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
* Checks whether a document satisfies a filter.
*
* Handles top-level logical operators ($and, $or, $not) first via
* recursion, then evaluates remaining field-level conditions.
*/
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) return true;
// Top-level logical operators.
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
// Field-level conditions (skip logical operator keys).
for (const [key, value] of Object.entries(filter)) {
if (isLogicalKey(key)) continue;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
@@ -289,7 +302,8 @@ export class StorageLocalStorage<
}
/**
* Evaluate a set of comparison operators against a single field value.
* Evaluate a set of comparison / string operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
if (ops.$eq !== undefined && fieldValue !== ops.$eq) return false;
@@ -298,6 +312,19 @@ export class StorageLocalStorage<
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
if (ops.$startsWith !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
}
if (ops.$contains !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
}
// Field-level $not: invert the enclosed operator set.
if (ops.$not !== undefined) {
if (this.matchesOperators(fieldValue, ops.$not)) return false;
}
return true;
}
@@ -312,7 +339,9 @@ export class StorageLocalStorage<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
for (const key of keys) {
const raw = localStorage.getItem(this.docKey(key));
@@ -447,18 +476,38 @@ export class StorageLocalStorage<
/**
* Try to resolve an operator filter against a single-field B+ Tree index.
* Returns a flat array of matching internal keys, or null if the
* operators can't be served by the tree ($ne).
* operators can't be efficiently served by the tree.
*
* Supported acceleration:
* - `$eq` → point lookup via `.get()`
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
* - `$startsWith` → converted to a range scan on the prefix
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
if (ops.$ne !== undefined) return null;
// Operators that prevent efficient index use.
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
if (ops.$startsWith !== undefined) {
const prefix = ops.$startsWith;
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = btree.range(prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return this.flattenEntryKeys(entries);
}
let min: any = undefined;
let max: any = undefined;
let lowerInclusive = true;

65
src/storage/storage-memory.ts
View File

@@ -5,6 +5,7 @@ import {
type Filter,
type ComparisonOperators,
isOperatorObject,
isLogicalKey,
} from './base-storage.js';
import { BPlusTree, type BPlusTreeEntry } from 'src/utils/btree.js';
@@ -202,15 +203,24 @@ export class StorageMemory<
// ---------------------------------------------------------------------------
/**
* Checks whether a document satisfies every field in the filter.
* Supports both plain equality values and comparison operator objects.
* Checks whether a document satisfies a filter.
*
* Handles top-level logical operators ($and, $or, $not) first via
* recursion, then evaluates remaining field-level conditions.
*/
private matchesFilter(item: T, filter?: Filter<T>): boolean {
if (!filter || Object.keys(filter).length === 0) {
return true;
}
// Top-level logical operators.
if (filter.$and && !filter.$and.every((f) => this.matchesFilter(item, f))) return false;
if (filter.$or && !filter.$or.some((f) => this.matchesFilter(item, f))) return false;
if (filter.$not && this.matchesFilter(item, filter.$not)) return false;
// Field-level conditions (skip logical operator keys).
for (const [key, value] of Object.entries(filter)) {
if (isLogicalKey(key)) continue;
if (isOperatorObject(value)) {
if (!this.matchesOperators(item[key], value)) return false;
} else {
@@ -221,7 +231,7 @@ export class StorageMemory<
}
/**
* Evaluate a set of comparison operators against a single field value.
* Evaluate a set of comparison / string operators against a single field value.
* All operators must pass for the field to match.
*/
private matchesOperators(fieldValue: any, ops: ComparisonOperators<any>): boolean {
@@ -231,6 +241,19 @@ export class StorageMemory<
if (ops.$lte !== undefined && !(fieldValue <= ops.$lte)) return false;
if (ops.$gt !== undefined && !(fieldValue > ops.$gt)) return false;
if (ops.$gte !== undefined && !(fieldValue >= ops.$gte)) return false;
if (ops.$startsWith !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.startsWith(ops.$startsWith)) return false;
}
if (ops.$contains !== undefined) {
if (typeof fieldValue !== 'string' || !fieldValue.includes(ops.$contains)) return false;
}
// Field-level $not: invert the enclosed operator set.
if (ops.$not !== undefined) {
if (this.matchesOperators(fieldValue, ops.$not)) return false;
}
return true;
}
@@ -244,7 +267,9 @@ export class StorageMemory<
if (resolution !== null) {
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const results: Array<[number, T]> = [];
for (const key of keys) {
@@ -376,20 +401,39 @@ export class StorageMemory<
/**
* Try to resolve an operator filter against a single-field B+ Tree index.
* Returns a flat array of matching internal keys, or null if the
* operators can't be served by the tree ($ne).
* operators can't be efficiently served by the tree.
*
* Supported acceleration:
* - `$eq` → point lookup via `.get()`
* - `$gt/$gte/$lt/$lte` → range scan via `.range()`
* - `$startsWith` → converted to a range scan on the prefix
* - `$ne`, `$contains`, `$not` → cannot use index, returns null
*/
private resolveOperatorViaTree(
btree: BPlusTree<any, number>,
ops: ComparisonOperators<any>,
): Iterable<number> | null {
// $ne prevents efficient index use.
if (ops.$ne !== undefined) return null;
// Operators that prevent efficient index use.
if (ops.$ne !== undefined || ops.$contains !== undefined || ops.$not !== undefined) return null;
// $eq is a point lookup.
if (ops.$eq !== undefined) {
return btree.get(ops.$eq) ?? [];
}
// $startsWith is converted to a range scan: "abc" → ["abc", "abd").
if (ops.$startsWith !== undefined) {
const prefix = ops.$startsWith;
if (prefix.length === 0) return null;
const upper = prefix.slice(0, -1)
+ String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
const entries = btree.range(prefix, upper, {
lowerInclusive: true,
upperInclusive: false,
});
return this.flattenEntryKeys(entries);
}
// Extract range bounds from the remaining operators.
let min: any = undefined;
let max: any = undefined;
@@ -437,7 +481,12 @@ export class StorageMemory<
const { keys, resolvedFields } = resolution;
const filterKeys = filter ? Object.keys(filter) : [];
const needsVerification = filterKeys.some((k) => !resolvedFields.includes(k));
// Logical operators ($and/$or/$not) are never resolved by the index,
// and any unresolved field conditions also require verification.
const hasLogicalOps = filterKeys.some(isLogicalKey);
const needsVerification = hasLogicalOps
|| filterKeys.some((k) => !isLogicalKey(k) && !resolvedFields.includes(k));
const results: T[] = [];
for (const key of keys) {