Draft Inverse Bloom Filter

packages/stack/src/utils/inverse-bloom-filter.ts

@@ -0,0 +1,134 @@
+// NOTE: Needs testing.
+//       Appears to work on 100,000 items though.
+//       Probably want to make it so that custom hash functions can be specified.
+//       Also, in future, we might want to be able to add to this on-the-fly (I forget the name of this particlar Bloom Filter, but there is one).
+//       Reason being that we might want to avoid re-computation of the filter each time a node requests it.
+//       That said, 100,000 only takes a few seconds.
+// Rough Flow: Node B contacts Node A and sends some basic stats (how many activities it has, etc - needs more thought)
+//             Node A replies with its own stats.
+//             Node B sends a Bloom Filter of its current activities.
+//             Node A replies with any that might be missing.
+
+interface Cell {
+  count: number;
+  idSum: bigint;
+  hashSum: bigint;
+}
+
+class InverseBloomFilter {
+  private size: number;
+  private cells: Cell[];
+  private items: Set<string>;
+
+  constructor(items: string[], expectedDiffSize?: number) {
+    this.items = new Set(items);
+    this.size = expectedDiffSize ?? Math.max(256, Math.ceil(items.length / 2));
+    this.cells = Array(this.size)
+      .fill(null)
+      .map(() => ({
+        count: 0,
+        idSum: 0n,
+        hashSum: 0n,
+      }));
+
+    for (const item of items) {
+      this.addItem(item);
+    }
+  }
+
+  private hashToIndex(value: string): number {
+    let hash = 2166136261;
+    for (let i = 0; i < value.length; i++) {
+      hash ^= value.charCodeAt(i);
+      hash +=
+        (hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24);
+      hash = hash >>> 0; // Keep as 32-bit unsigned
+    }
+    return hash % this.size;
+  }
+
+  private hashToId(value: string): bigint {
+    // Create a 64-bit max hash
+    let hash = 0n;
+    const mod = (1n << 63n) - 1n; // Use 63 bits to ensure we stay within signed 64-bit range
+
+    for (let i = 0; i < value.length; i++) {
+      hash = ((hash << 5n) - hash + BigInt(value.charCodeAt(i))) % mod;
+    }
+    return hash;
+  }
+
+  private addItem(item: string): void {
+    const idx = this.hashToIndex(item);
+    const itemId = this.hashToId(item);
+    const cell = this.cells[idx];
+
+    cell.count += 1;
+    cell.idSum ^= itemId;
+    cell.hashSum ^= itemId;
+  }
+
+  toBytes(): Uint8Array {
+    const cellSize = 4 + 8 + 8;
+    const bytes = new Uint8Array(8 + this.size * cellSize);
+    const view = new DataView(bytes.buffer);
+
+    view.setBigInt64(0, BigInt(this.size), true);
+
+    let offset = 8;
+    for (const cell of this.cells) {
+      view.setInt32(offset, cell.count, true);
+      view.setBigInt64(offset + 4, cell.idSum, true);
+      view.setBigInt64(offset + 12, cell.hashSum, true);
+      offset += cellSize;
+    }
+
+    return bytes;
+  }
+
+  private subtractFilter(otherBytes: Uint8Array): Cell[] {
+    const view = new DataView(otherBytes.buffer);
+    const otherSize = Number(view.getBigInt64(0, true));
+
+    if (otherSize !== this.size) {
+      throw new Error('Filter sizes do not match');
+    }
+
+    const result = Array(this.size);
+    let offset = 8;
+    const cellSize = 20;
+
+    for (let i = 0; i < this.size; i++) {
+      const otherCount = view.getInt32(offset, true);
+      const otherIdSum = view.getBigInt64(offset + 4, true);
+      const otherHashSum = view.getBigInt64(offset + 12, true);
+
+      result[i] = {
+        count: this.cells[i].count - otherCount,
+        idSum: this.cells[i].idSum ^ otherIdSum,
+        hashSum: this.cells[i].hashSum ^ otherHashSum,
+      };
+
+      offset += cellSize;
+    }
+
+    return result;
+  }
+
+  findMaybeMissing(otherFilterBytes: Uint8Array): string[] {
+    const diffCells = this.subtractFilter(otherFilterBytes);
+    const missing: string[] = [];
+
+    for (const item of this.items) {
+      const idx = this.hashToIndex(item);
+      const itemId = this.hashToId(item);
+      const cell = diffCells[idx];
+
+      if (cell.count === 1 && cell.idSum === itemId) {
+        missing.push(item);
+      }
+    }
+
+    return missing;
+  }
+}