Doc commit tag conventions

packages/stack/src/storage/base-store.ts

@@ -1,4 +1,4 @@
-import { Encrypter, EventEmitter } from '../utils';
+import { Encrypter, EventEmitter, InverseBloomFilter } from '../utils';
 
 export type StorageOpts = {
   syncInMemory?: boolean;
@@ -268,4 +268,15 @@ export abstract class BaseStore extends EventEmitter<StoreEvents> {
       },
     };
   }
+
+  async computeInverseBloomFilter(expectedDiffSize?: number) {
+    // Get all keys.
+    const keys = await this.keys();
+
+    // Construct an Inverse Bloom Filter using the keys.
+    const ibf = InverseBloomFilter.from(keys, expectedDiffSize);
+
+    // Return the bloom filter.
+    return ibf;
+  }
 }

packages/stack/src/storage/index.ts

@@ -1,3 +1,4 @@
 export * from './base-store.js';
 export * from './store-inmemory.js';
 export * from './store-inmemory-synced.js';
+export * from './store-remote-http.js';

packages/stack/src/storage/store-composite.ts

@@ -0,0 +1,146 @@
+import { BaseStore } from './base-store.js';
+
+/**
+ * A store that accepts multiple stores and syncs them by key.
+ *
+ * @remarks
+ *
+ * Syncing is done by key - if values differ, then this will remain undetected.
+ * This is by design - local stores will sometimes prune irrelevant values as an optimization (while retaining the key).
+ * However, this cannot be done on the encrypted stores as the encrypted stores cannot inspect the data to know whether relevant/irrelevant.
+ * UUIDv7 keys are used to ensure uniqueness of keys.
+ */
+export class StoreComposite extends BaseStore {
+  static async from(localStore: BaseStore, remoteStores: Array<BaseStore>) {
+    const compositeStore = new this(localStore, remoteStores);
+    await compositeStore.sync();
+    return compositeStore;
+  }
+
+  constructor(
+    private localStore: BaseStore,
+    private remoteStores: Array<BaseStore>,
+  ) {
+    super();
+
+    // Local store events propagate to remote stores
+    this.localStore.on('set', async (payload) => {
+      // Propagate to all remote stores in parallel
+      await Promise.all(
+        this.remoteStores.map((remoteStore) =>
+          remoteStore.set(payload.key, payload.value),
+        ),
+      );
+    });
+
+    // Local store events propagate to remote stores
+    this.localStore.on('delete', async (payload) => {
+      // Propagate to all remote stores in parallel
+      await Promise.all(
+        this.remoteStores.map((remoteStore) => remoteStore.delete(payload.key)),
+      );
+    });
+
+    // Local store events propagate to remote stores
+    this.localStore.on('clear', async () => {
+      // Propagate to all remote stores in parallel
+      await Promise.all(
+        this.remoteStores.map((remoteStore) => remoteStore.clear()),
+      );
+    });
+
+    // Remote store events propagate to local store
+    for (const remoteStore of this.remoteStores) {
+      remoteStore.on('set', async (payload) => {
+        // Skip if the local store already has the key
+        if (await this.localStore.has(payload.key)) return;
+
+        await this.localStore.set(payload.key, payload.value);
+      });
+
+      remoteStore.on('delete', async (payload) => {
+        if (await this.localStore.has(payload.key)) {
+          await this.localStore.delete(payload.key);
+        }
+      });
+    }
+  }
+
+  async all<T>() {
+    return await this.localStore.all<T>();
+  }
+
+  async keys() {
+    return await this.localStore.keys();
+  }
+
+  async get<T>(key: string) {
+    return await this.localStore.get<T>(key);
+  }
+
+  async set<T>(key: string, value: T) {
+    await this.localStore.set(key, value);
+    this.emit('set', { key, value });
+  }
+
+  async delete(key: string) {
+    await this.localStore.delete(key);
+    this.emit('delete', { key });
+  }
+
+  async clear() {
+    await this.localStore.clear();
+    this.emit('clear', undefined);
+  }
+
+  async has(key: string) {
+    return await this.localStore.has(key);
+  }
+
+  async close() {
+    await Promise.all([
+      this.localStore.close(),
+      ...this.remoteStores.map((store) => store.close()),
+    ]);
+  }
+
+  async sync(): Promise<void> {
+    // Sync all remote stores in parallel
+    await Promise.all(
+      this.remoteStores.map(async (remoteStore) => {
+        // TODO: We want to negotiate using an Inverse Bloom Filter first to optimize the sync.
+        //       Currently, this is doing a naive set comparison.
+
+        // Get all keys from the remote store
+        const remoteKeys = await remoteStore.keys();
+        const localKeys = await this.localStore.keys();
+
+        // Find keys missing in local store
+        const missingLocalKeys = remoteKeys.filter(
+          (key) => !localKeys.includes(key),
+        );
+
+        // Find keys missing in remote store
+        const missingRemoteKeys = localKeys.filter(
+          (key) => !remoteKeys.includes(key),
+        );
+
+        // Sync missing keys from remote to local (in parallel)
+        await Promise.all(
+          missingLocalKeys.map(async (key) => {
+            const value = await remoteStore.get(key);
+            await this.localStore.set(key, value);
+          }),
+        );
+
+        // Sync missing keys from local to remote (in parallel)
+        await Promise.all(
+          missingRemoteKeys.map(async (key) => {
+            const value = await this.localStore.get(key);
+            await remoteStore.set(key, value);
+          }),
+        );
+      }),
+    );
+  }
+}

packages/stack/src/storage/store-remote-http.ts

@@ -0,0 +1,189 @@
+import { BaseStore } from './base-store.js';
+import { Encrypter, SSEClient } from '../utils';
+import { Bytes, PrivateKey, PublicKey } from '@xocash/primitives';
+import { pack, unpack } from 'msgpackr';
+import { z } from 'zod';
+
+export class StoreRemoteHTTP extends BaseStore {
+  static async from(
+    baseURL: string,
+    privateKey: PrivateKey,
+    encrypter: Encrypter,
+  ) {
+    const store = new this(baseURL, privateKey, encrypter);
+
+    await store.subscribeEvents();
+
+    return store;
+  }
+
+  // Optimizations/Caches.
+  protected publicKey: PublicKey;
+
+  constructor(
+    protected baseURL: string,
+    protected privateKey: PrivateKey,
+    protected encrypter: Encrypter,
+  ) {
+    super();
+
+    // Derive the public key and cache it to optimize performance.
+    this.publicKey = privateKey.derivePublicKey();
+  }
+
+  async keys() {
+    const response = await this.makeRequest('keys');
+    return response;
+  }
+
+  async get<T>(key: string) {
+    const response = await this.makeRequest('get', { key });
+
+    if (!response.value) {
+      return undefined;
+    }
+
+    // Decrypt the value using the encrypter
+    const decryptedValue = await this.encrypter.decrypt(
+      new Uint8Array(response.value),
+    );
+
+    // Parse the decrypted MessagePack value
+    return unpack(decryptedValue) as T;
+  }
+
+  async set<T>(key: string, value: T): Promise<void> {
+    // Pack the encoded value with MessagePack and encrypt it
+    const packedValue = pack(value);
+    const encryptedValue = await this.encrypter.encrypt(packedValue);
+
+    await this.makeRequest('set', {
+      key,
+      value: encryptedValue, // Convert Uint8Array to regular array for MessagePack
+    });
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.makeRequest('delete', { key });
+  }
+
+  async subscribeEvents() {
+    const timestamp = Math.floor(Date.now() / 1000);
+    const signature = await StoreRemoteHTTP.signRequest(
+      this.privateKey.toHex(),
+      timestamp,
+      'events',
+      {},
+    );
+
+    const headers = {
+      'Content-Type': 'application/json',
+      Accept: 'application/json',
+      'X-Public-Key': this.publicKey.toHex(),
+      'X-Signature': Bytes.fromBytes(signature).toHex(),
+      'X-Timestamp': timestamp.toString(),
+    };
+
+    const sseClient = new SSEClient(`${this.baseURL}events`, {
+      method: 'POST',
+      headers,
+      onMessage: (e) => {
+        // Validate the payload using Zod.
+        // TODO: Use msgpackr
+        const payload = StoreRemoteHTTPEventSchema.parse(JSON.parse(e.data));
+
+        // Emit the event (update) to any listeners.
+        this.emit(payload.action, {
+          key: payload.key,
+          value: payload.value,
+        });
+      },
+      body: JSON.stringify({}),
+    });
+
+    await sseClient.connect();
+  }
+
+  private async makeRequest(endpoint: string, payload: object = {}) {
+    const timestamp = Math.floor(Date.now() / 1000);
+    const signature = await StoreRemoteHTTP.signRequest(
+      this.privateKey.toHex(),
+      timestamp,
+      endpoint,
+      payload,
+    );
+
+    const headers = {
+      'Content-Type': 'application/msgpackr',
+      Accept: 'application/msgpackr',
+      'X-Public-Key': this.publicKey.toHex(),
+      'X-Signature': Bytes.fromBytes(signature).toHex(),
+      'X-Timestamp': timestamp.toString(),
+    };
+
+    const requestOptions = {
+      method: 'POST',
+      headers,
+      body: payload ? pack(payload) : pack({}),
+    };
+
+    // Use the URL class to compile the full path.
+    // NOTE: We do this to cater for those situations where the base URL might contain a trailing slash.
+    //       E.g. https://wallet-sync.xo.cash/store/ vs https://wallet-sync.xo.cash/store
+    // TODO: THis is broken and currently requires a trailing slash in the BaseURL!
+    const requestURL = new URL(endpoint, this.baseURL).toString();
+
+    const response = await fetch(requestURL, requestOptions);
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const responseBuffer = await response.arrayBuffer();
+    return unpack(new Uint8Array(responseBuffer));
+  }
+
+  static async signRequest(
+    privateKeyHex: string,
+    timestamp: number,
+    endpoint: string,
+    payload?: object,
+  ) {
+    const messageBin = pack([endpoint, timestamp, payload]);
+
+    return PrivateKey.fromHex(privateKeyHex).signMessageSchnorr(messageBin);
+  }
+
+  static async verifySignature(
+    publicKeyHex: string,
+    signatureHex: string,
+    timestamp: number,
+    endpoint: string,
+    payload?: object,
+  ) {
+    const now = Math.floor(Date.now() / 1000);
+    if (Math.abs(now - timestamp) > 60) {
+      throw new Error(`Request has expired`);
+    }
+
+    const messageBin = pack([endpoint, timestamp, payload]);
+
+    const isValidSignature = PublicKey.fromHex(
+      publicKeyHex,
+    ).verifyMessageSchnorr(messageBin, signatureHex);
+
+    if (!isValidSignature) {
+      throw new Error(`The provided signature is invalid`);
+    }
+  }
+}
+
+export const StoreRemoteHTTPEventSchema = z.object({
+  publicKey: z.string(),
+  action: z.enum(['set', 'delete', 'clear']),
+  key: z.string(),
+  value: z.string(),
+  timestamp: z.number(),
+});
+
+export type StoreRemoteHTTPEvent = z.infer<typeof StoreRemoteHTTPEventSchema>;