Add experiment encyption clients. Rewrite demos. Reduce implementation code. Add value decoding. Rename to base-client. Type inference and generics

demo/experimental/encrypted-client-per-resource.ts

@@ -0,0 +1,176 @@
+import { subtle, getRandomValues } from "node:crypto";
+import { fromExtendedJson, toExtendedJson } from "@xo-cash/utils";
+import { sha256, binToBase64, binToHex } from "@bitauth/libauth";
+
+import { BaseSyncClient, type Resource, type ResourcesRecord } from "../../src/shared/base-client.js";
+
+type ResourceKey = {
+  resourceId: string;
+  keyMaterial: Uint8Array;
+};
+
+/**
+ * Encrypts each resource with its own key, where the resource ID itself is the secret.
+ *
+ * The caller-facing resource ID is hashed into a 32 byte AES key, and that key is hashed again
+ * to get the resource ID sent to the server. The server never sees the AES key or the original
+ * resource ID, so the resource ID must be unguessable (high entropy) for this to be secure.
+ */
+export class EncryptedClientPerResource<T = unknown> extends BaseSyncClient<T> {
+  /** Maps the hashed (server-side) resource ID to the original resource ID and its key */
+  private readonly resourceKeys = new Map<string, ResourceKey>();
+
+  constructor(private readonly client: BaseSyncClient<string>) {
+    super();
+
+    this.client.on('resource-instance-updated', async (message) => {
+      // Ignore updates for resources we don't hold the key for
+      const resourceKey = this.resourceKeys.get(message.resourceId);
+      if (!resourceKey) return;
+
+      try {
+        this.emit('resource-instance-updated', {
+          ...message,
+          resourceId: resourceKey.resourceId,
+          value: await EncryptedClientPerResource.decrypt(resourceKey.keyMaterial, message.value) as T,
+        });
+      } catch (error) {
+        this.emit('error', error instanceof Error ? error : new Error(String(error)));
+      }
+    });
+  }
+
+  /**
+   * Derives the AES key and server-side resource ID from a resource ID, and remembers the mapping.
+   * @param resourceId - The resource ID (the secret)
+   * @returns The hashed resource ID to send to the server
+   */
+  private hashResourceId(resourceId: string): string {
+    const keyMaterial = sha256.hash(new TextEncoder().encode(resourceId));
+    const hashedResourceId = binToHex(sha256.hash(keyMaterial));
+
+    this.resourceKeys.set(hashedResourceId, { resourceId, keyMaterial });
+
+    return hashedResourceId;
+  }
+
+  /**
+   * Gets the original resource ID and key for a hashed resource ID.
+   * @param hashedResourceId - The hashed (server-side) resource ID
+   * @returns The original resource ID and its key material
+   */
+  private getResourceKey(hashedResourceId: string): ResourceKey {
+    const resourceKey = this.resourceKeys.get(hashedResourceId);
+
+    if (!resourceKey) {
+      throw new Error(`No key found for resource ${hashedResourceId}`);
+    }
+
+    return resourceKey;
+  }
+
+  /**
+   * Decrypts every instance of every resource in a record, and maps the hashed resource IDs back to the original resource IDs.
+   * @param resources - The encrypted resources record, keyed by hashed resource ID
+   * @returns The resources record with decrypted values, keyed by original resource ID
+   */
+  private async decryptResources(resources: ResourcesRecord<string>): Promise<ResourcesRecord<T>> {
+    const entries = await Promise.all(Object.entries(resources).map(async ([ hashedResourceId, resource ]) => {
+      const { resourceId, keyMaterial } = this.getResourceKey(hashedResourceId);
+
+      const instances: Resource<T>[] = await Promise.all(resource.instances.map(async (instance) => ({
+        ...instance,
+        resourceId,
+        value: await EncryptedClientPerResource.decrypt(keyMaterial, instance.value) as T,
+      })));
+
+      return [ resourceId, { ...resource, instances } ] as const;
+    }));
+
+    return Object.fromEntries(entries);
+  }
+
+  public async connect(): Promise<void> {
+    return this.client.connect();
+  }
+
+  public async disconnect(): Promise<void> {
+    return this.client.disconnect();
+  }
+
+  public async readMany(resourceIds: string[]): Promise<ResourcesRecord<T>> {
+    const hashedResourceIds = resourceIds.map((resourceId) => this.hashResourceId(resourceId));
+
+    const encryptedResources = await this.client.readMany(hashedResourceIds);
+
+    return this.decryptResources(encryptedResources);
+  }
+
+  public async writeMany(resources: Record<string, T>): Promise<ResourcesRecord<T>> {
+    const encryptedEntries = await Promise.all(Object.entries(resources).map(async ([ resourceId, value ]) => {
+      const hashedResourceId = this.hashResourceId(resourceId);
+      const { keyMaterial } = this.getResourceKey(hashedResourceId);
+
+      return [ hashedResourceId, await EncryptedClientPerResource.encrypt(keyMaterial, value) ] as const;
+    }));
+
+    const response = await this.client.writeMany(Object.fromEntries(encryptedEntries));
+
+    return this.decryptResources(response);
+  }
+
+  public async subscribe(resourceId: string): Promise<void> {
+    return this.client.subscribe(this.hashResourceId(resourceId));
+  }
+
+  public async unsubscribe(resourceId: string): Promise<void> {
+    return this.client.unsubscribe(this.hashResourceId(resourceId));
+  }
+
+  /**
+   * Imports raw key material as an AES-GCM key
+   * @param keyMaterial - The 32 byte key material
+   * @returns The AES-GCM CryptoKey
+   */
+  private static importKey(keyMaterial: Uint8Array) {
+    return subtle.importKey('raw', Buffer.from(keyMaterial), { name: 'AES-GCM' }, false, ['encrypt', 'decrypt']);
+  }
+
+  /**
+   * Encrypts a value using AES-GCM with a random 12 byte IV
+   * @param keyMaterial - The encryption key material
+   * @param value - The value to encrypt
+   * @returns The IV and ciphertext as a base64 string
+   */
+  public static async encrypt(keyMaterial: Uint8Array, value: unknown): Promise<string> {
+    const key = await this.importKey(keyMaterial);
+    const iv = getRandomValues(new Uint8Array(12));
+
+    // Encode the value as extended JSON and then to UTF-8 bytes
+    const encodedValue = new TextEncoder().encode(toExtendedJson(value));
+
+    const encrypted = await subtle.encrypt({ name: 'AES-GCM', iv }, key, encodedValue);
+
+    // Prepend the IV so it can be recovered when decrypting
+    return binToBase64(Buffer.concat([ iv, new Uint8Array(encrypted) ]));
+  }
+
+  /**
+   * Decrypts a value encrypted by `encrypt`
+   * @param keyMaterial - The encryption key material
+   * @param value - The IV and ciphertext as a base64 string
+   * @returns The decrypted value
+   */
+  public static async decrypt(keyMaterial: Uint8Array, value: string): Promise<unknown> {
+    const key = await this.importKey(keyMaterial);
+
+    // Decode the base64 first: the IV is 12 bytes, not 12 base64 characters
+    const bytes = Buffer.from(value, 'base64');
+    const iv = bytes.subarray(0, 12);
+    const encryptedValue = bytes.subarray(12);
+
+    const decrypted = await subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedValue);
+
+    return fromExtendedJson(new TextDecoder().decode(decrypted));
+  }
+}

demo/experimental/encrypted-client.ts

@@ -0,0 +1,127 @@
+import { subtle, getRandomValues } from "node:crypto";
+import { fromExtendedJson, toExtendedJson } from "@xo-cash/utils";
+import { sha256, binToBase64 } from "@bitauth/libauth";
+
+import { BaseSyncClient, type ResourcesRecord } from "../../src/shared/base-client.js";
+
+/**
+ * Encrypts every resource with a single shared key.
+ */
+export class EncryptedClient<T = unknown> extends BaseSyncClient<T> {
+  private readonly encryptionKey: Uint8Array;
+
+  constructor(private readonly client: BaseSyncClient<string>, encryptionKey: Uint8Array | string) {
+    super();
+
+    // Hash the encryption key into 32 bytes if it is a string
+    this.encryptionKey = typeof encryptionKey === 'string' ? sha256.hash(new TextEncoder().encode(encryptionKey)) : encryptionKey;
+
+    this.client.on('resource-instance-updated', async (message) => {
+      try {
+        this.emit('resource-instance-updated', {
+          ...message,
+          value: await EncryptedClient.decrypt(this.encryptionKey, message.value) as T,
+        });
+      } catch (error) {
+        this.emit('error', error instanceof Error ? error : new Error(String(error)));
+      }
+    });
+  }
+
+  /**
+   * Decrypts every instance of every resource in a record.
+   * @param resources - The encrypted resources record
+   * @returns The resources record with decrypted values
+   */
+  private async decryptResources(resources: ResourcesRecord<string>): Promise<ResourcesRecord<T>> {
+    const entries = await Promise.all(Object.entries(resources).map(async ([ resourceId, resource ]) => {
+      const instances = await Promise.all(resource.instances.map(async (instance) => ({
+        ...instance,
+        value: await EncryptedClient.decrypt(this.encryptionKey, instance.value) as T,
+      })));
+
+      return [ resourceId, { ...resource, instances } ] as const;
+    }));
+
+    return Object.fromEntries(entries);
+  }
+
+  public async connect(): Promise<void> {
+    return this.client.connect();
+  }
+
+  public async disconnect(): Promise<void> {
+    return this.client.disconnect();
+  }
+
+  public async readMany(resourceIds: string[]): Promise<ResourcesRecord<T>> {
+    const encryptedResources = await this.client.readMany(resourceIds);
+
+    return this.decryptResources(encryptedResources);
+  }
+
+  public async writeMany(resources: Record<string, T>): Promise<ResourcesRecord<T>> {
+    const encryptedEntries = await Promise.all(Object.entries(resources).map(async ([ resourceId, value ]) => {
+      return [ resourceId, await EncryptedClient.encrypt(this.encryptionKey, value) ] as const;
+    }));
+
+    const response = await this.client.writeMany(Object.fromEntries(encryptedEntries));
+
+    return this.decryptResources(response);
+  }
+
+  public async subscribe(resourceId: string): Promise<void> {
+    return this.client.subscribe(resourceId);
+  }
+
+  public async unsubscribe(resourceId: string): Promise<void> {
+    return this.client.unsubscribe(resourceId);
+  }
+
+  /**
+   * Imports raw key material as an AES-GCM key
+   * @param keyMaterial - The 32 byte key material
+   * @returns The AES-GCM CryptoKey
+   */
+  private static importKey(keyMaterial: Uint8Array) {
+    return subtle.importKey('raw', Buffer.from(keyMaterial), { name: 'AES-GCM' }, false, ['encrypt', 'decrypt']);
+  }
+
+  /**
+   * Encrypts a value using AES-GCM with a random 12 byte IV
+   * @param keyMaterial - The encryption key material
+   * @param value - The value to encrypt
+   * @returns The IV and ciphertext as a base64 string
+   */
+  public static async encrypt(keyMaterial: Uint8Array, value: unknown): Promise<string> {
+    const key = await this.importKey(keyMaterial);
+    const iv = getRandomValues(new Uint8Array(12));
+
+    // Encode the value as extended JSON and then to UTF-8 bytes
+    const encodedValue = new TextEncoder().encode(toExtendedJson(value));
+
+    const encrypted = await subtle.encrypt({ name: 'AES-GCM', iv }, key, encodedValue);
+
+    // Prepend the IV so it can be recovered when decrypting
+    return binToBase64(Buffer.concat([ iv, new Uint8Array(encrypted) ]));
+  }
+
+  /**
+   * Decrypts a value encrypted by `encrypt`
+   * @param keyMaterial - The encryption key material
+   * @param value - The IV and ciphertext as a base64 string
+   * @returns The decrypted value
+   */
+  public static async decrypt(keyMaterial: Uint8Array, value: string): Promise<unknown> {
+    const key = await this.importKey(keyMaterial);
+
+    // Decode the base64 first: the IV is 12 bytes, not 12 base64 characters
+    const bytes = Buffer.from(value, 'base64');
+    const iv = bytes.subarray(0, 12);
+    const encryptedValue = bytes.subarray(12);
+
+    const decrypted = await subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedValue);
+
+    return fromExtendedJson(new TextDecoder().decode(decrypted));
+  }
+}

demo/shared.ts

@@ -0,0 +1,26 @@
+import type { BaseSyncClient, ResourcesRecord } from "../src/shared/base-client.js";
+
+export const writeMany = async (client: BaseSyncClient<{ message: string }>): Promise<ResourcesRecord<{ message: string }>> => {
+  // Create a group of resources with changes
+  const resources = Object.fromEntries(Array.from({ length: 10 }, (_, index) => [
+    `resource-${index}`,
+    {
+      message: `Hello, world! this is the minimal sync client using ${client.constructor.name} ${index}`,
+    },
+  ]));
+
+  // Write the resources to the server
+  const res = await client.writeMany(resources);
+
+  return res;
+}
+
+export const readMany = async (client: BaseSyncClient<{ message: string }>,): Promise<ResourcesRecord<{ message: string }>> => {
+  // Create a list of resource IDs
+  const resources = Array.from({ length: 10 }, (_, index) => `resource-${index}`);
+
+  // Read the resources from the server
+  const res = await client.readMany(resources);
+
+  return res;
+}

demo/sse.ts

@@ -1,33 +0,0 @@
-import { PrivateKey } from "@xo-cash/primitives";
-
-import { SSEClient } from '../src/sse/client.js';
-
-const testSSE = async () => {
-  const privateKey = PrivateKey.fromString('c440ce6ac5b63ae7bc4af9891b80b7a88ccdeebba5e43246885e9ec56f912ac2');
-  const client = new SSEClient('https://v2.sync.xo.harvmaster.com', privateKey);
-
-  client.on('message', (message) => {
-    console.log(message);
-  });
-
-  // Connect to the sync server
-  await client.connect();
-
-  // Subscribe to the resource
-  await client.subscribe('test');
-
-  // Write a new value to the resource
-  await client.write('test', {'message': 'Hello, world! this is the minimal sync client using SSE'});
-
-  // Read the current value of the resource
-  const data = await client.read('test');
-  console.log(data);
-
-  // If the user presses Ctrl+C, disconnect the client and exit the program
-  process.on('SIGINT', async () => {
-    await client.disconnect();
-    process.exit(0);
-  });
-}
-
-testSSE();

demo/suite.ts

@@ -0,0 +1,89 @@
+import { PrivateKey } from "@xo-cash/primitives";
+
+// Core clients
+import type { BaseSyncClient } from "../src/shared/base-client.js";
+import { SSEClient } from '../src/sse/client.js';
+import { WsClient } from "../src/ws/client.js";
+
+// Experimental clients
+import { EncryptedClientPerResource } from "./experimental/encrypted-client-per-resource.js";
+import { EncryptedClient } from "./experimental/encrypted-client.js";
+
+// Shared functions
+import { writeMany, readMany } from "./shared.js";
+
+const privateKey = PrivateKey.fromString('c440ce6ac5b63ae7bc4af9891b80b7a88ccdeebba5e43246885e9ec56f912ac2');
+
+const clients = {
+  sse: new SSEClient<{ message: string }>('https://dev.v2.sync.xo.harvmaster.com', privateKey),
+  ws: new WsClient<{ message: string }>('wss://dev.v2.sync.xo.harvmaster.com', privateKey),
+  encrypted: new EncryptedClient<{ message: string }>(new SSEClient<string>('https://dev.v2.sync.xo.harvmaster.com', privateKey), 'secret'),
+  encryptedPerResource: new EncryptedClientPerResource<{ message: string }>(new SSEClient<string>('https://dev.v2.sync.xo.harvmaster.com', privateKey)),
+};
+
+// Test the suite for a given client
+const testSuite = async (client: BaseSyncClient<{ message: string }>) => {
+  client.on('resource-instance-updated', (message) => {
+    console.log(message);
+  });
+
+  // Connect to the sync server
+  await client.connect();
+
+  // Subscribe to the resource
+  await client.subscribe('test');
+
+  // Write a new value to the resource
+  await client.write('test', {'message': 'Hello, world! this is the minimal sync client using ' + client.constructor.name});
+
+  // Read the current value of the resource
+  const data = await client.read('test');
+  console.log(data.map((resource) => resource.value.message));
+
+  // Use the shared functions to write and read many resources
+  const writtenResources = await writeMany(client);
+  console.dir(writtenResources, { depth: null });
+
+  const readResources = await readMany(client);
+  console.dir(readResources, { depth: null });
+
+  // If the user presses Ctrl+C, disconnect the client and exit the program
+  process.on('SIGINT', async () => {
+    await client.disconnect();
+    process.exit(0);
+  });
+}
+
+// Get the clients to test
+const selectClients = (params: string[]): BaseSyncClient<{ message: string }>[] => {
+  // If no params are provided, test all clients
+  const names = params.length > 0 ? params : Object.keys(clients);
+
+  // Find values that are unknown to the clients object
+  const unknown = names.filter((n) => !(n in clients));
+  if (unknown.length > 0) {
+    console.error(`Invalid client type(s): ${unknown.join(', ')}. Expected: ${Object.keys(clients).join(', ')}`);
+    process.exit(1);
+  }
+
+  // Return the clients that are known to the clients object
+  return names.map((n) => clients[n as keyof typeof clients]);
+}
+
+// Get the params
+const params = process.argv.slice(2);
+
+// Get the clients to test
+const clientsToTest = selectClients(params);
+
+// Test the clients
+for (const client of clientsToTest) {
+  await testSuite(client);
+}
+
+// Wait for the user to press Ctrl+C
+await new Promise((resolve) => process.on('SIGINT', resolve));
+
+// Exit the program
+console.log('Exiting...');
+process.exit(0);

demo/tui.ts

@@ -0,0 +1,56 @@
+import { PrivateKey } from "@xo-cash/primitives";
+
+import type { BaseSyncClient } from "../src/shared/base-client.js";
+
+import { SSEClient } from "../src/sse/client.js";
+import { EncryptedClientPerResource } from "./experimental/encrypted-client-per-resource.js";
+
+const privateKey = PrivateKey.fromString('c440ce6ac5b63ae7bc4af9891b80b7a88ccdeebba5e43246885e9ec56f912ac2');
+
+// Define some formatting helpers
+const formatResourceId = (resourceId: string) => {
+  return `\x1b[32m${resourceId}\x1b[0m`;
+}
+const formatMessage = (message: string) => {
+  return `\x1b[34m${message}\x1b[0m`;
+}
+const resetFormatting = () => {
+  return `\x1b[0m`;
+}
+
+const startTUI = async (client: BaseSyncClient<string>) => {
+  // Listen and log out the resource change
+  client.on('resource-instance-updated', (message) => {
+    // Print the message with colours
+    console.log(`${formatResourceId(message.resourceId)}:${formatResourceId(message.publicKey)} updated: ${formatMessage(message.value)}${resetFormatting()}`);
+  });
+
+  // Connect to the sync server and subscribe to the resource
+  await client.connect();
+  await client.subscribe('encrypted-test');
+  
+  // Start the TUI loop
+  while (true) {
+    // Print the prompt
+    process.stdout.write('> ');
+
+    // Wait for the user to enter a command
+    const data = await new Promise((resolve) => process.stdin.once('data', resolve));
+
+    // If the user entered a command, write it to the resource
+    if (data) {
+      const message = data.toString().trim();
+      await client.write('encrypted-test', message);
+    }
+  }
+}
+
+const clients = {
+  'sse': new SSEClient<string>('https://dev.v2.sync.xo.harvmaster.com', privateKey),
+  'encrypted': new EncryptedClientPerResource<string>(new SSEClient<string>('https://dev.v2.sync.xo.harvmaster.com', privateKey)),
+}
+
+const params = process.argv.slice(2);
+const client = params[0] === 'false' ? clients.sse : clients.encrypted;
+
+startTUI(client);

demo/ws.ts

@@ -1,40 +0,0 @@
-import { PrivateKey } from "@xo-cash/primitives";
-
-import { WsClient } from '../src/ws/client.js';
-
-const testWS = async () => {
-  const privateKey = PrivateKey.fromString('c440ce6ac5b63ae7bc4af9891b80b7a88ccdeebba5e43246885e9ec56f912ac2');
-  const client = new WsClient('https://v2.sync.xo.harvmaster.com', privateKey, {
-    // onMessage: (message) => {
-    //   console.log(toExtendedJson(message));
-    // },
-    onError: (error) => {
-      console.error(error);
-    },
-  });
-
-  client.on('message', (message) => {
-    console.log(message);
-  });
-
-  // Connect to the sync server
-  await client.connect();
-
-  // Subscribe to the resource
-  client.subscribe('test');
-
-  // Write a new value to the resource
-  await client.write('test', {'message': 'Hello, world! this is the minimal sync client using WS'});
-  
-  // Read the current value of the resource
-  const data = await client.read('test');
-  console.log(data);
-
-  // If the user presses Ctrl+C, disconnect the client and exit the program
-  process.on('SIGINT', async () => {
-    await client.disconnect();
-    process.exit(0);
-  });
-}
-
-testWS();