Signed requests: X-Nonce header; composite: sync skips unconfirmed writes

- Requests carry a random X-Nonce, signed as ${timestamp}:${nonce}:${path}:${body}. The
  server rejects a nonce the same key already used within the timestamp window. Identical
  requests at the same moment (two tabs, parallel reads) are now distinct, replacing the
  per-process increasing-timestamp workaround.
- Composite: a sync no longer reverts keys with local writes the source hasn't confirmed
  yet (in flight or queued); they briefly disappeared when a sync's scan ran before the
  write reached the source. Pending keys are reloaded from a persistent queue on start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Qxbkp4tA7G5SCWjzWuxmE

src/kv-storage/kv-storage-composite.ts

@@ -51,7 +51,8 @@ export interface KVStorageComposite<S extends Spec = AnySpec> {
  *   client's own writes therefore changes nothing.
  * - A write the source can't take (`StorageUnavailableError`) is applied locally and stored in
  *   the `queue` store; later writes queue behind it, so order is kept.
- * - `sync()` replays the queue, then copies what changed on the source into the replicas. It runs
+ * - `sync()` replays the queue, then copies what changed on the source into the replicas, except
+ *   keys with local writes the source hasn't confirmed yet (in flight or queued). It runs
  *   on start, when the source reconnects (`resync`), and after a retry delay while offline.
  * - Reads wait for the first sync (until one succeeds, they go to the source). Whether a sync has
  *   completed is kept in the `queue` store, so with a persistent one a restart reads offline.
@@ -77,9 +78,11 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
     queued: number
     /** Remote changes are applied one after another. */
     applying: Promise<unknown>
+    /** Keys with local writes the source hasn't confirmed yet (in flight or queued) → count. */
+    pending: Map<string, number>
     retry?: ReturnType<typeof setTimeout>
     offs: OffCallback[]
-  } = { synced: false, queued: 0, applying: Promise.resolve(), offs: [] }
+  } = { synced: false, queued: 0, applying: Promise.resolve(), pending: new Map(), offs: [] }
 
   constructor(options: KVStorageCompositeOptions) {
     super()
@@ -140,20 +143,25 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
     const keys = changes.map(({ key }) => key)
     this.checkRange(keys)
     await this.start()
-    // Applied locally first, so the source relaying this write back finds nothing new.
+    // Applied locally first, so the source relaying this write back finds nothing new. Until the
+    // source confirms it, a sync must not replace it with the source's (older) value.
     const previous = await this.replicas[0]!.readMany(keys)
+    this.markPending(keys, 1)
     await this.apply(changes)
     // While syncing or with writes queued, queue this one too so the source sees writes in order.
-    if (this.state.queued === 0 && !this.state.syncing) {
-      try {
-        return await this.source.writeMany(changes)
-      } catch (error) {
-        if (!(error instanceof StorageUnavailableError)) {
-          await this.apply(keys.map((key, i) => ({ key, value: previous[i] })))
-          throw error
-        }
-      }
+    if (this.state.queued > 0 || this.state.syncing) return this.enqueue(changes)
+    try {
+      await this.source.writeMany(changes)
+    } catch (error) {
+      if (error instanceof StorageUnavailableError) return this.enqueue(changes)
+      this.markPending(keys, -1)
+      await this.apply(keys.map((key, i) => ({ key, value: previous[i] })))
+      throw error
     }
+    this.markPending(keys, -1)
+  }
+
+  private async enqueue(changes: KVChange[]): Promise<void> {
     await this.writes.set(this.ids.generate(), changes)
     this.state.queued++
     this.retryLater()
@@ -167,7 +175,13 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
 
   private async open(): Promise<void> {
     this.state.synced = (await this.queue.get('synced')) === true
-    for await (const _ of this.writes.keys()) this.state.queued++
+    for await (const [, changes] of this.writes.entries()) {
+      this.state.queued++
+      this.markPending(
+        (changes as KVChange[]).map(({ key }) => key),
+        1,
+      )
+    }
     await this.seed()
     this.state.offs.push(
       this.source.on('change', (event) => {
@@ -192,7 +206,10 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
     try {
       await this.replay()
       let batch: KVChange[] = []
-      for await (const change of this.diff()) if (batch.push(change) >= 500) await this.apply(batch.splice(0), true)
+      for await (const change of this.diff()) {
+        if (this.state.pending.has(change.key)) continue // a newer local write, not yet sent
+        if (batch.push(change) >= 500) await this.apply(batch.splice(0), true)
+      }
       await this.apply(batch, true)
       await this.replay() // writes made during the diff: send them, and re-apply them over it
       await this.queue.set('synced', true)
@@ -224,6 +241,10 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
       }
       await this.writes.delete(next[0])
       this.state.queued--
+      this.markPending(
+        changes.map(({ key }) => key),
+        -1,
+      )
     }
   }
 
@@ -275,6 +296,14 @@ export class KVStorageComposite<S extends Spec = AnySpec> extends BaseKVStorage<
     await flush()
   }
 
+  private markPending(keys: string[], delta: 1 | -1): void {
+    for (const key of keys) {
+      const count = (this.state.pending.get(key) ?? 0) + delta
+      if (count > 0) this.state.pending.set(key, count)
+      else this.state.pending.delete(key)
+    }
+  }
+
   private retryLater(): void {
     clearTimeout(this.state.retry)
     this.state.retry = setTimeout(() => void this.sync(), this.retryMs)

src/utils/request-auth.ts

@@ -19,6 +19,7 @@ export interface SignatureVerifier {
 export const AUTH_HEADERS = {
   publicKey: 'X-Public-Key',
   timestamp: 'X-Timestamp',
+  nonce: 'X-Nonce',
   signature: 'X-Signature',
   algorithm: 'X-Signature-Algorithm',
 } as const
@@ -30,17 +31,18 @@ export interface RequestAuthOptions {
 }
 
 /**
- * Signed-request authentication. The client signs `${timestamp}:${path}:${body}` (path = URL
- * pathname, body = the exact request body) and sends the public key, timestamp, signature (hex)
- * and algorithm as headers. The server checks the timestamp is recent, verifies the signature
- * with the verifier for that algorithm, and rejects a signature it has already seen within the
- * timestamp window (replay protection).
+ * Signed-request authentication. The client signs `${timestamp}:${nonce}:${path}:${body}` (nonce =
+ * random per request, path = URL pathname, body = the exact request body) and sends the public
+ * key, timestamp, nonce, signature (hex) and algorithm as headers. The server checks the
+ * timestamp is recent, verifies the signature with the verifier for that algorithm, and rejects a
+ * nonce the same key already used within the timestamp window (replay protection). The nonce also
+ * keeps identical requests made at the same moment distinct (Ed25519 signatures are deterministic).
  */
 export class RequestAuth {
   private readonly verifiers: Map<string, SignatureVerifier>
   private readonly maxSkewMs: number
   private readonly now: () => number
-  /** Signatures accepted within the window → when they expire; insertion order ≈ expiry order. */
+  /** `<publicKey>:<nonce>` accepted within the window → when it expires; insertion order ≈ expiry order. */
   private readonly seen = new Map<string, number>()
 
   constructor(verifiers: SignatureVerifier[], options: RequestAuthOptions = {}) {
@@ -50,27 +52,19 @@ export class RequestAuth {
     this.now = options.now ?? Date.now
   }
 
-  static payload(timestamp: string, path: string, body: string): Uint8Array {
-    return new TextEncoder().encode(`${timestamp}:${path}:${body}`)
+  static payload(timestamp: string, nonce: string, path: string, body: string): Uint8Array {
+    return new TextEncoder().encode(`${timestamp}:${nonce}:${path}:${body}`)
   }
 
-  /** Last timestamp used per public key in this process (see `headers`). */
-  private static readonly lastTimestamps = new Map<string, number>()
-
-  /**
-   * Auth headers for a request to `path` with `body`. Timestamps per public key strictly increase
-   * within a process: Ed25519 signatures are deterministic, so two identical requests in the same
-   * millisecond would otherwise carry the same signature, and the second would be rejected as a
-   * replay.
-   */
+  /** Auth headers for a request to `path` with `body`. */
   static async headers(signer: RequestSigner, path: string, body: string, now: number = Date.now()): Promise<Record<string, string>> {
-    const time = Math.max(now, (RequestAuth.lastTimestamps.get(signer.publicKey) ?? 0) + 1)
-    RequestAuth.lastTimestamps.set(signer.publicKey, time)
-    const timestamp = String(time)
-    const signature = await signer.sign(RequestAuth.payload(timestamp, path, body))
+    const timestamp = String(now)
+    const nonce = Hex.encode(crypto.getRandomValues(new Uint8Array(16)))
+    const signature = await signer.sign(RequestAuth.payload(timestamp, nonce, path, body))
     return {
       [AUTH_HEADERS.publicKey]: signer.publicKey,
       [AUTH_HEADERS.timestamp]: timestamp,
+      [AUTH_HEADERS.nonce]: nonce,
       [AUTH_HEADERS.signature]: Hex.encode(signature),
       [AUTH_HEADERS.algorithm]: signer.algorithm,
     }
@@ -80,32 +74,35 @@ export class RequestAuth {
   async verify(headers: Headers, path: string, body: string): Promise<string> {
     const publicKey = headers.get(AUTH_HEADERS.publicKey)
     const timestamp = headers.get(AUTH_HEADERS.timestamp)
+    const nonce = headers.get(AUTH_HEADERS.nonce)
     const signature = headers.get(AUTH_HEADERS.signature)
     const algorithm = headers.get(AUTH_HEADERS.algorithm)
-    if (!publicKey || !timestamp || !signature || !algorithm) throw new AuthError(401, 'Missing authentication headers')
+    if (!publicKey || !timestamp || !nonce || !signature || !algorithm) throw new AuthError(401, 'Missing authentication headers')
     const verifier = this.verifiers.get(algorithm)
     if (!verifier) throw new AuthError(401, `Unsupported signature algorithm "${algorithm}"`)
     if (!/^[0-9a-f]+$/.test(publicKey)) throw new AuthError(401, 'Public key must be lowercase hex')
+    if (!/^[0-9A-Za-z_-]{16,128}$/.test(nonce)) throw new AuthError(401, 'Nonce must be 16-128 characters of [0-9A-Za-z_-]')
     const now = this.now()
     const time = Number(timestamp)
     if (!Number.isInteger(time) || Math.abs(now - time) > this.maxSkewMs) throw new AuthError(401, 'Timestamp outside the accepted window')
     this.forgetExpired(now)
-    if (this.seen.has(signature)) throw new AuthError(401, 'Replayed request')
+    const used = `${publicKey}:${nonce}`
+    if (this.seen.has(used)) throw new AuthError(401, 'Replayed request')
     let valid = false
     try {
-      valid = await verifier.verify(publicKey, RequestAuth.payload(timestamp, path, body), Hex.decode(signature))
+      valid = await verifier.verify(publicKey, RequestAuth.payload(timestamp, nonce, path, body), Hex.decode(signature))
     } catch {
       valid = false
     }
     if (!valid) throw new AuthError(401, 'Invalid signature')
-    this.seen.set(signature, time + this.maxSkewMs)
+    this.seen.set(used, time + this.maxSkewMs)
     return publicKey
   }
 
   private forgetExpired(now: number): void {
-    for (const [signature, expires] of this.seen) {
+    for (const [used, expires] of this.seen) {
       if (expires >= now) break
-      this.seen.delete(signature)
+      this.seen.delete(used)
     }
   }
 }