Next-edit: n-gram speculative decoding on the companion server (0.6.2)

The rewritten region is mostly a copy of the current one, so prompt-lookup drafts are
accepted in long runs. Greedy decoding keeps the output identical. Laptop CPU, 160-token
region rewrite: generation 1.76 s -> 0.37 s. Timings now include errorsMs/promptMs/genMs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J39imRDT8L7xYbWzsVWd8A

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Extension versions are released as `vscode-v<version>` on the [Releases](https://git.harvmaster.com/Harvmaster/autocomplete-model/releases) page. Models are versioned by their name in [`models/manifest.json`](models/manifest.json).
 
+## vscode 0.6.2
+
+- **Next-edit is ~3–5× faster:** the next-edit server uses n-gram speculative decoding (the rewritten region is mostly
+  a copy of the current one, so drafted tokens are verified in batches). Same output, since decoding is greedy.
+  Rewriting a 13-line region on a laptop CPU: ~2.1 s → ~0.7 s (~0.4 s when the prompt is cached).
+- Next-edit timings in "Show Last Prompt" now split TS errors / prompt / generation time.
+
 ## vscode 0.6.1
 
 - **New default model `stage-d`**, with **`stage-e` for next-edit**. Both download automatically (~530 MB each); the

daemon/src/config.ts

@@ -32,6 +32,9 @@ export interface DaemonConfig {
     ctx: number
     /** Extra flags, e.g. ["-ngl", "99"] on Apple Silicon, ["-t", "16"] on the laptop CPU. */
     args: string[]
+    /** Extra flags for the next-edit companion server only. Default: n-gram speculative decoding (the output is
+     *  mostly a copy of the region, so drafts from the prompt are accepted; exact with greedy sampling). */
+    nextEditArgs?: string[]
   }
   models: Record<string, ModelProfile>
   defaultModel: string

daemon/src/main.ts

@@ -52,7 +52,8 @@ function loadModel(name: string) {
 }
 
 // Next-edit companion model (profile.nextEditModel): a second llama-server, started on first use.
-const neLlama = new LlamaServer({ ...config.llamaServer, port: config.llamaServer.port + 1 })
+const NEXT_EDIT_SPECULATION = ['--spec-type', 'ngram-simple', '--spec-draft-n-max', '64', '--spec-ngram-simple-size-n', '3']
+const neLlama = new LlamaServer({ ...config.llamaServer, port: config.llamaServer.port + 1, args: [...config.llamaServer.args, ...(config.llamaServer.nextEditArgs ?? NEXT_EDIT_SPECULATION)] })
 let neLoading: Promise<void> | null = null
 /** The server that answers next-edit for the active model, or an error message. */
 async function nextEditServer(): Promise<LlamaServer | string> {
@@ -273,9 +274,11 @@ async function handleNextEdit(body: NextEditRequest, res: http.ServerResponse, c
   const endLine = Math.min(lines.length, cursorLine + o.regionAfter + 1)
   const recentEdits = edits.observe(`${body.workspaceRoot ?? body.repo}/${body.path}`, body.path, body.text, cursorLine)
   const lang = symbolLang(body.languageId ?? '')
+  const tErrors = performance.now()
   const errors = body.workspaceRoot && lang
     ? await symbols.errors({ root: body.workspaceRoot, filePath: path.join(body.workspaceRoot, body.path), lang, text: body.text, cursor: body.cursor, waitMs: 300 })
     : undefined
+  const errorsMs = performance.now() - tErrors
   const ids = builder.buildNextEdit({ repo: body.repo, path: body.path, file: body.text, regionStartLine: startLine, regionEndLine: endLine, cursor: body.cursor, recentEdits: recentEdits || undefined, errors },
     { beforeTokens: o.beforeTokens, afterTokens: o.afterTokens })
 
@@ -294,7 +297,7 @@ async function handleNextEdit(body: NextEditRequest, res: http.ServerResponse, c
   const response = {
     requestId: body.requestId, noEdit, startLine, endLine, oldText, newText: noEdit ? null : newText,
     recentEdits, errors: errors ?? null, model: profile.nextEditModel ?? state.model,
-    timings: { promptTokens: ids.length, processedTokens: result.timings?.prompt_n ?? null, genTokens: result.timings?.predicted_n ?? null, totalMs: round(performance.now() - t0) },
+    timings: { promptTokens: ids.length, processedTokens: result.timings?.prompt_n ?? null, genTokens: result.timings?.predicted_n ?? null, errorsMs: round(errorsMs), promptMs: round(result.timings?.prompt_ms ?? 0), genMs: round(result.timings?.predicted_ms ?? 0), totalMs: round(performance.now() - t0) },
   }
   remember({ kind: 'next-edit', requestId: body.requestId, path: body.path, model: state.model, prompt: builder.decode(ids), raw, noEdit, timings: response.timings })
   json(res, 200, response)

extensions/vscode/package.json

@@ -3,7 +3,7 @@
   "displayName": "Local Autocomplete",
   "description": "Inline code completion from a local fine-tuned model via the acd daemon",
   "publisher": "harvmaster",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "license": "MIT",
   "engines": {
     "vscode": "^1.90.0"