diff --git a/CHANGELOG.md b/CHANGELOG.md index 0365739..1b531c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ Extension versions are released as `vscode-v` on the [Releases](https://git.harvmaster.com/Harvmaster/autocomplete-model/releases) page. Models are versioned by their name in [`models/manifest.json`](models/manifest.json). +## vscode 0.6.2 + +- **Next-edit is ~3–5× faster:** the next-edit server uses n-gram speculative decoding (the rewritten region is mostly + a copy of the current one, so drafted tokens are verified in batches). Same output, since decoding is greedy. + Rewriting a 13-line region on a laptop CPU: ~2.1 s → ~0.7 s (~0.4 s when the prompt is cached). +- Next-edit timings in "Show Last Prompt" now split TS errors / prompt / generation time. + ## vscode 0.6.1 - **New default model `stage-d`**, with **`stage-e` for next-edit**. Both download automatically (~530 MB each); the diff --git a/daemon/src/config.ts b/daemon/src/config.ts index c05d239..777acef 100644 --- a/daemon/src/config.ts +++ b/daemon/src/config.ts @@ -32,6 +32,9 @@ export interface DaemonConfig { ctx: number /** Extra flags, e.g. ["-ngl", "99"] on Apple Silicon, ["-t", "16"] on the laptop CPU. */ args: string[] + /** Extra flags for the next-edit companion server only. Default: n-gram speculative decoding (the output is + * mostly a copy of the region, so drafts from the prompt are accepted; exact with greedy sampling). */ + nextEditArgs?: string[] } models: Record defaultModel: string diff --git a/daemon/src/main.ts b/daemon/src/main.ts index 0cf85fa..69ac99c 100644 --- a/daemon/src/main.ts +++ b/daemon/src/main.ts @@ -52,7 +52,8 @@ function loadModel(name: string) { } // Next-edit companion model (profile.nextEditModel): a second llama-server, started on first use. -const neLlama = new LlamaServer({ ...config.llamaServer, port: config.llamaServer.port + 1 }) +const NEXT_EDIT_SPECULATION = ['--spec-type', 'ngram-simple', '--spec-draft-n-max', '64', '--spec-ngram-simple-size-n', '3'] +const neLlama = new LlamaServer({ ...config.llamaServer, port: config.llamaServer.port + 1, args: [...config.llamaServer.args, ...(config.llamaServer.nextEditArgs ?? NEXT_EDIT_SPECULATION)] }) let neLoading: Promise | null = null /** The server that answers next-edit for the active model, or an error message. */ async function nextEditServer(): Promise { @@ -273,9 +274,11 @@ async function handleNextEdit(body: NextEditRequest, res: http.ServerResponse, c const endLine = Math.min(lines.length, cursorLine + o.regionAfter + 1) const recentEdits = edits.observe(`${body.workspaceRoot ?? body.repo}/${body.path}`, body.path, body.text, cursorLine) const lang = symbolLang(body.languageId ?? '') + const tErrors = performance.now() const errors = body.workspaceRoot && lang ? await symbols.errors({ root: body.workspaceRoot, filePath: path.join(body.workspaceRoot, body.path), lang, text: body.text, cursor: body.cursor, waitMs: 300 }) : undefined + const errorsMs = performance.now() - tErrors const ids = builder.buildNextEdit({ repo: body.repo, path: body.path, file: body.text, regionStartLine: startLine, regionEndLine: endLine, cursor: body.cursor, recentEdits: recentEdits || undefined, errors }, { beforeTokens: o.beforeTokens, afterTokens: o.afterTokens }) @@ -294,7 +297,7 @@ async function handleNextEdit(body: NextEditRequest, res: http.ServerResponse, c const response = { requestId: body.requestId, noEdit, startLine, endLine, oldText, newText: noEdit ? null : newText, recentEdits, errors: errors ?? null, model: profile.nextEditModel ?? state.model, - timings: { promptTokens: ids.length, processedTokens: result.timings?.prompt_n ?? null, genTokens: result.timings?.predicted_n ?? null, totalMs: round(performance.now() - t0) }, + timings: { promptTokens: ids.length, processedTokens: result.timings?.prompt_n ?? null, genTokens: result.timings?.predicted_n ?? null, errorsMs: round(errorsMs), promptMs: round(result.timings?.prompt_ms ?? 0), genMs: round(result.timings?.predicted_ms ?? 0), totalMs: round(performance.now() - t0) }, } remember({ kind: 'next-edit', requestId: body.requestId, path: body.path, model: state.model, prompt: builder.decode(ids), raw, noEdit, timings: response.timings }) json(res, 200, response) diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index 55b8cbf..1aeaa22 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -3,7 +3,7 @@ "displayName": "Local Autocomplete", "description": "Inline code completion from a local fine-tuned model via the acd daemon", "publisher": "harvmaster", - "version": "0.6.1", + "version": "0.6.2", "license": "MIT", "engines": { "vscode": "^1.90.0"