Extension 0.6.0: next-edit, token healing, TS errors context, cursor-move, prewarm
ci / build (push) Canceled after 0s
ci / build (push) Canceled after 0s
- shared/nextedit.ts: next-edit prompt format (region rewrite or NO_EDIT, Qwen spare special tokens) - shared/diagnostics.ts: TS errors near the cursor as context; symbols format 2 adds namespace imports - shared/prompt.ts: token healing (cut the half-typed word, force it via grammar), errors header section - daemon: model profiles carry contextFormat/errors/members/nextEdit; /v1/next-edit with edit-burst tracking; /v1/prewarm; /v1/debug/last; token healing on by default - extension: next-edit decorations (Tab apply / Esc dismiss), suggest on cursor move, prewarm on focus, Show Last Prompt, update check against releases, diagnostics in the edit recorder - models/README: training-data attribution; release script profiles for stage-d/stage-e - .gitea/workflows/ci.yml: build + test, publish release on vscode-v* tags (needs a runner + RELEASE_TOKEN) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J39imRDT8L7xYbWzsVWd8A
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Next-edit prediction prompt format, shared by the training pipeline and the daemon.
|
||||
*
|
||||
* <|repo_name|>{repo}\n
|
||||
* [<|file_sep|>{dir}/__errors__.txt\n{errors}\n] (shared/diagnostics.ts, optional)
|
||||
* [<|file_sep|>__recent_edits__.diff\n{diff}\n] (most recent edit(s), optional)
|
||||
* <|file_sep|>{path}\n
|
||||
* {file before region}<REGION_START>{region, with <CURSOR> at the cursor}<REGION_END>{file after region}<OUTPUT_START>
|
||||
*
|
||||
* Target: the rewritten region, then <OUTPUT_END>; or <NO_EDIT><OUTPUT_END> when nothing should change.
|
||||
* Markers reuse Qwen2.5's vision special tokens, which never occur in code.
|
||||
*/
|
||||
import type { Encode } from './prompt.ts'
|
||||
|
||||
export const NEXT_EDIT_TOKENS = {
|
||||
regionStart: '<|quad_start|>',
|
||||
regionEnd: '<|quad_end|>',
|
||||
cursor: '<|box_start|>',
|
||||
outputStart: '<|object_ref_start|>',
|
||||
outputEnd: '<|object_ref_end|>',
|
||||
noEdit: '<|vision_pad|>',
|
||||
} as const
|
||||
|
||||
export type NextEditTokenIds = Record<keyof typeof NEXT_EDIT_TOKENS | 'repoName' | 'fileSep', number>
|
||||
|
||||
export interface NextEditInput {
|
||||
repo: string
|
||||
path: string
|
||||
/** Current file text (after the user's recent edit). */
|
||||
file: string
|
||||
/** Editable region: [startLine, endLine) in `file`, 0-based. */
|
||||
regionStartLine: number
|
||||
regionEndLine: number
|
||||
/** Cursor offset in `file` (must lie inside the region). */
|
||||
cursor: number
|
||||
recentEdits?: string
|
||||
errors?: string
|
||||
}
|
||||
|
||||
/** Character offsets of the start of each line, plus file length. */
|
||||
export function lineOffsets(text: string): number[] {
|
||||
const out = [0]
|
||||
for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) out.push(i + 1)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Region text for [startLine, endLine): whole lines, including the final newline when present. */
|
||||
export function regionText(text: string, startLine: number, endLine: number): string {
|
||||
const offs = lineOffsets(text)
|
||||
return text.slice(offs[startLine], endLine < offs.length ? offs[endLine] : text.length)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact line diff of one edit for the "recent edits" context:
|
||||
* @@ path:line
|
||||
* -old
|
||||
* +new
|
||||
*/
|
||||
export function lineDiff(path: string, before: string, after: string, maxLines = 12): string {
|
||||
const a = before.split('\n'), b = after.split('\n')
|
||||
let p = 0
|
||||
while (p < a.length && p < b.length && a[p] === b[p]) p++
|
||||
let s = 0
|
||||
while (s < a.length - p && s < b.length - p && a[a.length - 1 - s] === b[b.length - 1 - s]) s++
|
||||
const removed = a.slice(p, a.length - s).slice(0, maxLines)
|
||||
const added = b.slice(p, b.length - s).slice(0, maxLines)
|
||||
if (!removed.length && !added.length) return ''
|
||||
return [`@@ ${path}:${p + 1}`, ...removed.map(l => `-${l}`), ...added.map(l => `+${l}`)].join('\n')
|
||||
}
|
||||
|
||||
/** Changed line range of `after` relative to `before`, in `after`'s line numbers: [start, end). */
|
||||
export function changedLines(before: string, after: string): [number, number] | null {
|
||||
const a = before.split('\n'), b = after.split('\n')
|
||||
let p = 0
|
||||
while (p < a.length && p < b.length && a[p] === b[p]) p++
|
||||
if (p === a.length && p === b.length) return null
|
||||
let s = 0
|
||||
while (s < a.length - p && s < b.length - p && a[a.length - 1 - s] === b[b.length - 1 - s]) s++
|
||||
return [p, Math.max(p + 1, b.length - s)]
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks the editable region: covers the cursor line and every line that must change (both in `file`
|
||||
* coordinates), plus a margin; null if that would exceed maxLines.
|
||||
*/
|
||||
export function chooseRegion(file: string, cursorLine: number, mustCover: [number, number][], opts: { margin?: number; maxLines?: number } = {}): [number, number] | null {
|
||||
const margin = opts.margin ?? 3
|
||||
const maxLines = opts.maxLines ?? 40
|
||||
const nLines = file.split('\n').length
|
||||
let start = cursorLine, end = cursorLine + 1
|
||||
for (const [a, b] of mustCover) { start = Math.min(start, a); end = Math.max(end, b) }
|
||||
if (end - start > maxLines) return null
|
||||
start = Math.max(0, start - margin)
|
||||
end = Math.min(nLines, end + margin)
|
||||
return [start, end]
|
||||
}
|
||||
|
||||
export function nextEditTokenIds(tokenToId: (t: string) => number | undefined): NextEditTokenIds {
|
||||
const id = (t: string) => {
|
||||
const v = tokenToId(t)
|
||||
if (v === undefined) throw new Error(`tokenizer is missing ${t}`)
|
||||
return v
|
||||
}
|
||||
return {
|
||||
regionStart: id(NEXT_EDIT_TOKENS.regionStart), regionEnd: id(NEXT_EDIT_TOKENS.regionEnd), cursor: id(NEXT_EDIT_TOKENS.cursor),
|
||||
outputStart: id(NEXT_EDIT_TOKENS.outputStart), outputEnd: id(NEXT_EDIT_TOKENS.outputEnd), noEdit: id(NEXT_EDIT_TOKENS.noEdit),
|
||||
repoName: id('<|repo_name|>'), fileSep: id('<|file_sep|>'),
|
||||
}
|
||||
}
|
||||
|
||||
/** Prompt tokens up to and including OUTPUT_START. Context before/after the region is truncated at line boundaries. */
|
||||
export function buildNextEditPrompt(input: NextEditInput, encode: Encode, ids: NextEditTokenIds, opts: { beforeTokens?: number; afterTokens?: number } = {}): number[] {
|
||||
const beforeTokens = opts.beforeTokens ?? 1024
|
||||
const afterTokens = opts.afterTokens ?? 384
|
||||
const offs = lineOffsets(input.file)
|
||||
const rStart = offs[input.regionStartLine]
|
||||
const rEnd = input.regionEndLine < offs.length ? offs[input.regionEndLine] : input.file.length
|
||||
if (input.cursor < rStart || input.cursor > rEnd) throw new Error('cursor outside region')
|
||||
|
||||
const out: number[] = [ids.repoName, ...encode(`${input.repo}\n`)]
|
||||
const dir = input.path.includes('/') ? input.path.slice(0, input.path.lastIndexOf('/')) : '.'
|
||||
if (input.errors) out.push(ids.fileSep, ...encode(`${dir}/__errors__.txt\n${input.errors}\n`))
|
||||
if (input.recentEdits) out.push(ids.fileSep, ...encode(`__recent_edits__.diff\n${input.recentEdits}\n`))
|
||||
out.push(ids.fileSep, ...encode(`${input.path}\n`))
|
||||
|
||||
// Before the region: last lines that fit; after: first lines that fit.
|
||||
let before = input.file.slice(0, rStart)
|
||||
let beforeIds = encode(before)
|
||||
while (beforeIds.length > beforeTokens && before.includes('\n')) {
|
||||
before = before.slice(before.indexOf('\n') + 1)
|
||||
beforeIds = beforeIds.length > beforeTokens * 2 ? encode(before.slice(-beforeTokens * 6)) : encode(before)
|
||||
}
|
||||
let after = input.file.slice(rEnd)
|
||||
let afterIds = encode(after.slice(0, afterTokens * 8))
|
||||
if (afterIds.length > afterTokens) {
|
||||
after = after.slice(0, afterTokens * 8)
|
||||
while (afterIds.length > afterTokens && after.includes('\n')) {
|
||||
after = after.slice(0, after.lastIndexOf('\n', after.length - 2) + 1)
|
||||
afterIds = encode(after)
|
||||
}
|
||||
}
|
||||
out.push(...beforeIds, ids.regionStart,
|
||||
...encode(input.file.slice(rStart, input.cursor)), ids.cursor, ...encode(input.file.slice(input.cursor, rEnd)),
|
||||
ids.regionEnd, ...afterIds, ids.outputStart)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Target tokens: rewritten region (or NO_EDIT), then OUTPUT_END. The caller appends <|endoftext|>. */
|
||||
export function nextEditTarget(newRegion: string | null, encode: Encode, ids: NextEditTokenIds): number[] {
|
||||
return newRegion === null ? [ids.noEdit, ids.outputEnd] : [...encode(newRegion), ids.outputEnd]
|
||||
}
|
||||
Reference in New Issue
Block a user