autocomplete-model/daemon/test/smoke.ts

/**
 * End-to-end smoke test: starts acd, then exercises it the way an editor would.
 *
 *   node test/smoke.ts [--config acd.config.json] [--typing-samples 15]
 *
 * Symbols/this.-repair checks and the typing replay use the private training workspace's ../data
 * (held-out samples, installed repos) when present, and are skipped otherwise.
 */
import fs from 'node:fs'
import path from 'node:path'
import crypto from 'node:crypto'
import { spawn } from 'node:child_process'
import { parseArgs } from 'node:util'

const { values: args } = parseArgs({ options: { config: { type: 'string', default: 'acd.config.json' }, 'typing-samples': { type: 'string', default: '15' }, entry: { type: 'string', default: 'src/main.ts' } } })
const config = JSON.parse(fs.readFileSync(args.config!, 'utf8')) as { port: number; stateFile: string }
const base = `http://127.0.0.1:${config.port}`
let failures = 0
const check = (name: string, ok: boolean, detail = '') => {
  console.log(`${ok ? 'PASS' : 'FAIL'}  ${name}${detail ? `  (${detail})` : ''}`)
  if (!ok) failures++
}

// --entry ../extensions/vscode/dist/daemon/acd.mjs tests the bundled build the extension ships.
const daemon = spawn(process.execPath, [args.entry!, '--config', args.config!], { stdio: ['ignore', 'pipe', 'pipe'] })
let daemonLog = ''
daemon.stdout.on('data', d => { daemonLog += d })
daemon.stderr.on('data', d => { daemonLog += d })
const cleanup = () => { daemon.kill('SIGTERM') }
process.on('exit', cleanup)

async function api(method: string, route: string, body?: unknown, signal?: AbortSignal) {
  const res = await fetch(base + route, { method, headers: { 'content-type': 'application/json' }, body: body ? JSON.stringify(body) : undefined, signal })
  return { status: res.status, body: await res.json() as Record<string, any> }
}

for (let i = 0; ; i++) {
  await new Promise(r => setTimeout(r, 500))
  try { if ((await api('GET', '/health')).body.ok) break } catch { /* starting */ }
  if (i > 120) { console.error(daemonLog); throw new Error('daemon did not become healthy') }
}

// ---------- config / toggle
await api('PUT', '/v1/config', { model: 'stage-c', mode: 'simple', enabled: true })
const cfg = (await api('GET', '/v1/config')).body
check('config lists model profiles', !!cfg.models?.['stage-c']?.modes?.includes('symbols'), JSON.stringify(cfg.models?.['stage-c']))
check('invalid mode for model is rejected', (await api('PUT', '/v1/config', { mode: 'raw' })).status === 400)
const sym = (await api('PUT', '/v1/config', { mode: 'symbols' })).body
check('symbols mode accepted', sym.mode === 'symbols' && sym.effectiveMode === 'symbols')
await api('PUT', '/v1/config', { mode: 'simple' })
check('state persisted to disk', JSON.parse(fs.readFileSync(path.resolve(path.dirname(args.config!), config.stateFile), 'utf8')).mode === 'simple')

// ---------- correctness on hand-written cases
const fim = (prefix: string, suffix: string, extra: Record<string, unknown> = {}, signal?: AbortSignal) =>
  api('POST', '/v1/fim', { repo: 'smoke', path: 'src/example.ts', languageId: 'typescript', prefix, suffix, ...extra }, signal)

const zod = await fim("import { z } from 'zod'\n\nexport const configSchema = z.object({\n  port: z.coerce.number().default(3000),\n})\n\nexport type Config = ", '\n')
check('zod type completion', zod.body.text.includes('z.infer<typeof configSchema>'), JSON.stringify(zod.body.text))

const obj = await fim('export interface Invoice {\n  id: string\n  memo: string\n  paid: boolean\n}\n\nexport function createInvoice(memo = \'\'): Invoice {\n  return {\n    id: crypto.randomUUID(),\n    ', '\n  }\n}\n')
check('multi-line object completion stops before the suffix', obj.body.text.includes('paid') && !obj.body.text.includes('}\n}'), JSON.stringify(obj.body.text))

const single = await fim('const total = items.reduce((sum, item) => sum + item.', ', 0)\nconsole.log(total)\n')
check('single-line mode when code follows the cursor', single.body.singleLine === true && !single.body.text.includes('\n'), JSON.stringify(single.body.text))

// ---------- cancellation: a newer request cancels the in-flight one
const slow = fim('export async function retry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {\n  ', '\n}\n', { maxLines: 50 })
await new Promise(r => setTimeout(r, 30))
const fast = fim('const answer = ', '\n')
const [slowRes, fastRes] = await Promise.all([slow, fast])
check('newer request cancels in-flight one', slowRes.body.cancelled === true && fastRes.status === 200, `first=${JSON.stringify(slowRes.body).slice(0, 80)}`)

const aborter = new AbortController()
const aborted = fim('function fib(n: number): number {\n  ', '\n}\n', {}, aborter.signal).catch(e => ({ status: -1, body: { error: (e as Error).name } }))
setTimeout(() => aborter.abort(), 20)
check('client abort is handled', (await aborted).body.error === 'AbortError')
check('daemon still healthy after aborts', (await api('GET', '/health')).body.ok === true)

// ---------- symbols mode + this.-repair
// Needs a TS project with node_modules installed: SMOKE_TS_ROOT (project dir), SMOKE_TS_FILE (a .ts file in it,
// relative) and SMOKE_TS_ANCHOR (text in that file to put the cursor before). Skipped when not set.
const envRoot = process.env.SMOKE_TS_ROOT ? path.resolve(process.env.SMOKE_TS_ROOT) : ''
if (envRoot && fs.existsSync(envRoot) && process.env.SMOKE_TS_FILE && process.env.SMOKE_TS_ANCHOR) {
  const file = process.env.SMOKE_TS_FILE
  const content = fs.readFileSync(path.join(envRoot, file), 'utf8')
  const at = content.indexOf(process.env.SMOKE_TS_ANCHOR)
  const req = { repo: path.basename(envRoot), path: file, workspaceRoot: envRoot, languageId: 'typescript', prefix: content.slice(0, at), suffix: content.slice(at), mode: 'symbols' }
  let first = (await api('POST', '/v1/fim', req)).body
  for (let i = 0; i < 20 && first.symbols?.status !== 'hit' && !(first.symbols?.items > 0); i++) {
    await new Promise(r => setTimeout(r, 250)) // language service still building the program
    first = (await api('POST', '/v1/fim', req)).body
  }
  check('symbols mode attaches language-service context', first.mode === 'symbols' && first.symbols?.items > 0, JSON.stringify(first.symbols))
  const again = (await api('POST', '/v1/fim', { ...req, prefix: req.prefix + 'x' })).body
  check('same anchor reuses cached context (KV-stable header)', again.symbols?.status === 'hit', JSON.stringify(again.symbols))
  const outside = (await api('POST', '/v1/fim', { ...req, workspaceRoot: undefined })).body
  check('symbols without a workspace falls back to simple', outside.mode === 'simple')

  const cls = 'export class UserService {\n  private users = new Map<string, string>()\n\n  getUser(id: string) {\n    return this.users.get(id)\n  }\n\n  printUser(id: string) {\n    const user = getU'
  const repaired = (await api('POST', '/v1/fim', { repo: req.repo, path: 'src/probe-user-service.ts', workspaceRoot: envRoot, languageId: 'typescript', prefix: cls, suffix: '\n  }\n}\n' })).body
  check('missing this. is repaired as a replacement', repaired.replaceBefore === 4 && repaired.text.startsWith('this.getU') && repaired.filterText?.startsWith('getU'), JSON.stringify({ text: repaired.text, replaceBefore: repaired.replaceBefore, filterText: repaired.filterText }))
  const local = (await api('POST', '/v1/fim', { repo: req.repo, path: 'src/probe-user-service.ts', workspaceRoot: envRoot, languageId: 'typescript', prefix: cls.replace('const user = getU', 'const getUsername = () => id\n    const user = getU'), suffix: '\n  }\n}\n' })).body
  check('no repair when a local matches', local.replaceBefore === 0, JSON.stringify(local.text))
} else {
  console.log('SKIP  symbols / this.-repair checks (set SMOKE_TS_ROOT, SMOKE_TS_FILE, SMOKE_TS_ANCHOR)')
}

// ---------- edit recording endpoint
const recorded = await api('POST', '/v1/edits', { events: [{ t: Date.now(), repo: 'smoke', path: 'src/a.ts', version: 2, changes: [{ offset: 0, length: 0, text: 'x' }], snapshot: 'x' }] })
check('edit recordings are accepted', recorded.body.recorded === 1)

// ---------- warm typing latency on held-out code
// Optional typing replay over held-out FIM samples from the (private) training workspace.
const dataDir = path.resolve(process.env.SMOKE_DATA_DIR ?? '../../data')
const testFim = path.join(dataDir, 'fim/test-repo.jsonl')
if (fs.existsSync(testFim)) {
  const files = new Map(fs.readFileSync(path.join(dataDir, 'corpus/files.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l)).map(r => [r.id, r]))
  const rank = (id: string) => crypto.createHash('sha256').update(`smoke:${id}`).digest('hex')
  const samples = fs.readFileSync(testFim, 'utf8').trim().split('\n').map(l => JSON.parse(l))
    .filter(s => s.suffix_mode === 'full' && ['line-rest', 'statement', 'call-args'].includes(s.type) && s.middle.length >= 12 && !s.middle.includes('\n'))
    .sort((a, b) => rank(a.id).localeCompare(rank(b.id)))
    .slice(0, Number(args['typing-samples']))
  for (const mode of ['simple', 'symbols']) {
  await api('PUT', '/v1/config', { mode })
  const steps: { total: number; processed: number }[] = []
  let firstAtLocation: number[] = []
  for (const s of samples) {
    const content = files.get(s.file_id).content as string
    for (let typed = 0; typed <= s.middle.length; typed += 3) {
      const workspaceRoot = path.join(dataDir, 'ts-env', s.repo)
      const r = await api('POST', '/v1/fim', { repo: s.repo, path: s.path, workspaceRoot: fs.existsSync(workspaceRoot) ? workspaceRoot : undefined, languageId: s.lang, prefix: content.slice(0, s.start) + s.middle.slice(0, typed), suffix: content.slice(s.end) })
      if (typed === 0) { firstAtLocation.push(r.body.timings.totalMs); continue }
      steps.push({ total: r.body.timings.totalMs, processed: r.body.timings.processedTokens })
    }
  }
  const pct = (xs: number[], p: number) => [...xs].sort((a, b) => a - b)[Math.min(xs.length - 1, Math.floor(p * xs.length))]
  console.log(`\nwarm typing, mode=${mode} (${samples.length} completions, ${steps.length} keystroke requests, end-to-end through acd):`)
  console.log(`  first request at a new location: p50 ${pct(firstAtLocation, 0.5)} ms, p90 ${pct(firstAtLocation, 0.9)} ms`)
  console.log(`  per keystroke: total p50 ${pct(steps.map(s => s.total), 0.5)} ms, p90 ${pct(steps.map(s => s.total), 0.9)} ms; tokens re-processed p50 ${pct(steps.map(s => s.processed), 0.5)}, p90 ${pct(steps.map(s => s.processed), 0.9)}`)
  check(`${mode}: warm keystrokes reuse the KV cache (p50 re-processed tokens < 50)`, pct(steps.map(s => s.processed), 0.5) < 50)
  }
  await api('PUT', '/v1/config', { mode: 'simple' })
}

console.log(failures ? `\n${failures} FAILED` : '\nall passed')
cleanup()
process.exit(failures ? 1 : 0)