diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c52ac12..cbd429e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,7 +3,9 @@ name: check on: push: branches: [main] + # Stacked work is reviewed as PRs into claude/* branches, not only main. pull_request: + branches: [main, "claude/**"] jobs: typecheck-and-test: diff --git a/src/cli.tsx b/src/cli.tsx index 94f699b..fc937e3 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -29,7 +29,7 @@ import { draftMission, missionStatus, runMission, writePlan, type MissionSummary import { parseDuration } from "./session-commands.js"; import { passedChecksWords } from "./tiers.js"; import { judgeEffort, judgeTarget } from "./judge.js"; -import { commandsHere,draftCriteriaCritiqued, preflightCriteria, taskChecksFrom, type Draft } from "./criteria.js"; +import { commandsHere, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, taskChecksFrom, type Draft, type DrafterInputs } from "./criteria.js"; import { listProject, removeNew } from "./leftovers.js"; import { projectScripts } from "./interview.js"; import { @@ -1189,6 +1189,15 @@ async function cmdMission(args: Args): Promise { } } +/** What the criteria drafter may read, taken before the turn's first step. */ +function drafterSnapshotFor(args: Args): DrafterInputs { + return drafterSnapshot(args.task ?? "", args.cwd, { + commands: commandsHere(args.cwd), + scripts: projectScripts(args.cwd), + lessons: new Judgments(args.cwd).lessons(), + }); +} + /** * `--criteria auto`: the model writes the exam, and is then held to it. * @@ -1205,17 +1214,23 @@ async function autoDraft( engine: Engine, args: Args, soFar?: { draft?: Draft; sealed?: boolean; late?: boolean }, + inputs?: { snapshot: DrafterInputs; used: string[] }, ): Promise> { + // Taken once, now, before the first step: the second try below and every + // stage of each draft read this and never the folder the work is changing. + const snapshot = inputs?.snapshot ?? drafterSnapshotFor(args); const none: ReturnType = { taskChecks: [], taskNotes: [] }; // Drafted, then read cold by a critic against the task text: a check that // invents or guesses is dropped (with a task quote), and a draft where // nothing runs the deliverable is asked for once more. See criteria.ts. const draftOnce = () => draftCriteriaCritiqued({ - commands: commandsHere(args.cwd), - lessons: new Judgments(args.cwd).lessons(), + snapshot, + ...(snapshot.commands ? { commands: snapshot.commands } : {}), + lessons: [...snapshot.lessons], task: args.task ?? "", - scripts: projectScripts(args.cwd), + scripts: [...snapshot.scripts], + onInputs: (sha) => inputs?.used.push(sha), barChecks: (engine.cfg.bar?.checks ?? []).map((c) => c.name), ...judgeTarget({ baseUrl: args.url, apiKey: args.key, model: args.model }), cwd: args.cwd, @@ -1518,7 +1533,13 @@ async function cmdRun(args: Args, ask = false): Promise { // Drafted while the model starts reading: the engine seals them before the // first change or claim, so they still predate the work (see RunOptions). const soFar: { draft?: Draft; sealed?: boolean; late?: boolean } = {}; - const pendingCriteria = args.autoCriteria && !ask ? autoDraft(engine, args, soFar) : undefined; + // The drafter's inputs, frozen here; the engine journals their hash at turn + // start and compares it with what each drafter stage actually used. + const drafterInputs = args.autoCriteria && !ask ? { snapshot: drafterSnapshotFor(args), used: [] as string[] } : undefined; + const pendingCriteria = drafterInputs ? autoDraft(engine, args, soFar, drafterInputs) : undefined; + const draftInputs = drafterInputs + ? { sha: drafterInputsHash(drafterInputs.snapshot), used: () => [...drafterInputs.used] } + : undefined; const criteriaSoFar = pendingCriteria ? () => { // Nothing reviewed yet: leave the draft running to join at the claim @@ -1535,7 +1556,7 @@ async function cmdRun(args: Args, ask = false): Promise { // How long a timed turn waits for its drafted checks; for tests and debugging. const waitEnv = Number(env("CRITERIA_WAIT_MS")); const criteriaWait = Number.isFinite(waitEnv) && waitEnv > 0 ? { criteriaWaitMs: waitEnv } : {}; - for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar, referenceCheck, ...criteriaWait })) { + for await (const ev of engine.run(args.task, confirm, { ask, taskChecks, taskNotes, pendingCriteria, criteriaSoFar, referenceCheck, ...(draftInputs ? { draftInputs } : {}), ...criteriaWait })) { emit(ev); if (ev.kind === "proof_exhausted" && ev.result.undetermined?.length) undetermined = true; // The sentence that explains an undetermined bar arrives as an error, so diff --git a/src/criteria.ts b/src/criteria.ts index e532ad6..d54a503 100644 --- a/src/criteria.ts +++ b/src/criteria.ts @@ -22,14 +22,15 @@ * A sentence dressed as a check is worse than no check. */ import { execFileSync } from "node:child_process"; -import { openSync, readSync, closeSync, realpathSync, statSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { openSync, readSync, closeSync, mkdtempSync, realpathSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, relative } from "node:path"; +import { isAbsolute, join, relative } from "node:path"; import { topLevel } from "./brief.js"; import { namedInputs, profileLine } from "./inspect.js"; import { askModel, type AskOptions } from "./ask.js"; import { runCommand, draftedShell, bashPath } from "./run.js"; -import { lintAll, readTree, type LintCtx } from "./checklint.js"; +import { lintAll, readTree, type LintCtx, type Tree } from "./checklint.js"; import { diagnoseFailure } from "./bar.js"; import { normalizeRequirements } from "./signout.js"; import { evidenceTags } from "./tiers.js"; @@ -390,6 +391,61 @@ export function projectView(task: string, cwd?: string): string { return lines.join("\n"); } +/** + * Everything the drafter reads about the project, taken once, before the first + * step. Every drafter stage — the first draft, the lint redraft, the critic, + * the cover step, the redraft after review, and a whole second try — reads + * this and never the live folder: those stages run while the worker is + * already changing files, and a redraft that re-read the tree once saw the + * worker's out.txt and could have sealed its answer as the check. + */ +export type DrafterInputs = { + /** The task text as given, before any review findings are appended. */ + readonly task: string; + /** projectView(task, cwd) at turn start. */ + readonly view: string; + /** readTree(cwd) at turn start, for the seal-time lint. */ + readonly tree?: Tree; + readonly commands?: { present: string[]; missing: string[] }; + readonly scripts: readonly string[]; + readonly lessons: readonly string[]; +}; + +/** Take the drafter's inputs now. */ +export function drafterSnapshot( + task: string, + cwd: string | undefined, + rest: { commands?: { present: string[]; missing: string[] }; scripts?: string[]; lessons?: string[] }, +): DrafterInputs { + return Object.freeze({ + task, + view: projectView(task, cwd), + ...(cwd ? { tree: readTree(cwd) } : {}), + ...(rest.commands ? { commands: { present: [...rest.commands.present], missing: [...rest.commands.missing] } } : {}), + scripts: Object.freeze([...(rest.scripts ?? [])]), + lessons: Object.freeze([...(rest.lessons ?? [])]), + }); +} + +/** + * A hash of the drafter's inputs. Taken at turn start over the snapshot, and + * again by each drafter stage over the values it actually put in its prompt, + * so "drafted without sight of the work" is a comparison the record can fail. + */ +export function drafterInputsHash(d: DrafterInputs): string { + const h = createHash("sha256"); + const part = (k: string, v: string) => h.update(`\0${k}\0${v}`, "utf8"); + part("task", d.task); + part("view", d.view); + part("files", d.tree ? [...d.tree.files].sort().join("\n") : ""); + part("text", d.tree?.text ?? ""); + part("present", (d.commands?.present ?? []).join(",")); + part("missing", (d.commands?.missing ?? []).join(",")); + part("scripts", d.scripts.join("\n")); + part("lessons", d.lessons.join("\n")); + return h.digest("hex").slice(0, 16); +} + const SYSTEM = [ "You draft acceptance criteria for one coding task. You are not doing the task", "and you will not judge whether it was done — a person approves what you write", @@ -532,7 +588,7 @@ async function edgeProbes(opts: Parameters[0], view: strin model: opts.model, system: EDGE_SYSTEM, prompt: [`Task: ${opts.task}`, ...(view ? ["", view] : [])].join("\n"), - cwd: opts.cwd, + cwd: opts.askCwd ?? opts.cwd, what: "probing edge cases", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -590,21 +646,42 @@ export async function draftCriteria(opts: { * the mistakes not to seal again. */ lessons?: string[]; + /** + * The project as it was before the work (drafterSnapshot). When given, the + * view, scripts, commands and lessons come from it and the folder is not + * read again; without it they are read now. + */ + snapshot?: DrafterInputs; + /** Called with drafterInputsHash of what this call put in its prompt. */ + onInputs?: (sha: string) => void; + /** + * Where a subprocess (ACP) drafter is started. Kept apart from the work + * folder so the agent answering cannot open the worker's files. + */ + askCwd?: string; }): Promise<{ ok: true; draft: Draft } | { ok: false; error: string }> { - const view = projectView(opts.task, opts.cwd); + const snap = opts.snapshot; + const view = snap ? snap.view : projectView(opts.task, opts.cwd); + const scripts = snap ? [...snap.scripts] : opts.scripts; + const commands = snap ? snap.commands : opts.commands; + const lessons = snap ? [...snap.lessons] : opts.lessons; + opts.onInputs?.( + drafterInputsHash({ task: snap?.task ?? opts.task, view, ...(snap?.tree ? { tree: snap.tree } : {}), ...(commands ? { commands } : {}), scripts, lessons: lessons ?? [] }), + ); + const askCwd = opts.askCwd ?? opts.cwd; const context = [ `Task: ${opts.task}`, ...(view ? ["", view] : []), "", - `Scripts available: ${opts.scripts.length ? opts.scripts.join(", ") : "(none found)"}`, + `Scripts available: ${scripts.length ? scripts.join(", ") : "(none found)"}`, `The project already checks: ${opts.barChecks.length ? opts.barChecks.join(", ") : "(nothing)"}`, - ...(opts.commands ? [`Commands on this machine: ${opts.commands.present.join(", ")}${opts.commands.missing.length ? ` (not installed: ${opts.commands.missing.join(", ")})` : ""}`] : []), + ...(commands ? [`Commands on this machine: ${commands.present.join(", ")}${commands.missing.length ? ` (not installed: ${commands.missing.join(", ")})` : ""}`] : []), ...(process.platform === "darwin" ? ["This is macOS: BSD tools, not GNU. No `find -printf`, no `stat -c`, no GNU `touch -d`, `sed -i ''` needs the empty argument. Prefer python3 for anything beyond plain shell."] : []), "`.maat/` is Maat's own folder in the project: a check that lists or counts files must ignore it.", - ...(opts.lessons?.length - ? ["", "A person ruled these earlier drafted checks wrong in this project. Do not make the same mistake:", ...opts.lessons.map((l) => `- ${l}`)] + ...(lessons?.length + ? ["", "A person ruled these earlier drafted checks wrong in this project. Do not make the same mistake:", ...lessons.map((l) => `- ${l}`)] : []), "", "Do not repeat what the project already checks. Add only what is specific to", @@ -619,7 +696,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: prompted, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -639,7 +716,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: `${prompted}\n\nYour last reply could not be parsed as JSON. Reply with the JSON object only.`, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -663,7 +740,7 @@ export async function draftCriteria(opts: { model: opts.model, system: SYSTEM, prompt: context, - cwd: opts.cwd, + cwd: askCwd, what: "drafting criteria", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, @@ -821,8 +898,21 @@ export async function draftCriteriaCritiqued( // back, however many ways the function below returns. const sink: Sink = { requirements: [], lint: [] }; const withReqs = (d: Draft): Draft => (sink.requirements.length ? { ...d, requirements: sink.requirements } : d); - const r = await critiqued({ ...opts, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, sink); - return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; + // Every stage below reads the project as it is now, not as the worker + // leaves it: the later stages run while the work is under way. + const snapshot = + opts.snapshot ?? drafterSnapshot(opts.task, opts.cwd, { commands: opts.commands, scripts: opts.scripts, lessons: opts.lessons }); + // A subprocess drafter starts in an empty folder of its own, not the work. + const ownDir = opts.askCwd === undefined ? mkdtempSync(join(tmpdir(), "maat-draft-")) : undefined; + try { + const r = await critiqued( + { ...opts, snapshot, askCwd: opts.askCwd ?? ownDir, onProgress: opts.onProgress && ((d) => opts.onProgress!(withReqs(d))) }, + sink, + ); + return r.ok ? { ...r, draft: withReqs(r.draft), ...(sink.lint.length ? { lint: sink.lint } : {}) } : r; + } finally { + if (ownDir) rmSync(ownDir, { recursive: true, force: true }); + } } /** A drafted check the seal-time lint retired (src/checklint.ts), for the journal. */ @@ -838,14 +928,17 @@ async function critiqued( }, sink: Sink, ): Promise<{ ok: true; draft: Draft; critique: string[] } | { ok: false; error: string }> { + const snap = opts.snapshot ?? drafterSnapshot(opts.task, opts.cwd, { commands: opts.commands, scripts: opts.scripts, lessons: opts.lessons }); + opts = { ...opts, snapshot: snap }; + const commands = snap.commands; const fix = (d: Draft): Draft => - opts.commands ? { ...d, checks: d.checks.map((c) => ({ ...c, run: fixInterpreters(c.run, opts.commands!) })) } : d; - const view = projectView(opts.task, opts.cwd); - // The project as it is now, before the model has changed anything. + commands ? { ...d, checks: d.checks.map((c) => ({ ...c, run: fixInterpreters(c.run, commands) })) } : d; + const view = snap.view; + // The project as it was before the model changed anything (the snapshot). const lintCtx: LintCtx = { task: opts.task, shell: bashPath() ? "bash" : "sh", - ...(opts.cwd ? { cwd: opts.cwd, tree: readTree(opts.cwd) } : {}), + ...(opts.cwd ? { cwd: opts.cwd, tree: snap.tree ?? readTree(opts.cwd) } : {}), }; const said: string[] = []; /** Split a draft's checks into the ones that pass the lint and the ones that do not. Drops are recorded, never silent. */ @@ -904,7 +997,7 @@ async function critiqued( model: opts.model, system: CRITIC_SYSTEM, prompt: `TASK TEXT:\n${opts.task}\n\n${view ? `${view}\n\n` : ""}CHECKS:\n${d.checks.map((c) => `- ${c.name}: ${c.run}`).join("\n")}`, - cwd: opts.cwd, + cwd: opts.askCwd ?? opts.cwd, what: "reviewing the drafted checks", fetchFn: opts.fetchFn, acpSpawn: opts.acpSpawn, diff --git a/src/engine.ts b/src/engine.ts index 29657b8..1b69b42 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1036,6 +1036,16 @@ export type RunOptions = { referenceWaitMs?: number; /** Overrides how long a claim waits for checks still drafting after the time budget's cut. Tests only. */ lateCriteriaWaitMs?: number; + /** + * The drafter's inputs, frozen before the first step (criteria.ts + * drafterSnapshot): `sha` is their hash, `used` the hash each drafter stage + * computed over what it actually put in its prompt. When every stage used + * the snapshot, the drafts are independent of the work, and a claim of done + * may wait for them until the time budget's safety margin + * (claimWaitForDraftsMs). A mismatch means a stage read the folder the work + * changed: its checks are not joined. + */ + draftInputs?: { sha: string; used: () => string[] }; /** * Criteria stated in words rather than as commands. * @@ -1173,6 +1183,23 @@ export function lateCriteriaWaitMs(timeLeftMs: number | undefined): number { return Math.max(0, Math.min(120_000, Math.round(timeLeftMs * 0.6))); } +/** + * How long a claim of done waits for drafted checks still on their way, when + * the drafter's inputs are known to be sealed before the work (RunOptions + * .draftInputs): until what is left of the time budget, less a margin for + * running the checks and the review — the larger of 60 s and 15% of the + * budget. Never shorter than lateCriteriaWaitMs. A separate judge (Grok at + * 170–300 s, a local model past 6 min) routinely finished after the fixed + * two minutes, and correct work was labelled unverified. No budget, the old + * two minutes. + */ +export function claimWaitForDraftsMs(timeLeftMs: number | undefined, budgetMs: number): number { + const floor = lateCriteriaWaitMs(timeLeftMs); + if (timeLeftMs === undefined || !budgetMs) return floor; + const margin = Math.max(60_000, Math.round(budgetMs * 0.15)); + return Math.max(floor, timeLeftMs - margin); +} + /** * A hash of everything the drafter reads: the task text and the project's file * listing before the first step. Journalled at turn start and again when late @@ -4990,7 +5017,7 @@ export class Engine { let lateCriteria: typeof pendingCriteria; let draftInputs = ""; if (pendingCriteria && !opts.ask) { - draftInputs = draftInputsHash(userText, listProject(this.cwd)); + draftInputs = opts.draftInputs?.sha ?? draftInputsHash(userText, listProject(this.cwd)); log?.append("note", { kind: "draft-inputs", text: "drafter inputs fixed at turn start", inputsSha: draftInputs }); } /** Sealed mid-step: tell the model once this step's tool results are in. */ @@ -5164,6 +5191,17 @@ export class Engine { let bar = opts.ask ? withoutWriteChecks(withTaskChecks(this.cfg.bar, taskChecks)) : withTaskChecks(this.cfg.bar, taskChecks); + /** + * Whether the drafted checks are known to be independent of the work: + * the caller froze the drafter's inputs before the first step, and every + * drafter stage so far used exactly those. + */ + const draftsIndependent = (): boolean => + opts.draftInputs !== undefined && opts.draftInputs.used().every((h) => h === draftInputs); + /** How long a claim waits for checks still drafting (see claimWaitForDraftsMs). */ + const lateWait = (): number => + opts.lateCriteriaWaitMs ?? + (draftsIndependent() ? claimWaitForDraftsMs(self.timeLeftMs(), self.turnDeadlineMs) : lateCriteriaWaitMs(self.timeLeftMs())); /** * Seal criteria that were still being drafted. Called before the first * call that changes anything and before the first claim, so the checks @@ -5199,7 +5237,7 @@ export class Engine { kind: "info", text: `checks were still being drafted after ${Math.round(cap / 1000)}s of the time budget and none was ready; ` + - `not sealing an empty set — the first claim waits up to ${Math.round((opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs())) / 1000)}s for them and judges with them if they arrive`, + `not sealing an empty set — the first claim waits up to ${Math.round(lateWait() / 1000)}s for them and judges with them if they arrive`, }; } else { yield { @@ -5276,7 +5314,9 @@ export class Engine { async function* joinLateCriteria(): AsyncGenerator { const p = lateCriteria; if (!p) return; - const wait = opts.lateCriteriaWaitMs ?? lateCriteriaWaitMs(self.timeLeftMs()); + const independent = draftsIndependent(); + const wait = lateWait(); + const began = Date.now(); let timer: NodeJS.Timeout | undefined; const late = Symbol("late"); const got = await Promise.race([ @@ -5284,11 +5324,36 @@ export class Engine { new Promise((r) => { timer = setTimeout(() => r(late), wait); }), ]); clearTimeout(timer); + const waited = Math.round((Date.now() - began) / 1000); + const budgetNote = independent && self.turnDeadlineMs ? " (waited to the time budget's margin: the drafter's inputs were sealed before the work)" : ""; if (got === late) { - yield { kind: "info", text: `the drafted checks did not arrive within ${Math.round(wait / 1000)}s; this claim is judged without them, so it cannot be verified by task checks` }; + log?.append("note", { kind: "late-checks", text: `drafted checks did not arrive after ${waited}s`, waitedMs: Date.now() - began, arrived: false, inputsSha: draftInputs }); + yield { + kind: "info", + text: `waited ${waited}s for the drafted checks${budgetNote}; they did not arrive within ${Math.round(wait / 1000)}s, so this claim is judged without them and cannot be verified by task checks`, + }; return; } lateCriteria = undefined; + // Recomputed from what the drafter stages actually used, now that they + // are done: a stage that read the changed folder shows here. + const used = opts.draftInputs?.used() ?? []; + const strays = used.filter((h) => h !== draftInputs); + if (opts.draftInputs && strays.length) { + log?.append("note", { + kind: "late-checks", + text: "late drafted checks NOT joined: a drafter stage used inputs that differ from the turn-start snapshot", + inputsSha: draftInputs, + used, + arrived: true, + waitedMs: Date.now() - began, + }); + yield { + kind: "info", + text: `the drafted checks arrived after ${waited}s but were drafted from a view of the project that differs from the one taken before the work; they are not joined, so this claim cannot be verified by task checks`, + }; + return; + } const have = new Set(taskChecks.map((c) => c.name)); const added = (got?.taskChecks ?? []) .map((c) => Object.freeze({ ...c, name: c.name.startsWith("task:") ? c.name : `task:${c.name}` })) @@ -5305,14 +5370,16 @@ export class Engine { bar = withTaskChecks(self.cfg.bar, taskChecks); log?.append("note", { kind: "late-checks", - text: `late drafted checks joined the sealed checks: ${added.length} check(s)`, - inputsSha: draftInputs, + text: `late drafted checks joined the sealed checks: ${added.length} check(s) after ${waited}s`, + inputsSha: used.length ? used[used.length - 1] : draftInputs, + waitedMs: Date.now() - began, + arrived: true, seal: taskSeal, checks: added.map((c) => c.name), }); yield { kind: "info", - text: `${added.length} drafted check(s) joined this turn's checks — written from the task text and the project as it was before the work`, + text: `waited ${waited}s for the drafted checks${budgetNote}; ${added.length} arrived and joined this turn's checks — written from the task text and the project as it was before the work`, }; self.transcript.push({ role: "user", diff --git a/test/drafter-view.test.ts b/test/drafter-view.test.ts index 2b11e86..24219a6 100644 --- a/test/drafter-view.test.ts +++ b/test/drafter-view.test.ts @@ -3,10 +3,12 @@ * check that reaches outside the project is dropped before it is sealed. */ import assert from "node:assert/strict"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { draftCriteria, draftCriteriaCritiqued, preflightCriteria, projectView, strayPath } from "../src/criteria.js"; +import { draftCriteria, draftCriteriaCritiqued, drafterInputsHash, drafterSnapshot, preflightCriteria, projectView, strayPath } from "../src/criteria.js"; +import { ACP_AGENTS } from "../src/acp.js"; +import { scriptedAcpAgent } from "./acp-agent.js"; import { workspace } from "./helpers.js"; function capturing(texts: string[]): { fetchFn: typeof fetch; prompts: string[] } { @@ -140,3 +142,100 @@ describe("absolute paths in drafted checks", () => { } }); }); + +/** + * The drafter's later stages (cover step, redraft after review, lint redraft, + * a whole second try) run while the worker is already changing files. They + * read the snapshot taken before the work, never the live folder: a redraft + * that re-read the tree once saw the worker's out.txt, and could have sealed + * its answer as the check. + */ +describe("the drafter's inputs are frozen before the work", () => { + const TASK = "Write the word count of input.txt to out.txt"; + const draft = '{"checks":[{"name":"counts","run":"test \\"$(cat out.txt)\\" = 3"}],"notes":[]}'; + + /** A provider that writes the worker's out.txt the moment the first draft is in. */ + function workerWritesAfterFirstDraft(dir: string, critique: string) { + const prompts: string[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const msgs = (JSON.parse(String(init?.body)) as { messages: { role: string; content: string }[] }).messages; + const prompt = msgs.filter((m) => m.role !== "system").map((m) => m.content).join("\n"); + prompts.push(prompt); + // The worker, past the seal, writes its answer while the drafter works. + if (prompts.length === 1) writeFileSync(join(dir, "out.txt"), "3\n"); + const content = prompt.startsWith("TASK TEXT:") ? critique : draft; + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + }) as unknown as typeof fetch; + return { fetchFn, prompts }; + } + + for (const [path, critique] of [ + ["the cover step", '{"checks":[{"name":"counts","verdict":"runs","quote":""}],"uncovered":["word count of input.txt"],"requirements":[]}'], + ["the redraft after review", '{"checks":[{"name":"counts","verdict":"surface","quote":""}],"uncovered":[],"requirements":[]}'], + ] as const) { + it(`${path} never shows the worker's out.txt (the word-count leak)`, async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const snapshot = drafterSnapshot(TASK, ws.dir, { scripts: [], lessons: [] }); + const used: string[] = []; + const { fetchFn, prompts } = workerWritesAfterFirstDraft(ws.dir, critique); + const r = await draftCriteriaCritiqued({ ...base, task: TASK, cwd: ws.dir, fetchFn, snapshot, onInputs: (h) => used.push(h) }); + assert.ok(r.ok); + const drafts = prompts.filter((p) => p.startsWith("Task:")); + assert.ok(drafts.length >= 2, `a later drafter stage ran (${prompts.length} prompts)`); + assert.ok(existsSync(join(ws.dir, "out.txt")), "the worker's file is there while the later stages run"); + // The live view would now show it; the drafter must not. + assert.match(projectView(TASK, ws.dir), /out\.txt: .*\n\s+first line: 3/); + for (const p of prompts) { + assert.ok(!/out\.txt: \d+ bytes/.test(p) && !/first line: 3/.test(p), `a drafter prompt saw the worker's out.txt:\n${p}`); + assert.ok(!/Project files \(top level\): .*out\.txt/.test(p), "nor lists it"); + } + // Every stage hashed what it used, and it is the snapshot. + assert.ok(used.length >= 2); + for (const h of used) assert.equal(h, drafterInputsHash(snapshot)); + } finally { + ws.cleanup(); + } + }); + } + + it("the hash tells a live read from the snapshot, so the comparison can fail", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const snapshot = drafterSnapshot(TASK, ws.dir, { scripts: [], lessons: [] }); + writeFileSync(join(ws.dir, "out.txt"), "3\n"); + const reply = (async () => + ({ ok: true, status: 200, json: async () => ({ choices: [{ message: { content: '{"checks":[],"notes":[]}' }, finish_reason: "stop" }] }), text: async () => "" }) as unknown as Response) as unknown as typeof fetch; + const frozen: string[] = []; + const live: string[] = []; + await draftCriteria({ ...base, task: TASK, cwd: ws.dir, fetchFn: reply, snapshot, onInputs: (h) => frozen.push(h) }); + // A stage that reads the folder as it is now (the old behaviour). + await draftCriteria({ ...base, task: TASK, cwd: ws.dir, fetchFn: reply, onInputs: (h) => live.push(h) }); + assert.deepEqual(frozen, [drafterInputsHash(snapshot)]); + assert.equal(live.length, 1); + assert.notEqual(live[0], drafterInputsHash(snapshot), "reading the changed folder shows in the hash"); + } finally { + ws.cleanup(); + } + }); + + it("a subprocess (ACP) drafter is started in an empty folder of its own, not the work", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "input.txt"), "a b c\n"); + const agent = scriptedAcpAgent([{ text: '{"checks":[],"notes":[]}' }]); + const grok = ACP_AGENTS.find((a) => a.name === "grok-build")!; + const r = await draftCriteriaCritiqued({ ...base, baseUrl: grok.url, model: "grok-4.6", task: TASK, cwd: ws.dir, acpSpawn: agent.spawnFn }); + assert.ok(r.ok, r.ok ? "" : r.error); + const cwd = String(agent.sessionParams().cwd ?? ""); + assert.ok(cwd, "the session was opened with a folder"); + assert.notEqual(cwd, ws.dir); + assert.ok(!cwd.startsWith(ws.dir), `drafter started inside the work: ${cwd}`); + assert.ok(!existsSync(cwd), "and the folder is removed afterwards"); + } finally { + ws.cleanup(); + } + }); +}); diff --git a/test/late-criteria.test.ts b/test/late-criteria.test.ts index 56aea43..1713fb0 100644 --- a/test/late-criteria.test.ts +++ b/test/late-criteria.test.ts @@ -7,8 +7,9 @@ * runs, a zero-check seal followed, and every correct result went unverified. */ import assert from "node:assert/strict"; -import { describe, it } from "node:test"; -import { draftInputsHash, Engine } from "../src/engine.js"; +import { describe, it, mock } from "node:test"; +import { claimWaitForDraftsMs, draftInputsHash, Engine, lateCriteriaWaitMs } from "../src/engine.js"; +import type { EngineEvent } from "../src/types.js"; import { Journal } from "../src/journal.js"; import type { Check } from "../src/types.js"; import { allowAll, drain, scriptedProvider, workspace } from "./helpers.js"; @@ -120,3 +121,199 @@ describe("zero checks at the time budget's cut", () => { } }); }); + +/** + * A claim of done waits for drafts still on their way until the time budget's + * margin — not a fixed two minutes — when the drafter's inputs were frozen + * before the work (RunOptions.draftInputs). Measured 2026-10-07: a separate + * judge drafting for 170–300 s (a local model past 6 min) finished after the + * claim's 120 s, and correct work was labelled plain unverified. + */ +describe("a claim waits for late drafts until the time budget's margin", () => { + const realTimeout = setTimeout; + const slow = (name: string, run: string): Check => ({ ...mk(name, run), timeoutMs: 10_000_000 }); + /** Run a turn on a virtual clock: one simulated second per real 2 ms. */ + async function onVirtualClock(start: (at: (v: T, ms: number) => Promise) => AsyncGenerator): Promise<{ ev: EngineEvent[]; simMs: number }> { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: Date.now() }); + const t0 = Date.now(); + try { + const ev: EngineEvent[] = []; + let done = false; + let err: unknown; + const run = (async () => { + for await (const e of start((v, ms) => new Promise((r) => setTimeout(() => r(v), ms)))) ev.push(e); + })().then( + () => { done = true; }, + (e: unknown) => { err = e; done = true; }, + ); + for (let i = 0; i < 2_000 && !done; i++) { + await new Promise((r) => realTimeout(r, 2)); + mock.timers.tick(1_000); + } + await run; + if (err) throw err; + return { ev, simMs: Date.now() - t0 }; + } finally { + mock.timers.reset(); + } + } + const infos = (ev: EngineEvent[]) => ev.filter((e) => e.kind === "info").map((e) => (e as { text: string }).text); + const outcome = (ev: EngineEvent[]) => { + const end = ev.find((e) => e.kind === "job_end"); + return end && end.kind === "job_end" ? end.outcome : undefined; + }; + const sealed = { sha: "frozen-inputs", used: () => ["frozen-inputs"] }; + + it("the numbers: a 600 s budget waits to its 90 s margin, never less than before; no budget keeps 120 s", () => { + assert.equal(claimWaitForDraftsMs(590_000, 600_000), 500_000); + assert.ok(claimWaitForDraftsMs(590_000, 600_000) > 200_000, "drafts at 200 s are inside the wait"); + assert.equal(lateCriteriaWaitMs(590_000), 120_000, "the old wait ended before them"); + // The margin is the larger of 60 s and 15% of the budget. + assert.equal(claimWaitForDraftsMs(250_000, 300_000), 190_000); + // Late in the budget it never waits less than the old rule. + assert.equal(claimWaitForDraftsMs(80_000, 600_000), lateCriteriaWaitMs(80_000)); + assert.equal(claimWaitForDraftsMs(undefined, 0), 120_000); + }); + + it("drafts arriving at 200 s of a 600 s budget are used, and judge the claim", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-1"); + const { ev, simMs } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /^waited (19\d|20\d)s for the drafted checks \(waited to the time budget's margin.*1 arrived and joined/.test(t)), infos(ev).join("\n")); + const proof = ev.find((e) => e.kind === "proof_result"); + assert.ok(proof && proof.kind === "proof_result" && proof.result.results.some((r: { name: string }) => r.name === "task:greeting")); + assert.equal(outcome(ev), "verified"); + assert.ok(simMs < 600_000); + const join = notes(journal, "late-checks"); + assert.equal(join.length, 1); + assert.equal(join[0]!.data.arrived, true); + assert.ok(Number(join[0]!.data.waitedMs) >= 195_000); + assert.equal(notes(journal, "draft-inputs")[0]!.data.inputsSha, "frozen-inputs"); + assert.equal(join[0]!.data.inputsSha, "frozen-inputs"); + } finally { + ws.cleanup(); + } + }); + + it("a late check that fails the work goes down the refusal path, never dropped", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-2"); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("farewell", "grep -qx goodbye out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /1 arrived and joined/.test(t))); + assert.ok(ev.some((e) => (e.kind === "proof_refused" || e.kind === "proof_exhausted") && e.result.results.some((r: { name: string; ok: boolean }) => r.name === "task:farewell" && !r.ok))); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts that never arrive: the claim is judged without them at the deadline margin", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-3"); + const { ev, simMs } = await onVirtualClock(() => + engine.run(TASK, allowAll, { + pendingCriteria: new Promise(() => {}), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + const said = infos(ev).find((t) => /did not arrive/.test(t)); + assert.ok(said, infos(ev).join("\n")); + // 600 s budget, 90 s margin: about 510 s waited, then judged in what is left. + const waited = Number(/^waited (\d+)s/.exec(said!)?.[1]); + assert.ok(waited >= 500 && waited <= 512, said); + assert.match(said!, /judged without them/); + assert.notEqual(outcome(ev), "verified"); + assert.ok(ev.some((e) => e.kind === "job_end")); + assert.ok(simMs < 600_000, `ended inside the budget (${simMs} ms)`); + assert.equal(notes(journal, "late-checks")[0]!.data.arrived, false); + } finally { + ws.cleanup(); + } + }); + + it("with no time budget the old 120 s applies", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-4"); + engine.setTurnDeadline(0); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: sealed, + }), + ); + assert.ok(infos(ev).some((t) => /^waited 12\ds for the drafted checks; they did not arrive within 120s/.test(t)), infos(ev).join("\n")); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts whose inputs are not the frozen snapshot get the old 120 s, not the long wait", async () => { + const ws = workspace(); + try { + const { engine } = setup(ws.dir, "late-wait-5"); + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + pendingCriteria: at({ taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }, 200_000), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: { sha: "frozen-inputs", used: () => ["frozen-inputs", "read-the-live-folder"] }, + }), + ); + assert.ok(infos(ev).some((t) => /^waited 12\ds for the drafted checks; they did not arrive within 120s/.test(t)), infos(ev).join("\n")); + assert.notEqual(outcome(ev), "verified"); + } finally { + ws.cleanup(); + } + }); + + it("drafts that arrive having used other inputs are not joined, and the claim is not verified by them", async () => { + const ws = workspace(); + try { + const { engine, journal } = setup(ws.dir, "late-wait-6"); + const used = ["frozen-inputs"]; + const { ev } = await onVirtualClock((at) => + engine.run(TASK, allowAll, { + // A later stage read the changed folder; that shows once it is done. + pendingCriteria: at(null, 150_000).then(() => { + used.push("read-the-live-folder"); + return { taskChecks: [slow("greeting", "grep -qx hello out.txt")], taskNotes: [] }; + }), + criteriaSoFar: nothing, + criteriaWaitMs: 100, + draftInputs: { sha: "frozen-inputs", used: () => [...used] }, + }), + ); + assert.ok(infos(ev).some((t) => /arrived after \d+s but were drafted from a view of the project that differs/.test(t)), infos(ev).join("\n")); + assert.ok(!ev.some((e) => e.kind === "proof_result" && e.result.results.some((r: { name: string }) => r.name === "task:greeting"))); + assert.notEqual(outcome(ev), "verified"); + const n = notes(journal, "late-checks"); + assert.equal(n.length, 1); + assert.match(String(n[0]!.data.text), /NOT joined/); + } finally { + ws.cleanup(); + } + }); +}); diff --git a/test/local-ask.test.ts b/test/local-ask.test.ts index 84b1529..c49b75f 100644 --- a/test/local-ask.test.ts +++ b/test/local-ask.test.ts @@ -88,7 +88,17 @@ describe("an ask to a self-hosted server", () => { }); return new Response(body, { headers: { "content-type": "text/event-stream" } }); }) as unknown as typeof fetch; - const r = await askModel({ baseUrl: LOCAL, model: "m", system: "s", prompt: "p", fetchFn, timeoutMs: 30 }); + // AbortSignal.timeout's timer does not hold the event loop open, and a + // mock stream has no socket that would. On Node 22 the loop emptied before + // the 30 ms were up and the test runner cancelled the rest of the file. + // A real request's socket keeps the process alive; this stands in for it. + const keepAlive = setInterval(() => {}, 1000); + let r: Awaited>; + try { + r = await askModel({ baseUrl: LOCAL, model: "m", system: "s", prompt: "p", fetchFn, timeoutMs: 30 }); + } finally { + clearInterval(keepAlive); + } assert.equal(aborted, true); assert.match(!r.ok ? r.error : "", /did not answer within/); });