diff --git a/src/anthropic.ts b/src/anthropic.ts index e3b9303..7ccba0f 100644 --- a/src/anthropic.ts +++ b/src/anthropic.ts @@ -65,7 +65,11 @@ export function outputCeiling(body: string): number | null { const m = /max_tokens:\s*\d+\s*>\s*(\d+)/i.exec(body) ?? /maximum allowed number of output tokens[^\d]{0,40}(\d+)/i.exec(body) ?? - /max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body); + /max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body) ?? + // OpenAI: "max_tokens is too large: 32768. This model supports at most 16384 completion tokens". + /max_tokens is too large[\s\S]{0,80}?supports at most (\d+) completion tokens/i.exec(body) ?? + // DeepSeek: "the valid range of max_tokens is [1, 8192]". + /valid range of max_tokens is \[\d+,\s*(\d+)\]/i.exec(body); if (!m) return null; const n = Number(m[1]); return Number.isFinite(n) && n > 0 ? n : null; diff --git a/src/engine.ts b/src/engine.ts index 5b89a7d..fa2f83e 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -338,6 +338,124 @@ export type Overflow = { sent: number; }; +/** + * A refusal where the completion reserve, not the prompt, overflowed. + * + * vLLM-style servers count `max_tokens` against the window. GMI Cloud serves + * qwen3-235b with a 131,072-token TOTAL window, and the request went out with + * the model's full listed output (an OpenAI-shaped request with no max_tokens + * is given that by the router), so a 2,454-token prompt was refused: + * + * maximum context length of 131072 tokens. You requested a total of + * 133526 tokens: 2454 tokens from the input messages and 131072 tokens + * for the completion. + * + * Shedding history can never fix that. Every server words it differently: + * vLLM (both its old and current wordings), OpenAI, OpenRouter, Anthropic and + * TGI are read below. llama.cpp never refuses for the reserve (it stops the + * reply when the window fills), so its refusal is always a prompt overflow. + * Returns the window, the input and the completion the server counted, or + * null when the body does not say all three. + */ +export function completionOverflow(body: string): { window: number; input: number; completion: number } | null { + const ok = (window: number, input: number, completion: number) => + window > 0 && input >= 0 && completion > 0 ? { window, input, completion } : null; + + // Anthropic: "input length and `max_tokens` exceed context limit: 188240 + 21333 > 200000". + const anth = /max_tokens`?\s+exceed context limit:\s*(\d+)\s*\+\s*(\d+)\s*>\s*(\d+)/i.exec(body); + if (anth) return ok(Number(anth[3]), Number(anth[1]), Number(anth[2])); + + // Hugging Face TGI: "`inputs` tokens + `max_new_tokens` must be <= 4096. Given: 1000 `inputs` tokens and 4000 `max_new_tokens`". + const tgi = /must be <= (\d+)\.\s*Given:\s*(\d+) `?inputs`? tokens and (\d+) `?max_new_tokens`?/i.exec(body); + if (tgi) return ok(Number(tgi[1]), Number(tgi[2]), Number(tgi[3])); + + const win = /maximum context length (?:is|of) (\d+)/i.exec(body); + if (!win) return null; + const window = Number(win[1]); + + // vLLM (since vllm#36197): "you requested 32768 output tokens and your prompt contains 1000 input tokens". + const v2 = /you requested (\d+) output tokens? and your prompt contains (\d+) input tokens?/i.exec(body); + if (v2) return ok(window, Number(v2[2]), Number(v2[1])); + + // vLLM 0.8-0.16: "'max_tokens' or 'max_completion_tokens' is too large: 32768. This model's maximum + // context length is 32768 tokens and your request has 1000 input tokens (32768 > 32768 - 1000)." + const v1 = /is too large:\s*(\d+)[\s\S]{0,200}?your request has (\d+) input tokens?/i.exec(body); + if (v1) return ok(window, Number(v1[2]), Number(v1[1])); + + // GMI / vLLM: "A tokens from the input messages and B tokens for the completion". + const gmi = /(\d+) tokens? from the input messages and (\d+) tokens? for the completion/i.exec(body); + if (gmi) return ok(window, Number(gmi[1]), Number(gmi[2])); + + // OpenRouter: "(1000 of text input, 900 of tool input, 32768 in the output)". Every "of … input" + // part counts against the prompt (text, tool, image). + const orOut = /(\d+) in the output/i.exec(body); + if (orOut) { + const ins = [...body.matchAll(/(\d+) of [a-z ]{0,20}?input/gi)].map((m) => Number(m[1])); + if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(orOut[1])); + } + + // OpenAI: "(1232 in the messages, 32768 in the completion)", sometimes with "N in the functions". + const oai = /\(([^)]*?(\d+) in the completion)\)/i.exec(body); + if (oai) { + const ins = [...oai[1]!.matchAll(/(\d+) in the (?!completion)\w+/gi)].map((m) => Number(m[1])); + if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(oai[2])); + } + return null; +} + +/** The most an unasked output cap grows to after replies hit it. */ +export const OUTPUT_CAP_MAX = 131_072; + +/** + * A 400 that refuses the `max_tokens` field itself, as opposed to its value. + * Narrow on purpose: it must name the field and say it is unsupported. + */ +export function refusedMaxTokens(body: string): boolean { + return /max_tokens/i.test(body) && /unsupported|not supported|unrecognized|unknown (field|parameter)|extra inputs are not permitted|use 'max_completion_tokens'/i.test(body); +} + +/** Room left between the prompt and the window, kept free of the completion reserve. */ +export const COMPLETION_MARGIN = 256; +/** A completion cap below this is no reply worth asking for; the prompt is what has to shrink. */ +export const MIN_COMPLETION = 1_024; + +/** + * The completion cap that fits, when lowering it alone fixes the overflow: + * the window less the input and a margin. Null when even the smallest useful + * reply does not fit (the history is the problem, and shedding is the fix). + */ +export function fittedCompletion(o: { window: number; input: number; completion: number }): number | null { + const fit = o.window - o.input - Math.max(COMPLETION_MARGIN, Math.round(o.window * 0.01)); + return fit >= MIN_COMPLETION && fit < o.completion ? fit : null; +} + +/** The cap tried once when a window refusal names no counts molt can read. */ +export const FALLBACK_COMPLETION = 8_192; + +/** + * A smaller cap to try before shedding, for a window refusal whose wording + * completionOverflow does not know. + * + * Maat always sends max_tokens now, and a server that counts it against the + * window refuses in its own words. Shedding cannot fix a reserve overflow, so + * when the refusal is about the context or the length and the cap sent was + * large, one request with a smaller cap is cheaper than ending a turn "with + * nothing left to shed". `inputEst` is molt's scaled estimate of the prompt, + * used when the server named a window but no prompt count. Null when the + * prompt itself is known not to fit (shed instead), when the cap is already + * small, or when the body is not about the window at all. + */ +export function reserveFallback(body: string, cap: number, inputEst: number): number | null { + if (cap <= FALLBACK_COMPLETION || completionOverflow(body)) return null; + const over = contextOverflow(body); + if (!over && !(/context|length/i.test(body) && /tokens?/i.test(body))) return null; + if (!over || over.window <= 0) return FALLBACK_COMPLETION; + const input = over.sent > 0 ? over.sent : inputEst; + const fit = over.window - input - Math.max(COMPLETION_MARGIN, Math.round(over.window * 0.01)); + if (fit < MIN_COMPLETION) return null; + return Math.min(fit, Math.floor(cap / 2)); +} + export function contextOverflow(body: string): Overflow | null { // Overflow wording, not the bare word "context": a pinned OpenRouter provider // answered a rate-limit retry with a 400 that merely mentioned context, and a @@ -348,15 +466,25 @@ export function contextOverflow(body: string): Overflow | null { ) ) return null; + // Where the server split the prompt from the completion, the prompt's count + // is what molt's estimate is compared with. Never the output count: current + // vLLM puts that first ("you requested 32768 output tokens and your prompt + // contains 1000"), and reading it as the prompt scaled every estimate 8x. + const co = completionOverflow(body); + if (co) return { window: co.window, sent: co.input }; // Every field the common servers use, most specific first. const win = /"n_ctx"\s*:\s*(\d+)/.exec(body) ?? /context size \((\d+)\s*tokens?\)/i.exec(body) ?? - /maximum context length is (\d+)/i.exec(body); + /maximum context length (?:is|of) (\d+)/i.exec(body) ?? + // Anthropic: "prompt is too long: 210000 tokens > 200000 maximum". + /tokens? > (\d+) maximum/i.exec(body); const sent = /"n_prompt_tokens"\s*:\s*(\d+)/.exec(body) ?? /request \((\d+)\s*tokens?\)/i.exec(body) ?? - /you requested (\d+)/i.exec(body); + /prompt is too long:\s*(\d+)/i.exec(body) ?? + /your (?:prompt contains|request has|messages resulted in) (\d+)/i.exec(body) ?? + /you requested (?:a total of |about )?(\d+)(?!\d)(?! output)/i.exec(body); return { window: win ? Number(win[1]) : 0, sent: sent ? Number(sent[1]) : 0 }; } @@ -1978,10 +2106,30 @@ export class Engine { */ private modelMaxTokens?: number; + /** + * The output cap this session settled on without being asked: the default + * (DEFAULT_MAX_TOKENS, not the model's whole listed output), doubled each + * time a reply is cut off at it, up to OUTPUT_CAP_MAX. + */ + private outputCap = DEFAULT_MAX_TOKENS; + /** + * The completion reserve a server said fits its window beside the prompt + * (completionOverflow). Kept until the history is shed: the prompt grows + * between sheds, and after one there is room for a larger reply again. + */ + private fittedCompletion?: number; + /** + * The fitted cap above was a guess (reserveFallback), not read from the + * server's own counts. A reply that is cut off at a guess drops it. + */ + private fittedGuessed = false; + /** Set once a provider refuses `max_tokens` outright; it is then not sent. */ + private maxTokensUnsupported = false; + /** What to send as `max_tokens`: what was asked for, bounded by what fits. */ private maxTokensFor(): number { - const asked = this.cfg.maxTokens ?? DEFAULT_MAX_TOKENS; - return this.modelMaxTokens ? Math.min(asked, this.modelMaxTokens) : asked; + const asked = this.cfg.maxTokens ?? this.outputCap; + return Math.min(asked, this.modelMaxTokens ?? Infinity, this.fittedCompletion ?? Infinity); } /** * These three are derived from the endpoint, and the endpoint moves. @@ -2291,6 +2439,11 @@ export class Engine { // Output maximums are per-model, and the last one's says nothing // about this one. Discovered again on the next refusal if it matters. this.modelMaxTokens = undefined; + this.fittedCompletion = undefined; + this.fittedGuessed = false; + this.maxTokensUnsupported = false; + // A cap doubled on the last model says nothing about this one either. + this.outputCap = DEFAULT_MAX_TOKENS; } this.cfg.model = m; } @@ -3059,6 +3212,9 @@ export class Engine { } this.transcript.commitShed(plan); + // A cap fitted beside the old prompt is too small for the new one. + this.fittedCompletion = undefined; + this.fittedGuessed = false; this.ledger = staying; // Journalled here, on every path. It was journalled by the two auto-shed @@ -6080,6 +6236,10 @@ export class Engine { // ignored (see selfHostedThinking). Nothing on a cloud endpoint. ...selfHostedThinking(this.cfg.baseUrl, this.effortNow), ...openRouterProvider(this.cfg.baseUrl, this.cfg.model), + // Always a cap. Without one a router fills in the model's whole + // listed output, and a server that counts it against the window + // refuses every request (completionOverflow). + ...(this.maxTokensUnsupported ? {} : { max_tokens: this.maxTokensFor() }), ...(stream ? { stream: true } : {}), ...(withUsage ? { stream_options: { include_usage: true } } : {}), }; @@ -6215,7 +6375,7 @@ export class Engine { // so, and says what the maximum is. Same shape as the two // fallbacks above: retry once, and only believe the ceiling was // the problem if that works. - if (!res.ok && res.status === 400 && this.native && this.modelMaxTokens === undefined) { + if (!res.ok && res.status === 400 && this.modelMaxTokens === undefined && !this.maxTokensUnsupported) { const cap = outputCeiling(body); if (cap && cap < this.maxTokensFor()) { const was = this.maxTokensFor(); @@ -6236,6 +6396,77 @@ export class Engine { } } + // The completion reserve overflowed, not the prompt: ask for a + // smaller reply and send the same history again. Shedding cannot + // fix this one, and it was ending runs "too large and nothing left + // to shed" on a 2,454-token prompt. + for (let i = 0; i < 2 && !res.ok && res.status === 400; i++) { + const co = completionOverflow(body); + const fit = co ? fittedCompletion(co) : null; + if (!co || fit === null || fit >= this.maxTokensFor()) break; + const was = this.maxTokensFor(); + this.fittedCompletion = fit; + this.fittedGuessed = false; + log?.append("note", { + body: body.slice(0, 600), + text: + `the completion reserve overflowed the ${co.window}-token window (${co.input} input + ` + + `${co.completion} completion) — max_tokens ${was} → ${fit}, retried without shedding`, + }); + yield { + kind: "info", + text: `this endpoint counts the reply against its ${co.window}-token window — asking for at most ${fit} output tokens instead of ${was}.`, + }; + res = await send(askForUsage && !this.streamUsageUnsupported); + body = res.ok ? "" : (await res.text().catch(() => "")); + } + + // A window refusal in wording none of the above reads, from a + // request that carried a large cap. Try one smaller cap before + // shedding: shedding cannot fix a reserve overflow, and before + // max_tokens was always sent these servers fitted the reply + // themselves. Kept only if it works. + if (!res.ok && res.status === 400 && !this.maxTokensUnsupported) { + const was = this.maxTokensFor(); + const guess = reserveFallback(body, was, Math.round(this.bom().requestTotalEst * this.tokenScale)); + if (guess !== null && guess < was) { + const before = { fitted: this.fittedCompletion, guessed: this.fittedGuessed }; + this.fittedCompletion = guess; + this.fittedGuessed = true; + const retry = await send(askForUsage && !this.streamUsageUnsupported); + if (retry.ok) { + log?.append("note", { + body: body.slice(0, 600), + text: `window refusal with max_tokens ${was} — ${guess} was accepted, kept for the session`, + }); + yield { + kind: "info", + text: `this endpoint refused a ${was}-token reply reserve — asking for at most ${guess} output tokens instead.`, + }; + } else { + this.fittedCompletion = before.fitted; + this.fittedGuessed = before.guessed; + } + res = retry; + body = res.ok ? "" : (await res.text().catch(() => "")); + } + } + + // A provider that will not take `max_tokens` at all (OpenAI's + // reasoning models want max_completion_tokens). Try once without; + // only believe the field was the problem if that works. + if (!res.ok && res.status === 400 && !this.native && !this.maxTokensUnsupported && refusedMaxTokens(body)) { + this.maxTokensUnsupported = true; + const retry = await send(askForUsage && !this.streamUsageUnsupported); + if (retry.ok) { + log?.append("note", { text: "provider refused max_tokens — sending requests without an output cap" }); + } else { + this.maxTokensUnsupported = false; + } + res = retry; + body = res.ok ? "" : (await res.text().catch(() => "")); + } + if (!res.ok) { // "Too big" is not a verdict on the work, and it is the one // refusal that says how to fix itself. Shed and send less rather @@ -7077,15 +7308,29 @@ export class Engine { // sentence. if (finishReason === "length" && truncatedTurns < TRUNCATED_TURN_RETRIES) { truncatedTurns += 1; + // The cap this reply hit, before any change below: the note, the + // nudge and the info line report what happened, not what comes next. + const hit = this.maxTokensFor(); + // A reply that actually hit the default cap earns a larger one; a cap + // the person set, or one the server said is all that fits, stays. A + // fitted cap that was only a guess is dropped; if the full one is + // refused again, the refusal fits it again. + if (this.cfg.maxTokens === undefined && this.outputCap < OUTPUT_CAP_MAX && hit === this.outputCap) { + this.outputCap = Math.min(OUTPUT_CAP_MAX, this.outputCap * 2); + } else if (this.fittedGuessed && hit === this.fittedCompletion) { + this.fittedCompletion = undefined; + this.fittedGuessed = false; + } + const next = this.maxTokensFor(); log?.append("note", { - text: `reply cut off at the ${this.maxTokensFor()}-token output ceiling`, + text: `reply cut off at the ${hit}-token output ceiling` + (next !== hit ? `; the next request asks for ${next}` : ""), step, attempt: truncatedTurns, }); this.transcript.push({ role: "user", content: - `[molt: that reply hit the output ceiling of ${this.maxTokensFor()} tokens and stopped ` + + `[molt: that reply hit the output ceiling of ${hit} tokens and stopped ` + `part-way through, so it is not being read as a finished answer. Continue from where ` + `it stops, and keep what remains short.]`, molt: { nudge: true }, @@ -7094,7 +7339,7 @@ export class Engine { yield { kind: "info", text: - `the reply was cut off at the ${this.maxTokensFor()}-token output ceiling — asking it ` + + `the reply was cut off at the ${hit}-token output ceiling — asking it ` + `to continue rather than running the bar on half a sentence` + (this.cfg.maxTokens === undefined ? " (raise it with --max-tokens)" : ""), }; diff --git a/test/completion-reserve.test.ts b/test/completion-reserve.test.ts new file mode 100644 index 0000000..98e0750 --- /dev/null +++ b/test/completion-reserve.test.ts @@ -0,0 +1,381 @@ +/** + * A server that counts the completion reserve against its window. + * + * GMI Cloud serves qwen/qwen3-235b-a22b-2507 with a 131,072-token TOTAL + * window. The request carried no max_tokens, the router filled in the model's + * whole listed output, and even a 2,454-token prompt was refused; Maat then + * shed history, which cannot fix that, and 11 of 34 runs ended "too large and + * nothing left to shed". Mock providers only. + */ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { completionOverflow, contextOverflow, Engine, FALLBACK_COMPLETION, fittedCompletion, refusedMaxTokens, reserveFallback } from "../src/engine.js"; +import { outputCeiling } from "../src/anthropic.js"; +import { allowAll, drain, workspace } from "./helpers.js"; + +const GMI = + `{"object":"error","message":"Requested token count exceeds the model's maximum context length of 131072 tokens. ` + + `You requested a total of 133526 tokens: 2454 tokens from the input messages and 131072 tokens for the completion. ` + + `Please reduce the number of tokens in the input messages or the completion to fit within the limit.","type":"BadRequestError","param":null,"code":400}`; + +const gmiBody = (input: number, completion: number) => + GMI.replace("133526", String(input + completion)).replace("2454 tokens", `${input} tokens`).replace("131072 tokens for", `${completion} tokens for`); + +type Body = { max_tokens?: number; messages: unknown[] }; + +const okReply = (message: object, finish = "stop") => + ({ + ok: true, + status: 200, + json: async () => ({ choices: [{ message, finish_reason: finish }], usage: { prompt_tokens: 900, completion_tokens: 20 } }), + text: async () => "", + }) as unknown as Response; +const refused = (text: string) => ({ ok: false, status: 400, text: async () => text, json: async () => ({}) }) as unknown as Response; + +function engineWith(dir: string, fetchFn: typeof fetch, over: Record = {}): Engine { + return new Engine({ baseUrl: "http://provider.test/v1", model: "m", provider: "test", cwd: dir, bar: null, stream: false, fetchFn, retryBackoffMs: [5, 5, 5], ...over }); +} + +describe("reading a completion-reserve overflow", () => { + it("reads the window, the input and the completion out of the GMI body", () => { + assert.deepEqual(completionOverflow(GMI), { window: 131072, input: 2454, completion: 131072 }); + // The older OpenAI/vLLM wording too. + assert.deepEqual( + completionOverflow("This model's maximum context length is 32768 tokens. However, you requested 34000 tokens (1232 in the messages, 32768 in the completion)."), + { window: 32768, input: 1232, completion: 32768 }, + ); + assert.equal(completionOverflow('{"n_ctx":16384,"n_prompt_tokens":17222}'), null); + // contextOverflow's "sent" is the prompt's count, the number Maat's estimate is compared with. + assert.deepEqual(contextOverflow(GMI), { window: 131072, sent: 2454 }); + }); + + it("fits the completion when lowering it alone is enough, and says no when the prompt is the problem", () => { + const fit = fittedCompletion({ window: 131072, input: 2454, completion: 131072 }); + assert.ok(fit !== null && fit >= 1024 && fit <= 131072 - 2454 - 256, String(fit)); + assert.equal(fittedCompletion({ window: 131072, input: 130500, completion: 32768 }), null); + }); + + it("knows a refused max_tokens field and more output-ceiling wordings", () => { + assert.equal(refusedMaxTokens("Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."), true); + assert.equal(refusedMaxTokens(GMI), false); + assert.equal(outputCeiling("max_tokens is too large: 32768. This model supports at most 16384 completion tokens, whereas you provided 32768."), 16384); + assert.equal(outputCeiling("Invalid max_tokens value, the valid range of max_tokens is [1, 8192]"), 8192); + }); +}); + +/** Each provider's own refusal for a 1000-token prompt and a 32768 cap on a 32768 window. */ +const OPENROUTER = (input: number, completion: number) => + `{"error":{"message":"This endpoint's maximum context length is 32768 tokens. However, you requested about ${input + completion} tokens ` + + `(${input - 100} of text input, 100 of tool input, ${completion} in the output). Please reduce the length of either one, ` + + `or use the \"middle-out\" transform to compress your prompt automatically.","code":400}}`; +const VLLM_NEW = (input: number, completion: number) => + `{"object":"error","message":"This model's maximum context length is 32768 tokens. However, you requested ${completion} output tokens ` + + `and your prompt contains ${input} input tokens, for a total of ${input + completion} tokens. Please reduce the length of the input ` + + `prompt or the number of requested output tokens.","type":"BadRequestError","param":null,"code":400}`; +const VLLM_OLD = (input: number, completion: number) => + `{"object":"error","message":"'max_tokens' or 'max_completion_tokens' is too large: ${completion}. This model's maximum context ` + + `length is 32768 tokens and your request has ${input} input tokens (${completion} > 32768 - ${input}).","type":"BadRequestError","param":null,"code":400}`; +const OPENAI = (input: number, completion: number) => + `{"error":{"message":"This model's maximum context length is 32768 tokens. However, you requested ${input + completion} tokens ` + + `(${input - 200} in the messages, 200 in the functions, ${completion} in the completion). Please reduce the length of the messages, functions, or completion.",` + + `"type":"invalid_request_error","param":"messages","code":"context_length_exceeded"}}`; +const ANTHROPIC = (input: number, completion: number) => + `{"type":"error","error":{"type":"invalid_request_error","message":"input length and \`max_tokens\` exceed context limit: ${input} + ${completion} > 32768, ` + + `decrease input length or \`max_tokens\` and try again"}}`; +const LLAMACPP = (input: number) => + `{"error":{"code":400,"message":"the request exceeds the available context size, try increasing it","type":"exceed_context_size_error",` + + `"n_prompt_tokens":${input},"n_ctx":32768}}`; + +describe("each provider's refusal wording", () => { + for (const [name, make] of [ + ["OpenRouter", OPENROUTER], + ["vLLM (current)", VLLM_NEW], + ["vLLM 0.8-0.16", VLLM_OLD], + ["OpenAI", OPENAI], + ["Anthropic", ANTHROPIC], + ] as const) { + it(`${name}: reads the window, the prompt and the completion, and the prompt is what contextOverflow reports`, () => { + const body = make(1000, 32768); + assert.deepEqual(completionOverflow(body), { window: 32768, input: 1000, completion: 32768 }); + assert.deepEqual(contextOverflow(body), { window: 32768, sent: 1000 }, "never the output count as sent"); + assert.equal(refusedMaxTokens(body), false, "the field itself was not refused"); + const fit = fittedCompletion(completionOverflow(body)!); + assert.ok(fit !== null && fit <= 32768 - 1000 - 256 && fit >= 1024, String(fit)); + }); + } + + it("TGI: reads `inputs` + `max_new_tokens` must be <= N", () => { + const body = "Input validation error: `inputs` tokens + `max_new_tokens` must be <= 4096. Given: 1000 `inputs` tokens and 32768 `max_new_tokens`"; + assert.deepEqual(completionOverflow(body), { window: 4096, input: 1000, completion: 32768 }); + }); + + it("llama.cpp: a prompt overflow, not a reserve one — shed, no smaller cap", () => { + const body = LLAMACPP(40000); + assert.equal(completionOverflow(body), null); + assert.deepEqual(contextOverflow(body), { window: 32768, sent: 40000 }); + assert.equal(reserveFallback(body, 32768, 100), null, "the prompt alone does not fit"); + }); + + it("Anthropic: a prompt that is too long by itself is read as a prompt overflow", () => { + const body = `{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 210000 tokens > 200000 maximum"}}`; + assert.equal(completionOverflow(body), null); + assert.deepEqual(contextOverflow(body), { window: 200000, sent: 210000 }); + assert.equal(reserveFallback(body, 32768, 100), null); + }); + + it("an unknown window wording with a large cap gets one smaller cap to try; a small cap or an unrelated 400 does not", () => { + assert.equal(reserveFallback("Error: prompt + max_tokens exceeds the context length of this model", 32768, 1000), FALLBACK_COMPLETION); + // A window it named, with a prompt count it did not: fitted from molt's own estimate. + const g = reserveFallback("maximum context length is 32768 tokens; reduce your request", 32768, 1000); + assert.ok(g !== null && g <= 16384 && g >= 1024, String(g)); + assert.equal(reserveFallback("Error: prompt + max_tokens exceeds the context length of this model", 4096, 1000), null); + assert.equal(reserveFallback('{"error":{"message":"invalid tool_choice"}}', 32768, 1000), null); + }); + + it("refusedMaxTokens is as narrow as its comment: naming both fields is not a refusal of the field", () => { + assert.equal(refusedMaxTokens("max_tokens / max_completion_tokens must be <= 4096"), false); + assert.equal(refusedMaxTokens(VLLM_OLD(1000, 32768)), false); + assert.equal(refusedMaxTokens(`{"error":{"message":"Unrecognized request argument supplied: max_tokens"}}`), true); + }); +}); + +/** A server with a 32768 window that counts the reserve against it, refusing in `wording`. */ +function reserveServer(wording: (input: number, completion: number) => string, input = 1000) { + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + const asked = b.max_tokens ?? 32768 - input; + if (asked + input > 32768) { + refusals++; + return refused(wording(input, asked)); + } + return okReply({ role: "assistant", content: "answered." }); + }) as unknown as typeof fetch; + return { fetchFn, bodies, refusals: () => refusals }; +} + +describe("turn 1 against each provider's wording", () => { + for (const [name, make] of [ + ["OpenRouter", OPENROUTER], + ["vLLM (current)", VLLM_NEW], + ["vLLM 0.8-0.16", VLLM_OLD], + ["OpenAI", OPENAI], + ["Anthropic wording", ANTHROPIC], + ["an unknown wording", (i: number, c: number) => `{"error":{"message":"prompt (${i}) + max_tokens (${c}) exceeds the context length of this model"}}`], + ] as const) { + it(`${name}: answers on turn 1 with a smaller cap, no shed, max_tokens still sent`, async () => { + const ws = workspace(); + try { + const srv = reserveServer(make); + const engine = engineWith(ws.dir, srv.fetchFn); + const events = await drain(engine.run("say hi", allowAll)); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("answered.")), "the turn answered"); + assert.equal(events.filter((e) => e.kind === "shed").length, 0, "nothing was shed"); + assert.equal(srv.refusals(), 1); + assert.equal(srv.bodies[0]!.max_tokens, 32768); + const second = srv.bodies[1]!.max_tokens; + assert.ok(second !== undefined && second < 32768 - 1000 && second >= 1024, String(second)); + assert.deepEqual(srv.bodies[1]!.messages, srv.bodies[0]!.messages, "the same history went again"); + // The next turn does not pay the refusal again. + await drain(engine.run("again", allowAll)); + assert.equal(srv.refusals(), 1); + } finally { + ws.cleanup(); + } + }); + } + + it("llama.cpp: a prompt overflow sheds, and is not answered with a smaller cap", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "big.txt"), "x".repeat(6000)); + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 5) { + return okReply( + { role: "assistant", content: null, tool_calls: [{ id: `c${bodies.length}`, type: "function", function: { name: "read_file", arguments: '{"path":"big.txt"}' } }] }, + "tool_calls", + ); + } + if (refusals < 1) { + refusals++; + return refused(LLAMACPP(40000)); + } + return okReply({ role: "assistant", content: "fitted." }); + }) as unknown as typeof fetch; + const events = await drain(engineWith(ws.dir, fetchFn).run("read it a few times then answer", allowAll)); + assert.equal(refusals, 1); + assert.ok(events.some((e) => e.kind === "shed"), "the history was shed"); + assert.equal(bodies.at(-1)!.max_tokens, 32768, "max_tokens unchanged"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("fitted."))); + } finally { + ws.cleanup(); + } + }); + + it("a smaller cap that is refused too is forgotten, and the refusal goes on to the shed path", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 2) return refused(`{"error":{"message":"prompt + max_tokens exceeds the context length of this model"}}`); + return okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + await drain(engine.run("hi", allowAll)); + assert.equal(bodies[1]!.max_tokens, FALLBACK_COMPLETION, "one smaller cap was tried"); + assert.equal(bodies.length, 2, "then the shed path had nothing to shed on turn 1"); + const events = await drain(engine.run("again", allowAll)); + assert.equal(bodies[2]!.max_tokens, 32768, "the guess was not kept"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("ok."))); + } finally { + ws.cleanup(); + } + }); +}); + +describe("a request whose completion reserve overflows", () => { + it("is sent again with a smaller max_tokens, the same history and no shed, and the cap stays for the session", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + if ((b.max_tokens ?? 131072) + 2454 > 131072) { + refusals++; + return refused(gmiBody(2454, b.max_tokens ?? 131072)); + } + return okReply({ role: "assistant", content: "answered." }); + }) as unknown as typeof fetch; + // As the router did: the model's full listed output. + const engine = engineWith(ws.dir, fetchFn, { maxTokens: 131072 }); + const events = await drain(engine.run("say hi", allowAll)); + assert.equal(refusals, 1); + assert.equal(bodies.length, 2); + assert.equal(bodies[0]!.max_tokens, 131072); + assert.ok(bodies[1]!.max_tokens! <= 131072 - 2454 - 256 && bodies[1]!.max_tokens! >= 1024, String(bodies[1]!.max_tokens)); + assert.deepEqual(bodies[1]!.messages, bodies[0]!.messages, "the same history went again"); + assert.equal(events.filter((e) => e.kind === "shed").length, 0, "nothing was shed"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("answered."))); + // Remembered: the next turn does not pay the refusal again. + await drain(engine.run("again", allowAll)); + assert.equal(refusals, 1); + assert.equal(bodies[2]!.max_tokens, bodies[1]!.max_tokens); + } finally { + ws.cleanup(); + } + }); + + it("a real history overflow still sheds, and leaves max_tokens alone", async () => { + const ws = workspace(); + try { + writeFileSync(join(ws.dir, "big.txt"), "x".repeat(6000)); + const bodies: Body[] = []; + let refusals = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 5) { + return okReply( + { role: "assistant", content: null, tool_calls: [{ id: `c${bodies.length}`, type: "function", function: { name: "read_file", arguments: '{"path":"big.txt"}' } }] }, + "tool_calls", + ); + } + if (refusals < 1) { + refusals++; + return refused(gmiBody(130500, 32768)); + } + return okReply({ role: "assistant", content: "fitted." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + const events = await drain(engine.run("read it a few times then answer", allowAll)); + assert.equal(refusals, 1); + assert.ok(events.some((e) => e.kind === "shed"), "the history was shed"); + assert.equal(bodies.at(-1)!.max_tokens, bodies[0]!.max_tokens, "max_tokens unchanged"); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("fitted."))); + } finally { + ws.cleanup(); + } + }); +}); + +describe("the default output cap", () => { + it("is not the model's whole listed output, and doubles only when a reply hits it", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + return bodies.length === 1 + ? okReply({ role: "assistant", content: "a long answer that stops mid" }, "length") + : okReply({ role: "assistant", content: "word. Done." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + const events = await drain(engine.run("write", allowAll)); + assert.equal(bodies[0]!.max_tokens, 32768); + assert.equal(bodies[1]!.max_tokens, 65536); + // The cap the reply hit, not the doubled one, in the nudge the model reads and the info line. + assert.ok(JSON.stringify(bodies[1]!.messages).includes("output ceiling of 32768 tokens")); + assert.ok(!JSON.stringify(bodies[1]!.messages).includes("output ceiling of 65536")); + assert.ok(events.some((e) => e.kind === "info" && e.text.includes("cut off at the 32768-token output ceiling"))); + // A new model starts from the default again. + engine.setModel("other"); + await drain(engine.run("again", allowAll)); + assert.equal(bodies.at(-1)!.max_tokens, 32768); + } finally { + ws.cleanup(); + } + }); + + it("a provider that refuses the field gets requests without it", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + const b = JSON.parse(String(init?.body ?? "{}")) as Body; + bodies.push(b); + return "max_tokens" in b + ? refused(`{"error":{"message":"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."}}`) + : okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + const events = await drain(engine.run("hi", allowAll)); + assert.ok(events.some((e) => e.kind === "assistant_text" && e.text.includes("ok."))); + assert.equal("max_tokens" in bodies.at(-1)!, false); + // The next turn does not pay the refusal again. + const before = bodies.length; + await drain(engine.run("again", allowAll)); + assert.equal(bodies.length, before + 1); + assert.equal("max_tokens" in bodies.at(-1)!, false); + } finally { + ws.cleanup(); + } + }); + + it("a retry without the field that also fails puts max_tokens back", async () => { + const ws = workspace(); + try { + const bodies: Body[] = []; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body ?? "{}")) as Body); + if (bodies.length <= 2) return refused(`{"error":{"message":"Unrecognized request argument supplied: max_tokens"}}`); + return okReply({ role: "assistant", content: "ok." }); + }) as unknown as typeof fetch; + const engine = engineWith(ws.dir, fetchFn); + await drain(engine.run("hi", allowAll)); + assert.equal("max_tokens" in bodies[1]!, false, "tried once without"); + await drain(engine.run("again", allowAll)); + assert.equal(bodies.at(-1)!.max_tokens, 32768, "and put it back when that failed too"); + } finally { + ws.cleanup(); + } + }); +});