Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,11 @@ export function outputCeiling(body: string): number | null {
const m =
/max_tokens:\s*\d+\s*>\s*(\d+)/i.exec(body) ??
/maximum allowed number of output tokens[^\d]{0,40}(\d+)/i.exec(body) ??
/max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body);
/max_tokens\D{0,40}(?:less than or equal to|at most|maximum of)\D{0,10}(\d+)/i.exec(body) ??
// OpenAI: "max_tokens is too large: 32768. This model supports at most 16384 completion tokens".
/max_tokens is too large[\s\S]{0,80}?supports at most (\d+) completion tokens/i.exec(body) ??
// DeepSeek: "the valid range of max_tokens is [1, 8192]".
/valid range of max_tokens is \[\d+,\s*(\d+)\]/i.exec(body);
if (!m) return null;
const n = Number(m[1]);
return Number.isFinite(n) && n > 0 ? n : null;
Expand Down
261 changes: 253 additions & 8 deletions src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,124 @@ export type Overflow = {
sent: number;
};

/**
* A refusal where the completion reserve, not the prompt, overflowed.
*
* vLLM-style servers count `max_tokens` against the window. GMI Cloud serves
* qwen3-235b with a 131,072-token TOTAL window, and the request went out with
* the model's full listed output (an OpenAI-shaped request with no max_tokens
* is given that by the router), so a 2,454-token prompt was refused:
*
* maximum context length of 131072 tokens. You requested a total of
* 133526 tokens: 2454 tokens from the input messages and 131072 tokens
* for the completion.
*
* Shedding history can never fix that. Every server words it differently:
* vLLM (both its old and current wordings), OpenAI, OpenRouter, Anthropic and
* TGI are read below. llama.cpp never refuses for the reserve (it stops the
* reply when the window fills), so its refusal is always a prompt overflow.
* Returns the window, the input and the completion the server counted, or
* null when the body does not say all three.
*/
export function completionOverflow(body: string): { window: number; input: number; completion: number } | null {
const ok = (window: number, input: number, completion: number) =>
window > 0 && input >= 0 && completion > 0 ? { window, input, completion } : null;

// Anthropic: "input length and `max_tokens` exceed context limit: 188240 + 21333 > 200000".
const anth = /max_tokens`?\s+exceed context limit:\s*(\d+)\s*\+\s*(\d+)\s*>\s*(\d+)/i.exec(body);
if (anth) return ok(Number(anth[3]), Number(anth[1]), Number(anth[2]));

// Hugging Face TGI: "`inputs` tokens + `max_new_tokens` must be <= 4096. Given: 1000 `inputs` tokens and 4000 `max_new_tokens`".
const tgi = /must be <= (\d+)\.\s*Given:\s*(\d+) `?inputs`? tokens and (\d+) `?max_new_tokens`?/i.exec(body);
if (tgi) return ok(Number(tgi[1]), Number(tgi[2]), Number(tgi[3]));

const win = /maximum context length (?:is|of) (\d+)/i.exec(body);
if (!win) return null;
const window = Number(win[1]);

// vLLM (since vllm#36197): "you requested 32768 output tokens and your prompt contains 1000 input tokens".
const v2 = /you requested (\d+) output tokens? and your prompt contains (\d+) input tokens?/i.exec(body);
if (v2) return ok(window, Number(v2[2]), Number(v2[1]));

// vLLM 0.8-0.16: "'max_tokens' or 'max_completion_tokens' is too large: 32768. This model's maximum
// context length is 32768 tokens and your request has 1000 input tokens (32768 > 32768 - 1000)."
const v1 = /is too large:\s*(\d+)[\s\S]{0,200}?your request has (\d+) input tokens?/i.exec(body);
if (v1) return ok(window, Number(v1[2]), Number(v1[1]));

// GMI / vLLM: "A tokens from the input messages and B tokens for the completion".
const gmi = /(\d+) tokens? from the input messages and (\d+) tokens? for the completion/i.exec(body);
if (gmi) return ok(window, Number(gmi[1]), Number(gmi[2]));

// OpenRouter: "(1000 of text input, 900 of tool input, 32768 in the output)". Every "of … input"
// part counts against the prompt (text, tool, image).
const orOut = /(\d+) in the output/i.exec(body);
if (orOut) {
const ins = [...body.matchAll(/(\d+) of [a-z ]{0,20}?input/gi)].map((m) => Number(m[1]));
if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(orOut[1]));
}

// OpenAI: "(1232 in the messages, 32768 in the completion)", sometimes with "N in the functions".
const oai = /\(([^)]*?(\d+) in the completion)\)/i.exec(body);
if (oai) {
const ins = [...oai[1]!.matchAll(/(\d+) in the (?!completion)\w+/gi)].map((m) => Number(m[1]));
if (ins.length) return ok(window, ins.reduce((a, b) => a + b, 0), Number(oai[2]));
}
return null;
}

/** The most an unasked output cap grows to after replies hit it. */
export const OUTPUT_CAP_MAX = 131_072;

/**
* A 400 that refuses the `max_tokens` field itself, as opposed to its value.
* Narrow on purpose: it must name the field and say it is unsupported.
*/
export function refusedMaxTokens(body: string): boolean {
return /max_tokens/i.test(body) && /unsupported|not supported|unrecognized|unknown (field|parameter)|extra inputs are not permitted|use 'max_completion_tokens'/i.test(body);
}

/** Room left between the prompt and the window, kept free of the completion reserve. */
export const COMPLETION_MARGIN = 256;
/** A completion cap below this is no reply worth asking for; the prompt is what has to shrink. */
export const MIN_COMPLETION = 1_024;

/**
* The completion cap that fits, when lowering it alone fixes the overflow:
* the window less the input and a margin. Null when even the smallest useful
* reply does not fit (the history is the problem, and shedding is the fix).
*/
export function fittedCompletion(o: { window: number; input: number; completion: number }): number | null {
const fit = o.window - o.input - Math.max(COMPLETION_MARGIN, Math.round(o.window * 0.01));
return fit >= MIN_COMPLETION && fit < o.completion ? fit : null;
}

/** The cap tried once when a window refusal names no counts molt can read. */
export const FALLBACK_COMPLETION = 8_192;

/**
* A smaller cap to try before shedding, for a window refusal whose wording
* completionOverflow does not know.
*
* Maat always sends max_tokens now, and a server that counts it against the
* window refuses in its own words. Shedding cannot fix a reserve overflow, so
* when the refusal is about the context or the length and the cap sent was
* large, one request with a smaller cap is cheaper than ending a turn "with
* nothing left to shed". `inputEst` is molt's scaled estimate of the prompt,
* used when the server named a window but no prompt count. Null when the
* prompt itself is known not to fit (shed instead), when the cap is already
* small, or when the body is not about the window at all.
*/
export function reserveFallback(body: string, cap: number, inputEst: number): number | null {
if (cap <= FALLBACK_COMPLETION || completionOverflow(body)) return null;
const over = contextOverflow(body);
if (!over && !(/context|length/i.test(body) && /tokens?/i.test(body))) return null;
if (!over || over.window <= 0) return FALLBACK_COMPLETION;
const input = over.sent > 0 ? over.sent : inputEst;
const fit = over.window - input - Math.max(COMPLETION_MARGIN, Math.round(over.window * 0.01));
if (fit < MIN_COMPLETION) return null;
return Math.min(fit, Math.floor(cap / 2));
}

export function contextOverflow(body: string): Overflow | null {
// Overflow wording, not the bare word "context": a pinned OpenRouter provider
// answered a rate-limit retry with a 400 that merely mentioned context, and a
Expand All @@ -348,15 +466,25 @@ export function contextOverflow(body: string): Overflow | null {
)
)
return null;
// Where the server split the prompt from the completion, the prompt's count
// is what molt's estimate is compared with. Never the output count: current
// vLLM puts that first ("you requested 32768 output tokens and your prompt
// contains 1000"), and reading it as the prompt scaled every estimate 8x.
const co = completionOverflow(body);
if (co) return { window: co.window, sent: co.input };
// Every field the common servers use, most specific first.
const win =
/"n_ctx"\s*:\s*(\d+)/.exec(body) ??
/context size \((\d+)\s*tokens?\)/i.exec(body) ??
/maximum context length is (\d+)/i.exec(body);
/maximum context length (?:is|of) (\d+)/i.exec(body) ??
// Anthropic: "prompt is too long: 210000 tokens > 200000 maximum".
/tokens? > (\d+) maximum/i.exec(body);
const sent =
/"n_prompt_tokens"\s*:\s*(\d+)/.exec(body) ??
/request \((\d+)\s*tokens?\)/i.exec(body) ??
/you requested (\d+)/i.exec(body);
/prompt is too long:\s*(\d+)/i.exec(body) ??
/your (?:prompt contains|request has|messages resulted in) (\d+)/i.exec(body) ??
/you requested (?:a total of |about )?(\d+)(?!\d)(?! output)/i.exec(body);
return { window: win ? Number(win[1]) : 0, sent: sent ? Number(sent[1]) : 0 };
}

Expand Down Expand Up @@ -1978,10 +2106,30 @@ export class Engine {
*/
private modelMaxTokens?: number;

/**
* The output cap this session settled on without being asked: the default
* (DEFAULT_MAX_TOKENS, not the model's whole listed output), doubled each
* time a reply is cut off at it, up to OUTPUT_CAP_MAX.
*/
private outputCap = DEFAULT_MAX_TOKENS;
/**
* The completion reserve a server said fits its window beside the prompt
* (completionOverflow). Kept until the history is shed: the prompt grows
* between sheds, and after one there is room for a larger reply again.
*/
private fittedCompletion?: number;
/**
* The fitted cap above was a guess (reserveFallback), not read from the
* server's own counts. A reply that is cut off at a guess drops it.
*/
private fittedGuessed = false;
/** Set once a provider refuses `max_tokens` outright; it is then not sent. */
private maxTokensUnsupported = false;

/** What to send as `max_tokens`: what was asked for, bounded by what fits. */
private maxTokensFor(): number {
const asked = this.cfg.maxTokens ?? DEFAULT_MAX_TOKENS;
return this.modelMaxTokens ? Math.min(asked, this.modelMaxTokens) : asked;
const asked = this.cfg.maxTokens ?? this.outputCap;
return Math.min(asked, this.modelMaxTokens ?? Infinity, this.fittedCompletion ?? Infinity);
}
/**
* These three are derived from the endpoint, and the endpoint moves.
Expand Down Expand Up @@ -2291,6 +2439,11 @@ export class Engine {
// Output maximums are per-model, and the last one's says nothing
// about this one. Discovered again on the next refusal if it matters.
this.modelMaxTokens = undefined;
this.fittedCompletion = undefined;
this.fittedGuessed = false;
this.maxTokensUnsupported = false;
// A cap doubled on the last model says nothing about this one either.
this.outputCap = DEFAULT_MAX_TOKENS;
}
this.cfg.model = m;
}
Expand Down Expand Up @@ -3059,6 +3212,9 @@ export class Engine {
}

this.transcript.commitShed(plan);
// A cap fitted beside the old prompt is too small for the new one.
this.fittedCompletion = undefined;
this.fittedGuessed = false;

this.ledger = staying;
// Journalled here, on every path. It was journalled by the two auto-shed
Expand Down Expand Up @@ -6080,6 +6236,10 @@ export class Engine {
// ignored (see selfHostedThinking). Nothing on a cloud endpoint.
...selfHostedThinking(this.cfg.baseUrl, this.effortNow),
...openRouterProvider(this.cfg.baseUrl, this.cfg.model),
// Always a cap. Without one a router fills in the model's whole
// listed output, and a server that counts it against the window
// refuses every request (completionOverflow).
...(this.maxTokensUnsupported ? {} : { max_tokens: this.maxTokensFor() }),
...(stream ? { stream: true } : {}),
...(withUsage ? { stream_options: { include_usage: true } } : {}),
};
Expand Down Expand Up @@ -6215,7 +6375,7 @@ export class Engine {
// so, and says what the maximum is. Same shape as the two
// fallbacks above: retry once, and only believe the ceiling was
// the problem if that works.
if (!res.ok && res.status === 400 && this.native && this.modelMaxTokens === undefined) {
if (!res.ok && res.status === 400 && this.modelMaxTokens === undefined && !this.maxTokensUnsupported) {
const cap = outputCeiling(body);
if (cap && cap < this.maxTokensFor()) {
const was = this.maxTokensFor();
Expand All @@ -6236,6 +6396,77 @@ export class Engine {
}
}

// The completion reserve overflowed, not the prompt: ask for a
// smaller reply and send the same history again. Shedding cannot
// fix this one, and it was ending runs "too large and nothing left
// to shed" on a 2,454-token prompt.
for (let i = 0; i < 2 && !res.ok && res.status === 400; i++) {
const co = completionOverflow(body);
const fit = co ? fittedCompletion(co) : null;
if (!co || fit === null || fit >= this.maxTokensFor()) break;
const was = this.maxTokensFor();
this.fittedCompletion = fit;
this.fittedGuessed = false;
log?.append("note", {
body: body.slice(0, 600),
text:
`the completion reserve overflowed the ${co.window}-token window (${co.input} input + ` +
`${co.completion} completion) — max_tokens ${was} → ${fit}, retried without shedding`,
});
yield {
kind: "info",
text: `this endpoint counts the reply against its ${co.window}-token window — asking for at most ${fit} output tokens instead of ${was}.`,
};
res = await send(askForUsage && !this.streamUsageUnsupported);
body = res.ok ? "" : (await res.text().catch(() => ""));
}

// A window refusal in wording none of the above reads, from a
// request that carried a large cap. Try one smaller cap before
// shedding: shedding cannot fix a reserve overflow, and before
// max_tokens was always sent these servers fitted the reply
// themselves. Kept only if it works.
if (!res.ok && res.status === 400 && !this.maxTokensUnsupported) {
const was = this.maxTokensFor();
const guess = reserveFallback(body, was, Math.round(this.bom().requestTotalEst * this.tokenScale));
if (guess !== null && guess < was) {
const before = { fitted: this.fittedCompletion, guessed: this.fittedGuessed };
this.fittedCompletion = guess;
this.fittedGuessed = true;
const retry = await send(askForUsage && !this.streamUsageUnsupported);
if (retry.ok) {
log?.append("note", {
body: body.slice(0, 600),
text: `window refusal with max_tokens ${was} — ${guess} was accepted, kept for the session`,
});
yield {
kind: "info",
text: `this endpoint refused a ${was}-token reply reserve — asking for at most ${guess} output tokens instead.`,
};
} else {
this.fittedCompletion = before.fitted;
this.fittedGuessed = before.guessed;
}
res = retry;
body = res.ok ? "" : (await res.text().catch(() => ""));
}
}

// A provider that will not take `max_tokens` at all (OpenAI's
// reasoning models want max_completion_tokens). Try once without;
// only believe the field was the problem if that works.
if (!res.ok && res.status === 400 && !this.native && !this.maxTokensUnsupported && refusedMaxTokens(body)) {
this.maxTokensUnsupported = true;
const retry = await send(askForUsage && !this.streamUsageUnsupported);
if (retry.ok) {
log?.append("note", { text: "provider refused max_tokens — sending requests without an output cap" });
} else {
this.maxTokensUnsupported = false;
}
res = retry;
body = res.ok ? "" : (await res.text().catch(() => ""));
}

if (!res.ok) {
// "Too big" is not a verdict on the work, and it is the one
// refusal that says how to fix itself. Shed and send less rather
Expand Down Expand Up @@ -7077,15 +7308,29 @@ export class Engine {
// sentence.
if (finishReason === "length" && truncatedTurns < TRUNCATED_TURN_RETRIES) {
truncatedTurns += 1;
// The cap this reply hit, before any change below: the note, the
// nudge and the info line report what happened, not what comes next.
const hit = this.maxTokensFor();
// A reply that actually hit the default cap earns a larger one; a cap
// the person set, or one the server said is all that fits, stays. A
// fitted cap that was only a guess is dropped; if the full one is
// refused again, the refusal fits it again.
if (this.cfg.maxTokens === undefined && this.outputCap < OUTPUT_CAP_MAX && hit === this.outputCap) {
this.outputCap = Math.min(OUTPUT_CAP_MAX, this.outputCap * 2);
} else if (this.fittedGuessed && hit === this.fittedCompletion) {
this.fittedCompletion = undefined;
this.fittedGuessed = false;
}
const next = this.maxTokensFor();
log?.append("note", {
text: `reply cut off at the ${this.maxTokensFor()}-token output ceiling`,
text: `reply cut off at the ${hit}-token output ceiling` + (next !== hit ? `; the next request asks for ${next}` : ""),
step,
attempt: truncatedTurns,
});
this.transcript.push({
role: "user",
content:
`[molt: that reply hit the output ceiling of ${this.maxTokensFor()} tokens and stopped ` +
`[molt: that reply hit the output ceiling of ${hit} tokens and stopped ` +
`part-way through, so it is not being read as a finished answer. Continue from where ` +
`it stops, and keep what remains short.]`,
molt: { nudge: true },
Expand All @@ -7094,7 +7339,7 @@ export class Engine {
yield {
kind: "info",
text:
`the reply was cut off at the ${this.maxTokensFor()}-token output ceiling — asking it ` +
`the reply was cut off at the ${hit}-token output ceiling — asking it ` +
`to continue rather than running the bar on half a sentence` +
(this.cfg.maxTokens === undefined ? " (raise it with --max-tokens)" : ""),
};
Expand Down
Loading
Loading