Skip to content

Commit 30c62ce

Browse files
tmatupclaude
andauthored
fix(codex): crash a turn codex ended as failed instead of grading it (#230)
Codex delivers a normal turn/completed when it gives up on the provider, with turn.status == failed and a TurnError (for example rateLimitExceeded after its own stream retries ran out). The harness never read that status, so the turn booked as COMPLETED and the row was graded FAILURE on work the provider had cut off. In adhoc-2026-10-21_16-16-27 (codex, gpt-5.6-luna) 27 rows (11 v1, 16 v2) ended this way, after 0 to 39 commands, and read as skill failures in the v1-vs-v2 comparison. The turn now crashes with the provider's message, the same way Pi and OpenCode already treat a terminal provider error. categorize_error routes the "rate limit exceeded" text to AGENT_RATE_LIMIT and its retry budget; when retries run out the row lands as ERROR. The partial turn keeps its commands and provider_errors. Claude-Session: https://claude.ai/code/session_013fePq8hgqbtNvf2pNvA3WC Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 32f18e4 commit 30c62ce

3 files changed

Lines changed: 165 additions & 10 deletions

File tree

‎.claude/notes/agents.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,10 +267,15 @@ matter how much the run actually billed. So the CLI harnesses crash rather than
267267
arm keys on a step the CLI reported as FINISHED, its own claim that a generation
268268
completed, rather than on `usage.is_empty()` alone, which would also condemn a stream
269269
cut before any step could finish.
270-
- **A terminal provider error** (Pi, OpenCode) — `pi -p` exits 0 after exhausting its
270+
- **A terminal provider error** (Pi, OpenCode, Codex) — `pi -p` exits 0 after exhausting its
271271
internal retries, so without the crash the turn books as a clean COMPLETED
272272
(`FinalStatus.FAILURE`, category "failed"), silently depressing the measured pass rate.
273273
Crashing routes it to `FinalStatus.ERROR`, which is excluded from outcomes.
274+
Codex reports the same thing differently: its `turn/completed` arrives normally but
275+
carries `turn.status == failed` and a `TurnError` (for example `rateLimitExceeded`
276+
after its own stream retries ran out). The harness reads that status and crashes, with
277+
the provider's message, so `categorize_error` routes a rate limit to
278+
`AGENT_RATE_LIMIT` and its retry budget like any other agent crash.
274279
- **A CLI that closed its stream but would not exit** within the grace period.
275280

276281
Every arm is gated on `stopped_early` / `max_turns_exhausted`, because an intentional cut

‎src/coder_eval/agents/codex_agent.py‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,23 @@ def _status_value(status: Any) -> str:
259259
return str(value).lower() if value is not None else ""
260260

261261

262+
def _failed_turn_reason(turn: Any) -> str | None:
263+
"""The crash message for a Turn codex ended as ``failed``, or None for any other status.
264+
265+
Codex ends a turn ``failed`` when it gives up on the provider, e.g. a
266+
``rateLimitExceeded`` it will not retry. The message carries the provider's
267+
own text, so ``categorize_error`` routes it (``rate limit`` ->
268+
``AGENT_RATE_LIMIT``) like any other agent crash.
269+
"""
270+
if turn is None or _status_value(getattr(turn, "status", None)) != "failed":
271+
return None
272+
error = getattr(turn, "error", None)
273+
message = str(getattr(error, "message", "") or "").strip() or "no error message"
274+
kind, _ = _codex_error_kind(error)
275+
suffix = f" (codexErrorInfo={kind})" if kind else ""
276+
return f"Codex turn failed: {message}{suffix}"
277+
278+
262279
def _record_command_text(argv: Any) -> str:
263280
"""The command a CommandExecution record ran, without the shell wrapper Codex adds.
264281
@@ -384,24 +401,29 @@ def _message_uncached_input(m: AssistantMessage) -> int:
384401
_ZSH_PROFILE_NAMES = frozenset({".zshenv", ".zprofile", ".zshrc"})
385402

386403

387-
def _provider_error_from(payload: Any) -> ProviderError:
388-
"""Map a codex ``ErrorNotification`` payload onto a ``ProviderError``.
404+
def _codex_error_kind(error: Any) -> tuple[str | None, int | None]:
405+
"""``(kind, http_status)`` of a codex ``TurnError``'s ``codexErrorInfo``.
389406
390407
``codexErrorInfo`` is either a bare category (enum) or a one-key object whose
391408
key is the category and whose value may carry ``httpStatusCode``.
392409
"""
393-
error = getattr(payload, "error", None)
394-
kind: str | None = None
395-
http_status: int | None = None
396-
info = getattr(getattr(error, "codex_error_info", None), "root", None)
410+
info: Any = getattr(getattr(error, "codex_error_info", None), "root", None)
397411
if isinstance(info, Enum):
398-
kind = str(info.value)
399-
elif hasattr(info, "model_dump"):
412+
return str(info.value), None
413+
if hasattr(info, "model_dump"):
400414
dumped = info.model_dump(by_alias=True, mode="json")
401415
if len(dumped) == 1:
402416
kind, body = next(iter(dumped.items()))
403417
if isinstance(body, dict) and isinstance(body.get("httpStatusCode"), int):
404-
http_status = body["httpStatusCode"]
418+
return kind, body["httpStatusCode"]
419+
return kind, None
420+
return None, None
421+
422+
423+
def _provider_error_from(payload: Any) -> ProviderError:
424+
"""Map a codex ``ErrorNotification`` payload onto a ``ProviderError``."""
425+
error = getattr(payload, "error", None)
426+
kind, http_status = _codex_error_kind(error)
405427
return ProviderError(
406428
at=datetime.now(),
407429
message=str(getattr(error, "message", "") or ""),
@@ -1199,6 +1221,12 @@ def _on_turn_timeout() -> None:
11991221
# like every other timeout path.
12001222
assert timeout is not None
12011223
self._finalize_and_raise_timeout(state.finalize, timeout)
1224+
failed_reason = None if state.ended_cleanly else _failed_turn_reason(state.result_turn)
1225+
if failed_reason is not None:
1226+
# Codex gave up on the turn (a provider error it would not retry),
1227+
# so the agent never finished: crash, never score it.
1228+
# Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash
1229+
self._finalize_and_raise_crash(state.finalize, truncate_crash_message(failed_reason))
12021230
except (AgentCrashError, TurnTimeoutError):
12031231
# Already funneled through finalize by the inner handlers.
12041232
raise

‎tests/test_codex_agent.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2985,3 +2985,125 @@ def test_clean_turn_has_no_provider_errors(self):
29852985
st, collector = self._state()
29862986
st.finalize(AgentEndStatus.COMPLETED)
29872987
assert collector.build_turn_record().provider_errors == []
2988+
2989+
2990+
# The provider message codex reported in adhoc-2026-10-21_16-16-27
2991+
# (v2/skill-case-diagnose-formal-arg-slot-ids): five retried stream disconnects,
2992+
# then a rate limit it would not retry, then a `failed` turn/completed.
2993+
_RATE_LIMIT_MESSAGE = (
2994+
"rate limit exceeded: Your requests to gpt-5.6-luna for gpt-5.6-luna in eastus2 have exceeded token rate limit."
2995+
)
2996+
2997+
2998+
def _error_notification(error: dict, *, will_retry: bool) -> SimpleNamespace:
2999+
from openai_codex.generated.v2_all import ErrorNotification
3000+
3001+
payload = ErrorNotification.model_validate(
3002+
{"error": error, "threadId": "t1", "turnId": "u1", "willRetry": will_retry}
3003+
)
3004+
return SimpleNamespace(method="error", payload=payload)
3005+
3006+
3007+
def _turn_failed(error: dict | None) -> SimpleNamespace:
3008+
turn = Turn.model_validate(
3009+
{
3010+
"id": "turn_1",
3011+
"status": "failed",
3012+
"durationMs": 21847,
3013+
"error": error,
3014+
"items": [],
3015+
"itemsView": "notLoaded",
3016+
}
3017+
)
3018+
return SimpleNamespace(method="turn/completed", payload=TurnCompletedNotification(thread_id="th_1", turn=turn))
3019+
3020+
3021+
def _rate_limited_turn(*, commands: int) -> list[SimpleNamespace]:
3022+
notes: list[SimpleNamespace] = []
3023+
for i in range(commands):
3024+
root = SimpleNamespace(
3025+
type="commandExecution",
3026+
id=f"cmd_{i}",
3027+
command=f"uip maestro case validate {i}",
3028+
exit_code=0,
3029+
aggregated_output="ok\n",
3030+
duration_ms=5,
3031+
)
3032+
notes += [_item_notification("item/started", root), _item_notification("item/completed", root)]
3033+
notes.append(_delta("I'm using the Maestro Case skill to repair the plan."))
3034+
for attempt in range(1, 6):
3035+
notes.append(
3036+
_error_notification(
3037+
{
3038+
"message": f"Reconnecting... {attempt}/5",
3039+
"codexErrorInfo": {"responseStreamDisconnected": {"httpStatusCode": None}},
3040+
"additionalDetails": _RATE_LIMIT_MESSAGE,
3041+
},
3042+
will_retry=True,
3043+
)
3044+
)
3045+
notes.append(
3046+
_error_notification({"message": _RATE_LIMIT_MESSAGE, "codexErrorInfo": "rateLimitExceeded"}, will_retry=False)
3047+
)
3048+
notes.append(_turn_failed({"message": _RATE_LIMIT_MESSAGE, "codexErrorInfo": "rateLimitExceeded"}))
3049+
return notes
3050+
3051+
3052+
class TestFailedTurnIsACrash:
3053+
"""A `turn/completed` whose turn is `failed` is a crash, never a COMPLETED turn.
3054+
3055+
Before this, codex's terminal rate limit booked as COMPLETED and the row was
3056+
graded FAILURE: 27 of 114 rows in adhoc-2026-10-21_16-16-27 (codex,
3057+
gpt-5.6-luna) were scored on a turn the provider cut off.
3058+
"""
3059+
3060+
@pytest.mark.parametrize("commands", [0, 4])
3061+
async def test_rate_limited_turn_raises_crash_and_keeps_the_partial(self, commands):
3062+
agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), _rate_limited_turn(commands=commands))
3063+
3064+
with pytest.raises(AgentCrashError) as excinfo:
3065+
await agent.communicate("repair the case")
3066+
3067+
assert _RATE_LIMIT_MESSAGE in str(excinfo.value)
3068+
assert "codexErrorInfo=rateLimitExceeded" in str(excinfo.value)
3069+
assert agent.get_state() == AgentState.ERROR
3070+
3071+
partial = agent.pending_turn
3072+
assert partial is not None
3073+
assert partial.crashed is True
3074+
assert partial.crash_reason is not None and "rate limit exceeded" in partial.crash_reason
3075+
# The work done before the cut and the provider evidence both survive.
3076+
assert len(partial.commands) == commands
3077+
assert [(e.kind, e.will_retry) for e in partial.provider_errors] == [
3078+
*[("responseStreamDisconnected", True)] * 5,
3079+
("rateLimitExceeded", False),
3080+
]
3081+
3082+
async def test_rate_limit_crash_is_categorized_as_a_rate_limit(self):
3083+
from coder_eval.errors.categories import ErrorCategory
3084+
from coder_eval.errors.categorization import categorize_error
3085+
3086+
agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), _rate_limited_turn(commands=0))
3087+
with pytest.raises(AgentCrashError) as excinfo:
3088+
await agent.communicate("repair the case")
3089+
3090+
assert categorize_error(excinfo.value, {"component": "agent"}) is ErrorCategory.AGENT_RATE_LIMIT
3091+
3092+
async def test_failed_turn_without_error_details_still_crashes(self):
3093+
agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), [_delta("hi"), _turn_failed(None)])
3094+
3095+
with pytest.raises(AgentCrashError, match="no error message"):
3096+
await agent.communicate("go")
3097+
3098+
async def test_completed_turn_with_a_retried_provider_error_is_not_a_crash(self):
3099+
notes = [
3100+
_error_notification({"message": "Reconnecting... 1/5"}, will_retry=True),
3101+
_delta("done"),
3102+
_turn_completed(),
3103+
]
3104+
agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notes)
3105+
3106+
record = await agent.communicate("go")
3107+
3108+
assert record.crashed is False
3109+
assert agent.pending_turn is None

0 commit comments

Comments
 (0)