diff --git a/benchmarks/terminal_bench_2_1/prepare.py b/benchmarks/terminal_bench_2_1/prepare.py index 6a27e188bd..bf2710aa41 100644 --- a/benchmarks/terminal_bench_2_1/prepare.py +++ b/benchmarks/terminal_bench_2_1/prepare.py @@ -41,6 +41,9 @@ def prepare() -> Path: "task_name": task_toml["task"]["name"], "docker_image": task_toml["environment"]["docker_image"], "task_folder": str(task_dir.relative_to(BENCHMARK_DIR.parent.parent)), + # Terminal-Bench declares a per-task agent budget. Carry it through so the + # agent can honour it instead of applying one flat wall to every task. + "agent_timeout_sec": (task_toml.get("agent") or {}).get("timeout_sec"), } f_out.write(json.dumps(sample) + "\n") diff --git a/responses_api_agents/terminus_2_sandboxed_agent/app.py b/responses_api_agents/terminus_2_sandboxed_agent/app.py index 26e15789cc..352e3701a1 100644 --- a/responses_api_agents/terminus_2_sandboxed_agent/app.py +++ b/responses_api_agents/terminus_2_sandboxed_agent/app.py @@ -67,7 +67,10 @@ class Terminus2AgentConfig(BaseResponsesAPIAgentConfig): sandbox_provider: str sandbox_config: dict[str, Any] = Field(default_factory=dict) + # Fallback agent wall clock, used when the dataset row carries no per-task budget. sandbox_timeout: float + # Upper bound applied to the per-task budget, mirroring Harbor's agent.max_timeout_sec. + max_agent_timeout: Optional[float] = None remote_tmux_binary_path: Optional[str] @@ -326,13 +329,22 @@ async def _connect_sandbox(self, sandbox_id: str) -> AsyncSandbox: sandbox = await AsyncSandbox.connect({"sandbox_id": sandbox_id}, provider=provider) return sandbox + def _resolve_agent_timeout(self, task_timeout: float | None) -> float: + """Per-task budget when the dataset supplies one, else the flat fallback, then capped.""" + timeout = self.config.sandbox_timeout if task_timeout is None else float(task_timeout) + if self.config.max_agent_timeout is not None: + timeout = min(timeout, self.config.max_agent_timeout) + return timeout + async def _execute( self, request: Request, body: NeMoGymResponseCreateParamsNonStreaming, sandbox: AsyncSandbox, + task_timeout: float | None = None, ) -> Tuple[NeMoGymResponse, Dict[str, Any]]: start_time = perf_counter() + agent_timeout = self._resolve_agent_timeout(task_timeout) instruction = _instruction(body.input) model_base_url = ( @@ -383,7 +395,7 @@ async def _execute( await agent.setup(environment) try: - async with asyncio.timeout(self.config.sandbox_timeout): + async with asyncio.timeout(agent_timeout): await agent.run(instruction, environment, context) terminus2_completed = True error = None @@ -430,6 +442,7 @@ async def _execute( "command_exec_time_pct": 100 * total_command_exec_time / total_time, "model_call_time_pct": 100 * total_model_call_time / total_time, "terminus2_time_taken": total_time, + "agent_timeout": agent_timeout, "model_calls_gt_10min": llm._model_calls_gt_10min, "num_proactive_compactions": agent._num_proactive_compactions, "num_compactions": llm._num_compactions, @@ -462,7 +475,12 @@ async def run(self, request: Request, body: Terminus2AgentRunRequest) -> Terminu session_key = request.session[SESSION_ID_KEY] self._session_sandboxes[session_key] = sandbox - response, metrics = await self._execute(request, body.responses_create_params, sandbox) + response, metrics = await self._execute( + request, + body.responses_create_params, + sandbox, + task_timeout=getattr(body, "agent_timeout_sec", None), + ) verification = await self.server_client.post( server_name=self.config.resources_server.name, diff --git a/responses_api_agents/terminus_2_sandboxed_agent/configs/terminus_2_sandboxed_agent.yaml b/responses_api_agents/terminus_2_sandboxed_agent/configs/terminus_2_sandboxed_agent.yaml index cdcb64249f..f20d1e75a6 100644 --- a/responses_api_agents/terminus_2_sandboxed_agent/configs/terminus_2_sandboxed_agent.yaml +++ b/responses_api_agents/terminus_2_sandboxed_agent/configs/terminus_2_sandboxed_agent.yaml @@ -23,7 +23,8 @@ terminus_2_sandboxed_agent: llm_request_timeout: 600 # 10 mins, Litellm default sandbox_provider: sandbox - sandbox_timeout: 10800 # 3 hrs + sandbox_timeout: 10800 # 3 hrs; only used when the dataset row has no agent_timeout_sec + max_agent_timeout: null # cap on the per-task budget; null means uncapped sandbox_config: ttl_s: 18000 ready_timeout_s: 1200 diff --git a/responses_api_agents/terminus_2_sandboxed_agent/tests/test_app.py b/responses_api_agents/terminus_2_sandboxed_agent/tests/test_app.py index ac1d87b932..8cce91fdcc 100644 --- a/responses_api_agents/terminus_2_sandboxed_agent/tests/test_app.py +++ b/responses_api_agents/terminus_2_sandboxed_agent/tests/test_app.py @@ -163,6 +163,54 @@ async def create_response(self, **kwargs): ] +def _agent_config(**overrides): + defaults = dict( + host="0.0.0.0", + port=8080, + entrypoint="app.py", + name="terminus_2_1_agent", + resources_server=ResourcesServerRef(type="resources_servers", name="swebench_resources_server"), + model_server=ModelServerRef(type="responses_api_models", name="policy_model"), + max_turns=100, + enable_summarize=True, + proactive_summarization_threshold=8000, + tmux_pane_width=160, + tmux_pane_height=40, + dump_trajectory=False, + debug=False, + model_context_limit=32_000, + model_output_limit=4_000, + llm_request_timeout=60, + sandbox_provider="opensandbox", + sandbox_timeout=10800, + remote_tmux_binary_path=None, + ) + return Terminus2AgentConfig(**(defaults | overrides)) + + +@pytest.mark.parametrize( + "task_timeout, max_agent_timeout, expected", + [ + # No per-task budget in the dataset row: fall back to the flat wall. + (None, None, 10800), + # Per-task budget wins over the flat wall, in both directions. + (900, None, 900), + (12000, None, 12000), + # The cap only binds when the per-task budget exceeds it. + (12000, 7200, 7200), + (900, 7200, 900), + # The cap also bounds the fallback. + (None, 7200, 7200), + ], +) +def test_resolve_agent_timeout(task_timeout, max_agent_timeout, expected): + server = Terminus2Agent( + config=_agent_config(max_agent_timeout=max_agent_timeout), + server_client=MagicMock(spec=ServerClient), + ) + assert server._resolve_agent_timeout(task_timeout) == expected + + @pytest.mark.asyncio @pytest.mark.parametrize("dump_trajectory", [False, True]) @pytest.mark.parametrize("debug", [False, True]) @@ -267,6 +315,7 @@ async def request_json(): "command_exec_time_pct": 40.0, "model_call_time_pct": 60.0, "terminus2_time_taken": 10.0, + "agent_timeout": 10.0, "model_calls_gt_10min": 0, "num_proactive_compactions": 0, "num_compactions": 2,