feat(terminus-2): honour Terminal-Bench per-task agent timeouts

benchmarks/terminal_bench_2_1/prepare.py

@@ -41,6 +41,9 @@ def prepare() -> Path:
             "task_name": task_toml["task"]["name"],
             "docker_image": task_toml["environment"]["docker_image"],
             "task_folder": str(task_dir.relative_to(BENCHMARK_DIR.parent.parent)),
+            # Terminal-Bench declares a per-task agent budget. Carry it through so the
+            # agent can honour it instead of applying one flat wall to every task.
+            "agent_timeout_sec": (task_toml.get("agent") or {}).get("timeout_sec"),
         }
 
         f_out.write(json.dumps(sample) + "\n")

responses_api_agents/terminus_2_sandboxed_agent/app.py

@@ -67,7 +67,10 @@ class Terminus2AgentConfig(BaseResponsesAPIAgentConfig):
 
     sandbox_provider: str
     sandbox_config: dict[str, Any] = Field(default_factory=dict)
+    # Fallback agent wall clock, used when the dataset row carries no per-task budget.
     sandbox_timeout: float
+    # Upper bound applied to the per-task budget, mirroring Harbor's agent.max_timeout_sec.
+    max_agent_timeout: Optional[float] = None
     remote_tmux_binary_path: Optional[str]
 
 
@@ -326,13 +329,22 @@ async def _connect_sandbox(self, sandbox_id: str) -> AsyncSandbox:
         sandbox = await AsyncSandbox.connect({"sandbox_id": sandbox_id}, provider=provider)
         return sandbox
 
+    def _resolve_agent_timeout(self, task_timeout: float | None) -> float:
+        """Per-task budget when the dataset supplies one, else the flat fallback, then capped."""
+        timeout = self.config.sandbox_timeout if task_timeout is None else float(task_timeout)
+        if self.config.max_agent_timeout is not None:
+            timeout = min(timeout, self.config.max_agent_timeout)
+        return timeout
+
     async def _execute(
         self,
         request: Request,
         body: NeMoGymResponseCreateParamsNonStreaming,
         sandbox: AsyncSandbox,
+        task_timeout: float | None = None,
     ) -> Tuple[NeMoGymResponse, Dict[str, Any]]:
         start_time = perf_counter()
+        agent_timeout = self._resolve_agent_timeout(task_timeout)
         instruction = _instruction(body.input)
 
         model_base_url = (
@@ -383,7 +395,7 @@ async def _execute(
             await agent.setup(environment)
 
             try:
-                async with asyncio.timeout(self.config.sandbox_timeout):
+                async with asyncio.timeout(agent_timeout):
                     await agent.run(instruction, environment, context)
                 terminus2_completed = True
                 error = None
@@ -430,6 +442,7 @@ async def _execute(
             "command_exec_time_pct": 100 * total_command_exec_time / total_time,
             "model_call_time_pct": 100 * total_model_call_time / total_time,
             "terminus2_time_taken": total_time,
+            "agent_timeout": agent_timeout,
             "model_calls_gt_10min": llm._model_calls_gt_10min,
             "num_proactive_compactions": agent._num_proactive_compactions,
             "num_compactions": llm._num_compactions,
@@ -462,7 +475,12 @@ async def run(self, request: Request, body: Terminus2AgentRunRequest) -> Terminu
         session_key = request.session[SESSION_ID_KEY]
         self._session_sandboxes[session_key] = sandbox
 
-        response, metrics = await self._execute(request, body.responses_create_params, sandbox)
+        response, metrics = await self._execute(
+            request,
+            body.responses_create_params,
+            sandbox,
+            task_timeout=getattr(body, "agent_timeout_sec", None),
+        )
 
         verification = await self.server_client.post(
             server_name=self.config.resources_server.name,

responses_api_agents/terminus_2_sandboxed_agent/configs/terminus_2_sandboxed_agent.yaml

@@ -23,7 +23,8 @@ terminus_2_sandboxed_agent:
       llm_request_timeout: 600  # 10 mins, Litellm default
 
       sandbox_provider: sandbox
-      sandbox_timeout: 10800  # 3 hrs
+      sandbox_timeout: 10800  # 3 hrs; only used when the dataset row has no agent_timeout_sec
+      max_agent_timeout: null  # cap on the per-task budget; null means uncapped
       sandbox_config:
         ttl_s: 18000
         ready_timeout_s: 1200

responses_api_agents/terminus_2_sandboxed_agent/tests/test_app.py

@@ -163,6 +163,54 @@ async def create_response(self, **kwargs):
     ]
 
 
+def _agent_config(**overrides):
+    defaults = dict(
+        host="0.0.0.0",
+        port=8080,
+        entrypoint="app.py",
+        name="terminus_2_1_agent",
+        resources_server=ResourcesServerRef(type="resources_servers", name="swebench_resources_server"),
+        model_server=ModelServerRef(type="responses_api_models", name="policy_model"),
+        max_turns=100,
+        enable_summarize=True,
+        proactive_summarization_threshold=8000,
+        tmux_pane_width=160,
+        tmux_pane_height=40,
+        dump_trajectory=False,
+        debug=False,
+        model_context_limit=32_000,
+        model_output_limit=4_000,
+        llm_request_timeout=60,
+        sandbox_provider="opensandbox",
+        sandbox_timeout=10800,
+        remote_tmux_binary_path=None,
+    )
+    return Terminus2AgentConfig(**(defaults | overrides))
+
+
+@pytest.mark.parametrize(
+    "task_timeout, max_agent_timeout, expected",
+    [
+        # No per-task budget in the dataset row: fall back to the flat wall.
+        (None, None, 10800),
+        # Per-task budget wins over the flat wall, in both directions.
+        (900, None, 900),
+        (12000, None, 12000),
+        # The cap only binds when the per-task budget exceeds it.
+        (12000, 7200, 7200),
+        (900, 7200, 900),
+        # The cap also bounds the fallback.
+        (None, 7200, 7200),
+    ],
+)
+def test_resolve_agent_timeout(task_timeout, max_agent_timeout, expected):
+    server = Terminus2Agent(
+        config=_agent_config(max_agent_timeout=max_agent_timeout),
+        server_client=MagicMock(spec=ServerClient),
+    )
+    assert server._resolve_agent_timeout(task_timeout) == expected
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("dump_trajectory", [False, True])
 @pytest.mark.parametrize("debug", [False, True])
@@ -267,6 +315,7 @@ async def request_json():
         "command_exec_time_pct": 40.0,
         "model_call_time_pct": 60.0,
         "terminus2_time_taken": 10.0,
+        "agent_timeout": 10.0,
         "model_calls_gt_10min": 0,
         "num_proactive_compactions": 0,
         "num_compactions": 2,