fix(tests): route-specific command telemetry must not move the score

tests/tasks/uipath-maestro-flow/_shared/test_same_ground_corpus.py

@@ -42,6 +42,23 @@
 
 DEBUG_SOLUTION_ALLOWLIST = set()
 
+# Commands whose JOB the other route does differently or internally, so a
+# `command_executed` on one of them measures which route ran rather than what
+# the run produced. See `test_route_specific_command_telemetry_is_weightless`
+# for the evidence behind each family, and for what is deliberately NOT here.
+ROUTE_SPECIFIC_COMMANDS = (
+    # v1 mutates the graph a node at a time; the SDK loop writes `.flow.ts`.
+    r"flow node \(?(add|configure|remove|update)|flow edge ",
+    # v1 scaffolds the inline agent's sidecar with the CLI; the SDK's
+    # `conversationalAgent()` / `agent()` emit `agent.json` themselves.
+    r"agent (init|refresh) (?=.*(inline-in-flow|--conversational))",
+    # v1 walks the tenant by hand; `registry prepare` picks the connection,
+    # pages the collection and writes `bindings.json` in one call.
+    r"is connections list|is resources run list|is triggers \(?(objects|describe)",
+    # v1 refreshes the node manifest before searching it.
+    r"flow registry \(?pull",
+)
+
 # The two billing lookups name only a "Data Service entity", which since #3041
 # denotes two node families. Their prompts pin the connector so the graded
 # structure is deterministic; the sibling dispute-resolution task pins it in the
@@ -184,6 +201,76 @@ def test_v1_only_authoring_commands_match_the_temporary_allowlist() -> None:
     assert offenders == V1_AUTHORING_ALLOWLIST
 
 
+def _command_pattern(criterion: str) -> str:
+    """A `command_pattern` with its regex escaping flattened to plain words.
+
+    The corpus spells the same command several ways — `\\s+` in a single-quoted
+    scalar, `\\\\s+` in a double-quoted one — so matching families against the
+    raw text would miss half of them.
+    """
+    match = re.search(r"(?m)^\s+command_pattern:\s*(.*)$", criterion)
+    if match is None:
+        return ""
+    pattern = match.group(1).replace("\\\\", "\\")
+    pattern = re.sub(r"\\s\+?", " ", pattern)
+    return re.sub(r"\s+", " ", pattern.replace("\\", ""))
+
+
+def test_route_specific_command_telemetry_is_weightless() -> None:
+    """A criterion that grades WHICH ROUTE ran must not move the score.
+
+    THE GAP THIS EXISTS FOR. `pass_threshold: 0` was read as "this criterion is
+    advisory", and the sibling test above enforces only that. It is half the
+    idiom; coder_eval's own field docs carry the other half: "weight=0 excludes
+    from the score but NOT from the pass/fail gate ... To make a criterion truly
+    non-gating, also set pass_threshold=0." So `pass_threshold: 0` with
+    `weight: 1.5` never fails a task and always moves its score.
+
+    That only matters where the command itself is route-specific. In the
+    2026-09-23 same-ground run those criteria scored 1.0 for v1 and 0.0 for v2,
+    dragging tasks that passed every graded check down with them —
+    `datafabric_integration_create_get` returned SUCCESS at 0.55 on three
+    `flow node add` advisories weighing 5.0 against two graded criteria at 3.0.
+
+    WHAT IS NOT IN `ROUTE_SPECIFIC_COMMANDS`, and why. An earlier revision of
+    this test keyed on criterion TYPE — every non-gating `command_executed` —
+    and that was wrong. It swept in `solution init`, `flow init`,
+    `flow validate`, `flow debug` and `flow eval ...`, which both routes run on
+    the same artifact and which the run shows both routes passing (31 of the 54
+    observed criteria were BOTH PASS). Zeroing those removes real, satisfiable
+    signal and buys no neutrality. `flow registry get|search|list` is out for
+    the same measured reason: the SDK arm passes those in the IxP tasks, so the
+    registry is not a v1-only surface — only `pull` is listed, as the refresh
+    step the SDK loop has no need of.
+
+    A criterion is therefore in scope only when its COMMAND has no counterpart
+    in the other route. Where an arm then fails one of the survivors, that is a
+    finding about the arm, which is the point.
+
+    `stop_early` criteria are exempt: there `weight` is load-bearing for the
+    pass-stop floor, not just for the score (see `ixp/routing.yaml`, whose
+    sentinel says so).
+    """
+    offenders = set()
+    for relative, _, text in _tagged_tasks():
+        for criterion_type, criterion in _criterion_blocks(text):
+            if criterion_type != "command_executed":
+                continue
+            threshold = re.search(r"(?m)^\s+pass_threshold:\s*([0-9.]+)", criterion)
+            if threshold is None or float(threshold.group(1)) > 0:
+                continue
+            if re.search(r"(?m)^\s+stop_early:", criterion):
+                continue
+            pattern = _command_pattern(criterion)
+            if not any(re.search(family, pattern) for family in ROUTE_SPECIFIC_COMMANDS):
+                continue
+            weight = re.search(r"(?m)^\s+weight:\s*([0-9.]+)", criterion)
+            # An absent `weight` defaults to 1.0, so silence is not compliance.
+            if weight is None or float(weight.group(1)) != 0:
+                offenders.add(relative)
+    assert offenders == set()
+
+
 def test_gating_skill_telemetry_matches_the_temporary_allowlist() -> None:
     offenders = set()
     for relative, _, text in _tagged_tasks():

tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/integration_create_get.yaml

@@ -76,23 +76,23 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record'
     min_count: 1
-    weight: 1.5
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed
     description: "Advisory: live-v1 agent added Get Entity Record By Id nodes"
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.get-entity-record-by-id'
     min_count: 1
-    weight: 2.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed
     description: "Advisory: live-v1 agent added Delete Entity Record nodes"
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.delete-entity-record'
     min_count: 1
-    weight: 1.5
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed

tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_error.yaml

@@ -61,14 +61,14 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.create-entity-record'
     min_count: 1
-    weight: 1.5
+    weight: 0
     pass_threshold: 0.0
   - type: command_executed
     description: "Advisory: live-v1 agent added the follow-up Query nodes"
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records'
     min_count: 1
-    weight: 2.0
+    weight: 0
     pass_threshold: 0.0
   - type: command_executed
     description: "Flow validated"

tests/tasks/uipath-maestro-flow/connector_features/datafabric_connector/smoke_query.yaml

@@ -78,7 +78,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+node\s+add\s+\S+\s+"?uipath\.connector\.uipath-uipath-dataservice\.query-entity-records'
     min_count: 1
-    weight: 2.0
+    weight: 0
     pass_threshold: 0.0
   - type: command_executed
     description: "Flow validated"

tests/tasks/uipath-maestro-flow/connector_features/enum.yaml

@@ -44,7 +44,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list"
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/paginated_reference_lookup.yaml

@@ -65,7 +65,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: 'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)'
     min_count: 2
-    weight: 3.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed
@@ -74,7 +74,7 @@ success_criteria:
     command_pattern: '(uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?(curated_channels|conversations)[^\n]*nextPage=|registry\s+prepare\s+\\?"?uipath-salesforce-slack\\?"?[^\n]*--resolve\s+\\?"?channel:)'
     min_count: 1
     require_success: true
-    weight: 3.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed

tests/tasks/uipath-maestro-flow/connector_features/path_params.yaml

@@ -46,7 +46,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: "(uip|\\$UIP)\\s+is\\s+connections\\s+list"
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_attachments/testmanager_attachments.yaml

@@ -52,7 +52,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_crud_grounded/testmanager_crud_grounded.yaml

@@ -52,7 +52,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_execution_results/testmanager_execution_results.yaml

@@ -57,7 +57,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_generic_records/testmanager_generic_records.yaml

@@ -53,7 +53,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_requirement_lifecycle/testmanager_requirement_lifecycle.yaml

@@ -54,7 +54,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_testcase_lifecycle/testmanager_testcase_lifecycle.yaml

@@ -53,7 +53,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/connector_features/testmanager_testset_lifecycle/testmanager_testset_lifecycle.yaml

@@ -55,7 +55,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+maestro\s+flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: run_command

tests/tasks/uipath-maestro-flow/conversational/conversational_chat_loop.yaml

@@ -121,15 +121,15 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--conversational'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0
 
   - type: command_executed
     description: "Advisory: agent rebuilt contentTokens with uip agent refresh --inline-in-flow"
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+agent\s+refresh\s+.*--inline-in-flow'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0
 
   - type: command_executed

tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml

@@ -37,7 +37,7 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+pull'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0.0
 
   - type: command_executed

tests/tasks/uipath-maestro-flow/voice/voice_inbound_call.yaml

@@ -151,5 +151,5 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--conversational'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0

tests/tasks/uipath-maestro-flow/voice/voice_outbound_call.yaml

@@ -129,15 +129,15 @@ success_criteria:
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+agent\s+init\s+.*--conversational'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0
 
   - type: command_executed
     description: "Advisory: agent rebuilt contentTokens with uip agent refresh --inline-in-flow"
     tool_name: "Bash"
     command_pattern: '(uip|\$UIP)\s+agent\s+refresh\s+.*--inline-in-flow'
     min_count: 1
-    weight: 1.0
+    weight: 0
     pass_threshold: 0
 
   - type: command_executed