Skip to content

fix(ci): use Opus 4.8 for native eval judging #327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrizzi merged 1 commit into from
Oct 6, 2026
Merged

fix(ci): use Opus 4.8 for native eval judging #327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/run-native-fullsend-evals.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ EOF
# Native stdout/stderr/transcripts can contain credential paths or arbitrary
# PR-generated bytes. Keep raw logs private; only export allowlisted results.
status=0
python3.12 "$runner" run --cache "$cache" \
python3.12 "$runner" run --cache "$cache" --judge-model claude-opus-4-8 \
--plugin-root "$GITHUB_WORKSPACE/pr-head/plugins/sdlc-workflow" \
--output "$RUNNER_TEMP/tc6726-private" --report-dir "$RUNNER_TEMP/tc6726-safe" \
> "$RUNNER_TEMP/tc6726-private-run.log" 2>&1 || status=$?
Expand Down
15 changes: 14 additions & 1 deletion plugins/sdlc-workflow/scripts/test_native_fullsend_eval_ci.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ def run_credential_wrapper(tmp_path, output):
doubles = {
"git": '#!/bin/sh\n# SYNTHETIC TEST DATA — immutable checkout identities\ncase "$2" in *upstream-fullsend) echo d5f36921ac754705619f38c637ef692873809fbc;; *) echo bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb;; esac\n',
"jq": '#!/bin/sh\n# SYNTHETIC TEST DATA — host ADC type\necho external_account\n',
"python3.12": '#!/usr/bin/env python3\n# SYNTHETIC TEST DATA — capture parser output without inference\nimport json, os\nfrom pathlib import Path\nPath(os.environ["TC6742_CAPTURE"]).write_text(json.dumps({k: os.environ.get(k) for k in ["GOOGLE_APPLICATION_CREDENTIALS", "TC6726_SANDBOX_CREDENTIALS", "GCP_OIDC_TOKEN_FILE", "FULLSEND_GCP_OIDC_URL", "FULLSEND_GCP_OIDC_AUTH_FILE", "TC6742_UNEXPECTED"]}))\n',
"python3.12": '#!/usr/bin/env python3\n# SYNTHETIC TEST DATA — capture parser output without inference\nimport json, os, sys\nfrom pathlib import Path\nPath(os.environ["TC6742_CAPTURE"]).with_suffix(".argv.json").write_text(json.dumps(sys.argv[1:]))\nPath(os.environ["TC6742_CAPTURE"]).write_text(json.dumps({k: os.environ.get(k) for k in ["GOOGLE_APPLICATION_CREDENTIALS", "TC6726_SANDBOX_CREDENTIALS", "GCP_OIDC_TOKEN_FILE", "FULLSEND_GCP_OIDC_URL", "FULLSEND_GCP_OIDC_AUTH_FILE", "TC6742_UNEXPECTED"]}))\n',
}
for name, content in doubles.items():
path = tools / name
Expand Down Expand Up @@ -487,3 +487,16 @@ def test_multiline_credentials_register_individual_nonempty_masks(tmp_path):
assert "::add-mask::::warning::synthetic-second" in lines
assert "::add-mask::" not in lines
assert "::warning::synthetic-second" not in lines


def test_native_wrapper_passes_requested_judge_model(tmp_path):
"""The trusted wrapper overrides the reviewed runner's older judge default."""
output = ("GOOGLE_APPLICATION_CREDENTIALS=synthetic-sandbox-adc\n"
"GCP_OIDC_TOKEN_FILE=synthetic-token\n"
"FULLSEND_GCP_OIDC_URL=synthetic-url\n"
"FULLSEND_GCP_OIDC_AUTH_FILE=synthetic-auth\n")
result, _ = run_credential_wrapper(tmp_path, output)
assert result.returncode == 0
arguments = json.loads((tmp_path / "captured.argv.json").read_text())
assert arguments[1] == "run"
assert arguments[arguments.index("--judge-model") + 1] == "claude-opus-4-8"
Loading