From 77ff75d3562e738e5075d25e8ab8fce2a5932794 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:12:20 -0800 Subject: [PATCH 1/6] fix(ci): authorize candidate testing through PR reviews --- .../workflows/candidate-review-signal.yaml | 18 ++ .../candidate-upgrade-admission.yaml | 30 +- .github/workflows/installer-hash-check.yaml | 1 + .github/workflows/pr.yaml | 10 +- scripts/checks/candidate-openshell-sdk.mts | 6 +- .../checks/package-openshell-sdk-for-pr.mts | 2 +- ...ndidate-upgrade-admission-workflow.test.ts | 98 ++++-- test/e2e/README.md | 39 ++- .../candidate-upgrade-admission.test.ts | 278 ++++++++++++++---- .../protected-openshell-workspace.test.ts | 10 +- test/helpers/candidate-openshell-trust.ts | 18 +- test/helpers/openshell-installer-template.ts | 1 + .../candidate-openshell-sdk-artifact.test.ts | 4 +- .../package-openshell-sdk-for-pr.test.ts | 3 +- test/support/candidate-openshell-sdk.ts | 18 +- test/support/protected-openshell-workspace.ts | 11 +- tools/e2e/candidate-openshell-trust.mts | 2 +- tools/e2e/candidate-review-scope.mts | 109 +++++++ tools/e2e/candidate-upgrade-admission.mts | 99 ++++--- 19 files changed, 583 insertions(+), 174 deletions(-) create mode 100644 .github/workflows/candidate-review-signal.yaml create mode 100644 tools/e2e/candidate-review-scope.mts diff --git a/.github/workflows/candidate-review-signal.yaml b/.github/workflows/candidate-review-signal.yaml new file mode 100644 index 00000000000..010cbf1fd07 --- /dev/null +++ b/.github/workflows/candidate-review-signal.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# This PR-context workflow only wakes the default-branch admission controller. +# It has no credentials, checkout, artifacts, or authority to admit candidate inputs. +name: Security / Candidate Review Signal +on: + pull_request_review: + types: [submitted, edited, dismissed] +permissions: {} +jobs: + signal: + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.repo.full_name == 'NVIDIA/NemoClaw' }} + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: Signal a review change + run: echo 'Candidate admission will read the current PR reviews.' diff --git a/.github/workflows/candidate-upgrade-admission.yaml b/.github/workflows/candidate-upgrade-admission.yaml index 6e9e24f3319..f442a7936bc 100644 --- a/.github/workflows/candidate-upgrade-admission.yaml +++ b/.github/workflows/candidate-upgrade-admission.yaml @@ -7,16 +7,17 @@ name: Security / Candidate Upgrade Admission on: pull_request_target: types: [opened, reopened, synchronize, edited, ready_for_review] - issue_comment: - types: [created, edited, deleted] + workflow_run: + workflows: [Security / Candidate Review Signal] + types: [completed] permissions: contents: read pull-requests: read jobs: candidate-upgrade-admission: - if: ${{ github.repository == 'NVIDIA/NemoClaw' && ((github.event_name == 'pull_request_target' && github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.repo.full_name == 'NVIDIA/NemoClaw') || (github.event_name == 'issue_comment' && github.event.issue.pull_request && (contains(github.event.comment.body, 'NemoClaw-Upgrade-E2E:') || contains(github.event.changes.body.from, 'NemoClaw-Upgrade-E2E:')))) }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && ((github.event_name == 'pull_request_target' && github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.repo.full_name == 'NVIDIA/NemoClaw') || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request_review')) }} concurrency: - group: candidate-upgrade-admission-${{ github.event.pull_request.number || github.event.issue.number }} + group: candidate-upgrade-admission-${{ github.event.pull_request.head.ref || github.event.workflow_run.head_branch }} cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 5 @@ -35,13 +36,26 @@ jobs: with: script: | if (context.repo.owner !== "NVIDIA" || context.repo.repo !== "NemoClaw") throw new Error("Unexpected upgrade repository"); - if (!["pull_request_target", "issue_comment"].includes(context.eventName)) throw new Error("Unexpected upgrade event"); - const marker = value => typeof value === "string" && value.includes("NemoClaw-Upgrade-E2E:"); - if (context.eventName === "issue_comment" && (!context.payload.issue?.pull_request || (!marker(context.payload.comment?.body) && !marker(context.payload.changes?.body?.from)))) return; - const number = context.payload.pull_request?.number ?? context.payload.issue?.number; + if (!["pull_request_target", "workflow_run"].includes(context.eventName)) throw new Error("Unexpected upgrade event"); + const signal = context.payload.workflow_run; + let number = context.payload.pull_request?.number; + if (context.eventName === "workflow_run") { + if (signal?.event !== "pull_request_review" || signal?.name !== "Security / Candidate Review Signal" || + signal?.head_repository?.full_name !== "NVIDIA/NemoClaw" || + typeof signal?.head_branch !== "string" || !signal.head_branch) + throw new Error("Invalid candidate review signal"); + // Review-triggered runs can omit pull_requests. Resolve the current upstream PR; + // the signal only requests reevaluation and supplies no approval authority. + const { data: prs } = await github.rest.pulls.list({ ...context.repo, + head: context.repo.owner + ":" + signal.head_branch, base: "main", state: "open", per_page: 2 }); + if (!Array.isArray(prs) || prs.length > 1) throw new Error("Ambiguous candidate review signal"); + if (prs.length === 0) return; + number = prs[0].number; + } if (!Number.isSafeInteger(number) || number <= 0) throw new Error("Invalid upgrade PR number"); const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: number }); if (pr.state !== "open") return; + if (context.eventName === "workflow_run" && pr.head.ref !== signal.head_branch) throw new Error("Candidate review branch changed"); if (pr.head.repo?.full_name !== "NVIDIA/NemoClaw" || pr.base.repo?.full_name !== "NVIDIA/NemoClaw" || pr.base.ref !== "main") throw new Error("Unsupported upgrade PR identity"); if (!/^[a-f0-9]{40}$/.test(pr.head.sha) || !/^[a-f0-9]{40}$/.test(pr.base.sha)) throw new Error("Invalid upgrade commits"); await github.rest.repos.createCommitStatus({ diff --git a/.github/workflows/installer-hash-check.yaml b/.github/workflows/installer-hash-check.yaml index 2479e2f3608..54ac4ea34b7 100644 --- a/.github/workflows/installer-hash-check.yaml +++ b/.github/workflows/installer-hash-check.yaml @@ -74,6 +74,7 @@ jobs: ci/openshell-upgrade-trust.json tools/e2e/candidate-openshell-trust.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts sparse-checkout-cone-mode: false diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b4d34c5e8f7..9ffbc964420 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -146,6 +146,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -250,6 +251,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/npm-diagnostics.sh scripts/lib/openclaw-npm-remediation.mts @@ -304,7 +306,7 @@ jobs: and .status == "completed" and .conclusion == "success" and ((.event == "pull_request_target" and .head_sha == $sha and any(.pull_requests[]?; .number == $pr and .head.sha == $sha and .base.sha == $base)) - or (.event == "issue_comment" and .head_branch == "main")) + or (.event == "workflow_run" and .head_branch == "main")) else (.path | split("@")[0]) == ".github/workflows/main.yaml" and .head_branch == "main" and (.event == "push" or .event == "workflow_dispatch") @@ -398,6 +400,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -478,6 +481,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -555,6 +559,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -693,6 +698,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -835,6 +841,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts @@ -897,6 +904,7 @@ jobs: scripts/lib/reviewed-ci-npm-install.mts scripts/checks/candidate-openshell-sdk.mts tools/e2e/candidate-upgrade-admission.mts + tools/e2e/candidate-review-scope.mts tools/e2e/base-image-publication.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts diff --git a/scripts/checks/candidate-openshell-sdk.mts b/scripts/checks/candidate-openshell-sdk.mts index 59d1805460e..cd2c64618fe 100644 --- a/scripts/checks/candidate-openshell-sdk.mts +++ b/scripts/checks/candidate-openshell-sdk.mts @@ -174,7 +174,7 @@ export async function readApprovedOpenShellSdkConfig( request, identity.mergeSha, ); - if (merged.commentId !== admission.commentId || merged.reviewer !== admission.reviewer) + if (merged.reviewId !== admission.reviewId || merged.reviewer !== admission.reviewer) throw new Error("Candidate SDK admission changed during merge verification"); sources = { ...sources, ...merged.sources }; // The merge can update other dependencies, but cannot select a different SDK or audit policy. @@ -185,7 +185,7 @@ export async function readApprovedOpenShellSdkConfig( configSource: JSON.stringify(policy), sources, reviewer: admission.reviewer, - commentId: admission.commentId, + reviewId: admission.reviewId, }; } @@ -218,7 +218,7 @@ function artifactDecision( required: true, candidate: true, artifactName: parseAuditConfig(admitted.configSource).sourceRegistryPackage.artifactName, - bundleName: `candidate-openshell-sdk-${identity.prNumber}-${identity.candidateSha}-${identity.baseSha}-${admitted.commentId}`, + bundleName: `candidate-openshell-sdk-${identity.prNumber}-${identity.candidateSha}-${identity.baseSha}-${admitted.reviewId}`, } as const; } diff --git a/scripts/checks/package-openshell-sdk-for-pr.mts b/scripts/checks/package-openshell-sdk-for-pr.mts index 55ffaec009a..b5af028df93 100755 --- a/scripts/checks/package-openshell-sdk-for-pr.mts +++ b/scripts/checks/package-openshell-sdk-for-pr.mts @@ -102,7 +102,7 @@ export async function packageApprovedCandidateOpenShellSdk( if ( before.configSource !== after.configSource || JSON.stringify(before.sources) !== JSON.stringify(after.sources) || - before.commentId !== after.commentId || + before.reviewId !== after.reviewId || before.reviewer !== after.reviewer ) { throw new Error("Candidate SDK admission changed during packaging"); diff --git a/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts b/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts index 40dec070680..e38dca84faa 100644 --- a/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts +++ b/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts @@ -56,7 +56,7 @@ function packagingFixture() { function fixture() { const pr = { state: "open", - head: { sha: head, repo: { full_name: "NVIDIA/NemoClaw" } }, + head: { sha: head, ref: "codex/candidate", repo: { full_name: "NVIDIA/NemoClaw" } }, base: { sha: base, ref: "main", repo: { full_name: "NVIDIA/NemoClaw" } }, }; const context = { @@ -66,21 +66,27 @@ function fixture() { runId: 456, payload: { pull_request: { number: 123 }, - issue: { number: 123, pull_request: {} }, - comment: { body: `NemoClaw-Upgrade-E2E: approve ${head} ${base}` }, - changes: { body: { from: "" } }, + workflow_run: { + event: "pull_request_review", + name: "Security / Candidate Review Signal", + head_repository: { full_name: "NVIDIA/NemoClaw" }, + head_branch: "codex/candidate", + pull_requests: [] as { number: number }[], + }, }, }; const outputs: Record = {}; const status = vi.fn(async (_value: unknown) => ({})); const get = vi.fn(async () => ({ data: pr })); - const github = { rest: { pulls: { get }, repos: { createCommitStatus: status } } }; + const list = vi.fn(async () => ({ data: [{ number: 123 }] })); + const github = { rest: { pulls: { get, list }, repos: { createCommitStatus: status } } }; return { pr, context, status, outputs, get, + list, start: () => new AsyncFunction("github", "context", "core", start)(github, context, { setOutput: (name: string, value: string) => { @@ -140,34 +146,76 @@ describe("candidate upgrade admission status", () => { expect(f.status).toHaveBeenLastCalledWith(expect.objectContaining({ state: "failure" })); }); - it.each(["approve", "revoke", "quoted approve"])( - "invalidates the merge status for %s without granting permission", - async (action) => { + it("refreshes current admission after the read-only review signal", async () => { + const f = fixture(); + f.context.eventName = "workflow_run"; + await f.start(); + expect(f.status).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ sha: head, context: "check-hash", state: "pending" }), + ); + expect(f.outputs).toEqual({ pr_number: "123", head_sha: head, base_sha: base }); + expect(f.list).toHaveBeenCalledExactlyOnceWith({ + owner: "NVIDIA", + repo: "NemoClaw", + head: "NVIDIA:codex/candidate", + base: "main", + state: "open", + per_page: 2, + }); + }); + + it.each(["wrong-event", "wrong-workflow", "fork", "missing-branch"])( + "rejects a %s review signal", + async (kind) => { const f = fixture(); - f.context.eventName = "issue_comment"; - f.context.payload.comment.body = `NemoClaw-Upgrade-E2E: ${action} ${head} ${base}`; - await f.start(); - expect(f.status).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - sha: head, - context: "check-hash", - state: "pending", - }), - ); - expect(f.outputs).toEqual({ pr_number: "123", head_sha: head, base_sha: base }); + f.context.eventName = "workflow_run"; + const signal = f.context.payload.workflow_run; + const mutate: Record unknown> = { + "wrong-event": () => { + signal.event = "push"; + }, + "wrong-workflow": () => { + signal.name = "Other workflow"; + }, + fork: () => { + signal.head_repository.full_name = "other/NemoClaw"; + }, + "missing-branch": () => { + signal.head_branch = ""; + }, + }; + mutate[kind]!(); + await expect(f.start()).rejects.toThrow(/Invalid candidate review signal/); + expect(f.status).not.toHaveBeenCalled(); }, ); - it("refreshes when editing removes the approval marker", async () => { + it("ignores a review signal after its branch has no open PR", async () => { const f = fixture(); - f.context.eventName = "issue_comment"; - f.context.payload.changes.body.from = f.context.payload.comment.body; - f.context.payload.comment.body = "removed"; + f.context.eventName = "workflow_run"; + f.list.mockResolvedValueOnce({ data: [] }); await f.start(); - expect(f.status).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ state: "pending" })); + expect(f.status).not.toHaveBeenCalled(); + expect(f.outputs).toEqual({}); + }); + + it("rejects ambiguous open PRs for a review branch", async () => { + const f = fixture(); + f.context.eventName = "workflow_run"; + f.list.mockResolvedValueOnce({ data: [{ number: 123 }, { number: 124 }] }); + await expect(f.start()).rejects.toThrow(/Ambiguous/); + expect(f.status).not.toHaveBeenCalled(); + }); + + it("rejects a branch mismatch after resolving the review signal", async () => { + const f = fixture(); + f.context.eventName = "workflow_run"; + f.pr.head.ref = "other-branch"; + await expect(f.start()).rejects.toThrow(/branch changed/); + expect(f.status).not.toHaveBeenCalled(); }); - it.each(["pull_request", "push", "workflow_dispatch"])( + it.each(["pull_request", "push", "workflow_dispatch", "issue_comment", "pull_request_review"])( "rejects untrusted or unrelated event %s", async (event) => { const f = fixture(); diff --git a/test/e2e/README.md b/test/e2e/README.md index e9a59deede1..774d81e6612 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -427,13 +427,25 @@ retain prior records and revocations. Revoked hashes cannot authorize execution, even when retained in a historical record. The PR author may be a GitHub App bot on this repository. This manifest cannot -approve itself. A different human with current repository -`maintain` or `admin` permission must review the exact source and post one whole -comment: `NemoClaw-Upgrade-E2E: approve `. The corresponding -`revoke` command withdraws that decision. The latest eligible decision wins; -edited comments, PR-author comments, stale commits, forks, and closed PRs do not -authorize admission. The trusted reader rechecks the PR, comments, and current -permissions before and after fetching the fixed source paths. +approve itself. A different human with current repository `maintain` or `admin` +permission must submit an **Approve** PR review. That review also authorizes +candidate testing. No admission comment or copied commit identifiers are required. +An outstanding maintainer request for changes blocks admission. Dismissed reviews, +self-approvals, bot reviews, forks, and closed PRs do not authorize it. + +Approval can cover a later descendant commit when the consumer's selected source +files and governing verification policy are unchanged. Unrelated tests or documentation +can change without renewed test authorization. Installer admission also permits an +unrelated lockfile synchronization. SDK admission includes its manifests, locks, and +audit identity; changes to those files require another review. +The verifier compares the reviewed commit with the candidate and checks trusted-base +policy changes since their historical common ancestor. Policy changes require +synchronizing the branch and obtaining another review. An incomplete comparison fails +closed. The comparison includes renamed paths and file-mode changes. +The trusted reader rechecks PR identity, review state, and reviewer permissions before +and after fetching source data. GitHub review dismissal remains effective, including +any dismissal caused by branch protection. Test authorization does not preserve a +dismissed merge approval or replace evidence for the latest commit. The trusted installer checker compares local sources with the approved commit. It then verifies upstream checksum and formula records through the existing @@ -449,10 +461,13 @@ After merge, the verifier reads the registry beside its trusted checkout, never from the candidate working directory. Normal branch protection and independent review govern that transition. An open PR's temporary admission is not merge authority. `Security / Candidate Upgrade Admission` refreshes the `check-hash` commit status -on upstream PR changes and approval-comment creation, editing, or deletion. +on upstream PR changes and review submission, editing, or dismissal. +Review events run a credential-free signal workflow without a checkout. Its completion +wakes the default-branch controller, which reads current reviews through GitHub's API. +The signal provides no approval authority or candidate artifacts. When `check-hash` is required, both that status and the native hash check must pass. -An approval comment refreshes only the commit status. If the native hash check -failed before approval, rerun that job after approval. +A review refreshes the commit status and candidate SDK packaging. If the native hash +check failed before approval, rerun that job after approval. An interrupted refresh remains pending or fails; it cannot reuse an earlier success. For a new OpenShell TypeScript SDK version, the same approval also binds the root @@ -471,8 +486,8 @@ commit, with the approved base and candidate as its two parents. The verifier re that commit's manifests and locks and compares them with the local files. The merged dependency graph must retain the approved SDK identity and pass the trusted package policy. Direct candidate E2E continues to require the approved candidate's file bytes. -If the merge uses a newer base, synchronize the PR and refresh its commit-bound -approval before rerunning candidate SDK CI. +If the merge uses a newer base, synchronize the PR before rerunning candidate SDK CI. +Obtain another review only when the admitted inputs or governing policy changed. The API credential is limited to trusted preparation and removed before npm runs. If CI starts before the approved artifact exists, wait for the admission workflow's SDK job, then rerun CI. Changed or revoked approval cannot reuse the older artifact. diff --git a/test/e2e/support/candidate-upgrade-admission.test.ts b/test/e2e/support/candidate-upgrade-admission.test.ts index 083ac3b26bc..c98132e40e8 100644 --- a/test/e2e/support/candidate-upgrade-admission.test.ts +++ b/test/e2e/support/candidate-upgrade-admission.test.ts @@ -18,9 +18,9 @@ function fixture() { const approval = { id: 50, user: { login: "reviewer", type: "User" }, - created_at: "2026-10-14T00:00:00Z", - updated_at: "2026-10-14T00:00:00Z", - body: `NemoClaw-Upgrade-E2E: approve ${identity.candidateSha} ${identity.baseSha}`, + submitted_at: "2026-10-14T00:00:00Z", + state: "APPROVED", + commit_id: identity.candidateSha, }; const records = [approval]; const permission = { role_name: "maintain" }; @@ -34,10 +34,19 @@ function fixture() { }; const routes = new Map([ ["/repos/NVIDIA/NemoClaw/pulls/123", pr], - ["/repos/NVIDIA/NemoClaw/issues/123/comments?per_page=100&page=1", records], + ["/repos/NVIDIA/NemoClaw/pulls/123/reviews?per_page=100&page=1", records], ["/repos/NVIDIA/NemoClaw/collaborators/reviewer/permission", permission], [`/repos/NVIDIA/NemoClaw/contents/${sourcePath}?ref=${identity.candidateSha}`, blob], ]); + routes.set( + `/repos/NVIDIA/NemoClaw/compare/${identity.baseSha}...${identity.candidateSha}?per_page=1`, + { + status: "ahead", + base_commit: { sha: identity.baseSha }, + merge_base_commit: { sha: identity.baseSha }, + files: [], + }, + ); const request = vi.fn(async (path: string): Promise => routes.get(path)); return { pr, approval, records, permission, source, blob, request, routes }; } @@ -60,13 +69,179 @@ function mergeFixture() { return { ...f, pr, commit, mergeSha, mergeSource }; } +function retainedReviewFixture() { + const f = fixture(); + const reviewed = "d".repeat(40); + const oldBase = "e".repeat(40); + f.approval.commit_id = reviewed; + const changes = { + status: "ahead", + base_commit: { sha: reviewed }, + merge_base_commit: { sha: reviewed }, + files: [{ filename: "test/unrelated.test.ts" }], + }; + const policy = { + status: "ahead", + base_commit: { sha: oldBase }, + merge_base_commit: { sha: oldBase }, + files: [{ filename: "package-lock.json" }], + }; + f.routes.set( + `/repos/NVIDIA/NemoClaw/compare/${reviewed}...${identity.candidateSha}?per_page=1`, + changes, + ); + f.routes.set(`/repos/NVIDIA/NemoClaw/compare/${identity.baseSha}...${reviewed}?per_page=1`, { + status: "diverged", + base_commit: { sha: identity.baseSha }, + merge_base_commit: { sha: oldBase }, + }); + f.routes.set( + `/repos/NVIDIA/NemoClaw/compare/${oldBase}...${identity.baseSha}?per_page=1`, + policy, + ); + return { ...f, reviewed, changes, policy }; +} + describe("candidate upgrade admission", () => { + it("retains approval after an unrelated test edit and a main lockfile synchronization", async () => { + const f = retainedReviewFixture(); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).resolves.toEqual({ + sources: { [sourcePath]: f.source }, + reviewer: "reviewer", + reviewId: 50, + }); + }); + + it.each([ + sourcePath, + "tools/e2e/candidate-upgrade-admission.mts", + "scripts/checks/extract-installer-pins.mts", + ])("requires a new review after changing %s", async (filename) => { + const f = retainedReviewFixture(); + f.changes.files = [{ filename }]; + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /PR approval covering/, + ); + }); + + it("requires renewed review when main changes the trusted verification policy", async () => { + const f = retainedReviewFixture(); + f.policy.files = [{ filename: "tools/e2e/candidate-review-scope.mts" }]; + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /PR approval covering/, + ); + }); + + it("includes the previous name when a sensitive input is renamed", async () => { + const f = retainedReviewFixture(); + Object.assign(f.changes.files[0]!, { previous_filename: sourcePath }); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /PR approval covering/, + ); + }); + + it.each([ + "unrelated-history", + "truncated", + "missing-files", + "invalid-path", + "wrong-base", + "api-failure", + ])("denies reuse with %s comparison evidence", async (kind) => { + const f = retainedReviewFixture(); + const mutate: Record unknown> = { + "unrelated-history": () => { + f.changes.merge_base_commit.sha = "f".repeat(40); + }, + truncated: () => { + f.changes.files = Array.from({ length: 300 }, () => ({ filename: "docs/example.mdx" })); + }, + "missing-files": () => Object.assign(f.changes, { files: undefined }), + "invalid-path": () => Object.assign(f.changes, { files: [null] }), + "wrong-base": () => { + f.changes.base_commit.sha = "f".repeat(40); + }, + "api-failure": () => f.request.mockRejectedValueOnce(new Error("API unavailable")), + }; + mutate[kind]!(); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow(); + }); + + it.each(["package.json", "scripts/lib/reviewed-npm-archive.mts"])( + "requires a new SDK review when %s changes", + async (filename) => { + const f = retainedReviewFixture(); + f.changes.files = [{ filename }]; + await expect( + readApprovedCandidateUpgrade(identity, ["package.json"], f.request), + ).rejects.toThrow(/PR approval covering/); + }, + ); + + it("does not discard approval when the reviewer adds an ordinary comment", async () => { + const f = fixture(); + f.records.push({ ...f.approval, id: 51, state: "COMMENTED" }); + await expect( + readApprovedCandidateUpgrade(identity, [sourcePath], f.request), + ).resolves.toMatchObject({ reviewId: 50 }); + }); + + it("does not revive an older approval after dismissing the replacement review", async () => { + const f = fixture(); + f.records.push({ ...f.approval, id: 51, state: "DISMISSED" }); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /PR approval covering/, + ); + }); + + it("honors another maintainer's outstanding request for changes", async () => { + const f = fixture(); + f.records.push({ + ...f.approval, + id: 51, + user: { type: "User", login: "other" }, + state: "CHANGES_REQUESTED", + }); + f.routes.set("/repos/NVIDIA/NemoClaw/collaborators/other/permission", { + role_name: "maintain", + }); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /changes-requested/, + ); + }); + + it.each(["duplicate", "invalid-id", "invalid-time", "mutable-commit", "unknown-state"])( + "rejects %s review evidence", + async (kind) => { + const f = fixture(); + const mutate: Record unknown> = { + duplicate: () => f.records.push({ ...f.approval }), + "invalid-id": () => { + f.approval.id = 0; + }, + "invalid-time": () => { + f.approval.submitted_at = "invalid"; + }, + "mutable-commit": () => { + f.approval.commit_id = "main"; + }, + "unknown-state": () => { + f.approval.state = "unknown"; + }, + }; + mutate[kind]!(); + await expect( + readApprovedCandidateUpgrade(identity, [sourcePath], f.request), + ).rejects.toThrow(); + }, + ); + it("reads reviewed commit data and rechecks both PR identity and current maintainer authority", async () => { const f = fixture(); await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).resolves.toEqual({ sources: { [sourcePath]: f.source }, reviewer: "reviewer", - commentId: 50, + reviewId: 50, }); expect(f.request.mock.calls.filter(([path]) => path.endsWith("/permission"))).toHaveLength(2); expect(f.request.mock.calls.filter(([path]) => path.endsWith("/pulls/123"))).toHaveLength(2); @@ -78,7 +253,7 @@ describe("candidate upgrade admission", () => { await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).resolves.toEqual({ sources: { [sourcePath]: f.source }, reviewer: "reviewer", - commentId: 50, + reviewId: 50, }); f.records.length = 0; await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( @@ -86,13 +261,13 @@ describe("candidate upgrade admission", () => { ); }); - it("rejects a malformed bot author login before reading approval comments", async () => { + it("rejects a malformed bot author login before reading PR reviews", async () => { const f = fixture(); f.pr.user.login = "nvidia-nemopatch-writer[bot]suffix"; await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( /open upstream PR/, ); - expect(f.request.mock.calls.some(([path]) => path.includes("/comments"))).toBe(false); + expect(f.request.mock.calls.some(([path]) => path.includes("/reviews"))).toBe(false); }); it.each([{ prNumber: 0 }, { candidateSha: "main" }, { baseSha: "main" }])( @@ -118,52 +293,39 @@ describe("candidate upgrade admission", () => { }, ); - it.each([ - "author", - "bot", - "edited", - "quoted", - "newline", - "old-commit", - "old-base", - "revoke", - "deleted", - ])("rejects %s approval records", async (kind) => { - const f = fixture(); - const mutations: Record void> = { - author: () => { - f.approval.user.login = "AUTHOR"; - }, - bot: () => { - f.approval.user.type = "Bot"; - }, - edited: () => { - f.approval.updated_at = "2026-10-14T00:01:00Z"; - }, - quoted: () => { - f.approval.body = `> ${f.approval.body}`; - }, - newline: () => { - f.approval.body += "\n"; - }, - "old-commit": () => { - f.approval.body = f.approval.body.replace(identity.candidateSha, "c".repeat(40)); - }, - "old-base": () => { - f.approval.body = f.approval.body.replace(identity.baseSha, "c".repeat(40)); - }, - revoke: () => { - f.approval.body = f.approval.body.replace("approve", "revoke"); - }, - deleted: () => { - f.records.length = 0; - }, - }; - mutations[kind]!(); - await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( - /independent maintainer/, - ); - }); + it.each(["author", "bot", "dismissed", "commented", "pending", "changes-requested", "deleted"])( + "rejects %s review records", + async (kind) => { + const f = fixture(); + const mutate: Record void> = { + author: () => { + f.approval.user.login = "AUTHOR"; + }, + bot: () => { + f.approval.user.type = "Bot"; + }, + deleted: () => { + f.records.length = 0; + }, + dismissed: () => { + f.approval.state = "DISMISSED"; + }, + commented: () => { + f.approval.state = "COMMENTED"; + }, + pending: () => { + f.approval.state = "PENDING"; + }, + "changes-requested": () => { + f.approval.state = "CHANGES_REQUESTED"; + }, + }; + mutate[kind]!(); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + /independent maintainer/, + ); + }, + ); it.each(["closed", "fork", "base", "head"])( "rejects a PR with changed %s identity", @@ -193,11 +355,11 @@ describe("candidate upgrade admission", () => { it("reads every page and honors a later revocation", async () => { const f = fixture(); f.routes.set( - "/repos/NVIDIA/NemoClaw/issues/123/comments?per_page=100&page=1", + "/repos/NVIDIA/NemoClaw/pulls/123/reviews?per_page=100&page=1", Array.from({ length: 100 }, (_, i) => ({ ...f.approval, id: i + 1 })), ); - f.routes.set("/repos/NVIDIA/NemoClaw/issues/123/comments?per_page=100&page=2", [ - { ...f.approval, id: 101, body: f.approval.body.replace("approve", "revoke") }, + f.routes.set("/repos/NVIDIA/NemoClaw/pulls/123/reviews?per_page=100&page=2", [ + { ...f.approval, id: 101, state: "CHANGES_REQUESTED" }, ]); await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( /independent maintainer/, @@ -216,7 +378,7 @@ describe("candidate upgrade admission", () => { f.records.push({ ...f.approval, id: 51, - body: f.approval.body.replace("approve", "revoke"), + state: "CHANGES_REQUESTED", }); }, role: () => { diff --git a/test/e2e/support/protected-openshell-workspace.test.ts b/test/e2e/support/protected-openshell-workspace.test.ts index fb28df81816..8a787ab23a2 100644 --- a/test/e2e/support/protected-openshell-workspace.test.ts +++ b/test/e2e/support/protected-openshell-workspace.test.ts @@ -68,7 +68,7 @@ describe("protected OpenShell workspace", () => { [ "stale", ({ approval }) => { - approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${identity.candidateSha} ${identity.baseSha}`; + approval.approvals[0]!.commit_id = identity.candidateSha; }, /approval|decision/i, ], @@ -104,7 +104,7 @@ describe("protected OpenShell workspace", () => { git(input.candidateRoot, "add", "."); git(input.candidateRoot, "commit", "--quiet", "-m", "unapproved feature"); input.candidateSha = git(input.candidateRoot, "rev-parse", "HEAD"); - approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${input.candidateSha} ${identity.baseSha}`; + approval.approvals[0]!.commit_id = input.candidateSha; await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow( /feature gate operational template/, ); @@ -115,12 +115,12 @@ describe("protected OpenShell workspace", () => { const { input, approval } = approvedFixture(); await manageProtectedOpenShellWorkspace("prepare", input); const before = git(input.trustedRoot, "diff", "--no-ext-diff"); - approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: revoke ${input.candidateSha} ${identity.baseSha}`; + approval.approvals[0]!.state = "DISMISSED"; await expect(manageProtectedOpenShellWorkspace("verify", input)).rejects.toThrow( - "Candidate upgrade needs an independent maintainer's current commit-bound E2E approval", + "Candidate upgrade needs an independent maintainer's PR approval covering its inputs and policy", ); await expect(manageProtectedOpenShellWorkspace("restore", input)).rejects.toThrow( - "Candidate upgrade needs an independent maintainer's current commit-bound E2E approval", + "Candidate upgrade needs an independent maintainer's PR approval covering its inputs and policy", ); expect(git(input.trustedRoot, "diff", "--no-ext-diff")).toBe(before); }); diff --git a/test/helpers/candidate-openshell-trust.ts b/test/helpers/candidate-openshell-trust.ts index f407b432701..e16e9b84567 100644 --- a/test/helpers/candidate-openshell-trust.ts +++ b/test/helpers/candidate-openshell-trust.ts @@ -81,9 +81,9 @@ export function fixture() { { id: 50, user: { type: "User", login: "reviewer" }, - created_at: "2026-10-14T00:00:00Z", - updated_at: "2026-10-14T00:00:00Z", - body: `NemoClaw-Upgrade-E2E: approve ${identity.candidateSha} ${identity.baseSha}`, + submitted_at: "2026-10-14T00:00:00Z", + state: "APPROVED", + commit_id: identity.candidateSha, }, ]; const routes: Record = { @@ -94,10 +94,20 @@ export function fixture() { head: { sha: identity.candidateSha, repo: repository }, base: { sha: identity.baseSha, ref: "main", repo: repository }, }, - "/repos/NVIDIA/NemoClaw/issues/123/comments?per_page=100&page=1": approvals, + "/repos/NVIDIA/NemoClaw/pulls/123/reviews?per_page=100&page=1": approvals, "/repos/NVIDIA/NemoClaw/collaborators/reviewer/permission": { role_name: "maintain" }, }; const request = vi.fn(async (path: string): Promise => { + if ( + path === + `/repos/NVIDIA/NemoClaw/compare/${identity.baseSha}...${identity.candidateSha}?per_page=1` + ) + return { + status: "ahead", + base_commit: { sha: identity.baseSha }, + merge_base_commit: { sha: identity.baseSha }, + files: [], + }; const sourcePath = new URL(path, "https://api.github.com").pathname.replace( "/repos/NVIDIA/NemoClaw/contents/", "", diff --git a/test/helpers/openshell-installer-template.ts b/test/helpers/openshell-installer-template.ts index 3a654a0dcaa..823f8f838eb 100644 --- a/test/helpers/openshell-installer-template.ts +++ b/test/helpers/openshell-installer-template.ts @@ -17,6 +17,7 @@ export function copyTrustedInstallerVerifier(repoRoot: string, targetRoot: strin "tools/e2e/candidate-openshell-trust.mts", "tools/e2e/candidate-openshell-version.mts", "tools/e2e/candidate-upgrade-admission.mts", + "tools/e2e/candidate-review-scope.mts", "tools/e2e/base-image-publication.mts", ].forEach((file) => { const target = path.join(targetRoot, file); diff --git a/test/repository/candidate-openshell-sdk-artifact.test.ts b/test/repository/candidate-openshell-sdk-artifact.test.ts index 3678a959975..35069082aaa 100644 --- a/test/repository/candidate-openshell-sdk-artifact.test.ts +++ b/test/repository/candidate-openshell-sdk-artifact.test.ts @@ -57,9 +57,9 @@ describe("candidate SDK artifact provenance", () => { ); }); - it("selects an archive produced after an approval comment on main", () => { + it("selects an archive produced after a PR review refresh on main", () => { const f = fixture(); - f.run.event = "issue_comment"; + f.run.event = "workflow_run"; f.run.head_branch = "main"; f.run.head_sha = "d".repeat(40); const result = f.execute(); diff --git a/test/repository/package-openshell-sdk-for-pr.test.ts b/test/repository/package-openshell-sdk-for-pr.test.ts index 46d93b9460a..3ac1a8c465d 100644 --- a/test/repository/package-openshell-sdk-for-pr.test.ts +++ b/test/repository/package-openshell-sdk-for-pr.test.ts @@ -52,7 +52,8 @@ describe("reviewed OpenShell SDK transition packaging", () => { const prefix = "/repos/NVIDIA/NemoClaw"; const routes = [ `${prefix}/pulls/123`, - `${prefix}/issues/123/comments?per_page=100&page=1`, + `${prefix}/compare/${f.identity.baseSha}...${f.identity.candidateSha}?per_page=1`, + `${prefix}/pulls/123/reviews?per_page=100&page=1`, `${prefix}/collaborators/reviewer/permission`, `${prefix}/git/commits/${f.identity.mergeSha}`, ...Object.keys(f.sources).flatMap((file) => [ diff --git a/test/support/candidate-openshell-sdk.ts b/test/support/candidate-openshell-sdk.ts index c8f99476cb4..b9e01cc6f11 100644 --- a/test/support/candidate-openshell-sdk.ts +++ b/test/support/candidate-openshell-sdk.ts @@ -64,16 +64,26 @@ export function candidateSdkFixture() { const approval = { id: 50, user: { login: "reviewer", type: "User" }, - created_at: "2026-10-14T00:00:00Z", - updated_at: "2026-10-14T00:00:00Z", - body: `NemoClaw-Upgrade-E2E: approve ${sdkCandidateIdentity.candidateSha} ${sdkCandidateIdentity.baseSha}`, + submitted_at: "2026-10-14T00:00:00Z", + state: "APPROVED", + commit_id: sdkCandidateIdentity.candidateSha, }; const records = [approval]; const permission = { role_name: "maintain" }; const request = vi.fn(async (route: string): Promise => { if (route.endsWith("/pulls/123")) return pr; + if ( + route === + `/repos/NVIDIA/NemoClaw/compare/${sdkCandidateIdentity.baseSha}...${sdkCandidateIdentity.candidateSha}?per_page=1` + ) + return { + status: "ahead", + base_commit: { sha: sdkCandidateIdentity.baseSha }, + merge_base_commit: { sha: sdkCandidateIdentity.baseSha }, + files: [], + }; if (route.endsWith("/permission")) return permission; - if (route.includes("/comments?")) return records; + if (route.includes("/reviews?")) return records; const match = /^\/repos\/NVIDIA\/NemoClaw\/contents\/(.+)\?ref=([a-f0-9]{40})$/u.exec(route); if (!match || match[2] !== sdkCandidateIdentity.candidateSha || !sources[match[1]!]) { throw new Error("Unexpected SDK fixture request"); diff --git a/test/support/protected-openshell-workspace.ts b/test/support/protected-openshell-workspace.ts index 6d3d3d09bd2..11e88f552e9 100644 --- a/test/support/protected-openshell-workspace.ts +++ b/test/support/protected-openshell-workspace.ts @@ -133,8 +133,17 @@ export function approvedFixture() { git(input.candidateRoot, "add", "."); git(input.candidateRoot, "commit", "--quiet", "-m", "approved fixture"); input.candidateSha = git(input.candidateRoot, "rev-parse", "HEAD"); - approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${input.candidateSha} ${identity.baseSha}`; + approval.approvals[0]!.commit_id = input.candidateSha; const request = vi.fn(async (path: string) => { + const comparison = /\/compare\/([a-f0-9]{40})\.\.\.([a-f0-9]{40})\?per_page=1$/u.exec(path); + if (comparison) + return { + status: "ahead", + base_commit: { sha: comparison[1] }, + merge_base_commit: { sha: comparison[1] }, + files: + comparison[1] === identity.baseSha ? [] : [{ filename: CANDIDATE_OPENSHELL_TRUST_PATH }], + }; const result = await approval.request(path); if (path === "/repos/NVIDIA/NemoClaw/pulls/123") { const pr = result as { head: { sha: string } }; diff --git a/tools/e2e/candidate-openshell-trust.mts b/tools/e2e/candidate-openshell-trust.mts index 37ec6f9425d..ffaa715d9f2 100644 --- a/tools/e2e/candidate-openshell-trust.mts +++ b/tools/e2e/candidate-openshell-trust.mts @@ -107,7 +107,7 @@ export async function reviewedCandidateOpenShellTrust( releases, selected, reviewer: admission.reviewer, - commentId: admission.commentId, + reviewId: admission.reviewId, }; } diff --git a/tools/e2e/candidate-review-scope.mts b/tools/e2e/candidate-review-scope.mts new file mode 100644 index 00000000000..4a9d4e5690e --- /dev/null +++ b/tools/e2e/candidate-review-scope.mts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { CandidateUpgradeIdentity, UpgradeRequest } from "./candidate-upgrade-admission.mts"; + +const PREFIX = "/repos/NVIDIA/NemoClaw"; +const SHA = /^[a-f0-9]{40}$/u; + +// These are the trusted controllers and installer verifier, including the API transport. +// SDK admission also depends on the package verifier and its archive/audit helpers. +function governsAdmission(path: string, sdk: boolean): boolean { + return ( + path.startsWith("tools/e2e/candidate-") || + [ + "tools/e2e/base-image-publication.mts", + "tools/e2e/protected-openshell-workspace.mts", + "scripts/checks/extract-installer-pins.mts", + "scripts/checks/openshell-trust-data.mts", + "scripts/check-installer-hash.sh", + "ci/openshell-upgrade-trust.json", + ".github/actions/ci-installer-hash-check/action.yaml", + ".github/workflows/candidate-upgrade-admission.yaml", + ".github/workflows/candidate-review-signal.yaml", + ".github/workflows/installer-hash-check.yaml", + ".github/workflows/managed-images.yaml", + ".github/workflows/e2e.yaml", + ].includes(path) || + (sdk && + (path.startsWith("scripts/lib/") || + path.startsWith(".github/actions/") || + [ + "scripts/audit-reviewed-npm-graph.mts", + "ci/reviewed-npm-audit.json", + "scripts/checks/candidate-openshell-sdk.mts", + "scripts/checks/package-openshell-sdk-for-pr.mts", + "scripts/checks/prepare-ci-npm-install.mts", + "scripts/checks/resolve-managed-pr-base.sh", + ".github/workflows/pr.yaml", + ].includes(path))) + ); +} + +async function compare(request: UpgradeRequest, base: string, head: string) { + const value = await request(`${PREFIX}/compare/${base}...${head}?per_page=1`); + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Candidate review comparison is unavailable"); + const result = value as { + status?: string; + base_commit?: { sha?: string }; + merge_base_commit?: { sha?: string }; + files?: { filename: string; previous_filename?: string }[]; + }; + if ( + result.base_commit?.sha !== base || + typeof result.merge_base_commit?.sha !== "string" || + !SHA.test(result.merge_base_commit.sha) || + !["ahead", "behind", "diverged", "identical"].includes(result.status ?? "") + ) + throw new Error("Candidate review comparison has invalid commit identities"); + return result; +} + +function changedPaths(result: Awaited>): string[] { + // GitHub returns at most 300 files, only on the first comparison page. At the cap, + // completeness is unknown. A fresh review avoids carrying authority over that gap. + if (!Array.isArray(result.files) || result.files.length >= 300) + throw new Error("Candidate review comparison has an incomplete file list; refresh the review"); + return result.files.flatMap((file) => { + if ( + !file || + typeof file.filename !== "string" || + file.filename.length === 0 || + (file.previous_filename !== undefined && typeof file.previous_filename !== "string") + ) + throw new Error("Candidate review comparison has an invalid file path"); + return file.previous_filename === undefined + ? [file.filename] + : [file.filename, file.previous_filename]; + }); +} + +/** Reuse a review only for unchanged inputs and policy, never as test or merge evidence. */ +export async function candidateReviewCoversInputs( + identity: CandidateUpgradeIdentity, + reviewedSha: string, + paths: readonly string[], + request: UpgradeRequest, +): Promise { + const sdk = paths.some((path) => path === "package.json" || path === "nemoclaw/package.json"); + if (reviewedSha !== identity.candidateSha) { + const candidate = await compare(request, reviewedSha, identity.candidateSha); + if (candidate.status !== "ahead" || candidate.merge_base_commit!.sha !== reviewedSha) + return false; + if (changedPaths(candidate).some((path) => paths.includes(path) || governsAdmission(path, sdk))) + return false; + } + // GitHub reviews bind a candidate commit, not a base commit. Its common ancestor + // with main is a conservative policy baseline. Require synchronization and a new + // review when governing policy changed after that baseline. + const ancestry = await compare(request, identity.baseSha, reviewedSha); + const reviewedBase = ancestry.merge_base_commit!.sha!; + if (reviewedBase !== identity.baseSha) { + const policy = await compare(request, reviewedBase, identity.baseSha); + if (policy.status !== "ahead" || policy.merge_base_commit!.sha !== reviewedBase) + throw new Error("Candidate review policy history is inconsistent"); + if (changedPaths(policy).some((path) => governsAdmission(path, sdk))) return false; + } + return true; +} diff --git a/tools/e2e/candidate-upgrade-admission.mts b/tools/e2e/candidate-upgrade-admission.mts index fadc5f4a1a7..014b5ba9734 100644 --- a/tools/e2e/candidate-upgrade-admission.mts +++ b/tools/e2e/candidate-upgrade-admission.mts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { candidateReviewCoversInputs } from "./candidate-review-scope.mts"; + /** Authenticate permission to test candidate artifacts; never grants merge or host-code authority. */ export interface CandidateUpgradeIdentity { prNumber: number; @@ -62,76 +64,77 @@ function assertPullRequest( return author.login.toLowerCase(); } -async function comments(request: UpgradeRequest, prNumber: number): Promise { +async function reviews(request: UpgradeRequest, prNumber: number): Promise { const result: RecordValue[] = []; const seen = new Set(); for (let page = 1; page <= MAX_PAGES; page++) { - const values = await request(`${PREFIX}/issues/${prNumber}/comments?per_page=100&page=${page}`); + const values = await request(`${PREFIX}/pulls/${prNumber}/reviews?per_page=100&page=${page}`); if (!Array.isArray(values) || values.length > 100) { - throw new Error("Upgrade admission could not read complete PR comments"); + throw new Error("Upgrade admission could not read complete PR reviews"); } for (const value of values) { const entry = record(value); if (!Number.isSafeInteger(entry.id) || Number(entry.id) < 1 || seen.has(Number(entry.id))) { - throw new Error("Upgrade admission received duplicate or invalid comment identities"); + throw new Error("Upgrade admission received duplicate or invalid review identities"); } seen.add(Number(entry.id)); result.push(entry); } if (values.length < 100) return result; } - throw new Error("Upgrade admission comment pagination exceeded its bound"); + throw new Error("Upgrade admission review pagination exceeded its bound"); } -type Decision = { id: number; login: string; action: "approve" | "revoke" }; +type Decision = { id: number; login: string; commit: string }; async function currentDecision( request: UpgradeRequest, identity: CandidateUpgradeIdentity, author: string, + paths: readonly string[], ): Promise { - const candidates: Decision[] = []; - const permissions = new Map(); - for (const entry of await comments(request, identity.prNumber)) { - const match = - typeof entry.body === "string" - ? /^NemoClaw-Upgrade-E2E: (approve|revoke) ([a-f0-9]{40}) ([a-f0-9]{40})$/u.exec(entry.body) - : null; - if ( - !match || - match[0] !== entry.body || - match[2] !== identity.candidateSha || - match[3] !== identity.baseSha - ) - continue; - // Edited records are not authority: moderation can retain the original author's identity. - if ( - typeof entry.created_at !== "string" || - !Number.isFinite(Date.parse(entry.created_at)) || - entry.created_at !== entry.updated_at - ) - continue; + const latest = new Map(); + for (const entry of await reviews(request, identity.prNumber)) { const user = record(entry.user); const login = typeof user.login === "string" ? user.login : ""; if (user.type !== "User" || !LOGIN.test(login) || login.toLowerCase() === author) continue; + if (entry.state === "COMMENTED" || entry.state === "PENDING") continue; + if (!["APPROVED", "CHANGES_REQUESTED", "DISMISSED"].includes(String(entry.state))) + throw new Error("Upgrade admission received an unknown review state"); + if ( + typeof entry.submitted_at !== "string" || + !Number.isFinite(Date.parse(entry.submitted_at)) || + typeof entry.commit_id !== "string" || + !SHA.test(entry.commit_id) + ) + throw new Error("Upgrade admission received an invalid submitted review"); const key = login.toLowerCase(); - if (!permissions.has(key)) { - const response = record(await request(`${PREFIX}/collaborators/${login}/permission`)); - // Repository membership or author_association is not a permission check. - const role = response.role_name; - permissions.set(key, role === "maintain" || role === "admin"); - } - if (permissions.get(key)) { - candidates.push({ id: Number(entry.id), login, action: match[1] as Decision["action"] }); - } + const prior = latest.get(key); + if ( + !prior || + Date.parse(entry.submitted_at) > Date.parse(String(prior.submitted_at)) || + (entry.submitted_at === prior.submitted_at && Number(entry.id) > Number(prior.id)) + ) + latest.set(key, entry); } - const latest = candidates.sort((left, right) => right.id - left.id)[0]; - if (!latest || latest.action !== "approve") { - throw new Error( - "Candidate upgrade needs an independent maintainer's current commit-bound E2E approval", - ); + const candidates: Decision[] = []; + for (const entry of latest.values()) { + const login = String(record(entry.user).login); + const response = record(await request(`${PREFIX}/collaborators/${login}/permission`)); + if (response.role_name !== "maintain" && response.role_name !== "admin") continue; + // A maintainer's outstanding objection must not be overridden by another approval. + if (entry.state === "CHANGES_REQUESTED") + throw new Error("Candidate upgrade has an independent maintainer's changes-requested review"); + if (entry.state === "APPROVED") + candidates.push({ id: Number(entry.id), login, commit: String(entry.commit_id) }); + } + for (const candidate of candidates.sort((a, b) => b.id - a.id)) { + if (await candidateReviewCoversInputs(identity, candidate.commit, paths, request)) + return candidate; } - return latest; + throw new Error( + "Candidate upgrade needs an independent maintainer's PR approval covering its inputs and policy", + ); } export async function readCandidateUpgradeSource( @@ -179,7 +182,7 @@ export async function readApprovedCandidateUpgrade( paths: readonly string[], request: UpgradeRequest, mergeSha?: string, -): Promise<{ sources: Readonly>; reviewer: string; commentId: number }> { +): Promise<{ sources: Readonly>; reviewer: string; reviewId: number }> { const identity = Object.freeze({ ...input }); const selection = [...paths]; assertIdentity(identity); @@ -194,7 +197,7 @@ export async function readApprovedCandidateUpgrade( } const prPath = `${PREFIX}/pulls/${identity.prNumber}`; const author = assertPullRequest(await request(prPath), identity, mergeSha); - const before = await currentDecision(request, identity, author); + const before = await currentDecision(request, identity, author, selection); if (mergeSha !== undefined) { const merge = record(await request(`${PREFIX}/git/commits/${mergeSha}`)); if ( @@ -205,7 +208,7 @@ export async function readApprovedCandidateUpgrade( record(merge.parents[1]).sha !== identity.candidateSha ) throw new Error( - "Upgrade admission merge parents differ from the approved base and candidate; synchronize the PR and refresh its commit-bound approval", + "Upgrade admission merge parents differ from the approved base and candidate; synchronize the PR and obtain a review covering the changed inputs and policy", ); } const sources: Record = Object.create(null); @@ -218,9 +221,9 @@ export async function readApprovedCandidateUpgrade( if (assertPullRequest(await request(prPath), identity, mergeSha) !== author) { throw new Error("Upgrade admission PR author changed during source acquisition"); } - const after = await currentDecision(request, identity, author); - if (before.id !== after.id || before.login !== after.login) { + const after = await currentDecision(request, identity, author, selection); + if (before.id !== after.id || before.login !== after.login || before.commit !== after.commit) { throw new Error("Upgrade admission decision changed during source acquisition"); } - return { sources: Object.freeze(sources), reviewer: after.login, commentId: after.id }; + return { sources: Object.freeze(sources), reviewer: after.login, reviewId: after.id }; } From 9729720c82303c67337f9dfc66ca66dad60b53e0 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:16:10 -0800 Subject: [PATCH 2/6] test(ci): verify review handoff and comparison failures --- ci/source-shape-test-budget.json | 5 ++ ...ndidate-upgrade-admission-workflow.test.ts | 56 ++++++++++------ .../candidate-upgrade-admission.test.ts | 66 ++++++++++++------- 3 files changed, 83 insertions(+), 44 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index d80114d4b78..09828ea8e03 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -505,6 +505,11 @@ "file": "test/automation/e2e/platform-vitest-main-workflow.test.ts", "test": "budgets WSL shard 1 for serial live tests, cleanup and evidence upload", "category": "compatibility" + }, + { + "file": "test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts", + "test": "refreshes admission through the configured %s review handoff", + "category": "security" } ] } diff --git a/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts b/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts index e38dca84faa..87fe382a6ab 100644 --- a/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts +++ b/test/automation/pull-requests/candidate-upgrade-admission-workflow.test.ts @@ -5,9 +5,14 @@ import { describe, expect, it, vi } from "vitest"; import { join } from "node:path"; import { readYaml, type WorkflowJob } from "../../helpers/e2e-workflow-contract"; -const workflow = readYaml<{ jobs: Record }>( - ".github/workflows/candidate-upgrade-admission.yaml", -); +const workflow = readYaml<{ + on: { workflow_run: { workflows: string[]; types: string[] } }; + jobs: Record; +}>(".github/workflows/candidate-upgrade-admission.yaml"); +const reviewSignal = readYaml<{ + name: string; + on: { pull_request_review: { types: string[] } }; +}>(".github/workflows/candidate-review-signal.yaml"); const steps = workflow.jobs["candidate-upgrade-admission"]!.steps!; const start = steps.find((step) => step.id === "identity")!.with!.script as string; const finish = steps.find((step) => step.name === "Report current candidate upgrade admission")! @@ -146,23 +151,34 @@ describe("candidate upgrade admission status", () => { expect(f.status).toHaveBeenLastCalledWith(expect.objectContaining({ state: "failure" })); }); - it("refreshes current admission after the read-only review signal", async () => { - const f = fixture(); - f.context.eventName = "workflow_run"; - await f.start(); - expect(f.status).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ sha: head, context: "check-hash", state: "pending" }), - ); - expect(f.outputs).toEqual({ pr_number: "123", head_sha: head, base_sha: base }); - expect(f.list).toHaveBeenCalledExactlyOnceWith({ - owner: "NVIDIA", - repo: "NemoClaw", - head: "NVIDIA:codex/candidate", - base: "main", - state: "open", - per_page: 2, - }); - }); + // source-shape-contract: security -- GitHub must relay every review change to the trusted admission controller so dismissed approvals cannot leave candidate authorization stale. + it.each(["submitted", "edited", "dismissed"])( + "refreshes admission through the configured %s review handoff", + async (action) => { + expect(reviewSignal.on.pull_request_review.types).toContain(action); + expect(workflow.on.workflow_run).toEqual({ + workflows: [reviewSignal.name], + types: ["completed"], + }); + const f = fixture(); + f.context.eventName = "workflow_run"; + f.context.payload.workflow_run.name = reviewSignal.name; + f.context.payload.workflow_run.event = Object.keys(reviewSignal.on)[0]!; + await f.start(); + expect(f.status).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ sha: head, context: "check-hash", state: "pending" }), + ); + expect(f.outputs).toEqual({ pr_number: "123", head_sha: head, base_sha: base }); + expect(f.list).toHaveBeenCalledExactlyOnceWith({ + owner: "NVIDIA", + repo: "NemoClaw", + head: "NVIDIA:codex/candidate", + base: "main", + state: "open", + per_page: 2, + }); + }, + ); it.each(["wrong-event", "wrong-workflow", "fork", "missing-branch"])( "rejects a %s review signal", diff --git a/test/e2e/support/candidate-upgrade-admission.test.ts b/test/e2e/support/candidate-upgrade-admission.test.ts index c98132e40e8..deb9f263890 100644 --- a/test/e2e/support/candidate-upgrade-admission.test.ts +++ b/test/e2e/support/candidate-upgrade-admission.test.ts @@ -140,31 +140,49 @@ describe("candidate upgrade admission", () => { ); }); - it.each([ - "unrelated-history", - "truncated", - "missing-files", - "invalid-path", - "wrong-base", - "api-failure", - ])("denies reuse with %s comparison evidence", async (kind) => { + it.each(["unrelated-history", "truncated", "missing-files", "invalid-path", "wrong-base"])( + "denies reuse with %s comparison evidence", + async (kind) => { + const f = retainedReviewFixture(); + const mutate: Record unknown> = { + "unrelated-history": () => { + f.changes.merge_base_commit.sha = "f".repeat(40); + }, + truncated: () => { + f.changes.files = Array.from({ length: 300 }, () => ({ filename: "docs/example.mdx" })); + }, + "missing-files": () => Object.assign(f.changes, { files: undefined }), + "invalid-path": () => Object.assign(f.changes, { files: [null] }), + "wrong-base": () => { + f.changes.base_commit.sha = "f".repeat(40); + }, + }; + const error: Record = { + "unrelated-history": "PR approval covering", + truncated: "incomplete file list", + "missing-files": "incomplete file list", + "invalid-path": "invalid file path", + "wrong-base": "invalid commit identities", + }; + mutate[kind]!(); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + error[kind]!, + ); + }, + ); + + it("denies review reuse when the comparison API is unavailable", async () => { const f = retainedReviewFixture(); - const mutate: Record unknown> = { - "unrelated-history": () => { - f.changes.merge_base_commit.sha = "f".repeat(40); - }, - truncated: () => { - f.changes.files = Array.from({ length: 300 }, () => ({ filename: "docs/example.mdx" })); - }, - "missing-files": () => Object.assign(f.changes, { files: undefined }), - "invalid-path": () => Object.assign(f.changes, { files: [null] }), - "wrong-base": () => { - f.changes.base_commit.sha = "f".repeat(40); - }, - "api-failure": () => f.request.mockRejectedValueOnce(new Error("API unavailable")), - }; - mutate[kind]!(); - await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow(); + const comparison = `/repos/NVIDIA/NemoClaw/compare/${f.reviewed}...${identity.candidateSha}?per_page=1`; + const responses = new Map([...f.routes].map(([path, value]) => [path, async () => value])); + responses.set(comparison, async () => { + throw new Error("API unavailable"); + }); + f.request.mockImplementation(async (path) => responses.get(path)!()); + await expect(readApprovedCandidateUpgrade(identity, [sourcePath], f.request)).rejects.toThrow( + "API unavailable", + ); + expect(f.request).toHaveBeenCalledWith(comparison); }); it.each(["package.json", "scripts/lib/reviewed-npm-archive.mts"])( From eca5ab9e72c8147ef9372fb3f2c1f562b9f1807e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:48:50 -0800 Subject: [PATCH 3/6] docs(e2e): align protected runtime review scope --- test/e2e/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 774d81e6612..696054017a2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -2566,7 +2566,9 @@ Keep `include_staging_brev_launchable=false`. The candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. The GPU job reads OpenShell sources from the candidate commit as data, then validates their pins and operational templates. When the candidate needs trust absent from the controller's registry, preparation, both verifications, and restoration recheck the [candidate approval](#candidate-openshell-release-admission). -The approval must match the open PR, candidate commit, and base commit. +The approval must cover the open PR's admitted inputs and governing policy. +A later descendant commit remains eligible only when those inputs and policy are unchanged. +If governing policy changes on the base branch, synchronize the PR and obtain another review. The approved registry and installer sources must match the candidate's committed files. Candidates covered by the controller's registry need no additional approval. The job exposes its GitHub token as `GITHUB_TOKEN` only to the four trusted source-projection steps, with `contents: read` and `pull-requests: read` permissions. From d043bd09509fa28c69d1b55c9d420f6e790789e2 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:27:33 -0800 Subject: [PATCH 4/6] docs(ci): describe review signal token permissions accurately --- .github/workflows/candidate-review-signal.yaml | 2 +- test/e2e/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/candidate-review-signal.yaml b/.github/workflows/candidate-review-signal.yaml index 010cbf1fd07..374c487e1d9 100644 --- a/.github/workflows/candidate-review-signal.yaml +++ b/.github/workflows/candidate-review-signal.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # This PR-context workflow only wakes the default-branch admission controller. -# It has no credentials, checkout, artifacts, or authority to admit candidate inputs. +# It uses permissions: {} and has no checkout, artifacts, or admission authority. name: Security / Candidate Review Signal on: pull_request_review: diff --git a/test/e2e/README.md b/test/e2e/README.md index 696054017a2..217008bd088 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -462,7 +462,7 @@ from the candidate working directory. Normal branch protection and independent review govern that transition. An open PR's temporary admission is not merge authority. `Security / Candidate Upgrade Admission` refreshes the `check-hash` commit status on upstream PR changes and review submission, editing, or dismissal. -Review events run a credential-free signal workflow without a checkout. Its completion +Review events run a signal workflow with `permissions: {}` and no checkout. Its completion wakes the default-branch controller, which reads current reviews through GitHub's API. The signal provides no approval authority or candidate artifacts. When `check-hash` is required, both that status and the native hash check must pass. From b363eb839c7624cfd7d438bef60b1954ad47c48e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:24:05 -0800 Subject: [PATCH 5/6] docs(ci): clarify when approved candidate SDKs are packaged Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 217008bd088..2ae5b2f08f4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -466,8 +466,9 @@ Review events run a signal workflow with `permissions: {}` and no checkout. Its wakes the default-branch controller, which reads current reviews through GitHub's API. The signal provides no approval authority or candidate artifacts. When `check-hash` is required, both that status and the native hash check must pass. -A review refreshes the commit status and candidate SDK packaging. If the native hash -check failed before approval, rerun that job after approval. +Review events refresh the commit status. Candidate SDK packaging runs only when admission +succeeds and the PR requires a new SDK archive. Rejected admission produces no candidate +SDK artifact. If the native hash check failed before approval, rerun that job after approval. An interrupted refresh remains pending or fails; it cannot reuse an earlier success. For a new OpenShell TypeScript SDK version, the same approval also binds the root From f9a84a0b95fe122f33eee1edbe351a67dab0d89d Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:31:21 -0800 Subject: [PATCH 6/6] fix(ci): invalidate review after projection policy changes Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../candidate-upgrade-admission.test.ts | 30 +++++++++++++++++++ tools/e2e/candidate-review-scope.mts | 1 + 2 files changed, 31 insertions(+) diff --git a/test/e2e/support/candidate-upgrade-admission.test.ts b/test/e2e/support/candidate-upgrade-admission.test.ts index deb9f263890..3add0c88b3a 100644 --- a/test/e2e/support/candidate-upgrade-admission.test.ts +++ b/test/e2e/support/candidate-upgrade-admission.test.ts @@ -132,6 +132,36 @@ describe("candidate upgrade admission", () => { ); }); + it.each([ + { admission: "installer", path: sourcePath, history: "changes" as const, branch: "candidate" }, + { + admission: "installer", + path: sourcePath, + history: "policy" as const, + branch: "trusted base", + }, + { admission: "SDK", path: "package.json", history: "changes" as const, branch: "candidate" }, + { admission: "SDK", path: "package.json", history: "policy" as const, branch: "trusted base" }, + ])( + "invalidates $admission approval when the $branch changes projection policy", + async ({ path, history }) => { + const f = retainedReviewFixture(); + f.routes.set(`/repos/NVIDIA/NemoClaw/contents/${path}?ref=${identity.candidateSha}`, { + ...f.blob, + path, + }); + await expect( + readApprovedCandidateUpgrade(identity, [path], f.request), + ).resolves.toMatchObject({ + reviewId: 50, + }); + f[history].files = [{ filename: "tools/e2e/protected-openshell-projection.mts" }]; + await expect(readApprovedCandidateUpgrade(identity, [path], f.request)).rejects.toThrow( + /PR approval covering/, + ); + }, + ); + it("includes the previous name when a sensitive input is renamed", async () => { const f = retainedReviewFixture(); Object.assign(f.changes.files[0]!, { previous_filename: sourcePath }); diff --git a/tools/e2e/candidate-review-scope.mts b/tools/e2e/candidate-review-scope.mts index 4a9d4e5690e..54511d08a88 100644 --- a/tools/e2e/candidate-review-scope.mts +++ b/tools/e2e/candidate-review-scope.mts @@ -14,6 +14,7 @@ function governsAdmission(path: string, sdk: boolean): boolean { [ "tools/e2e/base-image-publication.mts", "tools/e2e/protected-openshell-workspace.mts", + "tools/e2e/protected-openshell-projection.mts", "scripts/checks/extract-installer-pins.mts", "scripts/checks/openshell-trust-data.mts", "scripts/check-installer-hash.sh",