From 5dcb30090d984355457b92c46c89550f2d3d3300 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:05:27 -0800 Subject: [PATCH] test(e2e): qualify recent upgrades with running and stopped sandboxes Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/README.md | 9 ++- .../live/openshell-gateway-upgrade.test.ts | 23 +++--- ...hell-gateway-upgrade-old-installer.test.ts | 30 ++++++-- ...-gateway-upgrade-workflow-boundary.test.ts | 77 ++++++++++--------- test/e2e/support/workflow-plan.test.ts | 8 +- .../e2e/openshell-gateway-upgrade-fixture.mts | 19 +++++ tools/e2e/target-catalogue.mts | 34 ++++++++ 7 files changed, 142 insertions(+), 58 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 25741886736..730e5cf0602 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -731,7 +731,7 @@ The historical fixtures retain these version boundaries: | Fixture | Required boundary | | --- | --- | -| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89 and v0.0.123 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. | +| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89, v0.0.123, and v0.0.141 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. | | `rebuild-openclaw` | Retain the reviewed old-base build in the target. Build and create the old sandbox before testing the candidate rebuild path. | These targets may restore the shared artifact for the candidate CLI. @@ -743,6 +743,13 @@ NemoClaw restore behavior, and Dockerfile patch behavior. The live target does n assert OpenClaw database tables, migration checkpoints, or other third-party storage details. +The v0.0.141 fixture adds recent-release compatibility with one running and one stopped sandbox. +It retains the released OpenShell 0.0.116 path, so it does not substitute for historical gateway replacement coverage. +The existing support tests verify fixture identity, archive pins, and adapter behavior. +The recent fixture uses the native agent RPC before and after upgrade because its convenience command requests administrator scope. +Manual PR selection uses the catalogue from trusted `main`; a newly added selector becomes available after that catalogue merges. +The OpenShell destination and workspace marker repairs in PR #14276 are prerequisites for completing upgrade qualification. + The retained live target owns the released-gateway upgrade and usable-survivor boundary. Deterministic rebuild-flow tests own post-backup recreate failure, preserved backup and registry state, recovery-journal retention, and successful diff --git a/test/e2e/live/openshell-gateway-upgrade.test.ts b/test/e2e/live/openshell-gateway-upgrade.test.ts index 5b60e53310f..d42c9a1ad12 100644 --- a/test/e2e/live/openshell-gateway-upgrade.test.ts +++ b/test/e2e/live/openshell-gateway-upgrade.test.ts @@ -22,6 +22,7 @@ import { REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; import { type ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -238,7 +239,10 @@ async function collectOpenClawCredentialBoundary( const requestOffset = fake.requests().length; const sessionId = `e2e-state-upgrade-${phase}`; const agentCommand = `agent --agent main --json --thinking off --session-id ${shellQuote(sessionId)} -m ${shellQuote("Reply with only: ok")}`; - // OpenClaw 2026.9.1's local convenience command requests operator.admin. + const useNativeAgentRpc = + phase === "upgraded" || + OLD_OPENCLAW_VERSION === REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion; + // OpenClaw 2026.9.2's local convenience command requests operator.admin. // The same native agent RPC needs operator.write for this ordinary message. const rpcParams = JSON.stringify({ message: "Reply with only: ok", @@ -253,14 +257,13 @@ async function collectOpenClawCredentialBoundary( redactionValues: [GATEWAY_CREDENTIAL], timeoutMs: 120_000, }; - const agent = - phase === "upgraded" - ? await bash( - host, - `nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`, - agentOptions, - ) - : await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions); + const agent = useNativeAgentRpc + ? await bash( + host, + `nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`, + agentOptions, + ) + : await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions); const requests = fake .requests() .slice(requestOffset) @@ -272,7 +275,7 @@ async function collectOpenClawCredentialBoundary( valid: secretNonExposure.exitCode === 0 && agent.exitCode === 0 && - (phase !== "upgraded" || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) && + (!useNativeAgentRpc || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) && parseOpenClawAgentText(agent.stdout).trim().toLowerCase() === "ok" && requests.length > 0 && requests.every((request) => request.auth === "ok" && request.authorizationSent === true), diff --git a/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts b/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts index 1ac9d881683..4e979e9462f 100644 --- a/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -9,6 +10,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { type ReviewedGatewayUpgradeFixture, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; @@ -33,6 +35,7 @@ function historicalReleaseCommitRef(nemoclawRef: string): string { function assertHistoricalReleaseIdentity(identity: { nemoclawCommit: string; nemoclawRef: string; + installerSha256: string; }): void { const resolved = spawnSync( "git", @@ -55,6 +58,15 @@ function assertHistoricalReleaseIdentity(identity: { resolved.stdout.trim(), `Historical NemoClaw release ${identity.nemoclawRef} must resolve to reviewed commit ${identity.nemoclawCommit}`, ).toBe(identity.nemoclawCommit); + const installer = spawnSync( + "git", + ["-C", REPO_ROOT, "show", `${identity.nemoclawCommit}:install.sh`], + { killSignal: "SIGKILL", timeout: SYNC_SUBPROCESS_TIMEOUT_MS }, + ); + expect(installer.status, installer.stderr.toString()).toBe(0); + expect(createHash("sha256").update(installer.stdout).digest("hex")).toBe( + identity.installerSha256, + ); } function writeInstallerHarness(sourceRoot: string): { @@ -266,13 +278,17 @@ describe("historical OpenShell gateway upgrade installer adapter", () => { expect(dockerfile).toContain("audit signatures"); }, 30_000); - it("accepts the v0.0.123 zero-advisory upgrade boundary (#11898)", () => { - const dockerfile = runReviewedHistoricalFixture(REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE); - expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low"); - expect(dockerfile).not.toContain( - "Skipping current advisory audit for the immutable historical mcporter lock", - ); - }, 30_000); + it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])( + "accepts the $nemoclawRef zero-advisory upgrade boundary (#14268)", + (fixture) => { + const dockerfile = runReviewedHistoricalFixture(fixture); + expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low"); + expect(dockerfile).not.toContain( + "Skipping current advisory audit for the immutable historical mcporter lock", + ); + }, + 30_000, + ); it("rejects an ambiguous historical advisory boundary", () => { const fixture = writeHistoricalFixture(2); diff --git a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts index 94b2c5b8097..8ff2f7861cd 100644 --- a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; @@ -183,6 +184,7 @@ describe("OpenShell gateway upgrade boundary", () => { ).toEqual([ "openshell-gateway-upgrade-v0-0-89-x86-64", "openshell-gateway-upgrade-v0-0-123-x86-64", + "openshell-gateway-upgrade-v0-0-141-x86-64", ]); const { environment, runner, shard } = catalogueTarget( @@ -214,43 +216,46 @@ describe("OpenShell gateway upgrade boundary", () => { }); }); - it("pins the v0.0.123 gateway-registration regression target to amd64 (#11898)", () => { - const { environment, runner, shard } = catalogueTarget( - "openshell-gateway-upgrade-v0-0-123-x86-64", - ); + it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])( + "pins the $nemoclawRef upgrade target to amd64 (#14268)", + (fixture) => { + const { environment, runner, shard } = catalogueTarget( + `openshell-gateway-upgrade-${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`, + ); - expect({ - runner, - shard, - nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, - commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, - installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, - overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled( - environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - ), - sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, - openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, - }).toEqual({ - runner: "ubuntu-latest", - shard: "v0-0-123-x86-64", - nemoclawRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawRef, - commit: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawCommit, - installerSha256: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.installerSha256, - overridesBaseImage: false, - sandboxBaseImageRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.sandboxBaseImageRef, - openShellVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openShellVersion, - openClawVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openclawVersion, - }); - validateLegacyGatewayUpgradeFixture({ - nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, - nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, - installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, - sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, - openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, - }); - }); + expect({ + runner, + shard, + nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, + commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, + installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, + overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled( + environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + ), + sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, + openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, + }).toEqual({ + runner: "ubuntu-latest", + shard: `${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`, + nemoclawRef: fixture.nemoclawRef, + commit: fixture.nemoclawCommit, + installerSha256: fixture.installerSha256, + overridesBaseImage: false, + sandboxBaseImageRef: fixture.sandboxBaseImageRef, + openShellVersion: fixture.openShellVersion, + openClawVersion: fixture.openclawVersion, + }); + validateLegacyGatewayUpgradeFixture({ + nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, + nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, + installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, + sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, + openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, + }); + }, + ); it("rejects reintroducing the superseded workflow job", () => { const workflow = readWorkflow() as { jobs: Record }; diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index b05852ca256..78b30499926 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -41,8 +41,7 @@ const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts"); const PLANNER_CLI_PREFIX = ["--import", "tsx", PLANNER_CLI]; const PLANNER_COMPILE_CACHE = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cache-")); -// Each case owns its output files; bound child-process overlap on CI and retain -// the child-process timeout under aggregate runner load. +// Each case owns its output files; bound CI overlap and retain the child-process timeout. vi.setConfig({ maxConcurrency: 4, testTimeout: 35_000 }); afterAll(() => rmSync(PLANNER_COMPILE_CACHE, { force: true, recursive: true })); @@ -102,7 +101,7 @@ describe("E2E workflow plan", () => { }), ]); expect(plan.hermesSelected).toBe(true); - expect(plan.coverageMatrix).toHaveLength(80); + expect(plan.coverageMatrix).toHaveLength(81); expect(selectedWorkflowJobs(plan)).toEqual([ "catalogue-github-read", "catalogue-nvidia-api", @@ -770,13 +769,14 @@ describe("E2E workflow plan", () => { "src/lib/actions/maintenance.ts", "src/lib/actions/sandbox/forward-recovery.ts", "src/lib/actions/upgrade-sandboxes.ts", - ])("selects both gateway-upgrade fixtures when %s changes", (changedFile) => { + ])("selects all reviewed gateway-upgrade fixtures when %s changes", (changedFile) => { expect(catalogueTargetsForChangedFiles([changedFile]).map((target) => target.id)).toEqual([ ...(changedFile === "scripts/install.sh" ? ["deferred-onboarding-hermes", "deferred-onboarding-langchain-deepagents-code"] : []), "openshell-gateway-upgrade-v0-0-89-x86-64", "openshell-gateway-upgrade-v0-0-123-x86-64", + "openshell-gateway-upgrade-v0-0-141-x86-64", ]); }); diff --git a/tools/e2e/openshell-gateway-upgrade-fixture.mts b/tools/e2e/openshell-gateway-upgrade-fixture.mts index 6b887f2b0ab..0cc5d39514a 100644 --- a/tools/e2e/openshell-gateway-upgrade-fixture.mts +++ b/tools/e2e/openshell-gateway-upgrade-fixture.mts @@ -40,9 +40,28 @@ export const REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE = Object.freeze({ sandboxBaseImageRef: "", } as const); +export const REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE = Object.freeze({ + additionalStoppedSandboxes: 1, + expectedAdvisoryAuditCount: 0, + installerSha256: "bafd48cd005e11e064e0c4caa0968a4c781453ccebcf4593855f85d28afbdeb8", + nemoclawCommit: "b7dcc9bb76197fb28ec673867c35974ac799eb15", + nemoclawRef: "v0.0.141", + openclawVersion: "2026.9.2", + openClawArchive: Object.freeze({ + expectedIntegrity: + "sha512-M6C7UsnX815nv26qBJFYGe6aGzv+ftZLRzV6S9oRXUtXg2Yn67eVntpssT94kgkquKVSeUxerUg0j1ONp4WYQg==", + label: "historical fixture OpenClaw 2026.9.2", + packageSpec: "openclaw@2026.9.2", + tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.9.2.tgz", + }), + openShellVersion: "0.0.116", + sandboxBaseImageRef: "", +} as const); + export const REVIEWED_GATEWAY_UPGRADE_FIXTURES = Object.freeze([ REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, ] as const); export type ReviewedGatewayUpgradeFixture = (typeof REVIEWED_GATEWAY_UPGRADE_FIXTURES)[number]; diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index e983d763171..be6aae9d15f 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -29,6 +29,7 @@ import { import { HERMES_ACP_E2E_OWNING_PATHS } from "./hermes-acp-owning-paths.mts"; import { REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, } from "./openshell-gateway-upgrade-fixture.mts"; import { SANDBOX_SURVIVAL_TARGET_TIMEOUT_MINUTES } from "./sandbox-survival-timeout-contract.mts"; @@ -460,9 +461,42 @@ const GATEWAY_REGISTRATION_UPGRADE_TARGET = dockerOnlyTarget( }, ); +const RECENT_GATEWAY_UPGRADE_TARGET = dockerOnlyTarget( + "openshell-gateway-upgrade-v0-0-141-x86-64", + { + targetId: "openshell-gateway-upgrade", + displayName: `Upgrade: restores ${REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef} running and stopped sandboxes on x86-64`, + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: + "x86-64 Ubuntu; GitHub release artifacts; host-local compatible inference endpoint", + profile: "github-read", + runner: "ubuntu-latest", + testFile: "test/e2e/live/openshell-gateway-upgrade.test.ts", + timeoutMinutes: 120, + installMode: "none", + restoreCli: true, + exposeCliBin: true, + shard: "v0-0-141-x86-64", + owningPaths: GATEWAY_UPGRADE_OWNING_PATHS, + environment: { + ...nonInteractive, + NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME: "e2e-gw-survivor", + NEMOCLAW_OLD_NEMOCLAW_REF: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef, + NEMOCLAW_OLD_NEMOCLAW_COMMIT: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawCommit, + NEMOCLAW_OLD_INSTALLER_SHA256: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.installerSha256, + NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF: + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.sandboxBaseImageRef, + NEMOCLAW_OLD_OPENSHELL_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openShellVersion, + NEMOCLAW_OLD_OPENCLAW_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion, + OPENSHELL_GATEWAY: "nemoclaw", + }, + }, +); + const GATEWAY_UPGRADE_TARGETS = Object.freeze([ GATEWAY_UPGRADE_TARGET, GATEWAY_REGISTRATION_UPGRADE_TARGET, + RECENT_GATEWAY_UPGRADE_TARGET, ]); export const E2E_CATALOGUE_EXCLUSION_REASONS = {