diff --git a/test/e2e/README.md b/test/e2e/README.md index 957e1c2d8e5..6dffe02b375 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -788,7 +788,7 @@ The historical fixtures retain these version boundaries: | Fixture | Required boundary | | --- | --- | -| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89 and v0.0.123 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prepare private OpenShell and npm install locations before the historical install, preserving existing private executables. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. | +| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89, v0.0.123, and v0.0.141 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prepare private OpenShell and npm install locations before the historical install, preserving existing private executables. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. | | `rebuild-openclaw` | Retain the reviewed old-base build in the target. Build and create the old sandbox before testing the candidate rebuild path. | The gateway fixture sets a canonical private `XDG_BIN_HOME` for both installer generations. @@ -807,6 +807,13 @@ NemoClaw restore behavior, and Dockerfile patch behavior. The live target does n assert OpenClaw database tables, migration checkpoints, or other third-party storage details. +The v0.0.141 fixture adds recent-release compatibility with one running and one stopped sandbox. +It retains the released OpenShell 0.0.116 path, so it does not substitute for historical gateway replacement coverage. +The existing support tests verify fixture identity, archive pins, and adapter behavior. +The recent fixture uses the native agent RPC before and after upgrade because its convenience command requests administrator scope. +Manual PR selection uses the catalogue from trusted `main`; a newly added selector becomes available after that catalogue merges. +The OpenShell destination and workspace marker repairs in PR #14276 are prerequisites for completing upgrade qualification. + The retained live target owns the released-gateway upgrade and usable-survivor boundary. Deterministic rebuild-flow tests own post-backup recreate failure, preserved backup and registry state, recovery-journal retention, and successful diff --git a/test/e2e/live/openshell-gateway-upgrade.test.ts b/test/e2e/live/openshell-gateway-upgrade.test.ts index e75694d5e3a..772cd4f659c 100644 --- a/test/e2e/live/openshell-gateway-upgrade.test.ts +++ b/test/e2e/live/openshell-gateway-upgrade.test.ts @@ -22,6 +22,7 @@ import { REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; import { type ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -245,7 +246,10 @@ async function collectOpenClawCredentialBoundary( const requestOffset = fake.requests().length; const sessionId = `e2e-state-upgrade-${phase}`; const agentCommand = `agent --agent main --json --thinking off --session-id ${shellQuote(sessionId)} -m ${shellQuote("Reply with only: ok")}`; - // OpenClaw 2026.9.1's local convenience command requests operator.admin. + const useNativeAgentRpc = + phase === "upgraded" || + OLD_OPENCLAW_VERSION === REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion; + // OpenClaw 2026.9.2's local convenience command requests operator.admin. // The same native agent RPC needs operator.write for this ordinary message. const rpcParams = JSON.stringify({ message: "Reply with only: ok", @@ -260,14 +264,13 @@ async function collectOpenClawCredentialBoundary( redactionValues: [GATEWAY_CREDENTIAL], timeoutMs: 120_000, }; - const agent = - phase === "upgraded" - ? await bash( - host, - `nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`, - agentOptions, - ) - : await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions); + const agent = useNativeAgentRpc + ? await bash( + host, + `nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`, + agentOptions, + ) + : await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions); const requests = fake .requests() .slice(requestOffset) @@ -279,7 +282,7 @@ async function collectOpenClawCredentialBoundary( valid: secretNonExposure.exitCode === 0 && agent.exitCode === 0 && - (phase !== "upgraded" || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) && + (!useNativeAgentRpc || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) && parseOpenClawAgentText(agent.stdout).trim().toLowerCase() === "ok" && requests.length > 0 && requests.every((request) => request.auth === "ok" && request.authorizationSent === true), diff --git a/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts b/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts index 29faba739a9..6358875079e 100644 --- a/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -9,6 +10,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { type ReviewedGatewayUpgradeFixture, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; @@ -33,6 +35,7 @@ function historicalReleaseCommitRef(nemoclawRef: string): string { function assertHistoricalReleaseIdentity(identity: { nemoclawCommit: string; nemoclawRef: string; + installerSha256: string; }): void { const resolved = spawnSync( "git", @@ -55,6 +58,15 @@ function assertHistoricalReleaseIdentity(identity: { resolved.stdout.trim(), `Historical NemoClaw release ${identity.nemoclawRef} must resolve to reviewed commit ${identity.nemoclawCommit}`, ).toBe(identity.nemoclawCommit); + const installer = spawnSync( + "git", + ["-C", REPO_ROOT, "show", `${identity.nemoclawCommit}:install.sh`], + { killSignal: "SIGKILL", timeout: SYNC_SUBPROCESS_TIMEOUT_MS }, + ); + expect(installer.status, installer.stderr.toString()).toBe(0); + expect(createHash("sha256").update(installer.stdout).digest("hex")).toBe( + identity.installerSha256, + ); } function writeInstallerHarness(sourceRoot: string): { @@ -298,13 +310,17 @@ describe("historical OpenShell gateway upgrade installer adapter", () => { expect(dockerfile).toContain("audit signatures"); }, 30_000); - it("accepts the v0.0.123 zero-advisory upgrade boundary (#11898)", () => { - const dockerfile = runReviewedHistoricalFixture(REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE); - expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low"); - expect(dockerfile).not.toContain( - "Skipping current advisory audit for the immutable historical mcporter lock", - ); - }, 30_000); + it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])( + "accepts the $nemoclawRef zero-advisory upgrade boundary (#14268)", + (fixture) => { + const dockerfile = runReviewedHistoricalFixture(fixture); + expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low"); + expect(dockerfile).not.toContain( + "Skipping current advisory audit for the immutable historical mcporter lock", + ); + }, + 30_000, + ); it("rejects an ambiguous historical advisory boundary", () => { const fixture = writeHistoricalFixture(2); diff --git a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts index c6123c9b8f7..a07150b8965 100644 --- a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURES, } from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts"; @@ -220,6 +221,7 @@ describe("OpenShell gateway upgrade boundary", () => { ).toEqual([ "openshell-gateway-upgrade-v0-0-89-x86-64", "openshell-gateway-upgrade-v0-0-123-x86-64", + "openshell-gateway-upgrade-v0-0-141-x86-64", ]); const { environment, runner, shard } = catalogueTarget( @@ -251,43 +253,46 @@ describe("OpenShell gateway upgrade boundary", () => { }); }); - it("pins the v0.0.123 gateway-registration regression target to amd64 (#11898)", () => { - const { environment, runner, shard } = catalogueTarget( - "openshell-gateway-upgrade-v0-0-123-x86-64", - ); + it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])( + "pins the $nemoclawRef upgrade target to amd64 (#14268)", + (fixture) => { + const { environment, runner, shard } = catalogueTarget( + `openshell-gateway-upgrade-${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`, + ); - expect({ - runner, - shard, - nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, - commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, - installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, - overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled( - environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - ), - sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, - openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, - }).toEqual({ - runner: "ubuntu-latest", - shard: "v0-0-123-x86-64", - nemoclawRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawRef, - commit: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawCommit, - installerSha256: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.installerSha256, - overridesBaseImage: false, - sandboxBaseImageRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.sandboxBaseImageRef, - openShellVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openShellVersion, - openClawVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openclawVersion, - }); - validateLegacyGatewayUpgradeFixture({ - nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, - nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, - installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, - sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, - openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, - openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, - }); - }); + expect({ + runner, + shard, + nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, + commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, + installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, + overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled( + environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + ), + sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, + openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, + }).toEqual({ + runner: "ubuntu-latest", + shard: `${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`, + nemoclawRef: fixture.nemoclawRef, + commit: fixture.nemoclawCommit, + installerSha256: fixture.installerSha256, + overridesBaseImage: false, + sandboxBaseImageRef: fixture.sandboxBaseImageRef, + openShellVersion: fixture.openShellVersion, + openClawVersion: fixture.openclawVersion, + }); + validateLegacyGatewayUpgradeFixture({ + nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF, + nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT, + installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256, + sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF, + openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION, + openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION, + }); + }, + ); it("rejects reintroducing the superseded workflow job", () => { const workflow = readWorkflow() as { jobs: Record }; diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 895f6672909..39374593cda 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -41,8 +41,7 @@ const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts"); const PLANNER_CLI_PREFIX = ["--import", "tsx", PLANNER_CLI]; const PLANNER_COMPILE_CACHE = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cache-")); -// Each case owns its output files; bound child-process overlap on CI and retain -// the child-process timeout under aggregate runner load. +// Each case owns its output files; bound CI overlap and retain the child-process timeout. vi.setConfig({ maxConcurrency: 4, testTimeout: 35_000 }); afterAll(() => rmSync(PLANNER_COMPILE_CACHE, { force: true, recursive: true })); @@ -102,7 +101,7 @@ describe("E2E workflow plan", () => { }), ]); expect(plan.hermesSelected).toBe(true); - expect(plan.coverageMatrix).toHaveLength(79); + expect(plan.coverageMatrix).toHaveLength(80); expect(selectedWorkflowJobs(plan)).toEqual([ "catalogue-github-read", "catalogue-nvidia-api", @@ -770,13 +769,14 @@ describe("E2E workflow plan", () => { "src/lib/actions/maintenance.ts", "src/lib/actions/sandbox/forward-recovery.ts", "src/lib/actions/upgrade-sandboxes.ts", - ])("selects both gateway-upgrade fixtures when %s changes", (changedFile) => { + ])("selects all reviewed gateway-upgrade fixtures when %s changes", (changedFile) => { expect(catalogueTargetsForChangedFiles([changedFile]).map((target) => target.id)).toEqual([ ...(changedFile === "scripts/install.sh" ? ["deferred-onboarding-hermes", "deferred-onboarding-langchain-deepagents-code"] : []), "openshell-gateway-upgrade-v0-0-89-x86-64", "openshell-gateway-upgrade-v0-0-123-x86-64", + "openshell-gateway-upgrade-v0-0-141-x86-64", ]); }); diff --git a/tools/e2e/openshell-gateway-upgrade-fixture.mts b/tools/e2e/openshell-gateway-upgrade-fixture.mts index 6b887f2b0ab..0cc5d39514a 100644 --- a/tools/e2e/openshell-gateway-upgrade-fixture.mts +++ b/tools/e2e/openshell-gateway-upgrade-fixture.mts @@ -40,9 +40,28 @@ export const REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE = Object.freeze({ sandboxBaseImageRef: "", } as const); +export const REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE = Object.freeze({ + additionalStoppedSandboxes: 1, + expectedAdvisoryAuditCount: 0, + installerSha256: "bafd48cd005e11e064e0c4caa0968a4c781453ccebcf4593855f85d28afbdeb8", + nemoclawCommit: "b7dcc9bb76197fb28ec673867c35974ac799eb15", + nemoclawRef: "v0.0.141", + openclawVersion: "2026.9.2", + openClawArchive: Object.freeze({ + expectedIntegrity: + "sha512-M6C7UsnX815nv26qBJFYGe6aGzv+ftZLRzV6S9oRXUtXg2Yn67eVntpssT94kgkquKVSeUxerUg0j1ONp4WYQg==", + label: "historical fixture OpenClaw 2026.9.2", + packageSpec: "openclaw@2026.9.2", + tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.9.2.tgz", + }), + openShellVersion: "0.0.116", + sandboxBaseImageRef: "", +} as const); + export const REVIEWED_GATEWAY_UPGRADE_FIXTURES = Object.freeze([ REVIEWED_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, ] as const); export type ReviewedGatewayUpgradeFixture = (typeof REVIEWED_GATEWAY_UPGRADE_FIXTURES)[number]; diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index 9a76eaf8fdc..115817a0e67 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -29,6 +29,7 @@ import { import { HERMES_ACP_E2E_OWNING_PATHS } from "./hermes-acp-owning-paths.mts"; import { REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE, REVIEWED_GATEWAY_UPGRADE_FIXTURE, } from "./openshell-gateway-upgrade-fixture.mts"; import { SANDBOX_SURVIVAL_TARGET_TIMEOUT_MINUTES } from "./sandbox-survival-timeout-contract.mts"; @@ -460,9 +461,42 @@ const GATEWAY_REGISTRATION_UPGRADE_TARGET = dockerOnlyTarget( }, ); +const RECENT_GATEWAY_UPGRADE_TARGET = dockerOnlyTarget( + "openshell-gateway-upgrade-v0-0-141-x86-64", + { + targetId: "openshell-gateway-upgrade", + displayName: `Upgrade: restores ${REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef} running and stopped sandboxes on x86-64`, + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: + "x86-64 Ubuntu; GitHub release artifacts; host-local compatible inference endpoint", + profile: "github-read", + runner: "ubuntu-latest", + testFile: "test/e2e/live/openshell-gateway-upgrade.test.ts", + timeoutMinutes: 120, + installMode: "none", + restoreCli: true, + exposeCliBin: true, + shard: "v0-0-141-x86-64", + owningPaths: GATEWAY_UPGRADE_OWNING_PATHS, + environment: { + ...nonInteractive, + NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME: "e2e-gw-survivor", + NEMOCLAW_OLD_NEMOCLAW_REF: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef, + NEMOCLAW_OLD_NEMOCLAW_COMMIT: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawCommit, + NEMOCLAW_OLD_INSTALLER_SHA256: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.installerSha256, + NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF: + REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.sandboxBaseImageRef, + NEMOCLAW_OLD_OPENSHELL_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openShellVersion, + NEMOCLAW_OLD_OPENCLAW_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion, + OPENSHELL_GATEWAY: "nemoclaw", + }, + }, +); + const GATEWAY_UPGRADE_TARGETS = Object.freeze([ GATEWAY_UPGRADE_TARGET, GATEWAY_REGISTRATION_UPGRADE_TARGET, + RECENT_GATEWAY_UPGRADE_TARGET, ]); export const E2E_CATALOGUE_EXCLUSION_REASONS = {