Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -788,7 +788,7 @@ The historical fixtures retain these version boundaries:

| Fixture | Required boundary |
| --- | --- |
| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89 and v0.0.123 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prepare private OpenShell and npm install locations before the historical install, preserving existing private executables. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. |
| `openshell-gateway-upgrade` | Retain the reviewed v0.0.89, v0.0.123, and v0.0.141 x86-64 fixtures with pinned installer commits, digests, and OpenClaw archives. Prepare private OpenShell and npm install locations before the historical install, preserving existing private executables. Prove credential custody and authenticated agent turns before and after upgrade. Require the survivor to preserve workspace state. Create each fixture-declared stopped sandbox, preserve its workspace state, and restore its stopped phase after a lifecycle check. |
| `rebuild-openclaw` | Retain the reviewed old-base build in the target. Build and create the old sandbox before testing the candidate rebuild path. |

The gateway fixture sets a canonical private `XDG_BIN_HOME` for both installer generations.
Expand All @@ -807,6 +807,13 @@ NemoClaw restore behavior, and Dockerfile patch behavior. The live target does n
assert OpenClaw database tables, migration checkpoints, or other third-party
storage details.

The v0.0.141 fixture adds recent-release compatibility with one running and one stopped sandbox.
It retains the released OpenShell 0.0.116 path, so it does not substitute for historical gateway replacement coverage.
The existing support tests verify fixture identity, archive pins, and adapter behavior.
The recent fixture uses the native agent RPC before and after upgrade because its convenience command requests administrator scope.
Manual PR selection uses the catalogue from trusted `main`; a newly added selector becomes available after that catalogue merges.
The OpenShell destination and workspace marker repairs in PR #14276 are prerequisites for completing upgrade qualification.

The retained live target owns the released-gateway upgrade and usable-survivor
boundary. Deterministic rebuild-flow tests own post-backup recreate failure,
preserved backup and registry state, recovery-journal retention, and successful
Expand Down
23 changes: 13 additions & 10 deletions test/e2e/live/openshell-gateway-upgrade.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
REVIEWED_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURES,
REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE,
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE,
} from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts";
import { type ArtifactSink } from "../fixtures/artifacts.ts";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
Expand Down Expand Up @@ -245,7 +246,10 @@ async function collectOpenClawCredentialBoundary(
const requestOffset = fake.requests().length;
const sessionId = `e2e-state-upgrade-${phase}`;
const agentCommand = `agent --agent main --json --thinking off --session-id ${shellQuote(sessionId)} -m ${shellQuote("Reply with only: ok")}`;
// OpenClaw 2026.9.1's local convenience command requests operator.admin.
const useNativeAgentRpc =
phase === "upgraded" ||
OLD_OPENCLAW_VERSION === REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion;
// OpenClaw 2026.9.2's local convenience command requests operator.admin.
// The same native agent RPC needs operator.write for this ordinary message.
const rpcParams = JSON.stringify({
message: "Reply with only: ok",
Expand All @@ -260,14 +264,13 @@ async function collectOpenClawCredentialBoundary(
redactionValues: [GATEWAY_CREDENTIAL],
timeoutMs: 120_000,
};
const agent =
phase === "upgraded"
? await bash(
host,
`nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`,
agentOptions,
)
: await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions);
const agent = useNativeAgentRpc
? await bash(
host,
`nemoclaw ${shellQuote(SURVIVOR_SANDBOX)} exec -- openclaw gateway call agent --json --expect-final --timeout 120000 --params ${shellQuote(rpcParams)}`,
agentOptions,
)
: await runInSurvivorSandbox(host, `openclaw ${agentCommand}`, agentOptions);
const requests = fake
.requests()
.slice(requestOffset)
Expand All @@ -279,7 +282,7 @@ async function collectOpenClawCredentialBoundary(
valid:
secretNonExposure.exitCode === 0 &&
agent.exitCode === 0 &&
(phase !== "upgraded" || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) &&
(!useNativeAgentRpc || gatewayUpgradeAgentResponseIsSuccessful(agent.stdout)) &&
parseOpenClawAgentText(agent.stdout).trim().toLowerCase() === "ok" &&
requests.length > 0 &&
requests.every((request) => request.auth === "ok" && request.authorizationSent === true),
Expand Down
30 changes: 23 additions & 7 deletions test/e2e/support/openshell-gateway-upgrade-old-installer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
type ReviewedGatewayUpgradeFixture,
REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE,
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURES,
} from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts";
Expand All @@ -33,6 +35,7 @@ function historicalReleaseCommitRef(nemoclawRef: string): string {
function assertHistoricalReleaseIdentity(identity: {
nemoclawCommit: string;
nemoclawRef: string;
installerSha256: string;
}): void {
const resolved = spawnSync(
"git",
Expand All @@ -55,6 +58,15 @@ function assertHistoricalReleaseIdentity(identity: {
resolved.stdout.trim(),
`Historical NemoClaw release ${identity.nemoclawRef} must resolve to reviewed commit ${identity.nemoclawCommit}`,
).toBe(identity.nemoclawCommit);
const installer = spawnSync(
"git",
["-C", REPO_ROOT, "show", `${identity.nemoclawCommit}:install.sh`],
{ killSignal: "SIGKILL", timeout: SYNC_SUBPROCESS_TIMEOUT_MS },
);
expect(installer.status, installer.stderr.toString()).toBe(0);
expect(createHash("sha256").update(installer.stdout).digest("hex")).toBe(
identity.installerSha256,
);
}

function writeInstallerHarness(sourceRoot: string): {
Expand Down Expand Up @@ -298,13 +310,17 @@ describe("historical OpenShell gateway upgrade installer adapter", () => {
expect(dockerfile).toContain("audit signatures");
}, 30_000);

it("accepts the v0.0.123 zero-advisory upgrade boundary (#11898)", () => {
const dockerfile = runReviewedHistoricalFixture(REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE);
expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low");
expect(dockerfile).not.toContain(
"Skipping current advisory audit for the immutable historical mcporter lock",
);
}, 30_000);
it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])(
"accepts the $nemoclawRef zero-advisory upgrade boundary (#14268)",
(fixture) => {
const dockerfile = runReviewedHistoricalFixture(fixture);
expect(dockerfile).not.toContain("audit --omit=dev --audit-level=low");
expect(dockerfile).not.toContain(
"Skipping current advisory audit for the immutable historical mcporter lock",
);
},
30_000,
);

it("rejects an ambiguous historical advisory boundary", () => {
const fixture = writeHistoricalFixture(2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE,
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURES,
} from "../../../tools/e2e/openshell-gateway-upgrade-fixture.mts";
Expand Down Expand Up @@ -220,6 +221,7 @@ describe("OpenShell gateway upgrade boundary", () => {
).toEqual([
"openshell-gateway-upgrade-v0-0-89-x86-64",
"openshell-gateway-upgrade-v0-0-123-x86-64",
"openshell-gateway-upgrade-v0-0-141-x86-64",
]);

const { environment, runner, shard } = catalogueTarget(
Expand Down Expand Up @@ -251,43 +253,46 @@ describe("OpenShell gateway upgrade boundary", () => {
});
});

it("pins the v0.0.123 gateway-registration regression target to amd64 (#11898)", () => {
const { environment, runner, shard } = catalogueTarget(
"openshell-gateway-upgrade-v0-0-123-x86-64",
);
it.each([REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE, REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE])(
"pins the $nemoclawRef upgrade target to amd64 (#14268)",
(fixture) => {
const { environment, runner, shard } = catalogueTarget(
`openshell-gateway-upgrade-${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`,
);

expect({
runner,
shard,
nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF,
commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT,
installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256,
overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled(
environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
),
sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION,
openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION,
}).toEqual({
runner: "ubuntu-latest",
shard: "v0-0-123-x86-64",
nemoclawRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawRef,
commit: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.nemoclawCommit,
installerSha256: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.installerSha256,
overridesBaseImage: false,
sandboxBaseImageRef: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.sandboxBaseImageRef,
openShellVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openShellVersion,
openClawVersion: REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE.openclawVersion,
});
validateLegacyGatewayUpgradeFixture({
nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF,
nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT,
installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256,
sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION,
openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION,
});
});
expect({
runner,
shard,
nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF,
commit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT,
installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256,
overridesBaseImage: legacyGatewayUpgradeBaseImageOverrideEnabled(
environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
),
sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION,
openClawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION,
}).toEqual({
runner: "ubuntu-latest",
shard: `${fixture.nemoclawRef.replaceAll(".", "-")}-x86-64`,
nemoclawRef: fixture.nemoclawRef,
commit: fixture.nemoclawCommit,
installerSha256: fixture.installerSha256,
overridesBaseImage: false,
sandboxBaseImageRef: fixture.sandboxBaseImageRef,
openShellVersion: fixture.openShellVersion,
openClawVersion: fixture.openclawVersion,
});
validateLegacyGatewayUpgradeFixture({
nemoclawRef: environment.NEMOCLAW_OLD_NEMOCLAW_REF,
nemoclawCommit: environment.NEMOCLAW_OLD_NEMOCLAW_COMMIT,
installerSha256: environment.NEMOCLAW_OLD_INSTALLER_SHA256,
sandboxBaseImageRef: environment.NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF,
openShellVersion: environment.NEMOCLAW_OLD_OPENSHELL_VERSION,
openclawVersion: environment.NEMOCLAW_OLD_OPENCLAW_VERSION,
});
},
);

it("rejects reintroducing the superseded workflow job", () => {
const workflow = readWorkflow() as { jobs: Record<string, unknown> };
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/support/workflow-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts");
const PLANNER_CLI_PREFIX = ["--import", "tsx", PLANNER_CLI];
const PLANNER_COMPILE_CACHE = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cache-"));

// Each case owns its output files; bound child-process overlap on CI and retain
// the child-process timeout under aggregate runner load.
// Each case owns its output files; bound CI overlap and retain the child-process timeout.
vi.setConfig({ maxConcurrency: 4, testTimeout: 35_000 });
afterAll(() => rmSync(PLANNER_COMPILE_CACHE, { force: true, recursive: true }));

Expand Down Expand Up @@ -102,7 +101,7 @@ describe("E2E workflow plan", () => {
}),
]);
expect(plan.hermesSelected).toBe(true);
expect(plan.coverageMatrix).toHaveLength(79);
expect(plan.coverageMatrix).toHaveLength(80);
expect(selectedWorkflowJobs(plan)).toEqual([
"catalogue-github-read",
"catalogue-nvidia-api",
Expand Down Expand Up @@ -770,13 +769,14 @@ describe("E2E workflow plan", () => {
"src/lib/actions/maintenance.ts",
"src/lib/actions/sandbox/forward-recovery.ts",
"src/lib/actions/upgrade-sandboxes.ts",
])("selects both gateway-upgrade fixtures when %s changes", (changedFile) => {
])("selects all reviewed gateway-upgrade fixtures when %s changes", (changedFile) => {
expect(catalogueTargetsForChangedFiles([changedFile]).map((target) => target.id)).toEqual([
...(changedFile === "scripts/install.sh"
? ["deferred-onboarding-hermes", "deferred-onboarding-langchain-deepagents-code"]
: []),
"openshell-gateway-upgrade-v0-0-89-x86-64",
"openshell-gateway-upgrade-v0-0-123-x86-64",
"openshell-gateway-upgrade-v0-0-141-x86-64",
]);
});

Expand Down
19 changes: 19 additions & 0 deletions tools/e2e/openshell-gateway-upgrade-fixture.mts
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,28 @@ export const REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE = Object.freeze({
sandboxBaseImageRef: "",
} as const);

export const REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE = Object.freeze({
additionalStoppedSandboxes: 1,
expectedAdvisoryAuditCount: 0,
installerSha256: "bafd48cd005e11e064e0c4caa0968a4c781453ccebcf4593855f85d28afbdeb8",
nemoclawCommit: "b7dcc9bb76197fb28ec673867c35974ac799eb15",
nemoclawRef: "v0.0.141",
openclawVersion: "2026.9.2",
openClawArchive: Object.freeze({
expectedIntegrity:
"sha512-M6C7UsnX815nv26qBJFYGe6aGzv+ftZLRzV6S9oRXUtXg2Yn67eVntpssT94kgkquKVSeUxerUg0j1ONp4WYQg==",
label: "historical fixture OpenClaw 2026.9.2",
packageSpec: "openclaw@2026.9.2",
tarballUrl: "https://registry.npmjs.org/openclaw/-/openclaw-2026.9.2.tgz",
}),
openShellVersion: "0.0.116",
sandboxBaseImageRef: "",
} as const);

export const REVIEWED_GATEWAY_UPGRADE_FIXTURES = Object.freeze([
REVIEWED_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE,
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE,
] as const);

export type ReviewedGatewayUpgradeFixture = (typeof REVIEWED_GATEWAY_UPGRADE_FIXTURES)[number];
34 changes: 34 additions & 0 deletions tools/e2e/target-catalogue.mts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import {
import { HERMES_ACP_E2E_OWNING_PATHS } from "./hermes-acp-owning-paths.mts";
import {
REVIEWED_GATEWAY_REGISTRATION_UPGRADE_FIXTURE,
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE,
REVIEWED_GATEWAY_UPGRADE_FIXTURE,
} from "./openshell-gateway-upgrade-fixture.mts";
import { SANDBOX_SURVIVAL_TARGET_TIMEOUT_MINUTES } from "./sandbox-survival-timeout-contract.mts";
Expand Down Expand Up @@ -460,9 +461,42 @@ const GATEWAY_REGISTRATION_UPGRADE_TARGET = dockerOnlyTarget(
},
);

const RECENT_GATEWAY_UPGRADE_TARGET = dockerOnlyTarget(
"openshell-gateway-upgrade-v0-0-141-x86-64",
{
targetId: "openshell-gateway-upgrade",
displayName: `Upgrade: restores ${REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef} running and stopped sandboxes on x86-64`,
agentRuntime: "openclaw",
environmentOrInferenceEndpoint:
"x86-64 Ubuntu; GitHub release artifacts; host-local compatible inference endpoint",
profile: "github-read",
runner: "ubuntu-latest",
testFile: "test/e2e/live/openshell-gateway-upgrade.test.ts",
timeoutMinutes: 120,
installMode: "none",
restoreCli: true,
exposeCliBin: true,
shard: "v0-0-141-x86-64",
owningPaths: GATEWAY_UPGRADE_OWNING_PATHS,
environment: {
...nonInteractive,
NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME: "e2e-gw-survivor",
NEMOCLAW_OLD_NEMOCLAW_REF: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawRef,
NEMOCLAW_OLD_NEMOCLAW_COMMIT: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.nemoclawCommit,
NEMOCLAW_OLD_INSTALLER_SHA256: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.installerSha256,
NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF:
REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.sandboxBaseImageRef,
NEMOCLAW_OLD_OPENSHELL_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openShellVersion,
NEMOCLAW_OLD_OPENCLAW_VERSION: REVIEWED_RECENT_GATEWAY_UPGRADE_FIXTURE.openclawVersion,
OPENSHELL_GATEWAY: "nemoclaw",
},
},
);

const GATEWAY_UPGRADE_TARGETS = Object.freeze([
GATEWAY_UPGRADE_TARGET,
GATEWAY_REGISTRATION_UPGRADE_TARGET,
RECENT_GATEWAY_UPGRADE_TARGET,
]);

export const E2E_CATALOGUE_EXCLUSION_REASONS = {
Expand Down
Loading