Check packaged stdio probes through native Codex loading

.github/workflows/ci.yml

@@ -24,3 +24,39 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm check
       - run: pnpm test
+  codex:
+    name: Native Codex (${{ matrix.os }})
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            binary: codex-x86_64-unknown-linux-musl
+          - os: windows-latest
+            binary: codex-x86_64-pc-windows-msvc.exe
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+      - uses: actions/setup-node@v7.0.0
+        with:
+          node-version: 24
+      - name: Install pinned Codex
+        shell: bash
+        working-directory: ${{ runner.temp }}
+        env:
+          CODEX_VERSION: 0.155.1
+          CODEX_BINARY: ${{ matrix.binary }}
+        run: |
+          curl --fail --location "https://github.com/openai/codex/releases/download/rust-v${CODEX_VERSION}/${CODEX_BINARY}.tar.gz" --output codex.tar.gz
+          tar -xzf codex.tar.gz
+      - name: Check native loading and reporting
+        shell: bash
+        env:
+          CODEX_BINARY: ${{ matrix.binary }}
+        run: node scripts/smoke-codex.mjs "$RUNNER_TEMP/$CODEX_BINARY"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: codex-${{ matrix.os }}
+          path: reports/codex/

README.md

@@ -36,4 +36,6 @@ pnpm check
 
 The root development package builds and tests the plugins.
 
-Automated tests exercise the packaged MCP probes through the official MCP SDK and invoke the skill-local reporting scripts. The test harness expands configuration and launches servers itself; these tests do not exercise a client's native plugin loader.
+Unit and integration tests exercise the packaged MCP probes through the official MCP SDK and invoke the skill-local reporting scripts. Their harness expands configuration and launches servers itself.
+
+CI also runs a pinned Codex client on Ubuntu and Windows. It installs the unchanged Core and reporting plugins, calls Core's stdio probes through Codex's native MCP runtime, and checks the saved report's covered results. The JSON report and client diagnostics are retained as CI artifacts. This smoke test requires no API key or model turn; it does not exercise an agent following the guiding skill.

scripts/smoke-codex.mjs

@@ -0,0 +1,180 @@
+import assert from 'node:assert/strict';
+import { spawn, spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { closeSync, openSync } from 'node:fs';
+import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { delimiter, dirname, isAbsolute, join, relative } from 'node:path';
+import { createInterface } from 'node:readline';
+import { fileURLToPath } from 'node:url';
+import { CASE_IDS } from '../plugins/agent-plugins-conformance/src/cases.mjs';
+
+const root = fileURLToPath(new URL('..', import.meta.url));
+const output = join(root, 'reports/codex');
+const reportPath = join(output, 'report.json');
+const [codex, ...extra] = process.argv.slice(2);
+assert.ok(codex && isAbsolute(codex) && extra.length === 0,
+  'Usage: node scripts/smoke-codex.mjs <absolute-codex-binary>');
+const names = ['agent-plugins-conformance-core', 'agent-plugins-conformance'];
+const servers = ['default', 'relative', 'root', 'data'];
+const coveredCases = CASE_IDS.filter((id) => id.startsWith('mcp.stdio.')
+  && id !== 'mcp.stdio.data.distinct-across-plugins'
+  && !id.startsWith('mcp.stdio.recovery.'));
+const temporary = await realpath(await mkdtemp(join(tmpdir(), 'apc-codex-smoke-')));
+const home = join(temporary, 'codex-home');
+const workspace = join(temporary, 'workspace');
+const marketplace = join(temporary, 'marketplace');
+const searchPath = Object.entries(process.env).find(([key]) => key.toUpperCase() === 'PATH')?.[1] ?? '';
+const env = Object.fromEntries(Object.entries(process.env)
+  .filter(([key]) => !/OPENAI|CODEX|CHATGPT|^APC_|^PLUGIN_|^PATH$/i.test(key)));
+env.CODEX_HOME = home;
+env.PATH = `${dirname(process.execPath)}${delimiter}${searchPath}`;
+let appServer;
+let stderr;
+let pending;
+let protocolError;
+let sequence = 0;
+
+function run(binary, args, input) {
+  const result = spawnSync(binary, args, {
+    cwd: workspace, env, input, encoding: 'utf8', timeout: 90_000,
+  });
+  if (result.error) throw result.error;
+  assert.equal(result.status, 0,
+    `${binary} ${args.join(' ')} failed (${result.status}):\n${result.stderr}\n${result.stdout}`);
+  return result.stdout;
+}
+
+async function hashes(directory) {
+  const files = (await readdir(directory, { recursive: true, withFileTypes: true }))
+    .filter((entry) => entry.isFile())
+    .map((entry) => join(entry.parentPath, entry.name)).sort();
+  return Promise.all(files.map(async (path) => [relative(directory, path),
+    createHash('sha256').update(await readFile(path)).digest('hex')]));
+}
+
+function failProtocol(error) {
+  protocolError = error;
+  pending?.reject(error);
+}
+
+function rpc(method, params) {
+  if (protocolError) return Promise.reject(protocolError);
+  const id = ++sequence;
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => failProtocol(new Error(`${method} timed out`)), 90_000);
+    const finish = (callback, value) => {
+      clearTimeout(timeout);
+      pending = undefined;
+      callback(value);
+    };
+    pending = { id, resolve: (value) => finish(resolve, value), reject: (error) => finish(reject, error) };
+    appServer.stdin.write(`${JSON.stringify({ id, method, params })}\n`);
+  });
+}
+
+function killTree(signal) {
+  if (process.platform === 'win32') {
+    spawnSync('taskkill', ['/PID', String(appServer.pid), '/T', '/F'], { timeout: 10_000 });
+  } else {
+    try { process.kill(-appServer.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; }
+  }
+}
+
+try {
+  await rm(output, { recursive: true, force: true });
+  await Promise.all([mkdir(output, { recursive: true }), mkdir(home), mkdir(workspace),
+    mkdir(join(marketplace, '.agents/plugins'), { recursive: true })]);
+  const version = run(codex, ['--version']).trim();
+  console.log(version);
+  await writeFile(join(output, 'codex-version.txt'), `${version}\n`);
+  for (const name of names) {
+    await cp(join(root, 'plugins', name), join(marketplace, 'plugins', name), { recursive: true });
+  }
+  await writeFile(join(marketplace, '.agents/plugins/marketplace.json'), JSON.stringify({
+    name: 'apc-native-smoke',
+    plugins: names.map((name) => ({ name, source: { source: 'local', path: `./plugins/${name}` },
+      policy: { installation: 'AVAILABLE', authentication: 'ON_INSTALL' } })),
+  }));
+  run(codex, ['plugin', 'marketplace', 'add', marketplace, '--json']);
+  const installed = [];
+  for (const name of names) {
+    const { installedPath } = JSON.parse(run(codex, ['plugin', 'add', `${name}@apc-native-smoke`, '--json']));
+    const original = await hashes(join(root, 'plugins', name));
+    assert.deepEqual(await hashes(installedPath), original, `${name}: installed package differs from source`);
+    installed.push(installedPath);
+  }
+  const reporter = join(installed[1], 'skills/run-conformance/scripts/report.mjs');
+  const record = (message) => run(process.execPath, [reporter, reportPath], JSON.stringify(message));
+  record({ action: 'start' });
+
+  stderr = openSync(join(output, 'app-server.stderr.log'), 'w');
+  appServer = spawn(codex, ['app-server', '--stdio'], {
+    cwd: workspace, env, stdio: ['pipe', 'pipe', stderr], detached: true,
+  });
+  appServer.on('error', failProtocol);
+  appServer.on('exit', (code, signal) => failProtocol(new Error(`app-server exited (${code ?? signal})`)));
+  appServer.stdin.on('error', failProtocol);
+  const lines = createInterface({ input: appServer.stdout });
+  lines.on('line', (line) => {
+    try {
+      const message = JSON.parse(line);
+      if (pending && message.id === pending.id) {
+        if (message.error) pending.reject(new Error(JSON.stringify(message.error)));
+        else pending.resolve(message.result);
+      } else if (message.method && message.id !== undefined) {
+        failProtocol(new Error(`Unexpected app-server request: ${message.method}`));
+      }
+    } catch (error) {
+      failProtocol(error);
+    }
+  });
+  await rpc('initialize', {
+    clientInfo: { name: 'apc_native_smoke', version: '1.0.0' },
+    capabilities: { experimentalApi: true },
+  });
+  appServer.stdin.write(`${JSON.stringify({ method: 'initialized' })}\n`);
+  const { thread } = await rpc('thread/start', {
+    cwd: workspace, ephemeral: true, approvalPolicy: 'never', sandbox: 'danger-full-access',
+  });
+  const status = await rpc('mcpServerStatus/list', { threadId: thread.id, detail: 'toolsAndAuthOnly' });
+  await writeFile(join(output, 'mcp-status.json'), `${JSON.stringify(status, null, 2)}\n`);
+  for (const server of servers) {
+    assert.equal(status.data.filter(({ name }) => name === server).length, 1,
+      `${server}: expected one native MCP server`);
+    const result = await rpc('mcpServer/tool/call', {
+      threadId: thread.id, server, tool: 'observe', arguments: {},
+    });
+    assert.ok(!result.isError && !result.error, `${server}: observe failed: ${JSON.stringify(result)}`);
+    const observation = result.structuredContent;
+    assert.equal(observation?.kind, 'mcp-stdio', `${server}: missing stdio observation`);
+    assert.equal(observation.server, server, `${server}: unexpected observation server`);
+    record({ action: 'record', observation });
+  }
+  const report = JSON.parse(await readFile(reportPath, 'utf8'));
+  assert.equal(report.summary.fail, 0, `Report contains failed cases: ${JSON.stringify(report.summary)}`);
+  for (const id of coveredCases) {
+    assert.equal(report.results.find((result) => result.id === id)?.status, 'pass', `${id} did not pass`);
+  }
+  console.log(`Native Codex smoke passed: ${coveredCases.length} Core stdio cases; ${report.summary.not_verified} not verified.`);
+  console.log(`Report: ${reportPath}`);
+} catch (error) {
+  console.error(`Native Codex smoke failed: ${error.message}`);
+  process.exitCode = 1;
+} finally {
+  if (appServer?.pid) {
+    // Stop the native MCP children along with app-server.
+    killTree('SIGTERM');
+    if (appServer.exitCode === null && appServer.signalCode === null) {
+      await new Promise((resolve) => {
+        const timeout = setTimeout(() => {
+          killTree('SIGKILL');
+          resolve();
+        }, 5_000);
+        appServer.once('exit', () => { clearTimeout(timeout); resolve(); });
+      });
+    }
+  }
+  if (stderr !== undefined) closeSync(stderr);
+  await rm(temporary, { recursive: true, force: true });
+}