Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/bundle-platform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ jobs:
run: docker buildx version || true
# The unit tests' first partition, with the doctests; Unit tests 2 of 2
# runs the other.
- name: Set up Python for image source-retention tests
if: inputs.unit && runner.os != 'Windows'
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Run tests
if: inputs.unit
run: cargo ci test --partition slice:1/2
Expand Down Expand Up @@ -256,6 +261,11 @@ jobs:
- name: Load Docker Buildx from disk
if: runner.os == 'Linux'
run: docker buildx version || true
- name: Set up Python for image source-retention tests
if: runner.os != 'Windows'
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Run tests
run: cargo ci test --partition slice:2/2
- name: Upload test timings
Expand Down
244 changes: 244 additions & 0 deletions crates/nemoclaw-build/tests/dependency_sources.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Exercise the image's source-retention command with authenticated artifact fixtures.

use serde_json::json;
use sha2::{Digest, Sha256};
use std::{fs, path::Path, process::Command};

fn run(root: &Path, artifact: &[u8], expected: &str) -> std::process::Output {
let cache = root.join("cache");
fs::create_dir(&cache).unwrap();
fs::write(cache.join("fixture-1.0.0.crate"), artifact).unwrap();
let sbom = json!({"components": [{
"name": "fixture", "version": "1.0.0", "purl": "pkg:cargo/fixture@1.0.0",
"bom-ref": "registry+https://github.com/rust-lang/crates.io-index#fixture@1.0.0",
"hashes": [{"alg": "SHA-256", "content": expected}]
}]});
for name in ["first", "second"] {
let dir = root.join(format!("{name}.dist-info/sboms"));
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("dependencies.json"), sbom.to_string()).unwrap();
}
let mut command = Command::new("python3");
command
.arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/fabric/dependency_sources.py"))
.arg("--sboms")
.arg(root)
.arg("--cache")
.arg(cache)
.arg("--output")
.arg(root.join("sources.tar.gz"));
if root.join("upstream-notices").exists() {
command
.arg("--upstream-notices")
.arg(root.join("upstream-notices"));
}
command.output().unwrap()
}

#[test]
fn source_bundles_keep_authenticated_artifacts_verbatim_without_duplicates() {
let root = tempfile::tempdir().unwrap();
let original = b"original upstream source and copyright notice\n";
let sha = nemoclaw_build::hex(&Sha256::digest(original));
let output = run(root.path(), original, &sha);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let unpack = Command::new("tar")
.arg("-xOf")
.arg(root.path().join("sources.tar.gz"))
.arg("fixture-1.0.0.crate")
.output()
.unwrap();
assert_eq!(
unpack.stdout, original,
"the bundle must preserve the original source artifact once"
);
use std::os::unix::fs::PermissionsExt;
assert_eq!(
fs::metadata(root.path().join("sources.tar.gz"))
.unwrap()
.permissions()
.mode()
& 0o444,
0o444,
"sandbox users must be able to read the retained sources"
);
}

#[test]
fn source_bundles_reject_bytes_that_do_not_match_the_sbom_pin() {
let root = tempfile::tempdir().unwrap();
let expected = nemoclaw_build::hex(&Sha256::digest(b"original source"));
let output = run(root.path(), b"changed source", &expected);
assert!(
!output.status.success(),
"a mismatched source must not be accepted"
);
assert!(
!root.path().join("sources.tar.gz").exists(),
"a rejected input must not leave a source bundle"
);
}

#[test]
fn source_bundles_include_registry_dependencies_from_python_source_locks() {
let root = tempfile::tempdir().unwrap();
let original = b"upstream source bytes";
let checksum = nemoclaw_build::hex(&Sha256::digest(original));
let cache = root.path().join("cache");
fs::create_dir(&cache).unwrap();
fs::write(cache.join("fixture-1.0.0.crate"), original).unwrap();
let source = root.path().join("python-source.tar.gz");
let gzip = flate2::write::GzEncoder::new(
fs::File::create(&source).unwrap(),
flate2::Compression::fast(),
);
let mut archive = tar::Builder::new(gzip);
let lock = format!(
"version = 4\n[[package]]\nname = \"fixture\"\nversion = \"1.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"{checksum}\"\n"
);
let mut header = tar::Header::new_gnu();
header.set_size(lock.len() as u64);
header.set_mode(0o644);
header.set_cksum();
archive
.append_data(&mut header, "python-source/Cargo.lock", lock.as_bytes())
.unwrap();
archive.into_inner().unwrap().finish().unwrap();
let bundle = root.path().join("bundle.tar.gz");
let output = Command::new("python3")
.arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/fabric/dependency_sources.py"))
.arg("--sdist")
.arg(source)
.arg("--cache")
.arg(cache)
.arg("--output")
.arg(&bundle)
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let unpack = Command::new("tar")
.arg("-xOf")
.arg(bundle)
.arg("fixture-1.0.0.crate")
.output()
.unwrap();
assert_eq!(
unpack.stdout, original,
"a source lock must retain its authenticated registry dependency"
);
}

#[test]
fn source_bundles_keep_the_openssl_source_pinned_by_cryptography_wheels() {
let root = tempfile::tempdir().unwrap();
let original = b"original OpenSSL source archive with its notices";
let checksum = nemoclaw_build::hex(&Sha256::digest(original));
let cache = root.path().join("cache");
fs::create_dir(&cache).unwrap();
fs::write(cache.join("openssl-4.0.3.tar.gz"), original).unwrap();
let dir = root.path().join("cryptography.dist-info/sboms");
fs::create_dir_all(&dir).unwrap();
fs::write(
dir.join("sbom.json"),
json!({"components": [{
"name": "openssl", "version": "4.0.3", "purl": "pkg:generic/openssl@4.0.3",
"hashes": [{"alg": "SHA-256", "content": checksum}]
}]})
.to_string(),
)
.unwrap();
let bundle = root.path().join("sources.tar.gz");
let output = Command::new("python3")
.arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/fabric/dependency_sources.py"))
.arg("--sboms")
.arg(root.path())
.arg("--cache")
.arg(cache)
.arg("--output")
.arg(&bundle)
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let unpack = Command::new("tar")
.arg("-xOf")
.arg(bundle)
.arg("openssl-4.0.3.tar.gz")
.output()
.unwrap();
assert_eq!(
unpack.stdout, original,
"the pinned native source must retain its original notices"
);
}

#[test]
fn source_bundles_add_missing_notices_from_the_matching_upstream_revision() {
let root = tempfile::tempdir().unwrap();
let manifest = br#"[package]
name = "fixture"
version = "1.0.0"
repository = "https://github.com/fixture/project"
"#;
let vcs = br#"{"git":{"sha1":"0000000000000000000000000000000000000000"}}"#;
let mut source = tar::Builder::new(flate2::write::GzEncoder::new(
Vec::new(),
flate2::Compression::fast(),
));
for (name, bytes) in [
("Cargo.toml", manifest.as_slice()),
(".cargo_vcs_info.json", vcs.as_slice()),
] {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(0o644);
header.set_cksum();
source
.append_data(&mut header, format!("fixture-1.0.0/{name}"), bytes)
.unwrap();
}
let artifact = source.into_inner().unwrap().finish().unwrap();
let checksum = nemoclaw_build::hex(&Sha256::digest(&artifact));
let notices = root.path().join("upstream-notices");
fs::create_dir(&notices).unwrap();
let matching =
"rust-notice-fixture--project--0000000000000000000000000000000000000000--LICENSE";
let original = b"original upstream permission notice";
fs::write(notices.join(matching), original).unwrap();
fs::write(notices.join("unrelated-LICENSE"), b"unrelated notice").unwrap();
let output = run(root.path(), &artifact, &checksum);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let unpack = Command::new("tar")
.arg("-xOf")
.arg(root.path().join("sources.tar.gz"))
.arg(format!("upstream-notices/{matching}"))
.output()
.unwrap();
assert_eq!(
unpack.stdout, original,
"the exact revision's original notice must be retained"
);
let listing = Command::new("tar")
.arg("-tf")
.arg(root.path().join("sources.tar.gz"))
.output()
.unwrap();
assert!(!String::from_utf8_lossy(&listing.stdout).contains("unrelated-LICENSE"));
}
2 changes: 2 additions & 0 deletions crates/nemoclaw-build/tests/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ mod changes;
mod ci;
#[cfg(unix)]
mod ci_lifecycle;
#[cfg(unix)]
mod dependency_sources;
#[path = "docs.rs"]
mod docs;
#[path = "fabric_cache.rs"]
Expand Down
23 changes: 23 additions & 0 deletions image/SOURCES.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,29 @@ The vendor trees can include build dependencies and dependencies for other platf
Their presence does not mean every package is linked into the executable.
The images separately retain the Rust standard-library notices; the standard-library source and toolchain are not included in these archives.

## Native Python Dependency Sources

Every Fabric image retains `/opt/nemoclaw/source/native-dependency-sources.tar.gz`.
It contains the original Cargo registry archives and OpenSSL releases identified by installed wheels' embedded SBOMs, verified against their SHA-256 pins.
The Deep Agents and mini-swe-agent archives additionally retain their pinned tiktoken, ormsgpack, fastuuid, or LiteLLM source distributions where installed, and the registry dependencies recorded by those distributions' Cargo locks.
Images that install the `nemo-relay` CLI also retain its matching 0.11.0 Python source distribution and the registry dependencies in its Cargo lock.
Hermes regenerates the archive after installing its additional wheels.

Extract the outer archive with `tar -xzf native-dependency-sources.tar.gz`; extract individual `.crate` files with the same command to read their original sources, notices, and copyright headers.
The original Python source distributions are under `python/`.
The archive is a conservative source set: SBOMs and Cargo locks can include build, optional, and other-platform dependencies.
It does not prove that every entry survives linking, supply every non-registry native dependency, or provide an offline build environment.
Deep Agents and mini-swe-agent build Tiktoken 0.13.0 from that unmodified source distribution with the retained `local/tiktoken-Cargo.lock` instead of installing the published wheel.
`tiktoken-cargo-source.tar.gz` retains that lock, its vendored dependencies and original notices, and relative Cargo source-replacement configuration.
Extract the original Python source distribution, then extract this Cargo archive into the same directory before rebuilding with Python 3.14, Rust 1.100.0, and a C toolchain:

```sh
CARGO_NET_OFFLINE=true python -m pip wheel --no-deps .
```

The Python build backend and its dependencies remain prerequisites.
The Cargo lock records this image's build; it does not reconstruct the dependencies of historical upstream wheels.

## Deep Agents' orjson Extension

The Deep Agents image retains orjson 3.11.9's original source distribution at `/opt/nemoclaw/source/deepagents/orjson-source.tar.gz`.
Expand Down
19 changes: 13 additions & 6 deletions image/fabric/CODEX-SOURCES.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,33 @@ tar -xzf codex-source.tar.gz -C codex-source --strip-components=1
```

The archive includes `codex-rs/vendor/bubblewrap`, Codex's Rust wrapper in `codex-rs/bwrap`, the Cargo lockfile, build scripts, and patches.
`codex/codex-cargo-source.tar.gz` retains Cargo's dependency sources, original notices and source headers, plus supplemental upstream notices omitted from published crates.
It also contains source-replacement configuration and the normalized lockfile; extract it into `codex-source/codex-rs` before building:

```sh
tar -xzf codex-cargo-source.tar.gz -C codex-source/codex-rs
```

`codex-bubblewrap-COPYING` separately retains Bubblewrap's LGPL-2.0-or-later license text.
Its original copyright notices remain in the archived C sources.
On Linux, building the wrapper requires the Rust version in the archived `rust-toolchain.toml`, a C toolchain, pkg-config, and libcap development files:

```sh
cd codex-source/codex-rs
cargo update --workspace
cargo build --locked --release -p codex-bwrap
cargo build --locked --offline --release -p codex-bwrap
```

The archived manifests use workspace version `0.154.0`, while the lockfile's local workspace entries use `0.0.0`.
The workspace-only update normalizes those local entries before the locked build; it does not update existing external dependency pins.
It can access the registry and Git sources when they are not already cached.
The retained dependency archive normalizes those local entries without changing existing external dependency pins.
It includes build dependencies and dependencies for other platforms; their presence does not establish that every package is linked into the shipped binaries.

`codex-libcap-source.tar.xz` and `codex-libcap-License` retain libcap 2.75's source and license, matching the version and checksum in Codex's `.github/scripts/install-musl-build-tools.sh`.
That archived script records its musl build flags and pkg-config configuration.
Cargo can download dependencies; they are not vendored in this archive.
These inputs do not establish complete transitive-source or relinking delivery for the shipped executable.
The dependency archive supports Cargo's offline source resolution; native build tools, the Rust toolchain, and libcap development files remain prerequisites.
These inputs do not establish identical release binaries or complete non-Cargo native source/relinking delivery.

`codex-rg-LICENSE-MIT` retains the MIT choice for the bundled ripgrep 15.2.0.
`codex-rg-source.tar.gz` retains its matching source revision and Cargo lock; the native dependency archive separately retains that lock's registry sources and original notices.
`codex-zsh-LICENCE` retains Zsh's license from `77045ef899e53b9598bebc5a41db93a548a40ca6`, the revision named by the bundled shell's patch level.
Codex's shell modification patch and build script are in its source archive.
The image copies only the Zsh executable, not the upstream shell-function collection.
Expand Down
Loading
Loading