Embed the registry and the version in the bundle (#107)

.github/workflows/ci.yml

@@ -289,8 +289,8 @@ jobs:
           if phantom: print("::error::SKILL.md names commands the CLI does not have:", sorted(phantom))
           sys.exit(1 if (missing or phantom) else 0)
           PY
-      # A package.json version is a RUNTIME value: the bundle reports the one beside it as
-      # `treasury --version` and `earn status`, and the manifests are what the marketplaces advertise.
+      # The bundle reports the package.json version it was built from as `treasury --version` and
+      # `earn status`, and the manifests are what the marketplaces advertise: this ties them together.
       - name: every declaration of this repository's own version agrees
         if: steps.pk.outputs.present == 'yes'
         run: bash scripts/check-versions.sh

CHANGELOG.md

@@ -42,6 +42,19 @@ Pre-1.0, minor versions may change tool names, schemas and behaviour.
   Kamino's own SDK by committed fixtures (`vendor/kamino/`); that SDK is not a dependency.
 - `registry:check` reconciles the Solana row against the vault's own state.
 
+### Changed — the registry and the version are inside the bundle
+- The vault registry and the package version are now compiled into the CLI bundle and the library build
+  instead of being read from files beside them. `treasury --version` reports the version the bundle was built
+  from, and `listVaults()` returns the shipped registry even when the library is re-bundled into another
+  application, where it used to look for `registry/vaults.json` beside the consumer's own bundle and read
+  whatever it found there. A registry change now also changes `dist/treasury.mjs`.
+- A bundler that builds on this package must accept JSON import attributes (`with { type: "json" }`);
+  esbuild, webpack 5 and Rollup with its JSON plugin do.
+- The CLI bundle carries only the `version` field of `package.json`, so editing anything else there (a
+  dev-dependency bump) leaves `dist/treasury.mjs` unchanged.
+- Importing the library unbundled on Node 22.0 to 22.11 prints Node's `ExperimentalWarning` for JSON modules;
+  it works, and Node 22.12 and later print nothing. The CLI is unaffected.
+
 ### Fixed
 - `earn balance` reads the Ethereum public endpoint's range refusal ("query exceeds max block range
   100000") as a 100,000-block window and walks the history in chunks, instead of failing the scan.

scripts/check-versions.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 # Every place this repository declares its own version must say the same thing.
 #
-# Why this is not bookkeeping: a package.json version is a RUNTIME value, not metadata. The bundled
-# CLI reads `../package.json` at module load and reports what it finds as `treasury --version` and in
-# `earn status`, so the plugin's copy reports plugin/package.json and an npm install reports the root
-# one. The manifests are what the marketplaces and the plugin hosts advertise. When any of them
-# disagree, a user installing "0.1.0" gets a CLI that introduces itself as something else: measured, the
-# identical bundle reported `0.1.1` from one tree and `0.1.0` from another, decided entirely by the
-# package.json beside it.
+# Why this is not bookkeeping: the bundled CLI inlines `package.json` at build time and reports that
+# version as `treasury --version` and in `earn status`, so a bundle built from one version and
+# shipped under manifests that say another introduces itself as something the user did not install.
+# The manifests are what the marketplaces and the plugin hosts advertise. This check is what ties them
+# to the bundle: when any of them disagree, a user installing "0.1.0" gets a CLI that reports something
+# else. (Before the version was inlined, the identical bundle reported `0.1.1` from one tree and `0.1.0`
+# from another, decided by the package.json beside it.)
 #
 # Usage: bash scripts/check-versions.sh            # all declarations must agree
 #        bash scripts/check-versions.sh 0.1.1      # ...and must equal this

scripts/sync-plugin.sh

@@ -5,8 +5,8 @@
 # its folder and, when that folder holds package.json AND a lockfile, runs `npm ci` inside it. The
 # root has both, so a root-level plugin would install every dependency (169 packages, measured)
 # for a CLI bundle that needs none of them. plugin/ therefore carries a version-only
-# package.json and no lockfile, plus the files the CLI reads beside itself at runtime
-# (../registry/vaults.json, ../package.json) and the licence files that must travel with it.
+# package.json and no lockfile, plus the registry and the licence files that must travel with it.
+# (The CLI no longer reads the registry or package.json beside itself: both are inlined into the bundle.)
 #
 # Every file below is a COPY of a file at the root, made in the same commit. Nothing is pinned to
 # a release and nothing is fetched: the check is byte identity against this tree.

src/registry.ts

@@ -1,18 +1,14 @@
-import { readFileSync, realpathSync } from "node:fs";
-import { fileURLToPath, pathToFileURL } from "node:url";
 import { registrySchema, erc4626Chassis, type Registry, type SolanaVaultEntry, type VaultEntry } from "./registry-schema.js";
+import registryJson from "../registry/vaults.json" with { type: "json" };
 import { EARN } from "./config/earn.js";
 import { CHAIN_INFO, chainIdForKey, SOLANA_INFO, supportedChainIds, type ChainKey, type SupportedChainId } from "./client.js";
 
-// From this file's real path, not a symlink's: see PACKAGE_JSON_URL in version.ts.
-const REGISTRY_URL = new URL("../registry/vaults.json", pathToFileURL(realpathSync(fileURLToPath(import.meta.url))));
-
 let cached: Registry | undefined;
 
 /** Parses and validates the shipped registry. Throws on any schema violation — a registry that does not parse is not a registry. */
 export function loadRegistry(): Registry {
   if (cached) return cached;
-  const raw = JSON.parse(readFileSync(fileURLToPath(REGISTRY_URL), "utf8")) as unknown;
+  const raw: unknown = registryJson;
   const parsed = registrySchema.parse(raw);
   // The schema holds one default per chain THAT HAS A VAULT, so it cannot see a registry with no
   // vault on the default chain at all — and then the discovery call itself is what fails, with an

src/version.ts

@@ -1,18 +1,12 @@
 /**
- * The package version, read from package.json — never a literal. The declared version follows the
+ * The package version, from package.json — never a literal. The declared version follows the
  * branch (v0.0.N ⇒ 0.0.N), and CI holds a version branch to that; anything that
  * reports a version reads it from here so a branch bump cannot leave a stale "0.1.0" behind
  * (a literal in server.ts once lagged package.json by two releases).
  *
- * Same resolution trick as registry.ts: this file sits at src/ top level and the bundle at dist/
- * top level, so `../package.json` is the package manifest at the repository root from both.
- * Resolved from this file's REAL path: under `--preserve-symlinks-main`, `import.meta.url` is the
- * path of a symlink to the bundle (npm's `.bin/treasury`), and `../package.json` from there is a
- * file in whatever directory holds the link.
+ * Imported, so it is inlined into the CLI bundle at build time: the bundle reports the version it was
+ * built from, whatever file sits beside it. `scripts/check-versions.sh` ties the manifests to it.
  */
-import { readFileSync, realpathSync } from "node:fs";
-import { fileURLToPath, pathToFileURL } from "node:url";
+import pkg from "../package.json" with { type: "json" };
 
-const PACKAGE_JSON_URL = new URL("../package.json", pathToFileURL(realpathSync(fileURLToPath(import.meta.url))));
-
-export const PACKAGE_VERSION: string = (JSON.parse(readFileSync(fileURLToPath(PACKAGE_JSON_URL), "utf8")) as { version: string }).version;
+export const PACKAGE_VERSION: string = pkg.version;

tests/boundary.unit.test.ts

@@ -159,6 +159,7 @@ const BUILTINS_ALLOWED_PER_FILE: Record<string, string[]> = {
   "tests/fork-solana.test.ts": ["child_process", "os"], //  spawns surfpool in a temporary directory; the key is generated per run
   "tests/registry-check.chain.test.ts": ["child_process"], // runs scripts/registry-check.ts against a mock chain
   "tests/entrypoint.unit.test.ts": ["child_process", "os"], // runs the committed bundle by symlink and by import; tmpdir for the symlink
+  "tests/embedded.unit.test.ts": ["child_process", "os"], //   runs the committed bundle and a consumer's re-bundle of it from a temporary folder
   "tests/licence.unit.test.ts": ["crypto"], //             pins the sha256 of the canonical Apache-2.0 text
   "tests/solana-kvault.unit.test.ts": ["crypto"], //       derives Anchor's instruction discriminators (sha256)
 };
@@ -170,6 +171,7 @@ const MAY_SPAWN: Record<string, string[]> = {
   "tests/fork-solana.test.ts": ["surfpool"],
   "tests/registry-check.chain.test.ts": ["npx"],
   "tests/entrypoint.unit.test.ts": ["node"],
+  "tests/embedded.unit.test.ts": ["node"],
 };
 
 const parse = (file: string) =>
@@ -236,6 +238,24 @@ const resolveTs = (from: string, spec: string): string | null => {
   return null;
 };
 
+/**
+ * The ONLY data files a module may import, by exact importing file and exact specifier. The registry and the
+ * package version are inlined into the bundle at build time instead of read from beside it, so the bundle
+ * needs nothing next to it. Nothing here is a pattern: a third JSON import, or either of these from another
+ * file, fails until it is added on purpose.
+ */
+const JSON_IMPORTS_ALLOWED: Record<string, string[]> = {
+  "src/registry.ts": ["../registry/vaults.json"],
+  "src/version.ts": ["../package.json"],
+};
+
+/** A `.json` specifier is data, not code: it must be allowlisted for this file and be a real file in this package. */
+function jsonReachProblem(file: string, spec: string): string | null {
+  if (!(JSON_IMPORTS_ALLOWED[rel(file)] ?? []).includes(spec)) return `${rel(file)} → ${spec} (a JSON import this file is not allowed)`;
+  const target = resolve(dirname(file), spec);
+  return existsSync(target) && target.startsWith(pkgRoot + sep) ? null : `${rel(file)} → ${spec} (does not resolve to a file in this package)`;
+}
+
 /** Walk the runtime import graph from the entry points. */
 function reachable(): { files: Set<string>; external: Set<string>; importers: Map<string, Set<string>>; escapes: string[] } {
   const files = new Set<string>();
@@ -256,6 +276,11 @@ function reachable(): { files: Set<string>; external: Set<string>; importers: Ma
         importers.set(spec, (importers.get(spec) ?? new Set()).add(rel(file)));
         continue;
       }
+      if (spec.endsWith(".json")) {
+        const problem = jsonReachProblem(file, spec);
+        if (problem) escapes.push(problem);
+        continue;
+      }
       const target = resolveTs(file, spec);
       if (!target) escapes.push(`${rel(file)} → ${spec} (does not resolve to a file in this package)`);
       else if (!target.startsWith(srcRoot + sep)) escapes.push(`${rel(file)} → ${spec} resolves OUTSIDE src/`);
@@ -324,6 +349,16 @@ describe("A. the runtime module graph cannot leave this package", () => {
     expect(escapes).toEqual([]);
   });
 
+  it("the JSON allowlist is exact: each entry is really imported, and anything else is refused (positive and negative controls)", () => {
+    for (const [file, specs] of Object.entries(JSON_IMPORTS_ALLOWED))
+      for (const spec of specs) expect(moduleReaches(resolve(pkgRoot, file)).specs, `${file} no longer imports ${spec}: drop the entry`).toContain(spec);
+    const registryTs = resolve(srcRoot, "registry.ts");
+    expect(jsonReachProblem(registryTs, "../registry/vaults.json")).toBeNull();
+    expect(jsonReachProblem(registryTs, "../package.json")).toMatch(/not allowed/); // another allowed file's JSON
+    expect(jsonReachProblem(resolve(srcRoot, "build.ts"), "../registry/vaults.json")).toMatch(/not allowed/); // the same JSON, wrong importer
+    expect(jsonReachProblem(registryTs, "../../outside.json")).toMatch(/not allowed/); // outside the package
+  });
+
   it("every external import is a DECLARED dependency or a node: builtin", () => {
     const declared = new Set(Object.keys(pkg.dependencies ?? {}));
     // By EXCLUSION, so nothing is exempt: a specifier is a declared dependency, or it is on the
@@ -511,7 +546,10 @@ describe("E. the AUDITED surface is every .ts under src/, scripts/ and tests/",
       const { specs, opaque } = moduleReaches(f);
       for (const o of opaque) bad.push(`${rel(f)}: unanalysable module reach — ${o}`);
       for (const spec of specs) {
-        if (spec.startsWith(".")) {
+        if (spec.endsWith(".json")) {
+          const problem = jsonReachProblem(f, spec);
+          if (problem) bad.push(problem);
+        } else if (spec.startsWith(".")) {
           const target = resolveTs(f, spec);
           if (!target || !target.startsWith(pkgRoot + sep)) bad.push(`${rel(f)} → ${spec} (escapes this package)`);
         } else {

tests/embedded.unit.test.ts

@@ -0,0 +1,113 @@
+import { describe, it, expect, afterAll } from "vitest";
+import { spawn } from "node:child_process";
+import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { build } from "tsup";
+
+/**
+ * The registry and the version are inlined into whatever bundles this package, so a bundle needs no file beside
+ * it, and a file that happens to sit beside it is never read. Two bundles are checked: the committed CLI copied
+ * to a bare folder, and a consumer's own bundle of `dist/index.js` (the library entry, which `tsc -b` emits
+ * and `npm test` builds first).
+ */
+
+const pkgRoot = resolve(__dirname, "..");
+const BUNDLE = join(pkgRoot, "dist/treasury.mjs");
+const shipped = JSON.parse(readFileSync(join(pkgRoot, "registry/vaults.json"), "utf8")) as { vaults: { symbol: string }[] };
+const VERSION = (JSON.parse(readFileSync(join(pkgRoot, "package.json"), "utf8")) as { version: string }).version;
+/** A registry and a manifest that must never be read: the right shape, different content. */
+const DECOY_REGISTRY = JSON.stringify({ vaults: [] });
+const DECOY_MANIFEST = JSON.stringify({ name: "decoy", version: "9.9.9" });
+
+const roots: string[] = [];
+const scratch = (): string => {
+  const root = mkdtempSync(join(tmpdir(), "treasury-embedded-"));
+  roots.push(root);
+  return root;
+};
+afterAll(() => {
+  for (const r of roots) rmSync(r, { recursive: true, force: true });
+});
+
+function node(args: string[]): Promise<{ stdout: string; stderr: string; code: number | null }> {
+  return new Promise((done) => {
+    const p = spawn("node", args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    p.stdout.on("data", (d) => (stdout += d));
+    p.stderr.on("data", (d) => (stderr += d));
+    const timer = setTimeout(() => p.kill("SIGKILL"), 20_000);
+    p.on("close", (code) => {
+      clearTimeout(timer);
+      done({ stdout, stderr, code });
+    });
+  });
+}
+
+describe("the registry and the version are inside the bundle", () => {
+  /** What `earn vaults` prints from the committed bundle run in place: the baseline the copies are compared with. */
+  const reference = async (): Promise<string> => (await node([BUNDLE, "earn", "vaults"])).stdout;
+
+  it("the committed CLI embeds the version and nothing else from package.json", () => {
+    // Anything else from the manifest would tie the committed bundle to edits that do not change the CLI: a
+    // dev-dependency bump would make dist/treasury.mjs stale, and CI's bundle check would fail on it.
+    const text = readFileSync(BUNDLE, "utf8");
+    const manifest = JSON.parse(readFileSync(join(pkgRoot, "package.json"), "utf8")) as { version: string; scripts: Record<string, string>; devDependencies: Record<string, string> };
+    expect(text).toContain(JSON.stringify(manifest.version));
+    expect(text).not.toMatch(/\bdevDependencies\b/);
+    for (const script of Object.values(manifest.scripts)) expect(text, `a package.json script is in the bundle: ${script}`).not.toContain(script);
+  });
+
+  it("the committed CLI, alone in a folder, reports its own version and the shipped registry", async () => {
+    const root = scratch();
+    copyFileSync(BUNDLE, join(root, "treasury.mjs"));
+    const v = await node([join(root, "treasury.mjs"), "--version"]);
+    expect({ out: v.stdout.trim(), err: v.stderr, code: v.code }).toEqual({ out: VERSION, err: "", code: 0 });
+    const vaults = await node([join(root, "treasury.mjs"), "earn", "vaults"]);
+    expect(vaults.code, vaults.stderr).toBe(0);
+    expect((JSON.parse(vaults.stdout) as { vaults: unknown[] }).vaults.length).toBeGreaterThan(0);
+    expect(vaults.stdout).toBe(await reference());
+  });
+
+  it("the same CLI ignores a registry and a package.json beside it (and beside its parent)", async () => {
+    const root = scratch();
+    mkdirSync(join(root, "dist"));
+    copyFileSync(BUNDLE, join(root, "dist/treasury.mjs"));
+    for (const dir of [root, join(root, "dist")]) {
+      mkdirSync(join(dir, "registry"), { recursive: true });
+      writeFileSync(join(dir, "registry/vaults.json"), DECOY_REGISTRY);
+      writeFileSync(join(dir, "package.json"), DECOY_MANIFEST);
+    }
+    const v = await node([join(root, "dist/treasury.mjs"), "--version"]);
+    expect(v.stdout.trim()).toBe(VERSION);
+    const vaults = await node([join(root, "dist/treasury.mjs"), "earn", "vaults"]);
+    expect(vaults.code, vaults.stderr).toBe(0);
+    expect((JSON.parse(vaults.stdout) as { vaults: unknown[] }).vaults.length).toBeGreaterThan(0);
+    expect(vaults.stdout).toBe(await reference());
+  });
+
+  it("a consumer's bundle of dist/index.js lists the shipped registry with no file beside it, and ignores a decoy", async () => {
+    expect(existsSync(join(pkgRoot, "dist/index.js")), "dist/index.js is missing: run `npm test` (it runs `tsc -b` first), not bare vitest").toBe(true);
+    const root = scratch();
+    const consumer = join(root, "consumer");
+    mkdirSync(join(consumer, "registry"), { recursive: true });
+    writeFileSync(join(consumer, "registry/vaults.json"), DECOY_REGISTRY);
+    writeFileSync(join(consumer, "package.json"), DECOY_MANIFEST);
+    writeFileSync(join(root, "entry.mjs"), `import { listVaults } from ${JSON.stringify(join(pkgRoot, "dist/index.js"))};\nconsole.log(JSON.stringify(listVaults().map((v) => v.symbol)));\n`);
+    await build({
+      config: false,
+      entry: { bundle: join(root, "entry.mjs") },
+      outDir: consumer,
+      format: ["esm"],
+      outExtension: () => ({ js: ".mjs" }),
+      platform: "node",
+      target: "node22",
+      noExternal: [/.*/],
+      silent: true,
+    });
+    const r = await node([join(consumer, "bundle.mjs")]);
+    expect(r.code, r.stderr).toBe(0);
+    expect(JSON.parse(r.stdout)).toEqual(shipped.vaults.map((v) => v.symbol));
+  }, 60_000);
+});