diff --git a/Cargo.lock b/Cargo.lock index 93e63b90..6966641c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -719,6 +719,7 @@ dependencies = [ "flate2", "futures", "futures-util", + "handlebars", "log", "log-fastly", "serde", diff --git a/crates/edgezero-adapter-fastly/Cargo.toml b/crates/edgezero-adapter-fastly/Cargo.toml index 08c4832d..02ae5a83 100644 --- a/crates/edgezero-adapter-fastly/Cargo.toml +++ b/crates/edgezero-adapter-fastly/Cargo.toml @@ -49,4 +49,5 @@ walkdir = { workspace = true, optional = true } [dev-dependencies] edgezero-core = { path = "../edgezero-core", features = ["test-utils"] } +handlebars = { workspace = true } tempfile = { workspace = true } diff --git a/crates/edgezero-adapter-fastly/src/chunked_config.rs b/crates/edgezero-adapter-fastly/src/chunked_config.rs index 6c8b0937..50e9569d 100644 --- a/crates/edgezero-adapter-fastly/src/chunked_config.rs +++ b/crates/edgezero-adapter-fastly/src/chunked_config.rs @@ -21,19 +21,27 @@ use sha2::{Digest as _, Sha256}; -/// Per-entry value limit enforced by Fastly Config Store. Used by the -/// CLI writer to gate direct-vs-chunked storage; the runtime resolver -/// reads chunk lengths from the pointer struct, not this constant. -#[cfg(any(feature = "cli", test))] +/// Per-entry value limit enforced by Fastly Config Store. Used by the CLI writer +/// to gate direct-vs-chunked storage, and by the pointer validator (which both +/// the CLI and the runtime resolver call) — a chunked envelope is always larger +/// than this, so a pointer claiming otherwise is not one this writer emitted. pub(crate) const FASTLY_CONFIG_ENTRY_LIMIT: usize = 8_000; -/// Target payload size per chunk (kept under the entry limit to leave -/// room for the key and any protocol overhead). CLI writer only. -#[cfg(any(feature = "cli", test))] +/// Maximum length of a Config Store KEY, in CHARACTERS. Fastly's docs disagree +/// (the guide says 255, the API reference says 256); we take the STRICTER 255 so +/// a key we emit is valid under either. The writer must not emit a physical key +/// longer than this — a derived chunk key adds ~85 characters to the root, so a +/// near-limit root would otherwise fail only mid-write. Also used by the pointer +/// validator (CLI and runtime): a pointer referencing an over-limit key is not +/// one this writer could have produced. +pub(crate) const FASTLY_CONFIG_KEY_LIMIT: usize = 255; +/// Target payload size per chunk (kept under the entry limit to leave room for +/// the key and any protocol overhead). The writer never emits a chunk larger +/// than this, so the pointer validator (CLI and runtime) uses it as an upper +/// bound on any single chunk's declared length. pub(crate) const CHUNK_PAYLOAD_TARGET: usize = 7_000; -/// Infix inserted between the root key and the content-address in a -/// chunk key: `.__edgezero_chunks..`. CLI writer -/// only; the resolver reads chunk keys from the pointer struct. -#[cfg(any(feature = "cli", test))] +/// Infix inserted between the root key and the content-address in a chunk key: +/// `.__edgezero_chunks..`. Used by the writer and by the +/// pointer validator (CLI and runtime) to recognise a canonical chunk key. pub(crate) const CHUNK_KEY_INFIX: &str = ".__edgezero_chunks."; /// `edgezero_kind` discriminant stored in the pointer JSON. Used by /// BOTH the writer (when serialising the pointer) AND the resolver @@ -61,19 +69,46 @@ struct FastlyChunkRef { sha256: String, } +/// A chunk reference from a validated pointer, for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) struct GcChunkRef { + pub key: String, + pub len: usize, + pub sha256: String, +} + +/// A validated v1 pointer's contents, for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) struct GcPointer { + /// Validated: non-empty, one generation, dense `0..n-1` in order. + pub chunks: Vec, + pub envelope_len: usize, + pub envelope_sha256: String, +} + +/// What a root's value IS, once classified for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) enum GcRootValue { + /// A valid v1 chunk pointer. Its METADATA is validated; its CONTENT must + /// still be verified against the store -- see [`gc_verify_generation`]. + Chunked(GcPointer), + /// A valid, integrity-checked envelope stored inline. References no chunks. + Direct, +} + // --------------------------------------------------------------------------- // Public helpers // --------------------------------------------------------------------------- /// Compute the lowercase-hex SHA-256 of `bytes`. -fn sha256_hex(bytes: &[u8]) -> String { +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } /// Prepare the physical Config Store entries for a single logical /// `(root_key, envelope_json)` pair. /// -/// - If `envelope_json.len() <= 8_000`: returns one `(root_key, envelope_json)` tuple unchanged. +/// - If `envelope_json` is at most 8 000 BYTES (a conservative UTF-8 byte check): returns one `(root_key, envelope_json)` tuple unchanged. /// - Otherwise: splits into UTF-8-safe 7 000-byte chunks, returns chunk /// entries first and the root pointer entry last. /// @@ -83,11 +118,54 @@ fn sha256_hex(bytes: &[u8]) -> String { /// /// Returns an error string if the pointer JSON itself would exceed 8 000 /// characters (extremely unlikely in practice; recommends restructuring). +/// Reject a physical Config Store key that exceeds the store's key limit, +/// before any write is attempted. +#[cfg(any(feature = "cli", test))] +fn check_config_key_len(key: &str) -> Result<(), String> { + // CHARACTERS, not bytes: Fastly's limit is a character count, so a non-ASCII + // `--key` must be measured by `chars().count()`, not `len()` (UTF-8 bytes). + let char_len = key.chars().count(); + if char_len > FASTLY_CONFIG_KEY_LIMIT { + return Err(format!( + "config-store key is {char_len} characters, over Fastly's \ + {FASTLY_CONFIG_KEY_LIMIT}-character limit. Chunked storage derives keys of the form \ + `.__edgezero_chunks.<64-char-sha>.`, adding ~85 characters to the root, \ + so a long store id or `--key` override can push the derived key past the limit. Use a \ + shorter store id / `--key`." + )); + } + Ok(()) +} + #[cfg(any(feature = "cli", test))] pub(crate) fn prepare_fastly_config_entries( root_key: &str, envelope_json: &str, ) -> Result, String> { + // An EMPTY root key is writer-valid but resolver-INVALID: canonical chunk + // parsing rejects an empty root, so a chunked empty-key push would write + // chunks that can never be reassembled (and a partial cloud push could commit + // them before the final pointer write fails). Reject it before any I/O. + if root_key.is_empty() { + return Err( + "config key is empty; an empty key cannot be resolved at runtime. Provide a store id \ + or a non-empty `--key`." + .to_owned(), + ); + } + // The root key itself is a physical key on both paths (the direct entry and + // the pointer entry), so it must fit the store's key limit before anything + // is written. + check_config_key_len(root_key)?; + + // Direct-vs-chunked uses a conservative UTF-8 BYTE-count check (spec + // 2026-06-16 permits this and the merge-base writer used it). Byte length is + // >= character length, so this chunks whenever a stricter character check + // would AND for a few multibyte values that fit the character cap — which is + // safe (it never over-stores) and, crucially, is what EXISTING v1 pointers + // were written against. A character-based threshold would leave a value that + // is <= 8000 chars but > 8000 bytes stored directly, and REJECT the existing + // chunked pointer for it on read (an HTTP 500) — a v1 read-compat break. if envelope_json.len() <= FASTLY_CONFIG_ENTRY_LIMIT { return Ok(vec![(root_key.to_owned(), envelope_json.to_owned())]); } @@ -105,16 +183,10 @@ pub(crate) fn prepare_fastly_config_entries( }) .unwrap_or_default(); - // Split env_bytes into UTF-8-safe chunks of at most CHUNK_PAYLOAD_TARGET bytes. + // Split env_bytes into UTF-8-safe chunks of at most CHUNK_PAYLOAD_TARGET + // bytes, using the shared span computation the resolver replays. let mut chunks: Vec<(String, String, usize)> = Vec::new(); // (key, value, len) - let mut start = 0_usize; - let mut idx = 0_usize; - while start < env_bytes.len() { - // Walk forward by CHUNK_PAYLOAD_TARGET bytes, then retreat to the - // last codepoint boundary so we never split a multi-byte codepoint. - let end_raw = (start.saturating_add(CHUNK_PAYLOAD_TARGET)).min(env_bytes.len()); - // Find the largest valid UTF-8 boundary <= end_raw. - let end = find_utf8_boundary(envelope_json, start, end_raw); + for (idx, (start, end)) in writer_chunk_spans(envelope_json).into_iter().enumerate() { let chunk_bytes = env_bytes.get(start..end).ok_or_else(|| { format!("chunk boundary [{start}..{end}] out of range for `{root_key}`") })?; @@ -122,9 +194,12 @@ pub(crate) fn prepare_fastly_config_entries( format!("chunk [{start}..{end}] for `{root_key}` is not valid UTF-8: {err}") })?; let chunk_key = format!("{root_key}{CHUNK_KEY_INFIX}{envelope_sha256}.{idx}"); + // A chunk key adds the infix + a 64-char SHA + an index to the root key + // (~85 chars). A root that is itself valid can push the derived key past + // the store limit; reject it here rather than fail mid-write with some + // chunks already committed. + check_config_key_len(&chunk_key)?; chunks.push((chunk_key, chunk_str.to_owned(), chunk_bytes.len())); - start = end; - idx = idx.saturating_add(1); } // Build chunk refs with per-chunk SHAs. @@ -171,8 +246,7 @@ pub(crate) fn prepare_fastly_config_entries( /// Find the largest byte offset `<= end_raw` that is a valid UTF-8 /// codepoint boundary within `src`. `start` is used as a hint so we -/// don't scan the entire string from zero each time. CLI writer only. -#[cfg(any(feature = "cli", test))] +/// don't scan the entire string from zero each time. fn find_utf8_boundary(src: &str, start: usize, end_raw: usize) -> usize { if end_raw >= src.len() { return src.len(); @@ -185,6 +259,74 @@ fn find_utf8_boundary(src: &str, start: usize, end_raw: usize) -> usize { boundary } +/// The exact byte spans this writer splits an over-limit envelope into: walk +/// forward `CHUNK_PAYLOAD_TARGET` bytes, then retreat to the previous UTF-8 +/// codepoint boundary so no chunk ever splits a codepoint. +/// +/// SINGLE SOURCE OF TRUTH for the split layout. `prepare_fastly_config_entries` +/// builds its chunks from these spans, and the resolver replays them against a +/// reconstructed envelope to prove the pointer's boundaries are exactly the ones +/// this writer emits — so the two can never drift. Un-gated on purpose: the +/// runtime resolver (no `cli` feature) needs it for that check. +fn writer_chunk_spans(envelope_json: &str) -> Vec<(usize, usize)> { + let len = envelope_json.len(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut start = 0_usize; + while start < len { + let end_raw = (start.saturating_add(CHUNK_PAYLOAD_TARGET)).min(len); + let end = find_utf8_boundary(envelope_json, start, end_raw); + spans.push((start, end)); + start = end; + } + spans +} + +/// Prove a pointer's chunk boundaries are EXACTLY the ones this writer emits for +/// the reconstructed envelope. +/// +/// Called after the per-chunk and whole-envelope SHAs have verified, so the +/// reconstructed BYTES are already proven correct; this pins the LAYOUT. Replay +/// the writer's span computation: the chunk count and every chunk's byte length +/// must match the pointer. That rejects a hand-authored pointer which reassembles +/// to the same bytes along boundaries the writer would never choose, so the +/// resolver accepts only writer-produced layouts -- the same guarantee +/// `prove_generation` enforces before a GC delete. +/// +/// Comparing LENGTHS is sufficient: the SHAs already pinned the bytes, so equal +/// boundaries pin each chunk to the writer's own chunk. +/// +/// Diagnostics name a chunk's POSITION, never its key: `chunks[].key` is +/// pointer-controlled and these messages are logged verbatim. +fn verify_writer_split_layout( + root_key: &str, + reconstructed: &str, + pointer: &FastlyChunkPointer, +) -> Result<(), String> { + let spans = writer_chunk_spans(reconstructed); + if spans.len() != pointer.chunks.len() { + return Err(format!( + "chunk pointer at `{root_key}` names {} chunk(s), but this writer would split its \ + own reconstructed envelope into {}", + pointer.chunks.len(), + spans.len() + )); + } + for (position, (&(start, end), chunk_ref)) in + spans.iter().zip(pointer.chunks.iter()).enumerate() + { + let expected_len = end.saturating_sub(start); + if expected_len != chunk_ref.len { + return Err(format!( + "chunk {position} referenced by pointer at `{root_key}` records {} bytes, but this \ + writer splits the reconstructed envelope into {expected_len} bytes at that \ + boundary", + chunk_ref.len + )); + } + } + Ok(()) +} + /// Resolve a logical `(root_key, root_value)` pair from the store, /// handling both direct `BlobEnvelope` values and chunk-pointer values. /// @@ -216,25 +358,41 @@ where // --- Path 1: direct BlobEnvelope --- if let Ok(envelope) = serde_json::from_str::(&root_value) { - envelope.verify().map_err(|err| { - format!("BlobEnvelope at key `{root_key}` failed integrity check: {err}") + // Redacted: `verify()`'s message names the stored hashes, which are + // config-controlled strings that may hold anything. + envelope.verify().map_err(|_err| { + format!( + "BlobEnvelope at key `{root_key}` failed its integrity check (details redacted)" + ) })?; return Ok(root_value); } // --- Path 2: chunk pointer --- + // The pointer entry itself is a single Config Store value, so the writer can + // never emit one larger than the entry limit. Reject an over-limit pointer up + // front — this bounds how many chunk refs it can carry (hence the fetch + // fan-out) before any parsing. + if root_value.len() > FASTLY_CONFIG_ENTRY_LIMIT { + return Err(format!( + "chunk pointer at `{root_key}` is larger than the {FASTLY_CONFIG_ENTRY_LIMIT}-character \ + entry limit and so is not one this writer could have stored" + )); + } let pointer: FastlyChunkPointer = serde_json::from_str(&root_value).map_err(|err| { format!( "value at key `{root_key}` is neither a valid BlobEnvelope nor a valid \ - chunk pointer: {err}" + chunk pointer ({})", + redact_json_err(&err) ) })?; if pointer.edgezero_kind != POINTER_KIND { + // The stored `edgezero_kind` is not echoed: it is a value-controlled + // string on a path whose diagnostics are logged. return Err(format!( - "chunk pointer at `{root_key}` has unknown edgezero_kind \ - `{}`; expected `{POINTER_KIND}`", - pointer.edgezero_kind + "chunk pointer at `{root_key}` has an unknown `edgezero_kind` (value redacted); \ + expected `{POINTER_KIND}`" )); } if pointer.version != 1 { @@ -244,35 +402,51 @@ where pointer.version )); } + // Validate the pointer METADATA before fetching anything: a shape the writer + // could never emit (non-canonical keys, mixed generations, gaps, per-chunk + // lengths over the payload target, a sum that disagrees with `envelope_len`, + // or an `envelope_len` small enough to have been stored directly) is + // rejected up front rather than driving a fan-out of chunk fetches whose + // hashes could only fail at the end. Same validator the CLI/GC path uses. + validate_pointer_chunks(root_key, &pointer)?; // Fetch, verify, and concatenate all chunks. - let mut reconstructed = String::with_capacity(pointer.envelope_len); - for chunk_ref in &pointer.chunks { - let chunk_value = fetch(&chunk_ref.key)?.ok_or_else(|| { - format!( - "missing chunk `{}` referenced by pointer at `{root_key}`", - chunk_ref.key - ) - })?; + // + // NOT `with_capacity(pointer.envelope_len)`: that length is untrusted stored + // metadata, and this runs in the edge guest. A pointer declaring `usize::MAX` + // would abort the worker on a capacity overflow before any of the checks + // below could reject it. Grow from the bytes we actually fetch instead. + let mut reconstructed = String::new(); + for (position, chunk_ref) in pointer.chunks.iter().enumerate() { + let chunk_value = fetch(&chunk_ref.key) + .map_err(|_err| { + // The callback's error carries the chunk KEY (pointer-controlled), + // so it is not propagated verbatim. Position locates the fault. + format!( + "chunk {position} referenced by pointer at `{root_key}` could not be fetched \ + (details redacted)" + ) + })? + .ok_or_else(|| { + format!("missing chunk {position} referenced by pointer at `{root_key}`") + })?; // Verify length. if chunk_value.len() != chunk_ref.len { return Err(format!( - "chunk `{}` (referenced by `{root_key}`) has length {} but pointer \ + "chunk {position} (referenced by `{root_key}`) has length {} but the pointer \ records {}", - chunk_ref.key, chunk_value.len(), chunk_ref.len, )); } - // Verify SHA. - let actual_sha = sha256_hex(chunk_value.as_bytes()); - if actual_sha != chunk_ref.sha256 { + // Verify SHA. Neither hash is echoed: the expected one comes from the + // stored pointer, so it is value-controlled. + if sha256_hex(chunk_value.as_bytes()) != chunk_ref.sha256 { return Err(format!( - "chunk `{}` (referenced by `{root_key}`) SHA mismatch: \ - expected `{}`, got `{actual_sha}`", - chunk_ref.key, chunk_ref.sha256, + "chunk {position} (referenced by `{root_key}`) does not match the SHA-256 the \ + pointer records for it (hashes redacted)" )); } @@ -292,26 +466,1136 @@ where // Verify total envelope SHA. let actual_env_sha = sha256_hex(reconstructed.as_bytes()); if actual_env_sha != pointer.envelope_sha256 { + // Neither SHA is echoed: the expected one is `pointer.envelope_sha256`, + // a stored, value-controlled string that a malformed pointer can set to + // anything (a secret), and this runs on the read path where diagnostics + // are logged. return Err(format!( - "reconstructed envelope for `{root_key}` SHA mismatch: \ - expected `{}`, got `{actual_env_sha}`", - pointer.envelope_sha256, + "reconstructed envelope for `{root_key}` does not match the pointer's \ + `envelope_sha256` (hashes redacted)" )); } + verify_writer_split_layout(root_key, &reconstructed, &pointer)?; + + // Parse AND verify the inner envelope, exactly as the direct path does. + // + // The outer checks above only prove the chunks reassemble to the bytes the + // pointer names; they say nothing about the `BlobEnvelope` INSIDE. Without + // this, a reconstructed value whose embedded `sha256` is wrong (a secret) + // sails through the resolver, and core's own `verify()` then formats that + // stored hash into an HTTP 500. Redact to a category so the resolver + // guarantees -- on BOTH paths -- that whatever it returns is a verified + // envelope and no stored value escapes. + let envelope: BlobEnvelope = serde_json::from_str(&reconstructed).map_err(|err| { + format!( + "reconstructed value at key `{root_key}` is not a valid config envelope ({})", + redact_json_err(&err) + ) + })?; + envelope.verify().map_err(|_err| { + format!( + "reconstructed envelope at key `{root_key}` failed its integrity check (details \ + redacted -- the stored hashes are value-controlled)" + ) + })?; + Ok(reconstructed) } +/// Validate a prior root value and return the chunk keys it referenced, +/// scoped to `root_key`'s own chunk namespace. Used only for chunk GC on +/// re-push (Stage 7 writeback). +/// +/// Parse as `serde_json::Value` FIRST so a pointer-kind value with +/// missing/invalid fields still reaches the warning path instead of being +/// silently dropped by a failed struct deserialize. +/// +/// Returns: +/// - `Ok(keys)` -- value is a valid v1 chunk pointer; `keys` are its +/// `chunks[].key` entries (all confirmed to match +/// `"{root_key}{CHUNK_KEY_INFIX}"`). +/// - `Ok(vec![])` -- value is a direct `BlobEnvelope`, absent, or not +/// pointer-shaped at all (normal: first push, or was-direct). Silent. +/// - `Err(msg)` -- value IS pointer-kind (`edgezero_kind == POINTER_KIND`) +/// but fails validation: malformed, unsupported `version`, or a +/// referenced key falls outside this root's chunk prefix. Callers log +/// `msg` as a warning and skip GC for this root (delete nothing). +#[cfg(any(feature = "cli", test))] +pub(crate) fn prior_chunk_keys(root_key: &str, raw: &str) -> Result, String> { + // 1. Parse loosely. Not-JSON, or not our pointer kind => silent. + let value: serde_json::Value = match serde_json::from_str(raw) { + Ok(value) => value, + Err(_) => return Ok(Vec::new()), + }; + if value + .get("edgezero_kind") + .and_then(serde_json::Value::as_str) + != Some(POINTER_KIND) + { + // Direct BlobEnvelope, unrelated JSON, or first push. + return Ok(Vec::new()); + } + + // 2. It IS pointer-kind: from here every failure WARNS (Err), never silent. + // The serde error is REDACTED -- its Display quotes the offending value + // ("invalid type: string \"hunter2\", expected u8"), and a stored config + // may hold credentials. Position + category only. + let pointer: FastlyChunkPointer = serde_json::from_value(value).map_err(|err| { + format!( + "prior chunk pointer at `{root_key}` is malformed ({}); skipping chunk GC", + redact_json_err(&err) + ) + })?; + if pointer.version != 1 { + return Err(format!( + "prior chunk pointer at `{root_key}` has unsupported version {}; skipping chunk GC", + pointer.version + )); + } + + // 3. The pointer must be INTERNALLY CONSISTENT, not merely well-typed. + // `chunks` is the authoritative live set on the GC path, so a pointer + // that parses but under-reports its chunks (empty list, an omitted + // index, a stale generation mixed in, lengths that cannot add up to the + // envelope) would leave real live chunks looking orphaned. Only accept a + // chunk list this writer could actually have emitted. + validate_pointer_chunks(root_key, &pointer) + .map_err(|err| format!("{err}; skipping chunk GC"))?; + + Ok(pointer.chunks.into_iter().map(|chunk| chunk.key).collect()) +} + +/// Describe a `serde_json` failure WITHOUT quoting the offending value. +/// +/// `serde_json::Error`'s `Display` embeds the input that failed to parse, and +/// these diagnostics are logged verbatim. Stored app config may hold secrets, so +/// only the position and category ever escape. +/// +/// Unconditional: the runtime resolver needs it too. A guest reading a malformed +/// pointer must not log the value either — that path runs on every request. +fn redact_json_err(err: &serde_json::Error) -> String { + use serde_json::error::Category; + + let category = match err.classify() { + Category::Io => "io", + Category::Syntax => "syntax", + Category::Data => "schema", + Category::Eof => "unexpected end of input", + }; + format!( + "{category} error at line {} column {} -- value redacted", + err.line(), + err.column() + ) +} + +/// Is this pointer's chunk list one `prepare_fastly_config_entries` could have +/// produced? Anything else is treated as unreadable rather than authoritative. +/// +/// Checks, in order: non-empty; every key canonical for this root; a SINGLE +/// generation matching `envelope_sha256`; indexes exactly `0..n-1`, each once, +/// in order; per-chunk lengths within what the writer emits; and +/// `sum(len) == envelope_len`, computed with CHECKED arithmetic. +/// +/// Called on BOTH the CLI/GC path and the runtime resolver, so a pointer shape +/// the writer could never emit is rejected everywhere, not just during GC. +/// +/// **Diagnostics name a chunk's POSITION, never its key.** `chunks[].key` is +/// pointer-controlled and, on this path, not yet validated -- a malformed +/// pointer can carry any string at all there (`prod-db-password=hunter2`), and +/// these messages are logged verbatim. +/// +/// **Lengths are untrusted input.** They are attacker-supplied `usize`s, so they +/// are bounded against what the writer can actually emit and summed with +/// `checked_add`. Saturating arithmetic would let a metadata-consistent pointer +/// declare `usize::MAX` and survive validation. +fn validate_pointer_chunks(root_key: &str, pointer: &FastlyChunkPointer) -> Result<(), String> { + let bad = |detail: &str| format!("chunk pointer at `{root_key}` {detail}"); + + if pointer.chunks.is_empty() { + // An oversized envelope always splits into >= 2 chunks, so an empty + // list is never something we wrote -- and it would report "references + // nothing", orphaning the real chunks. + return Err(bad("references no chunks at all")); + } + + let mut total_len = 0_usize; + for (position, chunk_ref) in pointer.chunks.iter().enumerate() { + // Every referenced key must be a CANONICAL chunk key of this root -- + // the same validator that gates deletion. + let Some((generation, index)) = chunk_key_parts(root_key, &chunk_ref.key) else { + return Err(bad(&format!( + "references a non-canonical chunk key at position {position}" + ))); + }; + // The physical key must fit the store's key limit — the writer never + // emits one that does not, so an over-limit key is not ours. + if chunk_ref.key.chars().count() > FASTLY_CONFIG_KEY_LIMIT { + return Err(bad(&format!( + "references a chunk key at position {position} longer than the \ + {FASTLY_CONFIG_KEY_LIMIT}-character store limit" + ))); + } + // One pointer names exactly one generation. A mixed list means some + // other generation's chunks are being reported as this one's. + if generation != pointer.envelope_sha256 { + return Err(bad(&format!( + "references a chunk at position {position} belonging to a different generation \ + than the pointer's own `envelope_sha256`" + ))); + } + // Indexes are dense, unique and ordered: 0, 1, ... n-1. This is what + // rejects an omitted, duplicated or reordered index -- each of which + // would silently shrink the live set. + if index != position { + return Err(bad(&format!( + "references chunk index {index} at position {position}, but a chunk list must be \ + indexed 0..n-1 with no gaps, duplicates or reordering" + ))); + } + if chunk_ref.len == 0 { + return Err(bad(&format!( + "references a zero-length chunk at position {position}" + ))); + } + // The writer never emits a chunk larger than its payload target, so a + // bigger one is not ours -- and this is what stops an absurd declared + // length ever reaching an allocation. + if chunk_ref.len > CHUNK_PAYLOAD_TARGET { + return Err(bad(&format!( + "references a chunk at position {position} declaring {} bytes, more than the \ + {CHUNK_PAYLOAD_TARGET}-byte maximum this writer emits", + chunk_ref.len + ))); + } + // Every chunk EXCEPT the last is a full payload: the writer walks + // `CHUNK_PAYLOAD_TARGET` bytes then retreats at most 3 to the previous + // UTF-8 boundary (the longest codepoint is 4 bytes), so a non-last chunk + // is always >= CHUNK_PAYLOAD_TARGET - 3. This pins the split layout to + // what the writer emits and bounds the chunk count (hence the runtime + // fetch fan-out) to about `envelope_len / CHUNK_PAYLOAD_TARGET`; a + // hand-authored pointer of many tiny chunks is rejected. + let is_last = position == pointer.chunks.len().saturating_sub(1); + if !is_last && chunk_ref.len < CHUNK_PAYLOAD_TARGET.saturating_sub(3) { + return Err(bad(&format!( + "references a non-final chunk at position {position} of only {} bytes; this writer \ + fills every chunk but the last to within 3 bytes of {CHUNK_PAYLOAD_TARGET}", + chunk_ref.len + ))); + } + total_len = total_len.checked_add(chunk_ref.len).ok_or_else(|| { + bad("declares chunk lengths that overflow when summed (not a real envelope)") + })?; + } + + // The parts must add up to the whole. If they don't, the pointer does not + // describe the envelope it claims to, so its chunk list is not trustworthy. + if total_len != pointer.envelope_len { + return Err(bad(&format!( + "declares `envelope_len` {} but its chunk lengths sum to {total_len}", + pointer.envelope_len + ))); + } + // We only chunk what does not fit directly, so a pointer describing an + // envelope that WOULD have fit is not one we wrote. + if pointer.envelope_len <= FASTLY_CONFIG_ENTRY_LIMIT { + return Err(bad(&format!( + "declares an envelope of {} bytes, which fits the {FASTLY_CONFIG_ENTRY_LIMIT}-byte \ + entry limit and so would never have been chunked by this writer", + pointer.envelope_len + ))); + } + + Ok(()) +} + +/// Does this value announce itself as a chunk pointer? +/// +/// A cheap discriminant for "this entry is a ROOT, wherever it happens to live". +/// GC must not decide root-vs-chunk by KEY SHAPE: the runtime resolver follows +/// whatever pointer it is handed, so a pointer parked at a chunk-shaped key +/// still makes its references live. Chunk payloads are raw envelope fragments +/// and essentially never parse as JSON, let alone carry `edgezero_kind`. +#[cfg(any(feature = "cli", test))] +pub(crate) fn value_is_pointer_kind(raw: &str) -> bool { + serde_json::from_str::(raw).is_ok_and(|value| { + value + .get("edgezero_kind") + .and_then(serde_json::Value::as_str) + == Some(POINTER_KIND) + }) +} + +/// Classify a ROOT value for `config gc` — the live-set input on a DESTRUCTIVE +/// path, so it is fail-closed. Unlike [`prior_chunk_keys`] (which is lenient for +/// the push path and treats unrelated/empty JSON as "references nothing"), this +/// accepts ONLY: +/// +/// - a valid, integrity-checked direct `BlobEnvelope` → `Direct`; +/// - a valid v1 chunk pointer → `Chunked` (metadata validated). +/// +/// Anything else — an empty string, a truncated/partial value, unrelated JSON, +/// or a pointer that fails validation — is `Err`. A root we cannot classify has +/// unknown references, so we must reclaim NOTHING rather than treat its live +/// chunks as orphaned. +/// +/// **Metadata is not proof.** A `Chunked` result says the pointer is +/// self-consistent, NOT that it honestly describes its generation: a pointer can +/// drop its last chunk ref AND restate `envelope_len` as the remaining sum, and +/// every metadata check still passes while the dropped chunk silently leaves the +/// live set. Callers MUST reconstruct the referenced chunks and put them through +/// [`gc_verify_generation`] before trusting the live set. +#[cfg(any(feature = "cli", test))] +pub(crate) fn gc_classify_root(root_key: &str, raw: &str) -> Result { + use edgezero_core::blob_envelope::BlobEnvelope; + + if raw.trim().is_empty() { + return Err(format!( + "root `{root_key}` has an empty value; refusing to reclaim (cannot tell what it references)" + )); + } + // Every diagnostic below is REDACTED: serde's Display quotes the offending + // input, `verify()`'s names the stored hashes, and BOTH are attacker- or + // config-controlled strings that may hold credentials. + let value: serde_json::Value = serde_json::from_str(raw).map_err(|err| { + format!( + "root `{root_key}` is not valid JSON ({}); refusing to reclaim", + redact_json_err(&err) + ) + })?; + if value + .get("edgezero_kind") + .and_then(serde_json::Value::as_str) + == Some(POINTER_KIND) + { + let pointer: FastlyChunkPointer = serde_json::from_value(value).map_err(|err| { + format!( + "root `{root_key}` is a chunk pointer but is malformed ({}); refusing to reclaim", + redact_json_err(&err) + ) + })?; + if pointer.version != 1 { + return Err(format!( + "root `{root_key}` chunk pointer has unsupported version {}; refusing to reclaim", + pointer.version + )); + } + validate_pointer_chunks(root_key, &pointer)?; + return Ok(GcRootValue::Chunked(GcPointer { + chunks: pointer + .chunks + .into_iter() + .map(|chunk| GcChunkRef { + key: chunk.key, + len: chunk.len, + sha256: chunk.sha256, + }) + .collect(), + envelope_len: pointer.envelope_len, + envelope_sha256: pointer.envelope_sha256, + })); + } + // Otherwise it must be a valid, integrity-checked direct envelope. + let envelope: BlobEnvelope = serde_json::from_value(value).map_err(|err| { + format!( + "root `{root_key}` is neither a valid chunk pointer nor a valid config envelope ({}); \ + refusing to reclaim", + redact_json_err(&err) + ) + })?; + envelope.verify().map_err(|_err| { + format!( + "root `{root_key}` envelope failed its integrity check (details redacted -- the \ + stored hashes are config-controlled); refusing to reclaim" + ) + })?; + Ok(GcRootValue::Direct) +} + +/// Check that `assembled` really is the generation named by `generation_sha`. +/// +/// A chunk key embeds the SHA-256 of the WHOLE envelope it belongs to, so +/// reassembling a chunk set either reproduces the content-address its own keys +/// name, or it does not. Nothing an inconsistent store says about lengths, +/// indexes or counts survives this. +/// +/// **This is a consistency check, NOT proof of authorship — do not mistake it +/// for one.** Content-addressing is not a signature: anyone can pick an envelope +/// E, compute `H = sha256(E)`, and store the pieces under `H`. No preimage +/// attack is involved, so passing here says the bytes are internally consistent, +/// not that `EdgeZero` wrote them. Callers deciding to DELETE must additionally +/// require the entries to be byte-identical to this writer's own output — see +/// `prove_generation` in `cli.rs`. +/// +/// Used for BOTH destructive decisions: +/// - a live pointer's chunks must reconstruct the envelope it claims (else its +/// chunk list is not the true live set); +/// - a delete candidate's generation must reconstruct a real envelope (a +/// necessary, but not sufficient, condition for reclaiming it). +#[cfg(any(feature = "cli", test))] +pub(crate) fn gc_verify_generation(generation_sha: &str, assembled: &str) -> Result<(), String> { + use edgezero_core::blob_envelope::BlobEnvelope; + + // Neither hash is echoed: `generation_sha` comes from the chunk KEYS + // (pointer-controlled) and `actual` is derived from the reassembled config + // bytes. Both are stored/derived strings, and this message is logged. + if sha256_hex(assembled.as_bytes()) != generation_sha { + return Err( + "the chunk set does not reassemble to the generation its own keys name (hashes \ + redacted), so these chunks are not the generation they claim" + .to_owned(), + ); + } + // Content-addressing proves the bytes are SELF-CONSISTENT, not that EdgeZero + // wrote them (a forger can content-address their own envelope -- see this + // function's doc comment). Parsing here just confirms the reassembled bytes + // are a config envelope at all; the authorship-adjacent decision (delete or + // not) is `prove_generation`'s round-trip against the writer, in cli.rs. + let envelope: BlobEnvelope = serde_json::from_str(assembled).map_err(|err| { + format!( + "the chunk set reassembles to its content-address but does not parse as a config \ + envelope ({})", + redact_json_err(&err) + ) + })?; + envelope.verify().map_err(|_err| { + "the chunk set reassembles to a config envelope that fails its own integrity check \ + (details redacted -- the stored hashes are config-controlled)" + .to_owned() + })?; + Ok(()) +} + +/// Extract the content-address (envelope SHA) of the generation a chunk key +/// belongs to, for `root_key`. Returns `None` when `key` is not a well-formed +/// chunk key of that root. +/// +/// This is how reclamation groups the store's ACTUAL keys into generations. It +/// validates the shape (`.__edgezero_chunks..`) rather +/// than trusting it: a hand-edited or foreign key never becomes a delete target. +#[cfg(any(feature = "cli", test))] +pub(crate) fn chunk_key_generation(root_key: &str, key: &str) -> Option { + chunk_key_parts(root_key, key).map(|(generation, _)| generation) +} + +/// Split a canonical chunk key into its `(generation, index)`. +/// +/// The validating half of [`chunk_key_generation`], which discards the index. +/// Pointer validation needs both: the generation to prove a chunk list names one +/// generation, the index to prove the list is dense and ordered. +fn chunk_key_parts(root_key: &str, key: &str) -> Option<(String, usize)> { + if root_key.is_empty() { + return None; + } + let prefix = format!("{root_key}{CHUNK_KEY_INFIX}"); + let rest = key.strip_prefix(&prefix)?; + let (sha, index) = rest.rsplit_once('.')?; + // Canonical shape ONLY — this gates a destructive delete, so it must match + // exactly what `prepare_fastly_config_entries` emits and nothing else: + // - a 64-char LOWERCASE hex SHA-256 (`format!("{:x}", Sha256::digest(..))`), + // - a canonical decimal index (no leading zeros; `usize` `Display`). + // Anything else (short/uppercase hash, `00`, `007`) is foreign or + // hand-edited and must never become a delete candidate. + if !is_canonical_sha256_hex(sha) { + return None; + } + let parsed = canonical_index(index)?; + Some((sha.to_owned(), parsed)) +} + +/// Exactly 64 lowercase hex characters. +fn is_canonical_sha256_hex(sha: &str) -> bool { + sha.len() == 64 + && sha + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse a canonical decimal index: all ASCII digits, no leading zero unless the +/// value is exactly `0`, and it must fit `usize`. +/// +/// Must be an index this writer could actually have emitted: the chunk loop +/// counts in `usize`, so a digit run that overflows it (or is absurdly long) is +/// not ours -- accept only what `prepare_fastly_config_entries` can produce. +fn canonical_index(index: &str) -> Option { + if index.is_empty() || !index.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + if index != "0" && index.starts_with('0') { + return None; + } + index.parse::().ok() +} + // --------------------------------------------------------------------------- // Unit tests // --------------------------------------------------------------------------- #[cfg(test)] mod tests { + use std::collections::HashMap; use super::*; + /// the index must be one this writer could emit. The + /// chunk loop counts in `usize`, so a digit run that overflows it is not + /// ours -- and this gate authorises DELETES. + #[test] + fn canonical_index_must_fit_usize() { + let sha = "a".repeat(64); + let root = "app_config"; + assert!(chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.0")).is_some()); + assert!(chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.7")).is_some()); + // 40 digits: all-ASCII-digit and no leading zero, but not a usize. + let overflow = "9".repeat(40); + assert!( + chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.{overflow}")) + .is_none(), + "an index that overflows usize is not a key we wrote" + ); + } + + // ---- gc_classify_root / pointer consistency () ---- + + /// A real pointer + its real chunks, for mutation in the tests below. + fn pointer_fixture(root: &str) -> (String, FastlyChunkPointer) { + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).expect("parse"); + (pointer_json, pointer) + } + + fn reserialise(pointer: &FastlyChunkPointer) -> String { + serde_json::to_string(pointer).expect("serialise") + } + + /// A valid pointer whose chunks are scoped to a CHUNK-SHAPED holder key + /// classifies successfully (returns `Chunked` with those keys), not `Err`. + /// The cross-root GC test only shows an unclassifiable pointer failing + /// closed; this pins that a well-formed self-scoped pointer is accepted. + #[test] + fn gc_classify_root_accepts_a_pointer_rooted_at_a_chunk_shaped_key() { + let holder = format!("app_config{CHUNK_KEY_INFIX}{}.0", "e".repeat(64)); + let (pointer_json, pointer) = pointer_fixture(&holder); + let classified = gc_classify_root(&holder, &pointer_json) + .expect("a valid self-scoped pointer classifies"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("a pointer value must classify as Chunked"); + }; + assert_eq!(gc_pointer.chunks.len(), pointer.chunks.len()); + for chunk in &gc_pointer.chunks { + assert!( + chunk.key.starts_with(&format!("{holder}{CHUNK_KEY_INFIX}")), + "each referenced key must be scoped to the holder: `{}`", + chunk.key + ); + } + } + + /// The happy path: a real pointer classifies as its own chunk keys. + #[test] + fn gc_classify_root_accepts_a_real_pointer() { + let root = "app_config"; + let (pointer_json, pointer) = pointer_fixture(root); + let classified = + gc_classify_root(root, &pointer_json).expect("a real pointer must classify"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("a pointer value must classify as Chunked"); + }; + assert!( + gc_pointer.chunks.len() >= 2, + "an oversized envelope always splits into >= 2" + ); + // The classifier must carry the pointer's refs VERBATIM: `config gc` + // checks each stored chunk against these `len`/`sha256` values before + // reassembling, so a lossy or reordered mapping here would silently + // weaken every downstream content check. + assert_eq!(gc_pointer.envelope_len, pointer.envelope_len); + assert_eq!(gc_pointer.envelope_sha256, pointer.envelope_sha256); + assert_eq!(gc_pointer.chunks.len(), pointer.chunks.len()); + for (got, want) in gc_pointer.chunks.iter().zip(pointer.chunks.iter()) { + assert_eq!(got.key, want.key); + assert_eq!(got.len, want.len); + assert_eq!(got.sha256, want.sha256); + } + // ...and those refs actually describe the stored chunks. + let entries = + prepare_fastly_config_entries(root, &make_envelope_json(20_000)).expect("expand"); + for (chunk, (_, value)) in gc_pointer.chunks.iter().zip(entries.iter()) { + assert_eq!(chunk.len, value.len(), "ref len must match the real chunk"); + assert_eq!( + chunk.sha256, + sha256_hex(value.as_bytes()), + "ref sha256 must match the real chunk" + ); + } + } + + /// A direct envelope is a root that references no chunks. + #[test] + fn gc_classify_root_accepts_a_direct_envelope() { + let envelope = make_envelope_json(200); + let classified = + gc_classify_root("app_config", &envelope).expect("a valid envelope must classify"); + assert!( + matches!(classified, GcRootValue::Direct), + "an inline envelope must classify as Direct (it references no chunks)" + ); + } + + /// values that are NOT a valid envelope or pointer must + /// fail closed. `Ok([])` here would mean "references nothing" and would make + /// the root's own live chunks look orphaned. + #[test] + fn gc_classify_root_fails_closed_on_unclassifiable_values() { + let root = "app_config"; + let (pointer_json, _) = pointer_fixture(root); + // A VALID-JSON partial pointer: `chunks` truncated to one entry. This is + // the case the earlier truncated-JSON test missed. + let mut partial: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + partial.chunks.truncate(1); + let cases: Vec<(&str, String)> = vec![ + ("empty", String::new()), + ("whitespace", " ".to_owned()), + ("not json", "not-json-at-all".to_owned()), + ("unrelated json", r#"{"some":"value"}"#.to_owned()), + ("json scalar", "42".to_owned()), + ("valid-JSON partial pointer", reserialise(&partial)), + ]; + for (label, raw) in cases { + assert!( + gc_classify_root(root, &raw).is_err(), + "a {label} root value must fail closed, not classify as \"references nothing\"" + ); + } + } + + /// a pointer that PARSES but under-reports its chunks + /// would shrink the live set and get real live chunks deleted. Every way a + /// chunk list can lie must be rejected. + #[test] + fn pointer_chunk_list_must_be_internally_consistent() { + let root = "app_config"; + let (_, base) = pointer_fixture(root); + + // 1. Empty list: never emitted (an oversized envelope splits into >= 2). + let mut empty = clone_pointer(&base); + empty.chunks.clear(); + // 2. Omitted index: chunks [0, 2] -- 1 is live but unreferenced. + let mut omitted = clone_pointer(&base); + omitted.chunks.remove(1); + // 3. Duplicate index: [0, 0] -- reports fewer distinct keys than exist. + let mut duplicated = clone_pointer(&base); + let first = clone_ref(&base.chunks[0]); + duplicated.chunks = vec![clone_ref(&first), first]; + // 4. Reordered: [1, 0] -- not a shape the writer emits. + let mut reordered = clone_pointer(&base); + reordered.chunks.reverse(); + // 5. Mixed generation: a key from another envelope's content-address. + let mut mixed = clone_pointer(&base); + let other = make_envelope_json(19_000); + let other_entries = prepare_fastly_config_entries(root, &other).expect("expand"); + mixed.chunks[1] = FastlyChunkRef { + key: other_entries[1].0.clone(), + len: base.chunks[1].len, + sha256: base.chunks[1].sha256.clone(), + }; + // 6. Lengths that cannot add up to the declared envelope. + let mut bad_len = clone_pointer(&base); + bad_len.envelope_len = base.envelope_len.saturating_add(999); + // 7. Zero-length chunk. + let mut zero_len = clone_pointer(&base); + zero_len.chunks[0].len = 0; + + for (label, pointer) in [ + ("an empty chunk list", empty), + ("an omitted index", omitted), + ("a duplicated index", duplicated), + ("a reordered list", reordered), + ("a mixed generation", mixed), + ("an inconsistent envelope_len", bad_len), + ("a zero-length chunk", zero_len), + ] { + let raw = reserialise(&pointer); + assert!( + gc_classify_root(root, &raw).is_err(), + "a pointer with {label} must be rejected, not treated as the authoritative live set" + ); + assert!( + prior_chunk_keys(root, &raw).is_err(), + "a pointer with {label} must also warn on the push path, not prune from a bad list" + ); + } + + // Control: the unmutated fixture still passes, so the assertions above + // are rejecting the mutation and not something incidental. + gc_classify_root(root, &reserialise(&base)) + .expect("the unmutated fixture must still classify"); + } + + /// a parse diagnostic must never quote the stored + /// value -- app config may hold credentials and these lines are logged. + #[test] + fn pointer_parse_errors_do_not_leak_stored_values() { + const SENTINEL: &str = "s3cr3t-do-not-log"; + let root = "app_config"; + // `version` is typed `u8`; a string there makes serde quote it: + // `invalid type: string "s3cr3t-do-not-log", expected u8`. + let malformed = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":"{SENTINEL}","chunks":[],"data_sha256":"","envelope_len":0,"envelope_sha256":""}}"# + ); + let Err(gc_err) = gc_classify_root(root, &malformed) else { + panic!("a malformed pointer must fail closed on the gc path"); + }; + for err in [ + prior_chunk_keys(root, &malformed).expect_err("must warn"), + gc_err, + ] { + assert!( + !err.contains(SENTINEL), + "diagnostic leaked a stored value: {err}" + ); + assert!( + err.contains("redacted"), + "diagnostic should say the value was redacted: {err}" + ); + } + } + + fn clone_pointer(pointer: &FastlyChunkPointer) -> FastlyChunkPointer { + FastlyChunkPointer { + chunks: pointer.chunks.iter().map(clone_ref).collect(), + data_sha256: pointer.data_sha256.clone(), + edgezero_kind: pointer.edgezero_kind.clone(), + envelope_len: pointer.envelope_len, + envelope_sha256: pointer.envelope_sha256.clone(), + version: pointer.version, + } + } + + fn clone_ref(chunk: &FastlyChunkRef) -> FastlyChunkRef { + FastlyChunkRef { + key: chunk.key.clone(), + len: chunk.len, + sha256: chunk.sha256.clone(), + } + } + + /// Round-7 [P1]: a COORDINATED partial pointer -- drop the last chunk ref AND + /// restate `envelope_len` as the remaining sum. Generation, indexes, lengths + /// and the sum all agree; only the CONTENT disagrees. Metadata validation + /// cannot see it, so the dropped chunk would silently leave the live set and + /// become deletable. Only reassembling and hashing catches this. + #[test] + fn coordinated_partial_pointer_fails_content_verification() { + let root = "app_config"; + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + assert!(pointer.chunks.len() >= 3, "need >= 3 chunks for this case"); + + let dropped = pointer.chunks.pop().expect("last chunk"); + pointer.envelope_len = pointer.chunks.iter().map(|chunk| chunk.len).sum(); + let doctored = serde_json::to_string(&pointer).expect("serialise"); + + // METADATA validation passes -- that is the whole point of the attack. + let classified = gc_classify_root(root, &doctored) + .expect("the doctored pointer is metadata-consistent by construction"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("expected a chunked root"); + }; + assert!( + !gc_pointer + .chunks + .iter() + .any(|chunk| chunk.key == dropped.key), + "fixture check: the dropped chunk is absent from the pointer's list" + ); + + // CONTENT verification is what rejects it: the surviving chunks cannot + // hash to the envelope the generation names. + let assembled: String = entries[..entries.len().saturating_sub(2)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + let err = gc_verify_generation(&gc_pointer.envelope_sha256, &assembled) + .expect_err("a partial chunk set must not verify as its generation"); + assert!( + err.contains("not the generation they claim"), + "expected a content-address mismatch, got: {err}" + ); + + // And the honest, complete set still verifies -- so the assertion above + // is rejecting the omission, not something incidental. + let whole: String = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + gc_verify_generation(&gc_pointer.envelope_sha256, &whole) + .expect("the complete chunk set must verify"); + } + + /// Round-7/9 [P1]: an entry that merely LOOKS like a chunk key is not a chunk. + /// Reassembling to the content-address its keys name is NECESSARY but not + /// sufficient (a forger can content-address their own data with no preimage); + /// `prove_generation` adds the writer round-trip. Here we check the necessary + /// half: a value that does not even hash to its generation is never ours. + #[test] + fn only_a_real_generation_verifies() { + let root = "app_config"; + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let generation = chunk_key_generation(root, &entries[0].0).expect("canonical key"); + + for (label, value) in [ + ("plain text", "just some plain text"), + ("unrelated json", r#"{"some":"value"}"#), + ("someone's real config", make_envelope_json(200).as_str()), + ] { + assert!( + gc_verify_generation(&generation, value).is_err(), + "a {label} value must never verify as a generation we wrote" + ); + } + + // The genuine article does verify. + let whole: String = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + gc_verify_generation(&generation, &whole).expect("the real generation must verify"); + } + + /// `chunks[].key` is pointer-controlled and NOT yet + /// validated where we report it, so a malformed pointer can smuggle a secret + /// into a log line. Diagnostics must name a position, never the key. + #[test] + fn pointer_key_does_not_leak_into_diagnostics() { + const SENTINEL: &str = "prod-db-password=hunter2"; + let root = "app_config"; + let malformed = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{SENTINEL}","len":10,"sha256":"x"}}],"data_sha256":"","envelope_len":10,"envelope_sha256":"{}"}}"#, + "a".repeat(64) + ); + let err = prior_chunk_keys(root, &malformed).expect_err("must warn"); + assert!( + !err.contains(SENTINEL), + "a pointer-controlled key must never reach a diagnostic: {err}" + ); + } + + /// `envelope_len` and the per-chunk lengths are + /// untrusted `usize`s that later size allocations. Saturating arithmetic let + /// a metadata-consistent pointer declare `usize::MAX` and pass validation, + /// which then reached `String::with_capacity` and aborted the process. + #[test] + fn absurd_pointer_lengths_are_rejected_before_allocating() { + let root = "app_config"; + let sha = "a".repeat(64); + let huge = usize::MAX; + let overflowing = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":{huge},"sha256":"x"}},{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.1","len":{huge},"sha256":"y"}}],"data_sha256":"","envelope_len":{huge},"envelope_sha256":"{sha}"}}"# + ); + assert!( + prior_chunk_keys(root, &overflowing).is_err(), + "chunk lengths that overflow when summed must be rejected" + ); + + // A single absurd chunk: no overflow, but far beyond what we emit. + let oversized = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":{huge},"sha256":"x"}}],"data_sha256":"","envelope_len":{huge},"envelope_sha256":"{sha}"}}"# + ); + assert!( + prior_chunk_keys(root, &oversized).is_err(), + "a chunk larger than the writer's payload target must be rejected" + ); + } + + /// root-vs-chunk is decided by VALUE. A pointer value is a + /// root wherever it lives; a chunk payload (raw envelope fragment) is not. + #[test] + fn value_is_pointer_kind_detects_pointers_only() { + let root = "app_config"; + let (pointer_json, _) = pointer_fixture(root); + assert!( + value_is_pointer_kind(&pointer_json), + "a real pointer value must be recognised as pointer-kind" + ); + + // A genuine chunk PAYLOAD: a raw fragment of envelope JSON. + let entries = + prepare_fastly_config_entries(root, &make_envelope_json(20_000)).expect("expand"); + assert!( + !value_is_pointer_kind(&entries[0].1), + "a chunk payload must NOT be mistaken for a pointer" + ); + // Direct envelopes, unrelated JSON and non-JSON are not pointers either. + assert!(!value_is_pointer_kind(&make_envelope_json(200))); + assert!(!value_is_pointer_kind(r#"{"some":"value"}"#)); + assert!(!value_is_pointer_kind("not json at all")); + } + + /// The runtime resolver rejects a pointer shape the writer could never emit + /// BEFORE fetching any chunk -- the same metadata validation the CLI/GC path + /// runs, so invariant 14 holds on the read path too. + #[test] + fn runtime_resolver_rejects_non_writer_pointer_shapes() { + let root = "app_config"; + let sha = "a".repeat(64); + // A metadata-consistent-looking pointer whose per-chunk length is far + // over the writer's payload target (and whose envelope is too small to + // have been chunked at all). + let bogus = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":99999,"sha256":"x"}},{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.1","len":1,"sha256":"y"}}],"data_sha256":"","envelope_len":100000,"envelope_sha256":"{sha}"}}"# + ); + // No fetch should even be attempted: fail on metadata alone. + let mut fetched = false; + let err = resolve_fastly_config_value(root, bogus, |_key| { + fetched = true; + Ok(None) + }) + .expect_err("a non-writer pointer shape must be rejected"); + assert!( + !fetched, + "validation must reject before any chunk fetch: {err}" + ); + + // The real thing still resolves. + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries(root, &envelope).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect("a real chunked value must still resolve"); + assert_eq!(resolved, envelope); + } + + /// A hand-authored pointer of MANY tiny chunks (metadata-consistent: dense, + /// summing to `envelope_len`) is rejected — its non-final chunks are far + /// below a full payload, so it is not writer output. This bounds the runtime + /// fetch fan-out. + #[test] + fn pointer_with_many_tiny_chunks_is_rejected() { + use std::fmt::Write as _; + let root = "app_config"; + let sha = "a".repeat(64); + // 100 chunks of 100 bytes each = 10_000 bytes (> entry limit, dense). + let mut chunks = String::new(); + for idx in 0..100_u32 { + if idx > 0 { + chunks.push(','); + } + write!( + chunks, + r#"{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.{idx}","len":100,"sha256":"x"}}"# + ) + .expect("write to String"); + } + let bogus = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{chunks}],"data_sha256":"","envelope_len":10000,"envelope_sha256":"{sha}"}}"# + ); + let mut fetched = false; + let err = resolve_fastly_config_value(root, bogus, |_key| { + fetched = true; + Ok(None) + }) + .expect_err("a tiny-chunk fan-out must be rejected"); + assert!(!fetched, "must reject before fetching 100 chunks: {err}"); + } + + /// RESOLVER-TO-WRITER ROUND TRIP: for a spread of over-limit sizes, the + /// writer's own output must resolve back to the exact input. This is the + /// positive half of the exact split-boundary contract — whatever the writer + /// emits, the resolver (including its exact-boundary replay) accepts. + #[test] + fn writer_output_round_trips_through_the_resolver() { + let root = "app_config"; + // Sizes that land on, just past, and well past chunk boundaries. + for target in [ + FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1), + CHUNK_PAYLOAD_TARGET.saturating_add(50), + CHUNK_PAYLOAD_TARGET.saturating_mul(2).saturating_add(10), + 20_000, + 35_000, + ] { + let envelope = make_envelope_json(target); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (root_key, pointer_json) = entries.last().expect("pointer").clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .unwrap_or_else(|err| panic!("size {target} must round-trip: {err}")); + assert_eq!(resolved, envelope, "size {target} must reconstruct exactly"); + } + } + + /// EXACT SPLIT BOUNDARIES: a pointer whose chunks reassemble to the correct + /// bytes but along boundaries this writer would never choose is rejected. Here + /// two bytes are moved from the first (full) chunk into the last (short) one: + /// every metadata check still passes (dense indexes, single generation, both + /// lengths in range, sum == `envelope_len`, per-chunk and whole-envelope SHAs + /// correct), yet the writer splits the first chunk at 7 000 bytes, not 6 998, + /// so the resolver's boundary replay must reject it. + #[test] + fn resolver_rejects_a_non_writer_split_that_reassembles_correctly() { + let root = "app_config"; + // Just over the entry limit: chunk 0 is a full CHUNK_PAYLOAD_TARGET, chunk + // 1 is a short remainder with room to absorb the shifted bytes. + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(100)); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + assert_eq!(entries.len(), 3, "expected exactly 2 chunks + pointer"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let real: FastlyChunkPointer = serde_json::from_str(&pointer_json).expect("parse"); + + // Re-split the SAME bytes 2 bytes earlier: chunk 0 shrinks to 6 998 + // (still >= CHUNK_PAYLOAD_TARGET - 3, so the pre-fetch bound passes), + // chunk 1 grows correspondingly (still <= CHUNK_PAYLOAD_TARGET). + let cut = CHUNK_PAYLOAD_TARGET.saturating_sub(2); + let head = envelope.get(..cut).expect("cut is a char boundary"); + let tail = envelope.get(cut..).expect("cut is a char boundary"); + let key0 = format!("{root}{CHUNK_KEY_INFIX}{}.0", real.envelope_sha256); + let key1 = format!("{root}{CHUNK_KEY_INFIX}{}.1", real.envelope_sha256); + let shifted = FastlyChunkPointer { + chunks: vec![ + FastlyChunkRef { + key: key0.clone(), + len: head.len(), + sha256: sha256_hex(head.as_bytes()), + }, + FastlyChunkRef { + key: key1.clone(), + len: tail.len(), + sha256: sha256_hex(tail.as_bytes()), + }, + ], + data_sha256: real.data_sha256.clone(), + edgezero_kind: real.edgezero_kind.clone(), + envelope_len: real.envelope_len, + envelope_sha256: real.envelope_sha256.clone(), + version: real.version, + }; + let chunk_map: HashMap = + HashMap::from([(key0, head.to_owned()), (key1, tail.to_owned())]); + let err = resolve_fastly_config_value(root, reserialise(&shifted), |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect_err("a non-writer split must be rejected even if it reassembles correctly"); + assert!( + err.contains("writer splits the reconstructed envelope"), + "must fail on the exact-boundary check, got: {err}" + ); + } + + /// v1 READ-COMPATIBILITY: an envelope UNDER 8 000 characters but OVER 8 000 + /// UTF-8 BYTES is chunked by this writer's conservative byte-count threshold + /// (the merge-base behaviour), and MUST reconstruct and resolve — a + /// character-based threshold would leave it stored directly and reject the + /// existing chunked pointer on read (an HTTP 500 for previously stored data). + #[test] + fn chunked_value_under_char_limit_but_over_byte_limit_resolves() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + // ~2001 crabs of data: the serialised envelope is > 8000 BYTES but + // < 8000 CHARACTERS. + let data = json!({ "crabs": "\u{1F980}".repeat(2_001) }); + let envelope = + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); + assert!( + envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT, + "must exceed the byte limit" + ); + assert!( + envelope.chars().count() <= FASTLY_CONFIG_ENTRY_LIMIT, + "must fit the character limit -- this is the compatibility gap" + ); + + // The writer chunks it (byte-based), exactly as the merge-base did. + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + assert!( + entries.len() >= 3, + "must have chunked (>= 2 chunks + pointer)" + ); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + + // The resolver must ACCEPT it and reconstruct the original. + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect("an existing v1 byte-chunked value must still resolve"); + assert_eq!(resolved, envelope); + } + + /// An empty root key is rejected before any output: writer-valid but + /// resolver-invalid (canonical chunk parsing rejects an empty root). + #[test] + fn empty_root_key_is_rejected() { + assert!( + prepare_fastly_config_entries("", "{}").is_err(), + "an empty key must be rejected on the direct path" + ); + let big = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!( + prepare_fastly_config_entries("", &big).is_err(), + "an empty key must be rejected on the chunked path too" + ); + } + + /// A root that is itself valid but long enough that its DERIVED chunk keys + /// exceed the store limit must be rejected up front, not fail mid-write. + #[test] + fn oversized_derived_chunk_keys_are_rejected_before_write() { + // A root ~200 chars: valid as a key on its own, but + ~85 for the chunk + // suffix exceeds the limit. + let long_root = "r".repeat(200); + let big_envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let err = prepare_fastly_config_entries(&long_root, &big_envelope) + .expect_err("derived chunk keys over the limit must be rejected"); + assert!( + err.contains(&FASTLY_CONFIG_KEY_LIMIT.to_string()) && err.contains("limit"), + "error must name the key limit: {err}" + ); + + // A root over the limit is rejected even for a DIRECT (unchunked) value. + let over_limit_root = "r".repeat(FASTLY_CONFIG_KEY_LIMIT.saturating_add(1)); + assert!( + prepare_fastly_config_entries(&over_limit_root, "{}").is_err(), + "a root key over the limit must be rejected on the direct path too" + ); + + // A normal root is unaffected. + prepare_fastly_config_entries("app_config", &big_envelope) + .expect("a normal root must still expand"); + + // The limit is CHARACTERS, not bytes: a multi-byte key at exactly the + // character limit must be accepted even though its byte length exceeds + // it. U+00E9 ('é') is 2 bytes, so 255 of them is 510 bytes, 255 chars. + let multibyte_root = "\u{e9}".repeat(FASTLY_CONFIG_KEY_LIMIT); + assert!( + multibyte_root.len() > FASTLY_CONFIG_KEY_LIMIT, + "fixture must be multi-byte" + ); + prepare_fastly_config_entries(&multibyte_root, "{}") + .expect("a key at the CHARACTER limit must be accepted regardless of byte length"); + } + // ---- helpers ---- /// Build a valid `BlobEnvelope` JSON string of approximately `target_len` @@ -387,14 +1671,15 @@ mod tests { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; - // crab emoji: 4 bytes each; 3000 crabs = 12 000 bytes of payload + // crab emoji: 4 bytes each. The direct-vs-chunked threshold is a + // conservative BYTE count, so 3 000 crabs (12 000 bytes) chunks. let emoji_block = "\u{1F980}".repeat(3_000); let data = json!({ "crabs": emoji_block }); let envelope = serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); assert!( envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT, - "emoji envelope must be oversized" + "emoji envelope must exceed the byte limit so it chunks" ); let entries = prepare_fastly_config_entries("emoji_key", &envelope).unwrap(); @@ -533,12 +1818,20 @@ mod tests { }) .expect_err("corrupt chunk must error"); assert!( - err.contains("SHA mismatch") || err.contains("sha") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" + err.contains("does not match the SHA-256"), + "error must say what failed: {err}" ); + // the diagnostic identifies the chunk by POSITION, not by + // key. The key comes from the stored pointer, so echoing it would let a + // malformed pointer smuggle a secret into a log line. Same for the + // hashes it records. assert!( - err.contains(&first_chunk_key), - "error must name the failing chunk key: {err}" + err.contains("chunk 0"), + "error must locate the failing chunk by position: {err}" + ); + assert!( + !err.contains(&first_chunk_key), + "a pointer-controlled key must not be echoed into a log line: {err}" ); } @@ -568,13 +1861,14 @@ mod tests { } }) .expect_err("length-mismatched chunk must error"); + assert!(err.contains("length"), "error must mention length: {err}"); assert!( - err.contains("length") || err.contains("len"), - "error must mention length: {err}" + err.contains("chunk 0"), + "error must locate the failing chunk by position: {err}" ); assert!( - err.contains(&first_chunk_key), - "error must name the failing chunk key: {err}" + !err.contains(&first_chunk_key), + "a pointer-controlled key must not be echoed into a log line: {err}" ); } @@ -591,22 +1885,92 @@ mod tests { .cloned() .collect(); - // Tamper with envelope_sha256 in the pointer JSON. + // `envelope_sha256` is a stored, value-controlled string. + // Set it to a SENTINEL secret and require the diagnostic not to echo it. + let sentinel = "SUPER_SECRET_ENVELOPE_HASH"; let mut pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); - pointer.envelope_sha256 = "ff".repeat(32); + pointer.envelope_sha256 = sentinel.to_owned(); let tampered_pointer_json = serde_json::to_string(&pointer).unwrap(); let err = resolve_fastly_config_value(&root_key, tampered_pointer_json, |chunk_key| { Ok(chunk_map.get(chunk_key).cloned()) }) .expect_err("envelope hash mismatch must error"); + // Whether metadata validation (the chunk keys name a different + // generation than the tampered `envelope_sha256`) or the full-envelope + // check catches it, the stored hash must never reach the diagnostic. assert!( - err.contains("SHA mismatch") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" + !err.contains(sentinel), + "the stored envelope_sha256 must never reach a diagnostic: {err}" ); assert!(err.contains("root"), "error must name the root key: {err}"); } + /// the CHUNKED read path returns reconstructed bytes + /// after checking only the OUTER pointer hash -- it never parses/verifies the + /// inner `BlobEnvelope` the way the DIRECT path does. So an envelope whose + /// embedded `sha256` is a secret passes the resolver, and core's later + /// `verify()` formats that stored hash into an HTTP 500. The resolver must + /// reject it here, redacted. + #[test] + fn chunked_read_verifies_inner_envelope() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + const SENTINEL: &str = "SUPER_SECRET_INNER_SHA"; + + // A structurally valid envelope whose inner `sha256` is a sentinel that + // does NOT match its data -- so `verify()` fails and would name it. + let mut envelope = BlobEnvelope::new(json!({"k":"v"}), "2026-06-22T00:00:00Z".into()); + envelope.sha256 = SENTINEL.to_owned(); + // Pad the SERIALISED envelope past the entry limit so it chunks. + let mut envelope_json = serde_json::to_string(&envelope).unwrap(); + envelope_json.push_str(&" ".repeat(FASTLY_CONFIG_ENTRY_LIMIT)); + let entries = prepare_fastly_config_entries("root", &envelope_json).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + + let err = resolve_fastly_config_value(&root_key, pointer_json, |chunk_key| { + Ok(chunk_map.get(chunk_key).cloned()) + }) + .expect_err("a reconstructed envelope with a bad inner sha must be rejected"); + assert!( + !err.contains(SENTINEL), + "the inner envelope hash must never reach a diagnostic: {err}" + ); + } + + /// a fetch-callback failure carries the pointer-controlled + /// chunk KEY. The resolver must locate the fault by position, not echo it. + #[test] + fn resolver_fetch_error_does_not_leak_chunk_key() { + const SENTINEL: &str = "SUPER_SECRET_IN_A_CHUNK_KEY"; + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("root", &envelope).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + // A VALID pointer (so metadata validation passes), whose first chunk key + // the callback names in its failure — as the real store lookup would. + let pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + let secret_key = pointer.chunks[0].key.clone(); + + let err = resolve_fastly_config_value(&root_key, pointer_json, |chunk_key| { + Err(format!( + "config store lookup failed for `{chunk_key}` -- {SENTINEL}" + )) + }) + .expect_err("a fetch failure must error"); + assert!( + !err.contains(SENTINEL) && !err.contains(&secret_key), + "a fetch failure must not echo the pointer-controlled chunk key: {err}" + ); + assert!( + err.contains("chunk 0") && err.contains("could not be fetched"), + "the error must locate the fault by position: {err}" + ); + } + #[test] fn resolver_errors_on_malformed_pointer() { // A value that is neither a valid BlobEnvelope nor a valid pointer. @@ -657,4 +2021,144 @@ mod tests { ); assert!(err.contains("my_key"), "error must name root key: {err}"); } + + // ---- prior_chunk_keys tests ---- + + #[test] + fn prior_chunk_keys_returns_valid_v1_pointer_keys() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + + let keys = prior_chunk_keys("app_config", pointer_json).expect("valid pointer"); + + let expected: Vec = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + assert_eq!(keys, expected); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_direct_envelope() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT); + assert_eq!( + prior_chunk_keys("app_config", &envelope).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_unrelated_json() { + assert_eq!( + prior_chunk_keys("app_config", r#"{"hello":"world"}"#).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_wrong_kind() { + let raw = r#"{"edgezero_kind":"other","version":1,"chunks":[],"data_sha256":"","envelope_len":0,"envelope_sha256":""}"#; + assert_eq!( + prior_chunk_keys("app_config", raw).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_rejects_unsupported_pointer_version() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(pointer_json).unwrap(); + pointer.version = 2; + let raw = serde_json::to_string(&pointer).unwrap(); + + let err = prior_chunk_keys("app_config", &raw).expect_err("version 2 should warn"); + assert!(err.contains("unsupported version"), "{err}"); + } + + #[test] + fn prior_chunk_keys_rejects_foreign_chunk_prefix() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(pointer_json).unwrap(); + pointer.chunks[0].key = + pointer.chunks[0] + .key + .replacen("app_config", "app_config_staging", 1); + let raw = serde_json::to_string(&pointer).unwrap(); + + let err = prior_chunk_keys("app_config", &raw).expect_err("foreign chunk should warn"); + assert!(err.contains("non-canonical"), "{err}"); + } + + // ---- chunk_key_generation (canonical only) ---- + + #[test] + fn chunk_key_generation_extracts_the_sha() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (chunk_key, _) = &entries[0]; + let sha = chunk_key_generation("app_config", chunk_key).expect("a chunk key"); + assert_eq!(sha.len(), 64, "64-char sha256 hex"); + assert!(chunk_key.contains(&sha)); + } + + #[test] + fn chunk_key_generation_requires_canonical_shape() { + let good = "a".repeat(64); + let base = format!("app_config.__edgezero_chunks.{good}"); + // Canonical. + assert!(chunk_key_generation("app_config", &format!("{base}.0")).is_some()); + assert!(chunk_key_generation("app_config", &format!("{base}.17")).is_some()); + + // Different root / the root itself. + assert!( + chunk_key_generation("app_config", &format!("other.__edgezero_chunks.{good}.0")) + .is_none() + ); + assert!(chunk_key_generation("app_config", "app_config").is_none()); + // Empty root. + assert!(chunk_key_generation("", &format!(".__edgezero_chunks.{good}.0")).is_none()); + // Short SHA (< 64). + assert!( + chunk_key_generation("app_config", "app_config.__edgezero_chunks.abc123.0").is_none() + ); + // Uppercase SHA. + let upper = "A".repeat(64); + assert!( + chunk_key_generation( + "app_config", + &format!("app_config.__edgezero_chunks.{upper}.0") + ) + .is_none() + ); + // Non-hex SHA of the right length. + let nonhex = "z".repeat(64); + assert!( + chunk_key_generation( + "app_config", + &format!("app_config.__edgezero_chunks.{nonhex}.0") + ) + .is_none() + ); + // Leading-zero / non-numeric / missing index. + assert!(chunk_key_generation("app_config", &format!("{base}.00")).is_none()); + assert!(chunk_key_generation("app_config", &format!("{base}.007")).is_none()); + assert!(chunk_key_generation("app_config", &format!("{base}.x")).is_none()); + assert!(chunk_key_generation("app_config", &base).is_none()); + } + + // Regression for the Value-first rule: a value that IS pointer-kind but + // is missing required fields (`chunks`, `data_sha256`, …) must WARN, not + // be silently dropped by a failed struct deserialize. + #[test] + fn prior_chunk_keys_warns_on_pointer_kind_with_missing_fields() { + let raw = r#"{"edgezero_kind":"fastly_config_chunks","version":2}"#; + let err = prior_chunk_keys("app_config", raw) + .expect_err("pointer-kind but malformed must warn, not Ok([])"); + assert!(!err.is_empty(), "{err}"); + } } diff --git a/crates/edgezero-adapter-fastly/src/cli.rs b/crates/edgezero-adapter-fastly/src/cli.rs index 678f7b3f..0a31cb56 100644 --- a/crates/edgezero-adapter-fastly/src/cli.rs +++ b/crates/edgezero-adapter-fastly/src/cli.rs @@ -1,11 +1,18 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; use std::env; +use std::fmt::Write as _; use std::fs; use std::io::{ErrorKind, Write as _}; use std::path::{Path, PathBuf}; use std::process::Command; use std::process::Stdio; +use std::time::{SystemTime, UNIX_EPOCH}; -use crate::chunked_config::{prepare_fastly_config_entries, resolve_fastly_config_value}; +use crate::chunked_config::{ + CHUNK_KEY_INFIX, GcPointer, GcRootValue, chunk_key_generation, gc_classify_root, + gc_verify_generation, prepare_fastly_config_entries, prior_chunk_keys, + resolve_fastly_config_value, sha256_hex, value_is_pointer_kind, +}; use ctor::ctor; use edgezero_adapter::cli_support::{ find_manifest_upwards, find_workspace_root, path_distance, read_package_name, run_native_cli, @@ -141,6 +148,81 @@ enum ConfigStoreLookup { SchemaDrift(String), } +/// The reclamation plan for `config gc`: the orphan chunk entries to delete +/// (with their ages) plus the counts for the summary line. Produced by +/// `plan_gc_reclamation` (which owns every safety guard); consumed by +/// `gc_fastly_config_store` (which reports and deletes). +struct GcPlan { + /// Whole generations to reclaim, each a list of `(key, age_secs)`. Grouped, + /// not flat: a generation is provable only as a UNIT (see + /// `prove_generation`), so deleting part of one destroys the very evidence + /// that licenses deleting the rest. + doomed: Vec>, + live_count: usize, + retained_recent: usize, + roots: usize, + /// Chunk-shaped entries we could NOT prove our writer produced, so left + /// untouched. Surfaced so an operator can see we declined to judge them. + unprovable: usize, +} + +/// What one pass of `config gc`'s delete loop actually did. +struct GcDeleteOutcome { + /// Entries whose delete returned success. + deleted: usize, + /// Keys whose delete returned non-zero. + failed: Vec, + /// Survivors of a generation in which an earlier sibling's delete had + /// ALREADY succeeded before a later one failed. These are definitely an + /// incomplete generation now, so they can never be proved (or reclaimed) + /// again -- manual removal only. + stranded: Vec, + /// Members of a generation whose ONLY failure was on a delete with no + /// confirmed prior sibling success. A failed remote delete has UNKNOWN + /// outcome (Fastly may have committed it before returning an error), so we + /// cannot say whether the generation is still whole. A re-run reclaims it if + /// it is, or reports it as an unprovable fragment if it is not. + uncertain: Vec, +} + +/// The result of classifying a store's entries for reclamation. +struct GcClassification { + /// Chunk keys a live root pointer references, each verified against its + /// content-address. Never deletable. + live: HashSet, + /// Keys whose OWN value is a runtime-readable root — a valid direct envelope + /// or a pointer — regardless of what their key looks like. Never deletable. + protected: HashSet, + /// Count of entries classified as roots, for the summary line. + roots: usize, +} + +/// One `config-store-entry list` item. +/// +/// `item_value` IS captured — `config gc` must parse root pointers to learn +/// which chunks are live, and one listing avoids a `describe` per root. It is +/// the config payload: it may be read in memory but must NEVER be logged or +/// surfaced (see `redact_describe_response` / `redact_stderr`). +struct ConfigStoreItem { + created_at: String, + item_key: String, + item_value: String, +} + +/// Per-root plan for the LOCAL path's eager prune. +/// +/// Local reclamation is safe to do immediately: `fastly.toml` is a single +/// file that Viceroy reads at startup — there is no propagation window and no +/// POP that could still be serving the previous pointer. (The cloud path +/// cannot do this; see `reclaim_orphan_generations`.) +struct FastlyConfigGcPlan { + /// Exact keep-set this push writes for the root (chunk keys + root key). + new_keys: HashSet, + /// Prior chunk keys to consider deleting, or a warning to surface + /// (suspicious prior pointer) that skips GC for this root. + prior_keys: Result, String>, +} + // The three `validate_*` trait methods exist on `Adapter` because // spin requires them (variable-name regex, `[component.*]` // discovery, flat-namespace collision). The trait surface is typed @@ -191,10 +273,46 @@ impl Adapter for FastlyCliAdapter { } } + fn gc_config_entries( + &self, + _manifest_root: &Path, + _adapter_manifest_path: Option<&str>, + _component_selector: Option<&str>, + store: &ResolvedStoreId, + _push_ctx: &AdapterPushContext<'_>, + older_than_secs: u64, + dry_run: bool, + ) -> Result, String> { + gc_fastly_config_store(store.platform.as_str(), older_than_secs, dry_run) + } + fn name(&self) -> &'static str { "fastly" } + fn preflight_config_write(&self, key: &str, body: &str) -> Result<(), String> { + // Reject an infeasible push here, BEFORE the CLI's remote read, so it + // fails offline rather than after a list/describe. The write path + // re-checks, so this is a strict early gate, not the only one. + // + // An empty key is writer-valid but resolver-invalid (canonical chunk + // parsing rejects an empty root); reject it before any I/O. + if key.is_empty() { + return Err( + "config key is empty; provide a store id or a non-empty `--key`".to_owned(), + ); + } + let entry = [(key.to_owned(), String::new())]; + reject_reserved_root_keys(&entry)?; + // Run the full chunk expansion OFFLINE (no I/O): exactly what the write + // path does, so every body-dependent feasibility failure — the root key + // over the store limit, a DERIVED chunk key over it once the value + // chunks, or a pointer that would not fit the entry limit — is caught + // here, before the remote read, instead of after it. + prepare_fastly_config_entries(key, body)?; + Ok(()) + } + fn provision( &self, manifest_root: &Path, @@ -368,21 +486,25 @@ impl Adapter for FastlyCliAdapter { "no config entries to push to fastly config-store `{name}` (logical id `{logical}`)" )]); } - // Expand each logical (key, envelope_json) into physical entries - // via the chunk-pointer helper. Entries ≤ 8 000 chars go through - // as a single direct entry; larger envelopes are split into - // content-addressed chunks with a root pointer written LAST. - // Collect all physical entries before any writes so pointer-too- - // large errors surface before touching the remote store. + // Reject reserved keys before any expansion or I/O. + reject_reserved_root_keys(entries)?; + reject_duplicate_root_keys(entries)?; + // Expand each logical root once: flatten for the commit, and keep + // the exact per-root keep-set + the value written at the root key + // for GC (no prefix scan of the flattened set). Collecting all + // physical entries first also surfaces pointer-too-large errors + // before touching the remote store. let mut physical_entries: Vec<(String, String)> = Vec::new(); + let mut roots: Vec<(String, HashSet, String)> = Vec::with_capacity(entries.len()); for (key, body) in entries { - let expanded = prepare_fastly_config_entries(key, body)?; + let (expanded, new_keys, new_root_value) = expand_root(key, body)?; physical_entries.extend(expanded); + roots.push((key.clone(), new_keys, new_root_value)); } if dry_run { - // Report intent without shelling out. One line per logical key - // noting whether it would be direct or chunked, plus chunk count. - let mut out = Vec::with_capacity(entries.len().saturating_add(1)); + // Report intent without shelling out. Stays fully offline: no + // store-id resolution, no remote read (so no GC count). + let mut out = Vec::with_capacity(entries.len().saturating_mul(2).saturating_add(1)); out.push(format!( "would resolve fastly config-store `{name}` (logical id `{logical}`) via `fastly config-store list --json` and push entries:" )); @@ -405,6 +527,21 @@ impl Adapter for FastlyCliAdapter { return Ok(out); } let resolved_id = resolve_remote_config_store_id(name)?; + // NOTE: a cloud push does NOT reclaim orphaned chunks. + // + // Fastly's config store is eventually consistent, so a generation may + // only be deleted once the pointer that referenced it has stopped being + // served everywhere. Fastly records no pointer-supersession time + // (`updated_at` is NOT bumped by `update --upsert` -- verified against + // the live API), offers no compare-and-swap with which to record one + // safely, and chunk `created_at` is NOT a proxy for it (a chunked -> + // direct -> direct transition leaves the old generation with no + // "successor" at all). Every attempt to synthesise that fact is unsound. + // + // So reclamation is an explicit, operator-invoked `config gc`: the + // operator supplies the one fact the platform cannot -- that the current + // config has been live long enough that nothing is serving the old + // pointers. See the spec's "Cloud reclamation". push_entries_with_committer(&physical_entries, |key, value| { create_config_store_entry(&resolved_id, key, value) })?; @@ -447,19 +584,26 @@ impl Adapter for FastlyCliAdapter { fastly_path.display() )]); } - // Expand logical entries into physical entries (chunks + pointer). + // Reject reserved keys before any expansion or I/O. + reject_reserved_root_keys(entries)?; + reject_duplicate_root_keys(entries)?; + // Expand each logical root once: flatten for the write, keep the + // exact per-root keep-set for GC (no prefix scan of the flattened set). let mut physical_entries: Vec<(String, String)> = Vec::new(); + let mut gc_roots: Vec<(String, HashSet)> = Vec::with_capacity(entries.len()); for (key, body) in entries { - let expanded = prepare_fastly_config_entries(key, body)?; + let (expanded, new_keys, _new_root) = expand_root(key, body)?; physical_entries.extend(expanded); + gc_roots.push((key.clone(), new_keys)); } if dry_run { - let mut out = Vec::with_capacity(entries.len().saturating_add(1)); + let counts = local_orphan_counts_for_dry_run(&fastly_path, name, entries); + let mut out = Vec::with_capacity(entries.len().saturating_mul(2).saturating_add(1)); out.push(format!( "would edit `[local_server.config_stores.{name}.contents]` in {} (logical id `{logical}`) with entries:", fastly_path.display(), )); - for (key, body) in entries { + for (idx, (key, body)) in entries.iter().enumerate() { let expanded = prepare_fastly_config_entries(key, body) .unwrap_or_else(|_| vec![(key.clone(), body.clone())]); if expanded.len() == 1 { @@ -474,16 +618,28 @@ impl Adapter for FastlyCliAdapter { body.len() )); } + match counts.get(idx).map(|(_, count)| count) { + Some(Ok(n)) => out.push(format!( + " would delete {n} orphan chunks from the previous generation of `{key}`" + )), + Some(Err(reason)) => out.push(format!( + " would delete an unknown number of orphan chunks from the previous generation of `{key}` (unknown: {reason})" + )), + None => {} + } } return Ok(out); } - write_fastly_local_config_store(&fastly_path, name, &physical_entries)?; - Ok(vec![format!( + let warnings = + write_fastly_local_config_store(&fastly_path, name, &physical_entries, &gc_roots)?; + let mut out = vec![format!( "wrote {} physical entries ({} logical) to `[local_server.config_stores.{name}.contents]` in {} (logical id `{logical}`); restart `fastly compute serve` to pick up changes", physical_entries.len(), entries.len(), fastly_path.display() - )]) + )]; + out.extend(warnings); + Ok(out) } fn read_config_entry( @@ -535,19 +691,21 @@ impl Adapter for FastlyCliAdapter { if output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); // Parse the JSON and extract the `item_value` field. - let parsed: serde_json::Value = - serde_json::from_str(&stdout).map_err(|err| { - format!( - "failed to parse `fastly config-store-entry describe` JSON: {err}\nraw stdout: {stdout}" - ) - })?; + let parsed: serde_json::Value = serde_json::from_str(&stdout).map_err(|_err| { + format!( + "failed to parse `fastly config-store-entry describe` JSON (parse error \ + redacted; response: {})", + redact_describe_response(&stdout) + ) + })?; let value = parsed .get("item_value") .and_then(serde_json::Value::as_str) .ok_or_else(|| { format!( "`fastly config-store-entry describe` JSON has no string `item_value` field; \ - fastly CLI may have changed its output schema. Raw stdout: {stdout}" + fastly CLI may have changed its output schema. (response: {})", + redact_describe_response(&stdout) ) })?; // Resolve chunk pointers: if `value` is a direct BlobEnvelope it @@ -567,7 +725,7 @@ impl Adapter for FastlyCliAdapter { Err(format!( "`fastly config-store-entry describe --store-id={store_id} --key={key} --json` exited with status {}\nstderr: {}", output.status, - stderr.trim() + redact_stderr(&stderr) )) } @@ -590,42 +748,84 @@ impl Adapter for FastlyCliAdapter { }; let fastly_path = manifest_root.join(rel); let name = store.platform.as_str(); + // A prior-state read failure must never BLOCK the command: the diff just + // cannot be computed, so it degrades to `Unsupported` ("cannot diff"). + // Downstream, a dry-run then reaches the writer's orphan-count + // degradation (spec 12.x) and a real push reaches the writer, which + // fails fatally on malformed TOML or overwrites otherwise. Erroring here + // would newly fail a dry-run that reads nothing today. let raw = match fs::read_to_string(&fastly_path) { Ok(text) => text, Err(err) if err.kind() == ErrorKind::NotFound => { return Ok(ReadConfigEntry::MissingStore); } - Err(err) => { - return Err(format!("failed to read {}: {err}", fastly_path.display())); + Err(_err) => { + return Ok(ReadConfigEntry::Unsupported( + "local fastly.toml could not be read; cannot diff the prior value", + )); } }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", fastly_path.display()))?; - // Probe `[local_server.config_stores.]` — if absent, the store - // has not been seeded locally yet. - let Some(contents) = doc - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(name)) - .and_then(|store_tbl| store_tbl.get("contents")) - else { - return Ok(ReadConfigEntry::MissingStore); + let Ok(doc) = raw.parse::() else { + return Ok(ReadConfigEntry::Unsupported( + "local fastly.toml is not valid TOML; cannot diff the prior value", + )); + }; + // Descend `[local_server.config_stores..contents]` level by level. + // At each level an ABSENT key means the store isn't seeded yet + // (MissingStore), but a key that is PRESENT yet not a table is malformed + // store state — distinct outcomes. Collapsing the malformed case into + // MissingStore (as a plain `.get().and_then()` chain does) would render an + // inaccurate "all values added" diff, so it degrades to "cannot diff". + // + // `descend` returns Ok(None) for absent (-> MissingStore) and + // Err(Unsupported) for present-but-not-a-table. + let descend = |parent: &'_ toml_edit::Item, + child: &str| + -> Result, ReadConfigEntry> { + match parent.get(child) { + None => Ok(None), + Some(item) if item.is_table_like() => Ok(Some(item.clone())), + Some(_) => Err(ReadConfigEntry::Unsupported( + "a local config-store parent table is not a table; cannot diff the prior value", + )), + } + }; + let root_item = toml_edit::Item::Table(doc.as_table().clone()); + let contents_item = (|| { + let Some(local_server) = descend(&root_item, "local_server")? else { + return Ok(None); + }; + let Some(config_stores) = descend(&local_server, "config_stores")? else { + return Ok(None); + }; + let Some(store_tbl) = descend(&config_stores, name)? else { + return Ok(None); + }; + descend(&store_tbl, "contents") + })(); + let contents = match contents_item { + Ok(Some(item)) => item, + Ok(None) => return Ok(ReadConfigEntry::MissingStore), + Err(unsupported) => return Ok(unsupported), + }; + // `contents` MUST be a table of `key = "value"` pairs. (Guaranteed by + // `descend` above, but re-borrow as a table to index it.) + let Some(contents_tbl) = contents.as_table_like() else { + return Ok(ReadConfigEntry::Unsupported( + "local config-store `contents` is not a table; cannot diff the prior value", + )); }; // The contents table is `key = "value"` pairs. - match contents.get(key) { + match contents_tbl.get(key) { Some(item) => { - let value = item.as_str().ok_or_else(|| { - format!( - "`[local_server.config_stores.{name}.contents].{key}` in {} is not a string", - fastly_path.display() - ) - })?; + let Some(value) = item.as_str() else { + return Ok(ReadConfigEntry::Unsupported( + "the local prior value is not a string; cannot diff the prior value", + )); + }; // Resolve chunk pointers using the same toml contents table. - let resolved = - resolve_fastly_config_value(key, value.to_owned(), |chunk_key| match contents - .get(chunk_key) - { + let resolved = resolve_fastly_config_value(key, value.to_owned(), |chunk_key| { + match contents_tbl.get(chunk_key) { Some(chunk_item) => { let chunk_val = chunk_item.as_str().ok_or_else(|| { format!( @@ -636,8 +836,25 @@ impl Adapter for FastlyCliAdapter { Ok(Some(chunk_val.to_owned())) } None => Ok(None), - })?; - Ok(ReadConfigEntry::Present(resolved)) + } + }); + match resolved { + Ok(body) => Ok(ReadConfigEntry::Present(body)), + // A corrupt/invalid prior value must NOT block a local push. + // The whole point of `config push --local` here is to + // OVERWRITE that broken state, and the local writer already + // fail-soft handles a suspicious prior pointer (overwrite, + // warn, prune nothing). Reporting `Unsupported` — "cannot + // diff against this" — lets the write proceed to that path + // instead of aborting the whole command on the diff read. + // (Local only: a single file we are about to replace. The + // cloud read keeps erroring, since we must not overwrite + // remote state we could not read.) + Err(_reason) => Ok(ReadConfigEntry::Unsupported( + "local prior value could not be resolved (corrupt or incomplete chunk \ + state); it will be overwritten by this push", + )), + } } None => Ok(ReadConfigEntry::MissingKey), } @@ -686,10 +903,11 @@ fn fetch_remote_config_store_entry(store_id: &str, key: &str) -> Result Result Result Result, String> { Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let doc: toml_edit::DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let svc = doc .get("service_id") .and_then(|item| item.as_str()) @@ -852,9 +1074,12 @@ fn setup_block_present(path: &Path, kind: &str, id: &str) -> Result return Ok(false), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let doc: toml_edit::DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let plural = format!("{kind}_stores"); Ok(doc .get("setup") @@ -883,9 +1108,12 @@ fn append_fastly_setup(path: &Path, kind: &str, id: &str) -> Result<(), String> Err(err) if err.kind() == ErrorKind::NotFound => String::new(), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let mut doc: DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let mut doc: DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let plural = format!("{kind}_stores"); let parent_entry = doc.entry("setup").or_insert_with(table); @@ -924,7 +1152,8 @@ fn write_fastly_local_config_store( path: &Path, platform_name: &str, entries: &[(String, String)], -) -> Result<(), String> { + gc_roots: &[(String, HashSet)], +) -> Result, String> { use toml_edit::{DocumentMut, Item, Table, Value, table}; let raw = match fs::read_to_string(path) { @@ -932,9 +1161,15 @@ fn write_fastly_local_config_store( Err(err) if err.kind() == ErrorKind::NotFound => String::new(), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let mut doc: DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + // Redacted: `toml_edit`'s parse error quotes the offending source LINE, which + // in a config-store `contents` table is a stored (possibly secret-bearing) + // value. The diff read redacts the same failure; the writer must too. + let mut doc: DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let local_server_entry = doc.entry("local_server").or_insert_with(table); let local_server_tbl = local_server_entry.as_table_mut().ok_or_else(|| { @@ -989,100 +1224,268 @@ fn write_fastly_local_config_store( path.display() ) })?; + // Snapshot prior chunk keys per GC root BEFORE the upsert, using the + // exact keep-set the caller computed for each root (no prefix scan). + let mut plans: Vec = Vec::with_capacity(gc_roots.len()); + for (root_key, new_keys) in gc_roots { + let prior_keys = contents_tbl + .get(root_key) + .and_then(toml_edit::Item::as_str) + .map_or_else(|| Ok(Vec::new()), |value| prior_chunk_keys(root_key, value)); + plans.push(FastlyConfigGcPlan { + new_keys: new_keys.clone(), + prior_keys, + }); + } + + // Upsert the new physical entries. for (key, value) in entries { contents_tbl.insert(key, Item::Value(Value::from(value.clone()))); } + // Prune orphans in the same in-memory rewrite; a suspicious prior + // pointer (Err) warns and deletes nothing. + let mut warnings = Vec::new(); + for plan in &plans { + match orphan_chunk_keys(plan) { + Ok(orphans) => { + for key in orphans { + // Never remove an orphan whose VALUE is itself a + // runtime-readable root (a valid direct envelope or a + // pointer). A chunk-shaped key can hold one — e.g. a small + // envelope padded so its first chunk is a whole envelope — + // and it is independently readable, so deleting it would drop + // live config. Mirrors the cloud GC value-based protection: a + // real chunk payload is a raw fragment (not pointer-kind, not + // a valid envelope) and prunes normally. + let is_root_like = contents_tbl + .get(&key) + .and_then(toml_edit::Item::as_str) + .is_some_and(|value| { + value_is_pointer_kind(value) || gc_classify_root(&key, value).is_ok() + }); + if is_root_like { + warnings.push(format!( + "warning: kept `{key}` -- its value is a runtime-readable root \ + (envelope or pointer), not a chunk payload" + )); + continue; + } + contents_tbl.remove(&key); + } + } + Err(err) => warnings.push(format!("warning: {err}")), + } + } + fs::write(path, doc.to_string()) .map_err(|err| format!("failed to write {}: {err}", path.display()))?; - Ok(()) + Ok(warnings) } // ------------------------------------------------------------------- -// `config push` helpers +// chunk GC helpers (Stage 7 re-push reclamation) // ------------------------------------------------------------------- -/// Shell out to `fastly config-store-entry create --store-id= -/// --key= --value=` for a single entry. Surfaces fastly's -/// stderr verbatim on failure — including the "entry already -/// exists" error, which is the operator's signal to delete the -/// entry (or use `config-store-entry update` manually) before -/// re-running push. -/// Drive a sequential per-entry commit loop and produce the -/// partial-failure diagnostic when the committer fails mid-way. -/// Pure (no I/O) so the diagnostic shape is unit-testable without -/// the fastly CLI on PATH; production calls it with a closure that -/// shells out via `create_config_store_entry`. On success returns -/// the count of committed entries; on failure returns an error -/// string naming committed / failed / not-attempted keys so the -/// operator can resume from a known boundary. -fn push_entries_with_committer( - entries: &[(String, String)], - mut committer: F, -) -> Result -where - F: FnMut(&str, &str) -> Result<(), String>, -{ - let mut pushed: Vec = Vec::with_capacity(entries.len()); - for (key, value) in entries { - if let Err(err) = committer(key, value) { - let remaining: Vec<&str> = entries - .iter() - .skip(pushed.len().saturating_add(1)) - .map(|(remaining_key, _)| remaining_key.as_str()) - .collect(); +/// Expand ONE logical `(root_key, body)` into its physical entries, the +/// exact keep-set for that root, and the value written at the root key. +/// No cross-root prefix scanning (a free-form `--key` can't mislead it). +#[expect( + clippy::type_complexity, + reason = "one-off internal return; a named type would not aid readability" +)] +fn expand_root( + root_key: &str, + body: &str, +) -> Result<(Vec<(String, String)>, HashSet, String), String> { + let expanded = prepare_fastly_config_entries(root_key, body)?; + let new_keys: HashSet = expanded.iter().map(|(key, _)| key.clone()).collect(); + // prepare_* always emits the root entry LAST (root pointer or direct + // value). Make the invariant explicit rather than silently defaulting. + let new_root_value = expanded + .last() + .map(|(_, value)| value.clone()) + .ok_or_else(|| format!("internal: no physical entries produced for root `{root_key}`"))?; + Ok((expanded, new_keys, new_root_value)) +} + +/// Orphans = prior chunk keys not in the new keep-set. Propagates a +/// suspicious-pointer `Err` so the caller can warn and skip GC. +fn orphan_chunk_keys(plan: &FastlyConfigGcPlan) -> Result, String> { + match &plan.prior_keys { + Ok(prior) => Ok(prior + .iter() + .filter(|key| !plan.new_keys.contains(*key)) + .cloned() + .collect()), + Err(err) => Err(err.clone()), + } +} + +/// Reject logical keys that collide with the reserved chunk namespace. +/// `--key` is free-form, so this is enforced at the Fastly adapter +/// boundary: such a key would let a push write into another key's chunk +/// space, and could not be reclaimed correctly. +fn reject_reserved_root_keys(entries: &[(String, String)]) -> Result<(), String> { + for (key, _) in entries { + if key.contains(CHUNK_KEY_INFIX) { return Err(format!( - "fastly push failed at entry `{key}` after committing {committed} of {total} entries; the remaining {remaining_count} entries were NOT pushed.\n Committed (safe to skip on retry): {pushed:?}\n Failed: `{key}` — {err}\n Not attempted (re-push these): {remaining:?}", - committed = pushed.len(), - total = entries.len(), - remaining_count = remaining.len() + "config key `{key}` contains the reserved infix `{CHUNK_KEY_INFIX}`, which collides with Fastly chunk storage; choose a different config key (or --key override)" )); } - pushed.push(key.clone()); } - Ok(pushed.len()) + Ok(()) } -/// Shell `fastly config-store-entry update --upsert --stdin` with -/// the value piped through stdin instead of `--value=` on -/// argv. -/// -/// Two reasons for this exact invocation: +/// Unix epoch seconds. Push-time only (the `cli` feature is native). +fn unix_now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Reject a batch that names the same logical root key more than once. /// -/// 1. `--upsert` (vs. the original `create` subcommand): the prior -/// `create` form errored on any key that already existed in the -/// config store, which made `config push` non-repeatable — -/// after the first push, every follow-up push triggered by a -/// config edit would fail at the first unchanged key. -/// `update --upsert` is documented as "insert or update", which -/// matches the convergent semantic the other config-push paths -/// already have (axum overwrites the JSON, cloudflare's -/// `wrangler kv bulk put` overwrites, spin's -/// `cloud key-value set` overwrites). +/// The adapter trait takes an entry slice and does not enforce uniqueness, +/// but GC builds one plan per entry and snapshots every plan against the +/// SAME prior generation. With `[(root, A), (root, B)]` the last tuple wins +/// the upsert (root = B), yet A's plan would still reclaim `prior - A_keys` +/// — which includes B's freshly-written chunks — leaving the final pointer +/// referencing missing chunks. Rejecting is safer than silently coalescing: +/// a duplicated key is a caller bug, and picking a winner would hide it. +fn reject_duplicate_root_keys(entries: &[(String, String)]) -> Result<(), String> { + let mut seen: HashSet<&str> = HashSet::with_capacity(entries.len()); + for (key, _) in entries { + if !seen.insert(key.as_str()) { + return Err(format!( + "config key `{key}` appears more than once in a single push; each logical key must be pushed exactly once" + )); + } + } + Ok(()) +} + +/// Best-effort per-root orphan count for `config push --local --dry-run`. +/// Navigate to `[local_server.config_stores..contents]` for the +/// dry-run counter. `Ok(None)` when any level is absent (no prior state); +/// `Err` when a level is present but the wrong type — prior state the real +/// writer would reject, so the count must degrade to "unknown", not 0. +fn local_contents_table<'doc>( + doc: &'doc toml_edit::DocumentMut, + platform_name: &str, +) -> Result, String> { + let malformed = || "could not read prior state".to_owned(); + let Some(server_item) = doc.get("local_server") else { + return Ok(None); + }; + let Some(server) = server_item.as_table() else { + return Err(malformed()); + }; + let Some(stores_item) = server.get("config_stores") else { + return Ok(None); + }; + let Some(stores) = stores_item.as_table() else { + return Err(malformed()); + }; + let Some(store_item) = stores.get(platform_name) else { + return Ok(None); + }; + let Some(store) = store_item.as_table() else { + return Err(malformed()); + }; + let Some(contents_item) = store.get("contents") else { + return Ok(None); + }; + contents_item + .as_table() + .map_or_else(|| Err(malformed()), |table| Ok(Some(table))) +} + +/// Reads the current `fastly.toml` (offline) and, for each logical +/// `(root_key, body)`, counts `prior_chunk_keys(root, old) - new_keys` +/// where `new_keys` is the root's OWN expansion. Never fails the dry-run: +/// on a missing file / no prior pointer / direct prior value it reports +/// `Ok(0)`; on unreadable or malformed prior state it reports `Err(reason)` +/// which the caller renders as an "unknown" line. +fn local_orphan_counts_for_dry_run( + path: &Path, + platform_name: &str, + entries: &[(String, String)], +) -> Vec<(String, Result)> { + use toml_edit::DocumentMut; + + // Parse the current file once (best-effort). Absent file => no prior. + let parsed: Result, String> = match fs::read_to_string(path) { + Ok(text) => text + .parse::() + .map(Some) + .map_err(|_err| "could not read prior state".to_owned()), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(_) => Err("could not read prior state".to_owned()), + }; + + entries + .iter() + .map(|(root_key, body)| { + let new_keys = match expand_root(root_key, body) { + Ok((_, keys, _)) => keys, + Err(err) => return (root_key.clone(), Err(err)), + }; + let count = match &parsed { + Err(reason) => Err(reason.clone()), + Ok(None) => Ok(0), + Ok(Some(doc)) => match local_contents_table(doc, platform_name) { + Err(reason) => Err(reason), + Ok(None) => Ok(0), + Ok(Some(contents)) => match contents.get(root_key) { + None => Ok(0), // no prior value for this root + Some(item) => match item.as_str() { + None => Err("could not read prior state".to_owned()), + Some(raw) => match prior_chunk_keys(root_key, raw) { + Ok(prior) => Ok(prior + .iter() + .filter(|key| !new_keys.contains(*key)) + // Match the real prune: an orphan whose value + // is a runtime-readable root is KEPT, so it is + // not counted as a deletion. + .filter(|key| { + contents + .get(key) + .and_then(toml_edit::Item::as_str) + .is_none_or(|text| { + !value_is_pointer_kind(text) + && gc_classify_root(key, text).is_err() + }) + }) + .count()), + Err(_) => Err("suspicious prior pointer".to_owned()), + }, + }, + }, + }, + }; + (root_key.clone(), count) + }) + .collect() +} + +// ------------------------------------------------------------------- +// `config push` helpers +// ------------------------------------------------------------------- + +/// Run `fastly config-store-entry list --store-id= --json` and return each +/// item's `item_key`, `item_value`, and `created_at`. /// -/// 2. `--stdin` (vs. `--value=`): `--value=` exposed every -/// config entry's bytes in `ps`/`/proc//cmdline` listings -/// AND was bounded by the host's `ARG_MAX` (4 KiB to 256 KiB -/// depending on platform — easy to trip with a JSON blob). -/// `--stdin` reads the value from stdin instead — keeps value -/// bytes out of argv and lifts the size cap to whatever the OS -/// pipe buffer + the CLI's read accept (megabytes in practice). -fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<(), String> { +/// The item VALUE is KEPT (not discarded): `config gc` classifies each root by +/// its value (`gc_classify_root`) and reconstructs live generations from the +/// chunk values, so all three fields are required. The value is used internally +/// only and is NEVER echoed into a diagnostic — parse failures redact it via +/// `redact_describe_response`. +fn list_config_store_entries(store_id: &str) -> Result, String> { let store_arg = format!("--store-id={store_id}"); - let key_arg = format!("--key={key}"); - let mut child = Command::new("fastly") - .args([ - "config-store-entry", - "update", - store_arg.as_str(), - key_arg.as_str(), - "--upsert", - "--stdin", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + let output = Command::new("fastly") + .args(["config-store-entry", "list", store_arg.as_str(), "--json"]) + .output() .map_err(|err| { if err.kind() == ErrorKind::NotFound { format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") @@ -1090,2247 +1493,5677 @@ fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<( format!("failed to spawn `fastly`: {err}") } })?; - // Move stdin OUT of child via `take` so the ChildStdin drops at - // end of scope — that closes the pipe and lets the CLI see EOF. - // `child.wait_with_output()` then consumes child cleanly. - let mut stdin = child - .stdin - .take() - .ok_or_else(|| "failed to open stdin pipe to `fastly`".to_owned())?; - stdin - .write_all(value.as_bytes()) - .map_err(|err| format!("failed to write value to `fastly` stdin: {err}"))?; - drop(stdin); - let output = child - .wait_with_output() - .map_err(|err| format!("failed to wait on `fastly`: {err}"))?; - if output.status.success() { - return Ok(()); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "`fastly config-store-entry list --store-id={store_id} --json` exited with status {}\nstderr: {}", + output.status, + redact_stderr(&stderr) + )); } - Err(format!( - "`fastly config-store-entry update --store-id={store_id} --key={key} --upsert --stdin` exited with status {}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )) + let stdout = String::from_utf8_lossy(&output.stdout); + let parsed: serde_json::Value = serde_json::from_str(&stdout).map_err(|_err| { + format!( + "failed to parse `fastly config-store-entry list` JSON (parse error redacted; \ + response: {})", + redact_describe_response(&stdout) + ) + })?; + // A BARE ARRAY ONLY. The installed Fastly CLI returns the complete store as + // a top-level array with no cursor/paging flags. Any other shape (e.g. an + // `{"items":[...], ...}` envelope) may carry pagination metadata we do not + // follow -- and a page that omitted a ROOT while listing its chunks would + // make live chunks look orphaned. The completeness guard cannot see a root + // that isn't there, so we refuse rather than reclaim from a partial view. + let array = parsed.as_array().ok_or_else(|| { + format!( + "refusing to reclaim: `fastly config-store-entry list --json` did not return a bare \ + array (response: {}). This build only supports an unpaginated listing; a partial view \ + could hide a root and orphan its live chunks. Nothing was deleted.", + redact_describe_response(&stdout) + ) + })?; + // FAIL CLOSED on any malformed entry. A missing/non-string field on a + // reclamation input must NEVER be silently skipped or defaulted to empty: + // skipping a root hides the chunks it references (they'd look orphaned and + // get deleted while live), and an empty `item_value` makes a real root + // parse as "references nothing" — same catastrophe. If we can't read the + // listing exactly, we delete nothing. + let mut items = Vec::with_capacity(array.len()); + for (idx, entry) in array.iter().enumerate() { + let field = |name: &str| -> Result { + let raw = entry + .get(name) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!( + "`fastly config-store-entry list` entry #{idx} is missing a string `{name}` \ + field; refusing to reclaim on an unreadable listing (nothing deleted)" + ) + })?; + // An EMPTY field is as dangerous as a missing one: an empty root + // value would classify as "references nothing" and orphan its live + // chunks. Reject it here rather than reason about it later. + if raw.is_empty() { + return Err(format!( + "`fastly config-store-entry list` entry #{idx} has an empty `{name}` field; \ + refusing to reclaim on an unreadable listing (nothing deleted)" + )); + } + Ok(raw.to_owned()) + }; + items.push(ConfigStoreItem { + created_at: field("created_at")?, + item_key: field("item_key")?, + item_value: field("item_value")?, + }); + } + + // DUPLICATE KEYS => fail closed. A key must appear once; a store cannot + // really hold two entries under one key, so duplicate rows mean we are not + // reading the store we think we are (a merged/paginated view, or a CLI + // change). Left alone, the last row silently wins for BOTH the live-set + // lookup and `created_at`, so conflicting rows could age a recent key into + // eligibility and schedule the same key for two deletes. + let mut seen: HashSet<&str> = HashSet::with_capacity(items.len()); + if let Some(duplicate) = items + .iter() + .find(|item| !seen.insert(item.item_key.as_str())) + { + return Err(format!( + "refusing to reclaim: `fastly config-store-entry list` returned key `{}` more than \ + once. A key is unique in a config store, so this listing does not describe one \ + consistent view of it (nothing was deleted).", + duplicate.item_key + )); + } + + Ok(items) } -/// Parse `fastly config-store list --json` output and return the -/// platform `id` of the store whose `name` matches `name`. Accepts -/// both a bare array (`[ {"id": "...", "name": "..."}, ... ]`) -/// and an `{"items": [...]}` envelope so this stays compatible -/// across fastly CLI versions. -fn find_config_store_id(stdout: &str, name: &str) -> ConfigStoreLookup { - let parsed: serde_json::Value = match serde_json::from_str(stdout) { - Ok(value) => value, - Err(err) => { - return ConfigStoreLookup::SchemaDrift(format!("stdout did not parse as JSON: {err}")); - } - }; - let Some(array) = parsed - .as_array() - .or_else(|| parsed.get("items").and_then(serde_json::Value::as_array)) - else { - return ConfigStoreLookup::SchemaDrift(format!( - "expected a bare array `[...]` or an `{{\"items\": [...]}}` envelope; got JSON of shape `{}`", - shape_summary(&parsed) +/// RFC 3339 (`2026-07-13T03:27:42Z`) -> unix seconds. +fn parse_rfc3339_secs(raw: &str) -> Option { + let stamp = chrono::DateTime::parse_from_rfc3339(raw).ok()?; + u64::try_from(stamp.timestamp()).ok() +} + +/// `config gc` for Fastly: delete chunk entries that no LIVE root pointer +/// references and that are older than the operator's `older_than_secs`. +/// +/// Why this is a separate, operator-invoked command rather than part of `config +/// push`: see `Adapter::gc_config_entries`. The operator's `--older-than` is the +/// safety assertion the platform cannot make. A dry-run prints exactly which +/// keys would go, with ages, so the assertion is reviewable. +/// +/// Fails CLOSED: if the listing is unreadable, or a root's value cannot be +/// classified, nothing is deleted. +fn gc_fastly_config_store( + store_name: &str, + older_than_secs: u64, + dry_run: bool, +) -> Result, String> { + // THE destructive boundary enforces its own precondition. The CLI rejects a + // zero window too, but `gc_config_entries` is a public trait method any + // caller can reach directly -- a safety rule that lives only in the CLI is + // not a safety rule. A zero window asserts nothing: it makes every orphan + // eligible, including one superseded a second ago whose pointer POPs are + // still serving. (A dry-run may preview at zero; it deletes nothing.) + if !dry_run && older_than_secs == 0 { + return Err( + "refusing to reclaim: a destructive `config gc` requires a non-zero `--older-than` \ + window. Zero asserts nothing -- it would make every orphan eligible, including \ + chunks a pointer POPs are still serving. Nothing was deleted." + .to_owned(), + ); + } + let resolved_id = resolve_remote_config_store_id(store_name)?; + let items = list_config_store_entries(&resolved_id)?; + let plan = plan_gc_reclamation(&items, unix_now_secs(), older_than_secs)?; + let GcPlan { + doomed, + live_count, + retained_recent, + roots, + unprovable, + } = plan; + + let doomed_count: usize = doomed.iter().map(Vec::len).sum(); + let mut out = vec![format!( + "fastly config-store `{store_name}` (id={resolved_id}): {} entries, {roots} root(s), {live_count} live chunk(s), {doomed_count} orphan(s) in {} generation(s) older than {older_than_secs}s, {retained_recent} orphan(s) too recent", + items.len(), + doomed.len(), + )]; + if unprovable > 0 { + // NEVER silent: these entries look like chunk keys but we could not + // prove our writer produced them, so we left them alone. Say so, or the + // summary reads as "everything reclaimable was reclaimed". + out.push(format!( + " {unprovable} chunk-shaped entr(ies) left untouched: they are not byte-identical to what this writer would produce (wrong content-address, a split this writer would not choose, an incomplete generation, or a count it would never emit), so EdgeZero cannot claim them" )); - }; - let mut any_well_formed = false; - for entry in array { - let entry_name = entry.get("name").and_then(serde_json::Value::as_str); - let entry_id = entry.get("id").and_then(serde_json::Value::as_str); - if entry_name.is_some() && entry_id.is_some() { - any_well_formed = true; - } - if entry_name == Some(name) { - return entry_id.map_or_else( - || { - ConfigStoreLookup::SchemaDrift(format!( - "entry matched name `{name}` but is missing a string `id` field" - )) - }, - |id| ConfigStoreLookup::Found(id.to_owned()), - ); - } } - if array.is_empty() || any_well_formed { - ConfigStoreLookup::NotFound - } else { - ConfigStoreLookup::SchemaDrift( - "no entry has both string `name` and `id` fields -- fastly CLI may have changed its output schema" - .to_owned(), + if doomed_count == 0 { + out.push("nothing to reclaim".to_owned()); + return Ok(out); + } + for (key, age) in doomed.iter().flatten() { + let verb = if dry_run { "would delete" } else { "deleting" }; + out.push(format!(" {verb} `{key}` (age {age}s)")); + } + if dry_run { + out.push(format!( + "dry-run: {doomed_count} orphan chunk(s) would be deleted; re-run with --yes to apply" + )); + return Ok(out); + } + + let GcDeleteOutcome { + deleted, + failed, + stranded, + uncertain, + } = execute_gc_deletes(&resolved_id, &doomed, &mut out); + out.push(format!( + "reclaimed {deleted} of {doomed_count} orphan chunk entries" + )); + if failed.is_empty() { + return Ok(out); + } + // Partial/total failure must be a non-zero exit so automation can see it. + let mut diagnostic = format!( + "{}\nconfig gc: {} of {doomed_count} deletes FAILED ({})", + out.join("\n"), + failed.len(), + failed.join(", ") + ); + // A generation whose only failure was on an unconfirmed delete: the outcome + // is UNKNOWN (Fastly may have committed it), so a re-run is worth trying but + // may find a fragment. + if !uncertain.is_empty() { + write!( + diagnostic, + ".\nNOTE: a failed remote delete has an unknown outcome -- Fastly may have applied it \ + before returning an error. Re-run `config gc`: it reclaims each affected generation \ + if it is still whole, or reports it as an unprovable fragment (\"left untouched\") if \ + a delete did commit. If reported as a fragment, remove the survivors by hand:\n{}", + recovery_commands(&resolved_id, &uncertain) + ) + .map_err(|err| format!("failed to format the gc diagnostic: {err}"))?; + } + // A generation with a CONFIRMED prior delete: definitely a fragment now. + if !stranded.is_empty() { + write!( + diagnostic, + ".\nWARNING: {} entr(ies) are now an INCOMPLETE generation because a sibling was \ + already deleted before the failure: {}. `config gc` proves a generation by \ + reassembling it, so it can no longer prove these and will never reclaim them -- \ + re-running will NOT help. They are inert (no pointer references them). Remove them \ + by hand once you are satisfied they are unreferenced:\n{}", + stranded.len(), + stranded.join(", "), + recovery_commands(&resolved_id, &stranded), ) + .map_err(|err| format!("failed to format the gc diagnostic: {err}"))?; } + Err(diagnostic) } -/// One-line type label for a `serde_json::Value` (for diagnostic -/// error messages — not a canonical JSON-schema description). -fn shape_summary(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", +/// Render copy-pasteable `fastly config-store-entry delete` commands, one per +/// key, with EVERY interpolated value single-quoted for POSIX shells. +/// +/// Root keys are free-form (`--key `), and a chunk key preserves its +/// root, so a key can contain `$(...)`, spaces, or `;`. Pasting an unquoted +/// command could execute or misparse it, so this is not cosmetic. +fn recovery_commands(store_id: &str, keys: &[String]) -> String { + keys.iter() + .map(|key| { + format!( + " fastly config-store-entry delete --store-id={} --key={} --auto-yes", + shell_single_quote(store_id), + shell_single_quote(key), + ) + }) + .collect::>() + .join("\n") +} + +/// Single-quote a value for a POSIX shell: wrap in `'...'` and rewrite each +/// embedded `'` as `'\''`. Inside single quotes every other byte -- `$`, spaces, +/// `;`, `$(...)`, backticks -- is literal, so this neutralises any hostile key. +fn shell_single_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +/// Delete each doomed generation, stopping a generation at its FIRST failure. +/// +/// A generation is provable only as a whole (`prove_generation` reassembles it), +/// so a half-deleted one can never be proved again: the next run sees a fragment, +/// cannot verify it, and correctly refuses to touch it — forever. Ploughing on +/// after a failure is therefore the one thing that turns a possibly-recoverable +/// error into permanent, unreclaimable litter. +/// +/// A failed remote delete has an UNKNOWN outcome — Fastly may commit it before +/// returning an error — so nothing here is promised as cleanly retryable. The +/// caller distinguishes two cases: a failure with a CONFIRMED prior sibling +/// delete strands the survivors for good (manual recovery), and a failure with +/// no confirmed prior delete leaves the generation in an UNCERTAIN state (a +/// re-run may reclaim it, or surface it as an unprovable fragment). Generations +/// are independent, so a failure in one does not stop the others. +fn execute_gc_deletes( + resolved_id: &str, + doomed: &[Vec<(String, u64)>], + out: &mut Vec, +) -> GcDeleteOutcome { + let mut outcome = GcDeleteOutcome { + deleted: 0, + failed: Vec::new(), + stranded: Vec::new(), + uncertain: Vec::new(), + }; + for generation in doomed { + let mut deleted_here: Vec<&str> = Vec::new(); + for (key, _) in generation { + match delete_config_store_entry(resolved_id, key) { + Ok(()) => { + outcome.deleted = outcome.deleted.saturating_add(1); + deleted_here.push(key.as_str()); + } + Err(err) => { + out.push(format!(" FAILED to delete `{key}` ({err})")); + outcome.failed.push(key.clone()); + // Everything in this generation we have NOT confirmed deleted + // -- the failed key itself, plus the ones we never reached. + let unconfirmed: Vec = generation + .iter() + .map(|(member, _)| member.clone()) + .filter(|member| !deleted_here.contains(&member.as_str())) + .collect(); + if deleted_here.is_empty() { + // No sibling is CONFIRMED gone. The failed delete's + // outcome is unknown: if it did not commit, the + // generation is whole and a re-run reclaims it; if it + // did, the re-run finds a fragment and reports it. Either + // way we must not claim clean retryability. + outcome.uncertain.extend(unconfirmed); + } else { + // A sibling is CONFIRMED gone, so this generation is + // definitely a fragment no future run can prove. + outcome.stranded.extend(unconfirmed); + } + break; // stop THIS generation; the others are independent + } + } + } } + outcome } -/// Resolve the platform config-store id on demand: shell out to -/// `fastly config-store list --json`, parse the JSON, match by -/// `name`. The provision flow doesn't persist this id, so push -/// has to re-fetch every time. -fn resolve_remote_config_store_id(name: &str) -> Result { - let output = Command::new("fastly") - .args(["config-store", "list", "--json"]) - .output() - .map_err(|err| { - if err.kind() == ErrorKind::NotFound { - format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") - } else { - format!("failed to spawn `fastly`: {err}") +/// Classify a store's entries: the live chunk set, the protected root keys, and +/// the root count. +/// +/// Root-vs-chunk is decided by VALUE, not key shape. The runtime resolver reads +/// whatever value sits at a key, so ANY entry whose value is a valid direct +/// envelope or a chunk pointer is a runtime-readable root and must never be +/// deleted — even at a chunk-shaped key. Two ways that happens: +/// +/// - a pointer parked at a chunk-shaped key makes its references LIVE; +/// - a value that is itself a valid direct envelope (e.g. a small envelope whose +/// first 7 000-byte chunk is the whole envelope plus trailing whitespace, and +/// so still parses and verifies) is a root in its own right. +/// +/// Only a value that is NEITHER — a raw envelope fragment, which does not parse — +/// is a delete candidate. In normal operation a chunk payload is exactly such a +/// fragment, so this protects the pathological cases at no cost to real GC. +fn classify_store_entries( + items: &[ConfigStoreItem], + value_by_key: &HashMap<&str, &str>, +) -> Result { + let mut live: HashSet = HashSet::new(); + let mut protected: HashSet = HashSet::new(); + let mut roots = 0_usize; + for item in items { + let is_chunk_shaped = chunk_key_generation_any(&item.item_key).is_some(); + let classified = match gc_classify_root(&item.item_key, &item.item_value) { + Ok(classified) => classified, + // A chunk-shaped key whose value we cannot classify is a genuine + // chunk fragment (a candidate) ONLY if that value is not itself + // root-like. A pointer-kind value is always root-like — the runtime + // reads it as a pointer — so an unclassifiable one (e.g. a + // cross-root pointer that fails this root's scope check) must FAIL + // CLOSED, never become a deletable candidate whose references we + // would orphan. + Err(_) if is_chunk_shaped && !value_is_pointer_kind(&item.item_value) => { + continue; // a chunk payload: a delete candidate + } + Err(err) => { + return Err(format!( + "refusing to reclaim: could not classify root `{}` ({err}); nothing was deleted", + item.item_key + )); } + }; + // A runtime-readable root, wherever it lives: never a delete candidate. + roots = roots.saturating_add(1); + protected.insert(item.item_key.clone()); + let GcRootValue::Chunked(pointer) = classified else { + continue; // A direct envelope references no chunks. + }; + // The pointer's METADATA is self-consistent by here. That is not proof + // that it honestly describes its generation: a pointer can drop its last + // chunk ref AND restate `envelope_len` as the remaining sum, and every + // metadata check still passes while the dropped chunk silently leaves + // the live set and becomes deletable. So reassemble what it references + // and hold the bytes against its content-address. + let assembled = assemble_pointer_chunks(&item.item_key, &pointer, value_by_key)?; + gc_verify_generation(&pointer.envelope_sha256, &assembled).map_err(|err| { + format!( + "refusing to reclaim: root `{}` names a chunk set that does not reconstruct the \ + envelope it claims ({err}). Its chunk list is therefore not a trustworthy live \ + set, and treating it as one could delete a live chunk. Nothing was deleted.", + item.item_key + ) })?; - if !output.status.success() { - return Err(format!( - "`fastly config-store list --json` exited with status {}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - let stdout = String::from_utf8_lossy(&output.stdout); - match find_config_store_id(&stdout, name) { - ConfigStoreLookup::Found(id) => Ok(id), - ConfigStoreLookup::NotFound => Err(format!( - "no fastly config-store matches `{name}` (did you run `edgezero provision --adapter fastly`?)" - )), - ConfigStoreLookup::SchemaDrift(detail) => Err(format!( - "could not parse `fastly config-store list --json` output: {detail}.\n The fastly CLI may have changed its JSON schema in a recent version. Please file a bug report at https://github.com/stackpop/edgezero/issues with the fastly CLI version (`fastly version`) and the raw stdout. Workaround: pin to a known-compatible fastly CLI version." - )), + live.extend(pointer.chunks.into_iter().map(|chunk| chunk.key)); } + Ok(GcClassification { + live, + protected, + roots, + }) } -/// # Errors -/// Returns an error if the Fastly CLI build command fails. -#[inline] -pub fn build(extra_args: &[String]) -> Result { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; - let cargo_manifest = manifest_dir.join("Cargo.toml"); - let crate_name = read_package_name(&cargo_manifest)?; - - let status = Command::new("cargo") - .args([ - "build", - "--release", - "--target", - "wasm32-wasip1", - "--manifest-path", - cargo_manifest - .to_str() - .ok_or("invalid Cargo manifest path")?, - ]) - .args(extra_args) - .status() - .map_err(|err| format!("failed to run cargo build: {err}"))?; - if !status.success() { - return Err(format!("cargo build failed with status {status}")); +/// The reclamation plan for one store: which orphan chunk entries to delete, and +/// the counts for the summary line. Deriving it is where every safety guard +/// lives, so it is fail-closed throughout — any unreadable/incomplete state +/// returns `Err` and the caller deletes nothing. +/// +/// The organising idea is that **content-addressing makes a chunk set +/// self-proving**: a chunk key embeds the SHA-256 of the whole envelope it +/// belongs to, so reassembling a generation either reproduces the +/// content-address its own keys name, or it does not. Every destructive decision +/// here rests on that hash — never on what the store's metadata claims about +/// itself, which is exactly what an inconsistent store gets wrong. +fn plan_gc_reclamation( + items: &[ConfigStoreItem], + now: u64, + older_than_secs: u64, +) -> Result { + let mut value_by_key: HashMap<&str, &str> = HashMap::with_capacity(items.len()); + let mut created_by_key: HashMap<&str, u64> = HashMap::with_capacity(items.len()); + for item in items { + let Some(created) = parse_rfc3339_secs(&item.created_at) else { + // Unparseable timestamp anywhere in the listing -> fail closed. On a + // DELETE path we will not guess an age. + return Err(format!( + "refusing to reclaim: entry `{}` has an unreadable `created_at`; nothing was deleted", + item.item_key + )); + }; + created_by_key.insert(item.item_key.as_str(), created); + value_by_key.insert(item.item_key.as_str(), item.item_value.as_str()); } - let workspace_root = find_workspace_root(manifest_dir); - let artifact = locate_artifact(&workspace_root, manifest_dir, &crate_name)?; - let pkg_dir = workspace_root.join("pkg"); - fs::create_dir_all(&pkg_dir) - .map_err(|err| format!("failed to create {}: {err}", pkg_dir.display()))?; - let dest = pkg_dir.join(format!("{}.wasm", crate_name.replace('-', "_"))); - fs::copy(&artifact, &dest) - .map_err(|err| format!("failed to copy artifact to {}: {err}", dest.display()))?; + // ---- 1. Classify entries: live chunks, protected roots, root count ---- + let GcClassification { + live, + protected, + roots, + } = classify_store_entries(items, &value_by_key)?; + + // ---- 2. Per-root live-config age (best-effort; see the guard below) ---- + // rsplit_once (the LAST infix): a chunk of a chunk-shaped root nests the infix + // twice, and its root is everything before the LAST one. Splitting on the + // first would attribute a nested chunk's age to the wrong (outer) root. + let root_live_since: HashMap<&str, u64> = live.iter().fold(HashMap::new(), |mut acc, key| { + if let Some((root, _)) = key.rsplit_once(CHUNK_KEY_INFIX) { + let created = *created_by_key.get(key.as_str()).unwrap_or(&0); + let slot = acc.entry(root).or_insert(0); + *slot = (*slot).max(created); + } + acc + }); - Ok(dest) -} + // ---- 3. Candidates, grouped by GENERATION and proven writer-produced ---- + // A per-key decision cannot be safe: an entry is only ours if the whole + // generation it belongs to reassembles to the content-address its keys name. + // So group first, prove second, and delete whole generations or none -- a + // partial delete would leave a corrupt generation behind. + let mut groups: BTreeMap<(&str, String), Vec<&ConfigStoreItem>> = BTreeMap::new(); + for item in items { + if live.contains(&item.item_key) { + continue; + } + // A key whose own value is a runtime-readable root is never a candidate, + // even when its key is chunk-shaped (a valid direct envelope can sit at + // one). Excluding it here also means any real chunk sharing that + // generation drops to an incomplete group, which prove_generation then + // leaves untouched — safe: we leak rather than delete a possible root. + if protected.contains(&item.item_key) { + continue; + } + // rsplit_once (the LAST infix): the same nested-chunk correctness the + // live-set scan and classification use — a chunk of a chunk-shaped root + // is grouped under THAT root, not the outer one, so nested orphans are + // grouped (and thus reclaimed or reported), not silently dropped. + let Some((root, _)) = item.item_key.rsplit_once(CHUNK_KEY_INFIX) else { + continue; // a root + }; + let Some(generation) = chunk_key_generation(root, &item.item_key) else { + continue; // chunk-shaped but NOT canonical => never a key we emit + }; + groups.entry((root, generation)).or_default().push(item); + } -/// # Errors -/// Returns an error if the Fastly CLI deploy command fails. -#[inline] -pub fn deploy(extra_args: &[String]) -> Result<(), String> { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + let mut doomed: Vec> = Vec::new(); + let mut retained_recent = 0_usize; + let mut unprovable = 0_usize; + for ((root, generation), group) in groups { + if prove_generation(root, &generation, &group).is_err() { + // We cannot prove we wrote this, so we do not touch it. It may be an + // ordinary entry that merely LOOKS like a chunk key (a store can + // predate this feature or be shared, and push-time reserved-key + // rejection cannot protect what already exists), or a half-written + // generation. Skipped rather than fatal: one foreign entry must not + // block reclamation of the store forever. Reported in the summary. + unprovable = unprovable.saturating_add(group.len()); + continue; + } - let status = Command::new("fastly") - .args(["compute", "deploy"]) - .args(extra_args) - .current_dir(manifest_dir) - .status() - .map_err(|err| format!("failed to run fastly CLI: {err}"))?; - if !status.success() { - return Err(format!("fastly compute deploy failed with status {status}")); + // Age the generation as a UNIT, by its youngest member: deleting a + // generation is one decision, so its most restrictive age governs. + let group_age = group + .iter() + .map(|item| { + now.saturating_sub(*created_by_key.get(item.item_key.as_str()).unwrap_or(&0)) + }) + .min() + .unwrap_or(0); + // BOTH ages must clear the operator's window; neither substitutes for + // the other, so take the more restrictive (the MINIMUM). + // + // - The chunks' OWN age is mandatory: a generation written seconds ago + // is inside the propagation window whatever its root looks like (e.g. + // a concurrent push wrote it and has not committed its pointer yet), + // so an old-looking root must never license deleting it. + // - The root's live-config age (when known) is an EXTRA restriction: it + // catches an old generation superseded recently, which its own age + // cannot see. + let effective_age = root_live_since.get(root).map_or(group_age, |live_since| { + group_age.min(now.saturating_sub(*live_since)) + }); + if effective_age < older_than_secs { + retained_recent = retained_recent.saturating_add(group.len()); + continue; + } + doomed.push( + group + .iter() + .map(|item| { + let age = now + .saturating_sub(*created_by_key.get(item.item_key.as_str()).unwrap_or(&0)); + (item.item_key.clone(), age) + }) + .collect(), + ); } - Ok(()) + Ok(GcPlan { + doomed, + live_count: live.len(), + retained_recent, + roots, + unprovable, + }) } -fn find_fastly_manifest(start: &Path) -> Result { - if let Some(found) = find_manifest_upwards(start, "fastly.toml") { - return Ok(found); +/// Reassemble the chunks a live pointer references, in index order, checking each +/// against the pointer's own per-chunk `len`/`sha256` along the way. +/// +/// Fails closed when a referenced key is absent from the listing. This subsumes +/// the old standalone completeness guard: an incomplete or paginated listing +/// cannot produce the bytes, so it can never reach a passing verification. +fn assemble_pointer_chunks( + root_key: &str, + pointer: &GcPointer, + value_by_key: &HashMap<&str, &str>, +) -> Result { + // NOT `with_capacity(pointer.envelope_len)`: that length is untrusted stored + // metadata. `validate_pointer_chunks` bounds it, but this is a destructive + // path -- do not reserve from a number the store supplied when growing from + // the bytes we actually read costs nothing. + let mut assembled = String::new(); + // The chunk KEY is pointer-controlled (a malformed pointer can carry any + // string there), so diagnostics name a POSITION, not the key. `root_key` is + // the operator's own logical entry key and is named for context, as the rest + // of the GC diagnostics do. + for (position, chunk) in pointer.chunks.iter().enumerate() { + let Some(value) = value_by_key.get(chunk.key.as_str()) else { + return Err(format!( + "refusing to reclaim: root `{root_key}` references chunk {position}, which is \ + absent from the store listing (the listing may be incomplete/paginated, or the \ + store is already inconsistent); nothing was deleted" + )); + }; + if value.len() != chunk.len { + return Err(format!( + "refusing to reclaim: root `{root_key}` says chunk {position} is {} bytes but the \ + store holds {}; nothing was deleted", + chunk.len, + value.len() + )); + } + if sha256_hex(value.as_bytes()) != chunk.sha256 { + return Err(format!( + "refusing to reclaim: the stored value of chunk {position} does not match the \ + SHA-256 that root `{root_key}` records for it; nothing was deleted" + )); + } + assembled.push_str(value); } + if assembled.len() != pointer.envelope_len { + return Err(format!( + "refusing to reclaim: root `{root_key}` declares an envelope of {} bytes but its \ + chunks reassemble to {}; nothing was deleted", + pointer.envelope_len, + assembled.len() + )); + } + Ok(assembled) +} - let root = find_workspace_root(start); - let mut candidates: Vec = WalkDir::new(&root) - .follow_links(true) - .max_depth(8) - .into_iter() - .filter_map(Result::ok) - .map(|entry| entry.path().to_path_buf()) - .filter(|path| { - path.file_name().is_some_and(|n| n == "fastly.toml") - && path - .parent() - .is_some_and(|dir| dir.join("Cargo.toml").exists()) - }) - .collect(); - - if candidates.is_empty() { - return Err("could not locate fastly.toml".to_owned()); +/// Is this candidate generation byte-identical to what THIS writer would have +/// produced for the bytes it contains? +/// +/// The gate on every delete. `group` is every listed entry sharing one +/// `(root, generation)`. +/// +/// **What this proves, precisely.** We reassemble the group in index order and +/// re-run `prepare_fastly_config_entries` over the result. If the writer, given +/// those exact bytes, would emit exactly these keys and these values, the entries +/// are indistinguishable from our own output: same direct-vs-chunked threshold, +/// same UTF-8-safe 7 000-byte boundaries, same content-addressed keys, same +/// count. A lone chunk fails automatically (an envelope small enough to store +/// directly round-trips to a single ROOT-keyed entry, and a large one to >= 2 +/// chunks), as does any set split at boundaries we would not choose. +/// +/// **What this does NOT prove: authorship.** Content-addressing is not a +/// signature. A foreign writer can pick envelope E, compute `H = sha256(E)`, +/// split E exactly as we would, and store the parts under our reserved +/// `.__edgezero_chunks.` namespace; that group is byte-identical to ours and we +/// will reclaim it. No preimage attack is needed, and no check over the stored +/// bytes alone can separate the two — telling them apart needs trusted +/// generation metadata or an authenticated marker, and the store offers neither +/// (any writer with store access could forge either). +/// +/// We accept that residual: the namespace is reserved by convention, push-time +/// validation rejects logical keys inside it, and anything passing this gate is +/// a faithful reproduction of our format. The spec documents it as a limitation +/// rather than claiming a guarantee we cannot make. +fn prove_generation( + root: &str, + generation: &str, + group: &[&ConfigStoreItem], +) -> Result<(), String> { + let mut ordered: Vec<(usize, &str)> = Vec::with_capacity(group.len()); + for item in group { + let index = item + .item_key + .rsplit_once('.') + .and_then(|(_, index)| index.parse::().ok()) + .ok_or_else(|| format!("`{}` has no readable index", item.item_key))?; + ordered.push((index, item.item_value.as_str())); + } + ordered.sort_by_key(|&(index, _)| index); + for (position, &(index, _)) in ordered.iter().enumerate() { + if index != position { + return Err(format!( + "indexes are not dense 0..n-1 (found {index} at position {position})" + )); + } + } + let assembled: String = ordered.iter().map(|&(_, value)| value).collect(); + + // 1. The bytes must be the generation the keys name, and a real envelope. + gc_verify_generation(generation, &assembled)?; + + // 2. ...and the writer, given those bytes, must produce EXACTLY these + // entries. This is what pins the split boundaries and the chunked-vs- + // direct threshold, so a set assembled by anything that does not + // reproduce our writer's output byte-for-byte is left alone. + let expected = prepare_fastly_config_entries(root, &assembled) + .map_err(|err| format!("this writer could not re-derive the generation ({err})"))?; + let Some(expected_chunks) = expected.get(..expected.len().saturating_sub(1)) else { + return Err("this writer produced no chunk entries for these bytes".to_owned()); + }; + if expected_chunks.is_empty() { + // The envelope fits directly, so the writer would never have chunked it: + // whatever these entries are, they are not ours. + return Err( + "these bytes fit the entry limit, so this writer would have stored them directly \ + rather than in chunks" + .to_owned(), + ); + } + if expected_chunks.len() != ordered.len() { + return Err(format!( + "this writer would split these bytes into {} chunk(s), not {}", + expected_chunks.len(), + ordered.len() + )); + } + for ((expected_key, expected_value), item) in + expected_chunks.iter().zip(group_in_index_order(group)) + { + if *expected_key != item.item_key { + return Err(format!( + "this writer would not have produced the key `{}`", + item.item_key + )); + } + if *expected_value != item.item_value { + return Err(format!( + "the stored value of `{}` is not the chunk this writer would have written at that \ + index", + item.item_key + )); + } } + Ok(()) +} - candidates.sort_by_key(|path| { - let parent = path.parent().unwrap_or(Path::new("")); - path_distance(start, parent) +/// `group` sorted by chunk index, so it lines up with the writer's output order. +fn group_in_index_order<'item>(group: &[&'item ConfigStoreItem]) -> Vec<&'item ConfigStoreItem> { + let mut ordered: Vec<&ConfigStoreItem> = group.to_vec(); + ordered.sort_by_key(|item| { + item.item_key + .rsplit_once('.') + .and_then(|(_, index)| index.parse::().ok()) + .unwrap_or(usize::MAX) }); - - Ok(candidates.remove(0)) + ordered } -fn locate_artifact( - workspace_root: &Path, - manifest_dir: &Path, - crate_name: &str, -) -> Result { - let target_triple = "wasm32-wasip1"; - let release_name = format!("{}.wasm", crate_name.replace('-', "_")); +/// Is this key a chunk key of ANY root? (`config gc` scans the whole store, so +/// it cannot scope to one root up front.) Validates the canonical shape. +fn chunk_key_generation_any(key: &str) -> Option { + // Split on the LAST infix, not the first: a chunk of a root that ITSELF + // contains the infix (a pointer parked at a chunk-shaped key with self-scoped + // chunks) has the infix twice, and its chunk suffix is after the LAST one. + // Splitting on the first would misread the doubly-nested chunk as a + // non-chunk, get it classified as an unclassifiable root, and abort the whole + // store's GC. For an ordinary single-infix key the root has no infix, so the + // last infix IS the first — this only changes the nested case. + let (root, _rest) = key.rsplit_once(CHUNK_KEY_INFIX)?; + chunk_key_generation(root, key) +} - if let Some(custom) = env::var_os("CARGO_TARGET_DIR") { - let candidate = PathBuf::from(custom) - .join(target_triple) - .join("release") +/// Drive a sequential per-entry commit loop and produce the +/// partial-failure diagnostic when the committer fails mid-way. +/// Pure (no I/O) so the diagnostic shape is unit-testable without +/// the fastly CLI on PATH; production calls it with a closure that +/// shells out via `create_config_store_entry`. On success returns +/// the count of committed entries; on failure returns an error +/// string naming committed / failed / not-attempted keys so the +/// operator can resume from a known boundary. +fn push_entries_with_committer( + entries: &[(String, String)], + mut committer: F, +) -> Result +where + F: FnMut(&str, &str) -> Result<(), String>, +{ + let mut pushed: Vec = Vec::with_capacity(entries.len()); + for (key, value) in entries { + if let Err(err) = committer(key, value) { + let remaining: Vec<&str> = entries + .iter() + .skip(pushed.len().saturating_add(1)) + .map(|(remaining_key, _)| remaining_key.as_str()) + .collect(); + return Err(format!( + "fastly push failed at entry `{key}` after committing {committed} of {total} entries; the remaining {remaining_count} entries were NOT pushed.\n Committed (safe to skip on retry): {pushed:?}\n Failed: `{key}` — {err}\n Not attempted (re-push these): {remaining:?}", + committed = pushed.len(), + total = entries.len(), + remaining_count = remaining.len() + )); + } + pushed.push(key.clone()); + } + Ok(pushed.len()) +} + +/// Shell `fastly config-store-entry update --upsert --stdin` with +/// the value piped through stdin instead of `--value=` on +/// argv. +/// +/// Two reasons for this exact invocation: +/// +/// 1. `--upsert` (vs. the original `create` subcommand): the prior +/// `create` form errored on any key that already existed in the +/// config store, which made `config push` non-repeatable — +/// after the first push, every follow-up push triggered by a +/// config edit would fail at the first unchanged key. +/// `update --upsert` is documented as "insert or update", which +/// matches the convergent semantic the other config-push paths +/// already have (axum overwrites the JSON, cloudflare's +/// `wrangler kv bulk put` overwrites, spin's +/// `cloud key-value set` overwrites). +/// +/// 2. `--stdin` (vs. `--value=`): `--value=` exposed every +/// config entry's bytes in `ps`/`/proc//cmdline` listings +/// AND was bounded by the host's `ARG_MAX` (4 KiB to 256 KiB +/// depending on platform — easy to trip with a JSON blob). +/// `--stdin` reads the value from stdin instead — keeps value +/// bytes out of argv and lifts the size cap to whatever the OS +/// pipe buffer + the CLI's read accept (megabytes in practice). +fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<(), String> { + let store_arg = format!("--store-id={store_id}"); + let key_arg = format!("--key={key}"); + let mut child = Command::new("fastly") + .args([ + "config-store-entry", + "update", + store_arg.as_str(), + key_arg.as_str(), + "--upsert", + "--stdin", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + // Move stdin OUT of child via `take` so the ChildStdin drops at + // end of scope — that closes the pipe and lets the CLI see EOF. + // `child.wait_with_output()` then consumes child cleanly. + let mut stdin = child + .stdin + .take() + .ok_or_else(|| "failed to open stdin pipe to `fastly`".to_owned())?; + stdin + .write_all(value.as_bytes()) + .map_err(|err| format!("failed to write value to `fastly` stdin: {err}"))?; + drop(stdin); + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait on `fastly`: {err}"))?; + if output.status.success() { + return Ok(()); + } + Err(format!( + "`fastly config-store-entry update --store-id={store_id} --key={key} --upsert --stdin` exited with status {}\nstderr: {}", + output.status, + redact_stderr(&String::from_utf8_lossy(&output.stderr)) + )) +} + +fn delete_config_store_entry(store_id: &str, key: &str) -> Result<(), String> { + let store_arg = format!("--store-id={store_id}"); + let key_arg = format!("--key={key}"); + let output = Command::new("fastly") + .args([ + "config-store-entry", + "delete", + store_arg.as_str(), + key_arg.as_str(), + "--auto-yes", + ]) + .output() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + if output.status.success() { + return Ok(()); + } + // EVERY non-zero delete is a failure -- no "already gone" special case. + // Pattern-matching stderr for "not found"/"404" cannot reliably tell "this + // key is already gone" from "the store does not exist", an auth failure, or + // a 500: messages like `config store abc does not exist while deleting key + // ` name the key AND say "does not exist". Reporting those as a + // successful reclamation is strictly worse than a retry, and a retry is + // free: `config gc` re-lists the store, so a key that really is gone simply + // will not appear as a candidate next run. + // Redact stderr: a Fastly error can quote the entry value back, which on the + // delete path would put a stored config value into CI logs. + let stderr = String::from_utf8_lossy(&output.stderr); + Err(format!( + "`fastly config-store-entry delete --store-id={store_id} --key={key} --auto-yes` exited with status {}\n{}", + output.status, + redact_stderr(&stderr) + )) +} + +/// Parse `fastly config-store list --json` output and return the +/// platform `id` of the store whose `name` matches `name`. Accepts +/// both a bare array (`[ {"id": "...", "name": "..."}, ... ]`) +/// and an `{"items": [...]}` envelope so this stays compatible +/// across fastly CLI versions. +fn find_config_store_id(stdout: &str, name: &str) -> ConfigStoreLookup { + let parsed: serde_json::Value = match serde_json::from_str(stdout) { + Ok(value) => value, + Err(err) => { + return ConfigStoreLookup::SchemaDrift(format!("stdout did not parse as JSON: {err}")); + } + }; + let Some(array) = parsed + .as_array() + .or_else(|| parsed.get("items").and_then(serde_json::Value::as_array)) + else { + return ConfigStoreLookup::SchemaDrift(format!( + "expected a bare array `[...]` or an `{{\"items\": [...]}}` envelope; got JSON of shape `{}`", + shape_summary(&parsed) + )); + }; + let mut any_well_formed = false; + for entry in array { + let entry_name = entry.get("name").and_then(serde_json::Value::as_str); + let entry_id = entry.get("id").and_then(serde_json::Value::as_str); + if entry_name.is_some() && entry_id.is_some() { + any_well_formed = true; + } + if entry_name == Some(name) { + return entry_id.map_or_else( + || { + ConfigStoreLookup::SchemaDrift(format!( + "entry matched name `{name}` but is missing a string `id` field" + )) + }, + |id| ConfigStoreLookup::Found(id.to_owned()), + ); + } + } + if array.is_empty() || any_well_formed { + ConfigStoreLookup::NotFound + } else { + ConfigStoreLookup::SchemaDrift( + "no entry has both string `name` and `id` fields -- fastly CLI may have changed its output schema" + .to_owned(), + ) + } +} + +/// Summarise a `fastly ... describe` response for diagnostics WITHOUT +/// leaking its contents. +/// +/// The response body is the stored config value. App config may hold +/// credentials, internal endpoints, or security policy, and this adapter +/// performs no secret stripping — while CLI status lines are logged +/// verbatim and CI logs are commonly retained and shared. So a schema-drift +/// diagnostic must never echo the payload: report only its size and its +/// top-level *shape* (field names for an object, type otherwise), never a +/// value. +fn redact_describe_response(stdout: &str) -> String { + let len = stdout.len(); + serde_json::from_str::(stdout).map_or_else( + |_err| format!("{len} bytes, not valid JSON"), + |value| match value { + serde_json::Value::Object(map) => { + // Object KEYS are stored/provider-controlled data (a wrong-shape + // response could be `{"": ...}`), so only the COUNT is + // reported, never the key names. + format!("{len} bytes, JSON object with {} field(s)", map.len()) + } + other @ (serde_json::Value::Null + | serde_json::Value::Bool(_) + | serde_json::Value::Number(_) + | serde_json::Value::String(_) + | serde_json::Value::Array(_)) => { + format!("{len} bytes, JSON {}", shape_summary(&other)) + } + }, + ) +} + +/// Summarise a failing `fastly` invocation's stderr WITHOUT echoing it. +/// +/// The `describe` and `update --stdin` paths carry the stored config value, so +/// a Fastly error that quotes the payload back would put credentials straight +/// into CI logs — the same exposure as the stdout leak, via the failure branch. +/// Not-found *classification* still inspects stderr internally; only the +/// user-facing string is redacted. +fn redact_stderr(stderr: &str) -> String { + let len = stderr.trim().len(); + format!( + "{len} bytes suppressed (may echo the stored config value); re-run the `fastly` command directly to inspect it" + ) +} + +/// One-line type label for a `serde_json::Value` (for diagnostic +/// error messages — not a canonical JSON-schema description). +fn shape_summary(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + +/// Resolve the platform config-store id on demand: shell out to +/// `fastly config-store list --json`, parse the JSON, match by +/// `name`. The provision flow doesn't persist this id, so push +/// has to re-fetch every time. +fn resolve_remote_config_store_id(name: &str) -> Result { + let output = Command::new("fastly") + .args(["config-store", "list", "--json"]) + .output() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + if !output.status.success() { + return Err(format!( + "`fastly config-store list --json` exited with status {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let stdout = String::from_utf8_lossy(&output.stdout); + match find_config_store_id(&stdout, name) { + ConfigStoreLookup::Found(id) => Ok(id), + ConfigStoreLookup::NotFound => Err(format!( + "no fastly config-store matches `{name}` (did you run `edgezero provision --adapter fastly`?)" + )), + ConfigStoreLookup::SchemaDrift(detail) => Err(format!( + "could not parse `fastly config-store list --json` output: {detail}.\n The fastly CLI may have changed its JSON schema in a recent version. Please file a bug report at https://github.com/stackpop/edgezero/issues with the fastly CLI version (`fastly version`) and the raw stdout. Workaround: pin to a known-compatible fastly CLI version." + )), + } +} + +/// # Errors +/// Returns an error if the Fastly CLI build command fails. +#[inline] +pub fn build(extra_args: &[String]) -> Result { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + let cargo_manifest = manifest_dir.join("Cargo.toml"); + let crate_name = read_package_name(&cargo_manifest)?; + + let status = Command::new("cargo") + .args([ + "build", + "--release", + "--target", + "wasm32-wasip1", + "--manifest-path", + cargo_manifest + .to_str() + .ok_or("invalid Cargo manifest path")?, + ]) + .args(extra_args) + .status() + .map_err(|err| format!("failed to run cargo build: {err}"))?; + if !status.success() { + return Err(format!("cargo build failed with status {status}")); + } + + let workspace_root = find_workspace_root(manifest_dir); + let artifact = locate_artifact(&workspace_root, manifest_dir, &crate_name)?; + let pkg_dir = workspace_root.join("pkg"); + fs::create_dir_all(&pkg_dir) + .map_err(|err| format!("failed to create {}: {err}", pkg_dir.display()))?; + let dest = pkg_dir.join(format!("{}.wasm", crate_name.replace('-', "_"))); + fs::copy(&artifact, &dest) + .map_err(|err| format!("failed to copy artifact to {}: {err}", dest.display()))?; + + Ok(dest) +} + +/// # Errors +/// Returns an error if the Fastly CLI deploy command fails. +#[inline] +pub fn deploy(extra_args: &[String]) -> Result<(), String> { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + + let status = Command::new("fastly") + .args(["compute", "deploy"]) + .args(extra_args) + .current_dir(manifest_dir) + .status() + .map_err(|err| format!("failed to run fastly CLI: {err}"))?; + if !status.success() { + return Err(format!("fastly compute deploy failed with status {status}")); + } + + Ok(()) +} + +fn find_fastly_manifest(start: &Path) -> Result { + if let Some(found) = find_manifest_upwards(start, "fastly.toml") { + return Ok(found); + } + + let root = find_workspace_root(start); + let mut candidates: Vec = WalkDir::new(&root) + .follow_links(true) + .max_depth(8) + .into_iter() + .filter_map(Result::ok) + .map(|entry| entry.path().to_path_buf()) + .filter(|path| { + path.file_name().is_some_and(|n| n == "fastly.toml") + && path + .parent() + .is_some_and(|dir| dir.join("Cargo.toml").exists()) + }) + .collect(); + + if candidates.is_empty() { + return Err("could not locate fastly.toml".to_owned()); + } + + candidates.sort_by_key(|path| { + let parent = path.parent().unwrap_or(Path::new("")); + path_distance(start, parent) + }); + + Ok(candidates.remove(0)) +} + +fn locate_artifact( + workspace_root: &Path, + manifest_dir: &Path, + crate_name: &str, +) -> Result { + let target_triple = "wasm32-wasip1"; + let release_name = format!("{}.wasm", crate_name.replace('-', "_")); + + if let Some(custom) = env::var_os("CARGO_TARGET_DIR") { + let candidate = PathBuf::from(custom) + .join(target_triple) + .join("release") .join(&release_name); if candidate.exists() { return Ok(candidate); } } - let manifest_target = manifest_dir - .join("target") - .join(target_triple) - .join("release") - .join(&release_name); - if manifest_target.exists() { - return Ok(manifest_target); + let manifest_target = manifest_dir + .join("target") + .join(target_triple) + .join("release") + .join(&release_name); + if manifest_target.exists() { + return Ok(manifest_target); + } + + let workspace_target = workspace_root + .join("target") + .join(target_triple) + .join("release") + .join(&release_name); + if workspace_target.exists() { + return Ok(workspace_target); + } + + Err(format!( + "compiled artifact not found (looked in {} and workspace target)", + manifest_dir.display() + )) +} + +#[inline] +pub fn register() { + register_adapter(&FASTLY_ADAPTER); + register_adapter_blueprint(&FASTLY_BLUEPRINT); +} + +#[ctor(unsafe)] +fn register_ctor() { + register(); +} + +/// # Errors +/// Returns an error if the Fastly CLI serve command (Viceroy) fails. +#[inline] +pub fn serve(extra_args: &[String]) -> Result<(), String> { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + + let status = Command::new("fastly") + .args(["compute", "serve"]) + .args(extra_args) + .current_dir(manifest_dir) + .status() + .map_err(|err| format!("failed to run fastly CLI: {err}"))?; + if !status.success() { + return Err(format!("fastly compute serve failed with status {status}")); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use edgezero_adapter::cli_support::read_package_name; + #[cfg(unix)] + use edgezero_core::test_env::PathPrepend; + use std::collections::HashSet; + + #[cfg(unix)] + use std::sync::Mutex; + use tempfile::tempdir; + + // Shared fixture names. Pinning these as consts (instead of + // inline `"sessions"` / `"app_config"` per call site) keeps the + // setup-vs-assertion pair in sync -- a typo in one place no + // longer silently divorces from the other, because both reference + // the same const. Also names the intent: these are the LOGICAL + // store ids the fastly adapter operates on, not arbitrary strings. + const TEST_KV_ID: &str = "sessions"; + const TEST_CONFIG_ID: &str = "app_config"; + const TEST_SECRET_ID: &str = "default"; + + #[test] + fn finds_closest_manifest_when_multiple_exist() { + let dir = tempdir().unwrap(); + let root = dir.path(); + fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); + + let first = root.join("crates/first"); + fs::create_dir_all(&first).unwrap(); + fs::write(first.join("Cargo.toml"), "[package]\nname=\"first\"").unwrap(); + fs::write(first.join("fastly.toml"), "name=\"first\"").unwrap(); + + let second = root.join("examples/second"); + fs::create_dir_all(&second).unwrap(); + fs::write(second.join("Cargo.toml"), "[package]\nname=\"second\"").unwrap(); + fs::write(second.join("fastly.toml"), "name=\"second\"").unwrap(); + + let found = find_fastly_manifest(&second).unwrap(); + assert_eq!(found, second.join("fastly.toml")); + } + + #[test] + fn finds_manifest_in_current_directory() { + let dir = tempdir().unwrap(); + let root = dir.path(); + fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); + fs::write(root.join("fastly.toml"), "name = \"demo\"").unwrap(); + + let manifest = find_fastly_manifest(root).expect("should find manifest"); + assert_eq!(manifest, root.join("fastly.toml")); + } + + #[test] + fn locate_artifact_considers_workspace_target() { + let dir = tempdir().unwrap(); + let workspace = dir.path(); + let manifest_dir = workspace.join("service"); + fs::create_dir_all(manifest_dir.join("target/wasm32-wasip1/release")).unwrap(); + let artifact = workspace.join("target/wasm32-wasip1/release/demo.wasm"); + fs::create_dir_all(artifact.parent().unwrap()).unwrap(); + fs::write(&artifact, "wasm").unwrap(); + + let located = locate_artifact(workspace, &manifest_dir, "demo").unwrap(); + assert_eq!(located, artifact); + } + + #[test] + fn read_package_falls_back_to_name() { + let dir = tempdir().unwrap(); + let manifest = dir.path().join("Cargo.toml"); + fs::write(&manifest, "name = \"demo\"").unwrap(); + let name = read_package_name(&manifest).unwrap(); + assert_eq!(name, "demo"); + } + + #[test] + fn read_package_prefers_package_table() { + let dir = tempdir().unwrap(); + let manifest = dir.path().join("Cargo.toml"); + fs::write(&manifest, "[package]\nname = \"demo\"\n").unwrap(); + let name = read_package_name(&manifest).unwrap(); + assert_eq!(name, "demo"); + } + + // ---------- push_entries_with_committer ---------- + + #[test] + fn push_entries_with_committer_returns_count_when_all_succeed() { + let entries = vec![ + ("a".to_owned(), "1".to_owned()), + ("b".to_owned(), "2".to_owned()), + ("c".to_owned(), "3".to_owned()), + ]; + let pushed = push_entries_with_committer(&entries, |_, _| Ok(())).expect("all succeed"); + assert_eq!(pushed, 3); + } + + #[test] + fn push_entries_with_committer_zero_entries_is_ok() { + let pushed = push_entries_with_committer(&[], |_, _| Ok(())).expect("empty is fine"); + assert_eq!(pushed, 0); + } + + #[test] + fn push_entries_with_committer_failure_surfaces_committed_failed_not_attempted() { + // Mock committer: succeed for first 2 keys, fail at third. + let entries = vec![ + ("k1".to_owned(), "v1".to_owned()), + ("k2".to_owned(), "v2".to_owned()), + ("k3".to_owned(), "v3".to_owned()), + ("k4".to_owned(), "v4".to_owned()), + ("k5".to_owned(), "v5".to_owned()), + ]; + let mut calls: usize = 0; + let err = push_entries_with_committer(&entries, |key, _| { + calls = calls.saturating_add(1); + if key == "k3" { + Err("simulated fastly stderr".to_owned()) + } else { + Ok(()) + } + }) + .expect_err("middle failure must error"); + // Committer was invoked for k1, k2, k3 and stopped. + assert_eq!(calls, 3_usize, "no retries beyond failure point"); + // Error names all three categories. + assert!(err.contains("k1") && err.contains("k2"), "committed: {err}"); + assert!( + err.contains("Failed: `k3`"), + "failed entry named exactly: {err}" + ); + assert!( + err.contains("k4") && err.contains("k5"), + "not-attempted: {err}" + ); + assert!(err.contains("simulated fastly stderr"), "inner err: {err}"); + // Counts are sane. + assert!( + err.contains("committing 2 of 5 entries"), + "committed/total count: {err}" + ); + } + + #[test] + fn push_entries_with_committer_first_entry_failure_reports_zero_committed() { + let entries = vec![ + ("only".to_owned(), "val".to_owned()), + ("never".to_owned(), "tried".to_owned()), + ]; + let err = push_entries_with_committer(&entries, |_, _| Err("nope".to_owned())) + .expect_err("first-entry failure"); + assert!(err.contains("committing 0 of 2"), "zero committed: {err}"); + assert!( + err.contains("Failed: `only`"), + "first-entry failure named: {err}" + ); + assert!( + err.contains("never"), + "second entry as not-attempted: {err}" + ); + } + + #[test] + fn push_entries_with_committer_last_entry_failure_reports_n_minus_one_committed() { + let entries = vec![ + ("a".to_owned(), "1".to_owned()), + ("b".to_owned(), "2".to_owned()), + ("c".to_owned(), "3".to_owned()), + ]; + let err = push_entries_with_committer(&entries, |key, _| { + if key == "c" { + Err("late failure".to_owned()) + } else { + Ok(()) + } + }) + .expect_err("last-entry failure"); + assert!(err.contains("committing 2 of 3"), "n-1 committed: {err}"); + assert!( + err.contains("the remaining 0 entries"), + "zero not-attempted when last fails: {err}" + ); + } + + // ---------- looks_like_already_exists ---------- + + #[test] + fn looks_like_already_exists_recognises_common_phrasings() { + // Real-shaped fastly CLI error strings (paraphrased; the + // CLI varies across versions). Each must be detected so + // create_fastly_store can treat it as idempotent success. + assert!(looks_like_already_exists( + "Error: a kv-store with that name already exists", + "kv", + )); + assert!(looks_like_already_exists( + "ERROR: Conflict (409): duplicate kv_store name", + "kv", + )); + assert!(looks_like_already_exists( + "A config-store with this name already exists", + "config", + )); + // Spaced form: some fastly CLI versions emit prose + // ("kv store"); accept it alongside the punctuated forms. + assert!(looks_like_already_exists( + "Error: kv store conflict: name already in use", + "kv", + )); + } + + #[test] + fn looks_like_already_exists_rejects_unrelated_errors() { + assert!(!looks_like_already_exists( + "Error: unauthenticated; run `fastly profile create`", + "kv", + )); + assert!(!looks_like_already_exists( + "Error: network unreachable", + "kv", + )); + assert!(!looks_like_already_exists("", "kv")); + } + + #[test] + fn looks_like_already_exists_rejects_unrelated_conflict_errors() { + // The earlier wider heuristic swallowed ANY stderr + // containing "conflict" or "already exists", which would + // misread an unrelated 409 from a different fastly + // subcommand (e.g. a service-version conflict during a + // parallel deploy) as idempotent store-create success. + // Now we require the kind context too, so unrelated + // conflicts surface as failures. + assert!( + !looks_like_already_exists( + "Error: 409 Conflict on /service/abc/version/42 -- already exists", + "kv", + ), + "service-version conflict must NOT be misread as kv-store idempotency" + ); + assert!( + !looks_like_already_exists( + "Error: invalid duplicate request; check name resolution", + "kv", + ), + "unrelated `duplicate ... name` AND-match must NOT trigger" + ); + // And the kind must match: a config-store conflict must + // not look-like-already-exists for a kv-store create call. + assert!( + !looks_like_already_exists("Error: a config-store with that name already exists", "kv",), + "wrong-kind conflict must NOT trigger" + ); + } + + // ---------- setup_block_present ---------- + + #[test] + fn setup_block_present_true_when_table_exists() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "name = \"demo\"\n[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n", + ) + .expect("write"); + assert!(setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + /// The three provisioning parsers must NOT echo a malformed fastly.toml's + /// source text (which can contain a stored secret) on a parse failure. + #[test] + fn provisioning_parsers_redact_malformed_toml() { + const SENTINEL: &str = "SUPER_SECRET_IN_A_BROKEN_LINE"; + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Malformed TOML whose offending line carries a secret. + fs::write(&path, format!("service_id = \"{SENTINEL}\" = broken\n")).expect("write"); + + let errs = [ + read_fastly_service_id(&path).expect_err("malformed toml must error"), + setup_block_present(&path, "kv", TEST_KV_ID).expect_err("malformed toml must error"), + append_fastly_setup(&path, "kv", TEST_KV_ID).expect_err("malformed toml must error"), + ]; + for err in &errs { + assert!( + !err.contains(SENTINEL), + "a parse error must not echo the stored value: {err}" + ); + assert!( + err.contains("redacted"), + "error should say it redacted: {err}" + ); + } + } + + #[test] + fn setup_block_present_false_when_id_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n[setup.kv_stores.other]\n").expect("write"); + assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + #[test] + fn setup_block_present_false_for_missing_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("does-not-exist.toml"); + assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + #[test] + fn setup_block_present_true_when_only_setup_exists() { + // Post-F6 (PR #269 round 2): `setup_block_present` only + // checks `[setup._stores.]`. The pre-fix check + // ALSO required `[local_server._stores.]`, but + // writing an empty `[local_server.*]` table didn't match + // fastly's local-server schema (config-stores need + // `format` + contents, kv/secret stores need a JSON file + // or `{key, data}` entries). Local-server seeding moved + // to `config push --adapter fastly --local`, so probe + // only cares about `[setup]` now. + let dir = tempdir().expect("tempdir"); + let only_setup = dir.path().join("only_setup.toml"); + fs::write(&only_setup, "name = \"demo\"\n[setup.kv_stores.sessions]\n").expect("write"); + assert!( + setup_block_present(&only_setup, "kv", TEST_KV_ID).expect("probe"), + "[setup.*] alone is now sufficient: {only_setup:?}" + ); + + let only_local = dir.path().join("only_local.toml"); + fs::write( + &only_local, + "name = \"demo\"\n[local_server.kv_stores.sessions]\n", + ) + .expect("write"); + assert!( + !setup_block_present(&only_local, "kv", TEST_KV_ID).expect("probe"), + "[local_server.*] alone is NOT a provisioned-setup signal" + ); + } + + // ---------- append_fastly_setup ---------- + + #[test] + fn append_fastly_setup_creates_setup_table_in_minimal_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "setup table added: {after}" + ); + // Post-F6: no `[local_server.*]` write — that empty stanza + // didn't satisfy fastly's local-server schema and made + // `fastly compute serve` error or skip the store. Local- + // server seeding is now `config push --adapter fastly + // --local`'s job. + assert!( + !after.contains("[local_server.kv_stores.sessions]"), + "[local_server.*] empty table no longer written by provision: {after}" + ); + assert!( + after.contains("name = \"demo\""), + "preserved original keys: {after}" + ); + } + + #[test] + fn append_fastly_setup_appends_alongside_existing_kind_tables() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "[setup.kv_stores.cache]\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.cache]"), + "existing entry kept: {after}" + ); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "new entry added: {after}" + ); + } + + #[test] + fn append_fastly_setup_is_idempotent_on_duplicate_id() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "[setup.kv_stores.sessions]\nfoo = \"keep\"\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("idempotent append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("foo = \"keep\""), + "did not stomp existing key: {after}" + ); + } + + #[test] + fn append_fastly_setup_creates_file_when_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Note: no fs::write — file starts absent. + append_fastly_setup(&path, "config", TEST_CONFIG_ID).expect("create"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains("[setup.config_stores.app_config]")); + assert!( + !after.contains("[local_server.config_stores.app_config]"), + "[local_server.*] no longer written by provision: {after}" + ); + } + + #[test] + fn append_fastly_setup_preserves_top_comments() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "# managed by hand -- please keep this line\nname = \"demo\"\n", + ) + .expect("write"); + append_fastly_setup(&path, "secret", TEST_SECRET_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("# managed by hand"), + "preserved comment: {after}" + ); + } + + // ---------- write_fastly_local_config_store (config push --local) ---------- + + #[test] + fn write_fastly_local_config_store_creates_inline_block_in_minimal_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let entries = vec![ + ("greeting".to_owned(), "hello".to_owned()), + ("service.timeout_ms".to_owned(), "1500".to_owned()), + ]; + write_fastly_local_config_store(&path, TEST_CONFIG_ID, &entries, &[]).expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains(&format!("[local_server.config_stores.{TEST_CONFIG_ID}]")), + "store table: {after}" + ); + assert!( + after.contains("format = \"inline-toml\""), + "format field: {after}" + ); + assert!( + after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + )), + "contents table: {after}" + ); + assert!(after.contains("greeting = \"hello\""), "key 1: {after}"); + assert!( + after.contains("\"service.timeout_ms\" = \"1500\""), + "dotted key quoted: {after}" + ); + assert!(after.contains("name = \"demo\""), "preserved: {after}"); + } + + #[test] + fn write_fastly_local_config_store_replaces_existing_block_on_re_push() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "stale".to_owned())], + &[], + ) + .expect("first write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "fresh".to_owned())], + &[], + ) + .expect("second write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains("greeting = \"fresh\""), "new value: {after}"); + assert!( + !after.contains("greeting = \"stale\""), + "stale value dropped: {after}" + ); + } + + #[test] + fn write_fastly_local_config_store_preserves_unrelated_blocks() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + let original = "\ +[setup.kv_stores.sessions] + +[[local_server.kv_stores.sessions]] +key = \"__init__\" +data = \"\" + +[scripts] +build = \"cargo build --release\" +"; + fs::write(&path, original).expect("write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "setup KV kept: {after}" + ); + assert!(after.contains("[scripts]"), "scripts table kept: {after}"); + assert!( + after.contains("build = \"cargo build --release\""), + "scripts value kept: {after}" + ); + assert!( + after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + )), + "new config_stores block added: {after}" + ); + } + + #[test] + fn write_fastly_local_config_store_creates_file_when_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // No fs::write — file absent. + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + ))); + assert!(after.contains("greeting = \"hi\"")); + } + + // ---------- provision (dry-run + error path) ---------- + + #[test] + fn provision_dry_run_does_not_invoke_fastly() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let config_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_CONFIG_ID]); + let secret_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_SECRET_ID]); + let stores = ProvisionStores { + config: &config_ids, + kv: &kv_ids, + secrets: &secret_ids, + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, true) + .expect("dry-run succeeds"); + // 1 KV + 1 config + 1 secret + 1 runtime-env = 4 status lines. + assert_eq!(out.len(), 4); + assert!(out[0].contains("would run `fastly kv-store create --name=sessions`")); + assert!(out[1].contains("would run `fastly config-store create --name=app_config`")); + assert!(out[2].contains("would run `fastly secret-store create --name=default`")); + assert!( + out[3].contains("would run `fastly config-store create --name=edgezero_runtime_env`"), + "runtime-env store row: {out:?}", + ); + // Manifest untouched. + let after = fs::read_to_string(&path).expect("read"); + assert_eq!(after, "name = \"demo\"\n", "dry-run mutated fastly.toml"); + } + + #[test] + fn provision_errors_when_adapter_manifest_path_missing() { + let dir = tempdir().expect("tempdir"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let stores = ProvisionStores { + config: &[], + kv: &kv_ids, + secrets: &[], + }; + let err = FastlyCliAdapter + .provision(dir.path(), None, None, &stores, true) + .expect_err("missing adapter manifest path must error"); + assert!( + err.contains("fastly.toml"), + "error names what's missing: {err}" + ); + } + + #[test] + fn provision_with_no_declared_stores_says_so() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Pre-populate the runtime-env block so the provision flow's + // unconditional runtime-env step skips (otherwise it would + // shell out to real `fastly` to create the store). + fs::write( + &path, + "name = \"demo\"\n[setup.config_stores.edgezero_runtime_env]\n", + ) + .expect("write"); + let stores = ProvisionStores { + config: &[], + kv: &[], + secrets: &[], + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, false) + .expect("no-store provision is fine"); + assert_eq!(out, vec!["fastly has no declared stores to provision"]); + } + + #[test] + fn provision_skips_id_when_setup_block_already_present() { + // setup_block_present's role in the flow: re-running + // provision after the user already declared a store in + // fastly.toml must be a no-op (no shell-out to fastly). + // We can verify this in a real (non-dry-run) call because + // the skip path bypasses create_fastly_store entirely. + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n\ + [setup.config_stores.edgezero_runtime_env]\n", + ) + .expect("write"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let stores = ProvisionStores { + config: &[], + kv: &kv_ids, + secrets: &[], + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, false) + .expect("skip path succeeds without invoking fastly"); + assert_eq!(out.len(), 1); + assert!(out[0].contains("already declared"), "got: {out:?}"); + } + + /// When `fastly.toml` declares `service_id`, the next + /// `fastly compute deploy` skips `[setup]` entirely. provision + /// must emit the `fastly resource-link create` remediation for + /// every store it creates -- including the implicit + /// `edgezero_runtime_env` store the runtime override path + /// depends on. Without this, a freshly-provisioned override + /// store would not be linked to the already-deployed service + /// and the runtime would silently fall back to baked defaults. + #[test] + fn provision_emits_resource_link_note_for_runtime_env_on_existing_service() { + // Dry-run only -- we just want to drive the resource_link_note + // helper for the runtime-env store branch. The real-create + // path can't run in tests (would shell out to `fastly`). + // The dry-run output line for runtime-env doesn't include the + // note (the helper only fires on real create), so we test the + // helper directly here. + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\nservice_id = \"abc123svc\"\n").expect("write"); + let note = resource_link_note(&path, "config", "edgezero_runtime_env") + .expect("read service_id") + .expect("note present when service_id set"); + assert!( + note.contains("service_id = \"abc123svc\""), + "note quotes the service id: {note}" + ); + assert!( + note.contains("fastly config-store list --json"), + "note tells operator how to find the store id: {note}" + ); + assert!( + note.contains("name=`edgezero_runtime_env`"), + "note names the runtime override store: {note}" + ); + assert!( + note.contains( + "fastly resource-link create --service-id=abc123svc --resource-id= --version=latest --autoclone --name=edgezero_runtime_env" + ), + "note carries the full resource-link command: {note}" + ); + } + + /// And the inverse: no `service_id` (a service that hasn't been + /// deployed yet) means `[setup]` will be applied on the next + /// `compute deploy`, so no manual resource-link step is needed. + /// The helper must return `None` to avoid noisy false-positive + /// guidance. + #[test] + fn provision_skips_resource_link_note_when_service_undeployed() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let note = + resource_link_note(&path, "config", "edgezero_runtime_env").expect("read service_id"); + assert!( + note.is_none(), + "no service_id => no resource-link prompt: {note:?}" + ); + } + + // ---------- find_config_store_id ---------- + + #[test] + fn find_config_store_id_matches_bare_array_by_name() { + let stdout = format!( + r#"[ + {{"id": "abc123", "name": "{TEST_CONFIG_ID}"}}, + {{"id": "def456", "name": "other_store"}} + ]"# + ); + match find_config_store_id(&stdout, TEST_CONFIG_ID) { + ConfigStoreLookup::Found(id) => assert_eq!(id, "abc123"), + ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), + ConfigStoreLookup::SchemaDrift(detail) => { + panic!("expected Found, got SchemaDrift({detail})") + } + } + } + + #[test] + fn find_config_store_id_tolerates_items_envelope() { + let stdout = format!( + r#"{{"items": [ + {{"id": "xyz789", "name": "{TEST_CONFIG_ID}"}} + ]}}"# + ); + match find_config_store_id(&stdout, TEST_CONFIG_ID) { + ConfigStoreLookup::Found(id) => assert_eq!(id, "xyz789"), + ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), + ConfigStoreLookup::SchemaDrift(detail) => { + panic!("expected Found, got SchemaDrift({detail})") + } + } + } + + #[test] + fn find_config_store_id_distinguishes_not_found_from_match_failure() { + // JSON parses cleanly, entries are well-formed + // (`name` + `id` strings present), but no entry matches + // → NotFound. Operator likely needs to run `provision`. + let stdout = r#"[{"id": "abc", "name": "other"}]"#; + assert!(matches!( + find_config_store_id(stdout, "missing"), + ConfigStoreLookup::NotFound + )); + } + + #[test] + fn find_config_store_id_flags_schema_drift_on_malformed_json() { + // Unparseable bytes are NOT a "store not found" — they're + // a "fastly CLI output format changed" signal. Operator + // needs different recovery (file a bug, pin CLI version) + // than for the "store doesn't exist yet" case. + let drift = find_config_store_id("not json", "anything"); + assert!( + matches!(drift, ConfigStoreLookup::SchemaDrift(_)), + "non-JSON stdout must be schema drift, got {drift:?}" + ); + let empty = find_config_store_id("", "anything"); + assert!( + matches!(empty, ConfigStoreLookup::SchemaDrift(_)), + "empty stdout must be schema drift, got {empty:?}" + ); + } + + #[test] + fn find_config_store_id_flags_schema_drift_when_shape_unexpected() { + // JSON parses but the top-level is neither a bare array + // nor an `{items: [...]}` envelope. + let stdout = r#"{"namespace": "fastly", "list": []}"#; + match find_config_store_id(stdout, "any") { + ConfigStoreLookup::SchemaDrift(detail) => { + assert!( + detail.contains("bare array") || detail.contains("items"), + "schema-drift detail names the expected shapes: {detail}" + ); + } + ConfigStoreLookup::Found(id) => panic!("expected SchemaDrift, got Found({id})"), + ConfigStoreLookup::NotFound => panic!("expected SchemaDrift, got NotFound"), + } + } + + #[test] + fn find_config_store_id_flags_schema_drift_when_entries_lack_name_id() { + // Array of objects but none have BOTH string `name` and + // string `id` fields — suggests schema rename (e.g. + // fastly renamed `name` → `title`). + let stdout = format!(r#"[{{"title": "{TEST_CONFIG_ID}", "uid": "abc"}}]"#); + let drift = find_config_store_id(&stdout, TEST_CONFIG_ID); + assert!( + matches!(drift, ConfigStoreLookup::SchemaDrift(_)), + "entries lacking name/id must be schema drift, got {drift:?}" + ); + } + + #[test] + fn find_config_store_id_returns_not_found_for_empty_array() { + // Empty array IS a valid "store doesn't exist yet" signal, + // not schema drift — fastly CLI legitimately returns `[]` + // when no config-stores exist. + let drift = find_config_store_id("[]", "any"); + assert!( + matches!(drift, ConfigStoreLookup::NotFound), + "empty array must be NotFound, got {drift:?}" + ); + } + + // ---------- push_config_entries (dry-run + error paths) ---------- + + #[test] + fn push_dry_run_does_not_invoke_fastly() { + let dir = tempdir().expect("tempdir"); + let entries = vec![ + ("greeting".to_owned(), "hello".to_owned()), + ("feature.new_checkout".to_owned(), "false".to_owned()), + ]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, + ) + .expect("dry-run succeeds"); + // First line names the resolve+publish flow; then one preview line per + // key. A push no longer reclaims anything (see `config gc`), so there is + // no GC-intent line. + assert_eq!(out.len(), 1 + entries.len(), "header + per-entry preview"); + assert!( + out[0].contains("would resolve fastly config-store `app_config`") + && out[0].contains("push entries"), + "dry-run header describes the would-be flow: {out:?}" + ); + assert!( + out.iter().any(|line| line.contains("`greeting`")), + "dry-run lists `greeting`: {out:?}" + ); + assert!( + out.iter() + .any(|line| line.contains("`feature.new_checkout`")), + "dry-run lists `feature.new_checkout`: {out:?}" + ); + } + + #[test] + fn push_with_no_entries_reports_no_op_without_invoking_fastly() { + let dir = tempdir().expect("tempdir"); + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[], + &AdapterPushContext::new(), + false, + ) + .expect("zero-entry push is fine"); + assert_eq!(out.len(), 1); + assert!( + out[0].contains("no config entries"), + "status line names the no-op: {out:?}" + ); + } + + // ---------- read_config_entry_local ---------- + + #[test] + fn read_local_returns_missing_store_when_fastly_toml_absent() { + let dir = tempdir().expect("tempdir"); + // No fastly.toml written — file missing. + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing file is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "absent fastly.toml => MissingStore" + ); + } + + #[test] + fn read_local_returns_missing_store_when_no_local_server_contents() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // fastly.toml exists but has no [local_server.config_stores.*] block. + fs::write(&path, "name = \"demo\"\n[setup.config_stores.app_config]\n").expect("write"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing local_server block is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "no local_server stanza => MissingStore" + ); + } + + #[test] + fn read_local_returns_missing_key_when_key_absent_from_contents() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Write a local_server block with a different key so the store exists + // but the requested key is absent. + fs::write( + &path, + format!( + "name = \"demo\"\n\ + [local_server.config_stores.{TEST_CONFIG_ID}]\n\ + format = \"inline-toml\"\n\ + [local_server.config_stores.{TEST_CONFIG_ID}.contents]\n\ + other_key = \"other_value\"\n" + ), + ) + .expect("write"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing key is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingKey), + "key absent from contents => MissingKey" + ); + } + + #[test] + fn read_local_returns_present_when_key_exists_in_contents() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write initial toml"); + + // Use a valid BlobEnvelope value — the resolver requires BlobEnvelope + // or chunk-pointer JSON; raw strings are not accepted post-chunking. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "fastly"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), envelope_json.clone())], + &[], + ) + .expect("setup write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("key present"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present variant"); + }; + assert_eq!(value, envelope_json, "value matches what was written"); + } + + #[test] + fn read_local_roundtrips_with_push_local() { + // Write via push_config_entries_local, then read via + // read_config_entry_local — the two must agree on the value. + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + + // push_config_entries_local passes the value through the chunk-pointer + // helper which stores it verbatim when ≤ 8 000 chars. The reader then + // resolves it through the same resolver that requires BlobEnvelope JSON. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "roundtrip"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entries = vec![("greeting".to_owned(), envelope_json.clone())]; + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push succeeds"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("read succeeds"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present after push+read roundtrip"); + }; + assert_eq!(value, envelope_json, "roundtrip value matches"); + } + + #[test] + fn read_local_requires_adapter_manifest_path() { + let dir = tempdir().expect("tempdir"); + let result = FastlyCliAdapter.read_config_entry_local( + dir.path(), + None, // adapter_manifest_path missing + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ); + match result { + Err(err) => assert!( + err.contains("[adapters.fastly.adapter].manifest"), + "error names the missing field: {err}" + ), + Ok(_) => panic!("expected Err when adapter_manifest_path is None"), + } + } + + // ---------- read_config_entry (fake fastly, remote shell-out) ---------- + + /// Build a tempdir containing a `fastly` shim script that: + /// - Responds to `config-store list --json` with a store-list JSON containing + /// `TEST_CONFIG_ID` mapped to `store-abc123`. + /// - Responds to `config-store-entry describe ...` with `stdout_body` on + /// stdout and `stderr_body` on stderr, exiting with `exit_code`. + /// + /// Payloads are written to separate sibling files so shell-active chars + /// in the content don't get re-interpreted by the script. + #[cfg(unix)] + fn fake_fastly_returning( + stdout_body: &str, + stderr_body: &str, + exit_code: i32, + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempdir().expect("tempdir"); + let script_path = dir.path().join("fastly"); + let stdout_file = dir.path().join("stdout_payload.txt"); + let stderr_file = dir.path().join("stderr_payload.txt"); + let list_file = dir.path().join("list_payload.txt"); + // Store-list JSON: bare array with one entry matching TEST_CONFIG_ID. + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + fs::write(&stdout_file, stdout_body).expect("write stdout payload"); + fs::write(&stderr_file, stderr_body).expect("write stderr payload"); + fs::write(&list_file, list_json).expect("write list payload"); + let script = format!( + "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\ncat '{}' >&2\nexit {exit_code}\n", + list_file.display(), + stdout_file.display(), + stderr_file.display(), + ); + fs::write(&script_path, script).expect("write fastly script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + dir + } + + /// Build a fake `fastly` that logs each argv token (one per line) to + /// `out_path`, handles the list call correctly, and exits 0 for both calls. + #[cfg(unix)] + fn fake_fastly_argv_log(out_path: &Path) -> tempfile::TempDir { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + use std::os::unix::fs::PermissionsExt as _; + let dir = tempdir().expect("tempdir"); + let script_path = dir.path().join("fastly"); + let list_file = dir.path().join("list_payload.txt"); + let entry_file = dir.path().join("entry_payload.txt"); + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + // item_value must be a valid BlobEnvelope JSON so the resolver accepts it. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"v": "logged"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entry_json = format!( + r#"{{"item_value":{},"store_id":"store-abc123"}}"#, + serde_json::to_string(&envelope_json).expect("escape") + ); + fs::write(&list_file, list_json).expect("write list payload"); + fs::write(&entry_file, &entry_json).expect("write entry payload"); + let script = format!( + "#!/bin/sh\nfor arg in \"$@\"; do printf '%s\\n' \"$arg\" >> '{}'; done\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\nexit 0\n", + out_path.display(), + list_file.display(), + entry_file.display(), + ); + fs::write(&script_path, script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + dir + } + + /// Process-wide mutex serialising PATH-mutating tests so parallel + /// test threads don't race on the environment variable. + #[cfg(unix)] + fn path_mutation_guard() -> &'static Mutex<()> { + use std::sync::OnceLock; + static GUARD: OnceLock> = OnceLock::new(); + GUARD.get_or_init(|| Mutex::new(())) + } + + #[cfg(unix)] + #[test] + fn read_remote_returns_present_on_success() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Fake fastly: list succeeds with app_config → store-abc123; + // describe returns valid JSON with item_value that is a BlobEnvelope. + let envelope = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "fastly"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entry_json = format!( + r#"{{"item_value":{},"store_id":"store-abc123"}}"#, + serde_json::to_string(&envelope).expect("escape") + ); + let fake = fake_fastly_returning(&entry_json, "", 0); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("fake fastly exit-0 must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, envelope); + } + + #[cfg(unix)] + #[test] + fn read_remote_returns_missing_key_on_not_found_stderr() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // describe exits non-zero with "not found" in stderr → MissingKey. + let fake = fake_fastly_returning("", "Error: item not found", 1); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("not-found maps to MissingKey (not Err)"); + assert!( + matches!(result, ReadConfigEntry::MissingKey), + "not-found stderr => MissingKey" + ); + } + + /// The Fastly impl distinguishes store-not-found from key-not-found via + /// `resolve_remote_config_store_id`: when the list call exits non-zero and + /// the error string contains "not found", `read_config_entry` returns + /// `MissingStore` without ever calling the describe subcommand. + #[cfg(unix)] + #[test] + fn read_remote_returns_missing_store_on_appropriate_stderr() { + use std::os::unix::fs::PermissionsExt as _; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Script that exits non-zero for the list call so resolve fails with + // a "not found" error, causing read_config_entry to return MissingStore. + let fake_dir = tempdir().expect("tempdir"); + let stderr_file = fake_dir.path().join("stderr_payload.txt"); + fs::write(&stderr_file, "Error: config store not found for service").expect("write stderr"); + let script_path = fake_dir.path().join("fastly"); + let script = format!( + "#!/bin/sh\ncat '{stderr}' >&2\nexit 1\n", + stderr = stderr_file.display(), + ); + fs::write(&script_path, script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + let _path = PathPrepend::new(fake_dir.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("list failure with not-found maps to MissingStore (not Err)"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "list not-found => MissingStore" + ); + } + + /// Verify that `read_config_entry` invokes + /// `fastly config-store-entry describe --store-id= --key= --json` + /// (after the resolve step that calls `fastly config-store list --json`). + #[cfg(unix)] + #[test] + fn read_remote_invokes_correct_argv() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("argv-log fake must succeed"); + assert!( + matches!(result, ReadConfigEntry::Present(_)), + "expected Present from argv-log fake" + ); + let captured = fs::read_to_string(&argv_log).expect("argv log"); + // The describe call must include these args (resolve call args + // are also captured but we only assert the describe shape here). + assert!( + captured.contains("config-store-entry"), + "must invoke config-store-entry; got:\n{captured}" + ); + assert!( + captured.contains("describe"), + "must pass describe subcommand; got:\n{captured}" + ); + assert!( + captured.contains("--store-id=store-abc123"), + "must pass resolved store id; got:\n{captured}" + ); + assert!( + captured.contains("--key=greeting"), + "must pass --key=; got:\n{captured}" + ); + assert!( + captured.contains("--json"), + "must pass --json flag; got:\n{captured}" + ); + } + + // ---------- chunked push integration tests ---------- + + /// Build a valid `BlobEnvelope` JSON string of approximately `target_len` bytes. + #[cfg(unix)] + fn make_test_envelope(target_len: usize) -> String { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let pad = "x".repeat(target_len.saturating_add(64)); + let data = json!({ "pad": pad }); + let raw = + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); + if raw.len() >= target_len { + let overhead = raw.len().saturating_sub(pad.len()); + let adjusted = "x".repeat(target_len.saturating_sub(overhead)); + let data2 = json!({ "pad": adjusted }); + serde_json::to_string(&BlobEnvelope::new(data2, "2026-06-22T00:00:00Z".into())).unwrap() + } else { + raw + } + } + + /// Build a fake `fastly` script whose describe response depends on + /// the `--key=` argument: `key_responses` maps key names to JSON + /// item-value responses. Falls back to exit 1 "not found" for unknown keys. + #[cfg(unix)] + fn fake_fastly_with_key_dispatch( + _dir: &Path, + key_responses: &[(String, String)], + ) -> tempfile::TempDir { + use std::fmt::Write as _; + use std::os::unix::fs::PermissionsExt as _; + let fake_dir = tempdir().expect("tempdir"); + let list_file = fake_dir.path().join("list.json"); + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + fs::write(&list_file, list_json).expect("write list"); + // Write each key response to a named file. + let mut dispatch_lines = String::new(); + for (key, response) in key_responses { + let resp_file = fake_dir.path().join(format!("resp_{key}.json")); + fs::write(&resp_file, response).expect("write resp"); + // Use exact-match: iterate argv and compare each token literally + // so that a root key like "app_config" does NOT match a chunk key + // like "app_config.__edgezero_chunks.abc.0". + writeln!( + dispatch_lines, + " for arg in \"$@\"; do if [ \"$arg\" = \"--key={key}\" ]; then cat '{}'; exit 0; fi; done", + resp_file.display() + ) + .expect("write to String is infallible"); + } + // Fallback outputs "not found" so fetch_remote_config_store_entry + // maps it to Ok(None) rather than Err. + let script = format!( + "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\n{dispatch_lines}echo 'Error: item not found' >&2\nexit 1\n", + list_file.display() + ); + let script_path = fake_dir.path().join("fastly"); + fs::write(&script_path, &script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + fake_dir + } + + /// Fake `fastly` for cloud chunk-GC tests. Logs each + /// `config-store-entry` op ("describe " / "update " / + /// "delete ", plus "delete-argv ") to `oplog`. + /// + /// `root_describe_seq` gives the successive raw `item_value`s returned when + /// the ROOT key is described (call 1 = the pre-commit prior read, call 2 = + /// the post-commit read-back). `entry_list` is served for + /// `config-store-entry list` and is what reclamation derives generations + /// and supersession times from. `fail_delete_key` makes that one delete + /// exit non-zero. `describe_hard_error` makes the FIRST describe of each key + /// fail hard (so the prior read errors while the read-back still works). + #[cfg(unix)] + fn fake_fastly_gc( + root_key: &str, + root_describe_seq: &[String], + entry_list: &[(String, String, String)], + fail_delete_key: Option<&str>, + describe_hard_error: bool, + oplog: &Path, + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + // Rendered with handlebars. Triple-stache `{{{ }}}` disables HTML + // escaping (paths are not markup); the shell's own `${var}` / + // `$(( ))` use single braces so they are literal text to handlebars. + const TEMPLATE: &str = r#"#!/bin/sh +if [ "$1" = "config-store" ]; then cat '{{{list}}}'; exit 0; fi +sub="$2" +key="" +for arg in "$@"; do case "$arg" in --key=*) key="${arg#--key=}";; esac; done +if [ "$sub" = "list" ]; then printf 'list\n' >> '{{{oplog}}}'; cat '{{{entries}}}'; exit 0; fi +if [ "$sub" = "update" ]; then cat >/dev/null; printf 'update %s\n' "$key" >> '{{{oplog}}}'; exit 0; fi +if [ "$sub" = "delete" ]; then printf 'delete %s\n' "$key" >> '{{{oplog}}}'; printf 'delete-argv %s\n' "$*" >> '{{{oplog}}}'; if [ "$key" = "{{{fail}}}" ]; then echo 'Error: 404 item not found' >&2; exit 1; fi; exit 0; fi +if [ "$sub" = "describe" ]; then + printf 'describe %s\n' "$key" >> '{{{oplog}}}' + cfile='{{{dir}}}/count_'"$key" + n=0; [ -f "$cfile" ] && n=$(cat "$cfile"); n=$((n+1)); printf '%s' "$n" > "$cfile" + {{#if hard_error}}if [ "$n" = "1" ]; then echo 'Error: internal server error' >&2; exit 1; fi{{/if}} + rf='{{{dir}}}/resp_'"$key"'_'"$n"'.json' + if [ -f "$rf" ]; then cat "$rf"; exit 0; fi + echo 'Error: item not found' >&2; exit 1 +fi +echo 'unexpected' >&2; exit 1 +"#; + let dir = tempdir().expect("tempdir"); + let list_file = dir.path().join("list.json"); + fs::write( + &list_file, + format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#), + ) + .expect("list"); + let entries_file = dir.path().join("entries.json"); + fs::write(&entries_file, entry_list_json(entry_list)).expect("entries"); + for (index, value) in root_describe_seq.iter().enumerate() { + let wrapped = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(value).expect("escape") + ); + let nth = index.saturating_add(1); + fs::write( + dir.path().join(format!("resp_{root_key}_{nth}.json")), + wrapped, + ) + .expect("resp"); + } + let data = serde_json::json!({ + "list": list_file.display().to_string(), + "entries": entries_file.display().to_string(), + "oplog": oplog.display().to_string(), + "dir": dir.path().display().to_string(), + "fail": fail_delete_key.unwrap_or(""), + "hard_error": describe_hard_error, + }); + let script = handlebars::Handlebars::new() + .render_template(TEMPLATE, &data) + .expect("render fake fastly script"); + let script_path = dir.path().join("fastly"); + fs::write(&script_path, script).expect("script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + dir + } + + /// Like `fake_fastly_gc`, but serves a VERBATIM `config-store-entry list` + /// payload so a test can present a shape `entry_list_json` cannot build + /// (e.g. a paginated envelope). + #[cfg(unix)] + fn fake_fastly_gc_raw_list( + root_key: &str, + raw_listing: &str, + oplog: &Path, + ) -> tempfile::TempDir { + let dir = fake_fastly_gc(root_key, &[], &[], None, false, oplog); + fs::write(dir.path().join("entries.json"), raw_listing).expect("raw entries"); + dir } - let workspace_target = workspace_root - .join("target") - .join(target_triple) - .join("release") - .join(&release_name); - if workspace_target.exists() { - return Ok(workspace_target); + /// A `config-store-entry list --json` payload. The item VALUE is a + /// placeholder: reclamation must only ever use keys and timestamps. + #[cfg(unix)] + fn entry_list_json(items: &[(String, String, String)]) -> String { + let entries: Vec = items + .iter() + .map(|(key, created, value)| { + serde_json::json!({ + "item_key": key, + "created_at": created, + "item_value": value, + }) + }) + .collect(); + serde_json::to_string(&entries).expect("entry list json") } - Err(format!( - "compiled artifact not found (looked in {} and workspace target)", - manifest_dir.display() - )) -} - -#[inline] -pub fn register() { - register_adapter(&FASTLY_ADAPTER); - register_adapter_blueprint(&FASTLY_BLUEPRINT); -} + /// An RFC-3339 stamp `secs` in the past (the shape Fastly returns). + #[cfg(unix)] + fn stamp_secs_ago(secs: u64) -> String { + let delta = chrono::Duration::seconds(i64::try_from(secs).unwrap_or(0)); + let now = chrono::Utc::now(); + now.checked_sub_signed(delta) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + } -#[ctor(unsafe)] -fn register_ctor() { - register(); -} + /// Every chunk of `envelope` as the listing would return it: REAL keys and + /// REAL payload bytes. + /// + /// The values are not decorative. `config gc` proves a generation is ours by + /// reassembling it and hashing the result against the content-address its + /// keys name, so a placeholder value would (correctly) fail verification and + /// never be reclaimed. Fixtures must be honest for these tests to mean + /// anything. + #[cfg(unix)] + fn listed_generation( + root_key: &str, + envelope: &str, + secs_ago: u64, + ) -> Vec<(String, String, String)> { + let (chunks, _) = chunked_parts(root_key, envelope); + let stamp = stamp_secs_ago(secs_ago); + chunks + .into_iter() + .map(|(key, value)| (key, stamp.clone(), value)) + .collect() + } -/// # Errors -/// Returns an error if the Fastly CLI serve command (Viceroy) fails. -#[inline] -pub fn serve(extra_args: &[String]) -> Result<(), String> { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + /// The ROOT entry as the listing would return it: its value is the pointer, + /// which is how `config gc` learns which chunks are live. + #[cfg(unix)] + fn listed_root(root_key: &str, envelope: &str, secs_ago: u64) -> (String, String, String) { + let (_, pointer) = chunked_parts(root_key, envelope); + (root_key.to_owned(), stamp_secs_ago(secs_ago), pointer) + } - let status = Command::new("fastly") - .args(["compute", "serve"]) - .args(extra_args) - .current_dir(manifest_dir) - .status() - .map_err(|err| format!("failed to run fastly CLI: {err}"))?; - if !status.success() { - return Err(format!("fastly compute serve failed with status {status}")); + /// A chunked envelope with a distinct payload per tag, padded to `pad` + /// characters so a caller can force a given number of chunks (7 000 bytes + /// each). + #[cfg(unix)] + fn gen_envelope_padded(tag: &str, pad: usize) -> String { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(pad) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") } - Ok(()) -} + /// A chunked envelope with a distinct payload per tag. + #[cfg(unix)] + fn gen_envelope(tag: &str) -> String { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") + } -#[cfg(test)] -mod tests { - use super::*; - use edgezero_adapter::cli_support::read_package_name; + /// Split a chunked envelope into (chunk `(key, value)` pairs, root pointer). #[cfg(unix)] - use edgezero_core::test_env::PathPrepend; + fn chunked_parts(root_key: &str, envelope: &str) -> (Vec<(String, String)>, String) { + let entries = prepare_fastly_config_entries(root_key, envelope).expect("expand"); + let (_, pointer) = entries.last().expect("pointer").clone(); + let chunks = entries[..entries.len().saturating_sub(1)].to_vec(); + (chunks, pointer) + } + /// Just the chunk KEYS of a generation (for delete assertions). #[cfg(unix)] - use std::sync::Mutex; - use tempfile::tempdir; + fn chunk_keys_of(root_key: &str, envelope: &str) -> Vec { + let (chunks, _) = chunked_parts(root_key, envelope); + chunks.into_iter().map(|(key, _)| key).collect() + } - // Shared fixture names. Pinning these as consts (instead of - // inline `"sessions"` / `"app_config"` per call site) keeps the - // setup-vs-assertion pair in sync -- a typo in one place no - // longer silently divorces from the other, because both reference - // the same const. Also names the intent: these are the LOGICAL - // store ids the fastly adapter operates on, not arbitrary strings. - const TEST_KV_ID: &str = "sessions"; - const TEST_CONFIG_ID: &str = "app_config"; - const TEST_SECRET_ID: &str = "default"; + #[cfg(unix)] + fn oplog_has(oplog: &Path, line: &str) -> bool { + fs::read_to_string(oplog) + .unwrap_or_default() + .lines() + .any(|entry| entry == line) + } + #[cfg(unix)] #[test] - fn finds_closest_manifest_when_multiple_exist() { - let dir = tempdir().unwrap(); - let root = dir.path(); - fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); - - let first = root.join("crates/first"); - fs::create_dir_all(&first).unwrap(); - fs::write(first.join("Cargo.toml"), "[package]\nname=\"first\"").unwrap(); - fs::write(first.join("fastly.toml"), "name=\"first\"").unwrap(); - - let second = root.join("examples/second"); - fs::create_dir_all(&second).unwrap(); - fs::write(second.join("Cargo.toml"), "[package]\nname=\"second\"").unwrap(); - fs::write(second.join("fastly.toml"), "name=\"second\"").unwrap(); - - let found = find_fastly_manifest(&second).unwrap(); - assert_eq!(found, second.join("fastly.toml")); + fn push_config_entries_rejects_reserved_key() { + let dir = tempdir().expect("tempdir"); + let bad_key = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + let err = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(bad_key.clone(), "{}".to_owned())], + &AdapterPushContext::new(), + false, + ) + .expect_err("reserved key must be rejected"); + assert!(err.contains(&bad_key), "names the key: {err}"); } + /// Schema drift must never echo the config payload — including OBJECT KEYS, + /// which are provider/stored data. App config can hold credentials; CLI + /// status lines are logged verbatim and CI logs are retained/shared. Only a + /// size + field COUNT may be reported. + #[cfg(unix)] #[test] - fn finds_manifest_in_current_directory() { - let dir = tempdir().unwrap(); - let root = dir.path(); - fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); - fs::write(root.join("fastly.toml"), "name = \"demo\"").unwrap(); + fn read_config_entry_schema_drift_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_abc123"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // The sentinel is an OBJECT KEY (not a value): the earlier redactor joined + // keys into the diagnostic, so this is what pins the key-disclosure fix. + let drift = format!(r#"{{"{SENTINEL}":"x"}}"#); + let fake = fake_fastly_returning(&drift, "", 0); + let _path = PathPrepend::new(fake.path()); - let manifest = find_fastly_manifest(root).expect("should find manifest"); - assert_eq!(manifest, root.join("fastly.toml")); + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("schema drift must error") + }; + assert!( + !err.contains(SENTINEL), + "error must not leak an object KEY from the config payload: {err}" + ); + assert!( + err.contains("bytes") && err.contains("field(s)"), + "error should carry a redacted size + field COUNT: {err}" + ); } + /// The FAILURE branch leaks too: a Fastly error that quotes the stored + /// value back in stderr must not reach the user-facing error. + #[cfg(unix)] #[test] - fn locate_artifact_considers_workspace_target() { - let dir = tempdir().unwrap(); - let workspace = dir.path(); - let manifest_dir = workspace.join("service"); - fs::create_dir_all(manifest_dir.join("target/wasm32-wasip1/release")).unwrap(); - let artifact = workspace.join("target/wasm32-wasip1/release/demo.wasm"); - fs::create_dir_all(artifact.parent().unwrap()).unwrap(); - fs::write(&artifact, "wasm").unwrap(); + fn read_config_entry_stderr_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_stderr1"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Not a "not found" — a hard failure that echoes the value. + let stderr = format!("Error: internal failure processing value {SENTINEL}"); + let fake = fake_fastly_returning("", &stderr, 1); + let _path = PathPrepend::new(fake.path()); - let located = locate_artifact(workspace, &manifest_dir, "demo").unwrap(); - assert_eq!(located, artifact); + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("hard stderr failure must error") + }; + assert!( + !err.contains(SENTINEL), + "stderr must be redacted, not echoed: {err}" + ); + assert!( + err.contains("suppressed"), + "error should say the stderr was suppressed: {err}" + ); } + /// The WRITE path leaks too: a failing `config-store-entry update --upsert` + /// whose stderr quotes the value being written must be redacted. + #[cfg(unix)] #[test] - fn read_package_falls_back_to_name() { - let dir = tempdir().unwrap(); - let manifest = dir.path().join("Cargo.toml"); - fs::write(&manifest, "name = \"demo\"").unwrap(); - let name = read_package_name(&manifest).unwrap(); - assert_eq!(name, "demo"); + fn upsert_stderr_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_upsert1"; + let _lock = path_mutation_guard().lock().expect("guard"); + // A fake `fastly` that fails every call, echoing the value in stderr. + let stderr = format!("Error: rejected value {SENTINEL}"); + let fake = fake_fastly_returning("", &stderr, 1); + let _path = PathPrepend::new(fake.path()); + + let err = create_config_store_entry("store-abc", "cfg", SENTINEL) + .expect_err("a failing upsert must error"); + assert!( + !err.contains(SENTINEL), + "upsert stderr must be redacted, not echoed: {err}" + ); + assert!( + err.contains("suppressed"), + "error should say the stderr was suppressed: {err}" + ); } + /// `config gc` reads `item_value` for every entry (to classify roots). A + /// malformed listing whose values carry secrets must fail closed WITHOUT + /// echoing any value. (Replaces the old push prior-read redaction tests, + /// which are now vacuous: a cloud push performs no pre-commit read.) + #[cfg(unix)] #[test] - fn read_package_prefers_package_table() { - let dir = tempdir().unwrap(); - let manifest = dir.path().join("Cargo.toml"); - fs::write(&manifest, "[package]\nname = \"demo\"\n").unwrap(); - let name = read_package_name(&manifest).unwrap(); - assert_eq!(name, "demo"); - } + fn gc_list_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_gc_list"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); - // ---------- push_entries_with_committer ---------- + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let good = entry_list_json(&listing); + // A valid entry whose VALUE contains the sentinel, plus a malformed + // sibling (no created_at) to trip the fail-closed path. + let mut array: serde_json::Value = serde_json::from_str(&good).unwrap(); + let arr = array.as_array_mut().unwrap(); + arr.push(serde_json::json!({ + "item_key": "some.__edgezero_chunks.deadbeef.0", + "item_value": SENTINEL, + })); + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + None, + false, + &dir.path().join("ops.log"), + ); + fs::write( + fake.path().join("entries.json"), + serde_json::to_string(&array).unwrap(), + ) + .expect("overwrite entries"); + let _path = PathPrepend::new(fake.path()); - #[test] - fn push_entries_with_committer_returns_count_when_all_succeed() { - let entries = vec![ - ("a".to_owned(), "1".to_owned()), - ("b".to_owned(), "2".to_owned()), - ("c".to_owned(), "3".to_owned()), - ]; - let pushed = push_entries_with_committer(&entries, |_, _| Ok(())).expect("all succeed"); - assert_eq!(pushed, 3); + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + !err.contains(SENTINEL), + "the fail-closed error must not echo a stored value: {err}" + ); } + #[cfg(unix)] #[test] - fn push_entries_with_committer_zero_entries_is_ok() { - let pushed = push_entries_with_committer(&[], |_, _| Ok(())).expect("empty is fine"); - assert_eq!(pushed, 0); - } + fn push_config_entries_writes_direct_entry_at_exactly_8000_chars() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); - #[test] - fn push_entries_with_committer_failure_surfaces_committed_failed_not_attempted() { - // Mock committer: succeed for first 2 keys, fail at third. - let entries = vec![ - ("k1".to_owned(), "v1".to_owned()), - ("k2".to_owned(), "v2".to_owned()), - ("k3".to_owned(), "v3".to_owned()), - ("k4".to_owned(), "v4".to_owned()), - ("k5".to_owned(), "v5".to_owned()), - ]; - let mut calls: usize = 0; - let err = push_entries_with_committer(&entries, |key, _| { - calls = calls.saturating_add(1); - if key == "k3" { - Err("simulated fastly stderr".to_owned()) - } else { - Ok(()) - } - }) - .expect_err("middle failure must error"); - // Committer was invoked for k1, k2, k3 and stopped. - assert_eq!(calls, 3_usize, "no retries beyond failure point"); - // Error names all three categories. - assert!(err.contains("k1") && err.contains("k2"), "committed: {err}"); - assert!( - err.contains("Failed: `k3`"), - "failed entry named exactly: {err}" - ); + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + assert_eq!(envelope.len(), FASTLY_CONFIG_ENTRY_LIMIT); + + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push must succeed"); + // One physical entry written (direct). + let captured = fs::read_to_string(&argv_log).expect("argv log"); assert!( - err.contains("k4") && err.contains("k5"), - "not-attempted: {err}" + captured.contains(&format!("--key={TEST_CONFIG_ID}")), + "must write root key directly: {captured}" ); - assert!(err.contains("simulated fastly stderr"), "inner err: {err}"); - // Counts are sane. assert!( - err.contains("committing 2 of 5 entries"), - "committed/total count: {err}" + out[0].contains("1 physical entries (1 logical)"), + "summary reports 1 physical entry: {out:?}" ); } + #[cfg(unix)] #[test] - fn push_entries_with_committer_first_entry_failure_reports_zero_committed() { - let entries = vec![ - ("only".to_owned(), "val".to_owned()), - ("never".to_owned(), "tried".to_owned()), - ]; - let err = push_entries_with_committer(&entries, |_, _| Err("nope".to_owned())) - .expect_err("first-entry failure"); - assert!(err.contains("committing 0 of 2"), "zero committed: {err}"); + fn push_config_entries_writes_chunks_and_root_pointer_for_8001_chars() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!(envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT); + + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push must succeed"); + let captured = fs::read_to_string(&argv_log).expect("argv log"); + // At least one chunk key must appear before the root key. assert!( - err.contains("Failed: `only`"), - "first-entry failure named: {err}" + captured.contains(".__edgezero_chunks."), + "chunk keys must be written: {captured}" ); + // Root pointer must also be written. assert!( - err.contains("never"), - "second entry as not-attempted: {err}" + captured.contains(&format!("--key={TEST_CONFIG_ID}")), + "root pointer must be written: {captured}" ); - } - - #[test] - fn push_entries_with_committer_last_entry_failure_reports_n_minus_one_committed() { - let entries = vec![ - ("a".to_owned(), "1".to_owned()), - ("b".to_owned(), "2".to_owned()), - ("c".to_owned(), "3".to_owned()), - ]; - let err = push_entries_with_committer(&entries, |key, _| { - if key == "c" { - Err("late failure".to_owned()) - } else { - Ok(()) - } - }) - .expect_err("last-entry failure"); - assert!(err.contains("committing 2 of 3"), "n-1 committed: {err}"); + // Root key must be LAST in the log (chunk lines come before it). + let root_pos = captured.rfind(&format!("--key={TEST_CONFIG_ID}")).unwrap(); + let chunk_pos = captured.find(".__edgezero_chunks.").unwrap(); assert!( - err.contains("the remaining 0 entries"), - "zero not-attempted when last fails: {err}" + chunk_pos < root_pos, + "chunk writes must precede root pointer write: chunk_pos={chunk_pos} root_pos={root_pos}" ); + assert!(out[0].contains("logical"), "summary line present: {out:?}"); } - // ---------- looks_like_already_exists ---------- - + #[cfg(unix)] #[test] - fn looks_like_already_exists_recognises_common_phrasings() { - // Real-shaped fastly CLI error strings (paraphrased; the - // CLI varies across versions). Each must be detected so - // create_fastly_store can treat it as idempotent success. - assert!(looks_like_already_exists( - "Error: a kv-store with that name already exists", - "kv", - )); - assert!(looks_like_already_exists( - "ERROR: Conflict (409): duplicate kv_store name", - "kv", - )); - assert!(looks_like_already_exists( - "A config-store with this name already exists", - "config", - )); - // Spaced form: some fastly CLI versions emit prose - // ("kv store"); accept it alongside the punctuated forms. - assert!(looks_like_already_exists( - "Error: kv store conflict: name already in use", - "kv", - )); - } + fn push_config_entries_dry_run_reports_direct_vs_chunked() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); - #[test] - fn looks_like_already_exists_rejects_unrelated_errors() { - assert!(!looks_like_already_exists( - "Error: unauthenticated; run `fastly profile create`", - "kv", - )); - assert!(!looks_like_already_exists( - "Error: network unreachable", - "kv", - )); - assert!(!looks_like_already_exists("", "kv")); - } + let direct_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let chunked_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - #[test] - fn looks_like_already_exists_rejects_unrelated_conflict_errors() { - // The earlier wider heuristic swallowed ANY stderr - // containing "conflict" or "already exists", which would - // misread an unrelated 409 from a different fastly - // subcommand (e.g. a service-version conflict during a - // parallel deploy) as idempotent store-create success. - // Now we require the kind context too, so unrelated - // conflicts surface as failures. - assert!( - !looks_like_already_exists( - "Error: 409 Conflict on /service/abc/version/42 -- already exists", - "kv", - ), - "service-version conflict must NOT be misread as kv-store idempotency" - ); + let entries = vec![ + ("cfg_direct".to_owned(), direct_envelope), + ("cfg_chunked".to_owned(), chunked_envelope), + ]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("dry-run must not error"); + + // No shellout happens; output must describe intent. + let combined = out.join("\n"); assert!( - !looks_like_already_exists( - "Error: invalid duplicate request; check name resolution", - "kv", - ), - "unrelated `duplicate ... name` AND-match must NOT trigger" + combined.contains("would push `cfg_direct` as direct entry"), + "must report direct: {combined}" ); - // And the kind must match: a config-store conflict must - // not look-like-already-exists for a kv-store create call. assert!( - !looks_like_already_exists("Error: a config-store with that name already exists", "kv",), - "wrong-kind conflict must NOT trigger" + combined.contains("would push `cfg_chunked` as chunked"), + "must report chunked: {combined}" ); } - // ---------- setup_block_present ---------- - + /// Spec 12.7: pushing two blobs under different root keys + /// (e.g. `app_config` + `app_config_staging`) must leave both + /// keys readable from the local fastly.toml so the runtime + /// `EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY` override can + /// switch between them. Prior to the upsert fix the second + /// push wholesale-replaced the per-store contents table. + #[cfg(unix)] #[test] - fn setup_block_present_true_when_table_exists() { + fn push_config_entries_local_preserves_sibling_keys() { let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "name = \"demo\"\n[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n", - ) - .expect("write"); - assert!(setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); - } + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + let store = ResolvedStoreId::from_logical(TEST_CONFIG_ID); + let ctx = AdapterPushContext::new(); - #[test] - fn setup_block_present_false_when_id_missing() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n[setup.kv_stores.other]\n").expect("write"); - assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); - } + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &store, + &[("app_config".to_owned(), "{\"envelope\":\"A\"}".to_owned())], + &ctx, + false, + ) + .expect("first push"); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &store, + &[( + "app_config_staging".to_owned(), + "{\"envelope\":\"B\"}".to_owned(), + )], + &ctx, + false, + ) + .expect("second push (sibling key)"); - #[test] - fn setup_block_present_false_for_missing_file() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("does-not-exist.toml"); - assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + let raw = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = raw.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents after sibling push"); + let app_config = contents + .get("app_config") + .and_then(toml_edit::Item::as_str) + .expect("default key must survive sibling push"); + assert_eq!( + app_config, "{\"envelope\":\"A\"}", + "default key value: {raw}" + ); + let staging = contents + .get("app_config_staging") + .and_then(toml_edit::Item::as_str) + .expect("staging key must be present"); + assert_eq!(staging, "{\"envelope\":\"B\"}", "staging key value: {raw}"); } + #[cfg(unix)] #[test] - fn setup_block_present_true_when_only_setup_exists() { - // Post-F6 (PR #269 round 2): `setup_block_present` only - // checks `[setup._stores.]`. The pre-fix check - // ALSO required `[local_server._stores.]`, but - // writing an empty `[local_server.*]` table didn't match - // fastly's local-server schema (config-stores need - // `format` + contents, kv/secret stores need a JSON file - // or `{key, data}` entries). Local-server seeding moved - // to `config push --adapter fastly --local`, so probe - // only cares about `[setup]` now. + fn push_config_entries_local_writes_literal_dotted_chunk_keys() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); - let only_setup = dir.path().join("only_setup.toml"); - fs::write(&only_setup, "name = \"demo\"\n[setup.kv_stores.sessions]\n").expect("write"); - assert!( - setup_block_present(&only_setup, "kv", TEST_KV_ID).expect("probe"), - "[setup.*] alone is now sufficient: {only_setup:?}" - ); - - let only_local = dir.path().join("only_local.toml"); - fs::write( - &only_local, - "name = \"demo\"\n[local_server.kv_stores.sessions]\n", - ) - .expect("write"); - assert!( - !setup_block_present(&only_local, "kv", TEST_KV_ID).expect("probe"), - "[local_server.*] alone is NOT a provisioned-setup signal" - ); - } + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("write"); - // ---------- append_fastly_setup ---------- + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("local push must succeed"); - #[test] - fn append_fastly_setup_creates_setup_table_in_minimal_file() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains("[setup.kv_stores.sessions]"), - "setup table added: {after}" - ); - // Post-F6: no `[local_server.*]` write — that empty stanza - // didn't satisfy fastly's local-server schema and made - // `fastly compute serve` error or skip the store. Local- - // server seeding is now `config push --adapter fastly - // --local`'s job. + let after = fs::read_to_string(&fastly_toml).expect("read back"); + // Chunk keys contain '.' and must appear as quoted string keys, + // not as TOML nested tables (which would look like [table.sub]). assert!( - !after.contains("[local_server.kv_stores.sessions]"), - "[local_server.*] empty table no longer written by provision: {after}" + after.contains(".__edgezero_chunks."), + "chunk keys written to fastly.toml: {after}" ); + // Parse with toml_edit and confirm chunk keys are string-keyed entries. + let doc: toml_edit::DocumentMut = after.parse().expect("must parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .expect("contents table must exist"); + // At least one chunk key must be present as a string value (not a table). + let has_chunk_string = contents.as_table().is_some_and(|tbl| { + tbl.iter() + .any(|(key, val)| key.contains(".__edgezero_chunks.") && val.as_value().is_some()) + }); assert!( - after.contains("name = \"demo\""), - "preserved original keys: {after}" + has_chunk_string, + "chunk keys must be literal string-valued entries, not nested tables: {after}" ); } + #[cfg(unix)] #[test] - fn append_fastly_setup_appends_alongside_existing_kind_tables() { + fn push_config_entries_local_dry_run_reports_chunking_and_does_not_edit_fastly_toml() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "[setup.kv_stores.cache]\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains("[setup.kv_stores.cache]"), - "existing entry kept: {after}" - ); - assert!( - after.contains("[setup.kv_stores.sessions]"), - "new entry added: {after}" - ); - } + let fastly_toml = dir.path().join("fastly.toml"); + let original = "name = \"demo\"\n"; + fs::write(&fastly_toml, original).expect("write"); - #[test] - fn append_fastly_setup_is_idempotent_on_duplicate_id() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "[setup.kv_stores.sessions]\nfoo = \"keep\"\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("idempotent append"); - let after = fs::read_to_string(&path).expect("read back"); + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("local dry-run must not error"); + + // File must be untouched. + let after = fs::read_to_string(&fastly_toml).expect("read back"); + assert_eq!(after, original, "dry-run must not edit fastly.toml"); + + // Output must describe chunking intent. + let combined = out.join("\n"); assert!( - after.contains("foo = \"keep\""), - "did not stomp existing key: {after}" + combined.contains("would set") && combined.contains("chunked"), + "must report chunked intent: {combined}" ); } + // ---------- chunked read integration tests ---------- + + #[cfg(unix)] #[test] - fn append_fastly_setup_creates_file_when_missing() { + fn read_config_entry_resolves_direct_value_unchanged() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Note: no fs::write — file starts absent. - append_fastly_setup(&path, "config", TEST_CONFIG_ID).expect("create"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains("[setup.config_stores.app_config]")); - assert!( - !after.contains("[local_server.config_stores.app_config]"), - "[local_server.*] no longer written by provision: {after}" + + let envelope = BlobEnvelope::new(json!({"hello": "world"}), "2026-06-22T00:00:00Z".into()); + let json_str = serde_json::to_string(&envelope).unwrap(); + let item_json = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(&json_str).unwrap() ); + let fake = fake_fastly_returning(&item_json, "", 0); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, json_str, "direct envelope passes through unchanged"); } + #[cfg(unix)] #[test] - fn append_fastly_setup_preserves_top_comments() { + fn read_config_entry_reconstructs_chunked_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "# managed by hand -- please keep this line\nname = \"demo\"\n", - ) - .expect("write"); - append_fastly_setup(&path, "secret", TEST_SECRET_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains("# managed by hand"), - "preserved comment: {after}" + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + // Build a key→response map for every physical entry. + let mut key_responses: Vec<(String, String)> = Vec::new(); + for (pk, pv) in &physical { + let resp = format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()); + key_responses.push((pk.clone(), resp)); + } + // The root key should return the pointer. + let ptr_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() ); - } + key_responses.push((TEST_CONFIG_ID.to_owned(), ptr_resp)); - // ---------- write_fastly_local_config_store (config push --local) ---------- + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); - #[test] - fn write_fastly_local_config_store_creates_inline_block_in_minimal_file() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let entries = vec![ - ("greeting".to_owned(), "hello".to_owned()), - ("service.timeout_ms".to_owned(), "1500".to_owned()), - ]; - write_fastly_local_config_store(&path, TEST_CONFIG_ID, &entries).expect("write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains(&format!("[local_server.config_stores.{TEST_CONFIG_ID}]")), - "store table: {after}" - ); - assert!( - after.contains("format = \"inline-toml\""), - "format field: {after}" - ); - assert!( - after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - )), - "contents table: {after}" - ); - assert!(after.contains("greeting = \"hello\""), "key 1: {after}"); - assert!( - after.contains("\"service.timeout_ms\" = \"1500\""), - "dotted key quoted: {after}" + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("chunked read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!( + value, envelope, + "reconstructed envelope must equal original" ); - assert!(after.contains("name = \"demo\""), "preserved: {after}"); } + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_replaces_existing_block_on_re_push() { + fn read_config_entry_errors_on_missing_chunk() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), "stale".to_owned())], - ) - .expect("first write"); - write_fastly_local_config_store( - &path, + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + // Only provide the root pointer; omit chunk responses so chunk fetch returns not-found. + let ptr_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() + ); + let key_responses = vec![(TEST_CONFIG_ID.to_owned(), ptr_resp)]; + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), TEST_CONFIG_ID, - &[("greeting".to_owned(), "fresh".to_owned())], - ) - .expect("second write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains("greeting = \"fresh\""), "new value: {after}"); + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("missing chunk must error") + }; assert!( - !after.contains("greeting = \"stale\""), - "stale value dropped: {after}" + err.contains("missing chunk"), + "error must mention missing chunk: {err}" ); } + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_preserves_unrelated_blocks() { + fn read_config_entry_errors_on_corrupt_chunk_hash() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - let original = "\ -[setup.kv_stores.sessions] -[[local_server.kv_stores.sessions]] -key = \"__init__\" -data = \"\" + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + let mut key_responses: Vec<(String, String)> = Vec::new(); + // Corrupt first chunk's content. + let (first_chunk_key, first_chunk_val) = &physical[0]; + let corrupted: String = first_chunk_val.chars().map(|_| 'Z').collect(); + let corrupt_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(&corrupted).unwrap() + ); + key_responses.push((first_chunk_key.clone(), corrupt_resp)); + // Remaining chunks as normal. + for (pk, pv) in physical + .iter() + .take(physical.len().saturating_sub(1)) + .skip(1) + { + key_responses.push(( + pk.clone(), + format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()), + )); + } + key_responses.push(( + TEST_CONFIG_ID.to_owned(), + format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() + ), + )); + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); -[scripts] -build = \"cargo build --release\" -"; - fs::write(&path, original).expect("write"); - write_fastly_local_config_store( - &path, + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), TEST_CONFIG_ID, - &[("greeting".to_owned(), "hi".to_owned())], - ) - .expect("write"); - let after = fs::read_to_string(&path).expect("read back"); + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("corrupt chunk must error") + }; assert!( - after.contains("[setup.kv_stores.sessions]"), - "setup KV kept: {after}" + err.contains("does not match the SHA-256"), + "error must say what failed: {err}" ); - assert!(after.contains("[scripts]"), "scripts table kept: {after}"); + // identified by POSITION, and neither hash echoed -- the + // expected one comes from the stored pointer, so it is value-controlled. assert!( - after.contains("build = \"cargo build --release\""), - "scripts value kept: {after}" + err.contains("chunk 0"), + "error must locate the failing chunk by position: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn read_config_entry_errors_on_malformed_pointer() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Root value is JSON but neither a BlobEnvelope nor a valid pointer. + let bad_json = r#"{"some_field":"not a pointer or envelope"}"#; + let item_json = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(bad_json).unwrap() + ); + let fake = fake_fastly_returning(&item_json, "", 0); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), ); + let Err(err) = result else { + panic!("malformed pointer must error") + }; assert!( - after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - )), - "new config_stores block added: {after}" + err.contains("neither a valid BlobEnvelope") || err.contains("chunk pointer"), + "error must describe parse failure: {err}" ); } + // ---------- local read integration tests ---------- + #[test] - fn write_fastly_local_config_store_creates_file_when_missing() { + fn read_config_entry_local_resolves_direct_value() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // No fs::write — file absent. + let fastly_toml = dir.path().join("fastly.toml"); + + let envelope = BlobEnvelope::new(json!({"x": 1_i32}), "2026-06-22T00:00:00Z".into()); + let json_str = serde_json::to_string(&envelope).unwrap(); + // Write directly as a single entry (not via push_config_entries_local so we + // control the exact TOML content). write_fastly_local_config_store( - &path, + &fastly_toml, TEST_CONFIG_ID, - &[("greeting".to_owned(), "hi".to_owned())], + &[("cfg".to_owned(), json_str.clone())], + &[], ) .expect("write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - ))); - assert!(after.contains("greeting = \"hi\"")); - } - // ---------- provision (dry-run + error path) ---------- + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("local read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, json_str, "direct envelope passes through unchanged"); + } #[test] - fn provision_dry_run_does_not_invoke_fastly() { + fn read_config_entry_local_reconstructs_chunked_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let config_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_CONFIG_ID]); - let secret_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_SECRET_ID]); - let stores = ProvisionStores { - config: &config_ids, - kv: &kv_ids, - secrets: &secret_ids, + let fastly_toml = dir.path().join("fastly.toml"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + // Write all physical entries (chunks + pointer) to the local store. + write_fastly_local_config_store(&fastly_toml, TEST_CONFIG_ID, &physical, &[]) + .expect("write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("local chunked read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, true) - .expect("dry-run succeeds"); - // 1 KV + 1 config + 1 secret + 1 runtime-env = 4 status lines. - assert_eq!(out.len(), 4); - assert!(out[0].contains("would run `fastly kv-store create --name=sessions`")); - assert!(out[1].contains("would run `fastly config-store create --name=app_config`")); - assert!(out[2].contains("would run `fastly secret-store create --name=default`")); - assert!( - out[3].contains("would run `fastly config-store create --name=edgezero_runtime_env`"), - "runtime-env store row: {out:?}", + assert_eq!( + value, envelope, + "reconstructed envelope must equal original" ); - // Manifest untouched. - let after = fs::read_to_string(&path).expect("read"); - assert_eq!(after, "name = \"demo\"\n", "dry-run mutated fastly.toml"); } + /// a corrupt/invalid prior value must NOT abort the + /// local read, or the CLI push aborts on the diff read before the writer's + /// fail-soft ("overwrite, warn, prune nothing") can repair the state. + /// `config push --local` is how an operator recovers, so the read reports + /// `Unsupported` ("cannot diff") and lets the write proceed. #[test] - fn provision_errors_when_adapter_manifest_path_missing() { + fn read_config_entry_local_degrades_corrupt_prior_to_unsupported() { + use crate::chunked_config::{CHUNK_KEY_INFIX, POINTER_KIND}; let dir = tempdir().expect("tempdir"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let stores = ProvisionStores { - config: &[], - kv: &kv_ids, - secrets: &[], - }; - let err = FastlyCliAdapter - .provision(dir.path(), None, None, &stores, true) - .expect_err("missing adapter manifest path must error"); + let fastly_toml = dir.path().join("fastly.toml"); + + // A pointer-KIND value that is invalid (missing the chunks it needs). + // The resolver would error on this; the local read must NOT propagate + // that as `Err`. + let broken_pointer = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"cfg{CHUNK_KEY_INFIX}{sha}.0","len":10,"sha256":"x"}}],"data_sha256":"","envelope_len":10,"envelope_sha256":"{sha}"}}"#, + sha = "a".repeat(64), + ); + write_fastly_local_config_store( + &fastly_toml, + TEST_CONFIG_ID, + &[("cfg".to_owned(), broken_pointer)], + &[], + ) + .expect("write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a corrupt local prior must NOT abort the read"); assert!( - err.contains("fastly.toml"), - "error names what's missing: {err}" + matches!(result, ReadConfigEntry::Unsupported(_)), + "a corrupt prior value must degrade to Unsupported so the push can overwrite it" ); } + /// A `contents` that is not a table (a scalar or array) is malformed store + /// state. It must degrade to `Unsupported`, not fall through to `MissingKey` + /// (which would render an inaccurate "all values added" diff). #[test] - fn provision_with_no_declared_stores_says_so() { + fn read_config_entry_local_non_table_contents_is_unsupported() { let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Pre-populate the runtime-env block so the provision flow's - // unconditional runtime-env step skips (otherwise it would - // shell out to real `fastly` to create the store). + let fastly_toml = dir.path().join("fastly.toml"); fs::write( - &path, - "name = \"demo\"\n[setup.config_stores.edgezero_runtime_env]\n", + &fastly_toml, + format!("[local_server.config_stores.{TEST_CONFIG_ID}]\ncontents = 42\n"), ) - .expect("write"); - let stores = ProvisionStores { - config: &[], - kv: &[], - secrets: &[], - }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, false) - .expect("no-store provision is fine"); - assert_eq!(out, vec!["fastly has no declared stores to provision"]); + .expect("seed"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a non-table contents must NOT abort the read"); + assert!( + matches!(result, ReadConfigEntry::Unsupported(_)), + "a non-table `contents` must degrade to Unsupported, not MissingKey" + ); } + /// A malformed PARENT table (`local_server` etc. as a scalar) must degrade to + /// Unsupported, not collapse to `MissingStore`'s "all values added" diff. #[test] - fn provision_skips_id_when_setup_block_already_present() { - // setup_block_present's role in the flow: re-running - // provision after the user already declared a store in - // fastly.toml must be a no-op (no shell-out to fastly). - // We can verify this in a real (non-dry-run) call because - // the skip path bypasses create_fastly_store entirely. + fn read_config_entry_local_non_table_parent_is_unsupported() { let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n\ - [setup.config_stores.edgezero_runtime_env]\n", - ) - .expect("write"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let stores = ProvisionStores { - config: &[], - kv: &kv_ids, - secrets: &[], - }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, false) - .expect("skip path succeeds without invoking fastly"); - assert_eq!(out.len(), 1); - assert!(out[0].contains("already declared"), "got: {out:?}"); + let fastly_toml = dir.path().join("fastly.toml"); + // `local_server` is a scalar, not a table. + fs::write(&fastly_toml, "local_server = 42\n").expect("seed"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a non-table parent must NOT abort the read"); + assert!( + matches!(result, ReadConfigEntry::Unsupported(_)), + "a non-table parent must degrade to Unsupported, not MissingStore" + ); } - /// When `fastly.toml` declares `service_id`, the next - /// `fastly compute deploy` skips `[setup]` entirely. provision - /// must emit the `fastly resource-link create` remediation for - /// every store it creates -- including the implicit - /// `edgezero_runtime_env` store the runtime override path - /// depends on. Without this, a freshly-provisioned override - /// store would not be linked to the already-deployed service - /// and the runtime would silently fall back to baked defaults. + /// Spec 12.3 + 9.3: a second oversized push must converge the + /// runtime on the NEW envelope — chunk keys are content-addressed + /// by the full-envelope SHA, so push B writes a new chunk-set and + /// installs a new root pointer. + /// + /// The local fastly.toml writer upserts per-key (so a sibling + /// `--key app_config_staging` push leaves `app_config` intact per + /// spec 12.7). Within the SAME root key, GC on re-push prunes the + /// prior generation: after envelope B's push, envelope A's chunks — + /// now unreferenced by the `app_config` pointer — are removed from + /// the contents table. A read after push B follows the active + /// pointer and reconstructs envelope B, not A. + #[cfg(unix)] #[test] - fn provision_emits_resource_link_note_for_runtime_env_on_existing_service() { - // Dry-run only -- we just want to drive the resource_link_note - // helper for the runtime-env store branch. The real-create - // path can't run in tests (would shell out to `fastly`). - // The dry-run output line for runtime-env doesn't include the - // note (the helper only fires on real create), so we test the - // helper directly here. + #[expect( + clippy::too_many_lines, + reason = "linear test scenario: push A, inspect, push B, inspect, read; splitting would obscure the chunk-set comparison" + )] + fn second_oversized_push_converges_runtime_on_new_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\nservice_id = \"abc123svc\"\n").expect("write"); - let note = resource_link_note(&path, "config", "edgezero_runtime_env") - .expect("read service_id") - .expect("note present when service_id set"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + // First push: envelope A. Records the chunk-key set so we can + // confirm they are pruned by the second push's GC. + let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_a.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("first push must succeed"); + + let after_a = fs::read_to_string(&fastly_toml).expect("read"); + let doc_a: toml_edit::DocumentMut = after_a.parse().expect("parse"); + let contents_a = doc_a + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents table after push A"); + let chunks_a: Vec = contents_a + .iter() + .map(|(key, _)| key.to_owned()) + .filter(|key| key.contains(".__edgezero_chunks.")) + .collect(); assert!( - note.contains("service_id = \"abc123svc\""), - "note quotes the service id: {note}" + !chunks_a.is_empty(), + "push A must have produced chunk entries: {after_a}" ); + + // Second push: a DIFFERENT oversized envelope B. The + // content-addressed chunk keys must shift to B's sha; GC then + // prunes the old A-chunks. Build envelope B with a distinct + // payload key so its SHA differs from A's even at the same + // total length. + let envelope_b = { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ "alt": "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:01Z".to_owned())) + .expect("envelope B serialises") + }; + assert_ne!(envelope_a, envelope_b, "test fixtures must differ"); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_b.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("second push must succeed"); + + let after_b = fs::read_to_string(&fastly_toml).expect("read"); + let doc_b: toml_edit::DocumentMut = after_b.parse().expect("parse"); + let contents_b = doc_b + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents table after push B"); + let chunks_b: Vec = contents_b + .iter() + .map(|(key, _)| key.to_owned()) + .filter(|key| key.contains(".__edgezero_chunks.")) + .collect(); assert!( - note.contains("fastly config-store list --json"), - "note tells operator how to find the store id: {note}" + !chunks_b.is_empty(), + "push B must have produced chunk entries: {after_b}" ); + + // Chunk keys are content-addressed by envelope SHA, so the B + // push installs a fresh chunk-set whose keys are all distinct + // from A's. GC on re-push prunes the now-unreferenced A-chunks. + let new_b_chunks: Vec<&String> = chunks_b + .iter() + .filter(|key| !chunks_a.contains(*key)) + .collect(); assert!( - note.contains("name=`edgezero_runtime_env`"), - "note names the runtime override store: {note}" + !new_b_chunks.is_empty(), + "push B must have added at least one new content-addressed chunk: A-set={chunks_a:?} B-set={chunks_b:?}" ); - assert!( - note.contains( - "fastly resource-link create --service-id=abc123svc --resource-id= --version=latest --autoclone --name=edgezero_runtime_env" - ), - "note carries the full resource-link command: {note}" + // Old A-chunks are pruned: GC deletes the prior generation the + // old pointer referenced once B's pointer supersedes it. + for chunk_key in &chunks_a { + assert!( + !chunks_b.contains(chunk_key), + "old A-chunk `{chunk_key}` must be pruned from the local table after push B; B-set={chunks_b:?}" + ); + } + + // Runtime-correctness property: a fresh read after push B + // reconstructs envelope B (NOT envelope A). + let read = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("local read after push B"); + let ReadConfigEntry::Present(value) = read else { + panic!("expected Present after push B"); + }; + assert_eq!( + value, envelope_b, + "read after second push must reconstruct envelope B, not A" + ); + assert_ne!( + value, envelope_a, + "old envelope A's chunks must be inert -- read must NOT return A" ); } - /// And the inverse: no `service_id` (a service that hasn't been - /// deployed yet) means `[setup]` will be applied on the next - /// `compute deploy`, so no manual resource-link step is needed. - /// The helper must return `None` to avoid noisy false-positive - /// guidance. - #[test] - fn provision_skips_resource_link_note_when_service_undeployed() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let note = - resource_link_note(&path, "config", "edgezero_runtime_env").expect("read service_id"); - assert!( - note.is_none(), - "no service_id => no resource-link prompt: {note:?}" - ); + // ---------- config gc (operator-invoked reclamation) ---------- + + #[cfg(unix)] + fn run_gc(dir: &Path, older_than_secs: u64, dry_run: bool) -> Result, String> { + FastlyCliAdapter.gc_config_entries( + dir, + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &AdapterPushContext::new(), + older_than_secs, + dry_run, + ) } - // ---------- find_config_store_id ---------- + /// gc never deletes a chunk the LIVE root pointer references, however old. + #[cfg(unix)] + #[test] + fn gc_never_deletes_live_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + // The live generation is ANCIENT, but it is referenced by the root. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); - #[test] - fn find_config_store_id_matches_bare_array_by_name() { - let stdout = format!( - r#"[ - {{"id": "abc123", "name": "{TEST_CONFIG_ID}"}}, - {{"id": "def456", "name": "other_store"}} - ]"# - ); - match find_config_store_id(&stdout, TEST_CONFIG_ID) { - ConfigStoreLookup::Found(id) => assert_eq!(id, "abc123"), - ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), - ConfigStoreLookup::SchemaDrift(detail) => { - panic!("expected Found, got SchemaDrift({detail})") - } + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 1, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "live chunk `{key}` must never be reclaimed; log:\n{log}\nout: {out:?}" + ); } } + /// gc reclaims unreferenced chunks older than the operator's threshold. + #[cfg(unix)] #[test] - fn find_config_store_id_tolerates_items_envelope() { - let stdout = format!( - r#"{{"items": [ - {{"id": "xyz789", "name": "{TEST_CONFIG_ID}"}} - ]}}"# - ); - match find_config_store_id(&stdout, TEST_CONFIG_ID) { - ConfigStoreLookup::Found(id) => assert_eq!(id, "xyz789"), - ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), - ConfigStoreLookup::SchemaDrift(detail) => { - panic!("expected Found, got SchemaDrift({detail})") - } + fn gc_reclaims_unreferenced_chunks_older_than_threshold() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + + // The live config has been stable for 2 days; the operator asserts a 1-day + // window. So everything superseded (<= when live went live, i.e. >= 2 + // days ago) is safely reclaimable. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); // a week old + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "orphan `{key}` older than the threshold must be reclaimed; out: {out:?}" + ); + } + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "live chunk `{key}` must survive" + ); } } + /// The soundness test (design-3 counterexample): a root whose + /// current config was deployed seconds ago must NOT have its prior generation + /// reclaimed, even if that generation's chunks are ANCIENT. The clock is the + /// live config's age, not the orphan chunk's own creation time. + #[cfg(unix)] #[test] - fn find_config_store_id_distinguishes_not_found_from_match_failure() { - // JSON parses cleanly, entries are well-formed - // (`name` + `id` strings present), but no entry matches - // → NotFound. Operator likely needs to run `provision`. - let stdout = r#"[{"id": "abc", "name": "other"}]"#; - assert!(matches!( - find_config_store_id(stdout, "missing"), - ConfigStoreLookup::NotFound - )); + fn gc_protects_recently_superseded_generation_with_old_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let prior = gen_envelope("prior"); + let prior_chunks = chunk_keys_of(TEST_CONFIG_ID, &prior); + + // Live config went live 30s ago; the prior generation's chunks are a year + // old but were superseded only 30s ago -> POPs may still serve them. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 30)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 30)); + listing.extend(listed_generation(TEST_CONFIG_ID, &prior, 31_536_000)); // ~1 year + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // Even a generous 1-day threshold must NOT delete the prior generation, + // because the live config has only been stable for 30 seconds. + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &prior_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a generation superseded 30s ago must be retained despite old chunks: `{key}`; log:\n{log}" + ); + } } + /// a live root whose pointer drops its + /// last chunk ref AND restates `envelope_len` as the remaining sum passes + /// every metadata check. The dropped chunk is then absent from the live set + /// and looks like a deletable orphan -- while the config still needs it. + /// + /// Guards the PLANNER's content verification (a unit test on + /// `gc_verify_generation` alone does not prove the planner calls it). + #[cfg(unix)] #[test] - fn find_config_store_id_flags_schema_drift_on_malformed_json() { - // Unparseable bytes are NOT a "store not found" — they're - // a "fastly CLI output format changed" signal. Operator - // needs different recovery (file a bug, pin CLI version) - // than for the "store doesn't exist yet" case. - let drift = find_config_store_id("not json", "anything"); + fn gc_fails_closed_when_a_live_pointer_underreports_its_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + // Padded so the generation is >= 3 chunks: this case needs a ref to + // drop that still leaves a plausible multi-chunk set behind. + let live = gen_envelope_padded("live", 20_000); + let (chunks, pointer_json) = chunked_parts(TEST_CONFIG_ID, &live); + assert!(chunks.len() >= 3, "need >= 3 chunks for this case"); + + // Doctor the pointer: drop the last ref, restate envelope_len to match + // the survivors. Generation, indexes, per-chunk lens and the sum all + // still agree -- only the CONTENT does not. + let mut pointer: serde_json::Value = serde_json::from_str(&pointer_json).expect("parse"); + let refs = pointer + .get_mut("chunks") + .and_then(serde_json::Value::as_array_mut) + .expect("chunks array"); + refs.pop().expect("drop the last chunk ref"); + let surviving_len: u64 = refs + .iter() + .filter_map(|chunk| chunk.get("len").and_then(serde_json::Value::as_u64)) + .sum(); + pointer["envelope_len"] = serde_json::json!(surviving_len); + let doctored = serde_json::to_string(&pointer).expect("serialise"); + + // The store still physically holds ALL the chunks, including the one the + // doctored pointer no longer names. + let orphaned_by_omission = chunks.last().expect("last chunk").0.clone(); + let stamp = stamp_secs_ago(999_999); + let mut listing = vec![(TEST_CONFIG_ID.to_owned(), stamp.clone(), doctored)]; + for (key, value) in chunks { + listing.push((key, stamp.clone(), value)); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); assert!( - matches!(drift, ConfigStoreLookup::SchemaDrift(_)), - "non-JSON stdout must be schema drift, got {drift:?}" + err.contains("does not reconstruct the envelope it claims"), + "expected a content-address mismatch on the live pointer, got: {err}" ); - let empty = find_config_store_id("", "anything"); assert!( - matches!(empty, ConfigStoreLookup::SchemaDrift(_)), - "empty stdout must be schema drift, got {empty:?}" + !oplog_has(&oplog, &format!("delete {orphaned_by_omission}")), + "a chunk the live config still needs must never be deleted because its pointer \ + under-reported it: `{orphaned_by_omission}`" ); } + /// a LONE entry whose value hashes to the generation + /// its own key names would otherwise "prove" itself and be deleted. But our + /// writer never emits a one-chunk generation (an oversized envelope always + /// splits into >= 2), so a group of one is never ours -- it is a root-like + /// value sitting at a chunk-shaped key. This is the case a pure hash check + /// cannot catch on its own. + #[cfg(unix)] #[test] - fn find_config_store_id_flags_schema_drift_when_shape_unexpected() { - // JSON parses but the top-level is neither a bare array - // nor an `{items: [...]}` envelope. - let stdout = r#"{"namespace": "fastly", "list": []}"#; - match find_config_store_id(stdout, "any") { - ConfigStoreLookup::SchemaDrift(detail) => { - assert!( - detail.contains("bare array") || detail.contains("items"), - "schema-drift detail names the expected shapes: {detail}" - ); - } - ConfigStoreLookup::Found(id) => panic!("expected SchemaDrift, got Found({id})"), - ConfigStoreLookup::NotFound => panic!("expected SchemaDrift, got NotFound"), - } - } + fn gc_never_reclaims_a_lone_self_consistent_chunk() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + // A complete envelope stored at a chunk-shaped key whose generation IS + // that envelope's own SHA -- so it reassembles to its content-address. + let squatter_value = gen_envelope("someones-real-config"); + let self_sha = sha256_hex(squatter_value.as_bytes()); + let squatter_key = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{self_sha}.0"); + listing.push(( + squatter_key.clone(), + stamp_secs_ago(31_536_000), + squatter_value, + )); - #[test] - fn find_config_store_id_flags_schema_drift_when_entries_lack_name_id() { - // Array of objects but none have BOTH string `name` and - // string `id` fields — suggests schema rename (e.g. - // fastly renamed `name` → `title`). - let stdout = format!(r#"[{{"title": "{TEST_CONFIG_ID}", "uid": "abc"}}]"#); - let drift = find_config_store_id(&stdout, TEST_CONFIG_ID); - assert!( - matches!(drift, ConfigStoreLookup::SchemaDrift(_)), - "entries lacking name/id must be schema drift, got {drift:?}" - ); - } + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); - #[test] - fn find_config_store_id_returns_not_found_for_empty_array() { - // Empty array IS a valid "store doesn't exist yet" signal, - // not schema drift — fastly CLI legitimately returns `[]` - // when no config-stores exist. - let drift = find_config_store_id("[]", "any"); + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - matches!(drift, ConfigStoreLookup::NotFound), - "empty array must be NotFound, got {drift:?}" + !oplog_has(&oplog, &format!("delete {squatter_key}")), + "a one-chunk 'generation' is never something this writer emitted, so it must not be \ + reclaimed even though it hashes to its own key: `{squatter_key}`; log:\n{log}" ); } - // ---------- push_config_entries (dry-run + error paths) ---------- - + /// a delete that fails on a generation's FIRST key has + /// an UNKNOWN outcome -- Fastly may have committed it before returning an + /// error. called this "whole and retryable", which is unsound: if the + /// failed delete did commit, a re-run finds a fragment. The honest report is + /// a NOTE that the outcome is uncertain, NOT a clean-retry promise. We still + /// stop the generation so a CONFIRMED partial delete cannot happen. + #[cfg(unix)] #[test] - fn push_dry_run_does_not_invoke_fastly() { + fn gc_first_delete_failure_is_reported_as_uncertain_not_clean_retry() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let entries = vec![ - ("greeting".to_owned(), "hello".to_owned()), - ("feature.new_checkout".to_owned(), "false".to_owned()), - ]; - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - true, - ) - .expect("dry-run succeeds"); - // First line names the resolve+publish flow; subsequent lines preview - // each key the push would create (so callers can eyeball the keyset - // before running for real). - assert_eq!(out.len(), 1 + entries.len(), "header + per-entry preview"); - assert!( - out[0].contains("would resolve fastly config-store `app_config`") - && out[0].contains("push entries"), - "dry-run header describes the would-be flow: {out:?}" + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // The FIRST chunk of the doomed generation fails to delete. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&dead_chunks[0]), + false, + &oplog, ); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); assert!( - out.iter().any(|line| line.contains("`greeting`")), - "dry-run lists `greeting`: {out:?}" + err.contains("unknown outcome"), + "a failed delete's outcome is unknown and must be reported as such: {err}" ); assert!( - out.iter() - .any(|line| line.contains("`feature.new_checkout`")), - "dry-run lists `feature.new_checkout`: {out:?}" + !err.contains("will retry them"), + "the disproven clean-retry promise must be gone: {err}" ); + // The siblings must NOT have been ATTEMPTED -- stopping is what prevents a + // CONFIRMED partial delete. + for key in dead_chunks.iter().skip(1) { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "after the first failure the generation must be left alone: `{key}`" + ); + } } + /// the stateful case. A remote delete that COMMITS but + /// still reports failure leaves a real fragment. On the SECOND run that + /// missing key makes the generation unprovable, so it must be reported as + /// left-untouched (surfaced), never silently dropped. + #[cfg(unix)] #[test] - fn push_with_no_entries_reports_no_op_without_invoking_fastly() { + fn gc_committed_but_failed_delete_surfaces_as_unprovable_next_run() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &[], - &AdapterPushContext::new(), - false, - ) - .expect("zero-entry push is fine"); - assert_eq!(out.len(), 1); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + + // SECOND run's world: the first chunk's delete committed last time, so it + // is gone. The generation is now a fragment. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let mut dead_gen = listed_generation(TEST_CONFIG_ID, &dead, 604_800); + let survivor = dead_gen[1].0.clone(); + dead_gen.remove(0); // the committed-deleted chunk is absent now + listing.extend(dead_gen); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); assert!( - out[0].contains("no config entries"), - "status line names the no-op: {out:?}" + !oplog_has(&oplog, &format!("delete {survivor}")), + "an unprovable fragment survivor must not be deleted: `{survivor}`" ); - } - - // ---------- read_config_entry_local ---------- - - #[test] - fn read_local_returns_missing_store_when_fastly_toml_absent() { - let dir = tempdir().expect("tempdir"); - // No fastly.toml written — file missing. - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing file is not an error"); assert!( - matches!(result, ReadConfigEntry::MissingStore), - "absent fastly.toml => MissingStore" + out.iter() + .any(|line| line.contains("not byte-identical to what this writer would produce")), + "the surviving fragment must be SURFACED as left-untouched, not silently dropped: {out:?}" ); } + /// if a delete fails PART-WAY through a generation, the + /// survivors are an incomplete generation that `prove_generation` can never + /// verify again -- so `gc` will never reclaim them. Claiming "re-run to + /// retry" there was false. Say plainly that recovery is manual. + #[cfg(unix)] #[test] - fn read_local_returns_missing_store_when_no_local_server_contents() { + fn gc_reports_stranded_survivors_as_manual_recovery() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // fastly.toml exists but has no [local_server.config_stores.*] block. - fs::write(&path, "name = \"demo\"\n[setup.config_stores.app_config]\n").expect("write"); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing local_server block is not an error"); - assert!( - matches!(result, ReadConfigEntry::MissingStore), - "no local_server stanza => MissingStore" + let oplog = dir.path().join("ops.log"); + + // Padded to >= 3 chunks so a mid-generation failure leaves survivors. + let live = gen_envelope("live"); + let dead = gen_envelope_padded("dead", 20_000); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 3, "need >= 3 chunks"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // The SECOND chunk fails: the first is already gone by then. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&dead_chunks[1]), + false, + &oplog, ); - } + let _path = PathPrepend::new(fake.path()); - #[test] - fn read_local_returns_missing_key_when_key_absent_from_contents() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Write a local_server block with a different key so the store exists - // but the requested key is absent. - fs::write( - &path, - format!( - "name = \"demo\"\n\ - [local_server.config_stores.{TEST_CONFIG_ID}]\n\ - format = \"inline-toml\"\n\ - [local_server.config_stores.{TEST_CONFIG_ID}.contents]\n\ - other_key = \"other_value\"\n" - ), - ) - .expect("write"); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing key is not an error"); + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); + assert!( + err.contains("INCOMPLETE generation") && err.contains("re-running will NOT help"), + "a stranded fragment must not be described as retryable: {err}" + ); + // It must name the survivors and how to remove them by hand. + for key in dead_chunks.iter().skip(2) { + assert!( + err.contains(key.as_str()), + "the operator needs the exact surviving keys: `{key}` missing from: {err}" + ); + } assert!( - matches!(result, ReadConfigEntry::MissingKey), - "key absent from contents => MissingKey" + err.contains("fastly config-store-entry delete"), + "give the operator the recovery command: {err}" ); + // And we stopped rather than deleting the rest. + for key in dead_chunks.iter().skip(2) { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "deletion must stop at the first failure in a generation: `{key}`" + ); + } } + /// root keys are free-form, so a chunk key can hold + /// shell metacharacters. Manual-recovery commands must render them so that + /// pasting cannot execute or misparse -- single-quoted, with embedded quotes + /// escaped. #[test] - fn read_local_returns_present_when_key_exists_in_contents() { + fn recovery_commands_are_shell_safe() { + // A key crafted to run `id` and to break argument parsing if unquoted. + let hostile = "app$(id).__edgezero_chunks.'; rm -rf /'.0".to_owned(); + let keys = [hostile.clone()]; + let rendered = recovery_commands("store-abc", &keys); + + // The dangerous substring is not sitting there unquoted. + assert!( + !rendered.contains("$(id)") || rendered.contains("'app$(id)"), + "shell-active text must be inside single quotes: {rendered}" + ); + // Every embedded single quote is closed-escaped-reopened, so no quote + // context leaks. + assert!( + rendered.contains(r"'\''"), + "embedded single quotes must be escaped as '\\'': {rendered}" + ); + // Sanity: what a POSIX shell would parse back out of our --key argument + // is EXACTLY the original key (round-trip through `sh`). + let key_arg = rendered + .split("--key=") + .nth(1) + .and_then(|rest| rest.split(" --auto-yes").next()) + .expect("a --key argument"); + let out = Command::new("sh") + .arg("-c") + .arg(format!("printf '%s' {key_arg}")) + .output() + .expect("run sh"); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + hostile, + "the shell must parse the quoted argument back to the exact key" + ); + } + + /// a valid DIRECT envelope at a chunk-shaped key is a + /// runtime-readable root, but round 9 only protected POINTER values there. + /// + /// Construction: pad a small valid envelope with trailing JSON whitespace + /// past the entry limit. The writer chunks it; chunk 0 (the first 7 000 + /// bytes) is the whole envelope plus trailing spaces, which STILL parses and + /// verifies as that envelope. So chunk 0's key holds a valid direct envelope + /// -- a root -- yet the generation round-trips through the writer and passes + /// every proof, so GC deletes chunk 0. + #[cfg(unix)] + #[test] + fn valid_envelope_at_chunk_shaped_key_is_a_protected_root() { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write initial toml"); + let oplog = dir.path().join("ops.log"); + + // A small valid envelope + trailing whitespace over the entry limit. + let envelope = BlobEnvelope::new(json!({"k":"v"}), "2026-06-22T00:00:00Z".into()); + let mut padded = serde_json::to_string(&envelope).unwrap(); + padded.push_str(&" ".repeat(8_200)); + let entries = prepare_fastly_config_entries(TEST_CONFIG_ID, &padded).expect("expand"); + assert!(entries.len() >= 3, "need >= 2 chunks + pointer"); + let holder_key = entries[0].0.clone(); + // Sanity: chunk 0's value IS a standalone valid envelope. + let parsed: BlobEnvelope = + serde_json::from_str(&entries[0].1).expect("chunk 0 must parse as an envelope"); + parsed.verify().expect("chunk 0 must verify as an envelope"); + + // Seed the store with the chunk entries only -- NO live pointer refers + // to them, so this generation looks orphaned. Aged old. + let stamp = stamp_secs_ago(604_800); + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + for (key, value) in &entries[..entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } - // Use a valid BlobEnvelope value — the resolver requires BlobEnvelope - // or chunk-pointer JSON; raw strings are not accepted post-chunking. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "fastly"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), envelope_json.clone())], - ) - .expect("setup write"); + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("key present"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present variant"); - }; - assert_eq!(value, envelope_json, "value matches what was written"); + drop(run_gc(dir.path(), 86_400, false)); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !oplog_has(&oplog, &format!("delete {holder_key}")), + "an entry whose value is a valid direct envelope is a runtime-readable root and must \ + never be deleted, whatever its key looks like: `{holder_key}`; log:\n{log}" + ); + // The SIBLING chunks must survive too: protecting the holder drops the + // generation to an incomplete group, which is left unprovable — so + // nothing in this generation is deleted, not just the holder. + for (key, _) in &entries[1..entries.len().saturating_sub(1)] { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a sibling of a protected root must also survive (the group is left \ + unprovable): `{key}`; log:\n{log}" + ); + } } + /// A self-scoped pointer at a chunk-shaped holder key (its chunks nest the + /// infix twice) must NOT abort store-wide GC: the doubly-nested chunks are + /// recognised as chunks (via the LAST infix), so the holder classifies as a + /// root, its references are counted live, and other roots still reclaim. + #[cfg(unix)] #[test] - fn read_local_roundtrips_with_push_local() { - // Write via push_config_entries_local, then read via - // read_config_entry_local — the two must agree on the value. - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - + fn gc_tolerates_a_self_scoped_pointer_at_a_chunk_shaped_root() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); + let oplog = dir.path().join("ops.log"); + + // A pointer parked at a chunk-shaped key, with chunks scoped to itself. + let holder_key = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "e".repeat(64)); + let nested = gen_envelope("nested"); + let nested_entries = prepare_fastly_config_entries(&holder_key, &nested).expect("expand"); + let (_, holder_pointer) = nested_entries.last().expect("pointer").clone(); + + // A normal live root, and a normal orphan generation that SHOULD reclaim. + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let stamp = stamp_secs_ago(604_800); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + listing.push((holder_key.clone(), stamp.clone(), holder_pointer)); + for (key, value) in &nested_entries[..nested_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } - // push_config_entries_local passes the value through the chunk-pointer - // helper which stores it verbatim when ≤ 8 000 chars. The reader then - // resolves it through the same resolver that requires BlobEnvelope JSON. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "roundtrip"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entries = vec![("greeting".to_owned(), envelope_json.clone())]; - FastlyCliAdapter - .push_config_entries_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - false, - ) - .expect("push succeeds"); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("read succeeds"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present after push+read roundtrip"); - }; - assert_eq!(value, envelope_json, "roundtrip value matches"); + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // The run must SUCCEED (not abort) and still reclaim the ordinary orphan. + run_gc(dir.path(), 86_400, false).expect("store-wide GC must not abort"); + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "an ordinary orphan must still be reclaimed despite the self-scoped pointer: `{key}`" + ); + } + assert!( + !oplog_has(&oplog, &format!("delete {holder_key}")), + "the chunk-shaped holder root must never be deleted" + ); } + /// A nested ORPHAN generation (chunks scoped to a chunk-shaped root, with NO + /// live pointer referencing them) must be grouped and reclaimed, not silently + /// dropped. Age and grouping split on the LAST infix, so the nested chunks are + /// attributed to their real (nested) root. + #[cfg(unix)] #[test] - fn read_local_requires_adapter_manifest_path() { + fn gc_reclaims_a_nested_orphan_generation() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let result = FastlyCliAdapter.read_config_entry_local( - dir.path(), - None, // adapter_manifest_path missing - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ); - match result { - Err(err) => assert!( - err.contains("[adapters.fastly.adapter].manifest"), - "error names the missing field: {err}" - ), - Ok(_) => panic!("expected Err when adapter_manifest_path is None"), + let oplog = dir.path().join("ops.log"); + + // A chunk-shaped root, and a full generation of chunks SCOPED to it — but + // no pointer references them, so they are orphaned. + let nested_root = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "f".repeat(64)); + let nested = gen_envelope("nested-orphan"); + let nested_entries = prepare_fastly_config_entries(&nested_root, &nested).expect("expand"); + let nested_chunks: Vec = nested_entries[..nested_entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + + let live = gen_envelope("live"); + let stamp = stamp_secs_ago(604_800); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + for (key, value) in &nested_entries[..nested_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &nested_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "a nested orphan generation must be reclaimed, not silently dropped: `{key}`; \ + log:\n{log}" + ); } } - // ---------- read_config_entry (fake fastly, remote shell-out) ---------- + /// Age attribution works per NESTED root: a nested orphan generation whose + /// nested root's live config went live RECENTLY must be RETAINED (POPs may + /// still serve the superseded generation), even though the orphan's own + /// chunks are old. This pins that `root_live_since` splits on the last infix. + #[cfg(unix)] + #[test] + fn gc_retains_a_nested_orphan_under_a_recently_changed_nested_root() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); - /// Build a tempdir containing a `fastly` shim script that: - /// - Responds to `config-store list --json` with a store-list JSON containing - /// `TEST_CONFIG_ID` mapped to `store-abc123`. - /// - Responds to `config-store-entry describe ...` with `stdout_body` on - /// stdout and `stderr_body` on stderr, exiting with `exit_code`. - /// - /// Payloads are written to separate sibling files so shell-active chars - /// in the content don't get re-interpreted by the script. + let nested_root = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "a".repeat(64)); + + // The nested root's CURRENT (live) generation, created 30s ago. + let live_nested = gen_envelope("live-nested"); + let live_entries = prepare_fastly_config_entries(&nested_root, &live_nested).expect("exp"); + let (_, live_pointer) = live_entries.last().expect("pointer").clone(); + + // An OLD orphan generation under the SAME nested root (a week old). + let old_nested = gen_envelope("old-nested-orphan"); + let old_entries = prepare_fastly_config_entries(&nested_root, &old_nested).expect("exp"); + let old_chunks: Vec = old_entries[..old_entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + // The nested root holds its live pointer; its live chunks are 30s old. + listing.push((nested_root.clone(), stamp_secs_ago(30), live_pointer)); + for (key, value) in &live_entries[..live_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp_secs_ago(30), value.clone())); + } + // The old orphan chunks are a week old. + for (key, value) in &old_entries[..old_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp_secs_ago(604_800), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // A generous 1-day window: the orphan's OWN chunks are older, but the + // nested root's live config is only 30s old, so its orphan is retained. + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &old_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a nested orphan under a recently-changed nested root must be retained: `{key}`; \ + log:\n{log}" + ); + } + } + + /// A generation is aged by its YOUNGEST member, so a generation with one + /// recent chunk is retained whole even if its other chunks are ancient. #[cfg(unix)] - fn fake_fastly_returning( - stdout_body: &str, - stderr_body: &str, - exit_code: i32, - ) -> tempfile::TempDir { - use std::os::unix::fs::PermissionsExt as _; + #[test] + fn gc_ages_a_generation_by_its_youngest_member() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let script_path = dir.path().join("fastly"); - let stdout_file = dir.path().join("stdout_payload.txt"); - let stderr_file = dir.path().join("stderr_payload.txt"); - let list_file = dir.path().join("list_payload.txt"); - // Store-list JSON: bare array with one entry matching TEST_CONFIG_ID. - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - fs::write(&stdout_file, stdout_body).expect("write stdout payload"); - fs::write(&stderr_file, stderr_body).expect("write stderr payload"); - fs::write(&list_file, list_json).expect("write list payload"); - let script = format!( - "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\ncat '{}' >&2\nexit {exit_code}\n", - list_file.display(), - stdout_file.display(), - stderr_file.display(), - ); - fs::write(&script_path, script).expect("write fastly script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - dir + let oplog = dir.path().join("ops.log"); + + // `app_config` live is direct, so there is no live-config age signal — + // aging falls to the generation's own chunks. + let live_direct = gen_envelope_padded("live-direct", 100); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + live_direct, + )]; + // The doomed generation: chunk 0 written 30s ago (YOUNG), the rest a week + // ago. Its youngest-member age (30s) is under the 1-day window. + let dead_parts = chunked_parts(TEST_CONFIG_ID, &dead).0; + for (idx, (key, value)) in dead_parts.iter().enumerate() { + let age = if idx == 0 { 30 } else { 604_800 }; + listing.push((key.clone(), stamp_secs_ago(age), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &dead_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a generation with a recent member must be retained WHOLE (aged by its youngest): \ + `{key}`; log:\n{log}" + ); + } } - /// Build a fake `fastly` that logs each argv token (one per line) to - /// `out_path`, handles the list call correctly, and exits 0 for both calls. + /// A delete failure in one generation must not stop an INDEPENDENT + /// generation's deletes. #[cfg(unix)] - fn fake_fastly_argv_log(out_path: &Path) -> tempfile::TempDir { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - use std::os::unix::fs::PermissionsExt as _; + #[test] + fn gc_failure_in_one_generation_does_not_stop_another() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let script_path = dir.path().join("fastly"); - let list_file = dir.path().join("list_payload.txt"); - let entry_file = dir.path().join("entry_payload.txt"); - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - // item_value must be a valid BlobEnvelope JSON so the resolver accepts it. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"v": "logged"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entry_json = format!( - r#"{{"item_value":{},"store_id":"store-abc123"}}"#, - serde_json::to_string(&envelope_json).expect("escape") - ); - fs::write(&list_file, list_json).expect("write list payload"); - fs::write(&entry_file, &entry_json).expect("write entry payload"); - let script = format!( - "#!/bin/sh\nfor arg in \"$@\"; do printf '%s\\n' \"$arg\" >> '{}'; done\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\nexit 0\n", - out_path.display(), - list_file.display(), - entry_file.display(), + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead_a = gen_envelope("dead-a"); + let dead_b = gen_envelope("dead-b"); + let a_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead_a); + let b_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead_b); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead_a, 604_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead_b, 604_800)); + + // Generation A's first delete fails. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&a_chunks[0]), + false, + &oplog, ); - fs::write(&script_path, script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - dir + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + // Generation B must still have been reclaimed despite A's failure. + for key in &b_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "an independent generation must still be reclaimed after another one fails: \ + `{key}`; err: {err}; log:\n{log}" + ); + } } - /// Process-wide mutex serialising PATH-mutating tests so parallel - /// test threads don't race on the environment variable. + /// key shape is not authoritative for ROOTS either. + /// + /// A valid pointer stored at a chunk-SHAPED key (`shadow.__edgezero_chunks. + /// .0`) is skipped by the live-set scan, which excludes chunk-shaped + /// keys up front. The runtime resolver follows any pointer it is given, so + /// that pointer's references ARE live -- but GC never sees them, calls the + /// generation orphaned, and deletes it. #[cfg(unix)] - fn path_mutation_guard() -> &'static Mutex<()> { - use std::sync::OnceLock; - static GUARD: OnceLock> = OnceLock::new(); - GUARD.get_or_init(|| Mutex::new(())) + #[test] + fn pointer_at_chunk_shaped_key_keeps_its_references_live() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + // `app_config`'s CURRENT config is small enough to store directly, so + // its own root references no chunks at all. + let live_direct = gen_envelope_padded("live-direct", 100); + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + live_direct, + )]; + + // An older chunked generation of `app_config` still exists... + let referenced = gen_envelope("still-referenced"); + let referenced_chunks = chunk_keys_of(TEST_CONFIG_ID, &referenced); + listing.extend(listed_generation(TEST_CONFIG_ID, &referenced, 604_800)); + + // ...and a pointer at a CHUNK-SHAPED key references it. The resolver + // would happily follow this, so those chunks are LIVE. + let (_, referenced_pointer) = chunked_parts(TEST_CONFIG_ID, &referenced); + let shadow_key = format!("shadow{CHUNK_KEY_INFIX}{}.0", "d".repeat(64)); + listing.push((shadow_key, stamp_secs_ago(604_800), referenced_pointer)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // The RESULT does not matter here (it may Err after the fix if the + // shadow pointer's own chunks are incomplete); the invariant is purely + // that no LIVE-referenced chunk is deleted, which the oplog proves. + drop(run_gc(dir.path(), 86_400, false)); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &referenced_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a chunk a live pointer references must never be deleted, whatever the KEY of \ + the entry holding that pointer looks like: `{key}`; log:\n{log}" + ); + } } + /// a FOREIGN writer needs NO preimage to satisfy a + /// content-address. Pick envelope E, compute H = sha256(E), split E however + /// you like, store the parts as `.__edgezero_chunks.H.0` / `.1`. Under + /// hash-only checking that group "proved" itself and was deleted. + /// + /// The round-trip closes it: the writer, given those same bytes, must emit + /// exactly these keys and values. A split at boundaries we would never + /// choose is not our output, so it is left alone. #[cfg(unix)] #[test] - fn read_remote_returns_present_on_success() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - + fn gc_never_reclaims_a_foreign_content_addressed_group() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - // Fake fastly: list succeeds with app_config → store-abc123; - // describe returns valid JSON with item_value that is a BlobEnvelope. - let envelope = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "fastly"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entry_json = format!( - r#"{{"item_value":{},"store_id":"store-abc123"}}"#, - serde_json::to_string(&envelope).expect("escape") - ); - let fake = fake_fastly_returning(&entry_json, "", 0); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + // A foreign writer's data: a valid envelope, content-addressed under our + // reserved namespace, but split at ITS OWN boundary (not our 7 000-byte + // UTF-8-safe one). Everything hashes correctly -- no preimage needed. + let foreign = gen_envelope_padded("foreign-tool", 20_000); + let generation = sha256_hex(foreign.as_bytes()); + let (head, tail) = foreign.split_at(1_234); + let foreign_keys = [ + format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{generation}.0"), + format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{generation}.1"), + ]; + listing.push(( + foreign_keys[0].clone(), + stamp_secs_ago(31_536_000), + head.to_owned(), + )); + listing.push(( + foreign_keys[1].clone(), + stamp_secs_ago(31_536_000), + tail.to_owned(), + )); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("fake fastly exit-0 must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, envelope); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &foreign_keys { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a group this writer would never have produced must not be reclaimed, however \ + well it hashes: `{key}`; log:\n{log}" + ); + } } + /// an entry can be chunk-SHAPED without being a chunk + /// -- a store may predate this feature or be shared, and push-time + /// reserved-key rejection cannot protect what already exists. Deleting one + /// would destroy live config. + /// + /// proof is CONTENT, not shape. A candidate generation is ours only + /// if it reassembles to the content-address its own keys name. Unprovable + /// entries are left UNTOUCHED and reported -- not fatal, because one foreign + /// entry must not block reclaiming the rest of the store forever. #[cfg(unix)] #[test] - fn read_remote_returns_missing_key_on_not_found_stderr() { + fn gc_leaves_unprovable_chunk_shaped_entries_untouched() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - // describe exits non-zero with "not found" in stderr → MissingKey. - let fake = fake_fastly_returning("", "Error: item not found", 1); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + // A real orphan generation: provable, old -> must still be reclaimed. + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // Pre-existing entries at chunk-shaped keys that we did NOT write: one + // holding somebody's real config envelope, one holding plain text. + // Both are old enough to look "eligible" on age alone. + let envelope_squatter = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "b".repeat(64)); + let text_squatter = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "c".repeat(64)); + listing.push(( + envelope_squatter.clone(), + stamp_secs_ago(31_536_000), + gen_envelope("someones-real-config"), + )); + listing.push(( + text_squatter.clone(), + stamp_secs_ago(31_536_000), + "just some plain text".to_owned(), + )); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("not-found maps to MissingKey (not Err)"); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + + for key in [&envelope_squatter, &text_squatter] { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "an entry we cannot prove we wrote must never be deleted: `{key}`; log:\n{log}" + ); + } + // Left untouched must not mean silently ignored. The wording must not + // over-claim either: these two entries fail for DIFFERENT reasons (a + // wrong content-address vs a count this writer never emits), so the + // summary says "not byte-identical to what this writer would produce" + // rather than naming one specific check. assert!( - matches!(result, ReadConfigEntry::MissingKey), - "not-found stderr => MissingKey" + out.iter() + .any(|line| line.contains("not byte-identical to what this writer would produce")), + "the summary must report what it declined to judge; out: {out:?}" ); + // ...and a genuine orphan generation is still reclaimed, so one foreign + // entry does not block the store. + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "a provable orphan generation must still be reclaimed: `{key}`; log:\n{log}" + ); + } } - /// The Fastly impl distinguishes store-not-found from key-not-found via - /// `resolve_remote_config_store_id`: when the list call exits non-zero and - /// the error string contains "not found", `read_config_entry` returns - /// `MissingStore` without ever calling the describe subcommand. + /// a key is unique in a config store, so duplicate rows + /// mean the listing is not one consistent view. Left alone, last-row-wins on + /// `created_at` could age a recent key into eligibility. #[cfg(unix)] #[test] - fn read_remote_returns_missing_store_on_appropriate_stderr() { - use std::os::unix::fs::PermissionsExt as _; + fn gc_fails_closed_on_duplicate_listing_keys() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - // Script that exits non-zero for the list call so resolve fails with - // a "not found" error, causing read_config_entry to return MissingStore. - let fake_dir = tempdir().expect("tempdir"); - let stderr_file = fake_dir.path().join("stderr_payload.txt"); - fs::write(&stderr_file, "Error: config store not found for service").expect("write stderr"); - let script_path = fake_dir.path().join("fastly"); - let script = format!( - "#!/bin/sh\ncat '{stderr}' >&2\nexit 1\n", - stderr = stderr_file.display(), + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let mut orphans = listed_generation(TEST_CONFIG_ID, &dead, 30); + // The same key twice, with conflicting ages: young (real) then ancient. + let (dup_key, _, dup_value) = orphans[0].clone(); + orphans.push((dup_key.clone(), stamp_secs_ago(31_536_000), dup_value)); + listing.extend(orphans); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + err.contains("more than once"), + "expected a refusal naming the duplicate key, got: {err}" ); - fs::write(&script_path, script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - let _path = PathPrepend::new(fake_dir.path()); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("list failure with not-found maps to MissingStore (not Err)"); assert!( - matches!(result, ReadConfigEntry::MissingStore), - "list not-found => MissingStore" + !oplog_has(&oplog, &format!("delete {dup_key}")), + "a duplicated row must not let a recent key be aged into eligibility" ); } - /// Verify that `read_config_entry` invokes - /// `fastly config-store-entry describe --store-id= --key= --json` - /// (after the resolve step that calls `fastly config-store list --json`). + /// `gc_config_entries` is a public trait method, so the + /// zero-window rule must live at the DESTRUCTIVE boundary, not only in the + /// CLI that usually calls it. Rejected before any `fastly` invocation. #[cfg(unix)] #[test] - fn read_remote_invokes_correct_argv() { + fn gc_adapter_boundary_rejects_a_zero_window() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("argv-log fake must succeed"); + + // Straight at the adapter, bypassing the CLI's own gate. + let err = run_gc(dir.path(), 0, false).expect_err("a destructive zero window must fail"); assert!( - matches!(result, ReadConfigEntry::Present(_)), - "expected Present from argv-log fake" + err.contains("non-zero `--older-than`"), + "expected the boundary itself to reject zero, got: {err}" ); - let captured = fs::read_to_string(&argv_log).expect("argv log"); - // The describe call must include these args (resolve call args - // are also captured but we only assert the describe shape here). assert!( - captured.contains("config-store-entry"), - "must invoke config-store-entry; got:\n{captured}" + !fs::read_to_string(&oplog) + .unwrap_or_default() + .contains("delete "), + "nothing may be deleted under a zero window" ); + // A DRY-RUN at zero is still allowed: it previews and deletes nothing. + run_gc(dir.path(), 0, true).expect("a dry-run may preview at zero"); + } + + /// a root whose value is TRUNCATED/unparseable must fail + /// closed. It is pointer-shaped garbage -- we cannot tell what it references, + /// so its (live!) chunks must not be judged orphaned. Regression guard: the + /// push-path helper returns `Ok([])` for a non-pointer value, which on THIS + /// path would read as "references nothing" and reclaim the whole store. + #[cfg(unix)] + #[test] + fn gc_fails_closed_on_truncated_root_pointer() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let (_, pointer) = chunked_parts(TEST_CONFIG_ID, &live); + // A write that landed half-way: a valid PREFIX of the real pointer that + // is no longer valid JSON. (Chars, not a byte slice -- never split a + // codepoint.) + let truncated: String = pointer.chars().take(40).collect(); assert!( - captured.contains("describe"), - "must pass describe subcommand; got:\n{captured}" + serde_json::from_str::(&truncated).is_err(), + "fixture must be unparseable to exercise the classifier: {truncated}" ); + + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + truncated, + )]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); assert!( - captured.contains("--store-id=store-abc123"), - "must pass resolved store id; got:\n{captured}" + err.contains("refusing to reclaim"), + "expected a fail-closed refusal, got: {err}" ); + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "nothing may be deleted when a root is unclassifiable: `{key}`" + ); + } + } + + /// an ENVELOPED listing (`{"items":[...]}`) may carry + /// pagination we do not follow. A page that omitted a root would make that + /// root's live chunks look orphaned -- and the completeness guard cannot see + /// a root that isn't there. Refuse the shape outright. + #[cfg(unix)] + #[test] + fn gc_fails_closed_on_enveloped_listing() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + let enveloped = format!( + r#"{{"items":{},"next_cursor":"abc"}}"#, + entry_list_json(&listing) + ); + + let fake = fake_fastly_gc_raw_list(TEST_CONFIG_ID, &enveloped, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); assert!( - captured.contains("--key=greeting"), - "must pass --key=; got:\n{captured}" + err.contains("bare array") && err.contains("Nothing was deleted"), + "expected a refusal naming the unsupported listing shape, got: {err}" ); assert!( - captured.contains("--json"), - "must pass --json flag; got:\n{captured}" + !fs::read_to_string(&oplog) + .unwrap_or_default() + .contains("delete "), + "an unsupported listing shape must delete nothing" ); } - // ---------- chunked push integration tests ---------- + /// a root with an EMPTY value is as dangerous as a + /// missing one -- it would classify as "references nothing" and orphan its + /// live chunks. The listing parser rejects it before any reasoning. + #[cfg(unix)] + #[test] + fn gc_fails_closed_on_empty_root_value() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); - /// Build a valid `BlobEnvelope` JSON string of approximately `target_len` bytes. + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + String::new(), + )]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); + assert!( + err.contains("empty `item_value`"), + "expected a refusal naming the empty field, got: {err}" + ); + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "nothing may be deleted on an unreadable listing: `{key}`" + ); + } + } + + /// the orphan's OWN age is mandatory -- an old root does + /// not license deleting a chunk written seconds ago (e.g. by a concurrent + /// push that has not committed its pointer yet). Both ages must clear the + /// window; the more restrictive wins. #[cfg(unix)] - fn make_test_envelope(target_len: usize) -> String { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - let pad = "x".repeat(target_len.saturating_add(64)); - let data = json!({ "pad": pad }); - let raw = - serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); - if raw.len() >= target_len { - let overhead = raw.len().saturating_sub(pad.len()); - let adjusted = "x".repeat(target_len.saturating_sub(overhead)); - let data2 = json!({ "pad": adjusted }); - serde_json::to_string(&BlobEnvelope::new(data2, "2026-06-22T00:00:00Z".into())).unwrap() - } else { - raw + #[test] + fn gc_retains_young_orphan_under_long_stable_root() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let fresh = gen_envelope("fresh"); + let fresh_chunks = chunk_keys_of(TEST_CONFIG_ID, &fresh); + + // The root's live config has been stable for a year -- so the live-config + // clock alone would happily reclaim. But these chunks were written 10s + // ago and no pointer names them yet. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 31_536_000)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 31_536_000)); + listing.extend(listed_generation(TEST_CONFIG_ID, &fresh, 10)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &fresh_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a chunk written 10s ago must be retained under a 1-day window regardless of \ + how stable its root is: `{key}`; log:\n{log}" + ); } } - /// Build a fake `fastly` script whose describe response depends on - /// the `--key=` argument: `key_responses` maps key names to JSON - /// item-value responses. Falls back to exit 1 "not found" for unknown keys. + /// A dry-run lists exactly what it would delete, and deletes nothing. #[cfg(unix)] - fn fake_fastly_with_key_dispatch( - _dir: &Path, - key_responses: &[(String, String)], - ) -> tempfile::TempDir { - use std::fmt::Write as _; - use std::os::unix::fs::PermissionsExt as _; - let fake_dir = tempdir().expect("tempdir"); - let list_file = fake_dir.path().join("list.json"); - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - fs::write(&list_file, list_json).expect("write list"); - // Write each key response to a named file. - let mut dispatch_lines = String::new(); - for (key, response) in key_responses { - let resp_file = fake_dir.path().join(format!("resp_{key}.json")); - fs::write(&resp_file, response).expect("write resp"); - // Use exact-match: iterate argv and compare each token literally - // so that a root key like "app_config" does NOT match a chunk key - // like "app_config.__edgezero_chunks.abc.0". - writeln!( - dispatch_lines, - " for arg in \"$@\"; do if [ \"$arg\" = \"--key={key}\" ]; then cat '{}'; exit 0; fi; done", - resp_file.display() - ) - .expect("write to String is infallible"); + #[test] + fn gc_dry_run_lists_but_deletes_nothing() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, true).expect("dry-run succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "a dry-run must not delete; log:\n{log}" + ); + let rendered = out.join("\n"); + assert!( + rendered.contains("would delete"), + "lists intent: {rendered}" + ); + for key in &dead_chunks { + assert!(rendered.contains(key.as_str()), "names `{key}`: {rendered}"); } - // Fallback outputs "not found" so fetch_remote_config_store_entry - // maps it to Ok(None) rather than Err. - let script = format!( - "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\n{dispatch_lines}echo 'Error: item not found' >&2\nexit 1\n", - list_file.display() + } + + /// An unreadable `created_at` on a DELETE path fails CLOSED. + #[cfg(unix)] + #[test] + fn gc_fails_closed_on_unreadable_timestamp() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 3_600)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 3_600)); + // An orphan whose timestamp is garbage. + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + for key in dead_chunks { + listing.push((key, "not-a-timestamp".to_owned(), "X".to_owned())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + err.contains("unreadable") && err.contains("nothing was deleted"), + "must refuse to reclaim: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when the state is unreadable; log:\n{log}" ); - let script_path = fake_dir.path().join("fastly"); - fs::write(&script_path, &script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod"); - fake_dir } + /// A root whose pointer cannot be classified fails CLOSED — we cannot know + /// what it references, so nothing may be deleted. #[cfg(unix)] #[test] - fn push_config_entries_writes_direct_entry_at_exactly_8000_chars() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + fn gc_fails_closed_on_unclassifiable_root() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); - let _path = PathPrepend::new(fake.path()); + let oplog = dir.path().join("ops.log"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); - assert_eq!(envelope.len(), FASTLY_CONFIG_ENTRY_LIMIT); + let dead = gen_envelope("dead"); + // Root value is pointer-kind but invalid. + let bad = r#"{"edgezero_kind":"fastly_config_chunks","version":2}"#.to_owned(); + let mut listing = vec![(TEST_CONFIG_ID.to_owned(), stamp_secs_ago(3_600), bad)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - None, - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - false, - ) - .expect("push must succeed"); - // One physical entry written (direct). - let captured = fs::read_to_string(&argv_log).expect("argv log"); + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - captured.contains(&format!("--key={TEST_CONFIG_ID}")), - "must write root key directly: {captured}" + err.contains("could not classify root") && err.contains("nothing was deleted"), + "must refuse to reclaim: {err}" ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - out[0].contains("1 physical entries (1 logical)"), - "summary reports 1 physical entry: {out:?}" + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when a root is unclassifiable; log:\n{log}" ); } + /// A listing entry missing a required field fails CLOSED — a defaulted/empty + /// field could make a real root look like it references nothing, deleting + /// live chunks. #[cfg(unix)] #[test] - fn push_config_entries_writes_chunks_and_root_pointer_for_8001_chars() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + fn gc_fails_closed_on_malformed_listing_entry() { let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let good = entry_list_json(&listing); + // Inject an entry with NO item_value (drop that field entirely). + let mut array: serde_json::Value = serde_json::from_str(&good).unwrap(); + array.as_array_mut().unwrap().push(serde_json::json!({ + "item_key": "some.__edgezero_chunks.deadbeef.0", + "created_at": stamp_secs_ago(1000), + })); + // Serve that hand-built listing. + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + fs::write( + fake.path().join("entries.json"), + serde_json::to_string(&array).unwrap(), + ) + .expect("overwrite entries"); let _path = PathPrepend::new(fake.path()); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - assert!(envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT); - - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - None, - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - false, - ) - .expect("push must succeed"); - let captured = fs::read_to_string(&argv_log).expect("argv log"); - // At least one chunk key must appear before the root key. + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - captured.contains(".__edgezero_chunks."), - "chunk keys must be written: {captured}" + err.contains("missing a string") && err.contains("item_value"), + "must name the missing field: {err}" ); - // Root pointer must also be written. + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - captured.contains(&format!("--key={TEST_CONFIG_ID}")), - "root pointer must be written: {captured}" + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted on a malformed listing; log:\n{log}" ); - // Root key must be LAST in the log (chunk lines come before it). - let root_pos = captured.rfind(&format!("--key={TEST_CONFIG_ID}")).unwrap(); - let chunk_pos = captured.find(".__edgezero_chunks.").unwrap(); + } + + /// A failed delete is a non-zero exit that names the failed key(s), so + /// automation can detect partial failure. + #[cfg(unix)] + #[test] + fn gc_delete_failure_is_non_zero_exit() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let fail_key = dead_chunks.first().expect("a chunk").clone(); + + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&fail_key), + false, + &oplog, + ); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete must be non-zero"); assert!( - chunk_pos < root_pos, - "chunk writes must precede root pointer write: chunk_pos={chunk_pos} root_pos={root_pos}" + err.contains("deletes FAILED") && err.contains(&fail_key), + "error names the failed key: {err}" ); - assert!(out[0].contains("logical"), "summary line present: {out:?}"); } + /// Every reclamation delete passes `--key` + `--auto-yes` and NEVER `--all`. #[cfg(unix)] #[test] - fn push_config_entries_dry_run_reports_direct_vs_chunked() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + fn gc_delete_uses_key_and_auto_yes_never_all() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); - let direct_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); - let chunked_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); - let entries = vec![ - ("cfg_direct".to_owned(), direct_envelope), - ("cfg_chunked".to_owned(), chunked_envelope), + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + let argv_lines: Vec<&str> = log + .lines() + .filter(|line| line.starts_with("delete-argv ")) + .collect(); + assert!(!argv_lines.is_empty(), "a delete happened: {log}"); + for line in argv_lines { + assert!( + line.contains("--auto-yes"), + "delete passes --auto-yes: {line}" + ); + assert!(line.contains("--key="), "delete targets a --key: {line}"); + assert!( + !line.contains("--all"), + "delete must NEVER pass --all: {line}" + ); + } + } + + /// A non-canonical chunk-like key (short/uppercase SHA, leading-zero index) + /// is NOT a delete candidate — the destructive validator is canonical-only. + #[cfg(unix)] + #[test] + fn gc_never_deletes_non_canonical_keys() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + // Foreign-shaped keys under the reserved infix but not canonical. + let noncanonical = [ + format!("{TEST_CONFIG_ID}.__edgezero_chunks.abc123.0"), // short sha + format!("{TEST_CONFIG_ID}.__edgezero_chunks.{}.00", "a".repeat(64)), // leading-zero idx + format!("{TEST_CONFIG_ID}.__edgezero_chunks.{}.0", "A".repeat(64)), // uppercase ]; - let out = FastlyCliAdapter - .push_config_entries( + for key in &noncanonical { + listing.push((key.clone(), stamp_secs_ago(604_800), "X".to_owned())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // A key that is NOT canonical is not one we wrote, so it is not a + // reclamation candidate. It sits in our reserved namespace, though, so + // it is also not an ordinary root: we cannot say what it is. Since the + // GC classifier fails closed on any root it cannot classify, the run + // aborts and names it -- which satisfies this test's invariant (a + // non-canonical key is never deleted) the strict way. + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + err.contains("refusing to reclaim"), + "expected a fail-closed refusal, got: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &noncanonical { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a non-canonical key must never be deleted: `{key}`; log:\n{log}" + ); + } + assert!( + !log.contains("delete "), + "a fail-closed run deletes nothing at all; log:\n{log}" + ); + } + + // ---------- local chunk GC ---------- + + /// Config shrinks from chunked back under the 8 000-char limit: the + /// new value is a direct envelope, so GC prunes every prior chunk. + #[cfg(unix)] + #[test] + fn push_config_entries_local_prunes_prior_chunks_when_value_shrinks_to_direct() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( dir.path(), + Some("fastly.toml"), None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), chunked)], + &AdapterPushContext::new(), + false, + ) + .expect("first push"); + + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), - true, // dry_run + false, ) - .expect("dry-run must not error"); + .expect("second push"); - // No shellout happens; output must describe intent. - let combined = out.join("\n"); - assert!( - combined.contains("would push `cfg_direct` as direct entry"), - "must report direct: {combined}" + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "root holds the direct envelope" ); assert!( - combined.contains("would push `cfg_chunked` as chunked"), - "must report chunked: {combined}" + !contents + .iter() + .any(|(key, _)| key.contains(CHUNK_KEY_INFIX)), + "prior chunks must be pruned: {after}" ); } - /// Spec 12.7: pushing two blobs under different root keys - /// (e.g. `app_config` + `app_config_staging`) must leave both - /// keys readable from the local fastly.toml so the runtime - /// `EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY` override can - /// switch between them. Prior to the upsert fix the second - /// push wholesale-replaced the per-store contents table. - #[cfg(unix)] + /// The local prune must NOT delete a prior chunk key whose VALUE is a + /// runtime-readable root (a valid direct envelope). A small envelope padded + /// with trailing whitespace chunks so that chunk 0 is itself a whole, + /// verifying envelope; deleting it would drop live config. #[test] - fn push_config_entries_local_preserves_sibling_keys() { + fn push_config_entries_local_keeps_a_chunk_key_holding_a_valid_envelope() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let store = ResolvedStoreId::from_logical(TEST_CONFIG_ID); - let ctx = AdapterPushContext::new(); + + // A padded envelope: chunk 0 is the whole envelope plus trailing spaces + // (still a valid, verifying envelope on its own). + let envelope = BlobEnvelope::new(json!({ "k": "v" }), "2026-06-22T00:00:00Z".into()); + let mut padded = serde_json::to_string(&envelope).unwrap(); + padded.push_str(&" ".repeat(8_200)); + let entries = prepare_fastly_config_entries(TEST_CONFIG_ID, &padded).expect("expand"); + let chunk0_key = entries[0].0.clone(); + // Confirm the fixture: chunk 0's value verifies as an envelope. + let parsed: BlobEnvelope = serde_json::from_str(&entries[0].1).expect("chunk0 parses"); + parsed.verify().expect("chunk0 verifies"); FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, - &store, - &[("app_config".to_owned(), "{\"envelope\":\"A\"}".to_owned())], - &ctx, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), padded)], + &AdapterPushContext::new(), false, ) .expect("first push"); - FastlyCliAdapter + + // Re-push a direct value: the prior generation's chunks become orphans. + let direct = make_test_envelope(100); + let expected_deletions = entries.len().saturating_sub(2); // chunks minus the protected chunk0 + + // DRY-RUN first: its count must MATCH what the real prune deletes, i.e. + // it must exclude the protected root-like chunk0. + let dry = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, - &store, - &[( - "app_config_staging".to_owned(), - "{\"envelope\":\"B\"}".to_owned(), - )], - &ctx, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], + &AdapterPushContext::new(), + true, + ) + .expect("dry-run"); + assert!( + dry.join("\n") + .contains(&format!("would delete {expected_deletions} orphan chunks")), + "dry-run must count only the prunable orphans (excluding the protected root): {dry:?}" + ); + + let warnings = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), false, ) - .expect("second push (sibling key)"); + .expect("second push"); - let raw = fs::read_to_string(&fastly_toml).expect("read"); - let doc: toml_edit::DocumentMut = raw.parse().expect("parse"); + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); let contents = doc .get("local_server") .and_then(|ls| ls.get("config_stores")) .and_then(|cs| cs.get(TEST_CONFIG_ID)) .and_then(|st| st.get("contents")) .and_then(toml_edit::Item::as_table) - .expect("contents after sibling push"); - let app_config = contents - .get("app_config") - .and_then(toml_edit::Item::as_str) - .expect("default key must survive sibling push"); - assert_eq!( - app_config, "{\"envelope\":\"A\"}", - "default key value: {raw}" + .expect("contents"); + + assert!( + contents.contains_key(&chunk0_key), + "a chunk key holding a valid envelope is a runtime-readable root and must be kept: \ + {after}" + ); + assert!( + warnings + .iter() + .any(|warning| warning.contains("runtime-readable root")), + "the operator must be warned that the key was kept: {warnings:?}" + ); + } + + /// `preflight_config_write` rejects an infeasible push BEFORE any provider + /// I/O: a reserved key, an empty key, and a body whose DERIVED chunk keys + /// would exceed the store limit (caught by running expansion offline). + #[test] + fn preflight_config_write_rejects_infeasible_pushes_offline() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let small = make_test_envelope(100); + + let reserved = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + assert!( + FastlyCliAdapter + .preflight_config_write(&reserved, &small) + .is_err_and(|err| err.contains("reserved infix")), + "a reserved-namespace key must be rejected" + ); + + assert!( + FastlyCliAdapter + .preflight_config_write("", &small) + .is_err_and(|err| err.contains("empty")), + "an empty key must be rejected" + ); + + // A ~200-char root with a CHUNKED body: derived chunk keys (root + ~85) + // exceed the 255-char limit. Caught offline by expansion. + let long_root = "r".repeat(200); + let big = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!( + FastlyCliAdapter + .preflight_config_write(&long_root, &big) + .is_err(), + "an over-limit derived chunk key must be rejected before I/O" + ); + + // A normal push passes. + FastlyCliAdapter + .preflight_config_write("app_config", &small) + .expect("a normal push must pass preflight"); + } + + /// A logical key containing the reserved chunk infix is rejected + /// before any file I/O (it would collide with the chunk namespace). + #[cfg(unix)] + #[test] + fn push_config_entries_local_rejects_reserved_key() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + let bad_key = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + + let err = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(bad_key.clone(), "{}".to_owned())], + &AdapterPushContext::new(), + false, + ) + .expect_err("reserved key must be rejected"); + assert!(err.contains(&bad_key), "error names the key: {err}"); + assert!( + !fastly_toml.exists(), + "rejection must happen before any write" ); - let staging = contents - .get("app_config_staging") - .and_then(toml_edit::Item::as_str) - .expect("staging key must be present"); - assert_eq!(staging, "{\"envelope\":\"B\"}", "staging key value: {raw}"); } + /// A suspicious prior pointer (pointer-kind but invalid) makes GC + /// warn and delete nothing — pre-seeded chunk keys must survive. #[cfg(unix)] #[test] - fn push_config_entries_local_writes_literal_dotted_chunk_keys() { + fn push_config_entries_local_warns_on_suspicious_prior_pointer_and_keeps_chunks() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - fs::write(&fastly_toml, "name = \"demo\"\n").expect("write"); + // Seed the root with a pointer-kind-but-invalid value AND a real + // chunk-like key so "no deletes" is non-vacuous. + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":2}\"\n", + "\"app_config.__edgezero_chunks.deadbeef.0\" = \"seeded-chunk-payload\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; - FastlyCliAdapter + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), false, ) - .expect("local push must succeed"); + .expect("push must still succeed"); - let after = fs::read_to_string(&fastly_toml).expect("read back"); - // Chunk keys contain '.' and must appear as quoted string keys, - // not as TOML nested tables (which would look like [table.sub]). + let combined = out.join("\n"); assert!( - after.contains(".__edgezero_chunks."), - "chunk keys written to fastly.toml: {after}" + combined.contains("skipping chunk GC"), + "must warn about the suspicious prior pointer: {combined}" ); - // Parse with toml_edit and confirm chunk keys are string-keyed entries. - let doc: toml_edit::DocumentMut = after.parse().expect("must parse"); + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); let contents = doc .get("local_server") .and_then(|ls| ls.get("config_stores")) .and_then(|cs| cs.get(TEST_CONFIG_ID)) .and_then(|st| st.get("contents")) - .expect("contents table must exist"); - // At least one chunk key must be present as a string value (not a table). - let has_chunk_string = contents.as_table().is_some_and(|tbl| { - tbl.iter() - .any(|(key, val)| key.contains(".__edgezero_chunks.") && val.as_value().is_some()) - }); + .and_then(toml_edit::Item::as_table) + .expect("contents"); assert!( - has_chunk_string, - "chunk keys must be literal string-valued entries, not nested tables: {after}" + contents + .get("app_config.__edgezero_chunks.deadbeef.0") + .is_some(), + "pre-seeded chunk key must survive a suspicious-pointer skip: {after}" + ); + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "new value still written" ); } + /// Dry-run reports the orphan count and writes nothing. #[cfg(unix)] #[test] - fn push_config_entries_local_dry_run_reports_chunking_and_does_not_edit_fastly_toml() { + fn push_config_entries_local_dry_run_reports_orphan_count() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - let original = "name = \"demo\"\n"; - fs::write(&fastly_toml, original).expect("write"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_a)], + &AdapterPushContext::new(), + false, + ) + .expect("seed push"); + let before = fs::read_to_string(&fastly_toml).expect("read"); + + let envelope_b = { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ "alt": "y".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:02Z".to_owned())) + .expect("envelope B") + }; let out = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), envelope_b)], &AdapterPushContext::new(), true, // dry_run ) - .expect("local dry-run must not error"); - - // File must be untouched. - let after = fs::read_to_string(&fastly_toml).expect("read back"); - assert_eq!(after, original, "dry-run must not edit fastly.toml"); + .expect("dry-run"); - // Output must describe chunking intent. let combined = out.join("\n"); assert!( - combined.contains("would set") && combined.contains("chunked"), - "must report chunked intent: {combined}" + combined.contains("would delete") && combined.contains("orphan chunks"), + "dry-run must report orphan count: {combined}" + ); + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + before, + "dry-run must not edit fastly.toml" ); } - // ---------- chunked read integration tests ---------- - + /// PARITY: the dry-run's reported orphan count equals the number of chunk + /// keys the real (non-dry-run) push actually deletes, on ONE fixture. A + /// divergence would make the dry-run a misleading preview of the delete. #[cfg(unix)] #[test] - fn read_config_entry_resolves_direct_value_unchanged() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); + fn push_config_entries_local_dry_run_count_matches_real_deletions() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let envelope = BlobEnvelope::new(json!({"hello": "world"}), "2026-06-22T00:00:00Z".into()); - let json_str = serde_json::to_string(&envelope).unwrap(); - let item_json = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(&json_str).unwrap() - ); - let fake = fake_fastly_returning(&item_json, "", 0); - let _path = PathPrepend::new(fake.path()); + fn count_chunk_keys(toml_src: &str) -> usize { + let doc: toml_edit::DocumentMut = toml_src.parse().expect("parse"); + doc.get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .map_or(0, |table| { + table + .iter() + .filter(|(key, _)| key.contains(CHUNK_KEY_INFIX)) + .count() + }) + } + fn parse_would_delete_count(text: &str) -> Option { + let marker = "would delete "; + let idx = text.find(marker)?; + text.get(idx.saturating_add(marker.len())..)? + .split_whitespace() + .next()? + .parse::() + .ok() + } - let result = FastlyCliAdapter - .read_config_entry( + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + // Seed a multi-chunk generation, then measure how many chunk keys exist. + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); + FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", + &[(TEST_CONFIG_ID.to_owned(), chunked)], &AdapterPushContext::new(), + false, ) - .expect("read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, json_str, "direct envelope passes through unchanged"); - } + .expect("seed push"); + let seeded = fs::read_to_string(&fastly_toml).expect("read"); + let prior_chunk_count = count_chunk_keys(&seeded); + assert!(prior_chunk_count >= 2, "seed must have chunked: {seeded}"); - #[cfg(unix)] - #[test] - fn read_config_entry_reconstructs_chunked_envelope() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - // Build a key→response map for every physical entry. - let mut key_responses: Vec<(String, String)> = Vec::new(); - for (pk, pv) in &physical { - let resp = format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()); - key_responses.push((pk.clone(), resp)); - } - // The root key should return the pointer. - let ptr_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() - ); - key_responses.push((TEST_CONFIG_ID.to_owned(), ptr_resp)); - - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); - - let result = FastlyCliAdapter - .read_config_entry( + // Dry-run a shrink-to-direct re-push: capture the reported orphan count. + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), + true, // dry_run ) - .expect("chunked read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; + .expect("dry-run"); + let reported = parse_would_delete_count(&out.join("\n")) + .expect("dry-run must report a numeric orphan count"); assert_eq!( - value, envelope, - "reconstructed envelope must equal original" - ); - } - - #[cfg(unix)] - #[test] - fn read_config_entry_errors_on_missing_chunk() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - // Only provide the root pointer; omit chunk responses so chunk fetch returns not-found. - let ptr_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() - ); - let key_responses = vec![(TEST_CONFIG_ID.to_owned(), ptr_resp)]; - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); - - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, - &AdapterPushContext::new(), - ); - let Err(err) = result else { - panic!("missing chunk must error") - }; - assert!( - err.contains("missing chunk"), - "error must mention missing chunk: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn read_config_entry_errors_on_corrupt_chunk_hash() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - let mut key_responses: Vec<(String, String)> = Vec::new(); - // Corrupt first chunk's content. - let (first_chunk_key, first_chunk_val) = &physical[0]; - let corrupted: String = first_chunk_val.chars().map(|_| 'Z').collect(); - let corrupt_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(&corrupted).unwrap() + fs::read_to_string(&fastly_toml).expect("read"), + seeded, + "dry-run must not edit fastly.toml" ); - key_responses.push((first_chunk_key.clone(), corrupt_resp)); - // Remaining chunks as normal. - for (pk, pv) in physical - .iter() - .take(physical.len().saturating_sub(1)) - .skip(1) - { - key_responses.push(( - pk.clone(), - format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()), - )); - } - key_responses.push(( - TEST_CONFIG_ID.to_owned(), - format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() - ), - )); - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, - &AdapterPushContext::new(), - ); - let Err(err) = result else { - panic!("corrupt chunk must error") - }; - assert!( - err.contains("SHA mismatch") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" - ); - } + // Real re-push: count the chunk keys actually removed. + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + false, + ) + .expect("real push"); + let after = fs::read_to_string(&fastly_toml).expect("read"); + let actually_deleted = prior_chunk_count.saturating_sub(count_chunk_keys(&after)); - #[cfg(unix)] - #[test] - fn read_config_entry_errors_on_malformed_pointer() { - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - // Root value is JSON but neither a BlobEnvelope nor a valid pointer. - let bad_json = r#"{"some_field":"not a pointer or envelope"}"#; - let item_json = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(bad_json).unwrap() + assert_eq!( + reported, actually_deleted, + "dry-run count {reported} must equal real deletions {actually_deleted}" ); - let fake = fake_fastly_returning(&item_json, "", 0); - let _path = PathPrepend::new(fake.path()); - - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", - &AdapterPushContext::new(), + assert_eq!( + reported, prior_chunk_count, + "a shrink-to-direct re-push orphans every prior chunk" ); - let Err(err) = result else { - panic!("malformed pointer must error") - }; + } + + /// Real (non-dry-run) push over a MALFORMED prior pointer WARNS and deletes + /// nothing: its chunk list is untrustworthy, so no key is removed and the + /// root is simply overwritten with the new value. This is the real-push + /// counterpart to the dry-run "unknown" degradation on the same prior state. + #[cfg(unix)] + #[test] + fn push_config_entries_local_real_push_over_malformed_prior_warns_and_deletes_nothing() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + // A pointer-kind prior value missing its required fields — malformed, so + // `prior_chunk_keys` returns Err (warn, delete nothing). + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":2}\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); + + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("real push must not fail on a malformed prior"); + assert!( - err.contains("neither a valid BlobEnvelope") || err.contains("chunk pointer"), - "error must describe parse failure: {err}" + out.iter().any(|line| line.contains("skipping chunk GC")), + "must warn about the malformed prior pointer: {out:?}" ); - } - // ---------- local read integration tests ---------- + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "root is overwritten with the new direct envelope: {after}" + ); + } + /// Dry-run of an identical re-push reports zero orphans (new keys + /// equal prior keys — regression for expanding `new_keys`). + #[cfg(unix)] #[test] - fn read_config_entry_local_resolves_direct_value() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; + fn push_config_entries_local_dry_run_identical_repush_counts_zero() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let envelope = BlobEnvelope::new(json!({"x": 1_i32}), "2026-06-22T00:00:00Z".into()); - let json_str = serde_json::to_string(&envelope).unwrap(); - // Write directly as a single entry (not via push_config_entries_local so we - // control the exact TOML content). - write_fastly_local_config_store( - &fastly_toml, - TEST_CONFIG_ID, - &[("cfg".to_owned(), json_str.clone())], - ) - .expect("write"); + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("seed push"); - let result = FastlyCliAdapter - .read_config_entry_local( + let out = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", + &[(TEST_CONFIG_ID.to_owned(), envelope)], &AdapterPushContext::new(), + true, // dry_run, same bytes ) - .expect("local read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, json_str, "direct envelope passes through unchanged"); + .expect("dry-run"); + + assert!( + out.join("\n").contains("would delete 0 orphan chunks"), + "identical re-push must count 0 orphans: {out:?}" + ); } + /// Dry-run over a suspicious prior pointer reports an unknown count + /// and does not fail. + #[cfg(unix)] #[test] - fn read_config_entry_local_reconstructs_chunked_envelope() { + fn push_config_entries_local_dry_run_suspicious_prior_pointer_unknown() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":2}\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - // Write all physical entries (chunks + pointer) to the local store. - write_fastly_local_config_store(&fastly_toml, TEST_CONFIG_ID, &physical).expect("write"); - - let result = FastlyCliAdapter - .read_config_entry_local( + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, + &[(TEST_CONFIG_ID.to_owned(), direct)], &AdapterPushContext::new(), + true, // dry_run ) - .expect("local chunked read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!( - value, envelope, - "reconstructed envelope must equal original" + .expect("dry-run must not fail on suspicious pointer"); + + assert!( + out.join("\n").contains("unknown: suspicious prior pointer"), + "dry-run must degrade to unknown: {out:?}" ); } - /// Spec 12.3 + 9.3: a second oversized push must converge the - /// runtime on the NEW envelope — chunk keys are content-addressed - /// by the full-envelope SHA, so push B writes a new chunk-set and - /// installs a new root pointer. - /// - /// The local fastly.toml writer upserts per-key (so a sibling - /// `--key app_config_staging` push leaves `app_config` intact per - /// spec 12.7). Within the SAME root key, old chunks for envelope - /// A remain in the contents table after envelope B's push — they're - /// unreferenced (the root pointer at `app_config` now names B's - /// chunks), matching the remote Fastly behaviour where the - /// per-entry `update --upsert` shell-out has no atomic-delete - /// pairing. The runtime-correctness property holds either way: a - /// read after push B follows the active pointer and reconstructs - /// envelope B, not A. + /// A present-but-malformed `contents` (non-table) is prior state the + /// real writer would reject — the dry-run count must degrade to + /// `unknown: could not read prior state`, not silently report 0. #[cfg(unix)] #[test] - #[expect( - clippy::too_many_lines, - reason = "linear test scenario: push A, inspect, push B, inspect, read; splitting would obscure the chunk-set comparison" - )] - fn second_oversized_push_converges_runtime_on_new_envelope() { + fn push_config_entries_local_dry_run_non_table_contents_unknown() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n", + "contents = \"bad\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - // First push: envelope A. Records the chunk-key set so we can - // confirm they survive the second push (no garbage collection - // in v1 — spec 9.3 + Q6). - let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - FastlyCliAdapter + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let out = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &[(TEST_CONFIG_ID.to_owned(), envelope_a.clone())], + &[(TEST_CONFIG_ID.to_owned(), envelope)], &AdapterPushContext::new(), - false, + true, // dry_run ) - .expect("first push must succeed"); + .expect("dry-run must not fail on malformed contents"); - let after_a = fs::read_to_string(&fastly_toml).expect("read"); - let doc_a: toml_edit::DocumentMut = after_a.parse().expect("parse"); - let contents_a = doc_a - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .and_then(toml_edit::Item::as_table) - .expect("contents table after push A"); - let chunks_a: Vec = contents_a - .iter() - .map(|(key, _)| key.to_owned()) - .filter(|key| key.contains(".__edgezero_chunks.")) - .collect(); assert!( - !chunks_a.is_empty(), - "push A must have produced chunk entries: {after_a}" + out.join("\n") + .contains("unknown: could not read prior state"), + "non-table contents must degrade to unknown: {out:?}" ); + } - // Second push: a DIFFERENT oversized envelope B. The - // content-addressed chunk keys must shift to B's sha; old - // A-chunks may remain in the table (v1 doesn't GC). Build - // envelope B with a distinct payload key so its SHA differs - // from A's even at the same total length. - let envelope_b = { + /// A duplicate root key in one batch is rejected before any I/O. + /// Otherwise the earlier tuple's GC plan would reclaim the chunks the + /// LAST tuple just installed, leaving the final pointer dangling. + /// Regression: prior B, batch `[(root, A), (root, B)]` — the root must + /// still resolve afterwards. + #[cfg(unix)] + #[test] + fn push_config_entries_local_rejects_duplicate_root_keys() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let make = |tag: &str| { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; - let data = json!({ "alt": "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); - serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:01Z".to_owned())) - .expect("envelope B serialises") + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") }; - assert_ne!(envelope_a, envelope_b, "test fixtures must differ"); + let envelope_a = make("aaa"); + let envelope_b = make("bbb"); + + // Prior generation B is live. FastlyCliAdapter .push_config_entries_local( dir.path(), @@ -3341,50 +7174,35 @@ build = \"cargo build --release\" &AdapterPushContext::new(), false, ) - .expect("second push must succeed"); + .expect("seed push"); + let before = fs::read_to_string(&fastly_toml).expect("read"); - let after_b = fs::read_to_string(&fastly_toml).expect("read"); - let doc_b: toml_edit::DocumentMut = after_b.parse().expect("parse"); - let contents_b = doc_b - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .and_then(toml_edit::Item::as_table) - .expect("contents table after push B"); - let chunks_b: Vec = contents_b - .iter() - .map(|(key, _)| key.to_owned()) - .filter(|key| key.contains(".__edgezero_chunks.")) - .collect(); + // Duplicate-root batch must be rejected outright. + let err = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[ + (TEST_CONFIG_ID.to_owned(), envelope_a), + (TEST_CONFIG_ID.to_owned(), envelope_b.clone()), + ], + &AdapterPushContext::new(), + false, + ) + .expect_err("duplicate root keys must be rejected"); assert!( - !chunks_b.is_empty(), - "push B must have produced chunk entries: {after_b}" + err.contains("more than once"), + "error explains the duplicate: {err}" ); - - // Chunk keys are content-addressed by envelope SHA, so the B - // push installs a fresh chunk-set whose keys are all distinct - // from A's. Under the upsert semantic the A-chunks remain in - // the contents table (no GC in v1); B's chunks are simply added. - let new_b_chunks: Vec<&String> = chunks_b - .iter() - .filter(|key| !chunks_a.contains(*key)) - .collect(); - assert!( - !new_b_chunks.is_empty(), - "push B must have added at least one new content-addressed chunk: A-set={chunks_a:?} B-set={chunks_b:?}" + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + before, + "rejection must happen before any write" ); - // Old A-chunks remain in the table (orphan-but-present — - // matches the remote Fastly write-only-upsert semantic). - for chunk_key in &chunks_a { - assert!( - chunks_b.contains(chunk_key), - "old A-chunk `{chunk_key}` must remain in the local table after push B (v1 has no GC); B-set={chunks_b:?}" - ); - } - // Runtime-correctness property: a fresh read after push B - // reconstructs envelope B (NOT envelope A). + // The live root still resolves to B (nothing was reclaimed). let read = FastlyCliAdapter .read_config_entry_local( dir.path(), @@ -3394,17 +7212,134 @@ build = \"cargo build --release\" TEST_CONFIG_ID, &AdapterPushContext::new(), ) - .expect("local read after push B"); + .expect("root must still resolve"); let ReadConfigEntry::Present(value) = read else { - panic!("expected Present after push B"); + panic!("expected Present"); }; - assert_eq!( - value, envelope_b, - "read after second push must reconstruct envelope B, not A" - ); - assert_ne!( - value, envelope_a, - "old envelope A's chunks must be inert -- read must NOT return A" - ); + assert_eq!(value, envelope_b, "root still reconstructs envelope B"); + } + + /// GC of a chunked root must not touch a chunked SIBLING's chunks — + /// the prefix `app_config.__edgezero_chunks.` must not match + /// `app_config_staging.__edgezero_chunks.` (shared string prefix). + #[cfg(unix)] + #[test] + fn push_config_entries_local_gc_preserves_sibling_chunks() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let make = |tag: &str| { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") + }; + let push = |key: &str, body: String| { + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(key.to_owned(), body)], + &AdapterPushContext::new(), + false, + ) + .expect("push"); + }; + + // app_config gen X, then a chunked sibling, then app_config gen Z. + push("app_config", make("x1")); + push("app_config_staging", make("staging")); + let staging_chunks = chunk_keys_of("app_config_staging", &make("staging")); + push("app_config", make("z2")); // GCs app_config's gen-X chunks + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + for key in &staging_chunks { + assert!( + contents.get(key).is_some(), + "sibling chunk `{key}` must survive app_config GC: {after}" + ); + } + } + + // ---- chunk GC helpers ---- + + #[test] + fn reject_reserved_root_keys_accepts_clean_keys() { + let entries = vec![ + ("app_config".to_owned(), "{}".to_owned()), + ("app_config_staging".to_owned(), "{}".to_owned()), + ]; + reject_reserved_root_keys(&entries).expect("clean keys accepted"); + } + + #[test] + fn reject_reserved_root_keys_rejects_infix_key() { + let bad = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + let entries = vec![(bad.clone(), "{}".to_owned())]; + let err = reject_reserved_root_keys(&entries).expect_err("reserved infix must reject"); + assert!(err.contains(&bad), "error names the key: {err}"); + assert!(err.contains("reserved"), "error explains why: {err}"); + } + + #[test] + fn orphan_chunk_keys_subtracts_new_keys() { + let mut new_keys = HashSet::new(); + new_keys.insert("keep".to_owned()); + let plan = FastlyConfigGcPlan { + new_keys, + prior_keys: Ok(vec![ + "gone1".to_owned(), + "keep".to_owned(), + "gone2".to_owned(), + ]), + }; + let orphans = orphan_chunk_keys(&plan).expect("ok"); + assert_eq!(orphans, vec!["gone1".to_owned(), "gone2".to_owned()]); + } + + #[test] + fn orphan_chunk_keys_propagates_prior_err() { + let plan = FastlyConfigGcPlan { + new_keys: HashSet::new(), + prior_keys: Err("suspicious".to_owned()), + }; + orphan_chunk_keys(&plan).unwrap_err(); + } + + #[test] + fn expand_root_direct_value_has_single_entry() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let (expanded, new_keys, new_root_value) = expand_root(TEST_CONFIG_ID, &envelope).unwrap(); + assert_eq!(expanded.len(), 1); + assert_eq!(new_root_value, envelope); + assert!(new_keys.contains(TEST_CONFIG_ID)); + assert_eq!(new_keys.len(), 1); + } + + #[test] + fn expand_root_chunked_value_carries_pointer_as_root_value() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let (expanded, new_keys, new_root_value) = expand_root(TEST_CONFIG_ID, &envelope).unwrap(); + assert!(expanded.len() >= 2, "chunks + pointer"); + let (last_key, last_value) = expanded.last().unwrap(); + assert_eq!(last_key, TEST_CONFIG_ID); + assert_eq!(&new_root_value, last_value); + assert!(new_keys.contains(TEST_CONFIG_ID)); + assert_eq!(new_keys.len(), expanded.len()); } } diff --git a/crates/edgezero-adapter-fastly/src/config_store.rs b/crates/edgezero-adapter-fastly/src/config_store.rs index 6896369f..67306c38 100644 --- a/crates/edgezero-adapter-fastly/src/config_store.rs +++ b/crates/edgezero-adapter-fastly/src/config_store.rs @@ -32,9 +32,12 @@ impl FastlyConfigStore { /// Returns `Ok(Some(value))`, `Ok(None)` (missing), or `Err(message)`. fn get_sync(&self, key: &str) -> Result, String> { match &self.inner { - FastlyConfigStoreBackend::Fastly(inner) => inner - .try_get(key) - .map_err(|err| format!("config store lookup failed for `{key}`: {err}")), + FastlyConfigStoreBackend::Fastly(inner) => inner.try_get(key).map_err(|err| { + // The `key` here is a pointer-controlled chunk key; the resolver + // adds a safe position locator, so it is not echoed. The platform + // `err` is a fastly SDK type that does not embed the stored value. + format!("config store lookup failed: {err}") + }), #[cfg(test)] FastlyConfigStoreBackend::InMemory(data) => Ok(data.get(key).cloned()), } diff --git a/crates/edgezero-adapter/src/registry.rs b/crates/edgezero-adapter/src/registry.rs index 0f8850d0..286cc554 100644 --- a/crates/edgezero-adapter/src/registry.rs +++ b/crates/edgezero-adapter/src/registry.rs @@ -243,6 +243,51 @@ pub trait Adapter: Sync + Send { /// Returns an error string if the requested adapter action fails. fn execute(&self, action: AdapterAction, args: &[String]) -> Result<(), String>; + /// Reclaim chunk entries that no LIVE config pointer references. + /// + /// Deliberately NOT part of `config push`. On an eventually-consistent + /// store, a chunk may only be deleted once the pointer that referenced it + /// has stopped being served everywhere — and the platform may record no + /// pointer-supersession time (Fastly does not: `updated_at` is not bumped + /// by an upsert) and offer no compare-and-swap with which to record one + /// safely. Only the OPERATOR knows their deploy history, so `older_than` + /// carries that assertion. + /// + /// **`older_than` is a STORE-WIDE assertion, and it is stronger than "old + /// creations are no longer served".** `config gc` sweeps every root in the + /// selected store, so the caller asserts: **no root in this store changed + /// within the window, AND no writer is targeting the store** while `gc` runs. + /// A recently-changed sibling root — especially one that shrank to a direct + /// value, leaving no live chunk to age by — makes a wide window unsafe. Direct + /// trait callers MUST honour this stronger contract, not the weaker + /// paraphrase. A `dry_run` lists exactly what would be deleted, for review. + /// + /// # Errors + /// + /// Returns `Err` if the adapter has no `config gc` impl, if the platform + /// state cannot be read, or if it cannot be classified with confidence — + /// reclamation FAILS CLOSED: when in doubt, nothing is deleted. + #[inline] + #[expect( + clippy::too_many_arguments, + reason = "mirrors `push_config_entries`: manifest root, adapter manifest path, component selector, resolved store, push-time overlay, the operator's age assertion, and dry-run — each distinct; an aggregate struct is a worse ergonomic trade for implementers." + )] + fn gc_config_entries( + &self, + _manifest_root: &Path, + _adapter_manifest_path: Option<&str>, + _component_selector: Option<&str>, + _store: &ResolvedStoreId, + _push_ctx: &AdapterPushContext<'_>, + _older_than_secs: u64, + _dry_run: bool, + ) -> Result, String> { + Err(format!( + "adapter `{}` does not implement `config gc`", + self.name() + )) + } + /// Store kinds whose logical-id namespaces the adapter merges into /// a single backend at runtime — declaring the SAME logical id /// under two merged kinds causes silent write collisions because @@ -263,6 +308,26 @@ pub trait Adapter: Sync + Send { /// Name used to reference the adapter (case-insensitive). fn name(&self) -> &'static str; + /// Reject a config `(key, body)` that this adapter cannot store, BEFORE any + /// provider I/O. Called by `config push` ahead of the remote read, so an + /// infeasible push fails offline instead of after a `list`/`describe` + /// round-trip. + /// + /// `body` is the serialised blob-envelope JSON that would be written, so + /// body-dependent feasibility (e.g. Fastly's derived chunk-key length and + /// pointer-size limits, which depend on whether the value chunks) can be + /// checked here rather than during the later write expansion. + /// + /// Default: accept. The Fastly adapter overrides this to reject a reserved + /// or over-limit key and to run the full chunk expansion offline. + /// + /// # Errors + /// Returns a human-readable error string if the push is infeasible. + #[inline] + fn preflight_config_write(&self, _key: &str, _body: &str) -> Result<(), String> { + Ok(()) + } + /// Provision the platform resources backing each store id the /// user declared. Returns a list of human-readable /// status lines the CLI logs verbatim — one line per resource diff --git a/crates/edgezero-cli/src/args.rs b/crates/edgezero-cli/src/args.rs index 65f033d6..b12c458d 100644 --- a/crates/edgezero-cli/src/args.rs +++ b/crates/edgezero-cli/src/args.rs @@ -25,8 +25,14 @@ pub enum Command { Auth(AuthArgs), /// Build the project for a target edge. Build(BuildArgs), - /// Inspect or mutate the typed `.toml` app config. - #[command(subcommand, after_help = crate::args::STUB_POINTER_AFTER_HELP)] + /// Inspect or mutate the typed `.toml` app config, or reclaim + /// orphaned config-store entries (`gc`). + /// + /// NOTE: no group-level typed-CLI after-help here. `push`/`diff`/`validate` + /// each carry their own, because they need a typed `C`; `gc` does NOT -- + /// it inspects the store's physical entries and runs from this bundled + /// binary. A group-level notice would tell operators `gc` is unavailable. + #[command(subcommand)] Config(ConfigCmd), /// Run the bundled `app-demo` example locally (contributor-only). #[cfg(feature = "demo-example")] @@ -56,6 +62,23 @@ pub enum ConfigCmd { /// (Bundled `edgezero` stub — see after-help for the typed CLI.) #[command(after_help = STUB_POINTER_AFTER_HELP)] Diff(ConfigCmdStubArgs), + /// Reclaim chunk entries in the adapter's config store that no live + /// config pointer references. + /// + /// Deliberately NOT part of `config push`. On an eventually-consistent + /// store a chunk may only be deleted once the pointer that referenced it + /// has stopped being served everywhere — and the platform may record no + /// such timestamp (Fastly does not). Only YOU know your deploy history, + /// so `--older-than` is your assertion — see its help: it covers EVERY + /// root in the selected store, not only the config you have in mind. + /// + /// UNTYPED: unlike `push`/`diff`/`validate`, `gc` inspects the store's + /// physical entries rather than your `AppConfig`, so the bundled + /// `edgezero` binary can run it. + /// + /// SAFE BY DEFAULT: without `--yes` this only reports what it would + /// delete. Nothing is removed until you pass `--yes`. + Gc(ConfigGcArgs), /// Push the typed `.toml` as a single blob envelope to the /// adapter's config store. The blob carries every field verbatim /// (per spec 3.3 Model A — `#[secret]` fields store the key NAME, @@ -79,6 +102,68 @@ pub struct ConfigCmdStubArgs { pub trailing: Vec, } +/// Arguments for `config gc`. +/// +/// Unlike `push` / `diff`, `gc` needs no typed app-config: it reclaims +/// unreferenced chunk entries by inspecting the store, so it runs in-band in +/// the bundled `edgezero` binary. +#[derive(clap::Args, Debug)] +#[non_exhaustive] +pub struct ConfigGcArgs { + /// Adapter whose config store to reclaim (e.g. `fastly`). + #[arg(long)] + pub adapter: String, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Ignore `EDGEZERO__STORES__CONFIG____NAME` when resolving which + /// PHYSICAL store to sweep, so the logical id `` is used as the physical + /// store name instead. + /// + /// This is NOT the app-config overlay the other `config` subcommands mean by + /// `--no-env` — `gc` never loads your typed app config. On a DESTRUCTIVE + /// command it selects a DIFFERENT TARGET: if that variable is what maps your + /// logical id onto the real store, `--no-env` points `gc` at a different + /// store (or one that does not exist). Check the store id `gc` reports before + /// passing `--yes`. + #[arg(long)] + pub no_env: bool, + /// Only reclaim entries older than this. This is YOUR SAFETY ASSERTION, + /// and it is about the WHOLE PHYSICAL STORE, not just one config: `gc` + /// sweeps every root in the selected store, so you are asserting "NO root + /// in this store changed within this window, and no writer is targeting + /// it" — so nothing superseded more recently (which POPs may still serve) + /// is deleted. A sibling root you re-pushed minutes ago is enough to make + /// a wide window unsafe, especially if it changed to a value small enough + /// to store directly (that leaves no chunk for `gc` to date it by). + /// Accepts `s`/`m`/`h`/`d` suffixes (e.g. `7d`, `24h`, `90m`); a bare + /// number means seconds, and 0 is rejected for `--yes`. REQUIRED for + /// `--yes` (a destructive run must not guess it); a dry-run without it + /// previews every orphan and its age. + #[arg(long)] + pub older_than: Option, + /// Override the config-store id (defaults to the manifest's). + #[arg(long)] + pub store: Option, + /// Actually delete. Without this, `gc` only reports what it WOULD delete. + #[arg(long)] + pub yes: bool, +} + +impl Default for ConfigGcArgs { + #[inline] + fn default() -> Self { + Self { + adapter: String::new(), + manifest: PathBuf::from("edgezero.toml"), + no_env: false, + older_than: None, + store: None, + yes: false, + } + } +} + /// Arguments for the `auth` command. /// /// Intentionally has no `Default` impl: unlike the other `*Args` @@ -417,6 +502,44 @@ fn default_manifest_path() -> PathBuf { PathBuf::from("edgezero.toml") } +/// Parse a human duration (`7d`, `24h`, `90m`, `30s`, or bare seconds) into +/// seconds. +/// +/// # Errors +/// +/// Returns `Err` when the value is empty, non-numeric, or carries an unknown +/// suffix — a destructive command must never guess at its safety threshold. +#[must_use = "the parsed threshold gates a destructive command"] +#[inline] +pub fn parse_duration_secs(raw: &str) -> Result { + let trimmed = raw.trim(); + let unknown = || { + format!( + "could not parse `--older-than {raw}`; expected e.g. `7d`, `24h`, `90m`, `30s`, or a number of seconds" + ) + }; + let (digits, multiplier) = match trimmed.strip_suffix('s') { + Some(rest) => (rest, 1_u64), + None => match trimmed.strip_suffix('m') { + Some(rest) => (rest, 60_u64), + None => match trimmed.strip_suffix('h') { + Some(rest) => (rest, 3_600_u64), + None => match trimmed.strip_suffix('d') { + Some(rest) => (rest, 86_400_u64), + None => (trimmed, 1_u64), + }, + }, + }, + }; + if digits.is_empty() { + return Err(unknown()); + } + let value: u64 = digits.parse().map_err(|_err| unknown())?; + value + .checked_mul(multiplier) + .ok_or_else(|| format!("`--older-than {raw}` overflows")) +} + #[cfg(test)] mod tests { use super::*; @@ -844,4 +967,23 @@ mod tests { "`[TRAILING]` placeholder leaked into push help: {help}" ); } + #[test] + fn parse_duration_secs_accepts_suffixes_and_bare_seconds() { + assert_eq!(parse_duration_secs("30s").unwrap(), 30); + assert_eq!(parse_duration_secs("90m").unwrap(), 5_400); + assert_eq!(parse_duration_secs("24h").unwrap(), 86_400); + assert_eq!(parse_duration_secs("7d").unwrap(), 604_800); + assert_eq!(parse_duration_secs("3600").unwrap(), 3_600); + assert_eq!(parse_duration_secs(" 7d ").unwrap(), 604_800); + } + + /// A destructive command must never guess at its safety threshold. + #[test] + fn parse_duration_secs_rejects_garbage() { + parse_duration_secs("").unwrap_err(); + parse_duration_secs("soon").unwrap_err(); + parse_duration_secs("7w").unwrap_err(); + parse_duration_secs("-1d").unwrap_err(); + parse_duration_secs("d").unwrap_err(); + } } diff --git a/crates/edgezero-cli/src/config.rs b/crates/edgezero-cli/src/config.rs index ef4ac6a4..bd161026 100644 --- a/crates/edgezero-cli/src/config.rs +++ b/crates/edgezero-cli/src/config.rs @@ -18,7 +18,10 @@ //! env-overlay unless `--no-env` is passed, so the validation sees //! the values the runtime would. -use crate::args::{ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DiffFormat}; +use crate::args::{ + ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, DiffFormat, + parse_duration_secs, +}; use crate::diff::{collect_changes, render_json, render_structured}; use crate::ensure_adapter_defined; use edgezero_adapter::registry::{ @@ -36,6 +39,7 @@ use serde::de::DeserializeOwned; use similar::TextDiff; use std::collections::BTreeMap; use std::io::{Error as IoError, IsTerminal as _, Write, stdin}; +use std::iter; use std::path::{Path, PathBuf}; use toml::Value; use toml::value::Table; @@ -272,6 +276,126 @@ pub fn run_config_push(_args: &ConfigPushArgs) -> Result<(), String> { ) } +/// `config gc` — reclaim chunk entries no live config pointer references. +/// +/// Runs in-band in the bundled binary: unlike `push` / `diff` it needs no typed +/// app-config, only the store's own contents. +/// +/// SAFE BY DEFAULT: without `--yes` this is a dry-run. The adapter fails closed +/// — if it cannot classify the store's state with confidence it deletes nothing. +/// +/// # Errors +/// +/// Returns `Err` if the manifest/adapter cannot be resolved, `--older-than` is +/// unparseable, or the adapter refuses to reclaim (unreadable/unclassifiable +/// state). +#[inline] +pub fn run_config_gc(args: &ConfigGcArgs) -> Result<(), String> { + // A destructive run must not invent the operator's safety assertion. + let older_than_secs = match (args.yes, args.older_than.as_deref()) { + (true, None) => { + return Err( + "`config gc --yes` requires an explicit `--older-than `: a destructive run \ + must not guess it. It asserts that NO root in the selected store changed within \ + that window and that no writer is targeting the store, so nothing POPs may still \ + be serving is deleted -- `gc` sweeps the whole physical store, not just the \ + config you have in mind. Run without `--yes` first to preview every orphan and \ + its age." + .to_owned(), + ); + } + (yes, Some(raw)) => { + let secs = parse_duration_secs(raw)?; + // `--older-than 0 --yes` asserts nothing: it makes EVERY orphan + // eligible, including one superseded a second ago whose pointer + // POPs are still serving. Dry-run may preview at zero; a delete + // may not run at it. + if yes && secs == 0 { + return Err(format!( + "`config gc --yes --older-than {raw}` resolves to 0 seconds, which asserts \ + nothing: it would make every orphan eligible, including chunks a pointer POPs \ + are still serving. Pass a window at least as long as Fastly's propagation \ + time and no longer than the time since ANY root in this store last changed." + )); + } + secs + } + // Dry-run with no threshold: preview EVERY orphan (age >= 0) with ages, + // so the operator can choose `--older-than` from real data. + (false, None) => 0, + }; + + // Manifest-only resolution. Unlike `push`/`diff`, `gc` reclaims by + // inspecting the STORE, so it must NOT require the typed app-config file to + // exist — resolve the adapter + store straight from `edgezero.toml`. + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let manifest = manifest_loader.manifest(); + ensure_adapter_defined(&args.adapter, Some(&manifest_loader))?; + let adapter = adapter_registry::get_adapter(&args.adapter).ok_or_else(|| { + format!( + "adapter `{}` is declared in {} but not registered in this build (rebuild the CLI with its feature enabled)", + args.adapter, + args.manifest.display() + ) + })?; + let (_canonical, adapter_cfg) = manifest.adapter_entry(adapter.name()).ok_or_else(|| { + format!( + "adapter `{}` has no `[adapters.{}]` block", + args.adapter, args.adapter + ) + })?; + + let logical = resolve_config_store_id(args.store.as_deref(), manifest)?; + // `--no-env` means: do not overlay `EDGEZERO__*`. An empty config yields the + // logical id as the platform name (`store_name`'s fallback). + let env_config = if args.no_env { + EnvConfig::from_vars(iter::empty::<(String, String)>()) + } else { + EnvConfig::from_env() + }; + let platform = env_config.store_name("config", &logical); + let store = ResolvedStoreId::new(logical, platform); + + let manifest_root = args + .manifest + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let adapter_manifest_path = adapter_cfg.adapter.manifest.clone(); + let component_selector = adapter_cfg.adapter.component.clone(); + let mut push_ctx = adapter_registry::AdapterPushContext::new(); + if let Some(deploy_cmd) = adapter_cfg.commands.deploy.as_deref() { + push_ctx = push_ctx.with_manifest_adapter_deploy_cmd(deploy_cmd); + } + + // A run without --yes is a dry-run: report, delete nothing. + let dry_run = !args.yes; + let lines = adapter.gc_config_entries( + manifest_root, + adapter_manifest_path.as_deref(), + component_selector.as_deref(), + &store, + &push_ctx, + older_than_secs, + dry_run, + )?; + for line in lines { + log::info!("[edgezero] {line}"); + } + if dry_run { + match args.older_than.as_deref() { + Some(dur) => log::info!( + "[edgezero] dry-run (no --yes): nothing was deleted. Re-run with `--yes` to apply. `--older-than {dur}` asserts that NO root in this store changed within that window and that no writer is targeting it -- this sweeps the whole physical store, not just one config." + ), + None => log::info!( + "[edgezero] dry-run (no --yes): previewing ALL orphans and their ages. Choose an `--older-than ` that is (a) at least Fastly's propagation window, so POPs have stopped serving the superseded pointer, and (b) no longer than the time since ANY root in this store last changed -- gc sweeps every root here, so a sibling you re-pushed recently also constrains the window. Then re-run with `--yes`." + ), + } + } + Ok(()) +} + /// Typed flow — push the user's `C` struct. Runs strict pre-flight /// validation, then builds a `BlobEnvelope`, reads back the current /// remote for skip-on-equal + inline diff, prompts for consent, and @@ -323,6 +447,12 @@ where serde_json::from_str(&body).map_err(|err| format!("local envelope parse failed: {err}"))?; let local_sha = local_envelope.sha256.clone(); + // Reject an infeasible push (bad/over-limit key, over-limit derived chunk + // keys, oversized pointer) BEFORE any provider I/O — otherwise it would + // trigger a list/describe round-trip only to be rejected by the write path + // afterwards. `body` is the envelope JSON the write would store. + ctx.adapter.preflight_config_write(&key, &body)?; + // First read + diff. let remote = read_remote(ctx.adapter, args.local, &paths, &ctx.store, &key)?; let approved_remote_sha = @@ -486,11 +616,14 @@ where // Branch per variant, render, determine outcome. let outcome: DiffOutcome = match &remote { ReadConfigEntry::Present(body) => { - let remote_envelope: BlobEnvelope = serde_json::from_str(body) - .map_err(|err| format!("remote envelope parse failed: {err}"))?; + // Redacted: a serde parse error embeds the input (the stored blob, + // which may hold secrets); the verify error is redacted at source. + let remote_envelope: BlobEnvelope = serde_json::from_str(body).map_err(|_err| { + "remote value is not a valid envelope (details redacted)".to_owned() + })?; remote_envelope .verify() - .map_err(|err| format!("remote envelope verification failed: {err}"))?; + .map_err(|err| format!("remote envelope {err}"))?; if remote_envelope.sha256 == local_sha { diff_info(&format!("# no changes (sha256 matches: {local_sha})")); DiffOutcome::NoChanges @@ -590,7 +723,18 @@ fn dispatch_diff_format( /// Consent gate: 8.3 Spin Cloud four-branch UX when `remote` is /// `Unsupported`; 8.2 default flow otherwise. fn handle_consent(args: &ConfigPushArgs, remote: &ReadConfigEntry) -> Result<(), String> { + // The `Unsupported` branch below is Spin-Cloud-specific: a CLOUD read that + // cannot reach the backend, where a dry-run is genuinely impossible. A LOCAL + // read returns `Unsupported` only when the prior value could not be resolved + // (corrupt/incomplete chunk state) — a recoverable single-file overwrite, not + // an unreachable backend. That takes the NORMAL consent path, so a dry-run + // reaches the writer's report and a real write just needs `--yes`. Without + // this, a corrupt local prior would hit the Spin-Cloud dry-run rejection and + // the fail-soft ("overwrite, warn, prune nothing") would be unreachable. if let ReadConfigEntry::Unsupported(reason) = remote { + if args.local { + return require_consent(args, remote); + } if args.dry_run { return Err(format!( "config push --dry-run --adapter spin against Spin Cloud is unsupported \ @@ -685,11 +829,12 @@ fn recheck_before_write( ) -> Result { let remote_now = read_remote(adapter, args.local, paths, store, key)?; if let ReadConfigEntry::Present(body_now) = remote_now { - let remote_now_env: BlobEnvelope = serde_json::from_str(&body_now) - .map_err(|err| format!("post-consent remote envelope parse failed: {err}"))?; + let remote_now_env: BlobEnvelope = serde_json::from_str(&body_now).map_err(|_err| { + "post-consent remote value is not a valid envelope (details redacted)".to_owned() + })?; remote_now_env .verify() - .map_err(|err| format!("post-consent remote envelope verification failed: {err}"))?; + .map_err(|err| format!("post-consent remote envelope {err}"))?; if remote_now_env.sha256 == local_sha { push_info(&format!( "# concurrent push reached the same state (sha256 matches: {local_sha}); skipping write" @@ -747,11 +892,12 @@ fn render_first_read_diff( ) -> Result { match remote { ReadConfigEntry::Present(body_str) => { - let remote_envelope: BlobEnvelope = serde_json::from_str(body_str) - .map_err(|err| format!("remote envelope parse failed: {err}"))?; + let remote_envelope: BlobEnvelope = serde_json::from_str(body_str).map_err(|_err| { + "remote value is not a valid envelope (details redacted)".to_owned() + })?; remote_envelope .verify() - .map_err(|err| format!("remote envelope verification failed: {err}"))?; + .map_err(|err| format!("remote envelope {err}"))?; if remote_envelope.sha256 == local_sha { return Ok(FirstReadOutcome::NoChange); } @@ -1640,6 +1786,67 @@ mod tests { use std::sync::Mutex; use tempfile::TempDir; + // ---------- config gc argument gating ---------- + + /// A destructive `config gc --yes` MUST NOT invent the safety assertion: it + /// requires an explicit `--older-than`. The check runs before any manifest + /// or store access, so it is testable in isolation. + #[test] + fn config_gc_yes_requires_explicit_older_than() { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + yes: true, + older_than: None, + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args).expect_err("--yes without --older-than must be rejected"); + assert!( + err.contains("requires an explicit"), + "must explain the requirement: {err}" + ); + } + + /// `--older-than 0 --yes` parses, but asserts NOTHING -- + /// it makes every orphan eligible, including one superseded a second ago whose + /// pointer POPs still serve. A destructive run must reject it (a dry-run may + /// still preview at zero — see `config_gc_dry_run_allows_missing_older_than`). + #[test] + fn config_gc_yes_rejects_zero_older_than() { + for raw in ["0", "0s", "0d"] { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + yes: true, + older_than: Some(raw.to_owned()), + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args) + .expect_err("a zero window on a destructive run must be rejected"); + assert!( + err.contains("resolves to 0 seconds"), + "`--older-than {raw}` must be rejected as a no-op assertion, got: {err}" + ); + } + } + + /// A dry-run (no `--yes`) is allowed without `--older-than`; it fails later, + /// only because the fixture manifest/adapter is absent — NOT on the + /// threshold gate. (Proves the gate does not fire for a dry-run.) + #[test] + fn config_gc_dry_run_allows_missing_older_than() { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + manifest: PathBuf::from("does-not-exist.toml"), + yes: false, + older_than: None, + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args).expect_err("no manifest here"); + assert!( + !err.contains("requires an explicit"), + "the threshold gate must not fire for a dry-run: {err}" + ); + } + // ---------- shared fixtures ---------- const FIXTURE_APP_CONFIG: &str = r#" @@ -2739,6 +2946,226 @@ deep = true // ---------- typed push ---------- + /// A corrupt local prior value must not block `config push --local`: the + /// diff read degrades to `Unsupported`, and the shared handler must route a + /// LOCAL `Unsupported` through the normal consent path (reaching the writer) + /// rather than the Spin-Cloud dry-run rejection. Exercises the real + /// orchestration end-to-end, not the adapter methods in isolation. + #[test] + fn local_dry_run_over_corrupt_prior_reaches_the_writer() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + + // Every kind of unreadable prior STATE must let a local dry-run reach the + // writer's degrading count rather than hitting the Spin-Cloud rejection: + // a corrupt-but-parsing pointer, malformed TOML, and a non-string root. + let corrupt_pointer = format!( + "app_config = {}", + toml_string_literal(&format!( + "{{\"edgezero_kind\":\"fastly_config_chunks\",\"version\":1,\"chunks\":[{{\"key\":\"app_config.__edgezero_chunks.{sha}.0\",\"len\":10,\"sha256\":\"x\"}}],\"data_sha256\":\"\",\"envelope_len\":10,\"envelope_sha256\":\"{sha}\"}}", + sha = "a".repeat(64), + )), + ); + let cases = [ + ( + "corrupt pointer", + format!("[local_server.config_stores.app_config.contents]\n{corrupt_pointer}\n"), + ), + ( + "malformed TOML", + "[local_server.config_stores.app_config.contents]\nthis is not valid toml = = =" + .to_owned(), + ), + ( + "non-string root", + "[local_server.config_stores.app_config.contents]\napp_config = 42\n".to_owned(), + ), + ]; + + for (label, fastly_toml) in cases { + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + fs::write(dir.path().join("fastly.toml"), &fastly_toml).expect("write fastly.toml"); + + let mut args = push_args(&manifest, "fastly"); + args.local = true; + args.dry_run = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + run_config_push_typed::(&args).unwrap_or_else(|err| { + panic!("a local dry-run over {label} must reach the writer, not be rejected: {err}") + }); + } + } + + /// The dry-run degradation does NOT weaken the real push: a real + /// `config push --local` over malformed TOML still fails fatally at the + /// writer (which cannot parse the file to write into it). + #[test] + fn local_real_push_over_malformed_toml_still_fails() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + fs::write(dir.path().join("fastly.toml"), "this is not = = valid toml") + .expect("write fastly.toml"); + + let mut args = push_args(&manifest, "fastly"); + args.local = true; + args.yes = true; // real write, non-interactive + args.app_config = Some(dir.path().join("demo-app.toml")); + + run_config_push_typed::(&args) + .expect_err("a real push over malformed TOML must fail at the writer"); + } + + /// The body-aware preflight runs BEFORE any remote I/O: an infeasible cloud + /// push (here, a reserved `--key`) fails with the preflight error, not a + /// `fastly`-not-found / auth error from the remote read. If preflight ran + /// after `read_remote`, the error would be about the missing/failed shell-out. + #[test] + fn cloud_push_preflight_rejects_reserved_key_before_remote_io() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + + let mut args = push_args(&manifest, "fastly"); + // A reserved-namespace --key: infeasible, and preflight-detectable. + args.key = Some("app_config.__edgezero_chunks.deadbeef.0".to_owned()); + args.yes = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + let err = run_config_push_typed::(&args) + .expect_err("a reserved --key must be rejected"); + assert!( + err.contains("reserved infix"), + "must fail at preflight (before any remote read), not on a shell-out: {err}" + ); + } + + /// Stronger ordering proof: the generic push runs the FULL body-aware + /// preflight offline before ANY remote I/O. Unlike a reserved key (rejectable + /// by key SHAPE alone), a DERIVED-key overflow is only detectable by actually + /// expanding the envelope into chunks — a ~200-char root key is itself valid, + /// but once the >8 000-byte body chunks, the derived `.__edgezero_chunks. + /// <64-char-sha>.` key exceeds the 255-char store limit. If preflight + /// ran AFTER `read_remote`, the error would be a `fastly`-not-found shell-out + /// (no CLI on PATH in tests), never the key-limit message — so this pins that + /// the generic path performs no list/describe/update/delete before the offline + /// feasibility check has passed. + #[test] + fn cloud_push_preflight_rejects_derived_key_overflow_before_remote_io() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + // A body large enough to force chunking (envelope > 8 000 bytes). The + // huge `greeting` passes `#[validate(length(min = 1))]`. + let big_app_config = format!( + "api_token = \"demo_api_token\"\ngreeting = \"{}\"\nvault = \"default\"\n\n[service]\ntimeout_ms = 1500\n", + "x".repeat(9_000) + ); + fs::write(dir.path().join("demo-app.toml"), big_app_config).expect("write big app config"); + + let mut args = push_args(&manifest, "fastly"); + // A VALID root key (<= 255 chars, no reserved infix) whose DERIVED chunk + // key (+~85 chars) overflows the store's 255-char limit once chunked. + args.key = Some("r".repeat(200)); + args.yes = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + let err = run_config_push_typed::(&args) + .expect_err("a derived-key overflow must be rejected"); + assert!( + err.contains("character limit"), + "must fail at the offline body-aware preflight (before any remote read), not on a \ + shell-out: {err}" + ); + } + + /// Serialise a string as a TOML basic-string literal (test helper). + fn toml_string_literal(value: &str) -> String { + let mut out = String::from('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + other => out.push(other), + } + } + out.push('"'); + out + } + #[test] fn typed_push_writes_blob_envelope_to_local_config_file() { use edgezero_core::blob_envelope::{BlobEnvelope, ENVELOPE_VERSION_V1}; diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index 720d8243..eda4a01d 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -1345,6 +1345,7 @@ mod tests { for import in [ "AuthArgs", "BuildArgs", + "ConfigGcArgs", "ConfigPushArgs", "ConfigValidateArgs", "DeployArgs", @@ -1382,12 +1383,22 @@ mod tests { ); } + // `config gc` reclaims leaked chunk entries. It is UNTYPED on purpose: + // it inspects the store's physical entries, not the app-config struct, + // so it takes `ConfigGcArgs` and dispatches to the untyped runner. A + // generated CLI that omits it leaves operators no way to reclaim. + assert!( + main.contains("Gc(ConfigGcArgs)"), + "-cli ConfigCmd must include the untyped `Gc(ConfigGcArgs)` variant: {main}" + ); + // Typed dispatch — the whole reason a downstream CLI // exists. Raw push/validate would defeat the point. for call in [ "run_config_push_typed::", "run_config_validate_typed::", "edgezero_cli::run_auth", + "edgezero_cli::run_config_gc", "edgezero_cli::run_provision", ] { assert!( diff --git a/crates/edgezero-cli/src/lib.rs b/crates/edgezero-cli/src/lib.rs index 68eac9d8..512d42f9 100644 --- a/crates/edgezero-cli/src/lib.rs +++ b/crates/edgezero-cli/src/lib.rs @@ -48,8 +48,8 @@ pub mod args; pub use auth::run_auth; #[cfg(feature = "cli")] pub use config::{ - DiffExit, run_config_diff_typed, run_config_push, run_config_push_typed, run_config_validate, - run_config_validate_typed, + DiffExit, run_config_diff_typed, run_config_gc, run_config_push, run_config_push_typed, + run_config_validate, run_config_validate_typed, }; #[cfg(feature = "cli")] pub use provision::run_provision; diff --git a/crates/edgezero-cli/src/main.rs b/crates/edgezero-cli/src/main.rs index f94fe817..81d263af 100644 --- a/crates/edgezero-cli/src/main.rs +++ b/crates/edgezero-cli/src/main.rs @@ -25,6 +25,9 @@ fn main() { }; process::exit(2); } + // `config gc` needs no typed app-config -- it reclaims unreferenced + // chunk entries by inspecting the store -- so it runs in-band here. + Command::Config(ConfigCmd::Gc(cmd_args)) => edgezero_cli::run_config_gc(&cmd_args), Command::Config(ConfigCmd::Validate(cmd_args)) => { edgezero_cli::run_config_validate(&cmd_args) } diff --git a/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs b/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs index 3586a187..99721b1d 100644 --- a/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs +++ b/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs @@ -13,7 +13,8 @@ use clap::{Parser, Subcommand}; use edgezero_cli::DiffExit; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, NewArgs, + AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, + DeployArgs, NewArgs, ProvisionArgs, ServeArgs, }; use {{proj_core_mod}}::config::{{NameUpperCamel}}Config; @@ -67,6 +68,12 @@ enum {{NameUpperCamel}}ConfigCmd { /// Validate `edgezero.toml` and `{{name}}.toml` against the /// typed `{{NameUpperCamel}}Config` contract. Validate(ConfigValidateArgs), + /// Reclaim orphaned chunk entries the config store leaked from prior + /// oversized pushes. Store-derived and untyped (no `{{NameUpperCamel}}Config`). + /// A dry-run by default; deletes only with `--yes` + an explicit + /// `--older-than` (YOUR assertion that nothing superseded within that + /// window is still being served, and no push is running). + Gc(ConfigGcArgs), } fn main() { @@ -93,6 +100,9 @@ fn main() { Cmd::Config({{NameUpperCamel}}ConfigCmd::Validate(args)) => { edgezero_cli::run_config_validate_typed::<{{NameUpperCamel}}Config>(&args) } + // `gc` inspects the STORE, not the typed config, so it is not + // parameterised over `{{NameUpperCamel}}Config`. + Cmd::Config({{NameUpperCamel}}ConfigCmd::Gc(args)) => edgezero_cli::run_config_gc(&args), Cmd::Deploy(args) => edgezero_cli::run_deploy(&args), Cmd::New(args) => edgezero_cli::run_new(&args), Cmd::Provision(args) => edgezero_cli::run_provision(&args), diff --git a/crates/edgezero-core/src/blob_envelope.rs b/crates/edgezero-core/src/blob_envelope.rs index 8ca8b955..f0c319d1 100644 --- a/crates/edgezero-core/src/blob_envelope.rs +++ b/crates/edgezero-core/src/blob_envelope.rs @@ -5,6 +5,8 @@ //! { "data": {...}, "sha256": "", "version": 1, "generated_at": "" } //! ``` +use core::fmt; + use crate::canonical_form::canonical_data_sha256; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -21,14 +23,34 @@ pub struct BlobEnvelope { pub version: u32, } -#[derive(Debug, Error)] +#[derive(Error)] pub enum BlobEnvelopeError { - #[error("sha mismatch: stored {stored}, computed {computed}")] + // The stored/computed hashes are NOT interpolated into the Display message. + // `stored` is a blob-controlled string (a malformed envelope can set it to + // anything, including a secret), and this error is formatted into diagnostics + // that reach HTTP responses and logs. Redacting at the source covers every + // caller — the extractor, the chunk resolver, the CLI push/diff paths, and + // the introspection endpoint — instead of each having to remember to. The + // fields stay on the struct for programmatic inspection. + #[error("stored SHA-256 does not match the computed hash (hashes redacted)")] ShaMismatch { stored: String, computed: String }, #[error("unknown envelope version {0}; expected {expected}", expected = ENVELOPE_VERSION_V1)] UnknownVersion(u32), } +// Debug is hand-written (NOT derived): a derived `Debug` would print the +// `stored`/`computed` hashes, so `{err:?}` / `?err` (anyhow) would leak what the +// Display message deliberately redacts. Debug therefore mirrors Display. +impl fmt::Debug for BlobEnvelopeError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ShaMismatch { .. } => f.write_str("ShaMismatch { hashes redacted }"), + Self::UnknownVersion(version) => write!(f, "UnknownVersion({version})"), + } + } +} + impl BlobEnvelope { /// Consume the envelope and return only the inner `data` value. #[must_use] @@ -89,6 +111,29 @@ mod tests { envelope.verify().unwrap(); } + #[test] + fn sha_mismatch_display_does_not_leak_the_stored_hash() { + // The stored SHA is blob-controlled and this Display is formatted into + // diagnostics that reach HTTP responses and logs by many callers. + // Redacting here is the single point that covers all of them. + const SENTINEL: &str = "SUPER_SECRET_STORED_HASH"; + let mut envelope = BlobEnvelope::new(json!({ "greeting": "hi" }), "2026-06-17".into()); + envelope.sha256 = SENTINEL.to_owned(); + let err = envelope + .verify() + .expect_err("a tampered sha must fail verify"); + assert!( + !err.to_string().contains(SENTINEL), + "the stored hash must never appear in the error Display: {err}" + ); + // Debug must redact too: `{err:?}` / `?err` (anyhow) would otherwise + // print the stored hash even though Display redacts it. + assert!( + !format!("{err:?}").contains(SENTINEL), + "the stored hash must never appear in the error Debug: {err:?}" + ); + } + #[test] fn rejects_unknown_version() { let mut envelope = BlobEnvelope::new(json!({ "x": 1_i32 }), "2026-06-17T00:00:00Z".into()); diff --git a/crates/edgezero-core/src/error.rs b/crates/edgezero-core/src/error.rs index c5a4d7f7..3928f9ab 100644 --- a/crates/edgezero-core/src/error.rs +++ b/crates/edgezero-core/src/error.rs @@ -70,9 +70,29 @@ impl EdgeError { #[must_use] #[inline] pub fn config_out_of_date_from_serde(serde_err: &SerdePathError) -> Self { + // The serde message embeds the offending stored VALUE (e.g. `invalid + // type: string "hunter2", expected u32`), and this message is serialised + // into the HTTP error body. The config blob may hold secrets, so the + // VALUE must not escape — report only the category. + // + // The `field_path` STRING segments are redacted (structure kept): a map + // key is indistinguishable from a struct field here and may be a secret, + // and the redaction invariant forbids a stored string on any path. The + // exact path is available from a local `config validate`. See + // `redact_serde_path`. + use serde_json::error::Category; + let category = match serde_err.inner().classify() { + Category::Data => "wrong type or invalid value", + Category::Syntax => "malformed JSON", + Category::Eof => "unexpected end of input", + Category::Io => "i/o error while reading", + }; Self::ConfigOutOfDate { - message: serde_err.inner().to_string(), - field_path: serde_err.path().to_string(), + message: format!( + "typed app-config is out of date ({category}; value redacted) — \ + run ` config push` for this deploy" + ), + field_path: redact_serde_path(serde_err.path()), } } @@ -257,6 +277,49 @@ impl IntoResponse for EdgeError { } } +/// Render a `serde_path_to_error` path with its STRING segments redacted while +/// preserving structure (dots and sequence indices). +/// +/// A struct field and a map KEY are the same `Segment::Map` in this API +/// (verified empirically), so they cannot be told apart. A map key is stored +/// config DATA and may be a secret, and this path is serialised into the HTTP +/// error body — which the redaction invariant forbids for any stored string. So +/// every string segment becomes ``; sequence indices (positions, not +/// values) are kept. The operator recovers the EXACT path by running +/// `config validate` locally, which deserializes the same blob without an HTTP +/// boundary. See the `field_path` note in spec 2026-06-16. +fn redact_serde_path(path: &serde_path_to_error::Path) -> String { + use serde_path_to_error::Segment; + let mut out = String::new(); + for segment in path { + match segment { + Segment::Seq { index } => { + out.push('['); + out.push_str(&index.to_string()); + out.push(']'); + } + Segment::Map { .. } | Segment::Enum { .. } => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(""); + } + Segment::Unknown => { + if !out.is_empty() { + out.push('.'); + } + out.push('?'); + } + } + } + // No segments => a root-level error; keep serde's "." sentinel (a marker, + // not data). + if out.is_empty() { + return path.to_string(); + } + out +} + fn json_or_text(payload: &T) -> Body { Body::json(payload).unwrap_or_else(|_| Body::text("internal error")) } @@ -327,14 +390,68 @@ mod tests { let result: Result = serde_path_to_error::deserialize(de); let serde_err = result.expect_err("expected deserialization error"); - let expected_path = serde_err.path().to_string(); let err = EdgeError::config_out_of_date_from_serde(&serde_err); assert_eq!(err.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(!err.message().is_empty()); match err { EdgeError::ConfigOutOfDate { field_path, .. } => { - assert_eq!(field_path, expected_path); + // String segments redacted, structure preserved. + assert_eq!(field_path, "."); + } + EdgeError::BadRequest { .. } + | EdgeError::Internal { .. } + | EdgeError::MethodNotAllowed { .. } + | EdgeError::NotFound { .. } + | EdgeError::NotImplemented { .. } + | EdgeError::ServiceUnavailable { .. } + | EdgeError::Validation { .. } => panic!("expected ConfigOutOfDate"), + } + } + + #[test] + fn config_out_of_date_from_serde_redacts_map_key_from_path_and_message() { + use serde::Deserialize; + use std::collections::BTreeMap; + + #[derive(Debug, Deserialize)] + struct Outer { + #[expect(dead_code, reason = "only used to drive deserialization")] + items: BTreeMap, + } + #[derive(Debug, Deserialize)] + struct Inner { + #[expect(dead_code, reason = "only used to drive deserialization")] + port: u32, + } + + // A MAP KEY holding a secret. The type error is under that key, so the + // serde path is `items.SECRET_MAP_KEY.port`. It must reach NEITHER the + // message NOR the field_path: a map key is indistinguishable from a struct + // field in the serde path, so every string segment is redacted (structure + // kept). + const SENTINEL: &str = "SUPER_SECRET_MAP_KEY"; + let json = format!(r#"{{"items": {{"{SENTINEL}": {{"port": "nope"}}}}}}"#); + let de = &mut serde_json::Deserializer::from_str(&json); + let result: Result = serde_path_to_error::deserialize(de); + let serde_err = result.expect_err("expected deserialization error"); + + let err = EdgeError::config_out_of_date_from_serde(&serde_err); + match err { + EdgeError::ConfigOutOfDate { + field_path, + message, + } => { + assert!( + !message.contains(SENTINEL), + "a stored map key must never reach the message: {message}" + ); + assert!( + !field_path.contains(SENTINEL), + "a stored map key must never reach the field_path: {field_path}" + ); + // Structure is preserved (items..port -> redacted, dotted). + assert_eq!(field_path, ".."); } EdgeError::BadRequest { .. } | EdgeError::Internal { .. } diff --git a/crates/edgezero-core/src/extractor.rs b/crates/edgezero-core/src/extractor.rs index 42dda715..1c9baa0f 100644 --- a/crates/edgezero-core/src/extractor.rs +++ b/crates/edgezero-core/src/extractor.rs @@ -855,10 +855,20 @@ where String::new(), ) })?; - let envelope: BlobEnvelope = serde_json::from_str(&raw) - .map_err(|err| EdgeError::internal(anyhow::anyhow!("envelope parse failed: {err}")))?; - envelope.verify().map_err(|err| { - EdgeError::internal(anyhow::anyhow!("envelope verification failed: {err}")) + // Neither the parse error nor the verify error is echoed into the + // client-facing message: a serde error embeds the offending input, and a + // `BlobEnvelope` integrity failure names the stored hashes — both are + // config-store values that may hold secrets, and this message reaches the + // HTTP body. Report a category only. + let envelope: BlobEnvelope = serde_json::from_str(&raw).map_err(|_err| { + EdgeError::internal(anyhow::anyhow!( + "typed app-config blob is not a valid envelope (details redacted)" + )) + })?; + envelope.verify().map_err(|_err| { + EdgeError::internal(anyhow::anyhow!( + "typed app-config blob failed its integrity check (details redacted)" + )) })?; let mut data = envelope.into_data(); // Secret walk per spec 3.3.3. @@ -1039,10 +1049,13 @@ async fn resolve_leaf( })? .to_owned(); let bound = ctx.secret_store(&store_id_str).ok_or_else(|| { + // `store_id_str` is the blob's store_ref VALUE — config data that + // may be sensitive — and this message reaches the HTTP body. Name + // the field, not the stored id. EdgeError::config_out_of_date( format!( - "blob declared store_ref `{store_id_str}` but \ - [stores.secrets] has no such id" + "secret field `{leaf_path}` names a store_ref that is not declared in \ + [stores.secrets] (id redacted)" ), leaf_path.clone(), ) @@ -1062,23 +1075,33 @@ async fn resolve_leaf( fn map_secret_error( err: SecretError, field_name: &str, - store_id: &str, - key_name: &str, + // The stored key name, the store id, and the provider's message/source are + // deliberately UNUSED in the messages below: they are blob- or + // provider-controlled strings that may reveal the secret or infrastructure, + // and these messages reach the HTTP body / logs. Every branch names only the + // offending FIELD (schema, safe). Kept in the signature so the redaction is + // visible at the one place these values could have been formatted. + _store_id: &str, + _key_name: &str, ) -> EdgeError { match err { - SecretError::NotFound { name } => EdgeError::config_out_of_date( - format!("secret `{name}` in store `{store_id}` not found"), + SecretError::NotFound { .. } => EdgeError::config_out_of_date( + format!( + "the secret referenced by `{field_name}` was not found in its store (identifier redacted)" + ), field_name.to_owned(), ), - SecretError::Validation(msg) => EdgeError::config_out_of_date( - format!("secret `{key_name}` in store `{store_id}` rejected: {msg}"), + SecretError::Validation(_msg) => EdgeError::config_out_of_date( + format!( + "the secret referenced by `{field_name}` was rejected by its store (details redacted)" + ), field_name.to_owned(), ), - SecretError::Unavailable => { - EdgeError::service_unavailable(format!("secret store `{store_id}` unreachable")) - } - SecretError::Internal(source) => EdgeError::internal(anyhow::anyhow!( - "secret `{key_name}` in store `{store_id}` produced unexpected store error: {source}" + SecretError::Unavailable => EdgeError::service_unavailable(format!( + "the secret store for `{field_name}` is unreachable" + )), + SecretError::Internal(_source) => EdgeError::internal(anyhow::anyhow!( + "secret resolution for `{field_name}` failed (details redacted)" )), } } @@ -2308,16 +2331,20 @@ mod tests { #[test] fn app_config_extractor_returns_internal_on_sha_mismatch() { + const SENTINEL: &str = "SUPER_SECRET_STORED_HASH"; struct TamperedStore; #[async_trait(?Send)] impl ConfigStore for TamperedStore { async fn get(&self, _key: &str) -> Result, ConfigStoreError> { - // Build a valid envelope then corrupt its sha. + // A valid envelope whose stored sha is a SENTINEL secret. The + // stored hash is attacker-influenced (it comes from the config + // store), and this error becomes the HTTP 500 body — so the + // message must NOT echo it. let mut env = BlobEnvelope::new( serde_json::json!({ "greeting": "hi", "timeout_ms": 100_u32 }), "2026-01-01T00:00:00Z".into(), ); - env.sha256 = "ff".repeat(32); + env.sha256 = SENTINEL.to_owned(); Ok(Some(serde_json::to_string(&env).unwrap())) } } @@ -2330,12 +2357,58 @@ mod tests { matches!(err, EdgeError::Internal { .. }), "SHA mismatch must surface as Internal: {err:?}" ); + // The client-facing message (the HTTP body) must not carry the stored + // hash, only a redacted category. assert!( - err.message().contains("envelope verification failed"), - "message names the problem: {err:?}" + !err.message().contains(SENTINEL), + "the stored hash must never reach the client-facing message: {err:?}" + ); + assert!( + err.message().contains("integrity check"), + "message must still name the category: {err:?}" ); } + #[test] + fn app_config_extractor_does_not_leak_data_value_in_deserialize_error() { + const SENTINEL: &str = "SUPER_SECRET_FIELD_VALUE"; + struct TypeErrorStore; + #[async_trait(?Send)] + impl ConfigStore for TypeErrorStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + // A VALID envelope (verify passes) whose data has the wrong type + // for `timeout_ms` — a string sentinel where u32 is expected. + // The deserialize error names that value; it must not reach the + // client-facing message. + let env = BlobEnvelope::new( + serde_json::json!({ "greeting": "hi", "timeout_ms": SENTINEL }), + "2026-01-01T00:00:00Z".into(), + ); + Ok(Some(serde_json::to_string(&env).unwrap())) + } + } + + let ctx = ctx_with_config_store(TypeErrorStore, "key"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("a type mismatch in the typed config must error"); + assert!( + matches!(err, EdgeError::ConfigOutOfDate { .. }), + "a typed deserialize failure must be ConfigOutOfDate: {err:?}" + ); + assert!( + !err.message().contains(SENTINEL), + "the stored field value must never reach the client-facing message: {err:?}" + ); + // The field path segments are redacted (a map key is indistinguishable + // from a struct field and may be a secret); structure is kept. + if let EdgeError::ConfigOutOfDate { field_path, .. } = &err { + assert!( + !field_path.contains("timeout_ms") && field_path.contains(""), + "the field path must be redacted: {field_path}" + ); + } + } + #[test] fn app_config_extractor_returns_internal_on_bad_envelope_json() { struct GarbageStore; @@ -2354,9 +2427,15 @@ mod tests { "Envelope parse failure must be Internal: {err:?}" ); assert!( - err.message().contains("envelope parse failed"), + err.message().contains("not a valid envelope"), "message names the problem: {err:?}" ); + // The offending input is not echoed (a serde error would embed it, and a + // stored value may hold secrets). + assert!( + !err.message().contains("not-json-at-all"), + "the offending stored value must not reach the client message: {err:?}" + ); } #[test] @@ -2383,11 +2462,11 @@ mod tests { matches!(err, EdgeError::ConfigOutOfDate { .. }), "deserialise failure must be ConfigOutOfDate: {err:?}" ); - // serde_path_to_error should give us the field path. + // The field path segment is redacted (indistinguishable from a map key). if let EdgeError::ConfigOutOfDate { field_path, .. } = &err { assert_eq!( - field_path, "timeout_ms", - "serde_path_to_error must supply the field name: {err:?}" + field_path, "", + "the field path must be redacted: {err:?}" ); } } @@ -2479,6 +2558,32 @@ mod tests { } } + #[test] + fn app_config_secret_walk_error_does_not_leak_the_stored_key_name() { + use crate::config_store::{ConfigStore, ConfigStoreError}; + struct BlobStore(String); + #[async_trait(?Send)] + impl ConfigStore for BlobStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + Ok(Some(self.0.clone())) + } + } + + // The blob's secret field holds the secret's KEY NAME — config data that + // may be sensitive. When resolution fails, that name (and any store id / + // provider message) must not reach the error, which becomes the HTTP body. + const SENTINEL: &str = "SUPER_SECRET_KEY_NAME"; + let data = serde_json::json!({ "greeting": "hi", "api_token": SENTINEL }); + let blob = make_envelope(data); + let ctx = ctx_with_config_and_secrets(BlobStore(blob), "key", NoopSecretStore, "vault"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("missing secret must error"); + assert!( + !err.message().contains(SENTINEL), + "the stored secret key name must never reach the error message: {err:?}" + ); + } + // Build a RequestContext whose default secret store maps `default/{key}` -> // `value`, for exercising `secret_walk` directly. fn ctx_with_default_secret_store(key: &str, value: &str) -> RequestContext { diff --git a/crates/edgezero-core/src/introspection.rs b/crates/edgezero-core/src/introspection.rs index 9404f063..d74ba664 100644 --- a/crates/edgezero-core/src/introspection.rs +++ b/crates/edgezero-core/src/introspection.rs @@ -99,11 +99,17 @@ pub async fn config(ctx: RequestContext) -> Result { .await .map_err(EdgeError::from)? .ok_or_else(|| EdgeError::not_found("no config blob in default store"))?; - let envelope: BlobEnvelope = serde_json::from_str(&raw) - .map_err(|err| EdgeError::internal(anyhow::anyhow!("envelope parse failed: {err}")))?; - envelope.verify().map_err(|err| { - EdgeError::internal(anyhow::anyhow!("envelope verification failed: {err}")) + // Neither error echoes the stored value: a serde parse error embeds the + // input, and the envelope integrity error is redacted at its source. `raw` + // is the stored config blob and may hold secrets; this reaches the HTTP body. + let envelope: BlobEnvelope = serde_json::from_str(&raw).map_err(|_err| { + EdgeError::internal(anyhow::anyhow!( + "config blob is not a valid envelope (details redacted)" + )) })?; + envelope + .verify() + .map_err(|err| EdgeError::internal(anyhow::anyhow!("config blob {err}")))?; let body = Body::json(&envelope.into_data()).map_err(EdgeError::internal)?; json_response(StatusCode::OK, body) } diff --git a/docs/guide/blob-app-config-migration.md b/docs/guide/blob-app-config-migration.md index dfbba996..255f3fd5 100644 --- a/docs/guide/blob-app-config-migration.md +++ b/docs/guide/blob-app-config-migration.md @@ -117,8 +117,12 @@ The push uses `fastly config-store-entry update --upsert --stdin` to write the envelope as the value of one Config Store entry. **Oversized envelopes** are handled automatically. Fastly's per-entry -limit is 8,000 characters. If your envelope fits, it's stored -directly. Otherwise the adapter: +limit is 8,000 characters. The adapter applies a conservative +**UTF-8 byte** check (bytes are always ≥ characters, so an envelope +that fits by bytes always fits by characters): if the envelope JSON is +at most 8,000 bytes it's stored directly. This byte threshold is the +v1 read/write contract — it is stable across upgrades, so a value +stored by an earlier release always resolves. Otherwise the adapter: 1. Splits the envelope JSON into UTF-8-safe chunks (target 7,000 bytes each). @@ -306,16 +310,39 @@ Listing the orphans before deletion: # Cloudflare wrangler kv key list --namespace-id= --remote | jq -r '.[].name' -# Fastly -fastly config-store-entry list --store-id= --json | jq -r '.[].key' +# Fastly (the JSON field is `item_key`, not `key`) +fastly config-store-entry list --store-id= --json | jq -r '.[].item_key' # Spin sqlite3 .spin/sqlite_key_value.db "SELECT key FROM spin_key_value WHERE store=''" ``` -A future `config gc --adapter ` will automate this; v1 is -manual on the rationale that orphan cleanup is best done with operator -oversight. +> **`config gc` does NOT do this per-leaf cleanup.** It reclaims a different kind +> of orphan — the **chunk** entries an _oversized_ blob leaves behind when it is +> re-pushed (each generation is content-addressed, so a change orphans the whole +> previous chunk set). It never touches your old per-leaf keys. Run the per-leaf +> deletes above for the migration cutover; run `config gc` (below) as an ongoing +> reclamation for chunked stores. + +### Reclaiming orphaned chunk entries (Fastly, oversized configs) + +If your app-config blob exceeds Fastly's 8 000-character entry limit it is stored +as content-addressed chunks, and every re-push orphans the previous generation. +`config gc` reclaims them safely — it derives the live set from the store and +deletes only unreferenced chunk entries you have asserted are old enough: + +```sh +# Preview every orphan and its age (dry-run by default): +-cli config gc --adapter fastly + +# Delete, asserting that NO root in this store changed in the last 7 days AND no +# writer is targeting the store (config gc sweeps every root, store-wide): +-cli config gc --adapter fastly --older-than 7d --yes +``` + +`--older-than` is REQUIRED for `--yes` and is YOUR safety assertion; do not run +`config gc` while a `config push` to the same store is in flight (Fastly has no +compare-and-swap). Cloudflare/Spin orphan cleanup remains manual. ### Fastly chunk-pointer hygiene @@ -325,11 +352,11 @@ under 8,000 characters, the old chunks remain in the store unreferenced ```sh fastly config-store-entry list --store-id= --json \ - | jq -r '.[].key | select(contains(".__edgezero_chunks."))' + | jq -r '.[].item_key | select(contains(".__edgezero_chunks."))' ``` -They're safe to delete via the same per-key delete command above. -A future `config gc` will sweep them automatically. +`config gc --adapter fastly` (shown above) reclaims these safely — it will only +delete chunk keys no live pointer references. ## Troubleshooting @@ -356,13 +383,22 @@ project's typed downstream CLI. The bundled binary intentionally cannot push (it has no typed `C` in scope). Run `-cli config push` instead — `edgezero new` generates this CLI for you. -### Local fastly.toml writer wipes old chunks on re-push - -The local writer wholesale-replaces the per-store contents block on -every push so stale entries don't accumulate during dev work. This is -intentional (and STRONGER than the remote behaviour, where orphan -chunks remain inert). The runtime correctness property holds either -way: a read after push B reconstructs envelope B, not A. +### Local fastly.toml writer prunes old chunks on re-push + +The local writer **upserts the keys it is pushing and prunes the chunks the +prior generation of those roots referenced** — except any prior chunk key whose +VALUE is itself a runtime-readable root (a valid envelope or pointer), which is +kept with a warning rather than deleted. It does not wholesale-replace the +per-store contents block, so keys it isn't pushing (including sibling roots and +their chunks) are preserved. Pruning is safe +here in a way it is not in the cloud: `fastly.toml` is a single file that +Viceroy reads at startup, so there is no propagation window and no POP +that could still be serving the prior pointer. + +This is STRONGER than the remote behaviour, where a `config push` +reclaims nothing and orphan chunks remain inert until you run +`config gc`. The runtime correctness property holds either way: a read +after push B reconstructs envelope B, not A. ## Reference diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index f25c7404..469f9e43 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -236,14 +236,14 @@ edgezero config push --adapter [--manifest ] [--app-config ] - `--runtime-config ` — adapter runtime configuration file. Currently only consumed by Spin, which reads `[key_value_store.