|
| 1 | +//! Replay Broca's conformance captures against the 0.7.10 provider. |
| 2 | +use super::*; |
| 3 | +use sha2::{Digest, Sha256}; |
| 4 | + |
| 5 | +const ROOT: &str = concat!( |
| 6 | + env!("CARGO_MANIFEST_DIR"), |
| 7 | + "/testdata/broca-compaction-0710/" |
| 8 | +); |
| 9 | +const MANIFEST_SHA256: &str = "c679754e05680772890502cced3e0f1902bad79cce8d2f9569457e991a84c3ed"; |
| 10 | + |
| 11 | +fn fixture(name: &str) -> Value { |
| 12 | + serde_json::from_slice(&std::fs::read(Path::new(ROOT).join(name)).unwrap()).unwrap() |
| 13 | +} |
| 14 | + |
| 15 | +#[test] |
| 16 | +fn checksums_cover_all_31_captured_files() { |
| 17 | + let manifest = std::fs::read(Path::new(ROOT).join("SHA256SUMS")).unwrap(); |
| 18 | + assert_eq!(format!("{:x}", Sha256::digest(&manifest)), MANIFEST_SHA256); |
| 19 | + let mut listed = BTreeSet::new(); |
| 20 | + for line in std::str::from_utf8(&manifest).unwrap().lines() { |
| 21 | + let (expected, name) = line.split_once(" ").unwrap(); |
| 22 | + assert!(listed.insert(name.to_owned()), "duplicate fixture: {name}"); |
| 23 | + let bytes = std::fs::read(Path::new(ROOT).join(name)).unwrap(); |
| 24 | + assert_eq!(format!("{:x}", Sha256::digest(&bytes)), expected, "{name}"); |
| 25 | + } |
| 26 | + let present: BTreeSet<_> = std::fs::read_dir(ROOT) |
| 27 | + .unwrap() |
| 28 | + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) |
| 29 | + .filter(|name| name.ends_with(".json")) |
| 30 | + .collect(); |
| 31 | + assert_eq!(listed, present); |
| 32 | + assert_eq!(listed.len(), 31); |
| 33 | +} |
| 34 | + |
| 35 | +fn invalid_params(outcome: HandlerOutcome) -> String { |
| 36 | + match outcome { |
| 37 | + HandlerOutcome::Error { code, message } |
| 38 | + | HandlerOutcome::ErrorWithDetail { code, message, .. } => { |
| 39 | + assert_eq!(code, "invalid_params"); |
| 40 | + message |
| 41 | + } |
| 42 | + other => panic!("expected invalid_params, got {other:?}"), |
| 43 | + } |
| 44 | +} |
| 45 | + |
| 46 | +fn no_truncate(value: &Value) { |
| 47 | + match value { |
| 48 | + Value::Object(fields) => { |
| 49 | + assert!(!fields.contains_key("truncate"), "{value}"); |
| 50 | + for child in fields.values() { |
| 51 | + no_truncate(child); |
| 52 | + } |
| 53 | + } |
| 54 | + Value::Array(values) => { |
| 55 | + for child in values { |
| 56 | + no_truncate(child); |
| 57 | + } |
| 58 | + } |
| 59 | + _ => {} |
| 60 | + } |
| 61 | +} |
| 62 | + |
| 63 | +async fn replay_case(case: &str, fetches: usize, notices: usize, models: usize) { |
| 64 | + let dir = tempfile::tempdir().unwrap(); |
| 65 | + let setup_name = format!("{case}-setup-0.json"); |
| 66 | + let setup = fixture(&setup_name); |
| 67 | + let params = &setup["params"]; |
| 68 | + let session = params["session"].as_str().unwrap(); |
| 69 | + let harness = params["harness"].as_str().unwrap(); |
| 70 | + let rig = Rig::without_setup( |
| 71 | + dir.path(), |
| 72 | + "broca", |
| 73 | + ("runner", session), |
| 74 | + (harness, session), |
| 75 | + |_| {}, |
| 76 | + ) |
| 77 | + .await; |
| 78 | + |
| 79 | + // The published Setup decoder ignores future fields, not their whole |
| 80 | + // request. The captured scripted preset is not one ck-mc implements. |
| 81 | + let decoded: compact::setup::SetupRequest = decode(params).unwrap(); |
| 82 | + assert_eq!(decoded.view_due_max_ms(), Some(30_000)); |
| 83 | + assert_eq!(decoded.ride_wait_max_ms, Some(120_000)); |
| 84 | + assert_eq!( |
| 85 | + params["view_features"], |
| 86 | + json!(["truncate", "tag_bytes_edits"]) |
| 87 | + ); |
| 88 | + let known = serde_json::to_value(decoded).unwrap(); |
| 89 | + assert!(known.get("view_features").is_none()); |
| 90 | + let error = invalid_params(rig.h.dispatch_value(7, setup.clone()).await); |
| 91 | + assert_eq!(error, "unknown provider preset"); |
| 92 | + assert!(load_head(rig.h.store.get().unwrap(), &rig.reference) |
| 93 | + .unwrap() |
| 94 | + .is_none()); |
| 95 | + println!("{setup_name}: PASS decode (view_due_max_ms=30000, view_features ignored); handler REJECT invalid_params: {error}"); |
| 96 | + |
| 97 | + // This is a separate compatibility probe, not a rewrite of the capture: |
| 98 | + // remove only the future member and leave view_due_max_ms unchanged. |
| 99 | + let mut v0710 = params.clone(); |
| 100 | + v0710.as_object_mut().unwrap().remove("view_features"); |
| 101 | + let decoded: compact::setup::SetupRequest = decode(&v0710).unwrap(); |
| 102 | + assert_eq!(serde_json::to_value(decoded).unwrap(), known); |
| 103 | + let error = invalid_params( |
| 104 | + rig.h |
| 105 | + .dispatch_value(7, json!({"method":"compaction.setup","params":v0710})) |
| 106 | + .await, |
| 107 | + ); |
| 108 | + assert_eq!(error, "unknown provider preset"); |
| 109 | + println!("{setup_name}: without view_features, decode PASS (view_due_max_ms=30000); handler REJECT scripted preset"); |
| 110 | + |
| 111 | + for index in 0..notices { |
| 112 | + let name = format!("{case}-notice-{index}.json"); |
| 113 | + let headers: BTreeMap<String, String> = serde_json::from_value(fixture(&name)).unwrap(); |
| 114 | + let decoded = notice::AdvanceNotice::from_headers(&headers).unwrap(); |
| 115 | + assert_eq!(decoded.to_headers(), headers, "{name}: header lost"); |
| 116 | + let message = cortexkit_bus_trait::Message { |
| 117 | + subject: notice::event_subject("fixture", "broca", notice::ADVANCED_EVENT), |
| 118 | + id: bus_id(&decoded.session_ref, "advanced", decoded.seq), |
| 119 | + digest: decoded.digest().parse().unwrap(), |
| 120 | + headers, |
| 121 | + }; |
| 122 | + // Setup was refused. The actual notice handler acknowledges an unknown |
| 123 | + // conversation without reading the runner or publishing a view. |
| 124 | + rig.h.advance_compaction("fixture", &message).await.unwrap(); |
| 125 | + assert!(load_head(rig.h.store.get().unwrap(), &decoded.session_ref) |
| 126 | + .unwrap() |
| 127 | + .is_none()); |
| 128 | + let mut corrupt = message.clone(); |
| 129 | + corrupt.id.push_str("-wrong"); |
| 130 | + assert!(rig.h.advance_compaction("fixture", &corrupt).await.is_err()); |
| 131 | + println!("{name}: PASS decode and handler (unknown conversation acknowledged)"); |
| 132 | + } |
| 133 | + |
| 134 | + for index in 0..fetches { |
| 135 | + let name = format!("{case}-fetch-{index}.json"); |
| 136 | + let capture = fixture(&name); |
| 137 | + let _: view::ViewGetRequest = decode(&capture["request"]["params"]).unwrap(); |
| 138 | + let answer = response(rig.h.dispatch_value(7, capture["request"].clone()).await); |
| 139 | + let _: view::ViewGetAnswer = decode(&answer).unwrap(); |
| 140 | + assert_eq!(answer["answer"], "refuse"); |
| 141 | + assert_eq!(answer["code"], "unknown_session"); |
| 142 | + no_truncate(&answer); |
| 143 | + |
| 144 | + // These captured answers come from a scripted provider, not ck-mc. |
| 145 | + // An unknown truncate field decodes but cannot supply the action that |
| 146 | + // the 0.7.10 view validator requires (bytes or drop). |
| 147 | + let scripted: view::ViewGetAnswer = decode(&capture["answer"]).unwrap(); |
| 148 | + if case == "part_edits_truncate_renders_with_own_tag" { |
| 149 | + let view::ViewGetAnswer::View { view } = scripted else { |
| 150 | + panic!("expected a scripted view") |
| 151 | + }; |
| 152 | + assert_eq!(view.check(), Err(view::ViewProblem::EditAction)); |
| 153 | + println!("{name}: request PASS; ck-mc answer PASS (unknown_session); scripted answer PASS decode, FAIL validation (EditAction)"); |
| 154 | + } else { |
| 155 | + if let view::ViewGetAnswer::View { view } = scripted { |
| 156 | + assert_eq!(view.check(), Ok(()), "{name}"); |
| 157 | + } |
| 158 | + println!("{name}: request PASS; ck-mc answer PASS (unknown_session); scripted answer PASS decode/validation"); |
| 159 | + } |
| 160 | + } |
| 161 | + |
| 162 | + for index in 0..models { |
| 163 | + let name = format!("{case}-model-{index}.json"); |
| 164 | + let rows = fixture(&name); |
| 165 | + assert!(!rows.as_array().unwrap().is_empty()); |
| 166 | + for row in rows.as_array().unwrap() { |
| 167 | + assert!(matches!(row["role"].as_str(), Some("user" | "assistant"))); |
| 168 | + assert!(row["content"].is_string()); |
| 169 | + } |
| 170 | + println!("{name}: N/A model input (Broca -> model, not a ck-mc request/answer)"); |
| 171 | + } |
| 172 | + assert!(rig.runner.reads.lock().unwrap().is_empty()); |
| 173 | +} |
| 174 | + |
| 175 | +macro_rules! case { |
| 176 | + ($name:ident, $fetches:expr, $notices:expr, $models:expr) => { |
| 177 | + #[tokio::test] |
| 178 | + async fn $name() { |
| 179 | + replay_case(stringify!($name), $fetches, $notices, $models).await; |
| 180 | + } |
| 181 | + }; |
| 182 | +} |
| 183 | + |
| 184 | +case!(view_due_first_model_uses_view, 1, 2, 1); |
| 185 | +case!(view_due_later_step_retries, 2, 3, 1); |
| 186 | +case!(view_due_refused_fetch_blocks_model, 1, 1, 0); |
| 187 | +case!(part_edits_truncate_renders_with_own_tag, 1, 4, 2); |
| 188 | +case!(part_edits_bytes_render_with_own_tag, 1, 4, 2); |
| 189 | + |
| 190 | +#[tokio::test] |
| 191 | +async fn supported_broca_setup_and_published_edits_never_contain_truncate() { |
| 192 | + let dir = tempfile::tempdir().unwrap(); |
| 193 | + let rig = Rig::open( |
| 194 | + dir.path(), |
| 195 | + "broca", |
| 196 | + "broca", |
| 197 | + json!({"serializer_profile":"owned-broca"}), |
| 198 | + ) |
| 199 | + .await; |
| 200 | + let setup: compact::setup::SetupAnswer = decode(&rig.state().initial).unwrap(); |
| 201 | + no_truncate(&serde_json::to_value(setup).unwrap()); |
| 202 | + // A supported ck-mc session is a separate publication witness: the captures |
| 203 | + // omit session.read pages and their scripted Setup is not a ck-mc preset. |
| 204 | + *rig.runner.entries.lock().unwrap() = (0..40) |
| 205 | + .map(|n| { |
| 206 | + ReadMessage::new( |
| 207 | + n, |
| 208 | + format!("m{n}"), |
| 209 | + json!({"role":if n % 2 == 0 {"user"} else {"assistant"},"content":[{"type":"text","text":format!("message {n} ")+&"durable words ".repeat(16)}]}), |
| 210 | + ) |
| 211 | + .with_tags(vec![tags::PartTag::tagged(format!("m{n}#0"), format!("§{}§", n + 1))]) |
| 212 | + }) |
| 213 | + .collect(); |
| 214 | + rig.notice(1, 100, None).await; |
| 215 | + rig.queue(&[2]).await; |
| 216 | + let published = rig |
| 217 | + .ride("ttl_expired") |
| 218 | + .await |
| 219 | + .expect("a reduction publishes"); |
| 220 | + let answer = rig |
| 221 | + .op( |
| 222 | + "view.get", |
| 223 | + json!({"session_ref":rig.reference,"view_version":published.view_version}), |
| 224 | + ) |
| 225 | + .await; |
| 226 | + assert_eq!(answer["answer"], "view"); |
| 227 | + let _: view::ViewGetAnswer = decode(&answer).unwrap(); |
| 228 | + no_truncate(&answer); |
| 229 | + let edits: Vec<_> = answer["view"]["ranges"] |
| 230 | + .as_array() |
| 231 | + .unwrap() |
| 232 | + .iter() |
| 233 | + .filter_map(|range| range["edits"].as_array()) |
| 234 | + .flatten() |
| 235 | + .collect(); |
| 236 | + assert!( |
| 237 | + !edits.is_empty(), |
| 238 | + "exercise a real edit, not an unchanged view" |
| 239 | + ); |
| 240 | + assert!(edits |
| 241 | + .iter() |
| 242 | + .all(|edit| edit.get("drop").is_some() || edit.get("bytes").is_some())); |
| 243 | +} |
0 commit comments