From 1c85ad6745dd71c599b0af3fa2a2ba9177e5ebb9 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:37:56 +0200 Subject: [PATCH 01/40] fix(features): peak-bounds apex relocation, bounded/de-duplicated feature values - peak_bounds: relocate the apex to the profile maximum when the supplied apex sits at zero height, so the boundary walks no longer collapse to a zero-width window around the wrong scan. - ms1_ms2_apex_rt_delta: bound to [0,1) via d/(d+width); a degenerate (~0) reference width could previously drive it to ~1e9. - by_ratio_agreement: distinct bounded log-odds metric (tanh), no longer a duplicate of by_ratio_consistency and no longer saturating for single-series. - cosine_robust_trim3: hold the last non-degenerate value instead of collapsing to 0.0 when trimming removes all present fragments. - add n_peak_scans / peak_window_degenerate and both_series_present indicators. Co-Authored-By: Claude Fable 5 --- .../crates/mumdia/src/stages/features.rs | 25 ++++++++++++---- .../mumdia/src/stages/features/ion_series.rs | 16 +++++++++- .../crates/mumdia/src/stages/features/ms1.rs | 4 ++- .../mumdia/src/stages/features/peak_scans.rs | 30 +++++++++++++++++++ .../mumdia/src/stages/features/similarity.rs | 8 +++++ 5 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs diff --git a/rust/mumdia/crates/mumdia/src/stages/features.rs b/rust/mumdia/crates/mumdia/src/stages/features.rs index 956e6cc..cd7096c 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features.rs @@ -38,6 +38,7 @@ mod ms1; mod nonzero; mod novel; mod order_consistency; +mod peak_scans; mod rt; mod similarity; @@ -58,6 +59,7 @@ const FAMILIES: &[(&[&str], FamilyFn)] = &[ (novel::NAMES, novel::values), (nonzero::NAMES, nonzero::values), (order_consistency::NAMES, order_consistency::values), + (peak_scans::NAMES, peak_scans::values), ]; /// Names already used by the Minimal/Rich sets, which the extended battery must @@ -853,11 +855,24 @@ pub(crate) fn peak_bounds(prof: &[f64], ai: usize, frac: f64, grace: usize) -> ( if n < 3 { return (0, n.saturating_sub(1)); } - let peak = if prof[ai] > 0.0 { - prof[ai] - } else { - prof.iter().cloned().fold(0.0, f64::max) - }; + // If the supplied apex sits at zero profile height, relocate it to the global + // maximum. Using the max only for the threshold while walking from the zero + // `ai` collapses both walks to a zero-width window around the wrong scan. + let mut ai = ai; + if prof[ai] <= 0.0 { + ai = prof + .iter() + .enumerate() + .fold((0usize, f64::NEG_INFINITY), |(bi, bv), (i, &v)| { + if v > bv { + (i, v) + } else { + (bi, bv) + } + }) + .0; + } + let peak = prof[ai]; if peak <= 0.0 { return (0, n - 1); } diff --git a/rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs b/rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs index 126d41a..2037171 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs @@ -50,6 +50,7 @@ pub const NAMES: &[&str] = &[ "mean_matched_ordinal_norm", "by_ion_contiguous_intensity", "by_ion_contiguous_lib_frac", + "both_series_present", ]; const EPS: f64 = 1e-9; @@ -194,9 +195,21 @@ pub fn values(e: &Evidence) -> Vec { }; let by_ratio_agreement = { + // Bounded observed-vs-predicted b/y LOG-ODDS discrepancy, squashed to + // [0,1) by tanh. This is the log-odds analog (tail-sensitive), distinct + // from the linear-fraction `by_ratio_consistency` below; the earlier + // raw log-ratio version saturated to ~35 for single-series peptides. let lo = ((bo + EPS) / (yo + EPS)).ln(); let lp = ((bp + EPS) / (yp + EPS)).ln(); - fin((lo - lp).abs()) + fin((0.5 * (lo - lp)).tanh().abs()) + }; + + // Both fragment series observed: encodes single-series as a category rather + // than letting the ratio features carry it as a saturated magnitude. + let both_series_present = if n_matched_b > 0.0 && n_matched_y > 0.0 { + 1.0 + } else { + 0.0 }; let by_ratio_consistency = { @@ -559,6 +572,7 @@ pub fn values(e: &Evidence) -> Vec { mean_matched_ordinal_norm, by_ion_contiguous_intensity, by_ion_contiguous_lib_frac, + both_series_present, ]; debug_assert_eq!(out.len(), NAMES.len()); out.into_iter().map(fin).collect() diff --git a/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs b/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs index 18d4029..6d4cf26 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs @@ -283,7 +283,9 @@ fn xic_features(e: &Evidence) -> (f64, f64, f64, f64, f64, f64, f64, f64, f64, f }; let base_width = fwhm(axis, r); let d = (axis[am_mono] - axis[am_r]).abs(); - fin(d / (base_width + EPS)) + // Bounded to [0,1): d/(d+width) instead of d/width, so a degenerate + // (~0) reference width can't send the feature to ~1e9. + fin(d / (d + base_width + EPS)) } else { 0.0 } diff --git a/rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs b/rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs new file mode 100644 index 0000000..96629bd --- /dev/null +++ b/rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs @@ -0,0 +1,30 @@ +//! Extended feature family: peak-scan count / window-degeneracy indicator. +//! +//! Label-blind and emitted for EVERY PSM (never early-returns to zeros), so the +//! rescorer can distinguish an *undefined* zero from a *measured* zero. When the +//! extraction window is mis-centered (e.g. an RT-calibration error puts the true +//! apex at the window edge), the peak collapses to 1-2 non-empty scans and the +//! window-based families (order_consistency, peak_completeness, self-cosine) +//! degenerate to 0.0. Those zeros are indistinguishable from a genuine decoy-like +//! zero unless the model also sees how many peak scans actually existed. This +//! family exposes exactly that. +//! +//! Contract: `NAMES` and `values(&Evidence)` return the same number of items in +//! the same order; every value is finite; the length is stable. +use super::Evidence; + +pub const NAMES: &[&str] = &["n_peak_scans", "peak_window_degenerate"]; + +/// Scan count below which the window-based families early-return all-zeros +/// (mirrors order_consistency::MIN_SCANS). +const MIN_SCANS: usize = 3; + +pub fn values(e: &Evidence) -> Vec { + // Number of scan positions where any fragment carries observed intensity. + let np = e.traces.iter().map(|t| t.len()).min().unwrap_or(0); + let n_scans = (0..np) + .filter(|&j| e.traces.iter().any(|t| t[j] > 0.0)) + .count(); + let degenerate = if n_scans < MIN_SCANS { 1.0 } else { 0.0 }; + vec![n_scans as f64, degenerate] +} diff --git a/rust/mumdia/crates/mumdia/src/stages/features/similarity.rs b/rust/mumdia/crates/mumdia/src/stages/features/similarity.rs index d97b1a2..bfe7297 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/similarity.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/similarity.rs @@ -714,6 +714,14 @@ pub fn values(e: &Evidence) -> Vec { keep.remove(worst); let ot: Vec = keep.iter().map(|&i| o[i]).collect(); let lt: Vec = keep.iter().map(|&i| l[i]).collect(); + // Do not let trimming collapse to an all-absent observed vector: the + // trim ranks by residual and can remove every present fragment first, + // leaving only predicted-but-absent ones, for which cosine is a + // spurious 0.0 that reads as decoy-like for few-fragment peptides. + // Stop and hold the last non-degenerate value instead. + if ot.iter().map(|x| x.abs()).sum::() <= EPS { + break; + } let c = fin(cosine(&ot, <)); for s in step..3 { out[s] = c; From 020e4c007a262190bc417b97839cf823c30d2ce0 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:38:08 +0200 Subject: [PATCH 02/40] fix(scoring/fdr): conservative tie-safe q, OOF-safe grouped-q, calibration + index guards - fdr: (n_decoys+1)/n_targets numerator and collapse tied-score blocks to one q (was optimistic and dependent on arbitrary within-tie order). - rescore grouped_q: assign the group q only to the winning row per key; losing sibling targets no longer inherit the winner's low peptide/PG q. - rescore: precursor-level competition (CompeteGroupBy::PeptidoformCharge), experiment-wide multi-file rescore keyed on a unique row index, entrapment score readback keyed on row_id (candidate_id repeats across runs). - rt_im_train: use target-only RT-calibration anchors (decoys were admitted) and require enough anchors before trusting the residual window (avoids the ~1s overfit window with a handful of seeds). - index: fail if precursors are not ascending by precursor_mz (the fragment index's partition_point search assumes it) and warn on a decoy-free library. - report: label the count as precursor (peptidoform+charge), log a distinct stripped-sequence count. Co-Authored-By: Claude Fable 5 --- rust/mumdia/crates/mumdia/src/fdr.rs | 53 +++++++++++----- rust/mumdia/crates/mumdia/src/index.rs | 29 +++++++++ .../crates/mumdia/src/stages/compete.rs | 27 ++++++++ .../mumdia/crates/mumdia/src/stages/report.rs | 12 +++- .../crates/mumdia/src/stages/rescore.rs | 63 ++++++++++++++++--- .../crates/mumdia/src/stages/rt_im_train.rs | 20 +++++- 6 files changed, 176 insertions(+), 28 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/fdr.rs b/rust/mumdia/crates/mumdia/src/fdr.rs index a75c3dd..8d8185a 100644 --- a/rust/mumdia/crates/mumdia/src/fdr.rs +++ b/rust/mumdia/crates/mumdia/src/fdr.rs @@ -1,6 +1,6 @@ //! Native target-decoy FDR / q-values (PLAN.md Section 4 Stage F, Section 8: -//! DIA-NN no-pi0 estimator `q = n_decoys / max(1,n_targets)`, monotonized). -//! Shared by search-seed and rescore. +//! no-pi0 estimator `q = (n_decoys + 1) / max(1, n_targets)`, monotonized, with +//! tied scores collapsed to a single block q). Shared by search-seed and rescore. /// Compute per-record q-values from (score, is_decoy). Higher score is better. /// Returns q aligned to the input order. @@ -18,13 +18,28 @@ pub fn target_decoy_q(scores: &[(f64, bool)]) -> Vec { }); let (mut td, mut tt) = (0usize, 0usize); let mut fdr_at = vec![1.0f64; n]; - for (rank, &i) in order.iter().enumerate() { - if scores[i].1 { - td += 1; - } else { - tt += 1; + // Walk in score order, processing tied-score blocks together so every PSM in + // a block gets the same FDR (its within-tie order is arbitrary and must not + // change the q). Numerator uses `n_decoys + 1` (the standard conservative + // target-decoy estimate); the bare `n_decoys / n_targets` is optimistic in + // the low-count regime. + let mut rank = 0usize; + while rank < n { + let s = scores[order[rank]].0; + let mut end = rank; + while end < n && scores[order[end]].0 == s { + if scores[order[end]].1 { + td += 1; + } else { + tt += 1; + } + end += 1; + } + let f = (td as f64 + 1.0) / (tt.max(1) as f64); + for r in rank..end { + fdr_at[r] = f; } - fdr_at[rank] = (td.max(0) as f64) / (tt.max(1) as f64); + rank = end; } // Monotonize from worst-scoring to best so q is non-increasing with score. let mut q = vec![1.0f64; n]; @@ -98,19 +113,29 @@ mod tests { use super::*; #[test] - fn perfect_separation_gives_zero_q_for_targets() { - // all targets score above all decoys + fn perfect_separation_q_is_conservative_plus_one() { + // all targets score above all decoys; with the (n_decoys+1)/n_targets + // estimator the best targets get q = 1/n_targets (not 0). let s = vec![(10.0, false), (9.0, false), (1.0, true), (0.5, true)]; let q = target_decoy_q(&s); - // both targets outrank all decoys -> q = 0 for both - assert!(q[0] < 1e-9 && q[1] < 1e-9); - assert_eq!(count_targets_at_q(&q, &[false, false, true, true], 0.01), 2); - // With interleave the top target has q=0 + // 2 targets, 0 decoys ranked above them -> q = (0+1)/2 = 0.5 for both + assert!((q[0] - 0.5).abs() < 1e-9 && (q[1] - 0.5).abs() < 1e-9); + assert_eq!(count_targets_at_q(&q, &[false, false, true, true], 0.5), 2); + // 3 targets above 1 decoy -> best target q = (0+1)/3 = 1/3 let s2 = vec![(10.0, false), (9.0, false), (8.0, false), (1.0, true)]; let q2 = target_decoy_q(&s2); assert_eq!(count_targets_at_q(&q2, &[false, false, false, true], 0.34), 3); } + #[test] + fn tied_scores_share_one_q() { + // three PSMs at the same score must all receive the same q regardless of + // their arbitrary within-tie order (target/decoy interleave in a block). + let s = vec![(5.0, false), (5.0, true), (5.0, false)]; + let q = target_decoy_q(&s); + assert!((q[0] - q[1]).abs() < 1e-12 && (q[1] - q[2]).abs() < 1e-12); + } + #[test] fn entrapment_q_ranks_real_above_spike_in() { // Two real targets score highest, then an entrapment, then a real, then diff --git a/rust/mumdia/crates/mumdia/src/index.rs b/rust/mumdia/crates/mumdia/src/index.rs index a05b9c0..eacd758 100644 --- a/rust/mumdia/crates/mumdia/src/index.rs +++ b/rust/mumdia/crates/mumdia/src/index.rs @@ -18,6 +18,7 @@ use anyhow::Result; use mumdia_core::constants::{ppm_bounds, PROTON}; use mumdia_io::table::Table; use rayon::prelude::*; +use tracing::warn; #[derive(Clone, Debug)] pub struct Candidate { @@ -126,6 +127,34 @@ impl Library { prec_mz.push(pmz[c]); } + // Precondition for `candidate_range`: precursors ascending by m/z. The + // fragment-index `partition_point` search over `prec_mz` assumes this; + // an unsorted import (e.g. `import_diann_lib.py` output fed directly, + // skipping the sorting decoy builder) would silently return wrong + // candidate windows. Check explicitly and fail loudly. + for c in 1..ncand { + if prec_mz[c] < prec_mz[c - 1] { + anyhow::bail!( + "library precursors must be ascending by precursor_mz (row {c} m/z \ + {} < row {} m/z {}); sort/reindex the library (the decoy-builder \ + scripts do this)", + prec_mz[c], + c - 1, + prec_mz[c - 1] + ); + } + } + // A decoy-free library gives q = (0+1)/n_targets, i.e. silently-invalid + // near-zero FDR downstream. Warn loudly (bailing could break intentional + // decoy-free diagnostics, so this is a warning, not an error). + let n_decoy = cands.iter().filter(|c| c.is_decoy).count(); + if n_decoy == 0 { + warn!( + "library has 0 decoy candidates; target-decoy q-values will be invalid \ + (add decoys, e.g. via make_reverse_decoys.py)" + ); + } + // Build flat index entries. let mut entries: Vec<(f32, u32, f32)> = Vec::with_capacity(ft.nrows); for c in 0..ncand { diff --git a/rust/mumdia/crates/mumdia/src/stages/compete.rs b/rust/mumdia/crates/mumdia/src/stages/compete.rs index 56add5e..77ee9dc 100644 --- a/rust/mumdia/crates/mumdia/src/stages/compete.rs +++ b/rust/mumdia/crates/mumdia/src/stages/compete.rs @@ -39,6 +39,27 @@ pub fn run(p: CompeteParams) -> Result { let feat_names = &schema.feature_columns; let feat_cols: Vec> = feat_names.iter().map(|c| t.f64(c).unwrap()).collect(); + // `charge` is a minimal feature column (present in every set), stored as f64. + // Only the peptidoform-charge grouping needs it. + let charge = t.f64("charge").ok(); + if matches!(p.cfg.group_by, CompeteGroupBy::PeptidoformCharge) && charge.is_none() { + anyhow::bail!("compete group_by=peptidoform_charge requires a 'charge' feature column"); + } + // Dense peptidoform id by first appearance (deterministic) so the fixed-size + // tuple key can separate modforms without allocating a String per PSM. Built + // only for the peptidoform-charge grouping; empty otherwise. + let pform_id: Vec = if matches!(p.cfg.group_by, CompeteGroupBy::PeptidoformCharge) { + let mut ids = Vec::with_capacity(t.nrows); + let mut seen: HashMap<&str, u32> = HashMap::new(); + for i in 0..t.nrows { + let next = seen.len() as u32; + ids.push(*seen.entry(pform[i].as_str()).or_insert(next)); + } + ids + } else { + Vec::new() + }; + // Winner per competition group. The label is part of the key so a target is // NOT competed against its own decoy: the decoy population must survive for // the rescorer/FDR to have a valid null (otherwise decoys are depleted and @@ -61,6 +82,12 @@ pub fn run(p: CompeteParams) -> Result { let bucket = (apex_rt[i] / p.cfg.apex_rt_tolerance_s).round() as i64; (base[i], label_code, bucket) } + CompeteGroupBy::PeptidoformCharge => { + // pform_id separates modforms; charge in the bucket separates + // charges -> one group per peptidoform+charge (precursor-level). + let c = charge.as_ref().unwrap()[i].round() as i64; + (pform_id[i], label_code, c) + } }; winner .entry(key) diff --git a/rust/mumdia/crates/mumdia/src/stages/report.rs b/rust/mumdia/crates/mumdia/src/stages/report.rs index 50f2398..3217c47 100644 --- a/rust/mumdia/crates/mumdia/src/stages/report.rs +++ b/rust/mumdia/crates/mumdia/src/stages/report.rs @@ -82,13 +82,17 @@ pub fn run(p: ReportParams) -> Result<(u64, u64)> { let mut order: Vec = (0..n).collect(); order.sort_by(|&a, &b| pep_q[a].partial_cmp(&pep_q[b]).unwrap_or(std::cmp::Ordering::Equal)); let mut seen: HashSet<(String, i32)> = HashSet::new(); + let mut seen_strip: HashSet = HashSet::new(); let mut w = std::io::BufWriter::new(std::fs::File::create(p.out_peptides)?); - writeln!(w, "peptide\tstripped_sequence\tcharge\tprotein\tq_value\tscore\tquantity")?; + // The row unit here is the precursor (peptidoform + charge), NOT the stripped + // sequence; the header and the returned count reflect that. + writeln!(w, "precursor\tstripped_sequence\tcharge\tprotein\tq_value\tscore\tquantity")?; let mut npep = 0u64; for &i in &order { if label[i] != "target" || !(pep_q[i] <= p.q_threshold) { continue; } + seen_strip.insert(strip(&pform[i])); let key = (pform[i].clone(), charge[i]); if !seen.insert(key.clone()) { continue; @@ -123,6 +127,12 @@ pub fn run(p: ReportParams) -> Result<(u64, u64)> { } w2.flush()?; + tracing::info!( + precursors = npep, + stripped_sequences = seen_strip.len() as u64, + protein_groups = nprot, + "report: done (peptides.tsv rows are precursors, not stripped sequences)" + ); Ok((npep, nprot)) } diff --git a/rust/mumdia/crates/mumdia/src/stages/rescore.rs b/rust/mumdia/crates/mumdia/src/stages/rescore.rs index b372d01..1dd1fa2 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rescore.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rescore.rs @@ -53,10 +53,16 @@ pub fn run(p: RescoreParams) -> Result { ); let mut feats: Vec> = Vec::new(); let mut mz: Vec = Vec::new(); + // `source` = index of the competed input each PSM came from (0..N). For a + // single-run rescore this is all-zero; for an experiment-wide rescore over + // several files it lets quant map each scored PSM back to its run, and it is + // why the Mokapot PIN below keys on a unique row index rather than + // candidate_id (which is the library index and repeats across runs). + let mut source: Vec = Vec::new(); // Feature list is taken from the schema companion of the first input so the // classifier input matches the set the features stage produced. let feat_names = FeatureSchema::read(&p.competed[0])?.feature_columns; - for path in p.competed { + for (src, path) in p.competed.iter().enumerate() { let t = Table::read(path)?; let c = t.u32("candidate_id")?; let l = t.str("label")?; @@ -76,6 +82,7 @@ pub fn run(p: RescoreParams) -> Result { charge.push(z[i] as i32); prelim.push(pl[i]); mz.push(pm[i]); + source.push(src as u32); feats.push((0..feat_names.len()).map(|k| fcols[k][i]).collect()); } } @@ -258,6 +265,9 @@ pub fn run(p: RescoreParams) -> Result { Col::F64("pg_q_value".into(), pg_q), Col::F64("global_q_value".into(), global_q), Col::F64("prelim_score".into(), prelim), + // Run identity for experiment-wide rescore (index into --competed); + // all-zero for a single-run rescore. Lets quant map scores per file. + Col::U32("source".into(), source), ], )?; @@ -387,7 +397,25 @@ fn grouped_q( } }; let qmap: HashMap = ks.into_iter().zip(qv).collect(); - (0..n).map(|i| qmap[&keys[i]]).collect() + // Assign the group q ONLY to the winning (max-score) row of each group. A + // losing sibling (a lower-scoring charge/mod variant, which may itself be a + // false target) must not inherit the winner's low q; it gets 1.0. The + // report/counts dedup by key on the winner, so peptide/PG counts are + // unchanged, but per-PSM peptide_q/pg_q no longer propagate to losers. + let mut winner: HashMap = HashMap::new(); + for i in 0..n { + let e = winner + .entry(keys[i].clone()) + .or_insert((f64::NEG_INFINITY, i)); + if scores[i] > e.0 { + *e = (scores[i], i); + } + } + let mut out = vec![1.0f64; n]; + for (k, (_, i)) in winner { + out[i] = qmap[&k]; + } + out } /// Run the entrapment GBM sidecar: write a Parquet of features + meta columns, @@ -415,6 +443,10 @@ fn run_entrapment_gbm( let outp = format!("{}/entrapment_out.parquet", p.work_dir); let mut cols = vec![ + // Unique per-row id for score readback: candidate_id repeats across + // competed runs, so mapping scores back by candidate_id collides (later + // runs overwrite earlier). Map by this flat row index instead. + Col::U32("row_id".into(), (0..cid.len()).map(|i| i as u32).collect()), Col::U32("candidate_id".into(), cid.to_vec()), Col::U32("base_peptide_id".into(), base.to_vec()), Col::I32("is_entrapment".into(), is_entrapment.iter().map(|&b| b as i32).collect()), @@ -438,11 +470,13 @@ fn run_entrapment_gbm( } let t = Table::read(&outp)?; - let ocid = t.u32("candidate_id")?; + let orid = t.u32("row_id")?; let osc = t.f64("score")?; - let map: HashMap = ocid.into_iter().zip(osc).collect(); + let map: HashMap = orid.into_iter().zip(osc).collect(); let min = map.values().cloned().fold(f64::INFINITY, f64::min); - Ok(cid.iter().map(|c| *map.get(c).unwrap_or(&(min - 1.0))).collect()) + Ok((0..cid.len()) + .map(|i| *map.get(&(i as u32)).unwrap_or(&(min - 1.0))) + .collect()) } /// Run Mokapot over a PIN written from the competed set; return scores aligned @@ -472,9 +506,15 @@ fn run_mokapot( s.push_str("SpecId\tLabel\tScanNr\tExpMass\tCalcMass\t"); s.push_str(&feat_names.join("\t")); s.push_str("\tPeptide\tProteins\n"); + // Key the PIN on the unique row index i (SpecId=psm_i, ScanNr=i), NOT + // candidate_id: candidate_id is the library index and repeats across runs, so + // an experiment-wide (multi-file) PIN would collide on ScanNr and mokapot's + // per-spectrum competition would collapse the runs. The row index is unique + // across the whole concatenation. Single-run behaviour is unchanged (the + // mapping is bijective and mokapot does not use SpecId/ScanNr as features). for i in 0..cid.len() { let lab = if label[i] == "decoy" { -1 } else { 1 }; - write!(s, "cand_{}\t{}\t{}\t{:.5}\t{:.5}\t", cid[i], lab, cid[i], mz[i], mz[i]).ok(); + write!(s, "psm_{}\t{}\t{}\t{:.5}\t{:.5}\t", i, lab, i, mz[i], mz[i]).ok(); for fi in 0..feat_names.len() { write!(s, "{:.6}\t", feats[i][fi]).ok(); } @@ -493,10 +533,13 @@ fn run_mokapot( anyhow::bail!("mokapot worker exited with {status}"); } + // The worker echoes the PIN's SpecId tail as `candidate_id`, which here is the + // row index i. Map scores back by row index; a missing row (should not happen + // now the worker scores all PSMs) gets the worst score. let t = Table::read(&outp)?; - let ocid = t.u32("candidate_id")?; + let orow = t.u32("candidate_id")?; let osc = t.f64("score")?; - let map: HashMap = ocid.into_iter().zip(osc).collect(); - let min = map.values().cloned().fold(f64::INFINITY, f64::min); - Ok(cid.iter().map(|c| *map.get(c).unwrap_or(&(min - 1.0))).collect()) + let map: HashMap = orow.into_iter().zip(osc).collect(); + let worst = map.values().cloned().fold(f64::INFINITY, f64::min) - 1.0; + Ok((0..cid.len() as u32).map(|i| *map.get(&i).unwrap_or(&worst)).collect()) } diff --git a/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs b/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs index 55ca9e5..5facf8c 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs @@ -46,12 +46,18 @@ pub fn run(p: RtImTrainParams) -> Result { let s_q = seed.f64("spectrum_q")?; let s_score = seed.f64("score")?; let s_rt = seed.f64("observed_rt")?; + let s_label = seed.str("label")?; let mut best_per_pep: HashMap = HashMap::new(); // base -> (score, irt, rt) for i in 0..seed.nrows { if s_q[i] >= p.cfg.q_train { continue; } + // Only target PSMs may anchor the RT calibration; a decoy anchor injects a + // random iRT<->RT pair into the fit. + if s_label[i] != "target" { + continue; + } let irt = match irt_by_cid.get(&s_cid[i]) { Some(v) => *v, None => continue, @@ -83,8 +89,13 @@ pub fn run(p: RtImTrainParams) -> Result { } }; - // Residuals and RT window. - let (w_rt, status) = if n_train >= 2 { + // Residuals and RT window. Require enough anchors before trusting the + // residual-percentile window: with only a handful of points a linear fit + // passes ~exactly through them, so residuals ~0 and the window collapses to + // the 1s floor (which then discards nearly every true co-elution). Below the + // threshold, use the configured fixed fallback instead. + let min_anchors = p.cfg.min_seed_for_calibration.max(2); + let (w_rt, status) = if n_train >= min_anchors { let resid: Vec = train_irt .iter() .zip(&train_rt) @@ -95,7 +106,10 @@ pub fn run(p: RtImTrainParams) -> Result { let status = if use_loess { "loess" } else { "linear" }; (w, status.to_string()) } else { - warn!("rt-im-train: too few seeds; using fallback fixed RT window"); + warn!( + n_train, + min_anchors, "rt-im-train: too few target anchors; using fallback fixed RT window" + ); (p.cfg.fallback_rt_window_s, "fallback_fixed".to_string()) }; From b341388b92cddc99302083be5002f84331b7697e Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:38:22 +0200 Subject: [PATCH 03/40] fix(predict-frag): per-charge-group MS2PIP normalization; charge-2 fragments for z2 - MS2PIP charge-1 (TIC-fraction) and native charge-2 fallback intensities are max-normalized per charge group before the shared top-N ranking, so MS2PIP ions are no longer buried by the differently-scaled native charge-2 values. - charge2_from_precursor_charge default 3 -> 2: allow charge-2 fragments for charge-2 precursors (DIA-NN uses them for ~16% of z2 transitions). - config: CompeteGroupBy::PeptidoformCharge variant (precursor-level competition). Co-Authored-By: Claude Fable 5 --- rust/mumdia/crates/mumdia-core/src/config.rs | 15 +++++++++--- .../crates/mumdia/src/stages/predict_frag.rs | 23 ++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 327c84e..3a28326 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -334,7 +334,9 @@ pub struct PredictFragConfig { pub predictor: FragPredictorKind, pub rt_predictor: RtPredictorKind, /// Fragment charges rule: charge 1 always; charge 2 added for precursor - /// charge >= this threshold (PLAN.md Decision 3). + /// charge >= this threshold (PLAN.md Decision 3). Default 2: DIA-NN uses + /// doubly-charged fragments for ~16% of charge-2 precursors' transitions, so + /// blocking them (the old default of 3) discarded real signal. pub charge2_from_precursor_charge: i32, pub top_n_fragments: usize, pub ms2pip_model: String, @@ -350,7 +352,7 @@ impl Default for PredictFragConfig { Self { predictor: t(), rt_predictor: t(), - charge2_from_precursor_charge: 3, + charge2_from_precursor_charge: 2, top_n_fragments: 6, ms2pip_model: "HCD".to_string(), ms2pip_python: None, @@ -577,7 +579,10 @@ impl Default for FeaturesConfig { #[serde(default, deny_unknown_fields)] pub struct CompeteConfig { /// competition grouping: `precursor` groups target/decoy pairs and charge - /// variants of one peptide; `apex` also groups by rounded apex RT. + /// variants of one peptide; `apex` also groups by rounded apex RT; + /// `peptidoform_charge` keeps each peptidoform+charge as its own group + /// (precursor-level, as DIA-NN/Spectronaut report), so sibling charges of one + /// peptide are not collapsed. pub group_by: CompeteGroupBy, pub apex_rt_tolerance_s: f64, } @@ -595,6 +600,10 @@ impl Default for CompeteConfig { pub enum CompeteGroupBy { Precursor, Apex, + /// Precursor-level: separate every distinct peptidoform+charge. Recovers + /// sibling charges the peptide-level `Precursor` grouping collapses; the + /// label stays in the key so a target never competes against its own decoy. + PeptidoformCharge, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs b/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs index cd54e4e..80c4326 100644 --- a/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs +++ b/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs @@ -310,7 +310,7 @@ fn assign_intensities(p: &PredictFragParams, raws: &mut [Raw]) -> Result Some(per) if !per.is_empty() => { // native as fallback for fragment charges MS2PIP does not emit (charge 2) let nat = native.predict_intensities(&r.parsed, &r.frags); - r.frag_int = r + let mut vals: Vec = r .frags .iter() .enumerate() @@ -323,6 +323,27 @@ fn assign_intensities(p: &PredictFragParams, raws: &mut [Raw]) -> Result } }) .collect(); + // MS2PIP (charge-1, TIC-fraction, ~0.02-0.3) and the native + // charge-2 fallback (max-normalized, ~0.19-0.5) live on + // different scales; ranking them together in top-N buries + // MS2PIP. Max-normalize each charge group to its own peak so + // the two compete fairly. + let gmax = |want2: bool| { + r.frags + .iter() + .zip(&vals) + .filter(|(fr, _)| (fr.charge >= 2) == want2) + .map(|(_, v)| *v) + .fold(0.0f32, f32::max) + }; + let (m1, m2) = (gmax(false), gmax(true)); + for (k, fr) in r.frags.iter().enumerate() { + let m = if fr.charge >= 2 { m2 } else { m1 }; + if m > 0.0 { + vals[k] /= m; + } + } + r.frag_int = vals; } _ => { r.frag_int = native.predict_intensities(&r.parsed, &r.frags); From 2479b716bf96a76274c42825885e8ea34d5fcadd Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:38:23 +0200 Subject: [PATCH 04/40] fix(sidecars): mokapot out-of-fold scores, collision-free entrapment readback, import sort - mokapot_worker: use mokapot's out-of-fold confidence estimates (targets + decoys) instead of averaging all fold models over all rows (which is not OOF). Falls back to fold-model averaging if the confidence API differs. - entrapment_worker: pass through a unique row_id so the caller maps scores back without candidate_id collisions across competed runs. - entrapment_corrected.py: exclude human decoys from the entrapment-false population (add & ~dec) at both counting sites. - import_diann_lib.py: sort precursors by m/z before assigning candidate_id so a direct target-only import is fragment-index-valid. Co-Authored-By: Claude Fable 5 --- scripts/entrapment_corrected.py | 4 +- scripts/entrapment_worker.py | 6 ++- scripts/import_diann_lib.py | 6 ++- scripts/mokapot_worker.py | 71 +++++++++++++++++++++++++++------ 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/scripts/entrapment_corrected.py b/scripts/entrapment_corrected.py index c5b9a30..bef3665 100644 --- a/scripts/entrapment_corrected.py +++ b/scripts/entrapment_corrected.py @@ -96,7 +96,7 @@ def contam_mask(acct): def eco_at_true1(acct): contam = contam_mask(acct) - is_entrap = hum & ~contam # valid false population + is_entrap = hum & ~contam & ~dec # valid false population (exclude human decoys) is_real = ~dec & ~hum # E. coli targets q = ent_q(SCORE, is_entrap, is_real, RATIO) gate = q <= 0.01 @@ -117,7 +117,7 @@ def eco_at_true1(acct): print(f" [A] at shipped gate (q<=0.01): E.coli={eco_fixed} true FDR by accounting:") for a in ("raw", "crap", "diann"): contam = contam_mask(a) - false_h = c.loc[(hum & ~contam) & gate, "strip"].nunique() + false_h = c.loc[(hum & ~contam & ~dec) & gate, "strip"].nunique() print(f" acct={a:6}: true FDR = {false_h*RATIO/max(1,eco_fixed)*100:4.2f}% (false human seqs={false_h})") # Framing B: re-threshold to a genuine true 1% under each accounting (score sweep) diff --git a/scripts/entrapment_worker.py b/scripts/entrapment_worker.py index 0d92e7d..fb7d851 100644 --- a/scripts/entrapment_worker.py +++ b/scripts/entrapment_worker.py @@ -65,11 +65,14 @@ def main(): folds = int(sys.argv[3]) if len(sys.argv) > 3 else 3 t = pq.read_table(inp).to_pandas() - meta = ["candidate_id", "base_peptide_id", "is_entrapment", "is_decoy"] + meta = ["row_id", "candidate_id", "base_peptide_id", "is_entrapment", "is_decoy"] feat_cols = [c for c in t.columns if c not in meta] X = np.nan_to_num(t[feat_cols].to_numpy(dtype=np.float64), posinf=0.0, neginf=0.0) cid = t["candidate_id"].to_numpy() + # Unique flat row id for collision-free score readback (candidate_id repeats + # across competed runs). Fall back to positional index if absent. + rid = t["row_id"].to_numpy() if "row_id" in t.columns else np.arange(len(t), dtype=np.uint32) grp = t["base_peptide_id"].to_numpy() is_ent = t["is_entrapment"].to_numpy() != 0 is_dec = t["is_decoy"].to_numpy() != 0 @@ -105,6 +108,7 @@ def main(): scores[gap] = mf.predict_proba(X[gap])[:, 1] out = pa.table({ + "row_id": pa.array(rid.astype("uint32")), "candidate_id": pa.array(cid.astype("uint32")), "score": pa.array(scores.astype(np.float64)), }) diff --git a/scripts/import_diann_lib.py b/scripts/import_diann_lib.py index 255c3ae..1ef7878 100644 --- a/scripts/import_diann_lib.py +++ b/scripts/import_diann_lib.py @@ -45,7 +45,11 @@ def main(): prot_col = "Protein.Names" if "Protein.Names" in df.columns else "Protein.Ids" df["protein_str"] = df[prot_col].astype(str) - keys = df.drop_duplicates("key").reset_index(drop=True) + # Sort precursors by m/z before assigning candidate_id, so the emitted library + # is monotonic in precursor_mz (the fragment index's candidate_range assumes + # this). The decoy builder re-sorts too, but this makes a direct target-only + # import index-valid on its own. mergesort = stable for reproducibility. + keys = df.drop_duplicates("key").sort_values("Precursor.Mz", kind="mergesort").reset_index(drop=True) keys["candidate_id"] = np.arange(len(keys), dtype=np.uint32) key2cand = dict(zip(keys["key"], keys["candidate_id"])) keys["base_peptide_id"] = pd.factorize(keys["Stripped.Sequence"])[0].astype(np.uint32) diff --git a/scripts/mokapot_worker.py b/scripts/mokapot_worker.py index 6ad067e..9b29d74 100644 --- a/scripts/mokapot_worker.py +++ b/scripts/mokapot_worker.py @@ -50,9 +50,29 @@ def make_model(mokapot): ) # scaler=None -> StandardScaler (features are on mixed scales). return mokapot.Model(net, train_fdr=0.01, max_iter=brew_iters, rng=0) + if kind in ("xgb", "xgboost"): + # Gradient-boosted trees: nonlinear, captures feature interactions the + # linear models miss. Used as the second stage after a cheap prefilter, + # where the reduced PSM count keeps it tractable. Hist tree method + + # capped depth/estimators for speed on millions of PSMs. + from xgboost import XGBClassifier + + net = XGBClassifier( + n_estimators=int(os.environ.get("MUMDIA_XGB_TREES", "200")), + max_depth=int(os.environ.get("MUMDIA_XGB_DEPTH", "6")), + learning_rate=float(os.environ.get("MUMDIA_XGB_LR", "0.1")), + subsample=0.8, + colsample_bytree=0.8, + tree_method="hist", + n_jobs=int(os.environ.get("MUMDIA_XGB_JOBS", "0")) or None, + eval_metric="logloss", + random_state=0, + ) + # scaler=None -> StandardScaler; harmless for trees, keeps the API uniform. + return mokapot.Model(net, train_fdr=0.01, max_iter=brew_iters, rng=0) if kind != "nn": raise ValueError( - f"unknown MUMDIA_RESCORE_MODEL={kind!r} (want nn|logreg|percolator)" + f"unknown MUMDIA_RESCORE_MODEL={kind!r} (want nn|logreg|xgb|percolator)" ) from sklearn.neural_network import MLPClassifier @@ -102,19 +122,44 @@ def main(): else: _results, models = mokapot.brew(psms, model=model, rng=0, max_workers=workers) - # Score EVERY PSM (targets AND decoys) with the trained fold models. mokapot's - # confidence table is targets-only; returning only targets starves the decoys - # of scores downstream, so the caller's target-decoy q recomputation collapses - # (unscored decoys sink, so nearly every target passes -> huge false count). - # Model.predict == decision_function over the whole dataset; averaging across - # the cross-validation fold models gives every PSM a proper score in data order. - ml = list(models) if hasattr(models, "__len__") else [models] - score_mat = np.vstack( - [np.asarray(m.predict(psms), dtype=np.float64) for m in ml] - ) - scores = score_mat.mean(axis=0) - specids = psms.data["SpecId"].astype(str).to_numpy() + + # Prefer mokapot's OUT-OF-FOLD scores: brew returns held-out confidence tables + # (targets in confidence_estimates, decoys in decoy_confidence_estimates), each + # PSM scored only by the fold that did not train on it. The previous approach + # (averaging all fold models over all rows) is NOT out-of-fold - 2 of 3 folds + # trained on each row - which inflates apparent sensitivity (~2.5% external FDR + # at a nominal 1% cut). Fall back to fold-model averaging only if the mokapot + # confidence API differs from what we expect. + # NOTE: unvalidated in the unit suite (no mokapot there); confirm on a real run. + def _oof_scores(): + conf = _results[0] if isinstance(_results, (list, tuple)) else _results + tdf = conf.confidence_estimates["psms"] + ddf = conf.decoy_confidence_estimates["psms"] + + def scol(df): + for c in ("mokapot score", "score"): + if c in df.columns: + return c + return next(c for c in df.columns if "score" in c.lower()) + + m = {} + for df in (ddf, tdf): + m.update(dict(zip(df["SpecId"].astype(str), df[scol(df)].astype(float)))) + if len(m) < 0.5 * len(specids): + raise RuntimeError(f"OOF tables cover only {len(m)}/{len(specids)} PSMs") + worst = min(m.values()) - 1.0 + return np.array([m.get(s, worst) for s in specids], dtype=np.float64) + + try: + scores = _oof_scores() + print("mokapot_worker: using out-of-fold confidence scores", flush=True) + except Exception as e: + print(f"mokapot_worker: OOF unavailable ({e}); fold-model average fallback", flush=True) + ml = list(models) if hasattr(models, "__len__") else [models] + scores = np.vstack( + [np.asarray(m.predict(psms), dtype=np.float64) for m in ml] + ).mean(axis=0) if len(specids) != len(scores): raise RuntimeError( f"specid/score length mismatch: {len(specids)} vs {len(scores)}" From a79f6a02ded9fdca31de1c69741a5cd773df1d12 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:43:20 +0200 Subject: [PATCH 05/40] fix(native-rescore): standardize within fold, fold by peptide family - Fit the feature standardizer on each training fold only (was fit on the full data before splitting, leaking test-fold statistics into the scaling). - Cross-validation fold key is now base_peptide_id (was candidate_id % K), so all charge/modification variants of a peptide share a fold and none leaks between train and test. Co-Authored-By: Claude Fable 5 --- rust/mumdia/crates/mumdia/src/rescoring.rs | 157 +++++++++--------- .../crates/mumdia/src/stages/rescore.rs | 16 +- 2 files changed, 89 insertions(+), 84 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/rescoring.rs b/rust/mumdia/crates/mumdia/src/rescoring.rs index 59981ff..e2991b2 100644 --- a/rust/mumdia/crates/mumdia/src/rescoring.rs +++ b/rust/mumdia/crates/mumdia/src/rescoring.rs @@ -8,38 +8,40 @@ use crate::fdr::target_decoy_q; use rayon::prelude::*; -/// Standardize columns to zero mean / unit variance (guarded). -fn standardize(x: &[Vec]) -> Vec> { - if x.is_empty() { - return Vec::new(); - } - let n = x.len(); - let d = x[0].len(); +/// Column mean/std over a SUBSET of rows (guarded; std < 1e-9 -> 1.0). Fitting the +/// scaler on the training fold only avoids leaking test-fold statistics into the +/// standardization. +fn fit_standardizer(x: &[Vec], idx: &[usize]) -> (Vec, Vec) { + let d = x.first().map(|r| r.len()).unwrap_or(0); + let n = idx.len().max(1) as f64; let mut mean = vec![0.0; d]; - for row in x { + for &i in idx { for j in 0..d { - mean[j] += row[j]; + mean[j] += x[i][j]; } } for m in &mut mean { - *m /= n as f64; + *m /= n; } let mut std = vec![0.0; d]; - for row in x { + for &i in idx { for j in 0..d { - let dd = row[j] - mean[j]; + let dd = x[i][j] - mean[j]; std[j] += dd * dd; } } for s in &mut std { - *s = (*s / n as f64).sqrt(); + *s = (*s / n).sqrt(); if *s < 1e-9 { *s = 1.0; } } - x.iter() - .map(|row| (0..d).map(|j| (row[j] - mean[j]) / std[j]).collect()) - .collect() + (mean, std) +} + +#[inline] +fn std_row(row: &[f64], mean: &[f64], std: &[f64]) -> Vec { + (0..row.len()).map(|j| (row[j] - mean[j]) / std[j]).collect() } /// Logistic regression by full-batch gradient descent with L2. Weight[0] = bias. @@ -83,7 +85,9 @@ fn score_row(w: &[f64], r: &[f64]) -> f64 { pub struct RescoreInput<'a> { pub features: &'a [Vec], pub is_decoy: &'a [bool], - pub candidate_id: &'a [u32], + /// Cross-validation fold key. Use base_peptide_id so every charge/mod variant + /// of a peptide lands in the same fold (no peptide leaks train<->test). + pub fold_key: &'a [u32], pub init_score: &'a [f64], pub folds: usize, pub num_iter: usize, @@ -96,21 +100,15 @@ pub fn percolator_lite(inp: RescoreInput) -> Vec { if n == 0 { return Vec::new(); } - let xs = standardize(inp.features); let folds = inp.folds.max(1); - // deterministic fold assignment by candidate_id - let fold_of: Vec = inp - .candidate_id - .iter() - .map(|c| (*c as usize) % folds) - .collect(); + // Fold assignment by fold_key (base peptide): all charge/mod variants of a + // peptide share a fold, so none leaks between train and test. + let fold_of: Vec = inp.fold_key.iter().map(|c| (*c as usize) % folds).collect(); - // Folds are independent: each trains its own weights and scores only its own - // (disjoint) test set. Parallelize across folds and merge into `final_score` - // afterward. This is bit-identical to the serial loop: every fold's inner - // computation is unchanged, and the per-fold writes target disjoint indices - // (i belongs to exactly one test fold), so merge order does not matter. + // Folds are independent: each fits its own scaler + weights on its training + // rows and scores only its own (disjoint) test set. Standardization is fit on + // the TRAIN fold only (no test leakage), which is why each fold owns its scaler. let per_fold: Vec> = (0..folds) .into_par_iter() .map(|test_fold| { @@ -119,59 +117,66 @@ pub fn percolator_lite(inp: RescoreInput) -> Vec { if train_idx.is_empty() || test_idx.is_empty() { return Vec::new(); } - // iterate positive-set selection using current scores on the train set + let (mean, std) = fit_standardizer(inp.features, &train_idx); + // standardized train matrix (owned, so we can take &[f64] slices) + let xtr: Vec> = train_idx + .iter() + .map(|&i| std_row(&inp.features[i], &mean, &std)) + .collect(); let mut train_scores: Vec = train_idx.iter().map(|&i| inp.init_score[i]).collect(); - let mut w = vec![0.0; xs[0].len() + 1]; - // Reused scratch buffer for the (score, is_decoy) pairs; cleared and - // refilled each iteration so contents/order match a fresh allocation. - let mut sd: Vec<(f64, bool)> = Vec::with_capacity(train_idx.len()); - for _ in 0..inp.num_iter.max(1) { - sd.clear(); - sd.extend( - train_idx - .iter() - .enumerate() - .map(|(k, &i)| (train_scores[k], inp.is_decoy[i])), - ); - let q = target_decoy_q(&sd); - // positive set: confident targets; negatives: all decoys - let mut rows: Vec<&[f64]> = Vec::new(); - let mut ys: Vec = Vec::new(); - let mut n_pos = 0; - for (k, &i) in train_idx.iter().enumerate() { - if inp.is_decoy[i] { - rows.push(&xs[i]); - ys.push(0.0); - } else if q[k] <= inp.train_fdr { - rows.push(&xs[i]); - ys.push(1.0); - n_pos += 1; - } - } - // fallback: if too few confident targets, take the top-scoring half - if n_pos < 10 { - let mut order: Vec = (0..train_idx.len()).collect(); - order.sort_by(|&a, &b| train_scores[b].partial_cmp(&train_scores[a]).unwrap()); - let take = (train_idx.len() / 2).max(1); - rows.clear(); - ys.clear(); - for (rank, &k) in order.iter().enumerate() { - let i = train_idx[k]; + let mut w = vec![0.0; inp.features[0].len() + 1]; + let mut sd: Vec<(f64, bool)> = Vec::with_capacity(train_idx.len()); + for _ in 0..inp.num_iter.max(1) { + sd.clear(); + sd.extend( + train_idx + .iter() + .enumerate() + .map(|(k, &i)| (train_scores[k], inp.is_decoy[i])), + ); + let q = target_decoy_q(&sd); + // positive set: confident targets; negatives: all decoys + let mut rows: Vec<&[f64]> = Vec::new(); + let mut ys: Vec = Vec::new(); + let mut n_pos = 0; + for (k, &i) in train_idx.iter().enumerate() { if inp.is_decoy[i] { - rows.push(&xs[i]); + rows.push(&xtr[k]); ys.push(0.0); - } else if rank < take { - rows.push(&xs[i]); + } else if q[k] <= inp.train_fdr { + rows.push(&xtr[k]); ys.push(1.0); + n_pos += 1; + } + } + // fallback: if too few confident targets, take the top-scoring half + if n_pos < 10 { + let mut order: Vec = (0..train_idx.len()).collect(); + order.sort_by(|&a, &b| train_scores[b].partial_cmp(&train_scores[a]).unwrap()); + let take = (train_idx.len() / 2).max(1); + rows.clear(); + ys.clear(); + for (rank, &k) in order.iter().enumerate() { + let i = train_idx[k]; + if inp.is_decoy[i] { + rows.push(&xtr[k]); + ys.push(0.0); + } else if rank < take { + rows.push(&xtr[k]); + ys.push(1.0); + } } } + w = logreg_fit(&rows, &ys, 1e-3, 200, 0.5); + train_scores = (0..train_idx.len()).map(|k| score_row(&w, &xtr[k])).collect(); } - w = logreg_fit(&rows, &ys, 1e-3, 200, 0.5); - train_scores = train_idx.iter().map(|&i| score_row(&w, &xs[i])).collect(); - } - test_idx.iter().map(|&i| (i, score_row(&w, &xs[i]))).collect() - }) - .collect(); + // score the held-out test fold with this fold's scaler + weights + test_idx + .iter() + .map(|&i| (i, score_row(&w, &std_row(&inp.features[i], &mean, &std)))) + .collect() + }) + .collect(); let mut final_score = inp.init_score.to_vec(); for fold_scores in per_fold { @@ -205,7 +210,7 @@ mod tests { let s = percolator_lite(RescoreInput { features: &features, is_decoy: &is_decoy, - candidate_id: &cid, + fold_key: &cid, init_score: &init, folds: 3, num_iter: 5, diff --git a/rust/mumdia/crates/mumdia/src/stages/rescore.rs b/rust/mumdia/crates/mumdia/src/stages/rescore.rs index 1dd1fa2..07c5716 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rescore.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rescore.rs @@ -113,7 +113,7 @@ pub fn run(p: RescoreParams) -> Result { anyhow::bail!("rescore: Mokapot sidecar failed ({e}) and rescore.strict=true"); } warn!("rescore: Mokapot failed ({e}); falling back to native_tda"); - native_scores(&p, &feats, &is_decoy, &cid, &prelim) + native_scores(&p, &feats, &is_decoy, &base, &prelim) } }, RescorerKind::Percolator => { @@ -121,9 +121,9 @@ pub fn run(p: RescoreParams) -> Result { anyhow::bail!("rescore: classifier=percolator but percolator.exe is not wired, and rescore.strict=true"); } warn!("rescore: percolator.exe path not wired; using native_tda"); - native_scores(&p, &feats, &is_decoy, &cid, &prelim) + native_scores(&p, &feats, &is_decoy, &base, &prelim) } - RescorerKind::NativeTda => native_scores(&p, &feats, &is_decoy, &cid, &prelim), + RescorerKind::NativeTda => native_scores(&p, &feats, &is_decoy, &base, &prelim), RescorerKind::Entrapment => { let n_ent = is_entrapment.iter().filter(|&&b| b).count(); if p.cfg.entrapment_marker.is_none() || n_ent == 0 { @@ -141,7 +141,7 @@ pub fn run(p: RescoreParams) -> Result { (set rescore.entrapment_marker to the spike-in accession \ substring); falling back to native_tda" ); - native_scores(&p, &feats, &is_decoy, &cid, &prelim) + native_scores(&p, &feats, &is_decoy, &base, &prelim) } else if p.cfg.python.is_some() { match run_entrapment_gbm(&p, &feat_names, &cid, &base, &is_entrapment, &is_decoy, &feats) { Ok(s) => { @@ -159,7 +159,7 @@ pub fn run(p: RescoreParams) -> Result { classifier_used = "entrapment_native"; model_identity = "native-percolator-lite-entrapment-v1".to_string(); qmode = QMode::Entrapment; - native_scores(&p, &feats, &is_entrapment, &cid, &prelim) + native_scores(&p, &feats, &is_entrapment, &base, &prelim) } } } else { @@ -170,7 +170,7 @@ pub fn run(p: RescoreParams) -> Result { classifier_used = "entrapment_native"; model_identity = "native-percolator-lite-entrapment-v1".to_string(); qmode = QMode::Entrapment; - native_scores(&p, &feats, &is_entrapment, &cid, &prelim) + native_scores(&p, &feats, &is_entrapment, &base, &prelim) } } } @@ -311,13 +311,13 @@ fn native_scores( p: &RescoreParams, feats: &[Vec], is_decoy: &[bool], - cid: &[u32], + fold_key: &[u32], prelim: &[f64], ) -> Vec { percolator_lite(RescoreInput { features: feats, is_decoy, - candidate_id: cid, + fold_key, init_score: prelim, folds: p.cfg.folds, num_iter: p.cfg.num_iter, From a5bbfb6d012e0065107d70b4bca465127ef4dbe7 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:47:38 +0200 Subject: [PATCH 06/40] fix(extract/config): emit all predicted transitions, signature-ion apex, dead-knob warnings - extract: emit a chromatogram row for every predicted transition, with a zero-intensity trace when the fragment was never observed. The feature families (similarity, entropy, ion-series, coelution, interference, nonzero) now see the full predicted set and penalize missing strong ions; mass-accuracy is unaffected because it already counts only fragments with obs_apex > 0. - extract: apex tiebreak on the top-K predicted signature ions' observed intensity (wires the previously-dead apex_top_fragments; 0 -> default 3) instead of the 3 brightest observed peaks, so a bright off-signature interferent cannot set the apex. - config: warn at load when a declared-but-unimplemented knob (precursor_tol_ppm, tolerance_regime, k_select, max_fragment_charge, scan_scale, decoy.source, decoy.ratio) is set away from its default, instead of silently ignoring it. Co-Authored-By: Claude Fable 5 --- rust/mumdia/crates/mumdia-core/src/config.rs | 30 +++++++++ .../crates/mumdia/src/stages/extract.rs | 64 +++++++++++++------ 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 3a28326..c33f5f0 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -833,6 +833,36 @@ impl Config { .into(), )); } + // Warn (not fail) when a declared-but-unimplemented knob is set away from + // its default: it silently has no effect, which otherwise misleads tuning. + let d = Self::default(); + let mut dead = Vec::new(); + if self.search_seed.precursor_tol_ppm != d.search_seed.precursor_tol_ppm { + dead.push("search_seed.precursor_tol_ppm"); + } + if self.rt_im_train.tolerance_regime != d.rt_im_train.tolerance_regime { + dead.push("rt_im_train.tolerance_regime"); + } + if self.extract.k_select != d.extract.k_select { + dead.push("extract.k_select"); + } + if self.extract.max_fragment_charge != d.extract.max_fragment_charge { + dead.push("extract.max_fragment_charge"); + } + if self.extract.scan_scale != d.extract.scan_scale { + dead.push("extract.scan_scale"); + } + if self.digest.decoy.source != d.digest.decoy.source { + dead.push("digest.decoy.source"); + } + if self.digest.decoy.ratio != d.digest.decoy.ratio { + dead.push("digest.decoy.ratio"); + } + for k in &dead { + eprintln!( + "config warning: `{k}` is set but not implemented in the engine; it has no effect" + ); + } Ok(()) } diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index fa68ecf..d95845a 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -663,23 +663,35 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { let rt_prior_sigma = p.cfg.apex_rt_prior_s; let rt_cal_c = rt_cal[cid as usize]; let use_prior = rt_prior_sigma > 0.0 && rt_cal_c > 0.0; + // Signature-ion apex tiebreak: sum the OBSERVED intensity of the top-K + // PREDICTED fragments (`apex_top_fragments`; 0 -> a default of 3) at each + // qualifying scan, instead of the 3 brightest observed peaks. A bright + // interferent on a non-signature ion can then no longer define the apex. + let k_sig = if p.cfg.apex_top_fragments > 0 { + p.cfg.apex_top_fragments + } else { + 3 + }; + let sig: Vec = { + let mut ord: Vec = (0..fints0.len()).collect(); + ord.sort_by(|&a, &b| fints0[b].partial_cmp(&fints0[a]).unwrap_or(std::cmp::Ordering::Equal)); + ord.into_iter().take(k_sig).map(|o| o as u16).collect() + }; let mut apex_rt = groups[0].0; let mut apex_sum = 0.0f32; - let mut best_top3 = f32::NEG_INFINITY; + let mut best_sig = f32::NEG_INFINITY; for (i, (rt, map)) in groups.iter().enumerate() { if map.is_empty() || smoothed[i] < thresh { continue; } - let mut vals: Vec = map.values().cloned().collect(); - vals.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); - let top3: f32 = vals.iter().take(3).sum(); + let sig_sum: f32 = sig.iter().map(|&o| map.get(&o).copied().unwrap_or(0.0)).sum(); let score = if use_prior { - top3 * (-0.5 * ((*rt - rt_cal_c) / rt_prior_sigma).powi(2)).exp() as f32 + sig_sum * (-0.5 * ((*rt - rt_cal_c) / rt_prior_sigma).powi(2)).exp() as f32 } else { - top3 + sig_sum }; - if score > best_top3 { - best_top3 = score; + if score > best_sig { + best_sig = score; apex_rt = *rt; apex_sum = map.values().sum(); // report full apex intensity } @@ -787,28 +799,40 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } else { Vec::new() }; - let mut frag_keys: Vec = per_frag.keys().cloned().collect(); - frag_keys.sort_unstable(); - for frag in frag_keys { - let fi = frag as usize; + // Emit EVERY predicted transition, not just the observed ones. A predicted + // fragment never matched anywhere gets a zero-intensity row (all-zero grid + // trace, or an empty series without the grid) so the feature families see + // the full predicted set: similarity/entropy/ion-series/coelution get the + // correct denominator and a missing strong ion is penalized. obs m/z falls + // back to the theoretical m/z, which is harmless because the mass-accuracy + // features only count fragments with obs_apex > 0. + for fi in 0..fmzs.len() { + let frag = fi as u16; let obs_mz = wsum .get(&frag) .map(|(sm, sw)| if *sw > 0.0 { sm / sw } else { fmzs[fi] }) .unwrap_or(fmzs[fi]); let (rts, ints): (Vec, Vec) = if !grid.is_empty() { - let m: HashMap = - per_frag[&frag].iter().map(|(r, i)| (r.to_bits(), *i)).collect(); + let m: HashMap = per_frag + .get(&frag) + .map(|v| v.iter().map(|(r, i)| (r.to_bits(), *i)).collect()) + .unwrap_or_default(); ( grid.iter().map(|r| *r as f32).collect(), grid.iter().map(|r| *m.get(&r.to_bits()).unwrap_or(&0.0)).collect(), ) } else { - let mut s = per_frag[&frag].clone(); - s.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - ( - s.iter().map(|(r, _)| *r as f32).collect(), - s.iter().map(|(_, i)| *i).collect(), - ) + match per_frag.get(&frag) { + Some(v) => { + let mut s = v.clone(); + s.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + ( + s.iter().map(|(r, _)| *r as f32).collect(), + s.iter().map(|(_, i)| *i).collect(), + ) + } + None => (Vec::new(), Vec::new()), + } }; chrom_rows.push((cid, fnames[fi].clone(), fmzs[fi], obs_mz, fints[fi], rts, ints)); } From 46434c85a5335f904dfb420da001b98ae06a94bc Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:56:31 +0200 Subject: [PATCH 07/40] fix(extract): acquisition-grid apex + co-elution; MS1 evidence before the gate - Project the sparse hit-groups onto the real acquisition scan grid (the covering isolation-window scans within the RT window) before apex counting and the co-elution run. Missing acquisition scans are now count-0 and break a run, so "consecutive scans" are truly consecutive (the sparse-group run counter previously ignored acquisition gaps, admitting transient/interferent matches). Falls back to sparse groups when no covering-window grid exists. - Compute MS1 apex isotope intensities BEFORE the fragment-Pearson gate and add an opt-in rescue (extract.ms1_rescue, default off): a candidate failing the single- scan Pearson is kept when it has adequate matched fragments and MS1 isotope support (mono present + plausible +1/mono ratio). Off by default because it relaxes acceptance; enable with FDR validation. Co-Authored-By: Claude Fable 5 --- rust/mumdia/crates/mumdia-core/src/config.rs | 7 ++ .../crates/mumdia/src/stages/extract.rs | 115 ++++++++++++------ 2 files changed, 84 insertions(+), 38 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index c33f5f0..d3fa27a 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -513,6 +513,12 @@ pub struct ExtractConfig { /// peptide persists across its elution. 0 disables (the `scan_window` floor still /// applies). This is the DIA analog of a "seen in >= N PSMs" requirement. pub min_coelution_run: usize, + /// Rescue a candidate that fails the single-scan fragment-Pearson gate when it + /// has adequate matched fragments AND MS1 isotope-pattern support (mono + a + /// plausible +1/mono ratio). Off by default: it relaxes acceptance, so enable + /// it only with target-decoy/entrapment FDR validation. MS1 evidence is now + /// computed before the gate so this can take effect. + pub ms1_rescue: bool, } impl Default for ExtractConfig { fn default() -> Self { @@ -544,6 +550,7 @@ impl Default for ExtractConfig { peak_claim_margin: 2.0, matcher: MatcherKind::Fragindex, min_coelution_run: 0, // disabled; scan_window floor still applies + ms1_rescue: false, // opt-in; relaxes acceptance, validate FDR first } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index d95845a..3251b40 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -624,6 +624,41 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { let (fmzs0, fints0, _) = lib.cand_frags(cid); + // Acquisition scan grid: the covering isolation-window scans within the + // RT window. Project the sparse hit-groups onto it so apex counting and + // the co-elution run see MISSING acquisition scans (count 0, and they + // break a run) rather than only scans that happened to carry a hit. When + // no covering-window grid is available, fall back to the sparse groups. + let grid: Vec = if !windows.is_empty() { + let pm = lib.cands[cid as usize].precursor_mz; + let (lo, hi) = (rt_lo[cid as usize], rt_hi[cid as usize]); + let mut g: Vec = Vec::new(); + for (wl, wu, rts) in &windows { + if *wl <= pm && pm <= *wu { + let a = rts.partition_point(|&r| r < lo); + let b = rts.partition_point(|&r| r <= hi); + g.extend_from_slice(&rts[a..b]); + } + } + g.sort_by(|a, b| a.partial_cmp(b).unwrap()); + g.dedup(); + g + } else { + Vec::new() + }; + if !grid.is_empty() { + let g2i: HashMap = + grid.iter().enumerate().map(|(j, r)| (r.to_bits(), j)).collect(); + let mut aligned: Vec<(f64, BTreeMap)> = + grid.iter().map(|&r| (r, BTreeMap::new())).collect(); + for (rt, map) in std::mem::take(&mut groups) { + if let Some(&j) = g2i.get(&rt.to_bits()) { + aligned[j].1 = map; + } + } + groups = aligned; + } + // Apex: the scan group with the most distinct matched fragments, allowing // scans within `apex_count_tol` of that maximum (so a slightly-lower-count // but much more intense scan can still win), then the one maximizing the @@ -720,8 +755,41 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { return None; } - // Optional tier-d Pearson gate (kept for configurability; matched - // fraction above is the primary symmetric discriminator). + let c = &lib.cands[cid as usize]; + + // MS1 apex isotope intensities (nearest MS1 scan to the apex RT), computed + // BEFORE the acceptance gate so MS1 evidence can rescue a candidate the + // single-scan fragment-Pearson gate would otherwise reject. + let (o_ms1_m1, o_ms1_mono, o_ms1_i1, o_ms1_i2) = if ms1_scans.is_empty() { + (None, None, None, None) + } else { + let j = nearest_index(&ms1_rts, apex_rt); + let s = &ms1_scans[j]; + let z = c.charge as f64; + let sp = ISOTOPE_SPACING / z; + let tol = p.cfg.prec_tol_ppm; + ( + Some(sum_near(&s.mz, &s.intensity, c.precursor_mz - sp, tol) as f64), + Some(sum_near(&s.mz, &s.intensity, c.precursor_mz, tol) as f64), + Some(sum_near(&s.mz, &s.intensity, c.precursor_mz + sp, tol) as f64), + Some(sum_near(&s.mz, &s.intensity, c.precursor_mz + 2.0 * sp, tol) as f64), + ) + }; + // Cheap MS1 support: mono present and the +1/mono ratio in a plausible + // averagine band. Used only as the rescue signal for the Pearson gate. + let ms1_support = { + let mono = o_ms1_mono.unwrap_or(0.0); + let i1 = o_ms1_i1.unwrap_or(0.0); + mono > 0.0 && i1 > 0.0 && { + let r = i1 / mono; + (0.1..=1.5).contains(&r) + } + }; + + // Optional tier-d Pearson gate (kept for configurability; matched fraction + // above is the primary symmetric discriminator). With `ms1_rescue`, a + // candidate that fails the single-scan fragment Pearson is kept when it has + // adequate matched fragments AND MS1 isotope-pattern support. let apex_map = groups .iter() .find(|(rt, _)| (*rt - apex_rt).abs() < 1e-9) @@ -733,34 +801,21 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { .collect(); let pred: Vec = fints0.iter().map(|x| *x as f64).collect(); if crate::stats::pearson(&obs, &pred) < p.cfg.min_frag_corr { - return None; + let rescued = p.cfg.ms1_rescue + && ms1_support + && distinct.len() >= p.cfg.presence_min_fragments.max(1); + if !rescued { + return None; + } } } } - let c = &lib.cands[cid as usize]; let contested_val = { let (w, l) = contested.get(&cid).copied().unwrap_or((0.0, 0.0)); if w + l > 0.0 { l / (w + l) } else { 0.0 } }; - // MS1 apex isotope intensities: nearest MS1 scan to the apex RT. - let (o_ms1_m1, o_ms1_mono, o_ms1_i1, o_ms1_i2) = if ms1_scans.is_empty() { - (None, None, None, None) - } else { - let j = nearest_index(&ms1_rts, apex_rt); - let s = &ms1_scans[j]; - let z = c.charge as f64; - let sp = ISOTOPE_SPACING / z; - let tol = p.cfg.prec_tol_ppm; - ( - Some(sum_near(&s.mz, &s.intensity, c.precursor_mz - sp, tol) as f64), - Some(sum_near(&s.mz, &s.intensity, c.precursor_mz, tol) as f64), - Some(sum_near(&s.mz, &s.intensity, c.precursor_mz + sp, tol) as f64), - Some(sum_near(&s.mz, &s.intensity, c.precursor_mz + 2.0 * sp, tol) as f64), - ) - }; - // Per-fragment intensity-weighted observed m/z (for mass accuracy). let mut wsum: HashMap = HashMap::new(); // frag -> (sum w*mz, sum w) for h in &hits { @@ -782,23 +837,7 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { per_frag.entry(frag).or_default().push((*rt, inten)); } } - let grid: Vec = if !windows.is_empty() { - let pm = lib.cands[cid as usize].precursor_mz; - let (lo, hi) = (rt_lo[cid as usize], rt_hi[cid as usize]); - let mut g: Vec = Vec::new(); - for (wl, wu, rts) in &windows { - if *wl <= pm && pm <= *wu { - let a = rts.partition_point(|&r| r < lo); - let b = rts.partition_point(|&r| r <= hi); - g.extend_from_slice(&rts[a..b]); - } - } - g.sort_by(|a, b| a.partial_cmp(b).unwrap()); - g.dedup(); - g - } else { - Vec::new() - }; + // (the acquisition-scan `grid` was computed above, before apex/co-elution) // Emit EVERY predicted transition, not just the observed ones. A predicted // fragment never matched anywhere gets a zero-intensity row (all-zero grid // trace, or an empty series without the grid) so the feature families see From e6deee1abdcadd6f33e392668a514b9dc09f8db3 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 11:58:02 +0200 Subject: [PATCH 08/40] feat(scripts): empirical transition reanalysis + held-out entrapment harness - empirical_reanalyse.py: first DIA-NN-`--reanalyse`-style pass. Learns per predicted-rank transition reliability from confident target IDs (fraction observed in-peak) and reweights or prunes the library fragments by that rank reliability. Applied symmetrically by rank to targets and decoys, so it does not by itself distort the target-decoy null; the user re-extracts with the new library. (Global-rank version; leakage-free out-of-fold learning is the P3 refinement.) - entrapment_holdout.py: honest held-out validation. Splits the human entrapment proteins into disjoint train/test halves by a stable protein hash, trains the classifier on E. coli targets vs human-TRAIN negatives, and evaluates FDR / E. coli sensitivity against the UNSEEN human-TEST null. Co-Authored-By: Claude Fable 5 --- scripts/empirical_reanalyse.py | 98 ++++++++++++++++++++++++++++++ scripts/entrapment_holdout.py | 105 +++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 scripts/empirical_reanalyse.py create mode 100644 scripts/entrapment_holdout.py diff --git a/scripts/empirical_reanalyse.py b/scripts/empirical_reanalyse.py new file mode 100644 index 0000000..df1c61a --- /dev/null +++ b/scripts/empirical_reanalyse.py @@ -0,0 +1,98 @@ +"""Empirical transition reanalysis (a first, DIA-NN-`--reanalyse`-style pass). + +From a first-pass rescored result, learn how reliably each PREDICTED-RANK transition +is actually observed in-peak among confident target IDs, then reweight (or prune) +the library fragments by that rank reliability. The transform depends only on a +transition's predicted-intensity RANK within its candidate, not on its identity, so +it is applied SYMMETRICALLY to targets and decoys and cannot by itself distort the +target-decoy null. The user then re-extracts/re-rescores with the new library. + +This mirrors DIA-NN's empirical library: weak/interfered low-rank transitions are +down-weighted or dropped, high-rank signature transitions kept. + +Usage: + python empirical_reanalyse.py + [--q 0.01] [--mode reweight|prune] [--prune-thr 0.25] + +Caveat (leakage): rank reliability is learned globally from all confident IDs here. +The rigorous version learns it out-of-fold / from other runs (see the audit's P3); +this global version is a usable first pass, not the leakage-free final. +""" +import re +import sys + +import numpy as np +import pyarrow.dataset as pds +import pyarrow.parquet as pq +import pyarrow as pa + +strip = lambda p: re.sub(r"\[[^\]]*\]", "", str(p)) + + +def arg(flag, default): + return sys.argv[sys.argv.index(flag) + 1] if flag in sys.argv else default + + +def main(): + scored_p, chrom_p, lib_in, lib_out = sys.argv[1:5] + q_conf = float(arg("--q", "0.01")) + mode = arg("--mode", "reweight") + prune_thr = float(arg("--prune-thr", "0.25")) + + # Confident target candidates from the first pass. + sc = pds.dataset(scored_p).to_table(columns=["candidate_id", "label", "q_value"]).to_pandas() + conf = set( + sc.candidate_id[(sc.label == "target") & (sc.q_value <= q_conf)].astype(int) + ) + if not conf: + raise SystemExit("empirical_reanalyse: no confident target IDs at the given q") + + # Library fragments; rank per candidate by predicted intensity (0 = strongest). + lf = pds.dataset(lib_in).to_table().to_pandas() + lf = lf.sort_values(["candidate_id", "predicted_intensity"], ascending=[True, False]) + lf["rank"] = lf.groupby("candidate_id").cumcount() + + # Observed-in-peak set from the chromatograms: a (candidate_id, frag_name) whose + # XIC has any nonzero intensity is "present". + ch = pds.dataset(chrom_p).to_table(columns=["candidate_id", "frag_name", "intensity"]).to_pandas() + ch = ch[ch.candidate_id.astype(int).isin(conf)] + present = set() + for cid, fn, inten in zip(ch.candidate_id, ch.frag_name, ch.intensity): + arr = np.asarray(inten, dtype=float) + if arr.size and arr.max() > 0.0: + present.add((int(cid), str(fn))) + + # Rank reliability = fraction of confident candidates whose rank-r transition is + # present in-peak (denominator = confident candidates that HAVE a rank-r transition). + conf_lf = lf[lf.candidate_id.astype(int).isin(conf)] + max_rank = int(conf_lf["rank"].max()) + retention = np.ones(max_rank + 1) + for r in range(max_rank + 1): + rows = conf_lf[conf_lf["rank"] == r] + if len(rows) == 0: + continue + obs = sum((int(c), str(n)) in present for c, n in zip(rows.candidate_id, rows.name)) + retention[r] = obs / len(rows) + print("rank reliability (fraction observed in-peak among confident IDs):") + for r in range(min(max_rank + 1, 15)): + print(f" rank {r:2d}: {retention[r]:.3f}") + + # Apply symmetrically by rank to the WHOLE library (targets + decoys). + rk = lf["rank"].to_numpy().clip(0, max_rank) + rel = retention[rk] + if mode == "prune": + keep = rel >= prune_thr + lf = lf[keep].copy() + print(f"prune: kept {int(keep.sum())}/{len(keep)} transitions (rank reliability >= {prune_thr})") + else: + lf = lf.copy() + lf["predicted_intensity"] = (lf["predicted_intensity"].to_numpy() * rel).astype(np.float32) + print(f"reweight: scaled {len(lf)} predicted intensities by rank reliability") + + lf = lf.drop(columns=["rank"]).sort_values("candidate_id").reset_index(drop=True) + pq.write_table(pa.Table.from_pandas(lf, preserve_index=False), lib_out) + print(f"wrote empirical library -> {lib_out} ({len(lf)} fragments)") + + +if __name__ == "__main__": + main() diff --git a/scripts/entrapment_holdout.py b/scripts/entrapment_holdout.py new file mode 100644 index 0000000..94e16fc --- /dev/null +++ b/scripts/entrapment_holdout.py @@ -0,0 +1,105 @@ +"""Held-out entrapment validation harness. + +The entrapment-trained scorer uses the human proteome as negatives, but training, +FDR, and reporting on the SAME human population overfits (research leak). This +harness splits the human entrapment proteins into two disjoint halves by a stable +protein hash, trains the classifier on E. coli targets (positive) vs human-TRAIN +(negative), and then evaluates FDR/sensitivity against the UNSEEN human-TEST null. +That held-out E. coli count at 1% is the honest number. + +Usage: + python entrapment_holdout.py [--folds 3] [--q 0.01] + +Requires an env with scikit-learn + pyarrow (py312_mumdia). Deterministic. +""" +import hashlib +import re +import sys + +import numpy as np +import pyarrow.dataset as pds +from sklearn.ensemble import HistGradientBoostingClassifier + +strip = lambda p: re.sub(r"\[[^\]]*\]", "", str(p)) + + +def arg(flag, default): + return sys.argv[sys.argv.index(flag) + 1] if flag in sys.argv else default + + +def half(protein: str) -> int: + # Stable (non-randomized) split of a protein string into half 0/1. + h = hashlib.md5(str(protein).encode("utf-8")).hexdigest() + return int(h, 16) & 1 + + +def ent_q(score, is_entrap, is_real, ratio): + order = np.argsort(-score, kind="stable") + ne = nr = 0 + fdr = np.ones(len(score)) + for rank, i in enumerate(order): + if is_entrap[i]: + ne += 1 + elif is_real[i]: + nr += 1 + fdr[rank] = ratio * ne / max(1, nr) + q = np.ones(len(score)) + qmin = 1.0 + for rank in range(len(score) - 1, -1, -1): + qmin = min(qmin, fdr[rank]) + q[order[rank]] = qmin + return q + + +def main(): + path = sys.argv[1] + q_cut = float(arg("--q", "0.01")) + t = pds.dataset(path).to_table().to_pandas() + pcol = "protein_group" if "protein_group" in t.columns else "protein" + dec = t.label.eq("decoy").to_numpy() + human = (t[pcol].str.contains("_HUMAN") & ~t[pcol].str.contains("_ECOLI")).to_numpy() + ecoli_tgt = (~dec) & (~human) + hh = t[pcol].map(half).to_numpy() + train_neg = human & (hh == 0) # human-TRAIN negatives + test_neg = human & (hh == 1) # human-TEST null (unseen) + + meta = {"candidate_id", "label", "peptidoform", "protein", "protein_group", + "base_peptide_id", "charge", "q_value", "peptide_q_value", "pg_q_value", + "global_q_value", "score", "prelim_score", "source"} + feats = [c for c in t.columns if c not in meta and np.issubdtype(t[c].dtype, np.number)] + X = np.nan_to_num(t[feats].to_numpy(np.float64), posinf=0.0, neginf=0.0) + + # Train ONLY on E. coli targets (pos) vs human-TRAIN (neg); human-TEST + decoys + # are never seen in training. + tr = ecoli_tgt | train_neg + y = np.where(ecoli_tgt, 1, 0)[tr] + if len(np.unique(y)) < 2: + raise SystemExit("entrapment_holdout: need both E.coli targets and human-train negatives") + m = HistGradientBoostingClassifier(random_state=0, early_stopping=False) + m.fit(X[tr], y) + score = m.predict_proba(X)[:, 1] + + # Held-out evaluation: null = human-TEST (unseen). Library-size ratio correction. + ratio = int(ecoli_tgt.sum()) / max(1, int(test_neg.sum())) + q = ent_q(score, test_neg, ecoli_tgt, ratio) + gate = q <= q_cut + eco_heldout = t.loc[ecoli_tgt & gate, "peptidoform"].map(strip).nunique() + + # For contrast: FDR at the shipped q_value cutoff, measured on the held-out null. + shipped = None + if "q_value" in t.columns: + g2 = t.q_value.to_numpy() <= q_cut + eco_s = t.loc[ecoli_tgt & g2, "peptidoform"].map(strip).nunique() + leak = int((test_neg & g2).sum()) + shipped = (eco_s, ratio * leak / max(1, int((ecoli_tgt & g2).sum())) * 100) + + print(f"=== held-out entrapment ({path}) ===") + print(f" E.coli targets={int(ecoli_tgt.sum())} human train-neg={int(train_neg.sum())} " + f"test-null={int(test_neg.sum())} ratio={ratio:.3f}") + print(f" held-out E.coli stripped seqs @ {q_cut:.0%} (unseen null): {eco_heldout}") + if shipped: + print(f" at shipped q<= {q_cut:.0%}: E.coli={shipped[0]}, true FDR on held-out null = {shipped[1]:.2f}%") + + +if __name__ == "__main__": + main() From 2187b0d43abf25b1c9e4924cc7de10095df091be Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 13:42:00 +0200 Subject: [PATCH 09/40] fix(extract): absent transitions emit an empty trace, not a grid-length zero vector emit-all-transitions gave every never-observed predicted fragment a full grid- length zero trace, which multiplied the chromatogram list-values past arrow's 32-bit ListArray offset limit and panicked (Option::unwrap on None) on large runs (observed only after ~2 files). Absent transitions now emit an empty trace: the row is still present so the feature families see pred>0/obs=0, but the values count stays near the observed-only total. Co-Authored-By: Claude Fable 5 --- .../crates/mumdia/src/stages/extract.rs | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index 3251b40..8ff3fe6 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -838,40 +838,38 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } } // (the acquisition-scan `grid` was computed above, before apex/co-elution) - // Emit EVERY predicted transition, not just the observed ones. A predicted - // fragment never matched anywhere gets a zero-intensity row (all-zero grid - // trace, or an empty series without the grid) so the feature families see - // the full predicted set: similarity/entropy/ion-series/coelution get the - // correct denominator and a missing strong ion is penalized. obs m/z falls - // back to the theoretical m/z, which is harmless because the mass-accuracy - // features only count fragments with obs_apex > 0. + // Emit a row for EVERY predicted transition so the feature families see the + // full predicted set (a missing strong ion is penalized). An OBSERVED + // fragment carries its grid-sampled (or sorted) trace; a NEVER-OBSERVED one + // carries an EMPTY trace, NOT a grid-length zero vector. The empty trace + // still yields obs_apex = 0 downstream, but avoids inflating the total + // chromatogram list-values past arrow's 32-bit ListArray offset limit + // (a grid-length zero per absent fragment overflowed it on large runs). + // obs m/z falls back to theoretical; harmless since mass-accuracy counts + // only fragments with obs_apex > 0. for fi in 0..fmzs.len() { let frag = fi as u16; let obs_mz = wsum .get(&frag) .map(|(sm, sw)| if *sw > 0.0 { sm / sw } else { fmzs[fi] }) .unwrap_or(fmzs[fi]); - let (rts, ints): (Vec, Vec) = if !grid.is_empty() { - let m: HashMap = per_frag - .get(&frag) - .map(|v| v.iter().map(|(r, i)| (r.to_bits(), *i)).collect()) - .unwrap_or_default(); - ( - grid.iter().map(|r| *r as f32).collect(), - grid.iter().map(|r| *m.get(&r.to_bits()).unwrap_or(&0.0)).collect(), - ) - } else { - match per_frag.get(&frag) { - Some(v) => { - let mut s = v.clone(); - s.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - ( - s.iter().map(|(r, _)| *r as f32).collect(), - s.iter().map(|(_, i)| *i).collect(), - ) - } - None => (Vec::new(), Vec::new()), + let (rts, ints): (Vec, Vec) = match per_frag.get(&frag) { + Some(v) if !grid.is_empty() => { + let m: HashMap = v.iter().map(|(r, i)| (r.to_bits(), *i)).collect(); + ( + grid.iter().map(|r| *r as f32).collect(), + grid.iter().map(|r| *m.get(&r.to_bits()).unwrap_or(&0.0)).collect(), + ) + } + Some(v) => { + let mut s = v.clone(); + s.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + ( + s.iter().map(|(r, _)| *r as f32).collect(), + s.iter().map(|(_, i)| *i).collect(), + ) } + None => (Vec::new(), Vec::new()), // absent predicted transition }; chrom_rows.push((cid, fnames[fi].clone(), fmzs[fi], obs_mz, fints[fi], rts, ints)); } From c6f268f62e60697d0584c3f9a1f92dd597bcbbbc Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 16:26:44 +0200 Subject: [PATCH 10/40] fix(io): 64-bit LargeList chrom columns to lift the 2.1B offset ceiling Chromatogram rt/intensity were written as an arrow ListArray (32-bit offsets), capping total list-values at ~2.1B. When extraction accepts a very large candidate set (correlation gates opened up, or large multi-file runs), the chrom ListArray offset buffer overflows and the builder panics mid-write. Add a Col::LargeListF32 variant (arrow 64-bit LargeList) and write the chrom rt/intensity columns with it. The list_f32 reader now accepts both List (existing artifacts) and LargeList encodings, so chrom parquet from either binary reads back identically. Spectra keep 32-bit ListF32 (small per-scan peak lists, read via a direct downcast in spectra.rs). Also gitignore /_archive/ (local, superseded working docs). Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 + rust/mumdia/crates/mumdia-io/src/table.rs | 68 ++++++++++++++----- .../crates/mumdia/src/stages/extract.rs | 14 ++-- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 9febef8..b83a8aa 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ scripts/_probe*.py # internal docs/specs/analysis: kept local, NOT published (public repo ships only READMEs) /*.md !/README.md +# local archive of superseded working docs (not published) +/_archive/ diff --git a/rust/mumdia/crates/mumdia-io/src/table.rs b/rust/mumdia/crates/mumdia-io/src/table.rs index f25e939..1621776 100644 --- a/rust/mumdia/crates/mumdia-io/src/table.rs +++ b/rust/mumdia/crates/mumdia-io/src/table.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use arrow::array::{ Array, ArrayRef, BooleanArray, Float32Array, Float32Builder, Float64Array, Int32Array, - Int64Array, ListArray, ListBuilder, StringArray, UInt32Array, + Int64Array, LargeListArray, LargeListBuilder, ListArray, ListBuilder, StringArray, UInt32Array, }; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -34,6 +34,11 @@ pub enum Col { OptStr(String, Vec>), ListF32(String, Vec>), ListF64(String, Vec>), + /// Like `ListF32` but encoded as an Arrow `LargeList` (64-bit offsets). + /// Required for columns whose total list-value count can exceed the ~2.1B + /// limit of the 32-bit `ListArray` offset buffer (e.g. per-fragment + /// chromatograms when extraction accepts a very large candidate set). + LargeListF32(String, Vec>), } impl Col { @@ -51,7 +56,8 @@ impl Col { | Col::OptI32(n, _) | Col::OptStr(n, _) | Col::ListF32(n, _) - | Col::ListF64(n, _) => n, + | Col::ListF64(n, _) + | Col::LargeListF32(n, _) => n, } } @@ -70,6 +76,7 @@ impl Col { Col::OptStr(_, v) => v.len(), Col::ListF32(_, v) => v.len(), Col::ListF64(_, v) => v.len(), + Col::LargeListF32(_, v) => v.len(), } } @@ -90,6 +97,7 @@ impl Col { Col::OptStr(n, _) => Field::new(n, DataType::Utf8, true), Col::ListF32(n, _) => Field::new(n, DataType::List(item32()), true), Col::ListF64(n, _) => Field::new(n, DataType::List(item64()), true), + Col::LargeListF32(n, _) => Field::new(n, DataType::LargeList(item32()), true), } } @@ -126,6 +134,14 @@ impl Col { } Arc::new(b.finish()) } + Col::LargeListF32(_, v) => { + let mut b = LargeListBuilder::new(Float32Builder::new()); + for row in &v { + b.values().append_slice(row); + b.append(true); + } + Arc::new(b.finish()) + } } } } @@ -367,26 +383,40 @@ impl Table { Ok(out) } + /// Read an f32 list column. Accepts both `List` (32-bit offsets) and + /// `LargeList` (64-bit offsets, written by `Col::LargeListF32`) encodings, + /// so chromatogram artifacts written by either binary read back the same. pub fn list_f32(&self, name: &str) -> Result>> { let i = self.idx(name)?; let mut out = Vec::with_capacity(self.nrows); - for b in &self.batches { - let a = b - .column(i) + let push_inner = |out: &mut Vec>, v: ArrayRef| -> Result<()> { + let f = v .as_any() - .downcast_ref::() - .ok_or_else(|| anyhow!("column '{name}' is not a list"))?; - for k in 0..a.len() { - if a.is_null(k) { - out.push(Vec::new()); - continue; + .downcast_ref::() + .ok_or_else(|| anyhow!("list '{name}' inner is not f32"))?; + out.push(f.values().to_vec()); + Ok(()) + }; + for b in &self.batches { + let col = b.column(i); + if let Some(a) = col.as_any().downcast_ref::() { + for k in 0..a.len() { + if a.is_null(k) { + out.push(Vec::new()); + } else { + push_inner(&mut out, a.value(k))?; + } + } + } else if let Some(a) = col.as_any().downcast_ref::() { + for k in 0..a.len() { + if a.is_null(k) { + out.push(Vec::new()); + } else { + push_inner(&mut out, a.value(k))?; + } } - let v = a.value(k); - let f = v - .as_any() - .downcast_ref::() - .ok_or_else(|| anyhow!("list '{name}' inner is not f32"))?; - out.push(f.values().to_vec()); + } else { + return Err(anyhow!("column '{name}' is not a list")); } } Ok(out) @@ -411,6 +441,7 @@ mod tests { Col::Str("name".into(), vec!["a".into(), "b".into(), "c".into()]), Col::OptF64("cal".into(), vec![Some(1.0), None, Some(3.0)]), Col::ListF32("trace".into(), vec![vec![1.0, 2.0], vec![], vec![9.0]]), + Col::LargeListF32("big".into(), vec![vec![5.0], vec![6.0, 7.0], vec![]]), ], ) .unwrap(); @@ -423,5 +454,8 @@ mod tests { assert_eq!(t.opt_f64("cal").unwrap(), vec![Some(1.0), None, Some(3.0)]); assert_eq!(t.list_f32("trace").unwrap()[0], vec![1.0, 2.0]); assert!(t.list_f32("trace").unwrap()[1].is_empty()); + // LargeListF32 (64-bit offsets) reads back through the same list_f32 path. + assert_eq!(t.list_f32("big").unwrap()[1], vec![6.0, 7.0]); + assert!(t.list_f32("big").unwrap()[2].is_empty()); } } diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index 8ff3fe6..b72f793 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -842,9 +842,10 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { // full predicted set (a missing strong ion is penalized). An OBSERVED // fragment carries its grid-sampled (or sorted) trace; a NEVER-OBSERVED one // carries an EMPTY trace, NOT a grid-length zero vector. The empty trace - // still yields obs_apex = 0 downstream, but avoids inflating the total - // chromatogram list-values past arrow's 32-bit ListArray offset limit - // (a grid-length zero per absent fragment overflowed it on large runs). + // still yields obs_apex = 0 downstream, and keeps the total chromatogram + // list-value count down (a grid-length zero per absent fragment would + // bloat it needlessly; the column itself is now a 64-bit LargeList, so the + // old ~2.1B 32-bit offset ceiling no longer applies). // obs m/z falls back to theoretical; harmless since mass-accuracy counts // only fragments with obs_apex > 0. for fi in 0..fmzs.len() { @@ -990,8 +991,11 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { Col::F64("frag_mz".into(), ch_fmz), Col::F64("frag_obs_mz".into(), ch_obsmz), Col::F32("predicted_intensity".into(), ch_pint), - Col::ListF32("rt".into(), ch_rt), - Col::ListF32("intensity".into(), ch_int), + // LargeList (64-bit offsets): the total chromatogram list-value count + // can exceed the ~2.1B limit of a 32-bit ListArray offset buffer when + // extraction accepts a very large candidate set (e.g. gates opened up). + Col::LargeListF32("rt".into(), ch_rt), + Col::LargeListF32("intensity".into(), ch_int), ], )?; From 2f46d6d189d650abf5f992209f6239101e8f44af Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 16:47:22 +0200 Subject: [PATCH 11/40] feat(sensitivity): rejection reasons, top-K peak enumeration, competition-mode + config scaffolding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundational, additive, behaviour-preserving building blocks for the sensitivity program (sensitivity_plan/). - mumdia-core::rejection::RejectionReason: the spec's earliest-loss reason codes (01 §4 / P0.3) with stable string codes, a pipeline-stage ordering, and an `earliest()` combinator. Unit tested. - mumdia::peaks::enumerate_peaks: non-destructive top-K chromatographic peak enumeration (local maxima + fractional-height boundaries + evidence rank), the core of "retain the correct peak instead of committing to one apex" (04 §3 / P1). Pure + unit tested with synthetic chromatograms (single peak, dominant interferent + retained true peak, truncation, prominence filter, shoulder collapse, determinism). - config: ExtractConfig.retain_top_peaks (default 1 = legacy) + validation; ExtractConfig.emit_candidate_audit (default false); CompetitionMode enum (winner_take_all default / none / features_only / unique_evidence / margin_gated) + CompeteConfig.mode/margin/unique_evidence_min_fragments/ emit_competition_audit. All serde-default so existing configs are unchanged. No default behaviour changes; K=1 and winner_take_all reproduce current output. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 81 ++++++ rust/mumdia/crates/mumdia-core/src/lib.rs | 1 + .../crates/mumdia-core/src/rejection.rs | 163 +++++++++++ rust/mumdia/crates/mumdia/src/lib.rs | 1 + rust/mumdia/crates/mumdia/src/peaks.rs | 268 ++++++++++++++++++ 5 files changed, 514 insertions(+) create mode 100644 rust/mumdia/crates/mumdia-core/src/rejection.rs create mode 100644 rust/mumdia/crates/mumdia/src/peaks.rs diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index d3fa27a..5fd56ff 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -519,6 +519,18 @@ pub struct ExtractConfig { /// it only with target-decoy/entrapment FDR validation. MS1 evidence is now /// computed before the gate so this can take effect. pub ms1_rescue: bool, + /// Number of chromatographic peak groups to retain per candidate (sensitivity + /// program, spec 04 §3 / P1). `1` = legacy behaviour (one apex-level PSM per + /// candidate). `K>1` retains up to K local-maxima peak groups per candidate so + /// a wrong early apex does not discard the correct peak before rescoring. Each + /// retained peak carries its own apex, boundaries, initial evidence, and + /// `peak_rank`. K=1 is bit-for-bit compatible with the previous behaviour. + pub retain_top_peaks: usize, + /// Diagnostic candidate-audit: when true, extraction records, for every probed + /// candidate, either the survivor stage-flags or the earliest `RejectionReason`, + /// and writes `.audit.parquet` (spec 01 §4 / P0.3). Near-zero cost + /// when false (no per-candidate audit allocation). Default false (production). + pub emit_candidate_audit: bool, } impl Default for ExtractConfig { fn default() -> Self { @@ -551,6 +563,8 @@ impl Default for ExtractConfig { matcher: MatcherKind::Fragindex, min_coelution_run: 0, // disabled; scan_window floor still applies ms1_rescue: false, // opt-in; relaxes acceptance, validate FDR first + retain_top_peaks: 1, // legacy single-apex behaviour (K=1) + emit_candidate_audit: false, // diagnostic; off in production } } } @@ -592,12 +606,72 @@ pub struct CompeteConfig { /// peptide are not collapsed. pub group_by: CompeteGroupBy, pub apex_rt_tolerance_s: f64, + /// How within-group competition resolves (sensitivity program, spec 04 §6 / + /// P2.4). `winner_take_all` = legacy (keep only the top `prelim_score` per + /// group). The other modes preserve more candidate evidence for the rescorer/ + /// FDR to arbitrate. Default `winner_take_all` (unchanged behaviour). + pub mode: CompetitionMode, + /// Score margin (in `prelim_score` units) required to remove a loser under + /// `margin_gated`. A loser closer than this to the winner is kept. + pub margin: f64, + /// Minimum distinct unique-fragment count a loser must have to survive under + /// `unique_evidence` (needs the `unique_fragment_count` feature; falls back to + /// winner-take-all when the column is absent). + pub unique_evidence_min_fragments: usize, + /// Diagnostic: when true, write `.compete_audit.parquet` recording every + /// removed candidate with its group, winner, scores, and removal reason. + pub emit_competition_audit: bool, } impl Default for CompeteConfig { fn default() -> Self { Self { group_by: CompeteGroupBy::Precursor, apex_rt_tolerance_s: 5.0, + mode: CompetitionMode::WinnerTakeAll, + margin: 0.0, + unique_evidence_min_fragments: 2, + emit_competition_audit: false, + } + } +} + +/// Within-group competition resolution (spec 04 §6). Only `WinnerTakeAll` removes +/// candidates unconditionally; the others preserve candidates the rescorer can +/// still discriminate, which is the sensitivity program's central principle +/// ("preserve candidate evidence until the workflow can make a calibrated +/// decision"). Target/decoy labels remain part of the competition key in every +/// mode, so a target never competes against its own decoy (the null is preserved). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum CompetitionMode { + /// Legacy: keep only the highest `prelim_score` candidate per group. + #[default] + WinnerTakeAll, + /// Keep every candidate (no within-group removal); FDR handles ambiguity. + None, + /// Keep every candidate; conflict/contested features (added upstream) carry the + /// interference signal into rescoring. Same retained set as `None`; the name + /// documents intent for the experiment matrix. + FeaturesOnly, + /// Keep a loser when it has enough independent evidence + /// (`unique_fragment_count >= unique_evidence_min_fragments`); otherwise remove + /// it (winner-take-all fallback). + UniqueEvidence, + /// Remove a loser only when `winner_score - loser_score >= margin`; otherwise + /// keep it. Conservative removal for the low-FDR region. + MarginGated, +} + +impl CompetitionMode { + /// Parse a CLI/token spelling. + pub fn from_token(s: &str) -> Option { + match s.to_ascii_lowercase().replace('-', "_").as_str() { + "winner_take_all" | "winner" => Some(Self::WinnerTakeAll), + "none" => Some(Self::None), + "features_only" | "features" => Some(Self::FeaturesOnly), + "unique_evidence" | "unique" => Some(Self::UniqueEvidence), + "margin_gated" | "margin" => Some(Self::MarginGated), + _ => None, } } } @@ -840,6 +914,13 @@ impl Config { .into(), )); } + if self.extract.retain_top_peaks == 0 { + return Err(Invalid( + "extract.retain_top_peaks must be >= 1 (1 = legacy single-apex \ + behaviour; K>1 retains up to K peak groups per candidate)." + .into(), + )); + } // Warn (not fail) when a declared-but-unimplemented knob is set away from // its default: it silently has no effect, which otherwise misleads tuning. let d = Self::default(); diff --git a/rust/mumdia/crates/mumdia-core/src/lib.rs b/rust/mumdia/crates/mumdia-core/src/lib.rs index ffc3ad7..ecb704c 100644 --- a/rust/mumdia/crates/mumdia-core/src/lib.rs +++ b/rust/mumdia/crates/mumdia-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod constants; pub mod error; pub mod manifest; pub mod mass; +pub mod rejection; pub mod schema; pub mod types; diff --git a/rust/mumdia/crates/mumdia-core/src/rejection.rs b/rust/mumdia/crates/mumdia-core/src/rejection.rs new file mode 100644 index 0000000..1a9017b --- /dev/null +++ b/rust/mumdia/crates/mumdia-core/src/rejection.rs @@ -0,0 +1,163 @@ +//! Candidate rejection reason codes for the candidate-audit table. +//! +//! Sensitivity program (spec `01_workflow_and_gap_analysis.md` §4, +//! `02_sensitivity_diagnostic_plan.md` §5, backlog P0.3). Each variant names the +//! pipeline stage at which a candidate precursor is lost. A candidate's audit row +//! records the EARLIEST such stage, so the aggregate answers "where was each +//! DIA-NN-only precursor first lost?" without conflating later stages. +//! +//! The serialized spelling is SCREAMING_SNAKE_CASE and matches the reason strings +//! in the specification exactly (e.g. `NO_PEAK_GROUP`). Use [`RejectionReason::code`] +//! for the stable string written to Parquet/JSON (no serde round-trip cost). + +use serde::{Deserialize, Serialize}; + +/// Earliest-loss category for a candidate. Ordered along the pipeline so +/// [`RejectionReason::stage_order`] can pick the earliest when several apply. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum RejectionReason { + // --- search space (Stage A) --- + PeptideNotGenerated, + ModificationNotAllowed, + ChargeOutOfRange, + PrecursorMzOutOfRange, + NoValidFragments, + WrongIsolationWindow, + // --- candidate generation / pruning (Stage B) --- + RtPruned, + CandidateCapReached, + // --- extraction + peak formation (Stages C, D) --- + NoFragmentTraces, + NoPeakGroup, + // --- peak / peptide ranking (Stage E) --- + PeakNotSelected, + // --- competition (Stage G) --- + OutcompetedByTarget, + OutcompetedByDecoy, + // --- FDR + reporting (Stage H) --- + FailedPrecursorFdr, + FailedPeptideFdr, + RemovedDuringReporting, + /// Sentinel: not rejected; the candidate reached the final report. Present so + /// every candidate has exactly one audit row. + Reported, +} + +impl RejectionReason { + /// Stable string code (matches the specification's reason strings). Used for + /// Parquet columns and JSON without a serde round-trip. + pub fn code(&self) -> &'static str { + use RejectionReason::*; + match self { + PeptideNotGenerated => "PEPTIDE_NOT_GENERATED", + ModificationNotAllowed => "MODIFICATION_NOT_ALLOWED", + ChargeOutOfRange => "CHARGE_OUT_OF_RANGE", + PrecursorMzOutOfRange => "PRECURSOR_MZ_OUT_OF_RANGE", + NoValidFragments => "NO_VALID_FRAGMENTS", + WrongIsolationWindow => "WRONG_ISOLATION_WINDOW", + RtPruned => "RT_PRUNED", + CandidateCapReached => "CANDIDATE_CAP_REACHED", + NoFragmentTraces => "NO_FRAGMENT_TRACES", + NoPeakGroup => "NO_PEAK_GROUP", + PeakNotSelected => "PEAK_NOT_SELECTED", + OutcompetedByTarget => "OUTCOMPETED_BY_TARGET", + OutcompetedByDecoy => "OUTCOMPETED_BY_DECOY", + FailedPrecursorFdr => "FAILED_PRECURSOR_FDR", + FailedPeptideFdr => "FAILED_PEPTIDE_FDR", + RemovedDuringReporting => "REMOVED_DURING_REPORTING", + Reported => "REPORTED", + } + } + + /// Position on the identification-loss ladder (0 = earliest stage). The + /// `Reported` sentinel sorts last. When a candidate could be assigned more than + /// one reason across stages, keep the one with the smallest `stage_order`. + pub fn stage_order(&self) -> u8 { + use RejectionReason::*; + match self { + PeptideNotGenerated => 0, + ModificationNotAllowed => 1, + ChargeOutOfRange => 2, + PrecursorMzOutOfRange => 3, + NoValidFragments => 4, + WrongIsolationWindow => 5, + RtPruned => 6, + CandidateCapReached => 7, + NoFragmentTraces => 8, + NoPeakGroup => 9, + PeakNotSelected => 10, + OutcompetedByTarget => 11, + OutcompetedByDecoy => 12, + FailedPrecursorFdr => 13, + FailedPeptideFdr => 14, + RemovedDuringReporting => 15, + Reported => 255, + } + } + + /// True if the candidate was lost (any non-`Reported` reason). + pub fn is_rejection(&self) -> bool { + !matches!(self, RejectionReason::Reported) + } + + /// Keep the earlier of two losses (smaller `stage_order`). `Reported` never + /// overrides a real rejection. + pub fn earliest(self, other: RejectionReason) -> RejectionReason { + if other.stage_order() < self.stage_order() { + other + } else { + self + } + } +} + +#[cfg(test)] +mod tests { + use super::RejectionReason::*; + use super::*; + + #[test] + fn codes_match_spec_strings() { + assert_eq!(NoPeakGroup.code(), "NO_PEAK_GROUP"); + assert_eq!(PeakNotSelected.code(), "PEAK_NOT_SELECTED"); + assert_eq!(OutcompetedByDecoy.code(), "OUTCOMPETED_BY_DECOY"); + assert_eq!(Reported.code(), "REPORTED"); + } + + #[test] + fn serde_roundtrip_uses_spec_spelling() { + let j = serde_json::to_string(&FailedPrecursorFdr).unwrap(); + assert_eq!(j, "\"FAILED_PRECURSOR_FDR\""); + let back: RejectionReason = serde_json::from_str(&j).unwrap(); + assert_eq!(back, FailedPrecursorFdr); + } + + #[test] + fn earliest_keeps_smaller_stage() { + // an extraction loss precedes an FDR loss + assert_eq!(NoPeakGroup.earliest(FailedPrecursorFdr), NoPeakGroup); + assert_eq!(FailedPrecursorFdr.earliest(NoPeakGroup), NoPeakGroup); + // Reported never wins against a real rejection + assert_eq!(Reported.earliest(NoFragmentTraces), NoFragmentTraces); + assert_eq!(NoFragmentTraces.earliest(Reported), NoFragmentTraces); + } + + #[test] + fn is_rejection_flags_only_losses() { + assert!(NoPeakGroup.is_rejection()); + assert!(!Reported.is_rejection()); + } + + #[test] + fn stage_order_is_monotone_ladder() { + // ladder ordering across the major stages + assert!(PeptideNotGenerated.stage_order() < NoFragmentTraces.stage_order()); + assert!(NoFragmentTraces.stage_order() < NoPeakGroup.stage_order()); + assert!(NoPeakGroup.stage_order() < PeakNotSelected.stage_order()); + assert!(PeakNotSelected.stage_order() < OutcompetedByTarget.stage_order()); + assert!(OutcompetedByTarget.stage_order() < FailedPrecursorFdr.stage_order()); + assert!(FailedPrecursorFdr.stage_order() < RemovedDuringReporting.stage_order()); + assert!(RemovedDuringReporting.stage_order() < Reported.stage_order()); + } +} diff --git a/rust/mumdia/crates/mumdia/src/lib.rs b/rust/mumdia/crates/mumdia/src/lib.rs index da7098e..5e3e6ed 100644 --- a/rust/mumdia/crates/mumdia/src/lib.rs +++ b/rust/mumdia/crates/mumdia/src/lib.rs @@ -6,6 +6,7 @@ pub mod calibrate; pub mod fdr; pub mod index; pub mod matchers; +pub mod peaks; pub mod predict; pub mod quant_lfq; pub mod rescoring; diff --git a/rust/mumdia/crates/mumdia/src/peaks.rs b/rust/mumdia/crates/mumdia/src/peaks.rs new file mode 100644 index 0000000..acba087 --- /dev/null +++ b/rust/mumdia/crates/mumdia/src/peaks.rs @@ -0,0 +1,268 @@ +//! Top-K chromatographic peak enumeration (sensitivity program, spec +//! `04_peak_and_peptide_competition.md` §3, backlog P1). +//! +//! The central sensitivity hypothesis is that MuMDIA selects one chromatographic +//! apex too early, so the correct peak is discarded before the scorer ever sees it +//! (spec 01 §3.1). This module is the non-destructive alternative: given a +//! candidate's consensus elution profile (summed observed fragment intensity per +//! acquisition-scan group, aligned to a monotonic RT axis), it enumerates up to `K` +//! local-maximum peak groups, each with an apex, peak boundaries, an integrated +//! evidence score, and a rank. Downstream code can then compute features for every +//! retained peak and let an out-of-fold peak-selection model choose, instead of +//! committing to one apex up front. +//! +//! This module is intentionally pure and side-effect free so it is cheap to unit +//! test with synthetic chromatograms (see the tests) and carries no dependency on +//! the extraction hot path. `enumerate_peaks(.., k = 1, ..)` returns the single +//! strongest peak group (the global-argmax apex with the same fractional-height +//! boundary walk the features stage uses), so callers can adopt it incrementally. + +/// One retained chromatographic peak group for a candidate. +#[derive(Clone, Debug, PartialEq)] +pub struct PeakGroup { + /// Index of the apex within the input profile. + pub apex_idx: usize, + /// Inclusive left boundary index. + pub start_idx: usize, + /// Inclusive right boundary index. + pub end_idx: usize, + /// Intensity at the apex. + pub apex_intensity: f32, + /// Integrated intensity within `[start_idx, end_idx]` (the evidence score). + pub area: f32, + /// Rank by evidence (`area`), 0 = strongest. Assigned after sorting. + pub rank: usize, +} + +/// Enumerate up to `k` peak groups from a chromatographic `profile`. +/// +/// * `profile` — non-negative intensities per scan group along a monotonic RT axis. +/// * `k` — maximum peaks to return (>= 1; `k == 0` yields an empty vector). +/// * `bound_fraction` — peak-boundary threshold as a fraction of the local apex +/// height (matches `features.bound_peak_fraction`, default 1/3): the walk stops +/// when the profile drops below `bound_fraction * apex` or turns back upward +/// (a valley), whichever comes first. +/// * `min_prominence_frac` — a local maximum is ignored unless its height is at +/// least this fraction of the global maximum, suppressing noise flicker. Use +/// `0.0` to keep every local maximum. +/// +/// Peaks are returned strongest-first by integrated `area`, deduplicated so two +/// maxima inside one peak envelope collapse to the stronger one. Determinism: ties +/// break by earlier `apex_idx`. +pub fn enumerate_peaks( + profile: &[f32], + k: usize, + bound_fraction: f32, + min_prominence_frac: f32, +) -> Vec { + if k == 0 || profile.is_empty() { + return Vec::new(); + } + let n = profile.len(); + let global_max = profile.iter().cloned().fold(0.0f32, f32::max); + if global_max <= 0.0 { + return Vec::new(); + } + let prom_floor = min_prominence_frac.max(0.0) * global_max; + + // 1) Local maxima. `i` is a maximum when it is >= both neighbours and strictly + // greater than the left neighbour (so a flat plateau registers once, at its + // left edge). Edges count as maxima against their single neighbour. + let mut maxima: Vec = Vec::new(); + for i in 0..n { + let v = profile[i]; + if v <= 0.0 || v < prom_floor { + continue; + } + let left_ok = i == 0 || v > profile[i - 1]; + let right_ok = i + 1 == n || v >= profile[i + 1]; + if left_ok && right_ok { + maxima.push(i); + } + } + if maxima.is_empty() { + return Vec::new(); + } + + // 2) Boundaries for each maximum by the fractional-height descent walk. + let mut peaks: Vec = maxima + .iter() + .map(|&apex| { + let apex_v = profile[apex]; + let thr = bound_fraction.max(0.0) * apex_v; + // walk left: stop below threshold or when the profile turns upward + let mut start = apex; + while start > 0 { + let prev = profile[start - 1]; + if prev < thr || prev > profile[start] { + break; + } + start -= 1; + } + // walk right + let mut end = apex; + while end + 1 < n { + let next = profile[end + 1]; + if next < thr || next > profile[end] { + break; + } + end += 1; + } + let area: f32 = profile[start..=end].iter().sum(); + PeakGroup { + apex_idx: apex, + start_idx: start, + end_idx: end, + apex_intensity: apex_v, + area, + rank: 0, + } + }) + .collect(); + + // 3) Deduplicate maxima that fall inside another (stronger) peak's envelope. + // Sort strongest-first by area, then apex intensity, then earliest apex. + peaks.sort_by(|a, b| { + b.area + .partial_cmp(&a.area) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + b.apex_intensity + .partial_cmp(&a.apex_intensity) + .unwrap_or(std::cmp::Ordering::Equal), + ) + .then(a.apex_idx.cmp(&b.apex_idx)) + }); + let mut kept: Vec = Vec::new(); + for p in peaks { + let overlaps = kept + .iter() + .any(|q| p.apex_idx >= q.start_idx && p.apex_idx <= q.end_idx); + if !overlaps { + kept.push(p); + } + if kept.len() == k { + break; + } + } + for (r, p) in kept.iter_mut().enumerate() { + p.rank = r; + } + kept +} + +#[cfg(test)] +mod tests { + use super::*; + + const FR: f32 = 1.0 / 3.0; + + #[test] + fn empty_or_zero_profile_yields_no_peaks() { + assert!(enumerate_peaks(&[], 5, FR, 0.1).is_empty()); + assert!(enumerate_peaks(&[0.0, 0.0, 0.0], 5, FR, 0.1).is_empty()); + } + + #[test] + fn single_true_peak() { + // one clean triangular peak apexing at index 3 + let p = [0.0, 1.0, 4.0, 9.0, 4.0, 1.0, 0.0]; + let peaks = enumerate_peaks(&p, 5, FR, 0.1); + assert_eq!(peaks.len(), 1); + assert_eq!(peaks[0].apex_idx, 3); + assert_eq!(peaks[0].rank, 0); + // boundaries descend to >= 1/3 * 9 = 3.0: indices 2..=4 (value 4) are in, + // 1 and 5 (value 1) are below threshold. + assert_eq!(peaks[0].start_idx, 2); + assert_eq!(peaks[0].end_idx, 4); + } + + #[test] + fn k1_returns_only_the_strongest() { + // two peaks; K=1 keeps the strongest (area) one only + let p = [0.0, 9.0, 0.0, 0.0, 5.0, 0.0]; + let peaks = enumerate_peaks(&p, 1, FR, 0.1); + assert_eq!(peaks.len(), 1); + assert_eq!(peaks[0].apex_idx, 1); + } + + #[test] + fn interference_dominant_but_true_peak_retained_with_topk() { + // A dominant interference peak (broad+tall plateau, apex idx 1, largest + // integrated area) and a genuine but weaker true peak (apex idx 8). K=1 + // keeps only the dominant interferent and DISCARDS the true peak; K>=2 + // RETAINS the true peak so the scorer can still choose it. This is the core + // sensitivity behaviour: preserve the correct peak instead of discarding it + // early (spec 01 §3.1). + let p = [ + 0.0, 20.0, 20.0, 20.0, 0.0, 0.0, 6.0, 8.0, 9.0, 8.0, 6.0, 0.0, + ]; + let k1 = enumerate_peaks(&p, 1, FR, 0.05); + assert_eq!(k1.len(), 1); + assert_eq!(k1[0].apex_idx, 1, "K=1 keeps the dominant interferent"); + assert!( + !k1.iter().any(|pk| pk.apex_idx == 8), + "K=1 discards the true peak" + ); + let k3 = enumerate_peaks(&p, 3, FR, 0.05); + assert!( + k3.iter().any(|pk| pk.apex_idx == 8), + "K>1 must retain the true peak at idx 8" + ); + } + + #[test] + fn two_local_maxima_ranked_by_area() { + // broad peak (apex 2, larger area) vs sharp peak (apex 6, smaller area) + let p = [2.0, 5.0, 6.0, 5.0, 2.0, 3.0, 7.0, 3.0, 0.0]; + let peaks = enumerate_peaks(&p, 5, FR, 0.1); + assert!(peaks.len() >= 2); + // rank 0 is the larger-area (broad) peak around idx 2 + assert_eq!(peaks[0].rank, 0); + assert_eq!(peaks[0].apex_idx, 2); + assert!(peaks.iter().any(|pk| pk.apex_idx == 6)); + } + + #[test] + fn truncated_peak_at_left_edge() { + // apex at index 0 (peak truncated by the window start) + let p = [9.0, 6.0, 3.0, 1.0, 0.0]; + let peaks = enumerate_peaks(&p, 5, FR, 0.1); + assert_eq!(peaks.len(), 1); + assert_eq!(peaks[0].apex_idx, 0); + assert_eq!(peaks[0].start_idx, 0); + } + + #[test] + fn prominence_filter_suppresses_noise_flicker() { + // one real peak (apex 3, height 10) plus a tiny noise bump (height 1) + let p = [0.0, 0.0, 5.0, 10.0, 5.0, 0.0, 1.0, 0.0]; + // min_prominence_frac 0.2 -> floor 2.0 drops the height-1 bump + let peaks = enumerate_peaks(&p, 5, FR, 0.2); + assert_eq!(peaks.len(), 1); + assert_eq!(peaks[0].apex_idx, 3); + // with no prominence filter the noise bump is also returned + let peaks_all = enumerate_peaks(&p, 5, FR, 0.0); + assert!(peaks_all.iter().any(|pk| pk.apex_idx == 6)); + } + + #[test] + fn overlapping_maxima_collapse_to_stronger() { + // a shoulder (idx 2) on the side of a main peak (idx 4): the shoulder apex + // falls inside the main peak envelope and must not become a second peak + let p = [0.0, 3.0, 5.0, 7.0, 9.0, 6.0, 3.0, 0.0]; + let peaks = enumerate_peaks(&p, 5, FR, 0.1); + assert_eq!(peaks.len(), 1); + assert_eq!(peaks[0].apex_idx, 4); + } + + #[test] + fn determinism_ties_break_by_earlier_apex() { + // two identical peaks; equal area -> earlier apex ranks first, stable + let p = [0.0, 5.0, 0.0, 5.0, 0.0]; + let a = enumerate_peaks(&p, 5, FR, 0.1); + let b = enumerate_peaks(&p, 5, FR, 0.1); + assert_eq!(a, b); + assert_eq!(a[0].apex_idx, 1); + } +} From eb9da897447cc515fb9b14b2932a5efa6f91ad02 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 16:54:30 +0200 Subject: [PATCH 12/40] feat(sensitivity): candidate audit stage + `mumdia audit` (P0.3/P0.4) Non-destructive candidate-level observability. `mumdia audit` joins the artifact chain (library precursors -> psms -> competed -> scored) and writes candidate_audit.parquet with, per candidate, the stage flags (candidate_generated, traces_extracted, peak_generated, peak_selected, variant_selected, target_decoy_winner, passed_precursor_fdr, passed_peptide_fdr, reported) and the EARLIEST RejectionReason. Also writes .metrics.json with the identification- loss waterfall and stage recalls. Reruns no compute and changes no pipeline output, so it is safe on any completed search. Extraction losses collapse to NO_PEAK_GROUP at artifact resolution; an optional in-extract audit sidecar (.audit.parquet) refines them per candidate when present (hook in place, emitter is future work). Verified on the E. coli/HYE run: search_space=8,334,126 extracted=341,754 reported=8,568; waterfall NO_PEAK_GROUP=7,992,372 FAILED_PRECURSOR_FDR=332,120 FAILED_PEPTIDE_FDR=1,066 REPORTED=8,568. Unit tests cover the ladder assignment and entrapment labelling. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia/src/main.rs | 50 +++ rust/mumdia/crates/mumdia/src/stages/audit.rs | 378 ++++++++++++++++++ rust/mumdia/crates/mumdia/src/stages/mod.rs | 1 + 3 files changed, 429 insertions(+) create mode 100644 rust/mumdia/crates/mumdia/src/stages/audit.rs diff --git a/rust/mumdia/crates/mumdia/src/main.rs b/rust/mumdia/crates/mumdia/src/main.rs index 1294a2d..0ad691b 100644 --- a/rust/mumdia/crates/mumdia/src/main.rs +++ b/rust/mumdia/crates/mumdia/src/main.rs @@ -224,6 +224,35 @@ enum Cmd { Inspect { artifact: String, }, + /// Candidate audit: reconstruct per-candidate stage flags + earliest rejection + /// reason across the artifact chain and write candidate_audit.parquet + /// (sensitivity program, P0.3/P0.4). Non-destructive; reruns no compute. + Audit { + /// Library precursors parquet (the full candidate search space). + #[arg(long)] + library_precursors: String, + /// psms parquet from `extract`. + #[arg(long)] + psms: String, + /// competed parquet from `compete`. + #[arg(long)] + competed: String, + /// scored parquet from `rescore`. + #[arg(long)] + scored: String, + /// Output candidate_audit.parquet. + #[arg(long)] + out: String, + /// Precursor q-value threshold for passed_precursor_fdr / reported. + #[arg(long, default_value_t = 0.01)] + q: f64, + /// Run identifier stamped on every row. + #[arg(long, default_value = "run")] + run_id: String, + /// Optional protein substring marking entrapment candidates (e.g. _HUMAN). + #[arg(long, default_value = "")] + entrapment_substr: String, + }, /// Write peptides.tsv + proteins.tsv from a scored PSM table. Report { #[arg(long)] @@ -455,6 +484,27 @@ fn main() -> Result<()> { config_hash: &ch, })?; } + Cmd::Audit { + library_precursors, + psms, + competed, + scored, + out, + q, + run_id, + entrapment_substr, + } => { + stages::audit::run(stages::audit::AuditParams { + library_precursors: &library_precursors, + psms: &psms, + competed: &competed, + scored: &scored, + out: &out, + q_threshold: q, + run_id: &run_id, + entrapment_substr: &entrapment_substr, + })?; + } Cmd::Rescore { competed, out, config } => { let cfg = load_config(&config)?; let ch = mumdia_io::hash::blake3_str(&cfg.canonical_json()); diff --git a/rust/mumdia/crates/mumdia/src/stages/audit.rs b/rust/mumdia/crates/mumdia/src/stages/audit.rs new file mode 100644 index 0000000..1de2c82 --- /dev/null +++ b/rust/mumdia/crates/mumdia/src/stages/audit.rs @@ -0,0 +1,378 @@ +//! Candidate audit `mumdia audit` (sensitivity program, spec +//! `01_workflow_and_gap_analysis.md` §4, `02_sensitivity_diagnostic_plan.md` §5, +//! backlog P0.3 / P0.4). +//! +//! Non-destructive, post-hoc observability: reconstruct, for every candidate in +//! the search space (the library precursors), the pipeline stage flags and the +//! EARLIEST rejection reason, by tracking which `candidate_id`s survive across the +//! artifact chain library -> psms(extract) -> competed(compete) -> scored(rescore). +//! Writes `candidate_audit.parquet` and prints the identification-loss waterfall. +//! +//! This stage never re-runs compute and never changes any pipeline output, so it +//! is safe to run after any search. It answers "where was each candidate first +//! lost?" at the resolution the artifacts allow. The extraction stage collapses +//! "no fragment traces" and "traces but no accepted peak" into a single observable +//! event (a candidate is in `psms` or it is not); when a future in-extract audit +//! sidecar `.audit.parquet` is present, its precise per-candidate reason +//! refines the extract-stage bucket (see [`load_extract_reasons`]). + +use std::collections::{HashMap, HashSet}; +use std::time::Instant; + +use anyhow::{Context, Result}; +use mumdia_core::rejection::RejectionReason; +use mumdia_io::table::{write_table, Col, Table}; +use serde_json::json; +use tracing::info; + +pub struct AuditParams<'a> { + /// Library precursors parquet: the full candidate search space. + pub library_precursors: &'a str, + /// psms parquet written by `extract` (candidates that produced an accepted peak). + pub psms: &'a str, + /// competed parquet written by `compete` (survivors of within-group competition). + pub competed: &'a str, + /// scored parquet written by `rescore` (candidate_id + q_value). + pub scored: &'a str, + /// Output `candidate_audit.parquet`. + pub out: &'a str, + /// Precursor q-value threshold for `passed_precursor_fdr` / `reported`. + pub q_threshold: f64, + /// Run identifier stamped on every row. + pub run_id: &'a str, + /// Optional protein-substring marking entrapment candidates (e.g. `_HUMAN` for + /// an E. coli sample vs an HYE library). Empty = no entrapment labelling. + pub entrapment_substr: &'a str, +} + +/// Optional per-candidate extract-stage reason refinement written by a future +/// in-extract audit (`extract.emit_candidate_audit`). Returns a map +/// candidate_id -> reason code string. Absent file -> empty map (no refinement). +fn load_extract_reasons(psms_path: &str) -> HashMap { + let sidecar = format!("{psms_path}.audit.parquet"); + let mut out = HashMap::new(); + if let Ok(t) = Table::read(&sidecar) { + if let (Ok(cid), Ok(reason)) = (t.u32("candidate_id"), t.str("rejection_reason")) { + for (c, r) in cid.into_iter().zip(reason) { + out.insert(c, r); + } + } + } + out +} + +pub fn run(p: AuditParams) -> Result { + let t0 = Instant::now(); + + // Search space = all library precursors. + let lib = Table::read(p.library_precursors) + .with_context(|| format!("audit: reading library precursors {}", p.library_precursors))?; + let cid = lib.u32("candidate_id")?; + let pform = lib.str("peptidoform")?; + let charge = lib.i32("charge")?; + let label = lib.str("label")?; + let protein = lib.str("protein")?; + let n = cid.len(); + + // Survivor sets keyed by candidate_id from each downstream artifact. + let extracted: HashSet = Table::read(p.psms)?.u32("candidate_id")?.into_iter().collect(); + let competed: HashSet = Table::read(p.competed)? + .u32("candidate_id")? + .into_iter() + .collect(); + let scored_t = Table::read(p.scored)?; + let scored_cid = scored_t.u32("candidate_id")?; + let scored_q = scored_t.f64("q_value")?; + // peptide-level q is optional (only present in some scored schemas). + let scored_pep_q = scored_t.f64("peptide_q_value").ok(); + let mut q_by_cid: HashMap = HashMap::with_capacity(scored_cid.len()); + let mut pepq_by_cid: HashMap = HashMap::new(); + for (i, c) in scored_cid.iter().enumerate() { + q_by_cid.insert(*c, scored_q[i]); + if let Some(pq) = &scored_pep_q { + pepq_by_cid.insert(*c, pq[i]); + } + } + let extract_reasons = load_extract_reasons(p.psms); + + // Output columns. + let mut run_c: Vec = Vec::with_capacity(n); + let mut prec_c: Vec = Vec::with_capacity(n); + let mut seq_c: Vec = Vec::with_capacity(n); + let mut chg_c: Vec = Vec::with_capacity(n); + let mut td_c: Vec = Vec::with_capacity(n); + let mut entrap_c: Vec = Vec::with_capacity(n); + let mut f_generated: Vec = Vec::with_capacity(n); + let mut f_traces: Vec = Vec::with_capacity(n); + let mut f_peak: Vec = Vec::with_capacity(n); + let mut f_peak_sel: Vec = Vec::with_capacity(n); + let mut f_variant: Vec = Vec::with_capacity(n); + let mut f_td_winner: Vec = Vec::with_capacity(n); + let mut f_prec_fdr: Vec = Vec::with_capacity(n); + let mut f_pep_fdr: Vec = Vec::with_capacity(n); + let mut f_reported: Vec = Vec::with_capacity(n); + let mut reason_c: Vec = Vec::with_capacity(n); + + // Waterfall counters. + let mut waterfall: HashMap<&'static str, u64> = HashMap::new(); + + for i in 0..n { + let c = cid[i]; + let is_decoy = label[i] == "decoy"; + let traces = extracted.contains(&c); + let variant = competed.contains(&c); + let q = q_by_cid.get(&c).copied(); + let in_scored = q.is_some(); + let passed_prec = q.map(|v| v <= p.q_threshold).unwrap_or(false); + let passed_pep = pepq_by_cid + .get(&c) + .map(|v| *v <= p.q_threshold) + .unwrap_or(passed_prec); // fall back to precursor gate when no peptide-q + + // Earliest rejection reason along the ladder. + let reason: RejectionReason = if !traces { + // Extraction produced no accepted peak for this candidate. Refine with + // the in-extract audit sidecar if present; otherwise the generic bucket. + match extract_reasons.get(&c).map(String::as_str) { + Some("NO_FRAGMENT_TRACES") => RejectionReason::NoFragmentTraces, + Some("NO_VALID_FRAGMENTS") => RejectionReason::NoValidFragments, + Some("PEAK_NOT_SELECTED") => RejectionReason::PeakNotSelected, + Some("RT_PRUNED") => RejectionReason::RtPruned, + Some("WRONG_ISOLATION_WINDOW") => RejectionReason::WrongIsolationWindow, + _ => RejectionReason::NoPeakGroup, + } + } else if !variant { + if is_decoy { + RejectionReason::OutcompetedByDecoy + } else { + RejectionReason::OutcompetedByTarget + } + } else if !passed_prec { + RejectionReason::FailedPrecursorFdr + } else if !passed_pep { + RejectionReason::FailedPeptideFdr + } else { + RejectionReason::Reported + }; + *waterfall.entry(reason.code()).or_insert(0) += 1; + + run_c.push(p.run_id.to_string()); + prec_c.push(c); + seq_c.push(pform[i].clone()); + chg_c.push(charge[i]); + td_c.push(label[i].clone()); + entrap_c.push(!p.entrapment_substr.is_empty() && protein[i].contains(p.entrapment_substr)); + f_generated.push(true); // in the search space by construction + f_traces.push(traces); + f_peak.push(traces); // artifact resolution: an accepted peak == present in psms + f_peak_sel.push(traces); + f_variant.push(variant); + f_td_winner.push(in_scored); + f_prec_fdr.push(passed_prec); + f_pep_fdr.push(passed_pep && passed_prec); + f_reported.push(passed_prec); + reason_c.push(reason.code().to_string()); + } + + let rows = write_table( + p.out, + vec![ + Col::Str("run_id".into(), run_c), + Col::U32("precursor_id".into(), prec_c), + Col::Str("modified_sequence".into(), seq_c), + Col::I32("charge".into(), chg_c), + Col::Str("target_decoy_label".into(), td_c), + Col::Bool("entrapment_label".into(), entrap_c), + Col::Bool("candidate_generated".into(), f_generated), + Col::Bool("traces_extracted".into(), f_traces), + Col::Bool("peak_generated".into(), f_peak), + Col::Bool("peak_selected".into(), f_peak_sel), + Col::Bool("variant_selected".into(), f_variant), + Col::Bool("target_decoy_winner".into(), f_td_winner), + Col::Bool("passed_precursor_fdr".into(), f_prec_fdr), + Col::Bool("passed_peptide_fdr".into(), f_pep_fdr), + Col::Bool("reported".into(), f_reported), + Col::Str("rejection_reason".into(), reason_c), + ], + )?; + + // Stage-level metrics + waterfall (P0.4), written next to the audit table. + let n_extracted = extracted.len() as u64; + let n_competed = competed.len() as u64; + let n_reported = *waterfall.get("REPORTED").unwrap_or(&0); + let metrics = json!({ + "run_id": p.run_id, + "q_threshold": p.q_threshold, + "search_space": n, + "extracted": n_extracted, + "competed": n_competed, + "reported": n_reported, + "trace_recall": n_extracted as f64 / (n.max(1) as f64), + "waterfall": waterfall.iter().map(|(k, v)| (k.to_string(), *v)).collect::>(), + }); + mumdia_io::json::write_json(&format!("{}.metrics.json", p.out), &metrics)?; + + let elapsed = t0.elapsed().as_millis(); + let mut wf: Vec<(&&str, &u64)> = waterfall.iter().collect(); + wf.sort_by_key(|(_, v)| std::cmp::Reverse(**v)); + let wf_str: String = wf + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(" "); + info!( + search_space = n, + extracted = n_extracted, + competed = n_competed, + reported = n_reported, + elapsed_ms = elapsed, + "audit: done" + ); + info!("audit waterfall: {wf_str}"); + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(name: &str) -> String { + let dir = std::env::temp_dir().join("mumdia_audit_test"); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name).to_str().unwrap().to_string() + } + + fn write_lib(path: &str, cids: &[u32], labels: &[&str]) { + write_table( + path, + vec![ + Col::U32("candidate_id".into(), cids.to_vec()), + Col::Str( + "peptidoform".into(), + cids.iter().map(|c| format!("PEP{c}")).collect(), + ), + Col::I32("charge".into(), cids.iter().map(|_| 2i32).collect()), + Col::Str("label".into(), labels.iter().map(|s| s.to_string()).collect()), + Col::Str( + "protein".into(), + cids.iter().map(|_| "sp|X|ECOLI".to_string()).collect(), + ), + ], + ) + .unwrap(); + } + fn write_cid_only(path: &str, cids: &[u32]) { + write_table(path, vec![Col::U32("candidate_id".into(), cids.to_vec())]).unwrap(); + } + + #[test] + fn waterfall_assigns_earliest_loss_per_candidate() { + // 6 candidates in the library. Fates: + // 1 target -> reported (extract+compete+scored q<=0.01) + // 2 target -> failed precursor (scored q=0.5) + // 3 target -> outcompeted (extract yes, compete no) + // 4 target -> no peak group (not extracted) + // 5 decoy -> outcompeted decoy (extract yes, compete no) + // 6 decoy -> no peak group (not extracted) + let lib = tmp("lib.parquet"); + let psms = tmp("psms.parquet"); + let comp = tmp("comp.parquet"); + let scored = tmp("scored.parquet"); + let out = tmp("candidate_audit.parquet"); + write_lib( + &lib, + &[1, 2, 3, 4, 5, 6], + &["target", "target", "target", "target", "decoy", "decoy"], + ); + write_cid_only(&psms, &[1, 2, 3, 5]); // 4,6 never extracted + write_cid_only(&comp, &[1, 2]); // 3,5 outcompeted + write_table( + &scored, + vec![ + Col::U32("candidate_id".into(), vec![1, 2]), + Col::F64("q_value".into(), vec![0.001, 0.5]), + ], + ) + .unwrap(); + + let rows = run(AuditParams { + library_precursors: &lib, + psms: &psms, + competed: &comp, + scored: &scored, + out: &out, + q_threshold: 0.01, + run_id: "t", + entrapment_substr: "", + }) + .unwrap(); + assert_eq!(rows, 6); + + let a = Table::read(&out).unwrap(); + let cid = a.u32("precursor_id").unwrap(); + let reason = a.str("rejection_reason").unwrap(); + let reported = a.bool("reported").unwrap(); + let by: std::collections::HashMap = cid + .iter() + .cloned() + .zip(reason.into_iter().zip(reported)) + .map(|(c, (r, rep))| (c, (r, rep))) + .collect(); + assert_eq!(by[&1].0, "REPORTED"); + assert!(by[&1].1); + assert_eq!(by[&2].0, "FAILED_PRECURSOR_FDR"); + assert!(!by[&2].1); + assert_eq!(by[&3].0, "OUTCOMPETED_BY_TARGET"); + assert_eq!(by[&4].0, "NO_PEAK_GROUP"); + assert_eq!(by[&5].0, "OUTCOMPETED_BY_DECOY"); + assert_eq!(by[&6].0, "NO_PEAK_GROUP"); + } + + #[test] + fn entrapment_label_from_protein_substring() { + let lib = tmp("lib2.parquet"); + // one ECOLI, one HUMAN protein + write_table( + &lib, + vec![ + Col::U32("candidate_id".into(), vec![1, 2]), + Col::Str("peptidoform".into(), vec!["A".into(), "B".into()]), + Col::I32("charge".into(), vec![2, 3]), + Col::Str("label".into(), vec!["target".into(), "target".into()]), + Col::Str( + "protein".into(), + vec!["sp|X|EFTU_ECOLI".into(), "sp|Y|ALBU_HUMAN".into()], + ), + ], + ) + .unwrap(); + let psms = tmp("psms2.parquet"); + let comp = tmp("comp2.parquet"); + let scored = tmp("scored2.parquet"); + let out = tmp("audit2.parquet"); + write_cid_only(&psms, &[1, 2]); + write_cid_only(&comp, &[1, 2]); + write_table( + &scored, + vec![ + Col::U32("candidate_id".into(), vec![1, 2]), + Col::F64("q_value".into(), vec![0.001, 0.001]), + ], + ) + .unwrap(); + run(AuditParams { + library_precursors: &lib, + psms: &psms, + competed: &comp, + scored: &scored, + out: &out, + q_threshold: 0.01, + run_id: "t", + entrapment_substr: "_HUMAN", + }) + .unwrap(); + let a = Table::read(&out).unwrap(); + let entrap = a.bool("entrapment_label").unwrap(); + assert_eq!(entrap, vec![false, true]); + } +} diff --git a/rust/mumdia/crates/mumdia/src/stages/mod.rs b/rust/mumdia/crates/mumdia/src/stages/mod.rs index ffbe611..311a6e1 100644 --- a/rust/mumdia/crates/mumdia/src/stages/mod.rs +++ b/rust/mumdia/crates/mumdia/src/stages/mod.rs @@ -2,6 +2,7 @@ //! inputs and writing declared Parquet + a report (PLAN.md Section 3.5). pub mod align; +pub mod audit; pub mod compete; pub mod convert; pub mod digest; From de5ae2b7482be14ec13e8af2915918debab59b24 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 17:56:24 +0200 Subject: [PATCH 13/40] feat(sensitivity): configurable competition modes in compete (P2.4) CompeteConfig.mode drives within-group resolution via a pure, unit-tested resolve_competition(): - winner_take_all (default): unchanged legacy behaviour (keep top prelim per group; ties -> smallest index, bit-identical to the previous HashMap winner). - none / features_only: preserve every candidate (FDR/rescoring arbitrates); the sensitivity program's "preserve evidence until a calibrated decision". - unique_evidence: keep a loser with >= unique_evidence_min_fragments unique fragments (unique_fragment_count, else n_matched_fragments*(1-contested_frac), else n_matched_fragments; winner-take-all fallback when none present). - margin_gated: remove a loser only when winner_prelim - loser_prelim >= margin. Target/decoy label stays in the group key in every mode, so a target never competes its own decoy (null preserved). Optional emit_competition_audit writes .compete_audit.parquet (loser, winner, scores, OUTCOMPETED_BY_* reason). ArtifactReport now records mode + removed count. 7 new unit tests. Co-Authored-By: Claude Opus 4.8 --- .../crates/mumdia/src/stages/compete.rs | 296 ++++++++++++++++-- 1 file changed, 272 insertions(+), 24 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/stages/compete.rs b/rust/mumdia/crates/mumdia/src/stages/compete.rs index 77ee9dc..d7b9a7b 100644 --- a/rust/mumdia/crates/mumdia/src/stages/compete.rs +++ b/rust/mumdia/crates/mumdia/src/stages/compete.rs @@ -8,12 +8,13 @@ use std::collections::HashMap; use std::time::Instant; use anyhow::Result; -use mumdia_core::config::{CompeteConfig, CompeteGroupBy}; +use mumdia_core::config::{CompeteConfig, CompeteGroupBy, CompetitionMode}; +use mumdia_core::rejection::RejectionReason; use mumdia_core::schema::artifact; use mumdia_io::report::ArtifactReport; use mumdia_io::table::{write_table, Col, Table}; use serde_json::json; -use tracing::info; +use tracing::{info, warn}; use crate::stages::features::FeatureSchema; @@ -60,16 +61,15 @@ pub fn run(p: CompeteParams) -> Result { Vec::new() }; - // Winner per competition group. The label is part of the key so a target is - // NOT competed against its own decoy: the decoy population must survive for - // the rescorer/FDR to have a valid null (otherwise decoys are depleted and - // FDR is badly underestimated). Competition only removes redundant charge/ - // modification variants within targets and within decoys. - // Key is a fixed-size tuple (base_peptide_id, label_code, bucket) instead of a - // freshly-allocated String per PSM. The label is mapped to a small integer - // code so it stays part of the key exactly as before; Precursor grouping uses - // a constant bucket (0) so its equivalence classes are unchanged. - let mut winner: HashMap<(u32, u8, i64), usize> = HashMap::new(); + // Competition group members. The label is part of the key so a target is NOT + // competed against its own decoy: the decoy population must survive for the + // rescorer/FDR to have a valid null (otherwise decoys are depleted and FDR is + // badly underestimated). Competition only arbitrates redundant charge/mod + // variants within targets and within decoys. + // Key is a fixed-size tuple (base-or-pform id, label_code, bucket) instead of a + // freshly-allocated String per PSM. Precursor grouping uses a constant bucket + // (0) so its equivalence classes are unchanged. + let mut groups: HashMap<(u32, u8, i64), Vec> = HashMap::new(); for i in 0..t.nrows { let label_code = match label[i].as_str() { "target" => 0u8, @@ -89,17 +89,30 @@ pub fn run(p: CompeteParams) -> Result { (pform_id[i], label_code, c) } }; - winner - .entry(key) - .and_modify(|w| { - if prelim[i] > prelim[*w] { - *w = i; - } - }) - .or_insert(i); + groups.entry(key).or_default().push(i); } - let mut keep: Vec = winner.into_values().collect(); - keep.sort_unstable(); + + // Per-candidate unique-fragment evidence for the `unique_evidence` mode. Prefers + // an explicit `unique_fragment_count` feature; otherwise approximates it as + // matched-fragment count discounted by the contested fraction; None if neither + // is available (mode then falls back to winner-take-all). + let unique_ev = unique_evidence(&t); + if matches!(p.cfg.mode, CompetitionMode::UniqueEvidence) && unique_ev.is_none() { + warn!( + "compete mode=unique_evidence: no unique_fragment_count / \ + (n_matched_fragments, contested_frac) columns; falling back to winner-take-all" + ); + } + + // Resolve each group per competition mode (pure function; unit tested below). + let (keep, removed) = resolve_competition( + &groups, + &prelim, + p.cfg.mode, + p.cfg.margin, + p.cfg.unique_evidence_min_fragments, + unique_ev.as_deref(), + ); let sel = |v: &[f64]| keep.iter().map(|&i| v[i]).collect::>(); let mut cols: Vec = vec![ @@ -119,10 +132,58 @@ pub fn run(p: CompeteParams) -> Result { // Carry the feature schema forward for rescore. mumdia_io::json::write_json(&format!("{}.schema.json", p.out), &schema)?; + // Optional per-removal competition audit (spec 04 §2): every candidate removed + // by within-group competition, with its winner and removal reason. Within-label + // competition, so the sibling that outcompeted a loser shares its label. + if p.cfg.emit_competition_audit { + let reason_of = |i: usize| { + if label[i] == "decoy" { + RejectionReason::OutcompetedByDecoy + } else { + RejectionReason::OutcompetedByTarget + } + }; + let audit = format!("{}.compete_audit.parquet", p.out); + write_table( + &audit, + vec![ + Col::U32( + "candidate_id".into(), + removed.iter().map(|&(m, _)| cid[m]).collect(), + ), + Col::Str( + "label".into(), + removed.iter().map(|&(m, _)| label[m].clone()).collect(), + ), + Col::Str( + "peptidoform".into(), + removed.iter().map(|&(m, _)| pform[m].clone()).collect(), + ), + Col::U32( + "winner_candidate_id".into(), + removed.iter().map(|&(_, w)| cid[w]).collect(), + ), + Col::F64( + "loser_prelim".into(), + removed.iter().map(|&(m, _)| prelim[m]).collect(), + ), + Col::F64( + "winner_prelim".into(), + removed.iter().map(|&(_, w)| prelim[w]).collect(), + ), + Col::Str( + "rejection_reason".into(), + removed.iter().map(|&(m, _)| reason_of(m).code().to_string()).collect(), + ), + ], + )?; + } + let elapsed = t0.elapsed().as_millis(); let mut stats = std::collections::BTreeMap::new(); stats.insert("input_rows".to_string(), json!(t.nrows)); stats.insert("kept".to_string(), json!(rows)); + stats.insert("removed".to_string(), json!(removed.len())); ArtifactReport { logical_name: artifact::PSMS_COMPETED.0.to_string(), schema_name: artifact::PSMS_COMPETED.0.to_string(), @@ -130,13 +191,200 @@ pub fn run(p: CompeteParams) -> Result { stage: "compete".to_string(), rows, content_hash: mumdia_io::hash::blake3_file(p.out)?, - params: json!({"group_by": format!("{:?}", p.cfg.group_by)}), + params: json!({ + "group_by": format!("{:?}", p.cfg.group_by), + "mode": format!("{:?}", p.cfg.mode), + }), stats, model_identity: None, elapsed_ms: elapsed, } .write_for(p.out)?; - info!(input = t.nrows, kept = rows, elapsed_ms = elapsed, "compete: done"); + info!( + input = t.nrows, + kept = rows, + removed = removed.len(), + mode = ?p.cfg.mode, + "compete: done" + ); Ok(rows) } + +/// Per-candidate unique-fragment evidence for `CompetitionMode::UniqueEvidence`. +/// Prefers an explicit `unique_fragment_count` column; otherwise approximates it as +/// `n_matched_fragments * (1 - contested_frac)` (contested-discounted matched +/// count); falls back to raw `n_matched_fragments`; `None` if none are present. +fn unique_evidence(t: &Table) -> Option> { + if let Some(u) = col_f64(t, "unique_fragment_count") { + return Some(u); + } + let nm = col_f64(t, "n_matched_fragments")?; + match col_f64(t, "contested_frac") { + Some(cf) => Some( + nm.iter() + .zip(cf) + .map(|(n, c)| n * (1.0 - c).clamp(0.0, 1.0)) + .collect(), + ), + None => Some(nm), + } +} + +/// Read a numeric column as f64, accepting an f64 or i32 encoding. +fn col_f64(t: &Table, name: &str) -> Option> { + t.f64(name).ok().or_else(|| { + t.i32(name) + .ok() + .map(|v| v.into_iter().map(|x| x as f64).collect()) + }) +} + +/// Resolve within-group competition per [`CompetitionMode`]. Returns the +/// sorted-unique kept row indices and the `(loser, winner)` removal pairs. +/// Deterministic: groups are visited in sorted key order; the winner is the +/// highest `prelim` (ties broken by smallest index). +fn resolve_competition( + groups: &HashMap<(u32, u8, i64), Vec>, + prelim: &[f64], + mode: CompetitionMode, + margin: f64, + unique_min: usize, + unique_ev: Option<&[f64]>, +) -> (Vec, Vec<(usize, usize)>) { + use std::cmp::Ordering::Equal; + let mut group_keys: Vec<&(u32, u8, i64)> = groups.keys().collect(); + group_keys.sort_unstable(); + let mut keep: Vec = Vec::new(); + let mut removed: Vec<(usize, usize)> = Vec::new(); + for gk in group_keys { + let members = &groups[gk]; + let win = *members + .iter() + .min_by(|&&a, &&b| prelim[b].partial_cmp(&prelim[a]).unwrap_or(Equal).then(a.cmp(&b))) + .unwrap(); + match mode { + CompetitionMode::None | CompetitionMode::FeaturesOnly => { + keep.extend(members.iter().copied()); + } + CompetitionMode::WinnerTakeAll => { + keep.push(win); + removed.extend(members.iter().copied().filter(|&m| m != win).map(|m| (m, win))); + } + CompetitionMode::UniqueEvidence => { + keep.push(win); + let thr = unique_min as f64; + for &m in members { + if m == win { + continue; + } + if unique_ev.map(|u| u[m] >= thr).unwrap_or(false) { + keep.push(m); + } else { + removed.push((m, win)); + } + } + } + CompetitionMode::MarginGated => { + keep.push(win); + for &m in members { + if m == win { + continue; + } + if prelim[win] - prelim[m] >= margin { + removed.push((m, win)); + } else { + keep.push(m); + } + } + } + } + } + keep.sort_unstable(); + keep.dedup(); + (keep, removed) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn one_group(members: Vec) -> HashMap<(u32, u8, i64), Vec> { + let mut g = HashMap::new(); + g.insert((0u32, 0u8, 0i64), members); + g + } + + #[test] + fn winner_take_all_keeps_only_winner() { + let g = one_group(vec![0, 1, 2]); + let prelim = [0.1, 0.9, 0.5]; + let (keep, removed) = + resolve_competition(&g, &prelim, CompetitionMode::WinnerTakeAll, 0.0, 2, None); + assert_eq!(keep, vec![1]); + assert_eq!(removed.len(), 2); + } + + #[test] + fn none_and_features_only_keep_all() { + let g = one_group(vec![0, 1, 2]); + let prelim = [0.1, 0.9, 0.5]; + for mode in [CompetitionMode::None, CompetitionMode::FeaturesOnly] { + let (keep, removed) = resolve_competition(&g, &prelim, mode, 0.0, 2, None); + assert_eq!(keep, vec![0, 1, 2]); + assert!(removed.is_empty()); + } + } + + #[test] + fn margin_gated_keeps_close_losers_removes_distant() { + // winner idx1 (0.9); idx2 (0.85) within margin 0.1 -> kept; idx0 removed + let g = one_group(vec![0, 1, 2]); + let prelim = [0.1, 0.9, 0.85]; + let (keep, removed) = + resolve_competition(&g, &prelim, CompetitionMode::MarginGated, 0.1, 2, None); + assert_eq!(keep, vec![1, 2]); + assert_eq!(removed, vec![(0, 1)]); + } + + #[test] + fn unique_evidence_keeps_losers_with_enough_evidence() { + // winner idx1; idx0 unique 3 (>=2) kept; idx2 unique 1 removed + let g = one_group(vec![0, 1, 2]); + let prelim = [0.1, 0.9, 0.5]; + let ev = [3.0, 5.0, 1.0]; + let (keep, removed) = + resolve_competition(&g, &prelim, CompetitionMode::UniqueEvidence, 0.0, 2, Some(&ev)); + assert_eq!(keep, vec![0, 1]); + assert_eq!(removed, vec![(2, 1)]); + } + + #[test] + fn unique_evidence_without_data_falls_back_to_winner_take_all() { + let g = one_group(vec![0, 1, 2]); + let prelim = [0.1, 0.9, 0.5]; + let (keep, _) = + resolve_competition(&g, &prelim, CompetitionMode::UniqueEvidence, 0.0, 2, None); + assert_eq!(keep, vec![1]); + } + + #[test] + fn winner_take_all_is_deterministic_across_groups() { + let mut g = HashMap::new(); + g.insert((0u32, 0u8, 0i64), vec![0, 1]); + g.insert((1u32, 1u8, 0i64), vec![2, 3]); + let prelim = [0.2, 0.8, 0.9, 0.3]; + let (keep, _) = + resolve_competition(&g, &prelim, CompetitionMode::WinnerTakeAll, 0.0, 2, None); + assert_eq!(keep, vec![1, 2]); // winners of each group, sorted + } + + #[test] + fn winner_tie_breaks_to_smallest_index() { + let g = one_group(vec![0, 1, 2]); + let prelim = [0.9, 0.9, 0.1]; // tie between idx0 and idx1 + let (keep, _) = + resolve_competition(&g, &prelim, CompetitionMode::WinnerTakeAll, 0.0, 2, None); + assert_eq!(keep, vec![0]); + } +} From 22719532d818f47edc3220d61594c798133706b4 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 18:00:19 +0200 Subject: [PATCH 14/40] docs(sensitivity): architecture map, feature registry, status log, next steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ARCHITECTURE_MAP.md: 12-stage pipeline map with symbols/line numbers, an exhaustive "where candidates are dropped" table (~40 points -> reason codes), hook sites for each work item, and code-vs-spec differences (compete is within-label winner-take-all before rescore; target-decoy is q-values not a compete stage; one apex per candidate). - FEATURE_REGISTRY.md + feature_registry.yaml: all 360 features across 17 families with level/direction/source, a leakage audit vs spec 03 §3, and a gap table vs spec 03 §8. - IMPLEMENTATION_STATUS.md: baseline, assumptions, per-task status, tests, the candidate-audit waterfall on real data. - NEXT_STEPS.md: prioritized remaining work with exact hook sites (top-K extract wiring, in-extract audit emitter, competition-after-rescore, conflict features, new feature families, two-pass calibration, empirical-FDP acceptance loop). Co-Authored-By: Claude Opus 4.8 --- feature_registry.yaml | 1816 +++++++++++++++++ .../01_workflow_and_gap_analysis.md | 224 ++ sensitivity_plan/03_feature_evaluation.md | 439 ++++ .../04_peak_and_peptide_competition.md | 305 +++ sensitivity_plan/05_experiment_matrix.md | 199 ++ .../06_agent_implementation_backlog.md | 372 ++++ sensitivity_plan/ARCHITECTURE_MAP.md | 576 ++++++ sensitivity_plan/FEATURE_REGISTRY.md | 630 ++++++ sensitivity_plan/IMPLEMENTATION_STATUS.md | 90 + sensitivity_plan/NEXT_STEPS.md | 107 + sensitivity_plan/README.md | 81 + 11 files changed, 4839 insertions(+) create mode 100644 feature_registry.yaml create mode 100644 sensitivity_plan/01_workflow_and_gap_analysis.md create mode 100644 sensitivity_plan/03_feature_evaluation.md create mode 100644 sensitivity_plan/04_peak_and_peptide_competition.md create mode 100644 sensitivity_plan/05_experiment_matrix.md create mode 100644 sensitivity_plan/06_agent_implementation_backlog.md create mode 100644 sensitivity_plan/ARCHITECTURE_MAP.md create mode 100644 sensitivity_plan/FEATURE_REGISTRY.md create mode 100644 sensitivity_plan/IMPLEMENTATION_STATUS.md create mode 100644 sensitivity_plan/NEXT_STEPS.md create mode 100644 sensitivity_plan/README.md diff --git a/feature_registry.yaml b/feature_registry.yaml new file mode 100644 index 0000000..32923c8 --- /dev/null +++ b/feature_registry.yaml @@ -0,0 +1,1816 @@ +# MuMDIA feature registry (machine-readable) +# Auto-generated from _feat_inventory.json (360 features, 17 families). +# Schema per feature: family, level, direction, source_file. +# direction: higher_better | lower_better | neutral | ? +# level: fragment | peak | precursor | candidate | run | ? +# NOTE: 4 names collide with reserved Minimal/Rich names and are dropped +# from the scored schema; the dropped variant is keyed '@' +# and carries 'dropped_collision: true'. The active (reserved) row keeps +# the plain name. +total_features: 360 +n_families: 17 +features: + "rt_error_abs": + family: "minimal" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "rt_error_rel": + family: "minimal" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_matched_fragments": + family: "minimal" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "coelution_run": + family: "minimal" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "log_apex_intensity": + family: "minimal" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "frag_corr": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "frag_cosine": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "spectral_angle": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "coelution_mean": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "coelution_best": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_coelution_above": + family: "minimal" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "charge": + family: "minimal" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "peptide_length": + family: "minimal" + level: "candidate" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_proteins": + family: "minimal" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "library_norm_manhattan": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "library_rmsd": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "xcorr_coelution": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "xcorr_shape": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "sum_b_intensity": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "sum_y_intensity": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "diff_by_intensity": + family: "rich" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_b_ions": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_y_ions": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "weighted_mass_error": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "mean_mass_error": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "isotope_corr": + family: "rich" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "ms1_isom1_ratio": + family: "rich" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "log_mono_ms1": + family: "rich" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "has_ms1": + family: "rich" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "log_sn": + family: "rich" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_observations": + family: "rich" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "base_width_rt": + family: "rich" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "seed_score": + family: "rich" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "seed_identified": + family: "rich" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "matched_fraction": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "profile_cos": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "ref_corr": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "best_ref_corr": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "low_frag_coel": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "evidence": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "contrast_min": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "resid_corr": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "coel_clean": + family: "rich" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "shadow_frac": + family: "rich" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "peak_contested_frac": + family: "extended-extra (psms-derived)" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "n_charge_states": + family: "extended-extra (cross-candidate)" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "charge_multi_flag": + family: "extended-extra (cross-candidate)" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "cross_charge_intensity_log": + family: "extended-extra (cross-candidate)" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" + "spectrum_cosine_matched": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectrum_cosine_sqrt": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectrum_cosine_log": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_angle@similarity": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + dropped_collision: true + "spectral_angle_sqrt": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_angle_matched": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "pearson_intensity_matched": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "pearson_intensity_log": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spearman_intensity": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spearman_intensity_matched": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "kendall_tau_intensity": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "dot_product_raw": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "dot_product_norm": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "library_recall_intensity": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "manhattan_sim": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "manhattan_sqrt": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "rmsd_norm": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "mae_norm": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "mse_log": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "mae_weighted_pred": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "abs_diff_q3": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "max_positive_residual": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "chebyshev_dist": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "minkowski_p3": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "bray_curtis": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "bray_curtis_sqrt": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "canberra": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "canberra_matched": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "wave_hedges": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "chi_square_pearson": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "chi_square_symmetric": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "divergence_distance": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "bhattacharyya_coef": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "hellinger": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "squared_chord": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "harmonic_mean_sim": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "jaccard_presence": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "dice_presence": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "intensity_weighted_pearson": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "regression_slope": + family: "similarity" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "gini_diff": + family: "similarity" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "wasserstein_mz": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "footrule_norm": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "rank_overlap_top3": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "top1_frag_match": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "top1_predicted_observed": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "frac_top3_predicted_observed": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "count_strong_predicted_absent": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "frac_predicted_absent": + family: "similarity" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "pearson_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_angle_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_fullwindow": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "stein_scott_weighted_dot": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "log_dot_product": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_log_evidence": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "scribe_score": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "log_dot_product_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_log_evidence_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "scribe_score_area": + family: "similarity" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_high_ordinal": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_robust_trim1": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_robust_trim2": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "cosine_robust_trim3": + family: "similarity" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + "spectral_entropy_similarity": + family: "entropy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "weighted_spectral_entropy_similarity": + family: "entropy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "spectral_entropy_similarity_sqrt": + family: "entropy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "spectral_entropy_similarity_topk": + family: "entropy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "spectral_entropy_similarity_area": + family: "entropy" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "jensen_shannon_divergence": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "jeffreys_divergence": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "kl_obs_pred": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "kl_pred_obs": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "cross_entropy_obs_pred": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "obs_spectrum_entropy": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "pred_spectrum_entropy": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "entropy_diff": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "entropy_ratio": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "obs_normalized_entropy": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "normalized_entropy_diff": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "residual_spectrum_entropy": + family: "entropy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "entropy_weight_obs": + family: "entropy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" + "frag_ref_corr_mean": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_obsweighted": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_min": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_std": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_sq_mean": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_topk_weighted": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "n_frag_ref_corr_above_0_9": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frac_frag_ref_corr_above_0_8": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_ref_corr_mean_full": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "full_vs_peak_corr_gain": + family: "coelution" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_weighted": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_min": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_median": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_std": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_frac_negative": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_hi": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "pairwise_coelution_lo": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "coelution_hi_lo_contrast": + family: "coelution" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "coelution_corr_entropy": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_shape_mean": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_shape_min": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_shape_std": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_mean_abs": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_std": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_iqr": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_frac_zero": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_max_abs": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "xcorr_lag_entropy": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "ref_xcorr_lag_mean": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "ref_xcorr_shape_mean": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "observed_sum_vs_template_corr": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_loo_ref_corr_mean": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frag_loo_ref_corr_min": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "frac_frags_apex_aligned": + family: "coelution" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "top3_frag_ref_corr": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "by_cross_coelution": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "by_cross_lag_mean": + family: "coelution" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "charge_cross_coelution": + family: "coelution" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" + "explained_variance_ref": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "profile_residual_fraction": + family: "interference" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "n_interfered_fragments": + family: "interference" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "corrected_vs_raw_cos": + family: "interference" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "corrected_vs_raw_ratio": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "ifs_removed_count": + family: "interference" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "ifs_removed_intensity_frac": + family: "interference" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "ifs_corr_gain": + family: "interference" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "ifs_retained_frac": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "matched_frac_after_ifs": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "peak_to_full_area_ratio_profile": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "peak_to_full_area_ratio_frag_mean": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "peak_to_full_area_ratio_weighted": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "out_of_peak_intensity_frac": + family: "interference" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "profile_corr_full_vs_peak_delta": + family: "interference" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "frac_frag_ref_corr_below_0_5": + family: "interference" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "explained_apex_intensity_frac": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "apex_purity": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "interference_apex_residual_fraction": + family: "interference" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "dominant_frag_ref_corr": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "explained_variance_ratio": + family: "interference" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "second_component_fraction": + family: "interference" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "profile_second_peak_ratio": + family: "interference" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "n_competing_peaks_in_window": + family: "interference" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "matched_pred_intensity_fraction": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "top_pred_frag_matched": + family: "interference" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" + "gaussian_fit_r2": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "gaussian_cosine": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "emg_fit_improvement": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_prominence": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "profile_peak_snr": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "fwhm_seconds": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "fwhm_to_window_ratio": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "width_at_10pct": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "width_ratio_10_50": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "hwhm_asymmetry": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "tailing_factor_usp": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "asymmetry_factor_10pct": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_sharpness": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_curvature": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_to_boundary_ratio": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_dominance": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "zigzag_index": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "jaggedness": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "roughness_2nd_deriv": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "n_local_maxima": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "modality": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "rt_skewness": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "rt_excess_kurtosis": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "rt_std_seconds": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "mean_mode_offset": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "fraction_area_within_fwhm": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "triangle_area_similarity": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "baseline_fraction": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "peak_completeness": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "apex_centering_offset": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "intensity_score": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "total_xic_log": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_fwhm_cv": + family: "chromatographic" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_fwhm_mean": + family: "chromatographic" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_apex_rt_dispersion": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_apex_rt_dispersion_weighted": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_apex_offset_from_profile_mean": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_gaussianity_mean": + family: "chromatographic" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_gaussianity_weighted": + family: "chromatographic" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "frag_zigzag_mean": + family: "chromatographic" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "sumtrace_unweighted_gaussian_r2": + family: "chromatographic" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "reference_profile_rt_entropy_peak": + family: "chromatographic" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "reference_profile_rt_entropy_ratio": + family: "chromatographic" + level: "peak" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" + "median_abs_frag_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "signed_mean_frag_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "ppm_std": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "ppm_iqr": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "ppm_range": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "max_abs_frag_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "intensity_weighted_abs_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "intensity_weighted_signed_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "intensity_weighted_ppm_std": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "lib_weighted_abs_ppm": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "frac_frag_within_half_tol": + family: "mass_accuracy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "high_ppm_intensity_frac": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "ppm_intensity_anticorr": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "mass_error_mz_trend": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "mean_abs_mz_error_da": + family: "mass_accuracy" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "mass_evidence_gauss": + family: "mass_accuracy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "mass_log_evidence": + family: "mass_accuracy" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" + "n_matched_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "n_matched_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "frac_matched_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "frac_matched_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_count_balance": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_intensity_ratio": + family: "ion_series" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_ratio_agreement": + family: "ion_series" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_ratio_consistency": + family: "ion_series" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "longest_b_run": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "longest_y_run": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "longest_run_max": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "longest_run_frac_length": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "series_coverage_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "series_coverage_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "sequence_coverage": + family: "ion_series" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "series_gap_fraction": + family: "ion_series" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_complement_count": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_complement_mz_consistency": + family: "ion_series" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_complement_coelution": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "ordinal_intensity_concordance_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "ordinal_intensity_concordance_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "series_coelution_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "series_coelution_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "spectral_angle_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "spectral_angle_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "pearson_b": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "pearson_y": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "cosine_charge1": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "cosine_charge2": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "charge_corr_balance": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "mean_matched_ordinal_norm": + family: "ion_series" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_ion_contiguous_intensity": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "by_ion_contiguous_lib_frac": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "both_series_present": + family: "ion_series" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" + "ms1_isotope_cosine_apex": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_spectral_angle_apex": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_chi2_apex": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_manhattan_apex": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_ratio_1_0": + family: "ms1" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_ratio_2_0": + family: "ms1" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_plus1_ratio_dev": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_plus2_ratio_dev": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_minus_one_fraction": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "iso_overlap_flag": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "log_ms1_mono": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_total_isotope_log": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "has_ms1_signal": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_apex_entropy_3": + family: "ms1" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_m1_entropy_contribution": + family: "ms1" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_ms2_time_corr": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_ms2_envelope_time_corr": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_iso_coelution": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_ms2_apex_rt_delta": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_iso_ratio_stability": + family: "ms1" + level: "precursor" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_mono_gaussianity": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_ms2_fwhm_ratio": + family: "ms1" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_corr_xic": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_envelope_over_time_corr": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "ms1_isotope_xic_shape_consistency": + family: "ms1" + level: "precursor" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" + "rt_error_signed": + family: "rt" + level: "candidate" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_error_abs@rt": + family: "rt" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + dropped_collision: true + "rt_error_squared": + family: "rt" + level: "candidate" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_error_signed_norm_gradient": + family: "rt" + level: "run" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_error_abs_norm_gradient": + family: "rt" + level: "run" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "observed_rt_raw": + family: "rt" + level: "candidate" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "predicted_rt_raw": + family: "rt" + level: "candidate" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "observed_rt_fraction": + family: "rt" + level: "run" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "predicted_rt_fraction": + family: "rt" + level: "run" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_error_over_peak_width": + family: "rt" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_error_over_fwhm": + family: "rt" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "rt_diff_profile_apex": + family: "rt" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "predicted_rt_in_gradient": + family: "rt" + level: "run" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + "log_seed_hyperscore": + family: "novel" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "seed_hyperscore_per_matched": + family: "novel" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "seed_identified@novel": + family: "novel" + level: "candidate" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + dropped_collision: true + "peptide_length@novel": + family: "novel" + level: "candidate" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + dropped_collision: true + "precursor_charge": + family: "novel" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "charge_is_2": + family: "novel" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "charge_is_3": + family: "novel" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "charge_is_4plus": + family: "novel" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "precursor_mass": + family: "novel" + level: "precursor" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "log_total_matched_intensity": + family: "novel" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "n_matched_frags": + family: "novel" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "n_predicted_frags": + family: "novel" + level: "fragment" + direction: "neutral" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + "frag_corr_peakmax": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "frag_cosine_peakmax": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "spectral_angle_peakmax": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "frag_corr_matched_nz": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "frag_cosine_matched_nz": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "peakmax_apex_gain": + family: "nonzero" + level: "fragment" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "n_frag_present_inpeak": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "frac_frag_present_inpeak": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "coelution_mean_bothpos": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "coelution_mean_summpos": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "ref_corr_nz": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "profile_cos_nz": + family: "nonzero" + level: "fragment" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" + "rank_corr_vs_apex_mean": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "rank_corr_vs_apex_std": + family: "order_consistency" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "rank_corr_adjacent_mean": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "kendall_vs_apex_mean": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "top1_frag_persistence": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "top2_order_persistence": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "argmax_frag_entropy": + family: "order_consistency" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "self_cosine_vs_apex_mean": + family: "order_consistency" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" + "n_peak_scans": + family: "peak_scans" + level: "peak" + direction: "higher_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs" + "peak_window_degenerate": + family: "peak_scans" + level: "peak" + direction: "lower_better" + source_file: "rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs" diff --git a/sensitivity_plan/01_workflow_and_gap_analysis.md b/sensitivity_plan/01_workflow_and_gap_analysis.md new file mode 100644 index 0000000..25ccbbe --- /dev/null +++ b/sensitivity_plan/01_workflow_and_gap_analysis.md @@ -0,0 +1,224 @@ +# MuMDIA Workflow and Gap Analysis + +## 1. Purpose + +This document provides a working decomposition of the MuMDIA workflow so each stage can be evaluated independently. + +The intended pipeline model is: + +```text +FASTA and modification configuration + ↓ +peptide and peptidoform generation + ↓ +precursor charge-state generation + ↓ +fragment, retention-time, and optional mobility prediction + ↓ +candidate indexing and seed search + ↓ +run-specific calibration + ↓ +chromatogram extraction + ↓ +candidate peak-group generation + ↓ +peak-group feature calculation + ↓ +peak selection and precursor competition + ↓ +semi-supervised rescoring + ↓ +target-decoy competition and q-value estimation + ↓ +peptide and protein aggregation +``` + +The implementation agent must confirm which of these stages exist in the active MuMDIA branch and update this document when behavior differs. + +## 2. Primary comparison unit + +Use the following precursor identity as the primary comparison key: + +```text +normalized modified sequence + precursor charge +``` + +Also retain: + +- Run +- Protein assignment +- Precursor m/z +- Apex retention time +- Modification localization +- Decoy status +- Entrapment status + +Do not begin with protein counts. Protein inference can conceal precursor-level losses. + +## 3. Major differences to investigate relative to DIA-NN + +### 3.1 Timing of chromatographic peak selection + +Potential MuMDIA behavior: + +```text +extract traces + ↓ +choose one apex using heuristic evidence + ↓ +calculate final feature vector + ↓ +rescore +``` + +Recommended behavior: + +```text +extract traces + ↓ +generate top-K candidate peak groups + ↓ +calculate features for every peak group + ↓ +initial out-of-fold peak scoring + ↓ +select best peak per precursor + ↓ +final confidence scoring +``` + +A wrong early apex is unrecoverable if the correct peak is discarded before rescoring. + +### 3.2 Treatment of shared fragment evidence + +In wide-window DIA, a single observed fragment trace can match fragments from many candidate peptides. + +Potential failure: + +```text +one observed trace + ↓ +copied as full evidence to several candidates + ↓ +false candidates inherit real chromatographic signal + ↓ +target-decoy separation decreases +``` + +The preferred first step is not destructive peak ownership. Instead, calculate: + +- Number of candidates claiming each fragment +- Fraction of candidate intensity that is contested +- Fraction of evidence that is unique +- Best competing candidate score +- Shared-trace correlation +- Residual spectral similarity after stronger candidates are explained + +### 3.3 Candidate peak competition + +Distinguish: + +1. Multiple chromatographic peaks for the same precursor +2. Multiple peptides explaining the same local DIA evidence +3. Multiple peptidoforms explaining the same signal +4. Multiple charge states of the same peptide +5. Target-decoy competition +6. Duplicate reporting removal + +These must be separate stages with separate logs. + +### 3.4 Scoring architecture + +Potential differences to investigate: + +- Linear versus nonlinear feature interactions +- Separate peak-selection and confidence models +- Run normalization +- Prediction calibration +- Interference-aware scoring +- Unique-fragment evidence +- Candidate ambiguity +- Cross-charge corroboration +- Modification-aware residuals + +The question is not whether MuMDIA uses fewer or more features. The question is whether its features provide independent evidence in the low-FDR operating region. + +### 3.5 Calibration + +Evaluate whether MuMDIA uses: + +- Global or local mass calibration +- Nonlinear RT calibration +- Prediction uncertainty +- Run-specific peak-width distributions +- Ion-mobility calibration +- Modified-peptide-specific residual models + +A fixed extraction window may be simultaneously too wide for clean regions and too narrow for poorly calibrated regions. + +### 3.6 Decoy behavior and FDR + +More permissive extraction can increase both true and false candidates. If false candidates borrow real fragment traces, the high-scoring decoy tail can force a stricter score threshold. + +Therefore, sensitivity must be reported at: + +```text +matched empirical false discovery proportion +``` + +not only at each tool's nominal 1% q-value. + +## 4. Required workflow instrumentation + +For every candidate, store stage flags: + +```text +in_search_space +candidate_generated +traces_extracted +peak_group_generated +selected_as_peak_winner +selected_as_variant_winner +passed_initial_score +passed_target_decoy_competition +passed_precursor_fdr +passed_peptide_fdr +reported +``` + +Store one `rejection_reason` corresponding to the earliest failed stage. + +Suggested reasons: + +```text +PEPTIDE_NOT_GENERATED +MODIFICATION_NOT_ALLOWED +CHARGE_OUT_OF_RANGE +PRECURSOR_MZ_OUT_OF_RANGE +NO_VALID_FRAGMENTS +WRONG_ISOLATION_WINDOW +RT_PRUNED +CANDIDATE_CAP_REACHED +NO_FRAGMENT_TRACES +NO_PEAK_GROUP +PEAK_NOT_SELECTED +OUTCOMPETED_BY_TARGET +OUTCOMPETED_BY_DECOY +FAILED_PRECURSOR_FDR +FAILED_PEPTIDE_FDR +REMOVED_DURING_REPORTING +``` + +## 5. Questions the implementation must answer + +1. How many DIA-NN-only precursors are absent from the MuMDIA search space? +2. How many are generated in exhaustive candidate mode? +3. For how many is the DIA-NN apex within MuMDIA's top 1, 3, 5, and 10 candidate peaks? +4. How often does a wrong MuMDIA peak win over a peak near the DIA-NN apex? +5. How often is evidence dominated by shared fragments? +6. How many valid charge states or peptidoforms are removed by early competition? +7. How many targets rank well but fail q-value filtering? +8. Does competition improve empirical FDR enough to recover additional weak targets? +9. Which losses are enriched for low-intensity or modified peptides? +10. Which differences remain after matching the search space and prediction library? diff --git a/sensitivity_plan/03_feature_evaluation.md b/sensitivity_plan/03_feature_evaluation.md new file mode 100644 index 0000000..555a9ec --- /dev/null +++ b/sensitivity_plan/03_feature_evaluation.md @@ -0,0 +1,439 @@ +# Feature Evaluation and Feature Expansion + +## 1. Objective + +Determine whether each feature: + +1. Contains useful signal +2. Adds information beyond existing features +3. Generalizes across datasets +4. Improves identifications at controlled empirical FDR +5. Does not introduce leakage or unstable subgroup behavior + +Feature importance alone is not sufficient. + +## 2. Feature registry + +Maintain a registry: + +```yaml +features: + rt_residual_normalized: + family: retention_time + level: peak_group + direction: lower_is_better + requires_calibration: true + uses_cross_run_information: false + missing_value_policy: median_plus_indicator + computational_cost: low + description: absolute calibrated RT residual divided by local uncertainty +``` + +Required fields: + +- Name +- Definition +- Family +- Level +- Units +- Expected direction +- Valid range +- Missing-value policy +- Calibration dependency +- Cross-run dependency +- Potential leakage risk +- Computational cost + +## 3. Leakage checks + +Never use as normal scoring features: + +- Final q-value +- Final posterior error probability +- Final target-decoy winner +- Candidate rank generated by the same model +- DIA-NN score or identification status +- Protein evidence calculated after precursor scoring +- Cross-run evidence calculated using the held-out run +- Calibration values fit using the held-out candidate +- Features computed globally before cross-validation + +Fit within each training fold: + +- Imputation +- Scaling +- Calibration +- Feature transformation +- Scoring model + +## 4. Validation design + +### Outer validation + +Prefer: + +```text +leave-one-dataset-out +leave-one-instrument-out +leave-one-acquisition-method-out +``` + +### Inner validation + +Group by: + +```text +modified_sequence + charge +``` + +Keep all peak groups for one precursor in the same fold. + +Entrapment labels are evaluation labels, not training labels. + +## 5. Feature evidence ladder + +### Level 1: Data quality + +Report per feature: + +- Missing percentage +- Invalid values +- Unique-value count +- Quantiles +- Distribution per run +- Distribution per instrument +- Distribution for targets, decoys, and entrapments +- Correlation with intensity +- Correlation with candidate count +- Correlation with peptide length, charge, and modification count + +Flag: + +- Constant features +- Asymmetric missingness +- Run-specific artifacts +- Features dominated by intensity +- Features separating target and decoy only because of decoy construction + +### Level 2: Univariate utility + +Evaluate: + +- Precision-recall area +- Partial ROC area in the low-error region +- Targets at matched empirical FDP +- Target-entrapment separation +- Monotonicity with correctness +- Utility by intensity decile + +Full ROC AUC is not the main metric. + +### Level 3: Redundancy + +Calculate: + +- Spearman correlation +- Mutual information +- Correlation within targets +- Correlation within decoys +- Correlation after controlling for intensity + +Cluster redundant features and test: + +```text +all features +one representative per cluster +leave-one-feature-out +leave-one-cluster-out +``` + +### Level 4: Feature-family ablation + +Recommended families: + +- Retention time +- Ion mobility +- Precursor mass accuracy +- Fragment mass accuracy +- Spectral agreement +- Chromatographic shape +- Fragment coelution +- MS1 and isotope evidence +- Interference evidence +- Candidate competition +- Peak morphology +- Search-space priors +- Modification-aware evidence +- Run-context evidence + +For every family compare: + +```text +full model +full model minus family +minimal baseline plus family +``` + +### Level 5: Conditional permutation + +Perform permutation inside held-out folds. + +Evaluate: + +- Standard permutation importance +- Grouped family permutation +- Conditional permutation for correlated features + +Primary metric: + +```text +change in identifications at 1% empirical FDP +``` + +### Level 6: Explanation analysis + +SHAP or similar methods may be used to diagnose: + +- Direction of effect +- Dataset stability +- Subgroup behavior +- Nonlinear thresholds +- Interactions + +Do not use SHAP rank as the final retention criterion. + +## 6. Separate scoring tasks + +Evaluate features separately for: + +### Peak-selection model + +Question: + +```text +Which chromatographic peak belongs to this precursor? +``` + +Metrics: + +- Correct peak at rank 1 +- Correct peak in top 3 +- Apex RT error +- Peak-boundary agreement + +### Peptide-ranking model + +Question: + +```text +Which peptide best explains this local peak group? +``` + +Metrics: + +- Correct target rank +- Margin over second-best target +- Margin over best decoy +- Recall at matched FDP + +### Final confidence model + +Question: + +```text +Is the winning precursor sufficiently reliable to report? +``` + +Metrics: + +- Identifications at matched empirical FDP +- q-value calibration +- posterior calibration +- entrapment FDP + +A feature can be useful for one task and unhelpful for another. + +## 7. Model comparisons + +Always compare at least: + +- Logistic regression or linear discriminant model +- Gradient-boosted tree model + +A large nonlinear-model advantage suggests useful interactions or thresholds. A small advantage suggests feature quality matters more than model capacity. + +## 8. Candidate feature families to add + +### 8.1 Uncertainty-normalized residuals + +```text +abs(RT residual) / local RT uncertainty +abs(mobility residual) / local mobility uncertainty +abs(mass error) / local mass uncertainty +``` + +Also retain signed residuals. + +### 8.2 Fragment evidence distributions + +Add: + +- Median fragment coelution +- Minimum fragment coelution +- Coelution interquartile range +- Fraction of fragments above a coelution threshold +- Median fragment mass error +- Fragment mass-error dispersion +- Fraction of top predicted fragments observed +- Explained observed intensity +- Explained predicted intensity +- Effective fragment count +- Evidence concentration in the strongest fragment + +### 8.3 Apex dispersion + +Add: + +- Fragment apex RT standard deviation +- Fragment apex RT median absolute deviation +- Maximum apex deviation +- Precursor-to-fragment apex deviation + +### 8.4 Peak-shape evidence + +Add: + +- Consensus chromatographic profile +- Fraction of each fragment explained by consensus +- Peak symmetry +- Tailing +- Shoulder score +- Number of local maxima +- Boundary agreement +- Peak truncation indicator + +### 8.5 Interference and contested evidence + +Add: + +- Number of candidate claimants per fragment +- Contested intensity fraction +- Unique-fragment count +- Unique-fragment intensity fraction +- Shared-fragment count +- Shared-trace correlation +- Local candidate density +- Isolation-window precursor density +- Correlation with competing candidate profiles +- Residual similarity after stronger candidates are explained + +### 8.6 Candidate ambiguity + +Add: + +- Margin from best alternative peak +- Margin from best alternative peptide +- Margin from best decoy +- Number of candidates within a score threshold +- Candidate score entropy +- Number of near-isobaric candidates +- Number of alternative modification localizations + +Do not feed a margin produced by the final model back into that same model. Use an earlier-stage score or a second-stage model. + +### 8.7 MS1 and isotope evidence + +Add: + +- Monoisotopic trace evidence +- Isotope-pattern correlation +- Isotope-spacing accuracy +- Isotope coelution +- Theoretical versus observed isotope ratio +- Precursor-fragment apex agreement +- Incorrect monoisotope indicator + +### 8.8 Modification-aware evidence + +Add: + +- Modification count +- Modification class +- Prediction-training coverage indicator +- Modification-specific RT residual percentile +- Modification-specific spectral residual percentile +- Localization ambiguity +- Site-determining ion count +- Site-determining ion intensity +- Number of alternative peptidoforms + +Validate FDR independently per modification group. + +### 8.9 Run context + +Add: + +- Isolation-window width +- Cycle time +- Number of scans across peak +- Local signal density +- Local candidate density +- Run-level calibration quality +- Run-level prediction residual statistics + +Normalize context features by run. + +## 9. Feature acceptance criteria + +Retain a feature or family when it produces one or more of: + +- At least 1% additional precursor identifications at 1% empirical FDP on at least two held-out datasets +- At least 5% gain in a scientifically important weak subgroup +- At least 2% improvement in correct-peak recall +- Better q-value calibration +- Lower runtime without sensitivity loss + +Also require: + +- No material FDP inflation +- Stable direction across datasets +- No leakage +- Similar availability for targets and decoys +- Acceptable computational cost + +Reject or revise when: + +- Gain appears in one dataset only +- Gain disappears in conditional ablation +- Feature mainly separates targets from artificial decoys, not entrapments +- Modified-peptide FDP worsens +- Feature effect changes direction across datasets +- Gain is smaller than seed-to-seed variability + +## 10. Required output table + +```text +feature_family +baseline_identifications +new_identifications +relative_gain +empirical_fdp +correct_peak_gain +low_intensity_gain +modified_peptide_gain +runtime_change +stability_score +recommendation +``` + +Allowed recommendations: + +```text +KEEP +KEEP_FOR_SUBGROUP +REVISE +REDUNDANT +UNSTABLE +LEAKAGE_RISK +TOO_EXPENSIVE +``` diff --git a/sensitivity_plan/04_peak_and_peptide_competition.md b/sensitivity_plan/04_peak_and_peptide_competition.md new file mode 100644 index 0000000..bcb6e35 --- /dev/null +++ b/sensitivity_plan/04_peak_and_peptide_competition.md @@ -0,0 +1,305 @@ +# Peak and Peptide Competition Design + +## 1. Purpose + +Competition must prevent multiple false precursor hypotheses from borrowing the same physical DIA signal without suppressing genuinely coeluting peptides. + +Do not represent all competition as a single `compete` stage. + +## 2. Competition taxonomy + +Implement separate stages for: + +1. Chromatographic peak competition +2. Duplicate precursor competition +3. Peptide interference competition +4. Peptidoform competition +5. Modification-localization competition +6. Target-decoy competition +7. Reporting deduplication + +Each stage must record: + +- Input candidates +- Group identifier +- Winner +- Removed candidates +- Score used +- Removal reason + +## 3. Chromatographic peak competition + +### Problem + +One precursor may have several plausible chromatographic peaks. + +If one apex is selected before full scoring: + +```text +wrong peak selected + ↓ +correct peak discarded + ↓ +final scorer cannot recover precursor +``` + +### Recommended design + +1. Generate local maxima from consensus fragment evidence. +2. Retain top `K` peaks per precursor. +3. Calculate complete features for every peak group. +4. Score peaks out of fold. +5. Select the best peak for each modified-sequence and charge precursor. +6. Optionally retain a second peak when evidence supports repeated elution or unresolved ambiguity. + +Suggested initial values: + +```text +K ∈ {3, 5, 10} +``` + +### Evaluation + +Report reference-apex recall at top 1, 3, 5, and 10. + +## 4. Shared fragment evidence + +### Problem + +An observed MS2 trace may match fragments from many candidates: + +```text +observed m/z trace + ├── candidate A fragment + ├── candidate B fragment + ├── candidate C fragment + └── decoy D fragment +``` + +Giving full intensity to all candidates can create false high scores. Hard assignment can cause a high-abundance peptide to steal evidence from a real low-abundance peptide. + +### Initial non-destructive solution + +Do not reassign raw evidence in the first implementation. + +Calculate: + +- Claimant count per fragment +- Contested fragment count +- Contested intensity fraction +- Unique fragment count +- Unique intensity fraction +- Shared-trace correlation +- Strongest competitor score +- Score margin +- Residual spectral similarity +- Local conflict-group size + +Use these features during rescoring. + +## 5. Conflict graph + +Represent each candidate peak group as a node. + +Node identity: + +```text +run +modified sequence +charge +candidate apex +peak boundaries +``` + +Add an edge when candidates: + +1. Occur in compatible or overlapping DIA isolation windows +2. Have overlapping peak boundaries or close apexes +3. Share fragment m/z values within tolerance +4. Use a material amount of the same chromatographic evidence + +Edge features: + +```text +shared_fragment_count +shared_intensity_fraction_A +shared_intensity_fraction_B +apex_rt_difference +boundary_overlap +shared_trace_correlation +unique_fragment_count_A +unique_fragment_count_B +unique_intensity_fraction_A +unique_intensity_fraction_B +score_difference +``` + +## 6. Competition strategies to benchmark + +### Strategy A: No hard competition + +- Preserve all candidates +- Add conflict features +- Let final FDR handle ambiguity + +This is the baseline. + +### Strategy B: Hard winner-take-all + +Within sufficiently strong conflict groups: + +```text +retain highest initial discriminant score +``` + +This is simple but may suppress real coeluting peptides. + +### Strategy C: Unique-evidence-aware competition + +Allow multiple candidates to survive when each has enough independent evidence. + +Example rule: + +```text +unique_fragment_count >= 2 +and unique_intensity_fraction >= threshold +and unique_fragments_coelute +``` + +Otherwise, apply winner-take-all. + +### Strategy D: Margin-gated competition + +Remove a competitor only when: + +```text +winner_score - loser_score >= margin +``` + +and shared evidence exceeds a threshold. + +### Strategy E: Soft score penalty + +Do not remove candidates. Apply a penalty derived from: + +- Contested intensity +- Lack of unique evidence +- Stronger competing candidates +- Conflict-group size + +### Strategy F: Residual-evidence pass + +1. Score all candidates. +2. Select strongly supported candidates. +3. Model the fragment signal they explain. +4. Subtract or downweight explained signal. +5. Rescore weaker candidates using residual evidence. + +This is the most complex strategy and should be implemented only after A-E are benchmarked. + +## 7. Peptidoform and charge handling + +### Charge states + +Different charge states can both be real. + +Recommended rule: + +```text +do not directly compete distinct charge states +``` + +Instead, add cross-charge corroboration features. + +### Distinct modification states + +Unmodified and modified forms can co-exist. + +Do not compete all forms sharing the same base sequence. + +Compete only candidates that are mutually exclusive explanations of the same local signal. + +### Localization variants + +Candidates with the same composition but different modification sites require localization scoring. + +Retain them until site-determining ion evidence is evaluated. + +## 8. Target-decoy competition + +Target-decoy competition is not the same as interference competition. + +Recommended sequence: + +```text +peak scoring + ↓ +peak selection + ↓ +local peptide-interference handling + ↓ +peptidoform/localization handling + ↓ +final rescoring + ↓ +target-decoy competition + ↓ +q-value estimation +``` + +Document the competition unit explicitly: + +- Spectrum +- Peak group +- Precursor +- Peptide +- Peptidoform +- Protein group + +## 9. Competition feature audit + +For every competition feature, verify: + +- Symmetric calculation for targets and decoys +- No use of final labels +- Calculation within cross-validation folds where needed +- Stability across runs +- No dependence on candidate enumeration artifacts +- No direct reuse of final model score as an input to itself + +## 10. Primary experiment + +Run: + +| Mode | Peak candidates | Shared evidence | Variant competition | +|---|---:|---|---| +| A | 1 | full evidence | before rescoring | +| B | top 5 | full evidence | before rescoring | +| C | top 5 | conflict features | after initial rescoring | +| D | top 5 | unique-evidence-aware | after initial rescoring | +| E | top 5 | margin-gated hard competition | after initial rescoring | +| F | top 5 | residual-evidence pass | after initial rescoring | + +Measure: + +- Targets at matched empirical FDP +- Decoys and entrapments removed +- Correct targets removed +- Correct-peak recall +- Low-intensity identifications +- Modified-peptide identifications +- Conflict groups containing multiple independently supported targets +- Runtime and memory + +## 11. Recommended initial implementation + +Start with: + +```text +top-K peaks ++ non-destructive conflict graph ++ unique/contested evidence features ++ competition after initial rescoring +``` + +Do not start with raw centroid ownership or global winner-take-all assignment. diff --git a/sensitivity_plan/05_experiment_matrix.md b/sensitivity_plan/05_experiment_matrix.md new file mode 100644 index 0000000..46cb297 --- /dev/null +++ b/sensitivity_plan/05_experiment_matrix.md @@ -0,0 +1,199 @@ +# Experiment Matrix and Acceptance Criteria + +## 1. Experimental principles + +Each experiment must: + +- Change one component at a time +- Use fixed search spaces +- Use fixed random seeds +- Preserve raw candidate outputs +- Run on development and held-out datasets +- Report empirical FDP +- Report runtime and memory +- Report subgroup behavior + +## 2. Core experiment matrix + +| ID | Change | Main question | +|---|---|---| +| E0 | Current baseline | What is the present sensitivity gap? | +| E1 | Exhaustive candidate generation | Is pruning losing valid candidates? | +| E2 | Top-3 peak retention | Is early peak selection limiting sensitivity? | +| E3 | Top-5 peak retention | How much additional peak recall is gained? | +| E4 | RT oracle | Is RT calibration or pruning limiting sensitivity? | +| E5 | Peak oracle | Is the correct peak generated but ranked poorly? | +| E6 | Empirical fragment oracle | Are predicted transitions limiting sensitivity? | +| E7 | Wide extraction windows | Are tolerance settings too strict? | +| E8 | Adaptive calibrated windows | Can calibration improve signal-to-background? | +| E9 | No interference competition | Baseline effect of shared evidence | +| E10 | Conflict features only | Can non-destructive competition improve ranking? | +| E11 | Unique-evidence-aware competition | Can false candidates be suppressed safely? | +| E12 | Margin-gated hard competition | Does conservative removal improve FDR? | +| E13 | Residual-evidence rescoring | Can weaker coeluting peptides be recovered? | +| E14 | Move variant competition after rescoring | Is early competition removing real candidates? | +| E15 | Add MS1/isotope features | Does precursor evidence increase discrimination? | +| E16 | Add peak-shape features | Does chromatographic modeling improve peak ranking? | +| E17 | Add uncertainty-normalized residuals | Are raw prediction errors poorly calibrated? | +| E18 | Add candidate-ambiguity features | Does local competition improve confidence? | +| E19 | Linear versus LightGBM | Are nonlinear interactions important? | +| E20 | Alternative decoy design | Is FDR estimation limiting sensitivity? | +| E21 | Staged modification search | Is broad peptidoform competition too severe? | +| E22 | Run-specific prediction calibration | Does domain adaptation improve modified peptides? | +| E23 | Cross-run re-extraction | What is the experiment-level recovery ceiling? | + +## 3. Primary metrics + +### Sensitivity + +- Precursors at reported 1% q-value +- Precursors at 1% empirical FDP +- Peptides at 1% empirical FDP +- Modified peptides at 1% empirical FDP +- Protein groups at validated FDR + +### Pipeline recall + +- Search-space recall +- Candidate-generation recall +- Trace-extraction recall +- Correct peak in top 1 +- Correct peak in top 3 +- Correct peak in top 5 +- Correct peptide rank 1 +- Target-decoy win rate + +### Calibration + +- Reported q-value versus empirical FDP +- Posterior calibration +- Entrapment rate by score decile +- FDP by subgroup + +### Cost + +- Runtime +- Peak memory +- Candidate count +- Disk usage +- Feature-calculation time +- Model-training time + +## 4. Subgroup reporting + +Report all primary metrics by: + +- Intensity decile +- Precursor charge +- Peptide length +- Modification class +- Modification count +- Number of theoretical fragments +- Number of observed fragments +- Candidate density +- Isolation-window width +- Peak width +- RT region +- Instrument +- Acquisition method + +## 5. Decision rules + +### Candidate recall below 95% + +Prioritize: + +- Search-space generation +- Isolation-window assignment +- Modification enumeration +- RT pruning +- Candidate caps +- Isotope handling + +Do not prioritize final rescoring. + +### Candidate recall high, peak recall low + +Prioritize: + +- Top-K peak generation +- RT calibration +- Peak detection +- Peak boundaries +- Fragment consensus +- Adaptive peak widths + +### Correct peak available, peptide rank poor + +Prioritize: + +- Unique-fragment evidence +- Spectral agreement +- Fragment coelution +- Interference modeling +- Candidate ambiguity +- Prediction adaptation + +### Ranking good, q-value yield poor + +Prioritize: + +- Decoy construction +- Target-decoy competition +- Cross-validation +- Score calibration +- Group-specific FDR +- Removal of false candidates borrowing real signal + +### Single-run performance good, experiment coverage poor + +Prioritize: + +- Run alignment +- Empirical chromatogram libraries +- Controlled re-extraction +- Transfer-specific confidence estimation + +## 6. Acceptance criteria for code changes + +A change can be accepted when: + +- Empirical FDP remains controlled +- Gain reproduces on at least two held-out datasets +- No major subgroup degradation occurs +- Runtime and memory are acceptable +- Output is deterministic under fixed seeds +- All new rejection decisions are logged +- Feature and model versions are recorded + +Suggested quantitative thresholds: + +- At least 1% overall precursor gain at 1% empirical FDP, or +- At least 5% gain in a prespecified weak subgroup, or +- At least 2% gain in correct-peak recall, or +- Meaningful q-value calibration improvement + +## 7. Required statistical summaries + +For each experiment report: + +- Absolute identification difference +- Relative identification difference +- Bootstrap confidence interval by run or dataset +- Seed-to-seed variability +- Paired results by dataset +- Empirical FDP difference +- Runtime difference + +Do not select a feature based on one favorable run. + +## 8. Stop conditions + +Stop expanding model complexity when: + +- Candidate or peak recall, not scoring, remains the bottleneck +- Gains disappear on held-out datasets +- Empirical FDP degrades +- Runtime cost is disproportionate +- Additional features are redundant +- Feature effects are unstable across acquisition methods diff --git a/sensitivity_plan/06_agent_implementation_backlog.md b/sensitivity_plan/06_agent_implementation_backlog.md new file mode 100644 index 0000000..80e7848 --- /dev/null +++ b/sensitivity_plan/06_agent_implementation_backlog.md @@ -0,0 +1,372 @@ +# Agent Implementation Backlog + +## 1. Working conventions + +The agent should: + +- Implement small reviewable changes +- Add tests with every change +- Preserve backwards-compatible defaults where possible +- Record all configuration in output metadata +- Avoid irreversible candidate filtering in early stages +- Prefer diagnostic flags before production behavior changes +- Produce Parquet tables for large candidate-level outputs + +## 2. Priority 0: Observability and benchmark harness + +### P0.1 — Search-space manifest + +Implement a machine-readable search-space manifest and validation. + +Acceptance criteria: + +- Effective configuration is exported. +- MuMDIA and DIA-NN settings can be compared. +- The benchmark fails on important mismatches. +- Input hashes and software versions are stored. + +### P0.2 — Normalized output converter + +Implement a common output schema. + +Acceptance criteria: + +- All top candidates are retained. +- Final reported candidates are retained. +- Modified sequences are normalized consistently. +- Precursor keys are reproducible. + +### P0.3 — Candidate audit table + +Create `candidate_audit.parquet`. + +Required columns: + +```text +run_id +precursor_id +modified_sequence +charge +target_decoy_label +entrapment_label +candidate_generated +traces_extracted +peak_generated +peak_selected +variant_selected +target_decoy_winner +passed_precursor_fdr +passed_peptide_fdr +reported +rejection_reason +``` + +### P0.4 — Stage-level metrics + +Generate: + +- Search-space recall +- Candidate recall +- Trace recall +- Top-K peak recall +- Peptide-ranking recall +- Competition losses +- FDR losses + +### P0.5 — Entrapment runner + +Acceptance criteria: + +- Entrapment databases are generated reproducibly. +- Empirical FDP is reported at precursor, peptide, and protein levels. +- Results are stratified by modification and charge. + +## 3. Priority 1: Top-K chromatographic peaks + +### P1.1 — Candidate peak generator + +Implement configurable local peak generation. + +CLI: + +```text +--retain-top-peaks K +``` + +Acceptance criteria: + +- Supports `K = 1, 3, 5, 10`. +- Stores apex, boundaries, and initial score for every peak. +- Correct-peak top-K recall can be computed. +- Default behavior remains reproducible. + +### P1.2 — Peak-level feature calculation + +Calculate complete feature vectors independently for each candidate peak. + +Acceptance criteria: + +- No feature uses information from another candidate's final label. +- Peak features are cached. +- Computation is deterministic. + +### P1.3 — Peak-selection model + +Implement grouped out-of-fold peak scoring. + +Acceptance criteria: + +- Peak groups from one precursor remain in the same fold. +- Out-of-fold scores are stored. +- Top-1, top-3, and top-5 recall are reported. + +## 4. Priority 2: Competition graph and contested evidence + +### P2.1 — Fragment claimant index + +For every observed fragment trace, record candidate claimants. + +Acceptance criteria: + +- Claimant counts are available. +- Candidate-fragment mappings are queryable. +- Memory behavior is benchmarked. + +### P2.2 — Peak-group conflict graph + +Create graph edges for candidates with overlapping RT and fragment evidence. + +Acceptance criteria: + +- Edge thresholds are configurable. +- Edge features are stored. +- Graph construction is deterministic. +- Large connected components are diagnosed. + +### P2.3 — Conflict features + +Add: + +```text +contested_fragment_count +contested_intensity_fraction +unique_fragment_count +unique_intensity_fraction +strongest_competitor_score +score_margin +conflict_group_size +shared_trace_correlation +``` + +Acceptance criteria: + +- Target and decoy calculations are symmetric. +- Missing values are documented. +- Runtime is reported. + +### P2.4 — Competition modes + +CLI: + +```text +--competition-mode none +--competition-mode features-only +--competition-mode unique-evidence +--competition-mode margin-gated +``` + +Acceptance criteria: + +- Every removed candidate records its winner and reason. +- Competition happens after initial peak scoring. +- Empirical FDP is reported for every mode. + +## 5. Priority 3: Calibration and extraction + +### P3.1 — Two-pass mass calibration + +Acceptance criteria: + +- Robust precursor and fragment calibration. +- Local uncertainty estimate. +- Before-and-after residual plots. +- Fallback for insufficient calibrants. + +### P3.2 — Nonlinear RT calibration + +Acceptance criteria: + +- Monotonic mapping. +- Cross-validated residuals. +- Local RT uncertainty. +- Separate modified-peptide diagnostics. + +### P3.3 — Adaptive extraction windows + +Acceptance criteria: + +- Window width is based on calibrated uncertainty. +- Minimum and maximum bounds are configurable. +- Candidate recall and interference are compared with fixed windows. + +## 6. Priority 4: Feature evaluation framework + +### P4.1 — Feature registry + +Create `feature_registry.yaml`. + +Acceptance criteria: + +- Every feature has family, level, direction, missing policy, and leakage note. +- Documentation is generated automatically. +- Unknown features fail validation. + +### P4.2 — Feature audit command + +Suggested CLI: + +```bash +mumdia-feature-audit \ + --features candidate_features.parquet \ + --metadata candidate_metadata.parquet \ + --registry feature_registry.yaml \ + --output reports/feature_audit +``` + +Required outputs: + +- Missingness +- Distributions +- Correlation clusters +- Target/decoy/entrapment comparisons +- Run drift +- Leakage warnings + +### P4.3 — Ablation runner + +Suggested CLI: + +```bash +mumdia-feature-ablation \ + --config feature_experiments.yaml \ + --outer-group dataset_id \ + --inner-group precursor_id \ + --metric targets_at_empirical_fdp \ + --fdp 0.01 +``` + +Acceptance criteria: + +- Supports family removal and addition. +- Supports linear and tree models. +- Stores fold assignments. +- Produces paired dataset-level results. + +## 7. Priority 5: New features + +### P5.1 — Uncertainty-normalized residuals + +Implement RT, mass, and mobility normalized residuals. + +### P5.2 — Fragment evidence distributions + +Implement median, dispersion, threshold fractions, and effective fragment count. + +### P5.3 — Peak-shape and apex-dispersion features + +Implement consensus profile, boundary agreement, apex dispersion, symmetry, and shoulder score. + +### P5.4 — MS1 and isotope features + +Implement monoisotope, isotope correlation, spacing, coelution, and precursor-fragment agreement. + +### P5.5 — Candidate ambiguity features + +Implement alternative-target, alternative-peak, and decoy margins using an earlier-stage score. + +## 8. Priority 6: Peptidoforms and localization + +### P6.1 — Delay variant competition + +Acceptance criteria: + +- Different charge states are not prematurely collapsed. +- Modified and unmodified forms can coexist. +- Candidate counts before and after competition are reported. + +### P6.2 — Localization competition + +Acceptance criteria: + +- Localization variants remain until site-determining evidence is calculated. +- Site-determining ion count and intensity are available. +- Localization confidence is separated from precursor confidence. + +### P6.3 — Staged modification search + +Acceptance criteria: + +- Calibration stage and extended-modification stage are configurable. +- Combined confidence estimation is validated with entrapment. +- Modification-specific performance is reported. + +## 9. Priority 7: Reporting + +### P7.1 — HTML benchmark report + +Include: + +1. Search-space parity +2. Reported q-value versus empirical FDP +3. Identification counts at matched FDP +4. MuMDIA/DIA-NN overlap +5. Identification-loss waterfall +6. Candidate recall +7. Correct-peak recall +8. Peptide-ranking recall +9. Competition losses +10. Feature-family ablations +11. Performance by intensity +12. Performance by modification +13. Runtime and memory +14. Representative candidate chromatograms + +### P7.2 — Candidate diagnostic bundle + +For selected candidate IDs, export: + +- Fragment chromatograms +- MS1 traces +- Peak boundaries +- Predicted and observed spectra +- Feature values +- Conflict neighbors +- Removal reason + +## 10. Suggested first sprint + +Implement in this exact order: + +1. Search-space manifest +2. Candidate audit table +3. Rejection reason codes +4. Top-K peak retention +5. Reference-apex top-K analysis +6. Conflict graph +7. Unique and contested evidence features +8. Competition after initial rescoring +9. Entrapment validation +10. Feature-family ablation runner + +## 11. Completion checklist + +- [ ] Search-space parity is verified. +- [ ] Candidate-level audit output exists. +- [ ] At least 95% of missing precursors have a loss category. +- [ ] Top-K peak recall is measured. +- [ ] Competition stages are separated. +- [ ] Entrapment FDP is measured. +- [ ] Feature families have ablation results. +- [ ] Gains reproduce on held-out datasets. +- [ ] Modified peptides are evaluated separately. +- [ ] Runtime and memory are reported. diff --git a/sensitivity_plan/ARCHITECTURE_MAP.md b/sensitivity_plan/ARCHITECTURE_MAP.md new file mode 100644 index 0000000..f363ebd --- /dev/null +++ b/sensitivity_plan/ARCHITECTURE_MAP.md @@ -0,0 +1,576 @@ +# MuMDIA Architecture Map + +Stage-by-stage map of the real MuMDIA pipeline for the sensitivity-improvement +effort. It records where each stage lives, its key symbols with line numbers, its +input/output artifacts, the config knobs that govern it, its tests, and, most +importantly, every point at which a candidate precursor can be dropped. All paths +are repository-relative; all line numbers are 1-indexed against the working tree +on branch `feat/sensitivity-improvements`. + +Sources: the architecture-mapping workflow journal (7 stage maps: extract, +features, compete, rescore/FDR, configuration, IO+candidate-identity+calibration, +scripts) and direct reads of the source. The companion feature registry is +`FEATURE_REGISTRY.md` / `feature_registry.yaml`. + +## 0. Pipeline overview + +Each stage is an independent `mumdia ` subcommand reading path-addressable +inputs and writing Parquet plus an `.report.json` sidecar. The single-run +chain (orchestrated by `run`, `stages/run.rs`) is: + +``` +convert -> digest -> peptidoforms -> predict-frag -> search-seed -> rt-im-train + -> extract -> features -> compete -> rescore -> report (+ quant) +``` + +`candidate_id` is the dense identity key of the library and of every artifact from +extract onward. It is minted in `predict_frag.rs:163` as the `enumerate()` index +after sorting by precursor m/z. Pre-predict-frag stages (digest, peptidoforms) +have no `candidate_id`; they key on `base_peptide_id` / peptidoform id. +`Library::load` (`index.rs:78`) hard-asserts `candidate_id == 0..ncand` +contiguous and (`index.rs:135`) precursor-m/z ascending, so `candidate_id` is a +safe dense array index downstream. + +## 1. Modules added by the sensitivity lead agent + +Committed on `feat/sensitivity-improvements`: `2f46d6d` (rejection reasons, top-K +peak enumerator, config scaffolding), `eb9da89` (candidate `audit` stage + +subcommand), `de5ae2b` (competition modes wired into `compete`). + +Status of each addition: +- `mumdia audit` subcommand: WIRED and verified on real data (P0.3/P0.4). +- `CompetitionMode` in `compete`: WIRED (`de5ae2b`); default `WinnerTakeAll` + reproduces the legacy behaviour bit-for-bit; `none`/`features_only`/ + `unique_evidence`/`margin_gated` are selectable and unit tested. +- `enumerate_peaks` (`mumdia::peaks`): a tested pure helper, NOT yet called from + `extract` (extract still emits one apex per candidate). This is the one + remaining destructive-stage change; the exact hook site is in §4. +- `ExtractConfig.retain_top_peaks` / `emit_candidate_audit`: parsed and validated, + NOT yet consumed by `extract` (the top-K wiring and the in-extract audit sidecar + are the primary next step; see `NEXT_STEPS.md`). + +### `mumdia_core::rejection` (`rust/mumdia/crates/mumdia-core/src/rejection.rs`) + +- `RejectionReason` enum (`rejection.rs:19`): 16 loss categories plus a `Reported` + sentinel, `#[serde(rename_all = "SCREAMING_SNAKE_CASE")]`. The spellings match + spec 01 §4 exactly (`NO_PEAK_GROUP`, `OUTCOMPETED_BY_DECOY`, ...). +- `code()` (`rejection.rs:50`) stable string for Parquet/JSON; + `stage_order()` (`rejection.rs:76`) the identification-loss ladder (0 = earliest + stage, `Reported` = 255); `earliest()` (`rejection.rs:106`) keeps the earlier of + two losses; `is_rejection()` (`rejection.rs:100`). +- This is the type behind the audit table's `rejection_reason`; a candidate's row + records the earliest stage at which it was lost. + +### `mumdia::peaks` (`rust/mumdia/crates/mumdia/src/peaks.rs`) + +- `PeakGroup` struct (`peaks.rs:22`): `apex_idx`, `start_idx`, `end_idx`, + `apex_intensity`, `area`, `rank`. +- `enumerate_peaks(profile, k, bound_fraction, min_prominence_frac)` + (`peaks.rs:52`): pure, side-effect-free top-K local-maximum detector over a + consensus elution profile. Walks fractional-height boundaries (matching + `features.bound_peak_fraction`), drops maxima below a prominence floor, + deduplicates maxima inside a stronger peak's envelope, returns peaks strongest- + first by integrated `area`, deterministic (ties break by earlier `apex_idx`). + `k = 1` reproduces the single-strongest-apex behaviour, so callers can adopt it + incrementally. Fully unit-tested (`peaks.rs:154-268`), including the core + "interference-dominant but true peak retained with top-K" case + (`peaks.rs:190`). + +### `stages::audit` (`rust/mumdia/crates/mumdia/src/stages/audit.rs`, `mumdia audit`) + +- Wired as a subcommand (`main.rs:230`, `stages/mod.rs:5`). Non-destructive, + post-hoc: reconstructs per-candidate stage flags and the earliest + `RejectionReason` by tracking which `candidate_id`s survive across the artifact + chain library -> psms(extract) -> competed(compete) -> scored(rescore), then + writes `candidate_audit.parquet` and prints the identification-loss waterfall + (`audit.rs:64`, reason assignment `audit.rs:133-155`, waterfall `audit.rs:199`). +- `load_extract_reasons` (`audit.rs:51`) reads an optional + `.audit.parquet` sidecar to refine the coarse extract-stage bucket once an + in-extract audit is emitted. + +### New config fields (`rust/mumdia/crates/mumdia-core/src/config.rs`) + +- `ExtractConfig.retain_top_peaks: usize` (`config.rs:528`, default 1): K + chromatographic peak groups per candidate; 1 = legacy single apex. Validated + `>= 1` (`config.rs:917`). +- `ExtractConfig.emit_candidate_audit: bool` (`config.rs:533`, default false): + when true, extraction is to write `.audit.parquet` per-candidate + survivor flags / earliest reason. Near-zero cost when false. +- `CompeteConfig.mode: CompetitionMode` (`config.rs:613`, default `WinnerTakeAll`), + `margin: f64` (`config.rs:616`), `unique_evidence_min_fragments: usize` + (`config.rs:620`), `emit_competition_audit: bool` (`config.rs:623`). +- `CompetitionMode` enum (`config.rs:646`): `WinnerTakeAll` (legacy) / `None` / + `FeaturesOnly` / `UniqueEvidence` / `MarginGated`, with `from_token` + (`config.rs:667`). Maps to spec 04 §6 strategies A/B/C/D. CONSUMED in + `compete.rs` via the pure `resolve_competition()` (`de5ae2b`); `WinnerTakeAll` + is bit-identical to the previous behaviour. + +## 2. Stage-by-stage map + +For each stage: main source, key symbols (line), input -> output artifacts, the +config knobs that govern it, and existing tests. + +### convert (Stage 0) + +- Source: `stages/convert.rs`. Entry `run` (`convert.rs:103`); `centroid` + (`convert.rs:19`), `ConvertParams` (`convert.rs:86`), `ConvertOutputs` + (`convert.rs:96`). +- IO: mzML -> `spectra_ms1.parquet`, `spectra_ms2.parquet`, + `isolation_windows.parquet`, `ms2_to_ms1.parquet` (`convert.rs:172-175`). + Assigns a monotonic `scan_index`, converts scan time to seconds, centroids + profile spectra (local maxima), caps peaks top-N, synthesizes a full-range + window for zero-bounded AIF scans. +- Knobs: `max_spectra` (`convert.rs:89`/`124`) caps spectra for fast iteration + (run-level, not a candidate drop). No per-candidate drops. +- Tests: none at stage level (CLAUDE.md test gap). + +### digest (Stage A) + +- Source: `stages/digest.rs`. Entry `run` (`digest.rs:147`); `make_decoy` + (`digest.rs` ~95-117). +- IO: FASTA -> `peptides.parquet` (keyed on `id`/`target_id`). Fully-tryptic + in-silico digest, mints paired reverse/scramble decoys immediately, dedups + targets by stripped sequence (insertion order preserved for determinism). +- Knobs: `digest.enzyme` (`config.rs:261`), `missed_cleavages` (271), `min_len` + / `max_len` (272), `decoy.strategy` (244, `Reverse` default; `DiannShift`/`None` + produce no decoy and `DiannShift` is rejected by validate). +- Tests: `trypsin_p_cleaves_after_kr`, `reverse_decoy_keeps_cterm`, + `scramble_is_deterministic`. + +### peptidoforms (Stage A2) + +- Source: `stages/peptidoforms.rs`. Entry `run` (`peptidoforms.rs:68`); + `proforma` (`peptidoforms.rs:22`). +- IO: `peptides.parquet` -> `peptidoforms.parquet` (keyed on peptidoform `id` + + `base_peptide_id`). Expands stripped peptides into peptidoforms with fixed + + variable mods and charges, emits ProForma-lite. Known limitation: a second mod + at the same position is dropped; no terminal mods. +- Knobs: `peptidoforms.fixed_mods` (`config.rs:283`), `variable_mods` (284), + `max_variable_mods` (285), `charge_min` / `charge_max` (286-287), + `unknown_modification` (289, `Error` default). +- Tests: `proforma_places_mods`, `combos_bounded`. + +### predict-frag (Stage C) + +- Source: `stages/predict_frag.rs`. Entry `run` (`predict_frag.rs:50`); + candidate_id mint after precursor-m/z sort (`predict_frag.rs:137,163`). +- IO: `peptidoforms.parquet` -> `fragment_library_precursors.parquet` + + `fragment_library_fragments.parquet` (the library). Computes precursor and b/y + fragment m/z (shared mass model), intensities (native or MS2PIP), iRT (native + or DeepLC), top-N, and builds the contiguous `candidate_id` the inverted index + needs. This is the birth of candidate identity. +- Knobs: `predict_frag.predictor` (`config.rs:334`), `rt_predictor` (335), + `top_n_fragments` (341), `charge2_from_precursor_charge` (340), + `ms2pip_model`/`ms2pip_python`/`deeplc_python` (342-344). +- Tests: none at stage level. + +### search-seed (Stage S) + +- Source: `stages/search_seed.rs`. Entry `run` (`search_seed.rs:45`); masscal.json + writer (`search_seed.rs:186`). +- IO: library + `spectra_ms2.parquet` -> `seed_psms.parquet` (best-per-candidate) + + `.masscal.json` `{frag_ppm_offset, frag_tol_ppm, n_dev}`. Native + Sage-lite hyperscore for calibration only (not a library filter). Drives per-run + mass recalibration; fallback `{0.0, cfg.fragment_tol_ppm}` when `n_dev < 20`. +- Knobs: `search_seed.fdr_seed` (`config.rs:368`), `fragment_tol_ppm` (370), + `min_matched_peaks` (373/374), `report_psms` (371), `matcher` (381). + `precursor_tol_ppm` (369) is a dead knob (warns). +- Tests: none at stage level. + +### rt-im-train (Stage B) + +- Source: `stages/rt_im_train.rs`. Entry `run` (`rt_im_train.rs:28`); run_windows + writer (`rt_im_train.rs:135`), cal.json writer (`rt_im_train.rs:149`). +- IO: `seed_psms.parquet` + library -> `run_windows.parquet` + (`candidate_id, rt_pred_cal, rt_lo, rt_hi, im_*` (IM null)) + `.cal.json` + `{method, slope, intercept, w_rt, p_rt, multiplier, n_train, calibration_status}`. + Fits predicted_irt -> observed RT (linear always, LOESS when configured), sets a + single global RT window half-width `w_rt` applied uniformly to every candidate + (no per-candidate uncertainty). Optional DeepLC multitask fine-tune first. +- Knobs: `rt_im_train.calibration_method` (`config.rs:401`, `None` rejected), + `q_train` (402), `p_rt` / `rt_window_multiplier` (404-405), + `min_seed_for_calibration` (406), `loess_span` (408), `fallback_rt_window_s` + (410), `finetune_deeplc` (416). `tolerance_regime` (400) dead (warns). +- Tests: none at stage level. + +### extract (Stage D) - the core stage + +- Source: `stages/extract.rs`, `index.rs`, `matchers/fragindex.rs`. Entry `run` + (`extract.rs:222`); `Hit` (`extract.rs:80`), accumulator `acc` + (`extract.rs:302`), `extract_accumulate_windows` (`extract.rs:128`), per-candidate + parallel map `cand_hits`/`into_par_iter` (`extract.rs:566`), `CandOut` + (`extract.rs:569`), apex selection loop (`extract.rs:718`), smoothed rolling + count (`extract.rs:681`), signature ions (`extract.rs:710`), co-elution run + (`extract.rs:736`), acquisition-scan grid (`extract.rs:632`), chrom emission + (`extract.rs:851`). Index: `FragIndex` (`fragindex.rs:24`), `probe_peak` + (`fragindex.rs:152`), `Library::candidate_range` (`index.rs:233`), `cand_frags` + (`index.rs:208`), `page_search` (`index.rs:242`). +- IO: library + `run_windows.parquet` + spectra + masscal -> `psms_extracted.parquet` + (20 cols, one apex PSM per surviving candidate, write `extract.rs:960`) + + `chromatograms.parquet` (7 cols, `LargeListF32` traces, keyed by candidate_id, + write `extract.rs:986`). Peak-major over the SoA inverted index; a cheap-to- + expensive cascade (distinct-fragment presence -> co-elution run -> matched + fraction -> Pearson gate) accepts candidates. Emits exactly one apex per + candidate (`extract.rs:718`); no peak dimension exists yet. +- Knobs (all `ExtractConfig`, `config.rs:436-533`): `frag_tol_ppm` (440), + `prec_tol_ppm` (441), `presence_min_matched` (443), `presence_min_fragments` + (445), `presence_min_coelution` (447), `min_frag_corr` (452/453), + `min_matched_fraction` (458), `min_coelution_run` (515), `fixed_scan_window` + (439), `apex_top_fragments` (465), `apex_rt_prior_s` (469), `apex_count_tol` + (474), `apex_count_window` (484), `emit_window_grid` (489), `peak_claim` (498), + `emit_contested_features` (503), `peak_claim_margin` (507), `matcher` (509), + `ms1_rescue` (521), plus the new `retain_top_peaks` (528) and + `emit_candidate_audit` (533). Dead: `k_select` (491), `max_fragment_charge` + (495), `scan_scale` (438), `ScanWindowMode::PeakWidthDerived`. +- Tests: none at stage level (exercised only via full `run`). + +### features (Stage E) + +- Source: `stages/features.rs` + `stages/features/*.rs`. Entry `run` + (`features.rs:491`); `FAMILIES` (49), `active_features` (201), + `feature_schema_id` (220), `FeatureSchema` (226), `Evidence` (269), + `build_evidence` (336), `fragment_features` (915), `peak_bounds` (853), + `prelim_score` (722), per-PSM parallel block (608), cross-charge block (573), + `write_pin` (1216). See `FEATURE_REGISTRY.md` for the 17-family breakdown. +- IO: `psms_extracted.parquet` + `chromatograms.parquet` (+ `seed_psms.parquet`) + -> `features.parquet` + `.schema.json` + `run.pin`. Drop-free: one + output row per input PSM; missing chromatograms yield default/zero vectors + (see drop table). Emits `prelim_score` (bookkeeping, not a feature column). +- Knobs (`FeaturesConfig`, `config.rs:560`): `set` (561, `FeatureSet`), + `coelution_corr_threshold` (562), `bound_features` (567), `bound_peak_fraction` + (571). `prec_tol_ppm` (563) is declared but unused here. Several family + thresholds are hardcoded consts, not config (`FRAG_TOL_PPM` + `mass_accuracy.rs:44`, `MAXLAG` `coelution.rs:62`, `MIN_FRAGS/MIN_SCANS` + `order_consistency.rs:38`). +- Tests: `feature_sets_sized` (`features.rs:1263`, asserts tier sizes), + `peptide_length_ignores_mods`, `xcorr_aligned_traces`, and per-family tests in + `chromatographic.rs` and `order_consistency.rs`. Most family modules + (similarity, entropy, coelution, interference, mass_accuracy, ion_series, ms1, + rt, novel, nonzero, peak_scans) have no unit tests. + +### compete (Stage F, part 1) + +- Source: `stages/compete.rs`. Entry `run` (`compete.rs:28`); `CompeteParams` + (`compete.rs:20`), `pform_id` (51), winner map (72), `label_code` (74), key + match (79), winner selection (92-95), `keep` (101). +- IO: `features.parquet` -> `psms_competed.parquet` (survivors + carried feature + schema). Selects one winner per competition group by `prelim_score`. The label + is part of the group key (`compete.rs:74-78`), so a target never competes + against its own decoy (the decoy null is preserved). Runs BEFORE rescore + (`run.rs:243` between features `run.rs:231` and rescore `run.rs:252`); the + winner is chosen on `prelim_score`, not on any rescorer output. +- Knobs (`CompeteConfig`, `config.rs:601`): `group_by` (607, `CompeteGroupBy` + Precursor/Apex/PeptidoformCharge), `apex_rt_tolerance_s` (608), plus the new + scaffolded `mode`/`margin`/`unique_evidence_min_fragments`/`emit_competition_audit` + (613-623). +- Tests: `features_compete_rescore_run_on_crafted_input` + (`tests/pipeline.rs:166`, default Precursor grouping only). + +### rescore (Stage F, part 2) + target-decoy FDR + +- Source: `stages/rescore.rs`, `rescoring.rs`, `fdr.rs`. Entry `run` + (`rescore.rs:41`); `percolator_lite` (`rescoring.rs:98`), `logreg_fit` + (`rescoring.rs:48`), `fit_standardizer` (`rescoring.rs:14`), `RescoreInput.fold_key` + (`rescoring.rs:90`), `grouped_q` (`rescore.rs:367`), `classify_entrapment` + (`rescore.rs:335`), `QMode` (`rescore.rs:22`), `run_mokapot` (`rescore.rs:485`), + `run_entrapment_gbm` (`rescore.rs:427`). FDR: `target_decoy_q` (`fdr.rs:7`, + numerator `n_decoys + 1`, tie-block collapsed, monotonized), `entrapment_q` + (`fdr.rs:63`). +- IO: `psms_competed.parquet` (concatenated across inputs) -> `psms_scored.parquet` + with ALL rows incl decoys (`rescore.rs:251`, no drop). Attaches PSM / peptide + (`base_peptide_id`) / protein-group / global q-values. Native `percolator_lite` + folds by `base_peptide_id % folds` (no peptide leaks across folds); + standardizer fit on train rows only. +- Knobs (`RescoreConfig`, `config.rs:723`): `classifier` (724, `RescorerKind` + NativeTda/Mokapot/Percolator/Entrapment), `folds` (725), `train_fdr` (726), + `num_iter` (728), `python` (729), `percolator_bin` (730), + `entrapment_marker`/`entrapment_exclude`/`entrapment_contaminant_markers`/ + `entrapment_ratio` (734-748), `strict` (754). +- Tests: `perfect_separation_q_is_conservative_plus_one`, `tied_scores_share_one_q`, + `entrapment_q_ranks_real_above_spike_in`, `separates_targets_from_decoys`. Gap: + no test covers `rescore::run` itself, the sidecar branches, or `QMode::Entrapment` + end to end. + +### report + +- Source: `stages/report.rs`. Entry `run` (`report.rs:49`); peptide gate + (`report.rs:92`), peptide dedup (97), protein gate (118), protein dedup (121). +- IO: `psms_scored.parquet` -> `peptides.tsv` + `proteins.tsv`. This is where FDR + gating actually removes rows from output (targets only, q <= `q_threshold`). +- Knobs: `q_threshold` (`ReportParams`, `report.rs:19`; `run` uses 0.01). +- Tests: `strip_mods_and_decoy` (`report.rs:143`). + +### quant (Stage G, beyond-MVP) and quant-lfq + +- Source: `stages/quant.rs`. Entry `run` (`quant.rs:147`), `run_lfq_combine` + (`quant.rs:413`). Trapezoid XIC integration + top-N sum + protein-group rollup + + per-fragment export; MaxLFQ/directLFQ cross-run via `quant-lfq`. +- Knobs (`QuantConfig`, `config.rs:682`): `q_threshold` (682), `top_n_fragments` + (684), `top_n_peptides` (686), `rollup` (688), `bound_peak` (690), + `peak_fraction` (694), `peak_grace` (698), `peak_window_mode` (700), + `reliable_q` (703). +- `align` (Stage D2, `align.rs:52`) and MBR (Stage D3) are beyond-MVP / stub and + not in the `run` chain. + +## 3. Where candidates are dropped + +Every removal or decision point that can lose a candidate precursor, grouped by +stage, collected from all stage maps. `Reason` is the matching +`RejectionReason::code`. Stages before predict-frag have no `candidate_id`, so +their drops are audited at `base_peptide_id` / peptidoform-id level. + +### digest (Stage A) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `digest.rs:72` | `missed = j-i-1 > cfg.missed_cleavages` -> break | peptide spanning too many missed cleavages never enumerated | `PEPTIDE_NOT_GENERATED` | +| `digest.rs:77` | `len < min_len` or `len > max_len` -> continue | peptide outside length bounds dropped | `PEPTIDE_NOT_GENERATED` | +| `digest.rs:81` | non-standard residue (B/J/O/U/X/Z) -> continue | ambiguous-residue peptide dropped | `PEPTIDE_NOT_GENERATED` | +| `digest.rs:95` | `make_decoy: n < 3` -> None | no decoy minted for a <3-residue target | `PEPTIDE_NOT_GENERATED` | +| `digest.rs:117` | `DecoyStrategy::DiannShift`/`None` -> None | no sequence-rewrite decoy (DiannShift unrealized; rejected by validate) | `PEPTIDE_NOT_GENERATED` | +| `digest.rs:162` | target already seen (dedup by stripped seq) | duplicate peptide merged; collapses protein multiplicity | `PEPTIDE_NOT_GENERATED` | + +### peptidoforms (Stage A2) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `peptidoforms.rs:80` | `unimod_mass(mod).is_none()` -> `bail!` | unknown fixed/variable mod aborts the run (hard error) | `MODIFICATION_NOT_ALLOWED` | +| `peptidoforms.rs:22` | second mod at the same residue position | silently keeps only the first mod (documented limitation) | `MODIFICATION_NOT_ALLOWED` | +| `peptidoforms.rs:121` | `z` outside `[charge_min, charge_max]` | charge states outside the range never enumerated | `CHARGE_OUT_OF_RANGE` | + +### predict-frag (Stage C) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `predict_frag.rs:73` | `parse_peptidoform(pform)` is Err | ProForma parse failure dropped before candidate_id | `MODIFICATION_NOT_ALLOWED` | +| `predict_frag.rs:83` | `frags.is_empty()` | peptidoform with no b/y fragments dropped | `NO_VALID_FRAGMENTS` | +| `predict_frag.rs:134` | `retain(!frags.is_empty())` after top-N | candidate left with zero fragments after top-N truncation removed | `NO_VALID_FRAGMENTS` | + +### search-seed (Stage S) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `search_seed.rs:81` | seed matched count < `min_matched_peaks` | seed PSM not emitted (calibration only); does NOT remove the candidate, it just lacks a `seed_score`/`seed_identified` feature | `NO_FRAGMENT_TRACES` (soft) | + +### extract (Stage D) + +The three accumulation paths (parallel window, serial non-two-pass, two-pass) +repeat the same three peak-level gates. `hi <= lo` (empty candidate range), +the RT gate, and empty claimants are per-peak skips that only cause a candidate to +be dropped if all of its collisions are skipped (the implicit non-materialization +at `extract.rs:302/554`). The `return None` gates are the explicit per-candidate +drops. + +| file:line | condition | effect | reason | +|---|---|---|---| +| `extract.rs:302` / `:554` | candidate never inserted into `acc` (no in-window, in-RT fragment collision anywhere) | silent, irreversible non-materialization; never appears in `psms_extracted`. The single largest invisible drop class | `NO_FRAGMENT_TRACES` | +| `extract.rs:154` / `:327` / `:390` / `:433` | `hi <= lo`: isolation-window group maps to an empty candidate range | window contributes no hits to any candidate | `WRONG_ISOLATION_WINDOW` | +| `extract.rs:169` / `:341` / `:402` / `:446` | `rt < rt_lo[c]` or `rt > rt_hi[c]` | peak collision discarded for the candidate (outside calibrated RT window) | `RT_PRUNED` | +| `extract.rs:174` / `:348` / `:453` | `claimants.is_empty()` | peak matched no in-range/in-RT candidate; peak-level skip | `NO_FRAGMENT_TRACES` | +| `extract.rs:600` | `distinct.len() < presence_min_matched.max(1)` | first explicit per-candidate drop (tier-b presence) | `NO_FRAGMENT_TRACES` / `NO_VALID_FRAGMENTS` | +| `extract.rs:750` | `distinct.len() < presence_min_fragments` (acceptance sub-cond 1) | too few distinct matched fragments | `NO_VALID_FRAGMENTS` | +| `extract.rs:751` | `best_run < scan_window` (= `fixed_scan_window.max(1)`) | co-elution run shorter than the consecutive-scan floor | `NO_PEAK_GROUP` | +| `extract.rs:752` | `best_run < min_coelution_run` (default 0 = off) | transient (likely interferent) co-elution | `NO_PEAK_GROUP` | +| `extract.rs:753` | `matched_fraction < min_matched_fraction` (default 0 = off) | too small a fraction of predicted fragments observed | `NO_VALID_FRAGMENTS` | +| `extract.rs:808` | `pearson(obs_apex, pred) < min_frag_corr` and not MS1-rescued | apex fragment pattern disagrees with the predicted spectrum | `PEAK_NOT_SELECTED` / `NO_VALID_FRAGMENTS` | +| `extract.rs:718` | single-argmax apex; only the best scan-group emitted | secondary co-eluting peaks of the same candidate are never emitted (top-1 only). No trigger today; becomes reachable with top-K | `PEAK_NOT_SELECTED` (latent) | + +### features (Stage E) - not drops, but silent zeroing + +The features stage drops nothing (one row per input PSM), but several branches +turn a candidate into a maximally decoy-like all-zero vector with no signal that +the zero is undefined rather than measured. These are the "undefined-zero" +ambiguities the sensitivity work targets. + +| file:line | condition | effect | reason (undefined-zero) | +|---|---|---|---| +| `features.rs:611` | `chrom.get(cid)` None/empty | `FragFeatures::default()`: all legacy fragment/coelution/mass/interference features 0.0 | `NO_FRAGMENT_TRACES` | +| `features.rs:644` | Extended active and no chromatogram | entire extended battery zeroed for the candidate | `NO_FRAGMENT_TRACES` | +| `features.rs:748` | active column has no `fmap` entry | column filled with 0.0 (masks a missing/misnamed feature) | `REMOVED_DURING_REPORTING` | +| `features.rs:80` | extended name collides with reserved / repeats | FEATURE-column drop (not a candidate drop): `spectral_angle`, `rt_error_abs`, `novel::peptide_length`, `novel::seed_identified` removed from the schema | `REMOVED_DURING_REPORTING` | +| `features.rs:862` | `peak_bounds` apex at zero height | apex relocated to global max; changes peak-bounded feature values | `PEAK_NOT_SELECTED` | +| family early returns | `entropy.rs:124` (k==0), `chromatographic.rs:72` (empty axis), `mass_accuracy.rs:139` (no ppm), `order_consistency.rs:122` (<3 frags/scans), `ms1.rs:242` (`ms1_xic.len() < 3`) | family returns all-zero vector; the last is dead until extract persists `ms1_xic` | `NO_PEAK_GROUP` / `NO_VALID_FRAGMENTS` | + +### compete (Stage F, part 1) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `compete.rs:95` | same-key group, target loser: `prelim[i] <= prelim[w]` (ties keep first-seen) | lower-prelim charge/mod/localization sibling of a target removed before FDR, no audit row | `OUTCOMPETED_BY_TARGET` | +| `compete.rs:95` | same-key group, decoy loser (label in key) | lower-prelim decoy sibling removed; thins the decoy null symmetrically (keeps FDR trustworthy) | `OUTCOMPETED_BY_DECOY` | +| `compete.rs:82` | Apex mode: two peaks of same base+label round to the same `apex_rt` bucket | a genuinely distinct RT peak of the same peptide is dropped if it shares the bucket | `PEAK_NOT_SELECTED` | +| `compete.rs:46` | `PeptidoformCharge` mode and `charge` column absent | whole stage `bail`s (config/precondition error, not a per-candidate drop) | run failure | + +### rescore (Stage F, part 2) + +| file:line | condition | effect | reason | +|---|---|---|---| +| `rescore.rs:251` | none | writes every PSM incl decoys; drops nothing (all real drops deferred to report) | `REMOVED_DURING_REPORTING` (deferred) | +| `rescore.rs:415` | `grouped_q`: PSM is not the max-score row of its peptide / protein-group key | losing sibling's group q hard-set to 1.0 (silent demotion; will fail the report gate) | `OUTCOMPETED_BY_TARGET` / `FAILED_PEPTIDE_FDR` | +| `rescoring.rs:117` | degenerate fold (empty train_idx/test_idx) | those PSMs silently keep `init_score` (prelim) instead of a rescored value; scoring degradation, not a drop | (none) | + +### report + +| file:line | condition | effect | reason | +|---|---|---|---| +| `report.rs:92` | `label != "target"` or `peptide_q_value > q_threshold` | row excluded from `peptides.tsv` (decoys; targets failing peptide FDR) | `FAILED_PEPTIDE_FDR` | +| `report.rs:97` | `(peptidoform, charge)` already emitted | duplicate precursor dropped (best-q kept via sort) | `REMOVED_DURING_REPORTING` | +| `report.rs:118` | `label != "target"` or PG empty or `pg_q_value > q_threshold` | row excluded from `proteins.tsv` (no dedicated protein-FDR reason code exists) | `FAILED_PEPTIDE_FDR` / `REMOVED_DURING_REPORTING` | +| `report.rs:121` | `protein_group` already emitted | duplicate protein group dropped (best-q kept) | `REMOVED_DURING_REPORTING` | + +### configuration (run-level gates, not per-candidate) + +`Config::validate` (`config.rs:825`) blocks the whole run rather than dropping a +candidate: `DiannShift` decoy (`config.rs:827`) and `CalibrationMethod::None` +(`config.rs:836`) are hard errors; an unknown `--profile` (`config.rs:888`) aborts. +Seven dead knobs only warn (`search_seed.precursor_tol_ppm`, +`rt_im_train.tolerance_regime`, `extract.k_select`, `extract.max_fragment_charge`, +`extract.scan_scale`, `digest.decoy.source`, `digest.decoy.ratio`) and have no +runtime effect. `extract.k_select` (default 50, unimplemented) is the natural home +for a candidate-count cap (`CANDIDATE_CAP_REACHED`), which is not realized today. + +## 4. Hooks for the sensitivity work + +Exact file:line sites for the six work items, from the stage maps. + +### 4.1 Candidate audit (rejection reasons + per-candidate ledger) + +- Already available: `mumdia audit` post-hoc reconstruction (`audit.rs:64`), the + `RejectionReason` type (`rejection.rs:19`), the `emit_candidate_audit` flag + (`config.rs:533`), and the sidecar reader `load_extract_reasons` (`audit.rs:51`) + that consumes a future `.audit.parquet`. +- In-extract emission (to write that sidecar): the never-materialized cohort is + `union(candidate_range over all windows)` minus `acc.keys()` (`extract.rs:302`/ + `:554`); separate `RT_PRUNED` from `NO_FRAGMENT_TRACES` by tallying the RT gates + (`extract.rs:169`/`:341`/`:402`/`:446`); change the per-candidate parallel map + (`extract.rs:593-921`) to return `Accepted|Rejected{reason, evidence}` instead of + `Option`, carrying the failing gate (`extract.rs:600`, `:750-756`, + `:808`) and its numeric evidence. +- Pre-candidate-id drops (no candidate_id yet): audit at peptidoform-id level in + `predict_frag.rs:103` (RowOut match) and `:134` (retain). +- Compete losers: complement of `keep` at `compete.rs:101` (winner + `and_modify`/`or_insert` at `:92-98`); `emit_competition_audit` + (`config.rs:623`) is the flag. +- Rescore/report flags: per-candidate q outcomes at `rescore.rs:220-238`; the + terminal reported/rejected state at `report.rs:91-107`. +- IO: `mumdia-io/table.rs` needs a null-aware `opt_i32` reader for a nullable + reason column (`Col::OptI32` exists at `table.rs:33`; only `opt_f64`/`opt_*` + readers check `is_null` today), or use `Col::Str`/`Col::Bool`. + +### 4.2 Top-K peaks (one apex/candidate today) + +- Primary hook: the single-argmax apex loop `extract.rs:715-733`. Replace with a + local-maxima detector over the `score`/`smoothed` series; `peaks::enumerate_peaks` + (`peaks.rs:52`) is the ready-made pure function. +- Emission: `CandOut` (`extract.rs:569`), single-row append (`extract.rs:926-958`), + `psms_extracted` writer (`extract.rs:960`), chrom writer keyed by candidate_id + (`extract.rs:986`, key `:989`). Add a `peak_index`/`peak_rank` column; candidate_id + ceases to be unique. +- Downstream: `features.rs:524-543` chrom grouping must key by `(candidate_id, + peak)`; compete group key (`compete.rs:72`) and rescore grouping (`rescore.rs:194` + peptide, `:208` protein) must include `peak_rank` or explicitly collapse peaks. +- Config: `extract.retain_top_peaks` (`config.rs:528`). + +### 4.3 Fragment claimant / conflict graph + +- Per-peak conflict set: `claimants` buffer (`extract.rs:304`; per-path at + `:337-347`/`:398-408`/`:443-452`). Two-pass arbitration loop `extract.rs:456-511` + already picks a winner and computes shares; record edges `(scan_rt, peak_mz, + frag, winner_cid, {loser_cids}, shares)` there. +- Contested scalar already flows: `contested` map (`extract.rs:308`, `:479-485`, + `:814-817`) -> `contested_frac` column -> `features.rs:509` (read) / `:711` + (`peak_contested_frac` push). Extend with `n_claimants`, `unique_fragment_count`, + competitor score/margin the same way. +- Two-pass is active only when `peak_claim` is a `Coelution*` variant or + `emit_contested_features = true` (`extract.rs:311`). Config surface: + `PeakClaim` (`config.rs:187`), `emit_contested_features` (`config.rs:503`), + `peak_claim_margin` (`config.rs:507`). +- Feature side: add `Evidence` fields (`features.rs:269`) + a new family module + following the `NAMES` + `values(&Evidence)` contract and append to `FAMILIES` + (`features.rs:49`); dedup/schema/PIN flow automatically. + +### 4.4 Competition modes + +- Config scaffolding present: `CompeteConfig.mode/margin/unique_evidence_min_fragments/ + emit_competition_audit` (`config.rs:613-623`), `CompetitionMode` enum + (`config.rs:646`, `from_token` `:667`), `CompeteGroupBy` (`config.rs:681`). +- Consume in the winner loop `compete.rs:73-101`: `None`/`FeaturesOnly` set + `keep = 0..nrows` (no removal); `MarginGated` replaces the strict `prelim[i] > + prelim[*w]` (`compete.rs:95`) with a margin test; `UniqueEvidence` keeps a loser + that carries enough independent fragment evidence (needs the claimant graph and + a `unique_fragment_count` feature). `group_by` stays orthogonal (equivalence + class); `mode` is the removal policy. Not yet wired. + +### 4.5 Feature registry + +- `FAMILIES` (`features.rs:49`), `active_features` (`:201`), `extended_names` + (`:93`), `reserved_names` (`:67`), `feature_schema_id` (`:220`, blake3), + `FeatureSchema` (`:226`) persisted as `.schema.json` (`:755`) and read + back (`FeatureSchema::read`, `:233`). +- The schema is carried compete (`compete.rs:38`/`:120`) -> rescore + (`rescore.rs:64`) so the classifier never runs under a mismatched set. New + columns require no rescore change: rescore consumes every schema column + uniformly (`rescore.rs:75`, PIN at `:507`). + +### 4.6 Calibration + +- Mass: `masscal.json` writer (`search_seed.rs:186`), consumed at + `extract.rs:280-293` (offset applied to every probe m/z; learned tol sets the + index build tolerance). Uncertainty is only the tolerance width + `n_dev`. +- RT: `cal.json` writer (`rt_im_train.rs:149`), `run_windows` writer + (`rt_im_train.rs:135`). `w_rt` is a single global scalar applied uniformly + (`rt_im_train.rs:124-133`); there is no per-candidate RT uncertainty. +- Hook for per-candidate uncertainty (spec 03 §8.1 / 01 §3.5): capture the full + residual distribution into `cal.json` (`rt_im_train.rs:104-114`) and + `masscal.json` (`search_seed.rs:176-185`); add a `pred_sigma` field to + `Evidence` (`features.rs:279`) fed by a new library/chromatogram column. +- Config: `CalibrationMethod` (`config.rs:97`), `RtImTrainConfig` (`config.rs:399`), + `finetune_deeplc` (`config.rs:416`). + +## 5. Differences between the code and the sensitivity_plan docs + +- Competition is one stage, not the spec's taxonomy. Spec 04 §2 asks for seven + separate competition stages with separate logs (chromatographic peak, duplicate + precursor, peptide interference, peptidoform, localization, target-decoy, + reporting dedup). Reality: `compete.rs` does ONLY within-label winner-take-all + variant collapse on `prelim_score`, controlled by `CompeteGroupBy`. Peptide/ + protein grouping and duplicate removal are folded into `grouped_q` + (`rescore.rs:367`) and the `report` gates, not separate stages. +- Target-decoy "competition" is not a competition here. Spec 01/04 list + target-decoy competition as a distinct stage. In the code the label is part of + the compete key (`compete.rs:74-78`) precisely so targets and decoys never + compete head-to-head; the target-decoy relationship is realized as q-values in + `fdr::target_decoy_q` (`fdr.rs:7`), and the target-decoy q-value is the trusted + FDR estimate. The spec's `passed_target_decoy_competition` stage flag therefore + has no producing stage; `audit.rs` maps it to `q_value <= threshold` instead + (`audit.rs:146-155`). +- Competition happens before rescoring, not after. Spec 04 §11 and §10 recommend + competition after initial rescoring. In the code `compete` runs before `rescore` + (`run.rs:243` before `:252`) and picks the winner on `prelim_score`, a heuristic + (`features.rs:722`). A candidate a trained classifier would rank higher can be + eliminated before the classifier ever sees it. This is the single largest + in-stage sensitivity risk and the reason the `CompetitionMode::None`/`FeaturesOnly` + modes now exist (`de5ae2b`): selecting them preserves every candidate so the + rescorer, not `prelim_score`, arbitrates. Moving competition to AFTER an initial + rescoring pass (spec 04 §11) remains future work. +- One apex per candidate. Spec 01 §3.1 hypothesizes that a single apex is chosen + too early; confirmed: `extract.rs:718` emits exactly one apex PSM per candidate, + no peak dimension. `retain_top_peaks` (`config.rs:528`) and `peaks.rs` are the + answer but are not yet wired into extract. +- Registry metadata is thinner in code than in spec. Spec 03 §2 wants + `requires_calibration`, `uses_cross_run_information`, `missing_value_policy`, + and `computational_cost` per feature; the code registry (`FAMILIES`) stores only + ordered names + a `values` fn. Direction and level are documented in + `FEATURE_REGISTRY.md`/`feature_registry.yaml`, not in the code. `missing_value_policy` + is de facto "fill with 0.0" everywhere (see the features drop table), which the + spec's leakage/undefined-zero concern flags. +- Calibration is a global pre-CV fit. Spec 03 §3 requires calibration to be fit + within each training fold. The code fits standardization within-fold + (`rescoring.rs:120`) but mass recalibration, RT calibration, and the DeepLC + iRT fine-tune are one-shot whole-run fits whose derived features enter every + fold (see `FEATURE_REGISTRY.md` §4). This is a documented leakage path, not yet + addressed. +- "Candidate" scope. In the spec a candidate spans the whole search space; in the + code `candidate_id` exists only from `predict_frag` onward (minted at + `predict_frag.rs:163`). Digest/peptidoforms losses (`PEPTIDE_NOT_GENERATED`, + `MODIFICATION_NOT_ALLOWED`, `CHARGE_OUT_OF_RANGE`) precede candidate identity and + must be audited at `base_peptide_id`/peptidoform-id level, as `audit.rs` notes. diff --git a/sensitivity_plan/FEATURE_REGISTRY.md b/sensitivity_plan/FEATURE_REGISTRY.md new file mode 100644 index 0000000..c2c3db6 --- /dev/null +++ b/sensitivity_plan/FEATURE_REGISTRY.md @@ -0,0 +1,630 @@ +# MuMDIA Feature Registry + +Companion to `ARCHITECTURE_MAP.md`. This registry documents every scoring feature +MuMDIA can compute for a PSM, its family, its level, its expected monotone +direction, and its source file. It exists to satisfy the sensitivity program's +feature-registry requirement (spec `03_feature_evaluation.md` §2): before any +feature is added, removed, or ablated, the current set must be enumerated with +enough metadata to run the leakage checks (§3), the evidence ladder (§5), and the +family ablations (§4). + +A machine-readable copy of this table is `feature_registry.yaml` at the repository +root (one entry per feature: `family`, `level`, `direction`, `source_file`). + +## 1. Feature-set tiers + +The active feature set is selected by `features.set` (`FeatureSet` enum, +`rust/mumdia/crates/mumdia-core/src/config.rs:110`). Tiers are cumulative: + +| Tier | `FeatureSet` | Count | Definition | +|---|---|---:|---| +| Minimal | `Minimal` (default) | 14 | RT error, matched fragments, co-elution, apex intensity, library agreement, metadata (`MINIMAL_FEATURES`, `features.rs:149`). | +| Rich | `Rich` / `Custom` | 44 | Minimal + 30 (`RICH_EXTRA`, `features.rs:167`). `Custom` is aliased to `Rich`. | +| Extended | `Extended` | 356 | Rich + the 12-family Evidence battery (deduplicated) + 4 psms-derived / cross-candidate extras. Enabled by `--profile dia`. | + +The tier sizes are asserted by the `feature_sets_sized` test +(`features.rs:1263`): `Minimal = 14`, `Rich = 44`, and +`Extended = 14 + 30 + extended_names().len() + 4`. + +This registry documents 360 feature definitions across 17 families. Four of them +(`spectral_angle`, `rt_error_abs`, `peptide_length`, `seed_identified` in their +extended-family form) collide with reserved Minimal/Rich names and are dropped +from the scored schema (see §3 of the mechanism below), so the Extended tier +scores 356 distinct columns. + +## 2. Registry mechanism (how the schema is built and frozen) + +All feature logic lives in `rust/mumdia/crates/mumdia/src/stages/features.rs` and +the per-family modules under `stages/features/`. + +- `FAMILIES` (`features.rs:49`) is the ordered, append-only const array of the 12 + extended-battery families: `[(NAMES, values_fn)]`. Family order defines the + frozen extended-schema column order. New families are appended here. +- Each family module exposes the fixed contract `pub const NAMES: &[&str]` and + `pub fn values(&Evidence) -> Vec` of matching length. `Evidence` + (`features.rs:269`) is the per-PSM input struct handed to every family. +- `extended_names()` (`features.rs:93`) deduplicates the family names: it drops + names colliding with the reserved Minimal/Rich set (`reserved_names`, + `features.rs:67`) and cross-family repeats (keep-first). The four dropped + collisions above are removed here; their computed values are discarded. +- `active_features(set)` (`features.rs:201`) assembles the ordered active column + list for the configured tier. +- `feature_schema_id(cols)` (`features.rs:220`) is the hashed feature-schema + mechanism: blake3 of the comma-joined ordered active column names. Any change + to the set (add/drop/reorder) changes the id. +- `FeatureSchema` (`features.rs:226`) is the companion record + `{feature_columns, schema_id}`, written to `.schema.json` + (`features.rs:755`) and read back in compete and rescore + (`FeatureSchema::read`, `features.rs:233`) so the classifier never runs under a + mismatched feature set. + +`level` values: `fragment` (per-fragment spectral/co-elution kernels), +`peak` (peak-bounded chromatographic quantities), `precursor` (MS1 / charge), +`candidate` (identity/metadata), `run` (run-context). `direction` is the expected +monotone sign toward a correct target: `higher_better`, `lower_better`, +`neutral`, or `?` when undetermined. + +## 3. Feature tables by family + +Grouped by family in schema (first-appearance) order. Notes are abbreviated from +the inventory; source file is the basename under `stages/features/` (or +`features.rs` for the Minimal/Rich/extra rows). Rows marked "DROPPED from schema" +are computed but removed by the dedup at `features.rs:93` because they collide +with a reserved name. + +### minimal (14) + +Minimal tier base features (RT error, matched fragments, co-elution, apex intensity, library agreement, metadata). Always active. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `rt_error_abs` | candidate | lower_better | `features.rs` | \|apex_rt - rt_pred_cal\|; pushed at line 663 | +| `rt_error_rel` | candidate | lower_better | `features.rs` | rt_err/gradient; line 664 | +| `n_matched_fragments` | candidate | higher_better | `features.rs` | line 665 | +| `coelution_run` | peak | higher_better | `features.rs` | from psms; line 666 | +| `log_apex_intensity` | peak | higher_better | `features.rs` | line 667 | +| `frag_corr` | fragment | higher_better | `features.rs` | pearson(obs_apex,pred); CONTESTED/interference-relevant; line 668 | +| `frag_cosine` | fragment | higher_better | `features.rs` | line 669 | +| `spectral_angle` | fragment | higher_better | `features.rs` | normalized similarity in [0,1]; line 670; RESERVED name (shadows similarity::spectral_a... | +| `coelution_mean` | fragment | higher_better | `features.rs` | mean pairwise trace pearson; line 671 | +| `coelution_best` | fragment | higher_better | `features.rs` | line 672 | +| `n_coelution_above` | fragment | higher_better | `features.rs` | count pairwise corr>=coelution_corr_threshold; line 673 | +| `charge` | precursor | neutral | `features.rs` | line 674 | +| `peptide_length` | candidate | neutral | `features.rs` | line 675; RESERVED (shadows novel::peptide_length) | +| `n_proteins` | candidate | lower_better | `features.rs` | protein-group multiplicity; line 676 | + +### rich (30) + +Rich tier additions (library agreement, xcorr, ion-series sums, mass error, MS1 isotope, S/N, seed corroboration, DIA-NN profile/interference proxies). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `library_norm_manhattan` | fragment | lower_better | `features.rs` | line 677 | +| `library_rmsd` | fragment | lower_better | `features.rs` | line 678 | +| `xcorr_coelution` | fragment | lower_better | `features.rs` | mean abs xcorr lag; line 679 | +| `xcorr_shape` | fragment | higher_better | `features.rs` | line 680 | +| `sum_b_intensity` | fragment | higher_better | `features.rs` | line 681 | +| `sum_y_intensity` | fragment | higher_better | `features.rs` | line 682 | +| `diff_by_intensity` | fragment | neutral | `features.rs` | sum_b-sum_y; line 683 | +| `n_b_ions` | fragment | higher_better | `features.rs` | line 684 | +| `n_y_ions` | fragment | higher_better | `features.rs` | line 685 | +| `weighted_mass_error` | fragment | lower_better | `features.rs` | intensity-weighted mean \|ppm\|; line 686 | +| `mean_mass_error` | fragment | lower_better | `features.rs` | line 687 | +| `isotope_corr` | precursor | higher_better | `features.rs` | MS1 averagine corr; line 688 | +| `ms1_isom1_ratio` | precursor | lower_better | `features.rs` | iso -1/mono contamination; line 689 | +| `log_mono_ms1` | precursor | higher_better | `features.rs` | line 690 | +| `has_ms1` | precursor | higher_better | `features.rs` | line 691 | +| `log_sn` | peak | higher_better | `features.rs` | apex vs median trace; line 692 | +| `n_observations` | peak | neutral | `features.rs` | scans in peak; line 693 | +| `base_width_rt` | peak | neutral | `features.rs` | line 694 | +| `seed_score` | candidate | higher_better | `features.rs` | from seed_psms map; line 695 | +| `seed_identified` | candidate | higher_better | `features.rs` | seed spectrum_q<=0.01 flag; line 696; RESERVED (shadows novel::seed_identified) | +| `matched_fraction` | fragment | higher_better | `features.rs` | n_matched/n_pred; line 700 | +| `profile_cos` | fragment | higher_better | `features.rs` | DIA-NN pCos; line 702 | +| `ref_corr` | fragment | higher_better | `features.rs` | DIA-NN pTimeCorr; line 703 | +| `best_ref_corr` | fragment | higher_better | `features.rs` | line 704 | +| `low_frag_coel` | fragment | higher_better | `features.rs` | pResCorr proxy; line 705 | +| `evidence` | fragment | higher_better | `features.rs` | summed frag-vs-ref corr (DIA-NN Evidence); CONTESTED-relevant; line 706 | +| `contrast_min` | fragment | higher_better | `features.rs` | INTERFERENCE: min frag-vs-others corr (low=interfered); line 707 | +| `resid_corr` | fragment | lower_better | `features.rs` | INTERFERENCE: mean residual pairwise corr (high=shared interferent); line 708 | +| `coel_clean` | fragment | higher_better | `features.rs` | INTERFERENCE: co-elution after 1.5*r*ref capping; line 709 | +| `shadow_frac` | fragment | lower_better | `features.rs` | INTERFERENCE: fraction of intensity above cap; line 710 | + +### extended-extra (psms-derived) (1) + +Extended extra carried from the extract psms table (contested fraction), not an Evidence family. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `peak_contested_frac` | candidate | lower_better | `features.rs` | EXISTING CONTESTED FEATURE: reads extract `contested_frac` column (line 509), pushed li... | + +### extended-extra (cross-candidate) (3) + +Extended extra aggregated across candidates of one peptidoform (cross-charge corroboration); the only cross-candidate path in the feature stage. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `n_charge_states` | precursor | higher_better | `features.rs` | cross-candidate aggregate over peptidoform; line 579/729 | +| `charge_multi_flag` | precursor | higher_better | `features.rs` | >=2 charge states; line 580/730 | +| `cross_charge_intensity_log` | precursor | higher_better | `features.rs` | ln(1+summed apex of other charge states); unbounded; line 583/731 | + +### similarity (64) + +FAMILIES[0] observed-vs-library spectral agreement kernels (cosine/pearson/spearman/kendall/distances/presence/area/unbounded evidence). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `spectrum_cosine_matched` | fragment | higher_better | `similarity.rs` | | +| `spectrum_cosine_sqrt` | fragment | higher_better | `similarity.rs` | | +| `spectrum_cosine_log` | fragment | higher_better | `similarity.rs` | | +| `spectral_angle` | fragment | higher_better | `similarity.rs` | DROPPED from schema: collides with reserved minimal::spectral_angle | +| `spectral_angle_sqrt` | fragment | higher_better | `similarity.rs` | | +| `spectral_angle_matched` | fragment | higher_better | `similarity.rs` | | +| `pearson_intensity_matched` | fragment | higher_better | `similarity.rs` | | +| `pearson_intensity_log` | fragment | higher_better | `similarity.rs` | | +| `spearman_intensity` | fragment | higher_better | `similarity.rs` | | +| `spearman_intensity_matched` | fragment | higher_better | `similarity.rs` | | +| `kendall_tau_intensity` | fragment | higher_better | `similarity.rs` | | +| `dot_product_raw` | fragment | higher_better | `similarity.rs` | unbounded | +| `dot_product_norm` | fragment | higher_better | `similarity.rs` | | +| `library_recall_intensity` | fragment | higher_better | `similarity.rs` | predicted intensity fraction observed | +| `manhattan_sim` | fragment | higher_better | `similarity.rs` | 1-0.5*L1 | +| `manhattan_sqrt` | fragment | lower_better | `similarity.rs` | raw L1 of sqrt-renormalized | +| `rmsd_norm` | fragment | lower_better | `similarity.rs` | | +| `mae_norm` | fragment | lower_better | `similarity.rs` | | +| `mse_log` | fragment | lower_better | `similarity.rs` | | +| `mae_weighted_pred` | fragment | lower_better | `similarity.rs` | | +| `abs_diff_q3` | fragment | lower_better | `similarity.rs` | | +| `max_positive_residual` | fragment | lower_better | `similarity.rs` | | +| `chebyshev_dist` | fragment | lower_better | `similarity.rs` | | +| `minkowski_p3` | fragment | lower_better | `similarity.rs` | | +| `bray_curtis` | fragment | higher_better | `similarity.rs` | Ruzicka min/max | +| `bray_curtis_sqrt` | fragment | higher_better | `similarity.rs` | | +| `canberra` | fragment | lower_better | `similarity.rs` | | +| `canberra_matched` | fragment | lower_better | `similarity.rs` | | +| `wave_hedges` | fragment | lower_better | `similarity.rs` | | +| `chi_square_pearson` | fragment | lower_better | `similarity.rs` | | +| `chi_square_symmetric` | fragment | lower_better | `similarity.rs` | | +| `divergence_distance` | fragment | lower_better | `similarity.rs` | | +| `bhattacharyya_coef` | fragment | higher_better | `similarity.rs` | | +| `hellinger` | fragment | lower_better | `similarity.rs` | | +| `squared_chord` | fragment | lower_better | `similarity.rs` | | +| `harmonic_mean_sim` | fragment | higher_better | `similarity.rs` | | +| `jaccard_presence` | fragment | higher_better | `similarity.rs` | predicted vs observed>1%max presence | +| `dice_presence` | fragment | higher_better | `similarity.rs` | | +| `intensity_weighted_pearson` | fragment | higher_better | `similarity.rs` | | +| `regression_slope` | fragment | neutral | `similarity.rs` | cov(l,o)/var(l), ~1 ideal | +| `gini_diff` | fragment | neutral | `similarity.rs` | Gini(obs_matched)-Gini(lib) | +| `wasserstein_mz` | fragment | lower_better | `similarity.rs` | EMD over m/z-ordered CDFs | +| `footrule_norm` | fragment | higher_better | `similarity.rs` | | +| `rank_overlap_top3` | fragment | higher_better | `similarity.rs` | | +| `top1_frag_match` | fragment | higher_better | `similarity.rs` | argmax(l)==argmax(o) flag | +| `top1_predicted_observed` | fragment | higher_better | `similarity.rs` | | +| `frac_top3_predicted_observed` | fragment | higher_better | `similarity.rs` | | +| `count_strong_predicted_absent` | fragment | lower_better | `similarity.rs` | | +| `frac_predicted_absent` | fragment | lower_better | `similarity.rs` | (n_pred-n_matched)/n_pred | +| `cosine_area` | peak | higher_better | `similarity.rs` | peak-XIC trapezoid area vs lib | +| `pearson_area` | peak | higher_better | `similarity.rs` | | +| `spectral_angle_area` | peak | higher_better | `similarity.rs` | | +| `cosine_fullwindow` | peak | higher_better | `similarity.rs` | full-window area vs lib | +| `stein_scott_weighted_dot` | fragment | higher_better | `similarity.rs` | mz^3 * intensity^0.6 weighted cosine | +| `log_dot_product` | fragment | higher_better | `similarity.rs` | UNBOUNDED evidence (anti-saturation) | +| `spectral_log_evidence` | fragment | higher_better | `similarity.rs` | UNBOUNDED DIA-NN Evidence analog | +| `scribe_score` | fragment | higher_better | `similarity.rs` | EncyclopeDIA Scribe, UNBOUNDED | +| `log_dot_product_area` | peak | higher_better | `similarity.rs` | area twin | +| `spectral_log_evidence_area` | peak | higher_better | `similarity.rs` | | +| `scribe_score_area` | peak | higher_better | `similarity.rs` | | +| `cosine_high_ordinal` | fragment | higher_better | `similarity.rs` | ordinal>=3 only (drop co-isolation-prone b1/b2/y1/y2) | +| `cosine_robust_trim1` | fragment | higher_better | `similarity.rs` | cosine after dropping worst-residual fragment; interference trajectory | +| `cosine_robust_trim2` | fragment | higher_better | `similarity.rs` | | +| `cosine_robust_trim3` | fragment | higher_better | `similarity.rs` | | + +### entropy (18) + +FAMILIES[1] spectral-entropy / divergence features (Li entropy sim, JSD/KL/cross-entropy, obs/pred entropy). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `spectral_entropy_similarity` | fragment | higher_better | `entropy.rs` | Li entropy sim | +| `weighted_spectral_entropy_similarity` | fragment | higher_better | `entropy.rs` | | +| `spectral_entropy_similarity_sqrt` | fragment | higher_better | `entropy.rs` | | +| `spectral_entropy_similarity_topk` | fragment | higher_better | `entropy.rs` | top-6 by predicted | +| `spectral_entropy_similarity_area` | peak | higher_better | `entropy.rs` | | +| `jensen_shannon_divergence` | fragment | lower_better | `entropy.rs` | | +| `jeffreys_divergence` | fragment | lower_better | `entropy.rs` | symmetric KL | +| `kl_obs_pred` | fragment | lower_better | `entropy.rs` | | +| `kl_pred_obs` | fragment | lower_better | `entropy.rs` | | +| `cross_entropy_obs_pred` | fragment | lower_better | `entropy.rs` | | +| `obs_spectrum_entropy` | fragment | neutral | `entropy.rs` | | +| `pred_spectrum_entropy` | fragment | neutral | `entropy.rs` | | +| `entropy_diff` | fragment | neutral | `entropy.rs` | | +| `entropy_ratio` | fragment | neutral | `entropy.rs` | | +| `obs_normalized_entropy` | fragment | neutral | `entropy.rs` | Pielou evenness | +| `normalized_entropy_diff` | fragment | neutral | `entropy.rs` | | +| `residual_spectrum_entropy` | fragment | lower_better | `entropy.rs` | | +| `entropy_weight_obs` | fragment | neutral | `entropy.rs` | Li exponent | + +### coelution (38) + +FAMILIES[2] fragment-vs-reference and pairwise co-elution + cross-correlation-lag stats + b/y and charge cross co-elution. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `frag_ref_corr_mean` | fragment | higher_better | `coelution.rs` | | +| `frag_ref_corr_obsweighted` | fragment | higher_better | `coelution.rs` | | +| `frag_ref_corr_min` | fragment | higher_better | `coelution.rs` | | +| `frag_ref_corr_std` | fragment | lower_better | `coelution.rs` | | +| `frag_ref_corr_sq_mean` | fragment | higher_better | `coelution.rs` | | +| `frag_ref_corr_topk_weighted` | fragment | higher_better | `coelution.rs` | | +| `n_frag_ref_corr_above_0_9` | fragment | higher_better | `coelution.rs` | | +| `frac_frag_ref_corr_above_0_8` | fragment | higher_better | `coelution.rs` | | +| `frag_ref_corr_mean_full` | fragment | higher_better | `coelution.rs` | full window | +| `full_vs_peak_corr_gain` | fragment | neutral | `coelution.rs` | | +| `pairwise_coelution_weighted` | fragment | higher_better | `coelution.rs` | == coelution_weighted_mean alias (emitted once) | +| `pairwise_coelution_min` | fragment | higher_better | `coelution.rs` | | +| `pairwise_coelution_median` | fragment | higher_better | `coelution.rs` | | +| `pairwise_coelution_std` | fragment | lower_better | `coelution.rs` | | +| `pairwise_coelution_frac_negative` | fragment | lower_better | `coelution.rs` | INTERFERENCE indicator | +| `pairwise_coelution_hi` | fragment | higher_better | `coelution.rs` | top-half-pred fragments | +| `pairwise_coelution_lo` | fragment | higher_better | `coelution.rs` | | +| `coelution_hi_lo_contrast` | fragment | neutral | `coelution.rs` | | +| `coelution_corr_entropy` | fragment | lower_better | `coelution.rs` | | +| `xcorr_shape_mean` | fragment | higher_better | `coelution.rs` | | +| `xcorr_shape_min` | fragment | higher_better | `coelution.rs` | | +| `xcorr_shape_std` | fragment | lower_better | `coelution.rs` | | +| `xcorr_lag_mean_abs` | fragment | lower_better | `coelution.rs` | | +| `xcorr_lag_std` | fragment | lower_better | `coelution.rs` | | +| `xcorr_lag_iqr` | fragment | lower_better | `coelution.rs` | | +| `xcorr_lag_frac_zero` | fragment | higher_better | `coelution.rs` | | +| `xcorr_lag_max_abs` | fragment | lower_better | `coelution.rs` | | +| `xcorr_lag_entropy` | fragment | lower_better | `coelution.rs` | | +| `ref_xcorr_lag_mean` | fragment | lower_better | `coelution.rs` | each frag vs reference | +| `ref_xcorr_shape_mean` | fragment | higher_better | `coelution.rs` | | +| `observed_sum_vs_template_corr` | fragment | higher_better | `coelution.rs` | | +| `frag_loo_ref_corr_mean` | fragment | higher_better | `coelution.rs` | leave-one-out reference; INTERFERENCE-relevant | +| `frag_loo_ref_corr_min` | fragment | higher_better | `coelution.rs` | | +| `frac_frags_apex_aligned` | peak | higher_better | `coelution.rs` | apex-dispersion-relevant | +| `top3_frag_ref_corr` | fragment | higher_better | `coelution.rs` | | +| `by_cross_coelution` | fragment | higher_better | `coelution.rs` | b-vs-y cross pairs | +| `by_cross_lag_mean` | fragment | lower_better | `coelution.rs` | | +| `charge_cross_coelution` | fragment | higher_better | `coelution.rs` | charge-1 vs charge>=2 pairs | + +### interference (26) + +FAMILIES[3] contested/interference: explained variance, residual fraction, iterative prune, area ratios, apex purity, PCA rank, competing-peak counts. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `explained_variance_ref` | fragment | higher_better | `interference.rs` | CONTESTED family | +| `profile_residual_fraction` | fragment | lower_better | `interference.rs` | | +| `n_interfered_fragments` | fragment | lower_better | `interference.rs` | | +| `corrected_vs_raw_cos` | fragment | neutral | `interference.rs` | | +| `corrected_vs_raw_ratio` | fragment | higher_better | `interference.rs` | | +| `ifs_removed_count` | fragment | lower_better | `interference.rs` | remove_ifs iterative prune | +| `ifs_removed_intensity_frac` | fragment | lower_better | `interference.rs` | | +| `ifs_corr_gain` | fragment | neutral | `interference.rs` | large gain = was interfered | +| `ifs_retained_frac` | fragment | higher_better | `interference.rs` | | +| `matched_frac_after_ifs` | fragment | higher_better | `interference.rs` | | +| `peak_to_full_area_ratio_profile` | peak | higher_better | `interference.rs` | | +| `peak_to_full_area_ratio_frag_mean` | peak | higher_better | `interference.rs` | | +| `peak_to_full_area_ratio_weighted` | peak | higher_better | `interference.rs` | | +| `out_of_peak_intensity_frac` | peak | lower_better | `interference.rs` | | +| `profile_corr_full_vs_peak_delta` | fragment | neutral | `interference.rs` | | +| `frac_frag_ref_corr_below_0_5` | fragment | lower_better | `interference.rs` | | +| `explained_apex_intensity_frac` | peak | higher_better | `interference.rs` | | +| `apex_purity` | peak | higher_better | `interference.rs` | CONTESTED: fraction of apex intensity from coherent fragments | +| `interference_apex_residual_fraction` | peak | lower_better | `interference.rs` | | +| `dominant_frag_ref_corr` | fragment | higher_better | `interference.rs` | | +| `explained_variance_ratio` | peak | higher_better | `interference.rs` | top eigenvalue / trace of Gram (power_top) | +| `second_component_fraction` | peak | lower_better | `interference.rs` | CONTESTED: second eigenvalue fraction = 2-component (chimera) evidence | +| `profile_second_peak_ratio` | peak | lower_better | `interference.rs` | | +| `n_competing_peaks_in_window` | peak | lower_better | `interference.rs` | CONTESTED: competing chromatographic peaks in window | +| `matched_pred_intensity_fraction` | fragment | higher_better | `interference.rs` | | +| `top_pred_frag_matched` | fragment | higher_better | `interference.rs` | | + +### chromatographic (43) + +FAMILIES[4] peak shape: Gaussian/EMG fits, FWHM/asymmetry/tailing, roughness/zigzag, RT moments, apex dispersion, per-fragment gaussianity. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `gaussian_fit_r2` | peak | higher_better | `chromatographic.rs` | | +| `gaussian_cosine` | peak | higher_better | `chromatographic.rs` | | +| `emg_fit_improvement` | peak | neutral | `chromatographic.rs` | tailing indicator | +| `apex_prominence` | peak | higher_better | `chromatographic.rs` | | +| `profile_peak_snr` | peak | higher_better | `chromatographic.rs` | | +| `fwhm_seconds` | peak | neutral | `chromatographic.rs` | | +| `fwhm_to_window_ratio` | peak | neutral | `chromatographic.rs` | | +| `width_at_10pct` | peak | neutral | `chromatographic.rs` | | +| `width_ratio_10_50` | peak | neutral | `chromatographic.rs` | | +| `hwhm_asymmetry` | peak | lower_better | `chromatographic.rs` | abs toward 0 | +| `tailing_factor_usp` | peak | neutral | `chromatographic.rs` | ~1 ideal | +| `asymmetry_factor_10pct` | peak | neutral | `chromatographic.rs` | ~1 ideal | +| `apex_sharpness` | peak | higher_better | `chromatographic.rs` | | +| `apex_curvature` | peak | higher_better | `chromatographic.rs` | | +| `apex_to_boundary_ratio` | peak | higher_better | `chromatographic.rs` | | +| `apex_dominance` | peak | higher_better | `chromatographic.rs` | | +| `zigzag_index` | peak | lower_better | `chromatographic.rs` | | +| `jaggedness` | peak | lower_better | `chromatographic.rs` | | +| `roughness_2nd_deriv` | peak | lower_better | `chromatographic.rs` | | +| `n_local_maxima` | peak | lower_better | `chromatographic.rs` | | +| `modality` | peak | lower_better | `chromatographic.rs` | multimodality valley depth | +| `rt_skewness` | peak | neutral | `chromatographic.rs` | | +| `rt_excess_kurtosis` | peak | neutral | `chromatographic.rs` | | +| `rt_std_seconds` | peak | neutral | `chromatographic.rs` | | +| `mean_mode_offset` | peak | lower_better | `chromatographic.rs` | | +| `fraction_area_within_fwhm` | peak | higher_better | `chromatographic.rs` | | +| `triangle_area_similarity` | peak | lower_better | `chromatographic.rs` | | +| `baseline_fraction` | peak | neutral | `chromatographic.rs` | | +| `peak_completeness` | peak | higher_better | `chromatographic.rs` | window-edge/degeneracy indicator | +| `apex_centering_offset` | peak | lower_better | `chromatographic.rs` | apex distance from window center | +| `intensity_score` | peak | higher_better | `chromatographic.rs` | | +| `total_xic_log` | peak | higher_better | `chromatographic.rs` | | +| `frag_fwhm_cv` | fragment | lower_better | `chromatographic.rs` | | +| `frag_fwhm_mean` | fragment | neutral | `chromatographic.rs` | | +| `frag_apex_rt_dispersion` | peak | lower_better | `chromatographic.rs` | APEX DISPERSION (existing); std of per-fragment apex RTs | +| `frag_apex_rt_dispersion_weighted` | peak | lower_better | `chromatographic.rs` | APEX DISPERSION (existing), pred-weighted | +| `frag_apex_offset_from_profile_mean` | peak | lower_better | `chromatographic.rs` | APEX DISPERSION (existing) | +| `frag_gaussianity_mean` | fragment | higher_better | `chromatographic.rs` | | +| `frag_gaussianity_weighted` | fragment | higher_better | `chromatographic.rs` | | +| `frag_zigzag_mean` | fragment | lower_better | `chromatographic.rs` | | +| `sumtrace_unweighted_gaussian_r2` | peak | higher_better | `chromatographic.rs` | | +| `reference_profile_rt_entropy_peak` | peak | lower_better | `chromatographic.rs` | | +| `reference_profile_rt_entropy_ratio` | peak | neutral | `chromatographic.rs` | | + +### mass_accuracy (17) + +FAMILIES[5] fragment ppm-error distribution + positive mass-evidence. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `median_abs_frag_ppm` | fragment | lower_better | `mass_accuracy.rs` | | +| `signed_mean_frag_ppm` | fragment | neutral | `mass_accuracy.rs` | ~0 ideal | +| `ppm_std` | fragment | lower_better | `mass_accuracy.rs` | | +| `ppm_iqr` | fragment | lower_better | `mass_accuracy.rs` | | +| `ppm_range` | fragment | lower_better | `mass_accuracy.rs` | | +| `max_abs_frag_ppm` | fragment | lower_better | `mass_accuracy.rs` | | +| `intensity_weighted_abs_ppm` | fragment | lower_better | `mass_accuracy.rs` | | +| `intensity_weighted_signed_ppm` | fragment | neutral | `mass_accuracy.rs` | | +| `intensity_weighted_ppm_std` | fragment | lower_better | `mass_accuracy.rs` | | +| `lib_weighted_abs_ppm` | fragment | lower_better | `mass_accuracy.rs` | | +| `frac_frag_within_half_tol` | fragment | higher_better | `mass_accuracy.rs` | uses hardcoded HALF_TOL_PPM=10 | +| `high_ppm_intensity_frac` | fragment | lower_better | `mass_accuracy.rs` | | +| `ppm_intensity_anticorr` | fragment | lower_better | `mass_accuracy.rs` | pearson(\|ppm\|,intensity); real trends negative | +| `mass_error_mz_trend` | fragment | lower_better | `mass_accuracy.rs` | | +| `mean_abs_mz_error_da` | fragment | lower_better | `mass_accuracy.rs` | | +| `mass_evidence_gauss` | fragment | higher_better | `mass_accuracy.rs` | positive evidence; SIGMA_PPM=10 hardcoded | +| `mass_log_evidence` | fragment | higher_better | `mass_accuracy.rs` | UNBOUNDED DIA-NN Mass.Evidence analog | + +### ion_series (34) + +FAMILIES[6] b/y series coverage, ladder runs, complementarity, per-series similarity/co-elution, charge-resolved cosine. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `n_matched_b` | fragment | higher_better | `ion_series.rs` | | +| `n_matched_y` | fragment | higher_better | `ion_series.rs` | | +| `frac_matched_b` | fragment | higher_better | `ion_series.rs` | | +| `frac_matched_y` | fragment | higher_better | `ion_series.rs` | | +| `by_count_balance` | fragment | higher_better | `ion_series.rs` | | +| `by_intensity_ratio` | fragment | neutral | `ion_series.rs` | | +| `by_ratio_agreement` | fragment | lower_better | `ion_series.rs` | tanh log-odds discrepancy | +| `by_ratio_consistency` | fragment | lower_better | `ion_series.rs` | | +| `longest_b_run` | fragment | higher_better | `ion_series.rs` | | +| `longest_y_run` | fragment | higher_better | `ion_series.rs` | | +| `longest_run_max` | fragment | higher_better | `ion_series.rs` | | +| `longest_run_frac_length` | fragment | higher_better | `ion_series.rs` | | +| `series_coverage_b` | fragment | higher_better | `ion_series.rs` | | +| `series_coverage_y` | fragment | higher_better | `ion_series.rs` | | +| `sequence_coverage` | candidate | higher_better | `ion_series.rs` | | +| `series_gap_fraction` | fragment | lower_better | `ion_series.rs` | | +| `by_complement_count` | fragment | higher_better | `ion_series.rs` | | +| `by_complement_mz_consistency` | fragment | lower_better | `ion_series.rs` | ppm dev of b+y from M+2H | +| `by_complement_coelution` | fragment | higher_better | `ion_series.rs` | | +| `ordinal_intensity_concordance_y` | fragment | higher_better | `ion_series.rs` | | +| `ordinal_intensity_concordance_b` | fragment | higher_better | `ion_series.rs` | | +| `series_coelution_y` | fragment | higher_better | `ion_series.rs` | | +| `series_coelution_b` | fragment | higher_better | `ion_series.rs` | | +| `spectral_angle_b` | fragment | higher_better | `ion_series.rs` | | +| `spectral_angle_y` | fragment | higher_better | `ion_series.rs` | | +| `pearson_b` | fragment | higher_better | `ion_series.rs` | | +| `pearson_y` | fragment | higher_better | `ion_series.rs` | | +| `cosine_charge1` | fragment | higher_better | `ion_series.rs` | | +| `cosine_charge2` | fragment | higher_better | `ion_series.rs` | | +| `charge_corr_balance` | fragment | higher_better | `ion_series.rs` | | +| `mean_matched_ordinal_norm` | fragment | neutral | `ion_series.rs` | | +| `by_ion_contiguous_intensity` | fragment | higher_better | `ion_series.rs` | | +| `by_ion_contiguous_lib_frac` | fragment | higher_better | `ion_series.rs` | | +| `both_series_present` | fragment | higher_better | `ion_series.rs` | | + +### ms1 (25) + +FAMILIES[7] MS1 isotope-envelope agreement + MS1/MS2 XIC co-elution/shape (10 XIC features are 0.0 until extract persists ms1_xic). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `ms1_isotope_cosine_apex` | precursor | higher_better | `ms1.rs` | | +| `ms1_isotope_spectral_angle_apex` | precursor | higher_better | `ms1.rs` | | +| `ms1_isotope_chi2_apex` | precursor | lower_better | `ms1.rs` | | +| `ms1_isotope_manhattan_apex` | precursor | higher_better | `ms1.rs` | | +| `iso_ratio_1_0` | precursor | neutral | `ms1.rs` | | +| `iso_ratio_2_0` | precursor | neutral | `ms1.rs` | | +| `iso_plus1_ratio_dev` | precursor | lower_better | `ms1.rs` | | +| `iso_plus2_ratio_dev` | precursor | lower_better | `ms1.rs` | | +| `iso_minus_one_fraction` | precursor | lower_better | `ms1.rs` | co-isolation contamination | +| `iso_overlap_flag` | precursor | lower_better | `ms1.rs` | | +| `log_ms1_mono` | precursor | higher_better | `ms1.rs` | | +| `ms1_total_isotope_log` | precursor | higher_better | `ms1.rs` | | +| `has_ms1_signal` | precursor | higher_better | `ms1.rs` | | +| `ms1_isotope_apex_entropy_3` | precursor | neutral | `ms1.rs` | | +| `ms1_m1_entropy_contribution` | precursor | neutral | `ms1.rs` | | +| `ms1_ms2_time_corr` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted by extract | +| `ms1_ms2_envelope_time_corr` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_iso_coelution` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_ms2_apex_rt_delta` | precursor | lower_better | `ms1.rs` | 0.0 until ms1_xic persisted; was the mis-scaled/unbounded feature, now bounded d/(d+width) | +| `ms1_iso_ratio_stability` | precursor | lower_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_mono_gaussianity` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_ms2_fwhm_ratio` | precursor | neutral | `ms1.rs` | ~1 ideal; 0.0 until ms1_xic persisted | +| `ms1_isotope_corr_xic` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_envelope_over_time_corr` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | +| `ms1_isotope_xic_shape_consistency` | precursor | higher_better | `ms1.rs` | 0.0 until ms1_xic persisted | + +### rt (13) + +FAMILIES[8] RT-agreement (signed/abs/squared, gradient- and peak-width-normalized, profile-apex delta). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `rt_error_signed` | candidate | neutral | `rt.rs` | ~0 ideal | +| `rt_error_abs` | candidate | lower_better | `rt.rs` | DROPPED from schema: collides with reserved minimal::rt_error_abs | +| `rt_error_squared` | candidate | lower_better | `rt.rs` | | +| `rt_error_signed_norm_gradient` | run | neutral | `rt.rs` | | +| `rt_error_abs_norm_gradient` | run | lower_better | `rt.rs` | | +| `observed_rt_raw` | candidate | neutral | `rt.rs` | | +| `predicted_rt_raw` | candidate | neutral | `rt.rs` | | +| `observed_rt_fraction` | run | neutral | `rt.rs` | | +| `predicted_rt_fraction` | run | neutral | `rt.rs` | | +| `rt_error_over_peak_width` | peak | lower_better | `rt.rs` | | +| `rt_error_over_fwhm` | peak | lower_better | `rt.rs` | | +| `rt_diff_profile_apex` | peak | lower_better | `rt.rs` | | +| `predicted_rt_in_gradient` | run | higher_better | `rt.rs` | flag | + +### novel (12) + +FAMILIES[9] seed corroboration + precursor/charge metadata + count/intensity summaries. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `log_seed_hyperscore` | candidate | higher_better | `novel.rs` | | +| `seed_hyperscore_per_matched` | candidate | higher_better | `novel.rs` | | +| `seed_identified` | candidate | higher_better | `novel.rs` | DROPPED from schema: collides with reserved rich::seed_identified | +| `peptide_length` | candidate | neutral | `novel.rs` | DROPPED from schema: collides with reserved minimal::peptide_length | +| `precursor_charge` | precursor | neutral | `novel.rs` | | +| `charge_is_2` | precursor | neutral | `novel.rs` | flag | +| `charge_is_3` | precursor | neutral | `novel.rs` | flag | +| `charge_is_4plus` | precursor | neutral | `novel.rs` | flag | +| `precursor_mass` | precursor | neutral | `novel.rs` | | +| `log_total_matched_intensity` | fragment | higher_better | `novel.rs` | | +| `n_matched_frags` | fragment | higher_better | `novel.rs` | | +| `n_predicted_frags` | fragment | neutral | `novel.rs` | | + +### nonzero (12) + +FAMILIES[10] zero-ignoring variants (per-fragment peak-max spectral agreement, both-positive co-elution) to fix apex-scan-alignment zeros. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `frag_corr_peakmax` | fragment | higher_better | `nonzero.rs` | peak-max obs (fixes apex-scan-alignment zeros) | +| `frag_cosine_peakmax` | fragment | higher_better | `nonzero.rs` | | +| `spectral_angle_peakmax` | fragment | higher_better | `nonzero.rs` | | +| `frag_corr_matched_nz` | fragment | higher_better | `nonzero.rs` | | +| `frag_cosine_matched_nz` | fragment | higher_better | `nonzero.rs` | | +| `peakmax_apex_gain` | fragment | lower_better | `nonzero.rs` | how much apex scan understates peak; high=off-apex/chimera | +| `n_frag_present_inpeak` | fragment | higher_better | `nonzero.rs` | | +| `frac_frag_present_inpeak` | fragment | higher_better | `nonzero.rs` | | +| `coelution_mean_bothpos` | fragment | higher_better | `nonzero.rs` | | +| `coelution_mean_summpos` | fragment | higher_better | `nonzero.rs` | | +| `ref_corr_nz` | fragment | higher_better | `nonzero.rs` | | +| `profile_cos_nz` | fragment | higher_better | `nonzero.rs` | | + +### order_consistency (8) + +FAMILIES[11] prediction-free MS2-XIC rank-stability (per-scan Spearman/Kendall vs apex, top1/2 persistence, argmax entropy). + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `rank_corr_vs_apex_mean` | peak | higher_better | `order_consistency.rs` | prediction-free MS2-XIC rank stability | +| `rank_corr_vs_apex_std` | peak | lower_better | `order_consistency.rs` | | +| `rank_corr_adjacent_mean` | peak | higher_better | `order_consistency.rs` | | +| `kendall_vs_apex_mean` | peak | higher_better | `order_consistency.rs` | | +| `top1_frag_persistence` | peak | higher_better | `order_consistency.rs` | | +| `top2_order_persistence` | peak | higher_better | `order_consistency.rs` | | +| `argmax_frag_entropy` | peak | lower_better | `order_consistency.rs` | 0=one fragment dominates (clean), 1=noisy | +| `self_cosine_vs_apex_mean` | peak | higher_better | `order_consistency.rs` | | + +### peak_scans (2) + +FAMILIES[12] label-blind window-degeneracy indicators (n_peak_scans, peak_window_degenerate); the observability-model reference family. + +| name | level | direction | source file | note | +|---|---|---|---|---| +| `n_peak_scans` | peak | higher_better | `peak_scans.rs` | OBSERVABILITY: label-blind measured-scan count; disambiguates undefined vs genuine zero | +| `peak_window_degenerate` | peak | lower_better | `peak_scans.rs` | OBSERVABILITY: flag when <3 non-empty scans (window-based families collapse to 0) | +## 4. Leakage audit (spec 03 §3) + +Spec 03 §3 lists inputs that must never be used as normal scoring features, and +requires imputation/scaling/calibration/scoring to be fit within each training +fold. The rescore stage +(`rust/mumdia/crates/mumdia/src/stages/rescore.rs`) consumes every column in +`FeatureSchema.feature_columns` uniformly (`rescore.rs:64,75`) with no +label-leakage screen, so leakage prevention currently depends entirely on which +columns the features stage emits, not on a guard in the classifier. + +| Forbidden input (spec 03 §3) | Used as a feature today? | Assessment | +|---|---|---| +| Final q-value | No | `q_value` / `peptide_q_value` / `pg_q_value` are computed after scoring in rescore and written to `psms_scored`; they are not in the feature list. | +| Final posterior error probability | No | Not computed as a feature. | +| Final target-decoy winner | No | Competition winner is not fed back; `label` is used only for training targets/decoys, not as a feature column. | +| Candidate rank generated by the same model | No (with caveat) | `prelim_score` seeds the native semi-supervised loop as `init_score` (`rescore.rs:321`) but is bookkeeping, not a feature column. In the native path it is used only within-fold via `target_decoy_q` on train rows, so it is not a direct leak. It is not an output-model rank fed back to itself. | +| DIA-NN score or identification status | No | The imported DIA-NN library contributes predicted fragment intensities and iRT only; DIA-NN scores/IDs are not ingested as features. | +| Protein evidence computed after precursor scoring | No | `n_proteins` is protein-group multiplicity from the library/digest, available before scoring; `pg_q` is computed after and is not a feature. | +| Cross-run evidence using the held-out run | No | Single-run pipeline. `cross_charge_intensity_log` / `n_charge_states` aggregate charge siblings within the same run, not across runs. No cross-run feature exists. | +| Calibration values fit using the held-out candidate | RISK | RT features (`rt` family, `rt_error_*`) use per-run LOESS/linear RT calibration fit on all confident seed PSMs before cross-validation; mass-accuracy features use the per-run `masscal.json` offset/tolerance; the optional DeepLC iRT fine-tune is likewise global. The native rescorer folds by `base_peptide_id`, so a held-out peptide's own confident seed PSM can be inside the calibration anchor set. This is a genuine, documented pre-CV global-fit leakage path (fdr-rescore map, leakage risk #4). | +| Features computed globally before cross-validation | RISK | Same root cause: search-seed mass recalibration, rt-im-train RT calibration, and DeepLC fine-tune are one-shot whole-run fits, and the derived RT/mass/iRT features carry that global fit into every fold. Standardization itself is fit within-fold in the native path (`fit_standardizer` on train rows only, `rescoring.rs:120`), so the leak is in the upstream calibration, not the scaler. | + +Two further leakage risks recorded in the rescore/fdr stage map, not tied to a +single feature: + +1. Mokapot fold split is not peptide-grouped. `run_mokapot` writes the PIN keyed + on a flat row index (`rescore.rs:515-522`) and mokapot's internal brew CV can + place charge/mod variants of one peptide in different folds. The native + `percolator_lite` path avoids this by folding on `base_peptide_id` + (`rescoring.rs:107`); the mokapot sidecar carries a peptide-level leakage risk + the native path does not. +2. No label-leakage screen exists on the feature columns + (`rescore.rs:64,75`). Any future feature that encodes decoy/reverse status + (for example a sequence-derived quantity that differs systematically for the + documented reverse/scramble decoy scheme) would leak the label directly. This + is also the caution behind spec 04 §9 (competition features must be computed + symmetrically for targets and decoys). New contested/claimant features must be + audited for target/decoy symmetry before they are added. + +## 5. Gaps vs spec 03 §8 (candidate feature families to add) + +Status of each requested family against the current registry. "Exists" means a +populated, scored column; "present-but-dead" means the column is emitted but +returns a constant because an upstream input is not persisted; "partial" means +some members exist and others do not; "missing" means no analog is scored. + +| Spec 03 §8 family | Status | Evidence in the current registry / gap | +|---|---|---| +| 8.1 Uncertainty-normalized residuals | Partial | Signed/abs/squared and gradient- and peak-width-normalized RT residuals exist (`rt` family). They are NOT divided by a per-candidate RT uncertainty: `w_rt` is a single global scalar in `cal.json`, and `Evidence` carries only `e.pred` (a point prediction), no per-fragment `pred_sigma`. Mass errors (`mass_accuracy`) are not normalized by a predicted per-fragment mass uncertainty. Mobility residuals are missing (IM null, 3D MVP). | +| 8.2 Fragment evidence distributions | Mostly exists | `coelution` (38) covers median/min/IQR co-elution and `n_coelution_above` (fraction above threshold); `mass_accuracy` covers median ppm + dispersion; `similarity` covers `frac_top3_predicted_observed`, `library_recall_intensity` (explained predicted intensity), `gini_diff` (evidence concentration); `interference` covers explained observed intensity. Effective-fragment-count as a named feature is the main missing member. | +| 8.3 Apex dispersion | Exists | `chromatographic` has `frag_apex_rt_dispersion`, `frag_apex_rt_dispersion_weighted`, `frag_apex_offset_from_profile_mean`; `coelution` has `frac_frags_apex_aligned`. Precursor-to-fragment apex deviation exists in `ms1` (and the mis-scaled `ms1_ms2_apex_rt_delta` in `novel`, flagged for a fix in CLAUDE.md). | +| 8.4 Peak-shape evidence | Mostly exists | `chromatographic` (43) covers Gaussian/EMG fits, FWHM, asymmetry, tailing, roughness/zigzag, and bimodal detection (a local-maxima proxy). Missing: an explicit peak-truncation indicator, boundary-agreement, and a distinct shoulder score. Consensus profile and fraction-explained-by-consensus are computed internally (`ref_profile` / `interference`) but not all exported as named peak-shape features. | +| 8.5 Interference and contested evidence | Partial | Per-candidate proxies exist: `interference` (26) with contrast/residual/apex-purity/competing-peak counts/PCA rank, plus `peak_contested_frac` (contested intensity fraction from extract two-pass arbitration) and `resid_corr` (residual similarity). Missing the cross-candidate claimant-graph members: number of claimants per fragment, unique-fragment count, unique-fragment intensity fraction, shared-fragment count, shared-trace correlation, local/isolation-window candidate density, and correlation with competing candidate profiles. These require the fragment claimant/conflict graph (spec 04 §5), which extract computes internally (the `contested` map) but does not persist as edges. The scaffolded `CompetitionMode::UniqueEvidence` needs a `unique_fragment_count` feature that does not exist yet. This is the largest feature gap. | +| 8.6 Candidate ambiguity | Missing | No margin/rank-gap features are scored today. `compete` drops losers without recording margins; the new `emit_competition_audit` scaffolding would expose group size, winner/loser scores, and margins, but no ambiguity features (margin-from-best-alternative-peak/peptide/decoy, candidates-within-threshold, score entropy, near-isobaric count, alternative-localization count) are emitted. `n_charge_states` / `charge_multi_flag` are cross-charge corroboration, not ambiguity margins. Note spec 03 §8.6 forbids feeding the final-model margin back into the same model, so these must come from an earlier-stage score. | +| 8.7 MS1 and isotope evidence | Partial | Apex-level isotope agreement exists: `isotope_corr` (averagine correlation), `ms1_isom1_ratio` (incorrect-monoisotope / iso-1 contamination), `log_mono_ms1`, `has_ms1`, and the `ms1` family's isotope-envelope kernels. Present-but-dead: the 10 MS1/MS2 XIC co-elution/shape features (`ms1.rs`, `xic_features`) return 0.0 for every PSM until extract persists `ms1_xic`, so isotope co-elution and precursor-fragment apex agreement over the XIC are not yet live. | +| 8.8 Modification-aware evidence | Missing | No modification-count/class, prediction-training-coverage indicator, modification-specific RT/spectral residual percentile, localization ambiguity, site-determining-ion count/intensity, or alternative-peptidoform count. `novel` carries only `peptide_length` and charge metadata. The peptidoforms stage also drops a second mod at the same position and does no localization scoring, so the upstream evidence is not produced. | +| 8.9 Run context | Partial | Scans-across-peak exists (`n_observations`, `peak_scans::n_peak_scans`, `base_width_rt`). Missing as features: isolation-window width, cycle time, local signal density, local candidate density, run-level calibration quality (`w_rt` / `masscal` live in `cal.json` / `masscal.json` but are not exposed as feature columns), and run-level prediction-residual statistics. Run-context features must be normalized by run when added. | + +### Summary + +Well covered: fragment evidence distributions (8.2), apex dispersion (8.3), +peak shape (8.4), and apex-level isotope evidence (part of 8.7). + +Largest gaps, in priority order matching spec 04 and the sensitivity backlog: + +1. Cross-candidate contested/claimant-graph features (8.5) - blocked on + persisting the extract `contested` conflict edges; unlocks + `CompetitionMode::UniqueEvidence` and margin/ambiguity work. +2. Candidate-ambiguity margins (8.6) - blocked on the top-K peak dimension and + the competition-audit emission. +3. Uncertainty-normalized residuals (8.1) - blocked on persisting a per-candidate + RT/mass/iRT uncertainty (`pred_sigma`) rather than the global `w_rt` scalar. +4. Modification-aware evidence (8.8) - blocked on localization scoring in the + peptidoforms stage. +5. Live MS1 XIC isotope co-elution (8.7) - blocked on extract persisting + `ms1_xic`. +6. Run-context features (8.9) - straightforward to add from `cal.json` / + `masscal.json` and the acquisition grid. diff --git a/sensitivity_plan/IMPLEMENTATION_STATUS.md b/sensitivity_plan/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000..f956e62 --- /dev/null +++ b/sensitivity_plan/IMPLEMENTATION_STATUS.md @@ -0,0 +1,90 @@ +# MuMDIA Sensitivity Work — Implementation Status + +Living log for the autonomous sensitivity-improvement session. Updated throughout. + +## Phase 0 — Baseline (recorded) + +- **Repo:** `c:\Users\robbi\OneDrive - UGent\MuMDIA_NG` (git). +- **Branch created:** `feat/sensitivity-improvements`, forked from `fix/audit-correctness-evidence` @ `c6f268f`. + - The base branch carries the audit-correctness fixes + the 64-bit `LargeListF32` chrom fix (commit `c6f268f`). It is **not** on `main` and is **not pushed**. +- **Build target dir:** `C:/Users/robbi/mumdia_build/{debug,release}` (redirected off OneDrive by machine-local `rust/mumdia/.cargo/config.toml`; do not "fix"). +- **Build cmd:** `cd rust/mumdia && cargo build --release`. **Test cmd:** `cargo test`. +- **Baseline tests:** `cargo test` = **64 passed, 0 failed** (50 lib + 2 + 11 + 1 across binaries) — verified this session before any Phase-1 change. No pre-existing failures. +- **Baseline release build:** succeeds (`c6f268f`). +- **Empirical baseline (E. coli file, HYE library, audit-fixed binary), stripped E. coli seqs @1%:** + - mokapot logreg: 7,907 · native_tda: 6,914 · held-out entrapment (honest, unseen null): 7,909. + - DIA-NN reference on same file ≈ 10,072; pre-audit MuMDIA base ≈ 9,042. + - Gate-off probe (Pearson off, presence≥3, native): honest held-out 8,078 (+2.1%) — the Pearson gate discards ~170 real IDs recoverable only by a strong rescorer. + +## Assumptions (conservative; recorded per non-interactive policy) + +- A1. The primary comparison unit is `normalized modified sequence + charge` (spec 01 §2). MuMDIA's `peptidoform` string + `charge` is that key; `base_peptide_id` is the stripped-sequence key. +- A2. "Candidate" in the MuMDIA code == one row keyed by `candidate_id` (a library precursor peptidoform-charge). Extraction emits **one apex-level PSM per candidate** today (spec 01 §3.1 "one apex" hypothesis holds). +- A3. All new behavior is **default-off / K=1-compatible**; production defaults are unchanged unless an ablation proves a change and tests cover it. +- A4. Large candidate-level output uses **Parquet** via the existing `mumdia-io` typed `Col`/`Table` layer (dependency stack already supports it; spec P0 prefers Parquet). +- A5. Diagnostic instrumentation must be near-zero-cost when disabled (guarded by a config flag). + +## Plan (priority order, adapted to the real codebase) + +| # | Work | Spec | Status | Commit | +|---|---|---|---|---| +| 0 | Baseline + branch + status log | P0 | DONE | - | +| M | Architecture map (7-agent parallel workflow) | 01 | DONE | ARCHITECTURE_MAP.md | +| 1 | Rejection-reason enum | P0.3, 01 §4 | DONE | 2f46d6d | +| 2 | Candidate audit table + `mumdia audit` (+ metrics/waterfall) | P0.3/P0.4 | DONE | eb9da89 | +| 3 | Top-K peak enumerator + config `retain_top_peaks` (K=1 compat) + tests | P1 | PARTIAL (enumerator+config+tests done; extract wiring = NEXT_STEPS #1) | 2f46d6d | +| 4 | Competition-mode enum wired in compete (none/features-only/unique-evidence/margin-gated) | P2.4 | DONE | de5ae2b | +| 5 | `ARCHITECTURE_MAP.md`, `FEATURE_REGISTRY.md`, `feature_registry.yaml` | P4.1 | DONE | docs | +| 6 | Reference-apex top-K analysis script (Python) | P0.4, 02 §5 D | IN PROGRESS (benchmark agent) | - | +| 7 | Feature-ablation runner (Python) | P4.3 | IN PROGRESS (benchmark agent) | - | +| 8 | `BENCHMARK_GUIDE.md`, `NEXT_STEPS.md` | P7 | NEXT_STEPS done; BENCHMARK_GUIDE pending scripts | - | +| 9 | Fragment claimant / conflict features | P2.1-2.3 | DEFERRED -> NEXT_STEPS #4 (nucleus exists: contested_frac) | - | +| 10 | In-extract precise reason emitter | P0.3 | DEFERRED -> NEXT_STEPS #2 (audit reads sidecar already) | - | + +## Completed tasks + +- Phase 0 baseline recorded; dedicated branch `feat/sensitivity-improvements`. +- 7-agent architecture-map workflow (679k tokens); ARCHITECTURE_MAP.md written. +- `RejectionReason` enum (16 codes + Reported sentinel), tested. +- `mumdia::peaks::enumerate_peaks` top-K enumerator, 9 synthetic-chromatogram tests. +- Config: `retain_top_peaks` (+validation), `emit_candidate_audit`, `CompetitionMode` + + `CompeteConfig.mode/margin/unique_evidence_min_fragments/emit_competition_audit`. +- `mumdia audit` stage + subcommand: candidate_audit.parquet + metrics/waterfall, + verified on real E. coli/HYE data (2 tests). +- Competition modes wired into `compete` via pure `resolve_competition()` (7 tests). +- FEATURE_REGISTRY.md (360 features, 17 families) + feature_registry.yaml. + +## Partially completed + +- Top-K peaks: enumerator + config + tests done; NOT wired into `extract` (the + destructive-stage change). Exact hook site documented in NEXT_STEPS.md #1. + +## Tests run + +- `cargo test` baseline: 64 passed / 0 failed. +- `cargo test` after all Rust changes: 87 passed / 0 failed (68 mumdia lib incl + peaks(9)/audit(2)/compete(7), 16 mumdia-core incl rejection(5), 2 integration, 1). +- `mumdia audit` real-data smoke on out_ecoli: waterfall reproduced (below). + +## Key diagnostic result (candidate audit, E. coli file vs HYE library) + +``` +search_space = 8,334,126 extracted = 341,754 (trace_recall 4.1%) reported = 8,568 +waterfall: NO_PEAK_GROUP = 7,992,372 FAILED_PRECURSOR_FDR = 332,120 + FAILED_PEPTIDE_FDR = 1,066 REPORTED = 8,568 +``` + +Interpretation: with an 8.3M-candidate combined-species library searched against a +single-species (E. coli) sample, the vast majority of candidates correctly never +form a peak. The audit now makes every loss category countable and stratifiable, +which is the P0 prerequisite for targeting the recoverable losses (peak selection +and FDR), per the spec's decision rules (05 §5). + +## Known limitations / risks (running) + +- Empirical FDP validation here uses the single E. coli file + HYE entrapment null (one dataset); the spec's held-out multi-dataset reproduction cannot be completed in this environment without additional data. +- Real DIA-NN reference-apex tables are not loaded into the repo; the reference-apex top-K analysis is implemented as a runnable module pending a supplied DIA-NN report. + +## Recommended next steps + +- See `NEXT_STEPS.md` (written at end of session). diff --git a/sensitivity_plan/NEXT_STEPS.md b/sensitivity_plan/NEXT_STEPS.md new file mode 100644 index 0000000..f4a3a2c --- /dev/null +++ b/sensitivity_plan/NEXT_STEPS.md @@ -0,0 +1,107 @@ +# Sensitivity Program — Next Steps + +Prioritized, with exact hook sites (from `ARCHITECTURE_MAP.md`). Items are ordered +by expected sensitivity value per unit risk. Everything below builds on the +`feat/sensitivity-improvements` branch. + +## 1. Wire top-K peak retention into extraction (highest value, highest risk) + +The enumerator (`mumdia::peaks::enumerate_peaks`), config (`ExtractConfig.retain_top_peaks`), +and tests exist; extraction still emits one apex per candidate. + +- **Hook:** `extract.rs:718` (single-argmax apex loop) and `CandOut` (`extract.rs:569`), + which today carries one apex + one `Vec`. Multiply it to `Vec` (one + per retained peak) when `retain_top_peaks > 1`. +- **Approach:** after the scan-group build (`extract.rs:608`) and rolling count + (`extract.rs:681`), build the signature-ion summed profile over the grid and call + `enumerate_peaks(profile, k, bound_peak_fraction, prominence)`. For each returned + `PeakGroup`, run the existing apex/chrom emission restricted to `[start_idx, end_idx]`, + stamping a new `peak_rank` column on `psms_extracted` and `chromatograms`. +- **Compatibility:** `K == 1` must bypass the enumerator and keep the current path + (a regression test must show byte-identical `psms.parquet` for K=1 on a fixed input). +- **Downstream:** the features stage and `compete` key currently assume one row per + `candidate_id`. Add `peak_rank` to the competition key or add a peak-selection pass + (below) so multiple peaks of one candidate do not all survive to the report. +- **Validation:** run `scripts/reference_apex_topk.py` before/after; the SELF top-K + distribution predicts the achievable gain. Confirm entrapment FDP is not inflated. + +## 2. In-extract candidate-audit emitter (precise extract-stage reasons) + +`mumdia audit` already reconstructs the ladder from artifacts, but extraction losses +collapse to `NO_PEAK_GROUP`. Emit the precise reason per candidate. + +- **Hook:** the per-candidate cascade at `extract.rs:566` returns `Option`; + every `return None` (lines 600, 750-753, 808) is a mapped reason (see the drop + table in `ARCHITECTURE_MAP.md` §3). Change it to return + `Result` (or a small `enum CandEval`), collect via rayon, + and when `extract.emit_candidate_audit` is set write `.audit.parquet` + (candidate_id, rejection_reason). Also diff the library candidate range against the + accumulator keys (`extract.rs:302`) to emit `NO_FRAGMENT_TRACES` for never-materialized + candidates. +- `stages::audit::load_extract_reasons` already reads this sidecar and refines the + waterfall, so no audit-stage change is needed. +- **Cost guard:** only allocate the audit vector when the flag is set. + +## 3. Competition after an initial rescoring pass (spec 04 §11) + +`compete` runs before `rescore` on the heuristic `prelim_score` (`run.rs:243` before +`:252`), so a candidate a trained model would keep can be removed early. + +- **Interim (already available):** run with `compete.mode = none` (or `features_only`) + so competition removes nothing and the rescorer arbitrates. Benchmark this against + `winner_take_all` at matched empirical FDP (experiment E14). +- **Full:** add a first-pass rescore (native `percolator_lite` is cheap) that writes an + out-of-fold score, then run `compete` on that score instead of `prelim_score`. Keep + the fold grouping by peptidoform+charge to avoid leakage. + +## 4. Fragment claimant / conflict-graph features (spec 04 §5, P2.1-2.3) + +The nucleus exists: the per-peak `claimants` buffer (`extract.rs:304`) and the +two-pass `contested` map (`extract.rs:308`, feature `contested_frac` at +`extract.rs:814`). Extend to a candidate-level family: + +- `claimant_count` / `contested_fragment_count`, `unique_fragment_count`, + `unique_intensity_fraction`, `shared_trace_correlation`, `conflict_group_size`, + `strongest_competitor_score`, `score_margin`. +- **Hook:** compute in the two-pass arbitration (`extract.rs:456-511`) and emit as new + `psms_extracted` columns, or as a new `stages/features/*.rs` family reading the + chromatogram overlaps. Register in `feature_registry.yaml`. Keep target/decoy + computation symmetric (audit per spec 04 §9). +- These directly feed `compete.mode = unique_evidence`, which currently approximates + unique evidence from `n_matched_fragments * (1 - contested_frac)`. + +## 5. New feature families with existing data (spec 03 §8, P5) + +Lowest risk, additive. Priorities by data availability (see `FEATURE_REGISTRY.md` +gap table): + +- **Uncertainty-normalized residuals** (P5.1): `abs(rt_residual)/local_rt_sigma`, + `abs(mass_error)/local_mz_sigma`. Needs local uncertainty from calibration (item 6). +- **Apex dispersion** (P5.3): fragment-apex RT stddev/MAD, precursor-fragment apex + delta. Data already in the chromatograms; compute in a new features family. +- **Candidate ambiguity** (P5.5): margins to alternative peaks/peptides using + `prelim_score` (an earlier-stage score, not the final model, to avoid circularity). + +## 6. Two-pass calibration + local uncertainty (spec 03 §5, P3) + +`rt_im_train` (`rt_im_train.rs`) fits per-run RT calibration; mass calibration is in +`search_seed` (`masscal.json`). Add: robust two-pass precursor+fragment mass +calibration, a monotonic nonlinear RT map, and a LOCAL uncertainty estimate exported +per region. The uncertainty unlocks item 5's normalized residuals and adaptive +extraction windows (`window = max(min, scale * local_sigma)`), spec P3.3. + +## 7. Empirical-FDP-first evaluation loop + +`scripts/entrapment_holdout.py` already gives a leakage-free held-out entrapment count +on an unseen human null. Make it the acceptance gate for every change above (spec 05 +§6): a change ships only if held-out entrapment identifications rise without FDP +inflation, reproduced on a second dataset. The single E. coli/HYE file here is one +dataset; a second (the TTOF SWATH file, or a ProteoBench HYE run) is needed for the +spec's held-out reproduction criterion. + +## Cannot be completed in this environment + +- Held-out reproduction on a second dataset (needs a second labelled run loaded). +- Reference-apex recall vs DIA-NN (needs a DIA-NN report; `scripts/reference_apex_topk.py` + computes it once `--diann` is supplied). +- Ion-mobility / diaPASEF families (no 4D data). diff --git a/sensitivity_plan/README.md b/sensitivity_plan/README.md new file mode 100644 index 0000000..dc2a29c --- /dev/null +++ b/sensitivity_plan/README.md @@ -0,0 +1,81 @@ +# MuMDIA Sensitivity Evaluation and Improvement + +This documentation set describes a structured program for diagnosing and improving the sensitivity of MuMDIA relative to DIA-NN. + +The central principle is: + +> Do not optimize the final classifier until the workflow identifies where candidate precursors are lost. + +A DIA-NN-only identification can disappear because it was absent from the search space, never generated as a candidate, extracted incorrectly, assigned to the wrong chromatographic peak, outcompeted by another peptide interpretation, ranked poorly, or removed by false-discovery-rate filtering. + +## Documentation map + +1. [`01_workflow_and_gap_analysis.md`](01_workflow_and_gap_analysis.md) + Working model of the MuMDIA workflow and the major conceptual differences to investigate relative to DIA-NN. + +2. [`02_sensitivity_diagnostic_plan.md`](02_sensitivity_diagnostic_plan.md) + End-to-end identification-loss ladder, benchmark design, oracle experiments, and reporting requirements. + +3. [`03_feature_evaluation.md`](03_feature_evaluation.md) + Framework for auditing current features, testing new features, preventing leakage, and deciding which features to retain. + +4. [`04_peak_and_peptide_competition.md`](04_peak_and_peptide_competition.md) + Recommended design for chromatographic peak competition, shared-fragment competition, peptidoform competition, and target-decoy competition. + +5. [`05_experiment_matrix.md`](05_experiment_matrix.md) + Controlled experiment matrix, evaluation metrics, decision rules, and acceptance criteria. + +6. [`06_agent_implementation_backlog.md`](06_agent_implementation_backlog.md) + Agent-ready implementation tickets ordered by priority. + +## Main hypotheses + +The largest potential sensitivity losses are expected to arise from one or more of the following: + +1. A single chromatographic apex is selected before the final scoring model can compare alternative peak groups. +2. Candidate peptides can reuse the same observed fragment signal without sufficiently modeling contested evidence. +3. Charge states, modification states, or related precursor candidates may be competed too early. +4. Retention-time, mass-error, and spectral predictions are insufficiently calibrated to the current run. +5. The scorer lacks features describing unique evidence, contested evidence, candidate ambiguity, peak shape, or interference. +6. The decoy or competition design produces a high-scoring false-candidate tail, causing conservative q-value thresholds. +7. Feature-selection results are evaluated using target-decoy separation rather than empirical false discovery. + +## Recommended first implementation sequence + +1. Add full candidate observability and rejection reason codes. +2. Retain the top `K` chromatographic peak groups per precursor. +3. Build a peak-group conflict graph without removing candidates. +4. Add unique-evidence and contested-evidence features. +5. Move variant competition until after initial rescoring. +6. Introduce empirical entrapment validation. +7. Add two-pass RT and mass calibration. +8. Run grouped feature-family ablations. +9. Add conservative post-scoring interference competition. +10. Evaluate cross-run evidence only after single-run behavior is validated. + +## Evidence boundary + +The exact current internals of DIA-NN are not fully public and may differ by version. Treat DIA-NN-inspired competition strategies in these documents as hypotheses to benchmark, not as claims of exact implementation equivalence. + +Every comparison must record: + +- MuMDIA commit +- DIA-NN version +- Complete command lines +- Input hashes +- Search-space manifest +- Prediction-library version +- FDR columns and filtering rules +- Random seeds +- Runtime and memory + +## Definition of success + +The program is successful when: + +- At least 95% of DIA-NN-only precursors receive an explicit earliest-loss category. +- Candidate recall, correct-peak recall, target-ranking recall, and FDR losses are quantified separately. +- Improvements reproduce on held-out datasets. +- Identification gains are measured at matched empirical false discovery proportion. +- Modified peptides and low-intensity precursors are evaluated independently. +- Every retained feature or competition mechanism has an ablation result. From 2a056b3f5c4f4867546abe4e4842c29b8ab2cecf Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 18:14:18 +0200 Subject: [PATCH 15/40] feat(sensitivity): emit candidate_audit.parquet from `run` when enabled When extract.emit_candidate_audit is set, the `run` orchestrator invokes the audit stage after rescore, joining the run's library/psms/competed/scored artifacts into candidate_audit.parquet + its waterfall metrics. Off by default (one cheap join pass; no effect on the production chain). Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia/src/stages/run.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/rust/mumdia/crates/mumdia/src/stages/run.rs b/rust/mumdia/crates/mumdia/src/stages/run.rs index 431d510..9fe04bf 100644 --- a/rust/mumdia/crates/mumdia/src/stages/run.rs +++ b/rust/mumdia/crates/mumdia/src/stages/run.rs @@ -259,6 +259,23 @@ pub fn run(p: RunParams) -> Result<()> { })?; man.record(record_artifact(artifact::PSMS_SCORED.0, artifact::PSMS_SCORED, &scored, n, "rescore", &ch)?); + // Optional candidate audit (sensitivity program, P0.3): reconstruct the + // per-candidate identification-loss ladder from the artifact chain. Off by + // default (gated on extract.emit_candidate_audit); adds one cheap join pass. + if cfg.extract.emit_candidate_audit { + let audit_out = d("candidate_audit.parquet"); + audit::run(audit::AuditParams { + library_precursors: &lib_p, + psms: &psms, + competed: &competed, + scored: &scored, + out: &audit_out, + q_threshold: 0.01, + run_id: p.out_dir, + entrapment_substr: "", + })?; + } + let pep_q = d("peptide_quant.parquet"); let pg_q = d("protein_group_quant.parquet"); let frag_q = d("fragment_quant.parquet"); From ed44e2aa0769bdfe6e52e787adb84efe1dbc1912 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 18:36:05 +0200 Subject: [PATCH 16/40] feat(sensitivity): reference-apex top-K + feature-family ablation scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-invasive diagnostics over existing Parquet artifacts (no engine change). - scripts/reference_apex_topk.py (P0.4 / spec 02 §5 Stage D, peak oracle): builds each candidate's consensus fragment-elution profile, enumerates peaks (port of mumdia::peaks::enumerate_peaks), and reports where the selected apex ranks among peaks (top-1/3/5/10). Optional --diann report gives reference-apex-in-top-K recall (auto minutes->seconds). Smoke (20k candidates, E. coli): mean 10.35 peaks/candidate, 95.2% have >=2 peaks; selected apex is rank-1 only 52.5% (top-3 79.2%, top-5 88.3%, top-10 94.8%, in no peak 3.5%). Strong quantitative support for top-K retention. - scripts/feature_ablation.py (P4.3 / spec 03 §4-5): grouped-CV family ablation with in-fold standardization (winsorized +/-8 SD), logreg + HistGBT, targets at empirical-FDP metric via target-decoy q. Leakage-guarded (meta/q/score excluded, group by peptidoform+charge). Smoke (60k rows, logreg): most useful families similarity/rt/entropy; on this subset removing interference/rich/coelution raises the count (single-subset, one-model caveat). Co-Authored-By: Claude Opus 4.8 --- scripts/feature_ablation.py | 352 ++++++++++++++++++++++++++ scripts/reference_apex_topk.py | 445 +++++++++++++++++++++++++++++++++ 2 files changed, 797 insertions(+) create mode 100644 scripts/feature_ablation.py create mode 100644 scripts/reference_apex_topk.py diff --git a/scripts/feature_ablation.py b/scripts/feature_ablation.py new file mode 100644 index 0000000..7ab1d3f --- /dev/null +++ b/scripts/feature_ablation.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +"""Grouped feature-family ablation for MuMDIA (spec 03 Sections 4-5, backlog P4.3). + +Estimates how much each feature family contributes to sensitivity, with strict +leakage guards, on the comp.parquet feature table: + + * grouped cross-validation with the precursor (peptidoform + charge) held whole + inside a single fold, so no row of a precursor trains a model that then scores + another row of the same precursor; + * imputation and standardization fit inside each training fold only; + * target-decoy q-values computed from out-of-fold scores; the metric is the + number of target rows passing an empirical FDP threshold; + * two model families (L2 logistic regression, HistGradientBoosting). + +Ablations reported per family: full-minus-family (does removing it drop targets?) +and minimal-baseline-plus-family (does adding it to a small baseline help?). + +Reads Parquet only; deterministic (all hashing / seeds fixed). +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import sys +import warnings + +import numpy as np +import pyarrow.parquet as pq + +warnings.filterwarnings("ignore") # silence sklearn convergence chatter + +SEED = 0 + +# Columns that must never enter the feature matrix (ids, labels, targets, +# leakage-prone scores). `charge` is treated as meta per the task spec. +META_COLUMNS = { + "candidate_id", "label", "base_peptide_id", "peptidoform", "protein", + "apex_rt", "precursor_mz", "prelim_score", "charge", + "q_value", "peptide_q_value", "pg_q_value", "global_q_value", + "score", "protein_group", "source", +} + + +# --------------------------------------------------------------------------- # +def load_registry(path): + """Parse feature_registry.yaml (name -> family) without a YAML dependency. + + The file is a flat two-space-indented mapping under `features:`; each feature + name is a quoted key at indent 2 and `family:` is a quoted value at indent 4. + """ + fam = {} + cur = None + in_features = False + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + if line.rstrip("\n") == "features:": + in_features = True + continue + if not in_features: + continue + if line and not line[0].isspace() and line.strip(): + # a new top-level key ends the features block + break + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(line) - len(line.lstrip(" ")) + if indent == 2 and stripped.endswith(":"): + cur = stripped[:-1].strip().strip('"').strip("'") + elif indent == 4 and stripped.startswith("family:") and cur is not None: + val = stripped.split(":", 1)[1].strip().strip('"').strip("'") + fam[cur] = val + return fam + + +def stable_fold(key, folds): + h = hashlib.sha1(key.encode("utf-8")).hexdigest() + return int(h, 16) % folds + + +def target_ids_at_fdp(scores, is_target, fdp): + """Count target rows passing `fdp` using target-decoy q-values. + + q at a score threshold = (n_decoys + 1) / max(n_targets, 1); ties share the + group-end value; q-values are monotonised from the least confident end. + """ + order = np.argsort(-scores, kind="mergesort") + s = scores[order] + t = is_target[order].astype(np.int64) + d = 1 - t + cum_t = np.cumsum(t) + cum_d = np.cumsum(d) + fdr = (cum_d + 1.0) / np.maximum(cum_t, 1) + n = len(s) + # tie handling: every row in an equal-score group gets the group-end fdr + q = np.empty(n) + i = 0 + while i < n: + j = i + while j + 1 < n and s[j + 1] == s[i]: + j += 1 + q[i:j + 1] = fdr[j] + i = j + 1 + qv = np.minimum.accumulate(q[::-1])[::-1] + return int(((qv <= fdp) & (t == 1)).sum()) + + +# --------------------------------------------------------------------------- # +def build_model(kind): + from sklearn.ensemble import HistGradientBoostingClassifier + from sklearn.linear_model import LogisticRegression + + if kind == "logreg": + # lbfgs must be allowed to converge: on the full ~355 collinear standardized + # features it needs ~850 iterations, and an under-converged fit produces + # degenerate scores (0 targets at 1% FDP). It stops early once tol is met, so + # a high cap costs nothing on the small ablation subsets. liblinear coordinate + # descent converges but is ~15x slower here. + return LogisticRegression( + penalty="l2", C=1.0, solver="lbfgs", max_iter=2000, + tol=1e-3, random_state=SEED + ) + if kind == "hgb": + return HistGradientBoostingClassifier( + max_iter=200, learning_rate=0.1, max_depth=None, + l2_regularization=1.0, random_state=SEED + ) + raise ValueError(kind) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--features", required=True, help="comp.parquet") + ap.add_argument("--registry", required=True, help="feature_registry.yaml") + ap.add_argument("--out", default=None, help="output directory") + ap.add_argument("--folds", type=int, default=3) + ap.add_argument("--fdp", type=float, default=0.01) + ap.add_argument("--model", choices=["logreg", "hgb", "both"], default="both") + ap.add_argument("--max-rows", type=int, default=0, help="0 = all rows") + ap.add_argument("--clip-sd", type=float, default=8.0, + help="winsorize standardized features to +/- this many SD " + "(0 disables); tames outlier-driven logreg miscalibration") + args = ap.parse_args() + + from sklearn.impute import SimpleImputer + from sklearn.preprocessing import StandardScaler + + out_dir = args.out or os.path.join(os.path.dirname(os.path.abspath(args.features)), + "feature_ablation") + os.makedirs(out_dir, exist_ok=True) + + registry = load_registry(args.registry) + + # ---- load comp.parquet (optionally capped) -------------------------------- + schema = pq.read_schema(args.features) + import pyarrow as pa + numeric_cols = [ + n for n in schema.names + if pa.types.is_floating(schema.field(n).type) + or pa.types.is_integer(schema.field(n).type) + ] + feature_cols = [c for c in numeric_cols if c not in META_COLUMNS] + read_cols = feature_cols + ["label", "peptidoform", "charge"] + + if args.max_rows and args.max_rows > 0: + pf = pq.ParquetFile(args.features) + batches = [] + got = 0 + for b in pf.iter_batches(batch_size=50000, columns=read_cols): + batches.append(b) + got += b.num_rows + if got >= args.max_rows: + break + table = pa.Table.from_batches(batches).slice(0, args.max_rows) + else: + table = pq.read_table(args.features, columns=read_cols) + + n = table.num_rows + label = np.array(table.column("label").to_pylist()) + is_target = (label == "target").astype(np.int64) + peptidoform = table.column("peptidoform").to_pylist() + charge = table.column("charge").to_numpy() + + # ---- leakage guard -------------------------------------------------------- + leaked = [c for c in feature_cols if c in META_COLUMNS] + assert not leaked, f"leakage: meta columns in feature matrix: {leaked}" + + # ---- feature matrix ------------------------------------------------------- + X = table.select(feature_cols).to_pandas().to_numpy(dtype=np.float64) + X[~np.isfinite(X)] = np.nan # inf -> nan for the imputer + + # ---- family map ----------------------------------------------------------- + fam_of = {c: registry.get(c, "unknown") for c in feature_cols} + families = {} + for j, c in enumerate(feature_cols): + families.setdefault(fam_of[c], []).append(j) + fam_breakdown = {f: len(idx) for f, idx in sorted(families.items())} + + # ---- minimal baseline ----------------------------------------------------- + minimal_names = [c for c in feature_cols if registry.get(c) == "minimal"] + if len(minimal_names) >= 5: + baseline_names = minimal_names + else: + baseline_names = feature_cols[:5] # fallback: first 5 available + baseline_idx = [feature_cols.index(c) for c in baseline_names] + + print(f"[data] rows={n} targets={int(is_target.sum())} " + f"decoys={int((1 - is_target).sum())}") + print(f"[features] {len(feature_cols)} numeric feature columns " + f"(of {len(schema.names)} total)") + print(f"[families] {len(families)}: " + + ", ".join(f"{f}={c}" for f, c in fam_breakdown.items())) + print(f"[baseline] {len(baseline_idx)} features " + f"({'minimal tier' if len(minimal_names) >= 5 else 'first-5 fallback'})") + + # ---- grouped folds -------------------------------------------------------- + folds_arr = np.array( + [stable_fold(f"{peptidoform[i]}|{int(charge[i])}", args.folds) + for i in range(n)], + dtype=np.int64, + ) + fold_sizes = [int((folds_arr == k).sum()) for k in range(args.folds)] + print(f"[cv] {args.folds} grouped folds, sizes={fold_sizes}") + + # ---- configs to evaluate -------------------------------------------------- + all_idx = list(range(len(feature_cols))) + configs = {"full": all_idx, "baseline": baseline_idx} + for fam, idx in families.items(): + fam_set = set(idx) + configs[f"minus::{fam}"] = [j for j in all_idx if j not in fam_set] + configs[f"base+::{fam}"] = sorted(set(baseline_idx) | fam_set) + + models = ["logreg", "hgb"] if args.model == "both" else [args.model] + + # ---- run: per model, per fold fit shared imputer/scaler once -------------- + results = {} # (model) -> {config -> ids} + for mkind in models: + print(f"\n[model] {mkind}") + # precompute per-fold transformed full matrices + fold_data = [] + for k in range(args.folds): + te = folds_arr == k + tr = ~te + imp = SimpleImputer(strategy="median", keep_empty_features=True) + scl = StandardScaler() + Xtr = scl.fit_transform(imp.fit_transform(X[tr])) + Xte = scl.transform(imp.transform(X[te])) + if args.clip_sd and args.clip_sd > 0: + Xtr = np.clip(Xtr, -args.clip_sd, args.clip_sd) + Xte = np.clip(Xte, -args.clip_sd, args.clip_sd) + fold_data.append((tr, te, Xtr, Xte)) + + config_scores = {c: np.full(n, -np.inf) for c in configs} + for ci, (cname, idxs) in enumerate(configs.items()): + idxs = np.array(idxs, dtype=int) + for (tr, te, Xtr, Xte) in fold_data: + model = build_model(mkind) + model.fit(Xtr[:, idxs], is_target[tr]) + proba = model.predict_proba(Xte[:, idxs])[:, 1] + config_scores[cname][te] = proba + ids = {c: target_ids_at_fdp(config_scores[c], is_target, args.fdp) + for c in configs} + results[mkind] = ids + print(f" full={ids['full']} baseline={ids['baseline']}") + + # ---- build table ---------------------------------------------------------- + rows = [] + for mkind in models: + ids = results[mkind] + ids_full = ids["full"] + ids_base = ids["baseline"] + tol_n = max(1, round(0.001 * ids_full)) + for fam in sorted(families): + new_ids = ids[f"base+::{fam}"] + minus_ids = ids[f"minus::{fam}"] + rel_gain = (new_ids - ids_base) / ids_base if ids_base else 0.0 + delta_full = minus_ids - ids_full # negative => removing hurts + if delta_full < -tol_n: + rec = "KEEP" + elif delta_full > tol_n: + rec = "HARMFUL" # removing it improves the model + elif rel_gain > 0.01: + rec = "REDUNDANT_BUT_INFORMATIVE" # helps alone, redundant in full + else: + rec = "REDUNDANT" + rows.append({ + "feature_family": fam, + "n_features": len(families[fam]), + "baseline_identifications": ids_base, + "new_identifications": new_ids, + "relative_gain": round(rel_gain, 5), + "full_identifications": ids_full, + "minus_identifications": minus_ids, + "delta_vs_full": delta_full, + "model": mkind, + "recommendation": rec, + }) + + # ---- write ---------------------------------------------------------------- + csv_path = os.path.join(out_dir, "feature_ablation.csv") + json_path = os.path.join(out_dir, "feature_ablation.json") + fieldnames = ["feature_family", "n_features", "baseline_identifications", + "new_identifications", "relative_gain", "full_identifications", + "minus_identifications", "delta_vs_full", "model", "recommendation"] + with open(csv_path, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fieldnames) + w.writeheader() + for r in rows: + w.writerow(r) + summary = { + "params": {"folds": args.folds, "fdp": args.fdp, "model": args.model, + "max_rows": args.max_rows}, + "n_rows": n, + "n_targets": int(is_target.sum()), + "n_decoys": int((1 - is_target).sum()), + "n_features": len(feature_cols), + "family_breakdown": fam_breakdown, + "baseline_n_features": len(baseline_idx), + "full_identifications": {m: results[m]["full"] for m in models}, + "baseline_identifications": {m: results[m]["baseline"] for m in models}, + "table": rows, + } + with open(json_path, "w") as fh: + json.dump(summary, fh, indent=2) + + # ---- report --------------------------------------------------------------- + print(f"\n=== Feature-family ablation (fdp={args.fdp}) ===") + for mkind in models: + mrows = [r for r in rows if r["model"] == mkind] + by_delta = sorted(mrows, key=lambda r: r["delta_vs_full"]) # most negative first + print(f"\n[{mkind}] full={results[mkind]['full']} " + f"baseline={results[mkind]['baseline']}") + print(" most useful (largest target drop when removed):") + for r in by_delta[:3]: + print(f" {r['feature_family']:<28} delta_vs_full={r['delta_vs_full']:+d} " + f"rel_gain={r['relative_gain']:+.3f} {r['recommendation']}") + print(" least useful (removal helps or is neutral):") + for r in by_delta[-3:][::-1]: + print(f" {r['feature_family']:<28} delta_vs_full={r['delta_vs_full']:+d} " + f"rel_gain={r['relative_gain']:+.3f} {r['recommendation']}") + print(f"\n[written] {csv_path}") + print(f"[written] {json_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/reference_apex_topk.py b/scripts/reference_apex_topk.py new file mode 100644 index 0000000..00695db --- /dev/null +++ b/scripts/reference_apex_topk.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python +"""Top-K chromatographic-peak oracle for MuMDIA (spec 02 Section 5, backlog P0.4). + +The central sensitivity hypothesis is that MuMDIA commits to one chromatographic +apex too early, discarding the correct peak before the scorer sees it. This +diagnostic quantifies that opportunity non-invasively: it rebuilds each +candidate's consensus elution profile from the extracted fragment chromatograms +(chrom.parquet), enumerates its chromatographic peaks with the same semantics as +the Rust `enumerate_peaks` (rust/mumdia/crates/mumdia/src/peaks.rs), and asks + + SELF : where in the area-ranked peak list does the apex MuMDIA actually chose + (psms.apex_rt) fall? If it is rank 1 almost always, top-K rescue offers + little; if the true apex is often rank 2+ or in no peak, the top-K peak + model has headroom. + + REFERENCE (optional, --diann): for precursors also identified by DIA-NN, is the + DIA-NN apex RT within tolerance of one of the top-K MuMDIA peaks? This is + a peak-recall upper bound against an external reference. + +The script reads existing Parquet artifacts only; it changes no engine state and +is deterministic (fixed numeric order, no RNG). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import defaultdict + +import numpy as np +import pyarrow.parquet as pq + +# Deterministic: no stochastic component, but seed numpy for defensiveness. +np.random.seed(0) + +MOD_BRACKET = re.compile(r"\[[^\]]*\]") # ProForma-lite [UniMod:xx] / [+15.99] +MOD_PAREN = re.compile(r"\([^)]*\)") # DIA-NN (UniMod:xx) + + +# --------------------------------------------------------------------------- # +# Peak enumeration: faithful port of rust/.../peaks.rs::enumerate_peaks +# --------------------------------------------------------------------------- # +def enumerate_peaks(profile, k, bound_fraction, min_prominence_frac): + """Return peak groups as dicts, area-ranked (rank 0 = strongest). + + Each group: apex_idx, start_idx, end_idx, apex_intensity, area, rank. + Semantics match the Rust reference exactly (local maxima with strict left / + non-strict right, prominence floor, fractional-height boundary walk that also + stops at a valley, dedup of maxima inside a stronger envelope, area sort with + apex-intensity then earliest-apex tie breaks). + """ + n = len(profile) + if k == 0 or n == 0: + return [] + global_max = float(profile.max()) if n else 0.0 + if global_max <= 0.0: + return [] + prom_floor = max(min_prominence_frac, 0.0) * global_max + + maxima = [] + for i in range(n): + v = profile[i] + if v <= 0.0 or v < prom_floor: + continue + left_ok = i == 0 or v > profile[i - 1] + right_ok = i + 1 == n or v >= profile[i + 1] + if left_ok and right_ok: + maxima.append(i) + if not maxima: + return [] + + peaks = [] + bf = max(bound_fraction, 0.0) + for apex in maxima: + apex_v = profile[apex] + thr = bf * apex_v + start = apex + while start > 0: + prev = profile[start - 1] + if prev < thr or prev > profile[start]: + break + start -= 1 + end = apex + while end + 1 < n: + nxt = profile[end + 1] + if nxt < thr or nxt > profile[end]: + break + end += 1 + area = float(profile[start:end + 1].sum()) + peaks.append( + { + "apex_idx": apex, + "start_idx": start, + "end_idx": end, + "apex_intensity": float(apex_v), + "area": area, + "rank": 0, + } + ) + + # Strongest first by area, then apex intensity, then earliest apex. + peaks.sort(key=lambda p: (-p["area"], -p["apex_intensity"], p["apex_idx"])) + kept = [] + for p in peaks: + overlaps = any( + p["apex_idx"] >= q["start_idx"] and p["apex_idx"] <= q["end_idx"] + for q in kept + ) + if not overlaps: + kept.append(p) + if len(kept) == k: + break + for r, p in enumerate(kept): + p["rank"] = r + return kept + + +# --------------------------------------------------------------------------- # +# Consensus elution profile +# --------------------------------------------------------------------------- # +def build_consensus(rows, top_frags, rt_round): + """Sum the top-`top_frags` fragment traces (by predicted_intensity) onto the + union RT axis. `rows` is a list of (predicted_intensity, rt_list, int_list). + + Returns (rt_axis[np.float64], profile[np.float32]); empty arrays if no data. + """ + if not rows: + return np.empty(0), np.empty(0, dtype=np.float32) + # Select strongest predicted fragments; empty traces simply contribute nothing. + rows_sorted = sorted(rows, key=lambda r: -r[0])[:top_frags] + acc = defaultdict(float) + for _predi, rt_list, int_list in rows_sorted: + if not rt_list: + continue + for t, inten in zip(rt_list, int_list): + if inten is None: + continue + acc[round(float(t), rt_round)] += float(inten) + if not acc: + return np.empty(0), np.empty(0, dtype=np.float32) + keys = np.array(sorted(acc.keys()), dtype=np.float64) + prof = np.array([acc[k] for k in keys], dtype=np.float32) + return keys, prof + + +def locate_apex_peak(apex_rt, rt_axis, peaks, rt_tol): + """Return the rank of the peak the given apex_rt belongs to, or None. + + First, peaks whose [start_rt, end_rt] envelope contains apex_rt (nearest by + apex distance wins); otherwise the nearest peak apex within rt_tol. + """ + if not peaks: + return None + containing = [] + for p in peaks: + s_rt = rt_axis[p["start_idx"]] + e_rt = rt_axis[p["end_idx"]] + if s_rt <= apex_rt <= e_rt: + containing.append(p) + if containing: + best = min(containing, key=lambda p: abs(rt_axis[p["apex_idx"]] - apex_rt)) + return best["rank"] + best = min(peaks, key=lambda p: abs(rt_axis[p["apex_idx"]] - apex_rt)) + if abs(rt_axis[best["apex_idx"]] - apex_rt) <= rt_tol: + return best["rank"] + return None + + +# --------------------------------------------------------------------------- # +# DIA-NN reference loading +# --------------------------------------------------------------------------- # +def strip_mods(seq): + if seq is None: + return "" + s = MOD_BRACKET.sub("", str(seq)) + s = MOD_PAREN.sub("", s) + return s.strip().upper() + + +def load_diann(path): + """Load a DIA-NN report (.tsv/.parquet); return dict (stripped_seq, charge)->rt_seconds. + + Column names are matched defensively; RT minutes are detected and converted by + the caller against the MuMDIA RT range. + """ + import pandas as pd + + if path.lower().endswith(".parquet"): + df = pd.read_parquet(path) + else: + df = pd.read_csv(path, sep="\t") + cols = {c.lower(): c for c in df.columns} + + def pick(*cands): + for c in cands: + if c.lower() in cols: + return cols[c.lower()] + return None + + seq_col = pick("Stripped.Sequence", "Modified.Sequence", "Precursor.Id", "Sequence") + chg_col = pick("Precursor.Charge", "Charge") + rt_col = pick("RT", "RT.Start", "Retention.Time", "iRT") + if seq_col is None or chg_col is None or rt_col is None: + raise ValueError( + f"DIA-NN report missing required columns; found {list(df.columns)[:20]}" + ) + out = {} + rts = [] + for seq, chg, rt in zip(df[seq_col], df[chg_col], df[rt_col]): + try: + c = int(chg) + except (ValueError, TypeError): + continue + key = (strip_mods(seq), c) + try: + rtf = float(rt) + except (ValueError, TypeError): + continue + out[key] = rtf + rts.append(rtf) + return out, (min(rts) if rts else 0.0), (max(rts) if rts else 0.0) + + +# --------------------------------------------------------------------------- # +# Main +# --------------------------------------------------------------------------- # +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--psms", required=True) + ap.add_argument("--chrom", required=True) + ap.add_argument("--diann", default=None, help="optional DIA-NN report (.tsv/.parquet)") + ap.add_argument("--out", default=None, help="metrics JSON output path") + ap.add_argument("--max-candidates", type=int, default=0, + help="limit to the first N candidates (0 = all)") + ap.add_argument("--top-frags", type=int, default=6, + help="number of strongest predicted fragments summed into the consensus") + ap.add_argument("--bound-fraction", type=float, default=1.0 / 3.0) + ap.add_argument("--min-prominence", type=float, default=0.05) + ap.add_argument("--rt-tol-s", type=float, default=10.0) + ap.add_argument("--rt-round", type=int, default=3, help="RT rounding decimals for axis union") + ap.add_argument("--batch-size", type=int, default=100000) + args = ap.parse_args() + + # ---- psms: candidate -> apex_rt, and (stripped_seq,charge) for reference --- + pcols = ["candidate_id", "apex_rt", "label", "peptidoform", "charge"] + ptab = pq.read_table(args.psms, columns=pcols) + cand_ids = ptab.column("candidate_id").to_numpy() + apex_rt = ptab.column("apex_rt").to_numpy() + labels = ptab.column("label").to_pylist() + peptidoforms = ptab.column("peptidoform").to_pylist() + charges = ptab.column("charge").to_numpy() + + order = np.argsort(cand_ids, kind="mergesort") + cand_ids = cand_ids[order] + apex_rt = apex_rt[order] + labels = [labels[i] for i in order] + peptidoforms = [peptidoforms[i] for i in order] + charges = charges[order] + + n_total = len(cand_ids) + n_sel = n_total if args.max_candidates <= 0 else min(args.max_candidates, n_total) + sel_slice = slice(0, n_sel) + sel_ids = cand_ids[sel_slice] + sel_set = set(int(x) for x in sel_ids) + max_sel = int(sel_ids[-1]) if n_sel else -1 + apex_by_cand = {int(c): float(r) for c, r in zip(sel_ids, apex_rt[sel_slice])} + seqkey_by_cand = { + int(c): (strip_mods(pf), int(ch)) + for c, pf, ch in zip(sel_ids, peptidoforms[:n_sel], charges[sel_slice]) + } + mumdia_rt_min = float(np.nanmin(apex_rt[sel_slice])) if n_sel else 0.0 + mumdia_rt_max = float(np.nanmax(apex_rt[sel_slice])) if n_sel else 0.0 + + # ---- DIA-NN reference (optional) ------------------------------------------ + diann = None + if args.diann: + diann_map, drt_min, drt_max = load_diann(args.diann) + # Minutes detection: if the DIA-NN RT span is far below the MuMDIA span + # (seconds), assume minutes and scale by 60. + scale = 1.0 + if drt_max > 0 and mumdia_rt_max > 0 and drt_max <= mumdia_rt_max / 5.0: + scale = 60.0 + diann = {k: v * scale for k, v in diann_map.items()} + print(f"[diann] loaded {len(diann)} precursors, rt span " + f"{drt_min:.1f}-{drt_max:.1f} (scale x{scale:g} -> seconds)") + + # ---- stream chrom, grouped by candidate_id (sorted) ----------------------- + pf = pq.ParquetFile(args.chrom) + ccols = ["candidate_id", "predicted_intensity", "rt", "intensity"] + + # accumulators + ranks = [] # peak rank the self apex fell into (int) + n_no_peak = 0 # self apex matched no enumerated peak + n_no_chrom = 0 # selected candidate absent from chrom / empty profile + peaks_per_cand = [] # peak count per processed candidate + processed = set() + ref_hits = {1: 0, 3: 0, 5: 0, 10: 0} + ref_matched = 0 # candidates matched to a DIA-NN precursor with a profile + + def process_candidate(cid, rows): + nonlocal n_no_peak, ref_matched + rt_axis, prof = build_consensus(rows, args.top_frags, args.rt_round) + peaks = enumerate_peaks(prof, len(prof) if len(prof) else 0, + args.bound_fraction, args.min_prominence) + peaks_per_cand.append(len(peaks)) + processed.add(cid) + # SELF + a_rt = apex_by_cand.get(cid) + if a_rt is not None: + rank = locate_apex_peak(a_rt, rt_axis, peaks, args.rt_tol_s) + if rank is None: + n_no_peak += 1 + else: + ranks.append(rank) + # REFERENCE + if diann is not None and peaks: + key = seqkey_by_cand.get(cid) + d_rt = diann.get(key) if key else None + if d_rt is not None: + ref_matched += 1 + apex_rts = [rt_axis[p["apex_idx"]] for p in peaks] # rank-ordered + for K in (1, 3, 5, 10): + topk = apex_rts[:K] + if any(abs(d_rt - ar) <= args.rt_tol_s for ar in topk): + ref_hits[K] += 1 + + cur_cid = None + cur_rows = [] + stop = False + for batch in pf.iter_batches(batch_size=args.batch_size, columns=ccols): + cids = batch.column("candidate_id").to_numpy() + if len(cids) == 0: + continue + if int(cids[0]) > max_sel: + break # sorted; nothing selected remains + predi = batch.column("predicted_intensity").to_numpy() + rt_lists = batch.column("rt").to_pylist() + int_lists = batch.column("intensity").to_pylist() + for j in range(len(cids)): + cid = int(cids[j]) + if cid != cur_cid: + if cur_cid is not None and cur_cid in sel_set: + process_candidate(cur_cid, cur_rows) + if cur_cid is not None and cur_cid > max_sel: + stop = True + break + cur_cid = cid + cur_rows = [] + if cid in sel_set: + cur_rows.append((float(predi[j]), rt_lists[j], int_lists[j])) + if stop: + break + # flush last + if not stop and cur_cid is not None and cur_cid in sel_set: + process_candidate(cur_cid, cur_rows) + + # selected candidates never seen in chrom + n_no_chrom = n_sel - len(processed) + + # ---- metrics -------------------------------------------------------------- + ranks_arr = np.array(ranks, dtype=int) + # denominator: all selected candidates that had a psms apex_rt (all of them) + denom = n_sel + n_rank1 = int((ranks_arr == 0).sum()) + n_top3 = int((ranks_arr < 3).sum()) + n_top5 = int((ranks_arr < 5).sum()) + n_top10 = int((ranks_arr < 10).sum()) + # "no enumerated peak" = matched to no peak OR no chrom/empty profile at all + n_no_peak_total = n_no_peak + n_no_chrom + ppc = np.array(peaks_per_cand, dtype=float) + + metrics = { + "inputs": { + "psms": os.path.abspath(args.psms), + "chrom": os.path.abspath(args.chrom), + "diann": os.path.abspath(args.diann) if args.diann else None, + }, + "params": { + "max_candidates": args.max_candidates, + "top_frags": args.top_frags, + "bound_fraction": args.bound_fraction, + "min_prominence": args.min_prominence, + "rt_tol_s": args.rt_tol_s, + }, + "n_candidates_total": int(n_total), + "n_candidates_selected": int(n_sel), + "n_candidates_processed": int(len(processed)), + "n_no_chrom_or_empty": int(n_no_chrom), + "self": { + "denominator": int(denom), + "n_apex_matched_to_peak": int(len(ranks_arr)), + "frac_rank1": n_rank1 / denom if denom else 0.0, + "frac_top3": n_top3 / denom if denom else 0.0, + "frac_top5": n_top5 / denom if denom else 0.0, + "frac_top10": n_top10 / denom if denom else 0.0, + "frac_no_peak": n_no_peak_total / denom if denom else 0.0, + "mean_peaks_per_candidate": float(ppc.mean()) if ppc.size else 0.0, + "median_peaks_per_candidate": float(np.median(ppc)) if ppc.size else 0.0, + "frac_ge2_peaks": float((ppc >= 2).mean()) if ppc.size else 0.0, + }, + } + if diann is not None: + metrics["reference"] = { + "n_reference_matched": int(ref_matched), + "reference_apex_in_top_1": ref_hits[1] / ref_matched if ref_matched else 0.0, + "reference_apex_in_top_3": ref_hits[3] / ref_matched if ref_matched else 0.0, + "reference_apex_in_top_5": ref_hits[5] / ref_matched if ref_matched else 0.0, + "reference_apex_in_top_10": ref_hits[10] / ref_matched if ref_matched else 0.0, + } + + # ---- report --------------------------------------------------------------- + s = metrics["self"] + print("\n=== Top-K peak oracle (SELF) ===") + print(f"selected candidates : {n_sel} (processed {len(processed)}, " + f"no chrom/empty {n_no_chrom})") + print(f"mean peaks / candidate : {s['mean_peaks_per_candidate']:.2f} " + f"(median {s['median_peaks_per_candidate']:.0f})") + print(f"candidates with >=2 peaks: {s['frac_ge2_peaks']*100:.1f}% " + f"(the top-K opportunity size)") + print(f"chosen apex is peak rank1: {s['frac_rank1']*100:.1f}%") + print(f" within top-3 : {s['frac_top3']*100:.1f}%") + print(f" within top-5 : {s['frac_top5']*100:.1f}%") + print(f" within top-10 : {s['frac_top10']*100:.1f}%") + print(f" in NO peak : {s['frac_no_peak']*100:.1f}%") + if diann is not None: + r = metrics["reference"] + print("\n=== Reference-apex recall (DIA-NN) ===") + print(f"matched precursors : {r['n_reference_matched']}") + for K in (1, 3, 5, 10): + print(f"DIA-NN apex in top-{K:<2} : " + f"{r[f'reference_apex_in_top_{K}']*100:.1f}%") + + out = args.out or (os.path.splitext(args.psms)[0] + ".topk_oracle.json") + with open(out, "w") as fh: + json.dump(metrics, fh, indent=2) + print(f"\n[written] {os.path.abspath(out)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 184d66bd6684dc5c881cadd006fdef9a72bb3de1 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 18:37:32 +0200 Subject: [PATCH 17/40] docs(sensitivity): benchmark guide + status update with top-K + ablation findings - BENCHMARK_GUIDE.md: how to run + read the candidate audit, reference-apex top-K recall, feature-family ablation, and entrapment-FDP acceptance gate, with the real CLIs and the E. coli smoke numbers, plus the end-to-end experiment recipe. - IMPLEMENTATION_STATUS.md: tasks 6-8 done; records the top-K finding (selected apex is rank-1 only 52.5%, top-5 88.3%) and the ablation leads. Co-Authored-By: Claude Opus 4.8 --- sensitivity_plan/BENCHMARK_GUIDE.md | 153 ++++++++++++++++++++++ sensitivity_plan/IMPLEMENTATION_STATUS.md | 33 ++++- 2 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 sensitivity_plan/BENCHMARK_GUIDE.md diff --git a/sensitivity_plan/BENCHMARK_GUIDE.md b/sensitivity_plan/BENCHMARK_GUIDE.md new file mode 100644 index 0000000..adbd4f1 --- /dev/null +++ b/sensitivity_plan/BENCHMARK_GUIDE.md @@ -0,0 +1,153 @@ +# Sensitivity Benchmark Guide + +How to run the diagnostics added by the sensitivity program and how to read them. +All tools are non-destructive: they read existing artifacts and change no engine +output. Python tools use the `py312_mumdia` conda interpreter (pyarrow, pandas, +numpy, scikit-learn). + +Paths below use the E. coli / HYE example in `C:/proteobench/out_ecoli/`; substitute +your own run directory. + +## 1. Candidate audit (identification-loss waterfall) + +Reconstructs, per candidate, the pipeline stage flags and the EARLIEST rejection +reason from the artifact chain (library -> psms -> competed -> scored). + +``` +mumdia audit \ + --library-precursors lib/lib_precursors_ft.parquet \ + --psms out_ecoli/psms.parquet \ + --competed out_ecoli/comp.parquet \ + --scored out_ecoli/scored.parquet \ + --out out_ecoli/candidate_audit.parquet \ + --q 0.01 --run-id ecoli --entrapment-substr _HUMAN +``` + +Or automatically as part of a full run: set `extract.emit_candidate_audit = true` +in the config; `mumdia run` then writes `candidate_audit.parquet` after rescore. + +Outputs: +- `candidate_audit.parquet`: one row per candidate with `run_id, precursor_id, + modified_sequence, charge, target_decoy_label, entrapment_label, + candidate_generated, traces_extracted, peak_generated, peak_selected, + variant_selected, target_decoy_winner, passed_precursor_fdr, passed_peptide_fdr, + reported, rejection_reason`. +- `.metrics.json`: the waterfall counts + stage recalls. + +Example waterfall (E. coli file vs the 8.3M-candidate HYE library): + +``` +search_space = 8,334,126 extracted = 341,754 reported = 8,568 +NO_PEAK_GROUP = 7,992,372 FAILED_PRECURSOR_FDR = 332,120 +FAILED_PEPTIDE_FDR = 1,066 REPORTED = 8,568 +``` + +Reading it (spec 05 §5 decision rules): +- Large `NO_PEAK_GROUP` with a cross-species library is expected (most library + peptides are absent from a single-species sample). To make it actionable, restrict + the audit input library to candidates a reference tool (DIA-NN) identifies, or + stratify by `entrapment_label`. +- Large `FAILED_PRECURSOR_FDR` among candidates that DID extract points at scoring / + decoy / FDR (prioritize rescoring, decoy design). Large `OUTCOMPETED_*` points at + competition (try `compete.mode = none`). Large extraction loss among candidates + that should be present points at peak selection (top-K) or calibration. + +Limitation: at artifact resolution, extraction losses collapse to `NO_PEAK_GROUP`. +The in-extract audit sidecar (NEXT_STEPS #2) refines them to `NO_FRAGMENT_TRACES` / +`NO_VALID_FRAGMENTS` / `PEAK_NOT_SELECTED` / `RT_PRUNED`; `mumdia audit` already +reads `.audit.parquet` when present. + +## 2. Reference-apex top-K peak recall + +Quantifies the top-K peak opportunity: how often the selected apex is the strongest +peak, and (with a DIA-NN report) whether the reference apex is within the top-K +MuMDIA peaks. + +``` +python scripts/reference_apex_topk.py \ + --psms out_ecoli/psms.parquet \ + --chrom out_ecoli/chrom.parquet \ + [--diann diann_report.tsv] \ + [--out out_ecoli/topk_metrics.json] \ + [--max-candidates 20000] [--bound-fraction 0.333] [--rt-tol-s 10] +``` + +Self analysis (no DIA-NN needed) on 20,000 E. coli candidates: + +| metric | value | +|---|---| +| mean peaks / candidate | 10.35 (median 9) | +| candidates with >= 2 peaks | 95.2% | +| selected apex is peak rank-1 | 52.5% | +| selected apex in top-3 | 79.2% | +| selected apex in top-5 | 88.3% | +| selected apex in top-10 | 94.8% | +| selected apex in no enumerated peak | 3.5% | + +Reading it: the selected apex is the strongest peak only about half the time, and a +correct alternative peak exists within the top 5 for ~88% of candidates. This is the +quantitative case for `extract.retain_top_peaks = 5` (NEXT_STEPS #1): retain the +alternative peaks and let a peak-selection model choose, instead of committing to one +apex. Run this before and after the top-K wiring to measure recovered peak recall. + +With `--diann`: reports `reference_apex_in_top_{1,3,5,10}` by matching stripped +sequence + charge and comparing the DIA-NN apex RT (auto-converted minutes->seconds) +to the MuMDIA peak apexes. This is the spec's peak-oracle metric (02 §6). + +## 3. Feature-family ablation + +Grouped cross-validated ablation with leakage guards (in-fold standardization, +group by peptidoform+charge, targets at empirical FDP). + +``` +python scripts/feature_ablation.py \ + --features out_ecoli/comp.parquet \ + --registry feature_registry.yaml \ + --out out_ecoli/ablation \ + [--folds 3] [--fdp 0.01] [--model both] [--max-rows 60000] [--clip-sd 8.0] +``` + +Outputs a CSV + JSON per model with columns `feature_family, baseline_identifications, +new_identifications, relative_gain, delta_vs_full, model, recommendation`. + +Smoke (60,000 rows, logreg, 3 folds): 355 features / 17 families; full model = 509, +minimal baseline = 414 targets @1% FDP. Most useful families (removal hurts): +`similarity` (-393), `rt` (-382), `entropy` (-6). On this subset removing +`interference` / `rich` / `coelution` raised the count (+267 / +204 / +119), +indicating redundancy or subset overfit. + +Caveats (spec 03 §9, 05 §7): this is ONE dataset subset and (in the smoke) ONE model. +Do not retain or drop a family on a single favourable subset. Rerun with `--model +both`, all rows, and a second dataset before acting; a family that only helps on one +subset or whose gain flips sign across models/datasets is not a keep. + +## 4. Empirical entrapment FDP (the acceptance gate) + +Already present. Trains on E. coli targets vs a human-TRAIN half and evaluates on the +unseen human-TEST half, giving a leakage-free identification count at a genuinely +controlled FDP. + +``` +python scripts/entrapment_holdout.py out_ecoli/comp.parquet --q 0.01 +``` + +Use this as the accept/reject gate for every change (spec 05 §6): a change ships only +if held-out entrapment identifications rise without FDP inflation, reproduced on a +second dataset. + +## 5. End-to-end recipe for one experiment (spec 05) + +1. `mumdia run ... --config ` (set `extract.emit_candidate_audit=true`). +2. `mumdia audit ...` (or read the run's `candidate_audit.parquet`) -> waterfall. +3. `python scripts/reference_apex_topk.py ...` -> peak recall. +4. `python scripts/feature_ablation.py ...` -> family contributions. +5. `python scripts/entrapment_holdout.py ...` -> honest FDP + identification count. +6. Change ONE component (e.g. `compete.mode`, `retain_top_peaks`, a feature family), + rerun 1-5, and compare at matched empirical FDP. Keep raw candidate outputs. + +## Determinism and cost + +- All Rust stages are deterministic under a fixed seed; `mumdia audit` is a pure join. +- The Python tools seed RNG and use stable (SHA1) fold hashing. +- `reference_apex_topk.py` and `feature_ablation.py` support `--max-candidates` / + `--max-rows` for fast passes; full runs over ~1.3M rows are heavier (minutes). diff --git a/sensitivity_plan/IMPLEMENTATION_STATUS.md b/sensitivity_plan/IMPLEMENTATION_STATUS.md index f956e62..fd650f6 100644 --- a/sensitivity_plan/IMPLEMENTATION_STATUS.md +++ b/sensitivity_plan/IMPLEMENTATION_STATUS.md @@ -35,9 +35,9 @@ Living log for the autonomous sensitivity-improvement session. Updated throughou | 3 | Top-K peak enumerator + config `retain_top_peaks` (K=1 compat) + tests | P1 | PARTIAL (enumerator+config+tests done; extract wiring = NEXT_STEPS #1) | 2f46d6d | | 4 | Competition-mode enum wired in compete (none/features-only/unique-evidence/margin-gated) | P2.4 | DONE | de5ae2b | | 5 | `ARCHITECTURE_MAP.md`, `FEATURE_REGISTRY.md`, `feature_registry.yaml` | P4.1 | DONE | docs | -| 6 | Reference-apex top-K analysis script (Python) | P0.4, 02 §5 D | IN PROGRESS (benchmark agent) | - | -| 7 | Feature-ablation runner (Python) | P4.3 | IN PROGRESS (benchmark agent) | - | -| 8 | `BENCHMARK_GUIDE.md`, `NEXT_STEPS.md` | P7 | NEXT_STEPS done; BENCHMARK_GUIDE pending scripts | - | +| 6 | Reference-apex top-K analysis script (Python) | P0.4, 02 §5 D | DONE | ed44e2a | +| 7 | Feature-ablation runner (Python) | P4.3 | DONE | ed44e2a | +| 8 | `BENCHMARK_GUIDE.md`, `NEXT_STEPS.md` | P7 | DONE | docs | | 9 | Fragment claimant / conflict features | P2.1-2.3 | DEFERRED -> NEXT_STEPS #4 (nucleus exists: contested_frac) | - | | 10 | In-extract precise reason emitter | P0.3 | DEFERRED -> NEXT_STEPS #2 (audit reads sidecar already) | - | @@ -80,6 +80,33 @@ form a peak. The audit now makes every loss category countable and stratifiable, which is the P0 prerequisite for targeting the recoverable losses (peak selection and FDR), per the spec's decision rules (05 §5). +## Key diagnostic result (reference-apex top-K, 20,000 E. coli candidates) + +| metric | value | +|---|---| +| mean peaks / candidate | 10.35 | +| candidates with >= 2 peaks | 95.2% | +| selected apex is peak rank-1 | **52.5%** | +| selected apex in top-3 / top-5 / top-10 | 79.2% / 88.3% / 94.8% | +| selected apex in no enumerated peak | 3.5% | + +Interpretation: the current single-apex selection lands on the STRONGEST peak only +about half the time; a better-ranked alternative peak exists within the top 5 for +~88% of candidates. This is direct quantitative support for the spec's central +hypothesis (01 §3.1) and for `extract.retain_top_peaks = 5` (NEXT_STEPS #1): the +enumerator + config are in place; wiring them into `extract` is the highest-value +remaining change. (Whether recovering those peaks raises IDENTIFICATIONS at matched +empirical FDP must still be confirmed with the entrapment gate.) + +## Feature-family ablation (60,000 rows, logreg, 3-fold grouped CV) + +355 features / 17 families; full model 509 vs minimal-baseline 414 targets @1% FDP. +Most useful (removal hurts): similarity (-393), rt (-382), entropy (-6). On this +subset, removing interference / rich / coelution raised the count (+267 / +204 / ++119), indicating redundancy or subset overfit. One-subset, one-model result: not an +action, a lead. Rerun with `--model both`, all rows, and a second dataset (spec 05 §7) +before dropping any family. + ## Known limitations / risks (running) - Empirical FDP validation here uses the single E. coli file + HYE entrapment null (one dataset); the spec's held-out multi-dataset reproduction cannot be completed in this environment without additional data. From 048cccd405037477221c5d706db1a81602a2001d Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:42:49 +0200 Subject: [PATCH 18/40] feat(sensitivity): apex-dispersion/peak-shape + mass-uncertainty feature families (P5.1/P5.3) Two additive Extended-battery families computed from the per-PSM Evidence (peak-bounded traces, apex, mass errors), interference-resistant (shape/breadth, not absolute intensity): - apex_dispersion (13): fragment-apex RT scatter (std/mad/max/mean dev + agree fraction), precursor-fragment apex delta, and consensus peak shape (symmetry, tailing, local maxima, shoulder, FWHM, truncation, apex position in window). - mass_uncertainty (10): fragment mass-error distribution (median/abs-median/std/ IQR/max/range over matched fragments), effective fragment count (inverse participation ratio), evidence concentration, and fraction of the top-3/top-5 predicted ions observed (breadth of the strong ions). Append-only; the Extended-count test is dynamic so it stays green. 8 unit tests. Co-Authored-By: Claude Opus 4.8 --- .../crates/mumdia/src/stages/features.rs | 4 + .../src/stages/features/apex_dispersion.rs | 311 ++++++++++++++++++ .../src/stages/features/mass_uncertainty.rs | 202 ++++++++++++ 3 files changed, 517 insertions(+) create mode 100644 rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs create mode 100644 rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs diff --git a/rust/mumdia/crates/mumdia/src/stages/features.rs b/rust/mumdia/crates/mumdia/src/stages/features.rs index cd7096c..ca3509e 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features.rs @@ -28,10 +28,12 @@ use rayon::prelude::*; // OktoberFest analogs plus novel families. Kept separate so they can be built // and reviewed independently; the registry below concatenates them in a fixed // order that defines the extended schema. +mod apex_dispersion; mod chromatographic; mod coelution; mod entropy; mod interference; +mod mass_uncertainty; mod ion_series; mod mass_accuracy; mod ms1; @@ -60,6 +62,8 @@ const FAMILIES: &[(&[&str], FamilyFn)] = &[ (nonzero::NAMES, nonzero::values), (order_consistency::NAMES, order_consistency::values), (peak_scans::NAMES, peak_scans::values), + (apex_dispersion::NAMES, apex_dispersion::values), + (mass_uncertainty::NAMES, mass_uncertainty::values), ]; /// Names already used by the Minimal/Rich sets, which the extended battery must diff --git a/rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs b/rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs new file mode 100644 index 0000000..685dd42 --- /dev/null +++ b/rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs @@ -0,0 +1,311 @@ +//! Extended feature family: apex dispersion + peak shape (sensitivity_plan +//! spec 03 §8.3-8.4, backlog P5.3). +//! +//! A real peptide's fragments share one apex and one symmetric elution peak; a +//! chimeric/interfered match has fragments apexing at scattered retention times +//! and an irregular consensus peak (shoulders, tailing, truncation). These +//! features quantify that agreement from the peak-bounded traces, independent of +//! absolute intensity (which is chimeric in DIA), so they capture BREADTH-of- +//! coelution evidence rather than height. +//! +//! Contract: `NAMES` and `values(&Evidence)` return the same number of items in +//! the same order; every value is finite; label-blind; emitted for every PSM. +use super::Evidence; + +pub const NAMES: &[&str] = &[ + "frag_apex_rt_std", // scatter of per-fragment apex RTs (s) + "frag_apex_rt_mad", // robust scatter (median abs dev, s) + "frag_apex_max_dev", // max |fragment apex - consensus apex| (s) + "frag_apex_mean_dev", // mean |fragment apex - consensus apex| (s) + "frag_apex_agree_frac", // fraction of signal fragments apexing within 1 scan of consensus + "precursor_frag_apex_delta",// |MS1 mono apex - consensus apex| (s); 0 if no MS1 + "peak_symmetry", // right-area / (left-area+right-area) of consensus peak (~0.5 ideal) + "peak_tailing", // right half-width / left half-width at half height + "peak_n_local_maxima", // local maxima in consensus profile (>=1) + "peak_shoulder_score", // 2nd-highest local max / apex height + "peak_fwhm_scans", // full width at half maximum (scans) + "peak_truncation", // 1 if apex at window edge or boundary height > 0.5*apex + "apex_frac_of_window", // apex position within the window [0,1] (edge = mis-centered) +]; + +fn mean(v: &[f64]) -> f64 { + if v.is_empty() { + 0.0 + } else { + v.iter().sum::() / v.len() as f64 + } +} + +fn std(v: &[f64]) -> f64 { + if v.len() < 2 { + return 0.0; + } + let m = mean(v); + (v.iter().map(|x| (x - m) * (x - m)).sum::() / (v.len() as f64 - 1.0)).sqrt() +} + +fn median(v: &[f64]) -> f64 { + if v.is_empty() { + return 0.0; + } + let mut s = v.to_vec(); + s.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let n = s.len(); + if n % 2 == 1 { + s[n / 2] + } else { + 0.5 * (s[n / 2 - 1] + s[n / 2]) + } +} + +fn median_abs_dev(v: &[f64]) -> f64 { + if v.is_empty() { + return 0.0; + } + let m = median(v); + let dev: Vec = v.iter().map(|x| (x - m).abs()).collect(); + median(&dev) +} + +/// Index of the maximum of a slice (first on ties); None if empty or all <= 0. +fn argmax_pos(v: &[f64]) -> Option { + let mut best = 0usize; + let mut bv = f64::NEG_INFINITY; + let mut any = false; + for (i, &x) in v.iter().enumerate() { + if x > bv { + bv = x; + best = i; + any = true; + } + } + if any && bv > 0.0 { + Some(best) + } else { + None + } +} + +pub fn values(e: &Evidence) -> Vec { + let axis = &e.axis; + let n = axis.len(); + let consensus_rt = if e.apex_idx < n { axis[e.apex_idx] } else { 0.0 }; + let mean_scan_dt = if n >= 2 { + (axis[n - 1] - axis[0]) / (n as f64 - 1.0) + } else { + 0.0 + }; + + // Per-fragment apex RTs (only fragments that carry signal on the axis). + let mut apex_rts: Vec = Vec::new(); + for tr in &e.traces { + if let Some(pos) = argmax_pos(tr) { + if pos < n { + apex_rts.push(axis[pos]); + } + } + } + let devs: Vec = apex_rts.iter().map(|r| (r - consensus_rt).abs()).collect(); + let frag_apex_rt_std = std(&apex_rts); + let frag_apex_rt_mad = median_abs_dev(&apex_rts); + let frag_apex_max_dev = devs.iter().cloned().fold(0.0, f64::max); + let frag_apex_mean_dev = mean(&devs); + let frag_apex_agree_frac = if devs.is_empty() { + 0.0 + } else { + let tol = mean_scan_dt.max(1e-9); + devs.iter().filter(|d| **d <= tol).count() as f64 / devs.len() as f64 + }; + + // Precursor (MS1 mono) vs fragment consensus apex. + let precursor_frag_apex_delta = e + .ms1_xic + .first() + .and_then(|mono| argmax_pos(mono).map(|p| (axis.get(p).copied().unwrap_or(consensus_rt) - consensus_rt).abs())) + .unwrap_or(0.0); + + // Consensus profile shape (predicted-intensity-weighted reference profile). + let prof = &e.ref_profile; + let (peak_symmetry, peak_tailing, peak_fwhm_scans, peak_truncation, peak_n_local_maxima, peak_shoulder_score) = + profile_shape(prof, e.apex_idx); + + let apex_frac_of_window = if n >= 2 { + e.apex_idx as f64 / (n as f64 - 1.0) + } else { + 0.0 + }; + + let out = vec![ + frag_apex_rt_std, + frag_apex_rt_mad, + frag_apex_max_dev, + frag_apex_mean_dev, + frag_apex_agree_frac, + precursor_frag_apex_delta, + peak_symmetry, + peak_tailing, + peak_n_local_maxima, + peak_shoulder_score, + peak_fwhm_scans, + peak_truncation, + apex_frac_of_window, + ]; + // Guarantee finiteness. + out.into_iter() + .map(|x| if x.is_finite() { x } else { 0.0 }) + .collect() +} + +/// Shape descriptors of a consensus elution profile around `apex_idx`. +/// Returns (symmetry, tailing, fwhm_scans, truncation, n_local_maxima, shoulder). +fn profile_shape(prof: &[f64], apex_idx: usize) -> (f64, f64, f64, f64, f64, f64) { + let n = prof.len(); + if n == 0 { + return (0.5, 1.0, 0.0, 0.0, 0.0, 0.0); + } + let apex = apex_idx.min(n - 1); + let apex_h = prof[apex]; + if apex_h <= 0.0 { + return (0.5, 1.0, 0.0, 1.0, 0.0, 0.0); + } + // Left/right areas about the apex (symmetry). + let left_area: f64 = prof[..=apex].iter().sum(); + let right_area: f64 = prof[apex..].iter().sum(); + let total = left_area + right_area; + let symmetry = if total > 0.0 { right_area / total } else { 0.5 }; + + // Half-max widths (in scans) each side of the apex. + let half = 0.5 * apex_h; + let mut lw = 0usize; + let mut i = apex; + while i > 0 && prof[i - 1] >= half { + lw += 1; + i -= 1; + } + let mut rw = 0usize; + let mut j = apex; + while j + 1 < n && prof[j + 1] >= half { + rw += 1; + j += 1; + } + let tailing = if lw > 0 { + rw as f64 / lw as f64 + } else if rw > 0 { + 2.0 // right-only shoulder: strongly tailing + } else { + 1.0 + }; + let fwhm = (lw + rw + 1) as f64; + + // Truncation: apex at edge, or either boundary still above half the apex. + let truncation = if apex == 0 || apex == n - 1 || prof[0] > half || prof[n - 1] > half { + 1.0 + } else { + 0.0 + }; + + // Local maxima + shoulder (2nd-highest local max relative to apex). + let mut maxima: Vec = Vec::new(); + for k in 0..n { + let l = if k == 0 { 0.0 } else { prof[k - 1] }; + let r = if k + 1 == n { 0.0 } else { prof[k + 1] }; + if prof[k] > 0.0 && prof[k] >= l && prof[k] >= r && prof[k] > l { + maxima.push(prof[k]); + } + } + let n_local = maxima.len().max(1) as f64; + maxima.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + let shoulder = if maxima.len() >= 2 { + maxima[1] / apex_h + } else { + 0.0 + }; + (symmetry, tailing, fwhm, truncation, n_local, shoulder) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ev(axis: Vec, traces: Vec>, ref_profile: Vec, apex_idx: usize) -> Evidence { + Evidence { + axis, + traces, + axis_full: vec![], + traces_full: vec![], + pred: vec![], + obs_apex: vec![], + is_b: vec![], + ordinal: vec![], + frag_charge: vec![], + frag_mz: vec![], + frag_obs_mz: vec![], + mass_err_ppm: vec![], + apex_idx, + ref_profile, + apex_rt: 0.0, + rt_pred_cal: 0.0, + rt_err: 0.0, + gradient: 0.0, + precursor_mz: 0.0, + charge: 2, + seq_len: 0, + n_matched: 0, + n_predicted: 0, + seed_score: 0.0, + seed_identified: 0.0, + apex_intensity: 0.0, + ms1_mono: None, + ms1_iso1: None, + ms1_iso2: None, + ms1_isom1: None, + ms1_xic: vec![], + } + } + + #[test] + fn names_match_values_len_and_finite() { + let axis = vec![0.0, 1.0, 2.0, 3.0, 4.0]; + let tr = vec![vec![0.0, 1.0, 4.0, 1.0, 0.0], vec![0.0, 1.0, 3.0, 1.0, 0.0]]; + let prof = vec![0.0, 1.0, 4.0, 1.0, 0.0]; + let v = values(&ev(axis, tr, prof, 2)); + assert_eq!(v.len(), NAMES.len()); + assert!(v.iter().all(|x| x.is_finite())); + } + + #[test] + fn coeluting_fragments_have_low_apex_dispersion() { + // both fragments apex at index 2 + let axis = vec![0.0, 1.0, 2.0, 3.0, 4.0]; + let tr = vec![vec![0.0, 2.0, 9.0, 2.0, 0.0], vec![0.0, 1.0, 8.0, 1.0, 0.0]]; + let prof = vec![0.0, 1.5, 8.5, 1.5, 0.0]; + let v = values(&ev(axis, tr, prof, 2)); + let std_i = NAMES.iter().position(|n| *n == "frag_apex_rt_std").unwrap(); + let agree_i = NAMES.iter().position(|n| *n == "frag_apex_agree_frac").unwrap(); + assert_eq!(v[std_i], 0.0); // both apex at same scan + assert_eq!(v[agree_i], 1.0); + } + + #[test] + fn scattered_fragments_have_high_apex_dispersion() { + // fragments apex at index 1 and index 3 (2 scans apart) + let axis = vec![0.0, 1.0, 2.0, 3.0, 4.0]; + let tr = vec![vec![0.0, 9.0, 1.0, 0.0, 0.0], vec![0.0, 0.0, 1.0, 9.0, 0.0]]; + let prof = vec![0.0, 4.5, 1.0, 4.5, 0.0]; + let v = values(&ev(axis, tr, prof, 1)); + let std_i = NAMES.iter().position(|n| *n == "frag_apex_rt_std").unwrap(); + let maxdev_i = NAMES.iter().position(|n| *n == "frag_apex_max_dev").unwrap(); + assert!(v[std_i] > 0.0); + assert!(v[maxdev_i] >= 2.0); + } + + #[test] + fn truncation_flagged_when_apex_at_edge() { + let axis = vec![0.0, 1.0, 2.0]; + let tr = vec![vec![9.0, 4.0, 1.0]]; + let prof = vec![9.0, 4.0, 1.0]; + let v = values(&ev(axis, tr, prof, 0)); + let trunc_i = NAMES.iter().position(|n| *n == "peak_truncation").unwrap(); + assert_eq!(v[trunc_i], 1.0); + } +} diff --git a/rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs b/rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs new file mode 100644 index 0000000..6baa5d1 --- /dev/null +++ b/rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs @@ -0,0 +1,202 @@ +//! Extended feature family: fragment mass-error dispersion + evidence breadth +//! (sensitivity_plan spec 03 §8.1-8.2, backlog P5.1/P5.2). +//! +//! A correct identification matches its fragments with small AND mutually +//! consistent mass errors, and spreads its observed intensity across many of its +//! predicted transitions (breadth of evidence). A chimeric/interfered match tends +//! to have scattered mass errors and its signal concentrated in one or two +//! coincidental channels. These features summarize the DISTRIBUTION of per- +//! fragment mass errors and how the observed evidence is spread, complementing the +//! single-number mass-accuracy family. Intensity here is used only for shape +//! (participation ratio, concentration), not as an absolute magnitude, so it is +//! robust to the chimeric-intensity problem in DIA. +//! +//! Contract: `NAMES` and `values(&Evidence)` return the same number of items in +//! the same order; every value is finite; label-blind; emitted for every PSM. +use super::Evidence; + +pub const NAMES: &[&str] = &[ + "frag_mass_err_median", // median signed ppm over matched fragments + "frag_mass_err_abs_median", // median |ppm| + "frag_mass_err_std", // dispersion of ppm (consistency) + "frag_mass_err_iqr", // robust dispersion of ppm + "frag_mass_err_max_abs", // worst |ppm| + "frag_mass_err_range", // max - min ppm + "effective_frag_count", // inverse participation ratio of observed intensity + "evidence_concentration", // fraction of observed intensity in the strongest fragment + "frac_top3_pred_observed", // fraction of the top-3 predicted ions actually observed + "frac_top5_pred_observed", // fraction of the top-5 predicted ions actually observed +]; + +fn mean(v: &[f64]) -> f64 { + if v.is_empty() { + 0.0 + } else { + v.iter().sum::() / v.len() as f64 + } +} + +fn std(v: &[f64]) -> f64 { + if v.len() < 2 { + return 0.0; + } + let m = mean(v); + (v.iter().map(|x| (x - m) * (x - m)).sum::() / (v.len() as f64 - 1.0)).sqrt() +} + +/// Linear-interpolated percentile (q in [0,1]) of an unsorted slice. +fn percentile(sorted: &[f64], q: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + if sorted.len() == 1 { + return sorted[0]; + } + let pos = q * (sorted.len() as f64 - 1.0); + let lo = pos.floor() as usize; + let hi = pos.ceil() as usize; + let frac = pos - lo as f64; + sorted[lo] * (1.0 - frac) + sorted[hi] * frac +} + +/// Fraction of the top-`k` predicted fragments (by predicted intensity) that are +/// actually observed at the apex (obs_apex > 0). Breadth of the STRONG ions. +fn frac_top_pred_observed(pred: &[f64], obs: &[f64], k: usize) -> f64 { + let n = pred.len().min(obs.len()); + if n == 0 { + return 0.0; + } + let mut idx: Vec = (0..n).collect(); + idx.sort_by(|&a, &b| pred[b].partial_cmp(&pred[a]).unwrap_or(std::cmp::Ordering::Equal)); + let take = k.min(n); + if take == 0 { + return 0.0; + } + let hit = idx[..take].iter().filter(|&&i| obs[i] > 0.0).count(); + hit as f64 / take as f64 +} + +pub fn values(e: &Evidence) -> Vec { + // Mass errors are meaningful only for observed (matched) fragments. + let errs: Vec = e + .mass_err_ppm + .iter() + .zip(&e.obs_apex) + .filter(|(_, &o)| o > 0.0) + .map(|(&m, _)| m) + .collect(); + + let (median, abs_median, iqr, max_abs, range) = if errs.is_empty() { + (0.0, 0.0, 0.0, 0.0, 0.0) + } else { + let mut s = errs.clone(); + s.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let med = percentile(&s, 0.5); + let mut a: Vec = errs.iter().map(|x| x.abs()).collect(); + a.sort_by(|x, y| x.partial_cmp(y).unwrap_or(std::cmp::Ordering::Equal)); + let abs_med = percentile(&a, 0.5); + let iqr = percentile(&s, 0.75) - percentile(&s, 0.25); + let max_abs = a[a.len() - 1]; + let range = s[s.len() - 1] - s[0]; + (med, abs_med, iqr, max_abs, range) + }; + let err_std = std(&errs); + + // Evidence spread over observed intensity. + let obs_pos: Vec = e.obs_apex.iter().cloned().filter(|&x| x > 0.0).collect(); + let sum: f64 = obs_pos.iter().sum(); + let sum_sq: f64 = obs_pos.iter().map(|x| x * x).sum(); + let effective_frag_count = if sum_sq > 0.0 { sum * sum / sum_sq } else { 0.0 }; + let evidence_concentration = if sum > 0.0 { + obs_pos.iter().cloned().fold(0.0, f64::max) / sum + } else { + 0.0 + }; + + let out = vec![ + median, + abs_median, + err_std, + iqr, + max_abs, + range, + effective_frag_count, + evidence_concentration, + frac_top_pred_observed(&e.pred, &e.obs_apex, 3), + frac_top_pred_observed(&e.pred, &e.obs_apex, 5), + ]; + out.into_iter() + .map(|x| if x.is_finite() { x } else { 0.0 }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ev(pred: Vec, obs: Vec, err: Vec) -> Evidence { + Evidence { + axis: vec![], + traces: vec![], + axis_full: vec![], + traces_full: vec![], + pred, + obs_apex: obs, + is_b: vec![], + ordinal: vec![], + frag_charge: vec![], + frag_mz: vec![], + frag_obs_mz: vec![], + mass_err_ppm: err, + apex_idx: 0, + ref_profile: vec![], + apex_rt: 0.0, + rt_pred_cal: 0.0, + rt_err: 0.0, + gradient: 0.0, + precursor_mz: 0.0, + charge: 2, + seq_len: 0, + n_matched: 0, + n_predicted: 0, + seed_score: 0.0, + seed_identified: 0.0, + apex_intensity: 0.0, + ms1_mono: None, + ms1_iso1: None, + ms1_iso2: None, + ms1_isom1: None, + ms1_xic: vec![], + } + } + + #[test] + fn names_match_values_len_and_finite() { + let v = values(&ev(vec![9.0, 5.0, 1.0], vec![8.0, 4.0, 0.0], vec![1.0, -2.0, 50.0])); + assert_eq!(v.len(), NAMES.len()); + assert!(v.iter().all(|x| x.is_finite())); + } + + #[test] + fn only_observed_fragments_count_toward_mass_error() { + // third fragment is unobserved (obs 0) with a huge error -> excluded + let v = values(&ev(vec![9.0, 5.0, 1.0], vec![8.0, 4.0, 0.0], vec![1.0, -1.0, 999.0])); + let maxabs_i = NAMES.iter().position(|n| *n == "frag_mass_err_max_abs").unwrap(); + assert!(v[maxabs_i] <= 1.0 + 1e-9); // 999 excluded, only |1|,|-1| + } + + #[test] + fn breadth_of_top_predicted_ions() { + // top-3 predicted = frags 0,1,2 (pred 9,5,3); obs>0 for 0 and 1 only -> 2/3 + let v = values(&ev(vec![9.0, 5.0, 3.0, 1.0], vec![8.0, 4.0, 0.0, 2.0], vec![0.0; 4])); + let t3 = NAMES.iter().position(|n| *n == "frac_top3_pred_observed").unwrap(); + assert!((v[t3] - 2.0 / 3.0).abs() < 1e-9); + } + + #[test] + fn concentration_high_when_one_dominant_fragment() { + let v = values(&ev(vec![1.0, 1.0], vec![100.0, 1.0], vec![0.0, 0.0])); + let conc = NAMES.iter().position(|n| *n == "evidence_concentration").unwrap(); + assert!(v[conc] > 0.9); + } +} From 4e85cab23b4df4271e6666dadc238e48556414a1 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:42:50 +0200 Subject: [PATCH 19/40] feat(sensitivity): feature-audit, HTML report, diagnostic bundle, manifest, normalizer (P0.1/P0.2/P4.2/P7.1/P7.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-invasive Python diagnostics over existing artifacts (no engine change): - feature_audit.py (P4.2): per-feature missingness/quantiles/constant-flag, target/decoy/entrapment separation (Mann-Whitney), redundancy clusters (|rho|>=0.9), leakage + intensity-domination warnings, registry cross-check. - benchmark_report.py (P7.1): self-contained HTML (inlined CSS + base64 PNGs, no network) assembling waterfall, audit stratification, top-K recall, ablation, entrapment. - candidate_diagnostics.py (P7.2): per-candidate chromatogram + predicted-vs- observed plots + feature dump (pyarrow predicate pushdown, MS1 on twinx). - search_space_manifest.py (P0.1): effective search-space manifest from the library + DIA-NN/declared parity check with --fail-on-mismatch. - normalize_output.py (P0.2): MuMDIA scored -> spec 02 §4 common schema. Co-Authored-By: Claude Opus 4.8 --- scripts/benchmark_report.py | 768 +++++++++++++++++++++++++++++++ scripts/candidate_diagnostics.py | 537 +++++++++++++++++++++ scripts/feature_audit.py | 723 +++++++++++++++++++++++++++++ scripts/normalize_output.py | 295 ++++++++++++ scripts/search_space_manifest.py | 564 +++++++++++++++++++++++ 5 files changed, 2887 insertions(+) create mode 100644 scripts/benchmark_report.py create mode 100644 scripts/candidate_diagnostics.py create mode 100644 scripts/feature_audit.py create mode 100644 scripts/normalize_output.py create mode 100644 scripts/search_space_manifest.py diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py new file mode 100644 index 0000000..facaa1e --- /dev/null +++ b/scripts/benchmark_report.py @@ -0,0 +1,768 @@ +#!/usr/bin/env python +"""Self-contained HTML benchmark report for MuMDIA sensitivity diagnostics. + +Implements sensitivity_plan backlog P7.1 (spec 02 Section 8, spec 05). Assembles +the diagnostics this project already produces into a single, portable HTML file: + + * identification-loss waterfall (candidate audit metrics + stage counts); + * candidate-audit stratification (rejection reason by charge / entrapment); + * reference-apex top-K peak recall (self, and reference if present); + * feature-family ablation (per model, most useful first); + * empirical entrapment FDP (parsed from the holdout harness stdout). + +Non-invasive: it only reads existing JSON / CSV / Parquet outputs and writes one +HTML file with all CSS inlined and every chart embedded as a base64 PNG data URI. +There are no external assets and no network access, so the report opens correctly +straight from disk. Any missing input renders as "not provided" for its section. + +Deterministic: analysis and charts are reproducible for the same inputs. Only the +header timestamp varies (fix it with --stamp). Charts require matplotlib; if it is +unavailable the tables still render and the chart panels note the omission. + +Interpreter: C:/Users/robbi/anaconda3/envs/py312_mumdia/python.exe + (pyarrow, pandas, numpy, matplotlib). + +Usage: + python benchmark_report.py --out report.html + [--audit-metrics candidate_audit.parquet.metrics.json] + [--audit candidate_audit.parquet] + [--topk topk.json] + [--ablation ablation_dir_or_csv] + [--entrapment entrapment_stdout.txt] + [--title "..."] [--stamp "2026-07-17 12:00"] +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import datetime as _dt +import html +import io +import json +import os +import re +import sys + +# --------------------------------------------------------------------------- # +# matplotlib is optional at runtime: charts are omitted gracefully if absent. +try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + _HAVE_MPL = True +except Exception: # pragma: no cover - defensive + _HAVE_MPL = False + +# Fixed PNG metadata so byte output does not carry a matplotlib version string or +# creation time, keeping identical inputs byte-reproducible across runs. +_PNG_META = {"Software": "mumdia-benchmark-report"} + +# Canonical ordering of known rejection reasons, following the pipeline ladder +# (spec 02 Section 8). Unknown reasons are appended in first-seen order. +_REASON_ORDER = [ + "NOT_IN_SEARCH_SPACE", + "NO_CANDIDATE", + "NOT_GENERATED", + "NO_FRAGMENT_TRACES", + "NO_VALID_FRAGMENTS", + "NO_PEAK_GROUP", + "RT_PRUNED", + "PEAK_NOT_SELECTED", + "WRONG_VARIANT", + "OUTCOMPETED_PEAK", + "OUTCOMPETED_PEPTIDE", + "OUTCOMPETED_PEPTIDOFORM", + "OUTCOMPETED_LOCALIZATION", + "OUTCOMPETED_DUPLICATE", + "OUTCOMPETED_TARGET_DECOY", + "FAILED_PRECURSOR_FDR", + "FAILED_PEPTIDE_FDR", + "FAILED_PROTEIN_FDR", + "NOT_REPORTED", + "REPORTED", +] +_REASON_RANK = {r: i for i, r in enumerate(_REASON_ORDER)} + + +# --------------------------------------------------------------------------- # +# small helpers +def esc(x) -> str: + return html.escape("" if x is None else str(x)) + + +def fmt_int(x) -> str: + try: + return f"{int(round(float(x))):,}" + except (TypeError, ValueError): + return esc(x) + + +def fmt_pct(x, digits: int = 1) -> str: + try: + return f"{float(x) * 100:.{digits}f}%" + except (TypeError, ValueError): + return esc(x) + + +def fmt_num(x, digits: int = 4) -> str: + try: + return f"{float(x):.{digits}g}" + except (TypeError, ValueError): + return esc(x) + + +def load_json(path): + if not path or not os.path.isfile(path): + return None + with open(path, "r", encoding="utf-8") as fh: + return json.load(fh) + + +def reason_sort_key(reason: str): + return (_REASON_RANK.get(reason, len(_REASON_ORDER)), str(reason)) + + +def fig_to_data_uri(fig) -> str: + buf = io.BytesIO() + fig.savefig(buf, format="png", dpi=110, bbox_inches="tight", + facecolor="white", metadata=_PNG_META) + plt.close(fig) + b64 = base64.b64encode(buf.getvalue()).decode("ascii") + return f"data:image/png;base64,{b64}" + + +def not_provided(msg: str = "not provided") -> str: + return f'

{esc(msg)}

' + + +def html_table(headers, rows, align_right_from: int = 1) -> str: + """Render a responsive table. Columns from `align_right_from` are right-aligned.""" + th = "".join( + f'= align_right_from else ""}">{esc(h)}' + for i, h in enumerate(headers) + ) + body = [] + for row in rows: + tds = "".join( + f'= align_right_from else ""}">{c}' + for i, c in enumerate(row) + ) + body.append(f"{tds}") + return ( + '
' + f"{th}" + f'{"".join(body)}' + "
" + ) + + +def img_panel(uri: str, caption: str = "") -> str: + cap = f'
{esc(caption)}
' if caption else "" + return f'
{esc(caption)}{cap}
' + + +# --------------------------------------------------------------------------- # +# Section 2: identification-loss waterfall +def section_waterfall(metrics): + if not metrics: + return not_provided() + + wf = metrics.get("waterfall") or {} + parts = [] + + # stage-count funnel + stage_rows = [] + for key, label in ( + ("search_space", "Search space (candidates)"), + ("extracted", "Extracted"), + ("competed", "Competed"), + ("reported", "Reported"), + ): + if key in metrics and metrics[key] is not None: + stage_rows.append((label, fmt_int(metrics[key]))) + if metrics.get("trace_recall") is not None: + stage_rows.append(("Trace recall (extracted / search space)", + fmt_pct(metrics["trace_recall"], 3))) + if metrics.get("q_threshold") is not None: + stage_rows.append(("q threshold", fmt_num(metrics["q_threshold"]))) + if metrics.get("run_id"): + stage_rows.append(("Run id", esc(metrics["run_id"]))) + if stage_rows: + parts.append("

Stage counts

") + parts.append(html_table(["Stage", "Value"], stage_rows)) + + if not wf: + parts.append("

Waterfall

") + parts.append(not_provided("no waterfall block in metrics")) + return "".join(parts) + + ordered = sorted(wf.items(), key=lambda kv: reason_sort_key(kv[0])) + total = sum(v for _, v in ordered) or metrics.get("search_space") or 1 + + # table + wf_rows = [ + (esc(reason), fmt_int(count), fmt_pct(count / total, 2)) + for reason, count in ordered + ] + parts.append("

Earliest-loss waterfall

") + parts.append(html_table(["Rejection reason", "Candidates", "% of total"], wf_rows)) + + # chart (log scale: the earliest bucket dominates by orders of magnitude) + if _HAVE_MPL: + labels = [r for r, _ in ordered] + counts = [c for _, c in ordered] + fig, ax = plt.subplots(figsize=(8.2, max(2.2, 0.55 * len(labels) + 1.0))) + ypos = list(range(len(labels)))[::-1] + bars = ax.barh(ypos, counts, color="#3f6fb0", edgecolor="#26456f") + ax.set_yticks(ypos) + ax.set_yticklabels(labels, fontsize=9) + ax.set_xscale("log") + ax.set_xlabel("candidates (log scale)") + ax.set_title("Identification-loss waterfall") + for rect, c in zip(bars, counts): + ax.text(rect.get_width() * 1.05, rect.get_y() + rect.get_height() / 2, + f"{c:,}", va="center", ha="left", fontsize=8, color="#333") + ax.margins(x=0.18) + ax.grid(axis="x", linestyle=":", alpha=0.4) + parts.append(img_panel(fig_to_data_uri(fig), + "Candidates lost at each stage (log-scaled).")) + else: + parts.append(not_provided("matplotlib unavailable: chart omitted")) + + return "".join(parts) + + +# --------------------------------------------------------------------------- # +# Section 3: candidate-audit stratification +def _crosstab(df, index_col, col_col, col_order=None, col_fmt=str): + import pandas as pd + + ct = pd.crosstab(df[index_col], df[col_col]) + # order rows by the pipeline ladder + ct = ct.reindex(sorted(ct.index, key=reason_sort_key)) + if col_order is not None: + cols = [c for c in col_order if c in ct.columns] + cols += [c for c in ct.columns if c not in cols] + else: + cols = sorted(ct.columns, key=lambda c: (str(type(c)), c)) + # reindex (not ct[cols]) so a list of booleans is read as labels, not a mask + ct = ct.reindex(columns=cols) + headers = ["Rejection reason"] + [col_fmt(c) for c in ct.columns] + ["Total"] + rows = [] + for reason, series in ct.iterrows(): + vals = [fmt_int(v) for v in series.tolist()] + rows.append([esc(reason)] + vals + [fmt_int(series.sum())]) + totals = ["Total"] + [f"{fmt_int(ct[c].sum())}" for c in ct.columns] + totals += [f"{fmt_int(ct.values.sum())}"] + rows.append(totals) + return html_table(headers, rows) + + +def section_stratification(audit_path): + if not audit_path or not os.path.isfile(audit_path): + return not_provided() + try: + import pyarrow.parquet as pq + except Exception as exc: # pragma: no cover - defensive + return not_provided(f"pyarrow unavailable: {exc}") + + schema = pq.read_schema(audit_path) + want = [c for c in ("rejection_reason", "charge", "entrapment_label", + "target_decoy_label") if c in schema.names] + if "rejection_reason" not in want: + return not_provided("no rejection_reason column in audit parquet") + + df = pq.read_table(audit_path, columns=want).to_pandas() + n = len(df) + parts = [f'

{fmt_int(n)} candidate rows.

'] + + if "charge" in df.columns: + charges = sorted(c for c in df["charge"].dropna().unique()) + parts.append("

Rejection reason by charge

") + parts.append(_crosstab(df, "rejection_reason", "charge", + col_order=charges, col_fmt=lambda c: f"charge {int(c)}")) + + if "entrapment_label" in df.columns: + parts.append("

Rejection reason by entrapment label

") + parts.append(_crosstab(df, "rejection_reason", "entrapment_label", + col_order=[False, True], + col_fmt=lambda c: f"entrapment={bool(c)}")) + + if "target_decoy_label" in df.columns: + parts.append("

Rejection reason by target / decoy

") + parts.append(_crosstab(df, "rejection_reason", "target_decoy_label", + col_order=["target", "decoy"], + col_fmt=str)) + + return "".join(parts) + + +# --------------------------------------------------------------------------- # +# Section 4: top-K peak recall +_TOPK_KEYS = [ + ("frac_rank1", "Selected apex is peak rank-1"), + ("frac_top3", "Selected apex in top-3"), + ("frac_top5", "Selected apex in top-5"), + ("frac_top10", "Selected apex in top-10"), + ("frac_no_peak", "Selected apex in no enumerated peak"), +] + + +def section_topk(topk): + if not topk: + return not_provided() + + parts = [] + params = topk.get("params") or {} + ctx_rows = [] + for key, label in ( + ("n_candidates_total", "Candidates total"), + ("n_candidates_selected", "Candidates selected"), + ("n_candidates_processed", "Candidates processed"), + ("n_no_chrom_or_empty", "No chromatogram / empty"), + ): + if topk.get(key) is not None: + ctx_rows.append((label, fmt_int(topk[key]))) + if params.get("rt_tol_s") is not None: + ctx_rows.append(("RT tolerance (s)", fmt_num(params["rt_tol_s"]))) + if params.get("top_frags") is not None: + ctx_rows.append(("Top fragments", fmt_int(params["top_frags"]))) + if ctx_rows: + parts.append("

Context

") + parts.append(html_table(["Item", "Value"], ctx_rows)) + + self_m = topk.get("self") + if not self_m: + parts.append(not_provided("no 'self' block in topk json")) + return "".join(parts) + + rows = [] + for key, label in _TOPK_KEYS: + if key in self_m and self_m[key] is not None: + rows.append((label, fmt_pct(self_m[key], 2))) + for key, label in ( + ("mean_peaks_per_candidate", "Mean peaks / candidate"), + ("median_peaks_per_candidate", "Median peaks / candidate"), + ("frac_ge2_peaks", "Candidates with >= 2 peaks"), + ("n_apex_matched_to_peak", "Apexes matched to a peak"), + ("denominator", "Denominator"), + ): + if key in self_m and self_m[key] is not None: + val = fmt_pct(self_m[key], 2) if key == "frac_ge2_peaks" else ( + fmt_num(self_m[key]) if "peaks_per" in key else fmt_int(self_m[key])) + rows.append((label, val)) + parts.append("

Self peak recall

") + parts.append(html_table(["Metric", "Value"], rows)) + + # cumulative top-K bar chart + if _HAVE_MPL: + bar_keys = [("frac_rank1", "rank-1"), ("frac_top3", "top-3"), + ("frac_top5", "top-5"), ("frac_top10", "top-10")] + xs = [lbl for k, lbl in bar_keys if self_m.get(k) is not None] + ys = [self_m[k] for k, _ in bar_keys if self_m.get(k) is not None] + if ys: + fig, ax = plt.subplots(figsize=(6.0, 3.2)) + bars = ax.bar(xs, ys, color="#4c9a63", edgecolor="#2f6b40") + ax.set_ylim(0, 1.0) + ax.set_ylabel("fraction of candidates") + ax.set_title("Selected apex within cumulative top-K") + for rect, y in zip(bars, ys): + ax.text(rect.get_x() + rect.get_width() / 2, y + 0.02, + f"{y * 100:.1f}%", ha="center", va="bottom", fontsize=9) + ax.grid(axis="y", linestyle=":", alpha=0.4) + parts.append(img_panel(fig_to_data_uri(fig), + "Cumulative fraction where the selected apex is " + "among the top-K enumerated peaks.")) + + # reference (peak-oracle) block, if present + ref = topk.get("reference") + if isinstance(ref, dict) and ref: + ref_rows = [] + for k in ("reference_apex_in_top_1", "reference_apex_in_top_3", + "reference_apex_in_top_5", "reference_apex_in_top_10"): + if k in ref and ref[k] is not None: + ref_rows.append((k.replace("_", " "), fmt_pct(ref[k], 2))) + for k, v in ref.items(): + if not k.startswith("reference_apex_in_top_"): + ref_rows.append((esc(k), fmt_num(v) if isinstance(v, float) else fmt_int(v))) + parts.append("

Reference-apex peak oracle

") + parts.append(html_table(["Metric", "Value"], ref_rows)) + else: + parts.append('

Reference (DIA-NN) apex block not present; ' + "self analysis only.

") + + return "".join(parts) + + +# --------------------------------------------------------------------------- # +# Section 5: feature-family ablation +def _resolve_ablation_csvs(path): + if not path: + return [] + if os.path.isfile(path) and path.lower().endswith(".csv"): + return [path] + if os.path.isdir(path): + preferred = os.path.join(path, "feature_ablation.csv") + found = [] + if os.path.isfile(preferred): + found.append(preferred) + for name in sorted(os.listdir(path)): + full = os.path.join(path, name) + if full != preferred and name.lower().endswith(".csv") and os.path.isfile(full): + found.append(full) + return found + return [] + + +def _to_int(x): + try: + return int(float(x)) + except (TypeError, ValueError): + return None + + +def section_ablation(path): + csvs = _resolve_ablation_csvs(path) + if not csvs: + if path: + return not_provided(f"no ablation CSV found at {esc(path)}") + return not_provided() + + rows = [] + for cpath in csvs: + with open(cpath, "r", encoding="utf-8", newline="") as fh: + for r in csv.DictReader(fh): + rows.append(r) + if not rows: + return not_provided("ablation CSV(s) contained no rows") + + parts = [f'

Source: {esc(", ".join(os.path.basename(c) for c in csvs))}

'] + + models = [] + for r in rows: + m = r.get("model") or "model" + if m not in models: + models.append(m) + + for model in models: + mrows = [r for r in rows if (r.get("model") or "model") == model] + # most useful first: most negative delta_vs_full (removal hurts most) + mrows.sort(key=lambda r: (_to_int(r.get("delta_vs_full")) is None, + _to_int(r.get("delta_vs_full")) if + _to_int(r.get("delta_vs_full")) is not None else 0)) + headers = ["Feature family", "Baseline IDs", "New IDs", + "delta vs full", "Recommendation"] + trows = [] + for r in mrows: + delta = _to_int(r.get("delta_vs_full")) + delta_s = f"{delta:+,}" if delta is not None else esc(r.get("delta_vs_full")) + rec = r.get("recommendation") or "" + rec_cls = { + "KEEP": "rec-keep", "HARMFUL": "rec-harm", + "REDUNDANT_BUT_INFORMATIVE": "rec-info", "REDUNDANT": "rec-red", + }.get(rec, "") + rec_html = f'{esc(rec)}' if rec else "" + trows.append([ + esc(r.get("feature_family")), + fmt_int(r.get("baseline_identifications")), + fmt_int(r.get("new_identifications")), + delta_s, + rec_html, + ]) + parts.append(f"

Model: {esc(model)}

") + parts.append(html_table(headers, trows)) + return "".join(parts) + + +# --------------------------------------------------------------------------- # +# Section 6: empirical entrapment FDP +def section_entrapment(path): + if not path or not os.path.isfile(path): + return not_provided() + with open(path, "r", encoding="utf-8", errors="replace") as fh: + text = fh.read() + + rows = [] + m = re.search(r"E\.?coli targets\s*=\s*(\d+)", text) + if m: + rows.append(("E. coli targets", fmt_int(m.group(1)))) + m = re.search(r"human train-neg\s*=\s*(\d+)", text) + if m: + rows.append(("Human train-negatives", fmt_int(m.group(1)))) + m = re.search(r"test-null\s*=\s*(\d+)", text) + if m: + rows.append(("Held-out test null", fmt_int(m.group(1)))) + m = re.search(r"ratio\s*=\s*([\d.]+)", text) + if m: + rows.append(("Library-size ratio", fmt_num(m.group(1)))) + for m in re.finditer( + r"held-out E\.?coli stripped seqs @\s*(\d+)%[^:]*:\s*(\d+)", text + ): + rows.append((f"Held-out E. coli stripped seqs @ {m.group(1)}%", + fmt_int(m.group(2)))) + m = re.search( + r"at shipped q<=\s*(\d+)%:\s*E\.?coli\s*=\s*(\d+),\s*true FDR" + r"[^=]*=\s*([\d.]+)%", + text, + ) + if m: + rows.append((f"E. coli at shipped q <= {m.group(1)}%", fmt_int(m.group(2)))) + rows.append((f"True FDR on held-out null (q <= {m.group(1)}%)", + f"{m.group(3)}%")) + + parts = [] + if rows: + parts.append(html_table(["Metric", "Value"], rows)) + else: + parts.append(not_provided("no recognizable entrapment metrics in the file")) + parts.append("
Raw entrapment output" + f"
{esc(text.strip())}
") + return "".join(parts) + + +# --------------------------------------------------------------------------- # +# Section 7: limitations footer +def section_limitations(rendered): + absent = [name for name, ok in rendered.items() if not ok] + items = [ + "Diagnostics are computed on a single dataset (the E. coli / HYE example); " + "no family, threshold, or component decision should be made from one dataset " + "or one favourable subset (spec 03 Section 9, spec 05 Section 7).", + "The identification-loss waterfall collapses all extraction losses to " + "NO_PEAK_GROUP at artifact resolution; the in-extract audit sidecar is needed " + "to separate NO_FRAGMENT_TRACES / NO_VALID_FRAGMENTS / PEAK_NOT_SELECTED / " + "RT_PRUNED.", + "Top-K self recall measures peak-selection opportunity only; the reference " + "(DIA-NN) peak-oracle metric requires a reference report and is shown only " + "when a reference block is present.", + "Feature-family ablation is cross-validated on one dataset; a gain that flips " + "sign across models or datasets is not a keep. Rerun with both models and a " + "second dataset before acting.", + "The entrapment held-out FDP is the accept / reject gate; a sensitivity gain " + "that inflates empirical FDP must not be retained.", + ] + parts = ["
    "] + for it in items: + parts.append(f"
  • {esc(it)}
    • ") + parts.append("
") + if absent: + pretty = ", ".join(esc(a) for a in absent) + parts.append(f'

Sections not rendered (input absent): ' + f"{pretty}.

") + return "".join(parts) + + +# --------------------------------------------------------------------------- # +_CSS = """ +:root { color-scheme: light dark; } +* { box-sizing: border-box; } +body { + margin: 0; padding: 0 0 4rem 0; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, + Arial, sans-serif; + line-height: 1.5; color: #1c2530; background: #f4f6f9; +} +.container { max-width: 1040px; margin: 0 auto; padding: 1.5rem; } +header.report { + background: #26456f; color: #fff; padding: 1.6rem 1.5rem; border-radius: 0 0 10px 10px; +} +header.report h1 { margin: 0 0 .3rem 0; font-size: 1.55rem; } +header.report .stamp { opacity: .85; font-size: .9rem; } +section { + background: #fff; border: 1px solid #e2e7ee; border-radius: 10px; + padding: 1.2rem 1.3rem; margin: 1.2rem 0; box-shadow: 0 1px 2px rgba(0,0,0,.04); +} +section > h2 { + margin: 0 0 .4rem 0; font-size: 1.2rem; color: #26456f; + border-bottom: 2px solid #eef1f5; padding-bottom: .4rem; +} +section > h2 .sec-no { + display: inline-block; min-width: 1.6rem; color: #7a8aa0; font-weight: 600; +} +h3 { font-size: 1rem; margin: 1rem 0 .4rem 0; color: #33445c; } +p.desc { margin: .2rem 0 .8rem 0; color: #55627a; font-size: .92rem; } +p.np { + color: #8a94a6; font-style: italic; background: #f7f8fb; border: 1px dashed #d7dde7; + padding: .5rem .7rem; border-radius: 6px; display: inline-block; +} +p.muted { color: #6b7688; font-size: .88rem; } +.table-wrap { overflow-x: auto; -webkit-overflow-scrolling: touch; margin: .3rem 0 .6rem; } +table { border-collapse: collapse; width: 100%; font-size: .9rem; } +th, td { + text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #eef1f5; + white-space: nowrap; +} +th { background: #f0f3f8; color: #33445c; font-weight: 600; position: sticky; top: 0; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } +tbody tr:hover { background: #f7f9fc; } +figure.chart { margin: .8rem 0; text-align: center; } +figure.chart img { + max-width: 100%; height: auto; border: 1px solid #e2e7ee; border-radius: 8px; + background: #fff; +} +figcaption { color: #6b7688; font-size: .82rem; margin-top: .35rem; } +.toc { font-size: .92rem; } +.toc a { color: #26456f; text-decoration: none; } +.toc a:hover { text-decoration: underline; } +.rec { font-size: .78rem; padding: .1rem .45rem; border-radius: 10px; font-weight: 600; } +.rec-keep { background: #dff3e4; color: #1f6b39; } +.rec-harm { background: #fde3e0; color: #9c2b20; } +.rec-info { background: #e5eefb; color: #26456f; } +.rec-red { background: #eef0f4; color: #66707f; } +details { margin-top: .6rem; } +summary { cursor: pointer; color: #26456f; font-weight: 600; } +pre { + overflow-x: auto; background: #0f1826; color: #d7e0ee; padding: .8rem; + border-radius: 8px; font-size: .82rem; line-height: 1.45; +} +footer.report { color: #6b7688; font-size: .8rem; text-align: center; padding: 1rem; } +@media (prefers-color-scheme: dark) { + body { background: #10151d; color: #d6dde8; } + section { background: #1a212c; border-color: #2a3341; box-shadow: none; } + section > h2 { color: #9db9e6; border-bottom-color: #2a3341; } + h3 { color: #c3cede; } + th { background: #232c39; color: #c3cede; } + th, td { border-bottom-color: #2a3341; } + tbody tr:hover { background: #212a37; } + p.np { background: #202836; border-color: #2f3a49; color: #8a94a6; } + p.desc, p.muted, figcaption { color: #97a3b6; } + figure.chart img { border-color: #2a3341; } +} +""" + + +def build_html(title, stamp, sections): + """sections: list of (num, anchor, name, description, body_html).""" + toc = " · ".join( + f'{num}. {esc(name)}' + for num, anchor, name, _desc, _body in sections + ) + blocks = [] + for num, anchor, name, desc, body in sections: + desc_html = f'

{esc(desc)}

' if desc else "" + blocks.append( + f'
' + f'

{num}.{esc(name)}

' + f"{desc_html}{body}
" + ) + return ( + "\n" + '' + '' + f"{esc(title)}" + f"" + f'

{esc(title)}

' + f'
Generated {esc(stamp)}
' + f'
' + f'{"".join(blocks)}' + '
MuMDIA sensitivity benchmark report. ' + "All diagnostics are read-only over existing artifacts.
" + "
" + ) + + +# --------------------------------------------------------------------------- # +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--out", required=True, help="output HTML file") + ap.add_argument("--audit-metrics", default=None, + help="candidate_audit.parquet.metrics.json (waterfall)") + ap.add_argument("--audit", default=None, + help="candidate_audit.parquet (stratification)") + ap.add_argument("--topk", default=None, help="topk.json (peak recall)") + ap.add_argument("--ablation", default=None, + help="feature-family ablation CSV or directory") + ap.add_argument("--entrapment", default=None, + help="entrapment holdout stdout text file") + ap.add_argument("--title", default="MuMDIA Sensitivity Benchmark Report") + ap.add_argument("--stamp", default=None, + help="fixed header timestamp (default: current local time)") + args = ap.parse_args() + + stamp = args.stamp or _dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + metrics = load_json(args.audit_metrics) + topk = load_json(args.topk) + + # Section 1 body: run metadata + input inventory + inputs = [ + ("Audit metrics (JSON)", args.audit_metrics), + ("Candidate audit (Parquet)", args.audit), + ("Top-K peak recall (JSON)", args.topk), + ("Feature ablation (CSV/dir)", args.ablation), + ("Entrapment stdout (text)", args.entrapment), + ] + meta_rows = [] + if metrics and metrics.get("run_id"): + meta_rows.append(("Run id", esc(metrics["run_id"]))) + meta_rows.append(("Report generated", esc(stamp))) + for label, path in inputs: + present = bool(path) and os.path.exists(path) + status = "present" if present else ("missing" if path else "not provided") + shown = esc(path) if path else "—" + meta_rows.append((label, f"{shown} ({status})")) + header_body = html_table(["Item", "Value"], meta_rows, align_right_from=99) + + # build sections + wf_body = section_waterfall(metrics) + strat_body = section_stratification(args.audit) + topk_body = section_topk(topk) + abl_body = section_ablation(args.ablation) + ent_body = section_entrapment(args.entrapment) + + rendered = { + "Identification-loss waterfall": bool(metrics), + "Candidate-audit stratification": bool(args.audit and os.path.isfile(args.audit)), + "Top-K peak recall": bool(topk), + "Feature-family ablation": bool(_resolve_ablation_csvs(args.ablation)), + "Empirical entrapment FDP": bool(args.entrapment and os.path.isfile(args.entrapment)), + } + lim_body = section_limitations(rendered) + + sections = [ + (1, "run", "Run metadata", + "Report inputs and provenance. Each downstream section renders only when " + "its input is provided.", header_body), + (2, "waterfall", "Identification-loss waterfall", + "Where DIA-NN-only precursors are lost, from candidate audit metrics " + "(spec 02 Section 8).", wf_body), + (3, "stratification", "Candidate-audit stratification", + "Earliest rejection reason grouped by charge and entrapment label.", + strat_body), + (4, "topk", "Top-K peak recall", + "How often the selected apex is the strongest peak, and (with a reference) " + "whether the reference apex is within the top-K peaks.", topk_body), + (5, "ablation", "Feature-family ablation", + "Cross-validated contribution of each feature family, most useful first " + "(largest identification drop when removed).", abl_body), + (6, "entrapment", "Empirical entrapment FDP", + "Held-out entrapment identification count at a genuinely controlled FDP " + "(the accept / reject gate).", ent_body), + (7, "limitations", "Limitations", + "What is single-dataset or not yet measured.", lim_body), + ] + + doc = build_html(args.title, stamp, sections) + out_dir = os.path.dirname(os.path.abspath(args.out)) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(doc) + + n_rendered = sum(1 for v in rendered.values() if v) + print(f"[written] {args.out} ({os.path.getsize(args.out):,} bytes)") + print(f"[sections] {n_rendered + 2}/7 rendered " + f"(run metadata + limitations always render)") + for name, ok in rendered.items(): + print(f" {'rendered ' if ok else 'not-prov '} {name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/candidate_diagnostics.py b/scripts/candidate_diagnostics.py new file mode 100644 index 0000000..9aeee5a --- /dev/null +++ b/scripts/candidate_diagnostics.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python +"""Per-candidate diagnostic bundle export (sensitivity_plan backlog P7.2). + +Implements the individual-candidate diagnostic packet described in +sensitivity_plan/02_sensitivity_diagnostic_plan.md section 9. For a selected +list of candidate_ids the script reads the run artifacts (non-invasively, never +writing back to them) and produces, per candidate: + + fragments.png overlaid fragment chromatograms (MS2 left axis, MS1 + right axis via twinx, apex and reference RT markers) + predicted_vs_observed.png predicted vs observed-at-apex intensity per fragment + candidate.json metadata, per-fragment summary, and all comp features + +A top-level index.txt lists the exported candidates and their key fields. + +Design notes: + - Deterministic: fixed Agg backend, sorted candidate order, sorted fragment + order, no random state. + - Bounded memory: chrom / comp / psms / scored are read with a pyarrow + dataset filter (predicate pushdown) restricted to the requested + candidate_ids, so the full tables are never materialized. + - Robust to missing MS1 traces, missing comp, and missing scored inputs. + +Interpreter: C:/Users/robbi/anaconda3/envs/py312_mumdia/python.exe +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys + +import matplotlib + +matplotlib.use("Agg") # headless, deterministic rendering + +import matplotlib.pyplot as plt +import numpy as np +import pyarrow.dataset as ds + + +# --------------------------------------------------------------------------- # +# Fragment-name parsing and ordering +# --------------------------------------------------------------------------- # + +_FRAG_RE = re.compile(r"^(?P[a-zA-Z]+)(?P\d+)(?:\^(?P\d+))?$") + + +def is_ms1_name(name: str) -> bool: + """Return True for MS1 precursor-isotope trace rows (ms1_mono, ms1_iso1, ...).""" + return str(name).lower().startswith("ms1") + + +def frag_sort_key(name: str): + """Deterministic, human-readable ordering key for fragment names. + + MS2 ions sort by series letter, ordinal, then fragment charge. Names that do + not parse fall back to a lexical key placed after parsed ions. + """ + m = _FRAG_RE.match(str(name)) + if not m: + return (2, str(name), 0, 0) + series = m.group("series").lower() + ordinal = int(m.group("ordinal")) + charge = int(m.group("charge")) if m.group("charge") else 1 + return (0, series, ordinal, charge) + + +# --------------------------------------------------------------------------- # +# IO helpers +# --------------------------------------------------------------------------- # + +def load_filtered(path: str, cand_ids, columns=None): + """Load only the rows for cand_ids from a parquet file (predicate pushdown). + + Returns a pandas DataFrame, or None if the path is falsy. Missing columns + are tolerated: only columns present in the file schema are requested. + """ + if not path: + return None + dataset = ds.dataset(path, format="parquet") + if columns is not None: + available = set(dataset.schema.names) + columns = [c for c in columns if c in available] + if "candidate_id" not in columns: + columns = ["candidate_id"] + columns + table = dataset.to_table( + filter=ds.field("candidate_id").isin(list(cand_ids)), + columns=columns, + ) + return table.to_pandas() + + +def index_by_candidate(df): + """Return {candidate_id: first-row dict} for quick per-candidate lookup.""" + out = {} + if df is None: + return out + for row in df.to_dict("records"): + cid = int(row["candidate_id"]) + if cid not in out: # keep first occurrence for determinism + out[cid] = row + return out + + +def as_float_array(seq): + """Coerce a nullable list column value to a 1-D float ndarray (empty if None).""" + if seq is None: + return np.empty(0, dtype=float) + arr = np.asarray(list(seq), dtype=float) + return arr + + +def value_at_rt(rt, intensity, target_rt): + """Observed intensity at the scan nearest target_rt, or None if unavailable.""" + rt = as_float_array(rt) + intensity = as_float_array(intensity) + if rt.size == 0 or intensity.size == 0 or target_rt is None: + return None + n = min(rt.size, intensity.size) + idx = int(np.argmin(np.abs(rt[:n] - float(target_rt)))) + return float(intensity[idx]) + + +def count_nonzero(intensity): + intensity = as_float_array(intensity) + if intensity.size == 0: + return 0 + return int(np.count_nonzero(intensity > 0.0)) + + +def json_default(obj): + """Make numpy scalars / arrays JSON-serializable.""" + if isinstance(obj, (np.integer,)): + return int(obj) + if isinstance(obj, (np.floating,)): + v = float(obj) + return v if np.isfinite(v) else None + if isinstance(obj, (np.bool_,)): + return bool(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + return str(obj) + + +def clean_scalar(v): + """Normalize a scalar for JSON: NaN/inf -> None, numpy -> python.""" + if v is None: + return None + if isinstance(v, (np.integer,)): + return int(v) + if isinstance(v, (np.floating, float)): + v = float(v) + return v if np.isfinite(v) else None + if isinstance(v, (np.bool_,)): + return bool(v) + return v + + +# --------------------------------------------------------------------------- # +# Reference RT table +# --------------------------------------------------------------------------- # + +def parse_ref_rt_file(path): + """Parse a candidate_id,rt reference table. Returns {candidate_id: rt}. + + Accepts an optional header line and comma or whitespace separation. Lines + that do not parse as (int, float) are skipped. + """ + ref = {} + if not path: + return ref + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = re.split(r"[,\t ]+", line) + if len(parts) < 2: + continue + try: + cid = int(float(parts[0])) + rt = float(parts[1]) + except ValueError: + continue # header or malformed line + ref[cid] = rt + return ref + + +# --------------------------------------------------------------------------- # +# Candidate id selection +# --------------------------------------------------------------------------- # + +def parse_candidate_list(arg_value, arg_file): + ids = [] + if arg_value: + for tok in re.split(r"[,\s]+", arg_value.strip()): + if tok: + ids.append(int(tok)) + if arg_file: + with open(arg_file, "r", encoding="utf-8") as fh: + for line in fh: + for tok in re.split(r"[,\s]+", line.strip()): + if tok and not tok.startswith("#"): + ids.append(int(tok)) + # de-duplicate, preserve first-seen order, then sort for determinism + seen = set() + uniq = [] + for i in ids: + if i not in seen: + seen.add(i) + uniq.append(i) + return sorted(uniq) + + +# --------------------------------------------------------------------------- # +# Plotting +# --------------------------------------------------------------------------- # + +def plot_fragments(cand_id, frag_rows, ms1_rows, apex_rt, ref_rt, + ms1_scalars, title, out_path): + """Overlaid fragment chromatograms with MS1 on a separate twin axis.""" + fig, ax = plt.subplots(figsize=(11, 6)) + + cmap = plt.get_cmap("tab20") + handles, labels = [], [] + + # MS2 fragment traces on the left axis. + for i, row in enumerate(frag_rows): + rt = as_float_array(row["rt"]) + inten = as_float_array(row["intensity"]) + if rt.size == 0 or inten.size == 0: + continue + n = min(rt.size, inten.size) + color = cmap(i % 20) + (line,) = ax.plot(rt[:n], inten[:n], color=color, linewidth=1.2, + label=row["frag_name"]) + handles.append(line) + labels.append(row["frag_name"]) + + ax.set_xlabel("Retention time (s)") + ax.set_ylabel("MS2 fragment intensity") + ax.set_title(title) + + # MS1 traces on a separate twin axis (never normalized into MS2 scaling). + ax2 = None + if ms1_rows: + ax2 = ax.twinx() + ms1_cmap = plt.get_cmap("Dark2") + for j, row in enumerate(ms1_rows): + rt = as_float_array(row["rt"]) + inten = as_float_array(row["intensity"]) + if rt.size == 0 or inten.size == 0: + continue + n = min(rt.size, inten.size) + color = ms1_cmap(j % 8) + (line,) = ax2.plot(rt[:n], inten[:n], color=color, linewidth=1.4, + linestyle="--", label=row["frag_name"]) + handles.append(line) + labels.append(row["frag_name"]) + ax2.set_ylabel("MS1 precursor-isotope intensity") + elif ms1_scalars: + # No MS1 traces; show available MS1 XIC scalars as points at the apex. + ax2 = ax.twinx() + ms1_cmap = plt.get_cmap("Dark2") + for j, (name, val) in enumerate(sorted(ms1_scalars.items())): + if val is None or apex_rt is None: + continue + color = ms1_cmap(j % 8) + pt = ax2.scatter([apex_rt], [val], color=color, marker="D", s=40, + zorder=5, label=name + " (scalar)") + handles.append(pt) + labels.append(name + " (scalar)") + ax2.set_ylabel("MS1 precursor-isotope intensity (scalar)") + + # Apex and reference RT markers. + if apex_rt is not None: + vl = ax.axvline(apex_rt, color="black", linewidth=1.4, linestyle="-", + label="apex_rt") + handles.append(vl) + labels.append("apex_rt") + if ref_rt is not None: + vr = ax.axvline(ref_rt, color="red", linewidth=1.4, linestyle=":", + label="reference_rt") + handles.append(vr) + labels.append("reference_rt") + + if handles: + ax.legend(handles, labels, fontsize=7, ncol=2, loc="upper right", + framealpha=0.9) + + fig.tight_layout() + fig.savefig(out_path, dpi=120) + plt.close(fig) + + +def plot_predicted_vs_observed(frag_summ, title, out_path): + """Grouped bar chart: predicted intensity vs observed-at-apex per fragment.""" + names = [f["name"] for f in frag_summ] + pred = np.array([f["predicted_intensity"] or 0.0 for f in frag_summ], float) + obs = np.array( + [(f["observed_apex_intensity"] or 0.0) for f in frag_summ], float + ) + + fig, ax = plt.subplots(figsize=(max(8, 0.5 * len(names) + 2), 6)) + if names: + x = np.arange(len(names)) + width = 0.4 + # Normalize each series to its own max so predicted (relative) and + # observed (raw counts) shapes are comparable on one axis. + pred_n = pred / pred.max() if pred.max() > 0 else pred + obs_n = obs / obs.max() if obs.max() > 0 else obs + ax.bar(x - width / 2, pred_n, width, label="predicted (norm)", + color="#4C72B0") + ax.bar(x + width / 2, obs_n, width, label="observed at apex (norm)", + color="#DD8452") + ax.set_xticks(x) + ax.set_xticklabels(names, rotation=60, ha="right", fontsize=8) + ax.set_ylabel("Relative intensity (per-series max = 1)") + ax.legend(fontsize=8) + else: + ax.text(0.5, 0.5, "no MS2 fragments", ha="center", va="center") + ax.set_title(title) + fig.tight_layout() + fig.savefig(out_path, dpi=120) + plt.close(fig) + + +# --------------------------------------------------------------------------- # +# Per-candidate bundle +# --------------------------------------------------------------------------- # + +FEATURE_SKIP = { + "candidate_id", "label", "base_peptide_id", "peptidoform", "protein", + "apex_rt", "precursor_mz", +} + + +def build_bundle(cand_id, chrom_df, psms_row, comp_row, scored_row, ref_rt, + out_dir): + """Write fragments.png, predicted_vs_observed.png and candidate.json. + + Returns a summary dict used for index.txt. + """ + cdir = os.path.join(out_dir, str(cand_id)) + os.makedirs(cdir, exist_ok=True) + + # Split chrom rows into MS2 fragments and MS1 traces, sorted deterministically. + rows = chrom_df.to_dict("records") if chrom_df is not None else [] + frag_rows = [r for r in rows if not is_ms1_name(r["frag_name"])] + ms1_rows = [r for r in rows if is_ms1_name(r["frag_name"])] + frag_rows.sort(key=lambda r: frag_sort_key(r["frag_name"])) + ms1_rows.sort(key=lambda r: str(r["frag_name"])) + + # Metadata, tolerant of missing psms / scored rows. + def g(row, key, default=None): + if row is None or key not in row: + return default + return clean_scalar(row[key]) + + peptidoform = g(psms_row, "peptidoform") or g(scored_row, "peptidoform") \ + or g(comp_row, "peptidoform") + charge = g(psms_row, "charge") + if charge is None: + charge = g(scored_row, "charge") + if charge is None: + charge = g(comp_row, "charge") + label = g(psms_row, "label") or g(scored_row, "label") or g(comp_row, "label") + protein = g(psms_row, "protein") or g(scored_row, "protein") \ + or g(comp_row, "protein") + apex_rt = g(psms_row, "apex_rt") + if apex_rt is None: + apex_rt = g(comp_row, "apex_rt") + q_value = g(scored_row, "q_value") + n_matched = g(psms_row, "n_matched_fragments") + + # MS1 XIC scalars from psms (fallback plotting + JSON record). + ms1_scalars = {} + for key in ("ms1_isom1", "ms1_mono", "ms1_iso1", "ms1_iso2"): + val = g(psms_row, key) + if val is not None: + ms1_scalars[key] = val + + # Per-fragment summary (MS2 fragments only). + frag_summ = [] + for r in frag_rows: + frag_summ.append({ + "name": r["frag_name"], + "frag_mz": clean_scalar(r.get("frag_mz")), + "predicted_intensity": clean_scalar(r.get("predicted_intensity")), + "observed_apex_intensity": value_at_rt( + r.get("rt"), r.get("intensity"), apex_rt), + "n_nonzero_points": count_nonzero(r.get("intensity")), + }) + + charge_str = "" if charge is None else "+%d" % int(charge) + title = "%s%s %s (candidate %d)" % ( + peptidoform or "?", charge_str, label or "?", cand_id) + + plot_fragments( + cand_id, frag_rows, ms1_rows, apex_rt, ref_rt, ms1_scalars, title, + os.path.join(cdir, "fragments.png"), + ) + plot_predicted_vs_observed( + frag_summ, title, os.path.join(cdir, "predicted_vs_observed.png"), + ) + + # Feature values from comp (all columns except identity/context fields). + features = {} + if comp_row is not None: + for k, v in comp_row.items(): + if k in FEATURE_SKIP: + continue + features[k] = clean_scalar(v) + + bundle = { + "candidate_id": cand_id, + "peptidoform": peptidoform, + "charge": None if charge is None else int(charge), + "label": label, + "protein": protein, + "apex_rt": apex_rt, + "reference_rt": ref_rt, + "q_value": q_value, + "n_matched_fragments": None if n_matched is None else int(n_matched), + "n_fragments_chrom": len(frag_rows), + "n_ms1_traces": len(ms1_rows), + "ms1_scalars": ms1_scalars, + "fragments": frag_summ, + "features": features, + } + with open(os.path.join(cdir, "candidate.json"), "w", encoding="utf-8") as fh: + json.dump(bundle, fh, indent=2, default=json_default) + + return { + "candidate_id": cand_id, + "peptidoform": peptidoform or "?", + "charge": "" if charge is None else int(charge), + "label": label or "?", + "q_value": q_value, + "apex_rt": apex_rt, + "n_matched_fragments": n_matched, + "n_fragments": len(frag_rows), + "protein": protein or "?", + } + + +# --------------------------------------------------------------------------- # +# Main +# --------------------------------------------------------------------------- # + +def main(argv=None): + ap = argparse.ArgumentParser( + description="Export per-candidate diagnostic bundles (sensitivity_plan " + "P7.2, spec 02 section 9).", + ) + ap.add_argument("--chrom", required=True, help="chrom.parquet path") + ap.add_argument("--psms", required=True, help="psms.parquet path") + ap.add_argument("--comp", default=None, help="comp.parquet path (optional)") + ap.add_argument("--scored", default=None, + help="scored.parquet path (optional)") + ap.add_argument("--candidates", default=None, + help="comma-separated candidate_ids") + ap.add_argument("--candidates-file", default=None, + help="file with candidate_ids (comma/whitespace/newline)") + ap.add_argument("--out", default="candidate_diag", + help="output directory (default: candidate_diag)") + ap.add_argument("--ref-rt-file", default=None, + help="reference RT table: candidate_id,rt per line") + args = ap.parse_args(argv) + + cand_ids = parse_candidate_list(args.candidates, args.candidates_file) + if not cand_ids: + ap.error("no candidate_ids given (use --candidates or --candidates-file)") + + os.makedirs(args.out, exist_ok=True) + ref_rts = parse_ref_rt_file(args.ref_rt_file) + + # Bounded reads: only rows for the requested candidate_ids. + chrom_df = load_filtered(args.chrom, cand_ids) + psms_idx = index_by_candidate(load_filtered(args.psms, cand_ids)) + comp_idx = index_by_candidate(load_filtered(args.comp, cand_ids)) + scored_idx = index_by_candidate(load_filtered(args.scored, cand_ids)) + + # Group chrom rows per candidate once. + chrom_by_cand = {} + if chrom_df is not None and len(chrom_df): + for cid, grp in chrom_df.groupby("candidate_id"): + chrom_by_cand[int(cid)] = grp + + summaries = [] + for cid in cand_ids: + sub = chrom_by_cand.get(cid) + if sub is None: + print("warning: no chrom rows for candidate_id %d" % cid, + file=sys.stderr) + summ = build_bundle( + cid, + sub, + psms_idx.get(cid), + comp_idx.get(cid), + scored_idx.get(cid), + ref_rts.get(cid), + args.out, + ) + summaries.append(summ) + print("wrote %s" % os.path.join(args.out, str(cid))) + + # Top-level index. + index_path = os.path.join(args.out, "index.txt") + header = ["candidate_id", "peptidoform", "charge", "label", "q_value", + "apex_rt", "n_matched_fragments", "n_fragments", "protein"] + with open(index_path, "w", encoding="utf-8") as fh: + fh.write("\t".join(header) + "\n") + for s in summaries: + qv = "" if s["q_value"] is None else "%.6g" % s["q_value"] + ar = "" if s["apex_rt"] is None else "%.4f" % s["apex_rt"] + nm = "" if s["n_matched_fragments"] is None else str( + int(s["n_matched_fragments"])) + fh.write("\t".join([ + str(s["candidate_id"]), + str(s["peptidoform"]), + str(s["charge"]), + str(s["label"]), + qv, + ar, + nm, + str(s["n_fragments"]), + str(s["protein"]), + ]) + "\n") + print("wrote %s" % index_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/feature_audit.py b/scripts/feature_audit.py new file mode 100644 index 0000000..4de2f85 --- /dev/null +++ b/scripts/feature_audit.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python +"""Feature audit for MuMDIA scored/competed feature tables. + +Implements the sensitivity_plan backlog item P4.2 (spec +``sensitivity_plan/03_feature_evaluation.md`` section 5, evidence ladder +Level 1 to Level 3). The tool is non-invasive: it reads a competed features +Parquet plus the feature registry and writes an audit report. Nothing in the +engine or the input data is modified. + +Evidence ladder covered here: + +Level 1 (data quality), per feature: + missing percentage, NaN/inf percentage, unique-value count, quantiles + (5/25/50/75/95), constant flag, per-group distribution summaries for + targets, decoys and entrapments (mean/median), correlation with + prelim_score, correlation with peptide length and charge, and correlation + with log apex intensity. Flags: constant, asymmetric missingness between + targets and decoys, intensity-dominated, and target-vs-decoy separation + that is much larger than target-vs-entrapment separation (a decoy + construction artifact). + +Level 2 (univariate utility), per feature: + target-vs-decoy separation and target-vs-entrapment separation, both as a + Mann-Whitney U rank AUC and its rank-biserial magnitude. Features that + separate target from decoy strongly but target from entrapment weakly are + flagged LEAKAGE_RISK. + +Level 3 (redundancy): + Spearman correlation matrix over the non-constant features, single-linkage + clustering at ``|rho| >= threshold`` via union-find, with one representative + reported per cluster. + +Registry cross-check: + every audited feature must appear in the registry; features that do not are + reported as UNKNOWN, and per-family coverage is summarised. + +Entrapment framing: this is a target-decoy plus entrapment experiment. The +"real" target group is the target-labelled rows whose protein matches the +real-species substring (default ``_ECOLI``). The entrapment group is the +target-labelled rows whose protein matches the entrapment substring (default +``_HUMAN``); these are foreign-proteome spike-ins that behave as a null. The +decoy group is the decoy-labelled rows. A useful feature separates real +targets from both decoys and entrapments; a feature that separates real +targets from decoys but not from entrapments is likely exploiting the decoy +construction scheme. + +The tool is deterministic: sampling uses a fixed seed, and all iteration +orders are fixed. + +Interpreter: ``C:/Users/robbi/anaconda3/envs/py312_mumdia/python.exe``. + +Example: + + python feature_audit.py \ + --features C:/proteobench/out_ecoli/comp.parquet \ + --registry feature_registry.yaml \ + --max-rows 80000 --entrapment-substr _HUMAN +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import defaultdict, OrderedDict + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import yaml +from scipy.stats import rankdata + +# --- Meta columns that are never treated as scoring features. -------------- +META_COLUMNS = [ + "candidate_id", + "label", + "base_peptide_id", + "peptidoform", + "protein", + "apex_rt", + "precursor_mz", + "prelim_score", + "charge", + "q_value", + "peptide_q_value", + "pg_q_value", + "global_q_value", + "score", +] + +# --- Flag thresholds (documented, deterministic). -------------------------- +# A separation is "strong" when its rank-biserial magnitude reaches this. +SEP_STRONG = 0.15 +# Leakage: target-vs-entrapment separation below this fraction of the +# target-vs-decoy separation, while the latter is strong. +LEAKAGE_RATIO = 0.5 +# Intensity-dominated: absolute Spearman correlation with log apex intensity. +INTENSITY_CORR = 0.70 +# Asymmetric missingness: target vs decoy missing-rate ratio, with a floor on +# the larger rate so that trivially tiny rates are not flagged. +ASYM_RATIO = 5.0 +ASYM_MIN_RATE = 0.01 +# Redundancy clustering default correlation threshold. +DEFAULT_REDUNDANCY_THRESHOLD = 0.90 +# Default deterministic sampling seed. +DEFAULT_SEED = 1234 + +_BRACKET_RE = re.compile(r"\[[^\]]*\]") + + +# --------------------------------------------------------------------------- +# Registry parsing +# --------------------------------------------------------------------------- +def load_registry(path): + """Load the feature registry YAML. + + Returns a dict name -> {family, level, direction, source_file, ...}. Only + the active (non-collision) rows are usable as columns; rows carrying + ``dropped_collision: true`` (keyed like ``name@family``) are kept so that + coverage accounting is complete but are not expected as columns. + """ + with open(path, "r", encoding="utf-8") as handle: + doc = yaml.safe_load(handle) + features = doc.get("features", {}) or {} + reg = OrderedDict() + for name, meta in features.items(): + meta = meta or {} + reg[str(name)] = { + "family": str(meta.get("family", "?")), + "level": str(meta.get("level", "?")), + "direction": str(meta.get("direction", "?")), + "source_file": str(meta.get("source_file", "")), + "dropped_collision": bool(meta.get("dropped_collision", False)), + } + return reg + + +# --------------------------------------------------------------------------- +# Data loading (bounded memory via deterministic row sampling) +# --------------------------------------------------------------------------- +def load_features(path, max_rows, seed): + """Read the features Parquet, sampling rows deterministically if needed. + + Returns (dataframe, total_rows, sampled_flag). + """ + table = pq.read_table(path) + total_rows = table.num_rows + sampled = False + if max_rows and total_rows > max_rows: + rng = np.random.RandomState(seed) + idx = np.sort(rng.choice(total_rows, size=max_rows, replace=False)) + table = table.take(pa.array(idx)) + sampled = True + df = table.to_pandas() + del table + return df, total_rows, sampled + + +def peptide_length(pepform): + """Stripped peptide length from a ProForma-lite peptidoform. + + Drops a ``DECOY_`` prefix, removes bracketed modifications with the regex + ``\\[[^\\]]*\\]``, and counts uppercase residue letters. + """ + if not isinstance(pepform, str) or not pepform: + return np.nan + seq = pepform + if seq.startswith("DECOY_"): + seq = seq[len("DECOY_"):] + seq = _BRACKET_RE.sub("", seq) + return float(sum(1 for c in seq if "A" <= c <= "Z")) + + +# --------------------------------------------------------------------------- +# Statistics helpers +# --------------------------------------------------------------------------- +def safe_spearman(x, y): + """Spearman correlation over jointly finite pairs; NaN if undefined.""" + mask = np.isfinite(x) & np.isfinite(y) + if mask.sum() < 3: + return np.nan + xr = rankdata(x[mask]) + yr = rankdata(y[mask]) + if xr.std() == 0.0 or yr.std() == 0.0: + return np.nan + return float(np.corrcoef(xr, yr)[0, 1]) + + +def auc_mwu(pos, neg): + """Rank AUC = P(pos > neg) via the Mann-Whitney U statistic. + + Returns NaN when either group is empty. Ties contribute 0.5. + """ + pos = pos[np.isfinite(pos)] + neg = neg[np.isfinite(neg)] + n1 = len(pos) + n2 = len(neg) + if n1 == 0 or n2 == 0: + return np.nan + ranks = rankdata(np.concatenate([pos, neg])) + r1 = ranks[:n1].sum() + u1 = r1 - n1 * (n1 + 1) / 2.0 + return float(u1 / (n1 * n2)) + + +def separation(auc): + """Rank-biserial magnitude in [0, 1] from a rank AUC.""" + if auc is None or not np.isfinite(auc): + return np.nan + return abs(2.0 * auc - 1.0) + + +# --------------------------------------------------------------------------- +# Group masks +# --------------------------------------------------------------------------- +def substr_match(series, substrings): + """Boolean mask: True where any of the substrings occurs in the string.""" + result = np.zeros(len(series), dtype=bool) + for sub in substrings: + if sub: + result |= series.str.contains(re.escape(sub), regex=True).to_numpy() + return result + + +# --------------------------------------------------------------------------- +# Redundancy clustering +# --------------------------------------------------------------------------- +class UnionFind: + def __init__(self, items): + self.parent = {it: it for it in items} + + def find(self, a): + root = a + while self.parent[root] != root: + root = self.parent[root] + # Path compression. + while self.parent[a] != root: + self.parent[a], a = root, self.parent[a] + return root + + def union(self, a, b): + ra, rb = self.find(a), self.find(b) + if ra != rb: + # Deterministic: smaller name becomes the root. + if rb < ra: + ra, rb = rb, ra + self.parent[rb] = ra + + +def cluster_features(df, feature_cols, threshold): + """Cluster features by single-linkage Spearman ``|rho| >= threshold``. + + Returns (clusters, corr_dataframe) where clusters is a dict + root -> sorted member list, including singletons. + """ + if not feature_cols: + return {}, None + sub = df[feature_cols].replace([np.inf, -np.inf], np.nan) + corr = sub.corr(method="spearman") + cols = list(corr.columns) + values = corr.to_numpy() + uf = UnionFind(cols) + n = len(cols) + for i in range(n): + row = values[i] + for j in range(i + 1, n): + r = row[j] + if np.isfinite(r) and abs(r) >= threshold: + uf.union(cols[i], cols[j]) + groups = defaultdict(list) + for col in cols: + groups[uf.find(col)].append(col) + for root in groups: + groups[root].sort() + return dict(groups), corr + + +# --------------------------------------------------------------------------- +# Main audit +# --------------------------------------------------------------------------- +def run_audit(args): + registry = load_registry(args.registry) + df, total_rows, sampled = load_features(args.features, args.max_rows, args.seed) + n_used = len(df) + + if "label" not in df.columns: + raise SystemExit("features table has no 'label' column") + + # Reference vectors for Level 1 correlations. + label = df["label"].astype(str) + protein = df["protein"].astype(str) if "protein" in df.columns else pd.Series([""] * n_used) + prelim = df["prelim_score"].to_numpy(dtype=float) if "prelim_score" in df.columns else np.full(n_used, np.nan) + charge_ref = df["charge"].to_numpy(dtype=float) if "charge" in df.columns else np.full(n_used, np.nan) + pep_len = np.array([peptide_length(p) for p in df["peptidoform"]], dtype=float) if "peptidoform" in df.columns else np.full(n_used, np.nan) + has_intensity_col = "log_apex_intensity" in df.columns + log_int = df["log_apex_intensity"].to_numpy(dtype=float) if has_intensity_col else np.full(n_used, np.nan) + + real_subs = [s.strip() for s in args.real_substr.split(",") if s.strip()] + entrap_subs = [s.strip() for s in args.entrapment_substr.split(",") if s.strip()] + + is_target = (label == "target").to_numpy() + is_decoy = (label == "decoy").to_numpy() + real_mask_prot = substr_match(protein, real_subs) + entrap_mask_prot = substr_match(protein, entrap_subs) + + mask_target = is_target & real_mask_prot + mask_decoy = is_decoy + mask_entrap = is_target & entrap_mask_prot + + n_target = int(mask_target.sum()) + n_decoy = int(mask_decoy.sum()) + n_entrap = int(mask_entrap.sum()) + + # Feature columns: numeric, not meta. + meta_set = set(META_COLUMNS) + feature_cols = [] + for col in df.columns: + if col in meta_set: + continue + if pd.api.types.is_numeric_dtype(df[col]): + feature_cols.append(col) + feature_cols.sort() + + rows = [] + unknown_features = [] + for col in feature_cols: + arr = df[col].to_numpy(dtype=float) + finite = np.isfinite(arr) + n_nan = int(np.isnan(arr).sum()) + n_inf = int(np.isinf(arr).sum()) + finite_vals = arr[finite] + uniq = int(np.unique(finite_vals).size) if finite_vals.size else 0 + constant = uniq <= 1 + + if finite_vals.size: + q05, q25, q50, q75, q95 = ( + float(np.percentile(finite_vals, p)) for p in (5, 25, 50, 75, 95) + ) + else: + q05 = q25 = q50 = q75 = q95 = np.nan + + def grp_stats(mask): + vals = arr[mask & finite] + if vals.size == 0: + return np.nan, np.nan + return float(vals.mean()), float(np.median(vals)) + + mean_t, med_t = grp_stats(mask_target) + mean_d, med_d = grp_stats(mask_decoy) + mean_e, med_e = grp_stats(mask_entrap) + + corr_prelim = safe_spearman(arr, prelim) + corr_len = safe_spearman(arr, pep_len) + corr_charge = safe_spearman(arr, charge_ref) + corr_int = safe_spearman(arr, log_int) if has_intensity_col else np.nan + + # Missingness by label group for the asymmetry flag. + n_t_rows = int(is_target.sum()) + n_d_rows = int(is_decoy.sum()) + miss_rate_t = float(np.isnan(arr[is_target]).mean()) if n_t_rows else np.nan + miss_rate_d = float(np.isnan(arr[is_decoy]).mean()) if n_d_rows else np.nan + if np.isfinite(miss_rate_t) and np.isfinite(miss_rate_d): + hi = max(miss_rate_t, miss_rate_d) + lo = min(miss_rate_t, miss_rate_d) + asym_ratio = (hi + 1e-12) / (lo + 1e-12) + else: + hi = np.nan + asym_ratio = np.nan + + auc_td = auc_mwu(arr[mask_target], arr[mask_decoy]) + auc_te = auc_mwu(arr[mask_target], arr[mask_entrap]) + sep_td = separation(auc_td) + sep_te = separation(auc_te) + + # Flags. + flag_constant = constant + flag_asym = bool( + np.isfinite(asym_ratio) + and hi > ASYM_MIN_RATE + and asym_ratio > ASYM_RATIO + ) + flag_intensity = bool( + has_intensity_col + and col != "log_apex_intensity" + and np.isfinite(corr_int) + and abs(corr_int) >= INTENSITY_CORR + ) + flag_leakage = bool( + np.isfinite(sep_td) + and np.isfinite(sep_te) + and sep_td >= SEP_STRONG + and sep_te < LEAKAGE_RATIO * sep_td + ) + + reg = registry.get(col) + in_registry = reg is not None + if not in_registry: + unknown_features.append(col) + family = reg["family"] if in_registry else "UNKNOWN" + level = reg["level"] if in_registry else "?" + direction = reg["direction"] if in_registry else "?" + + rows.append( + { + "feature": col, + "family": family, + "level": level, + "direction": direction, + "in_registry": in_registry, + "n_rows_used": n_used, + "missing_pct": 100.0 * n_nan / n_used if n_used else np.nan, + "inf_pct": 100.0 * n_inf / n_used if n_used else np.nan, + "unique_count": uniq, + "constant": constant, + "q05": q05, + "q25": q25, + "q50": q50, + "q75": q75, + "q95": q95, + "n_target": n_target, + "mean_target": mean_t, + "median_target": med_t, + "n_decoy": n_decoy, + "mean_decoy": mean_d, + "median_decoy": med_d, + "n_entrap": n_entrap, + "mean_entrap": mean_e, + "median_entrap": med_e, + "corr_prelim_score": corr_prelim, + "corr_peptide_length": corr_len, + "corr_charge": corr_charge, + "corr_log_apex_intensity": corr_int, + "missing_rate_target": miss_rate_t, + "missing_rate_decoy": miss_rate_d, + "asym_missing_ratio": asym_ratio, + "auc_target_decoy": auc_td, + "sep_target_decoy": sep_td, + "auc_target_entrap": auc_te, + "sep_target_entrap": sep_te, + "sep_gap_decoy_minus_entrap": (sep_td - sep_te) if (np.isfinite(sep_td) and np.isfinite(sep_te)) else np.nan, + "flag_constant": flag_constant, + "flag_asymmetric_missing": flag_asym, + "flag_intensity_dominated": flag_intensity, + "flag_leakage": flag_leakage, + } + ) + + audit = pd.DataFrame(rows) + + # Level 3 redundancy over non-constant features. + nonconst = [r["feature"] for r in rows if not r["constant"]] + clusters, _corr = cluster_features(df, nonconst, args.redundancy_threshold) + + # Representative per cluster: highest target-vs-entrapment separation, + # then highest target-vs-decoy separation, then name. + sep_te_map = {r["feature"]: (r["sep_target_entrap"] if np.isfinite(r["sep_target_entrap"]) else -1.0) for r in rows} + sep_td_map = {r["feature"]: (r["sep_target_decoy"] if np.isfinite(r["sep_target_decoy"]) else -1.0) for r in rows} + + cluster_id_map = {} + representative_map = {} + multi_clusters = [] + cluster_counter = 0 + # Deterministic order: sort clusters by their sorted member list. + for root, members in sorted(clusters.items(), key=lambda kv: kv[1]): + if len(members) < 2: + continue + rep = sorted( + members, + key=lambda f: (-sep_te_map[f], -sep_td_map[f], f), + )[0] + cid = cluster_counter + cluster_counter += 1 + for m in members: + cluster_id_map[m] = cid + representative_map[m] = (m == rep) + multi_clusters.append( + { + "cluster_id": cid, + "size": len(members), + "representative": rep, + "representative_sep_target_entrap": (None if sep_te_map[rep] < 0 else round(sep_te_map[rep], 6)), + "members": members, + } + ) + + n_singletons = sum(1 for m in clusters.values() if len(m) == 1) + + audit["cluster_id"] = audit["feature"].map(lambda f: cluster_id_map.get(f, -1)) + audit["is_cluster_representative"] = audit["feature"].map(lambda f: representative_map.get(f, False)) + + return { + "audit": audit, + "rows": rows, + "clusters": multi_clusters, + "n_singletons": n_singletons, + "unknown_features": unknown_features, + "registry": registry, + "feature_cols": feature_cols, + "total_rows": total_rows, + "n_used": n_used, + "sampled": sampled, + "n_target": n_target, + "n_decoy": n_decoy, + "n_entrap": n_entrap, + "real_subs": real_subs, + "entrap_subs": entrap_subs, + "has_intensity_col": has_intensity_col, + } + + +# --------------------------------------------------------------------------- +# Output writers +# --------------------------------------------------------------------------- +def write_outputs(result, args, out_dir): + os.makedirs(out_dir, exist_ok=True) + audit = result["audit"] + rows = result["rows"] + registry = result["registry"] + + csv_path = os.path.join(out_dir, "feature_audit.csv") + audit.to_csv(csv_path, index=False, float_format="%.6g") + + clusters_path = os.path.join(out_dir, "redundancy_clusters.json") + with open(clusters_path, "w", encoding="utf-8") as handle: + json.dump( + { + "spearman_abs_threshold": args.redundancy_threshold, + "n_features_considered": int((~audit["constant"]).sum()), + "n_clusters": len(result["clusters"]), + "n_singletons": result["n_singletons"], + "clusters": result["clusters"], + }, + handle, + indent=2, + ) + + # Warnings file. + by_feature = {r["feature"]: r for r in rows} + constant_feats = [r["feature"] for r in rows if r["flag_constant"]] + asym_feats = [r["feature"] for r in rows if r["flag_asymmetric_missing"]] + intensity_feats = [r["feature"] for r in rows if r["flag_intensity_dominated"]] + leakage_feats = [r["feature"] for r in rows if r["flag_leakage"]] + unknown = result["unknown_features"] + + # Registry features absent from the table (active rows only). + table_names = set(result["feature_cols"]) + missing_from_table = [ + name + for name, meta in registry.items() + if not meta["dropped_collision"] and name not in table_names and name not in set(META_COLUMNS) + ] + + warnings_path = os.path.join(out_dir, "warnings.txt") + with open(warnings_path, "w", encoding="utf-8") as handle: + handle.write("MuMDIA feature audit warnings\n") + handle.write("=" * 60 + "\n\n") + + handle.write("CONSTANT features (%d)\n" % len(constant_feats)) + for f in constant_feats: + handle.write(" %s\n" % f) + handle.write("\n") + + handle.write("ASYMMETRIC MISSINGNESS target vs decoy > %gx (%d)\n" % (ASYM_RATIO, len(asym_feats))) + for f in asym_feats: + r = by_feature[f] + handle.write( + " %s target=%.4f decoy=%.4f ratio=%.1f\n" + % (f, r["missing_rate_target"], r["missing_rate_decoy"], r["asym_missing_ratio"]) + ) + handle.write("\n") + + handle.write("INTENSITY-DOMINATED |corr(log_apex_intensity)| >= %g (%d)\n" % (INTENSITY_CORR, len(intensity_feats))) + for f in intensity_feats: + handle.write(" %s corr=%.3f\n" % (f, by_feature[f]["corr_log_apex_intensity"])) + handle.write("\n") + + handle.write( + "LEAKAGE RISK sep(target,decoy) >= %g and sep(target,entrapment) < %g x sep(target,decoy) (%d)\n" + % (SEP_STRONG, LEAKAGE_RATIO, len(leakage_feats)) + ) + for f in leakage_feats: + r = by_feature[f] + handle.write( + " %s sep_td=%.3f sep_te=%.3f gap=%.3f\n" + % (f, r["sep_target_decoy"], r["sep_target_entrap"], r["sep_gap_decoy_minus_entrap"]) + ) + handle.write("\n") + + handle.write("UNKNOWN features not in registry (%d)\n" % len(unknown)) + for f in unknown: + handle.write(" %s\n" % f) + handle.write("\n") + + handle.write("REGISTRY features absent from the table (%d)\n" % len(missing_from_table)) + for f in missing_from_table: + handle.write(" %s\n" % f) + handle.write("\n") + + # Summary file. + summary_path = os.path.join(out_dir, "summary.txt") + n_feat = len(result["feature_cols"]) + with open(summary_path, "w", encoding="utf-8") as handle: + handle.write("MuMDIA feature audit summary\n") + handle.write("=" * 60 + "\n\n") + handle.write("Input features : %s\n" % os.path.abspath(args.features)) + handle.write("Registry : %s\n" % os.path.abspath(args.registry)) + handle.write("Output dir : %s\n" % os.path.abspath(out_dir)) + handle.write("\n") + if result["sampled"]: + handle.write( + "Sampling : %d of %d rows (seed=%d, deterministic)\n" + % (result["n_used"], result["total_rows"], args.seed) + ) + else: + handle.write("Sampling : none, all %d rows used\n" % result["total_rows"]) + handle.write("\n") + handle.write("Groups\n") + handle.write(" real targets (%s) : %d\n" % (",".join(result["real_subs"]), result["n_target"])) + handle.write(" decoys : %d\n" % result["n_decoy"]) + handle.write(" entrapments (%s) : %d\n" % (",".join(result["entrap_subs"]), result["n_entrap"])) + handle.write(" note: target-labelled rows matching neither the real nor the entrapment\n") + handle.write(" substring fall outside both the target and entrapment groups.\n") + if not result["has_intensity_col"]: + handle.write(" note: no log_apex_intensity column found; intensity checks skipped.\n") + handle.write(" note: per-run drift not computed (no run column in this input).\n") + handle.write("\n") + handle.write("Features audited : %d\n" % n_feat) + handle.write("Constant : %d\n" % len(constant_feats)) + handle.write("Asymmetric miss. : %d\n" % len(asym_feats)) + handle.write("Intensity-domin. : %d\n" % len(intensity_feats)) + handle.write("Leakage risk : %d\n" % len(leakage_feats)) + handle.write("Unknown (registry): %d\n" % len(unknown)) + handle.write("Redundancy clusters (size>=2): %d\n" % len(result["clusters"])) + handle.write("Redundancy singletons : %d\n" % result["n_singletons"]) + handle.write("\n") + + # Family coverage over audited features. + fam_counts = defaultdict(int) + for r in rows: + fam_counts[r["family"]] += 1 + handle.write("Family coverage (audited features)\n") + for fam in sorted(fam_counts): + handle.write(" %-32s %d\n" % (fam, fam_counts[fam])) + handle.write("\n") + + # Top separations. + def top_by(key, n=10): + valid = [r for r in rows if np.isfinite(r[key])] + return sorted(valid, key=lambda r: (-r[key], r["feature"]))[:n] + + handle.write("Top 10 target-vs-entrapment separation\n") + for r in top_by("sep_target_entrap"): + handle.write( + " %-40s sep_te=%.3f sep_td=%.3f family=%s\n" + % (r["feature"], r["sep_target_entrap"], r["sep_target_decoy"], r["family"]) + ) + handle.write("\n") + handle.write("Top 10 target-vs-decoy separation\n") + for r in top_by("sep_target_decoy"): + handle.write( + " %-40s sep_td=%.3f sep_te=%.3f family=%s\n" + % (r["feature"], r["sep_target_decoy"], r["sep_target_entrap"], r["family"]) + ) + handle.write("\n") + + return { + "csv_path": csv_path, + "clusters_path": clusters_path, + "warnings_path": warnings_path, + "summary_path": summary_path, + "constant_feats": constant_feats, + "leakage_feats": leakage_feats, + } + + +def parse_args(argv=None): + parser = argparse.ArgumentParser( + description="Audit a MuMDIA competed features Parquet against the feature registry." + ) + parser.add_argument("--features", required=True, help="competed features Parquet") + parser.add_argument("--registry", required=True, help="feature_registry.yaml") + parser.add_argument("--out", default=None, help="output directory (default /feature_audit/)") + parser.add_argument("--max-rows", type=int, default=None, help="deterministic row sample cap") + parser.add_argument("--entrapment-substr", default="_HUMAN", help="protein substring(s) for the entrapment null (comma-separated)") + parser.add_argument("--real-substr", default="_ECOLI", help="protein substring(s) for the real target species (comma-separated)") + parser.add_argument("--redundancy-threshold", type=float, default=DEFAULT_REDUNDANCY_THRESHOLD, help="abs Spearman rho for clustering") + parser.add_argument("--seed", type=int, default=DEFAULT_SEED, help="sampling seed") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + if args.out: + out_dir = args.out + else: + out_dir = os.path.join(os.path.dirname(os.path.abspath(args.features)), "feature_audit") + + result = run_audit(args) + paths = write_outputs(result, args, out_dir) + + rows = result["rows"] + top_te = sorted( + [r for r in rows if np.isfinite(r["sep_target_entrap"])], + key=lambda r: (-r["sep_target_entrap"], r["feature"]), + )[:3] + + print("feature_audit written to: %s" % os.path.abspath(out_dir)) + print("features audited : %d" % len(result["feature_cols"])) + print("constant : %d" % len(paths["constant_feats"])) + print("leakage-flagged : %d" % len(paths["leakage_feats"])) + print("redundancy clusters : %d (size>=2)" % len(result["clusters"])) + if result["sampled"]: + print("sampling : %d of %d rows (seed=%d)" % (result["n_used"], result["total_rows"], args.seed)) + print("top-3 target-vs-entrapment separation:") + for r in top_te: + print(" %-40s sep_te=%.3f (sep_td=%.3f)" % (r["feature"], r["sep_target_entrap"], r["sep_target_decoy"])) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/normalize_output.py b/scripts/normalize_output.py new file mode 100644 index 0000000..49a5449 --- /dev/null +++ b/scripts/normalize_output.py @@ -0,0 +1,295 @@ +"""Normalize a MuMDIA scored table to the common benchmark schema. + +Sensitivity-plan backlog item P0.2 (normalized output converter). The script is +non-invasive: it reads existing MuMDIA artifacts (a scored PSM table, and +optionally the psms and peptide-quant tables) and writes a single Parquet in the +common schema of sensitivity_plan spec 02 section 4: + + run_id, precursor_id, stripped_sequence, modified_sequence, charge, + protein_ids, is_decoy, is_entrapment, precursor_mz, apex_rt, score, q_value, + pep, quantity, engine, engine_version + +Two export modes support the spec requirement to keep both "all top-scoring +candidates before FDR" and "final reported candidates": + --all-candidates keep every row (default) + --reported-only keep rows with q_value <= --q + +The output is deterministic: rows are sorted by (run_id, precursor_id), and +precursor keys are reproducible because precursor_id is the stable MuMDIA +candidate_id. + +Interpreter: C:/Users/robbi/anaconda3/envs/py312_mumdia/python.exe +(pyarrow, pandas, numpy, stdlib tomllib). + +Example +------- +python normalize_output.py \ + --scored C:/proteobench/out_ecoli/scored.parquet \ + --psms C:/proteobench/out_ecoli/psms.parquet \ + --out C:/proteobench/accept/ecoli_normalized.parquet \ + --run-id ecoli +""" + +from __future__ import annotations + +import argparse +import re +import sys +import tomllib +from pathlib import Path + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +# Bracketed UniMod-style modification token and the decoy prefix. +_MOD_RE = re.compile(r"\[[^\]]*\]") +_DECOY_RE = re.compile(r"^DECOY_") + +# Preference order for the discriminant score. The final rescore "score" is +# preferred over the pre-rescore "prelim_score" so that the reported score and +# q_value come from the same model; override with --score-col. +_SCORE_PREF = ["score", "prelim_score"] +# Candidate column names for a posterior error probability, if present. +_PEP_PREF = ["pep", "posterior_error_prob", "PEP", "posterior_error_probability"] + +# Common schema, in the fixed spec-02 section-4 order. +_OUT_SCHEMA = pa.schema( + [ + ("run_id", pa.string()), + ("precursor_id", pa.uint32()), + ("stripped_sequence", pa.string()), + ("modified_sequence", pa.string()), + ("charge", pa.int32()), + ("protein_ids", pa.string()), + ("is_decoy", pa.bool_()), + ("is_entrapment", pa.bool_()), + ("precursor_mz", pa.float64()), + ("apex_rt", pa.float64()), + ("score", pa.float64()), + ("q_value", pa.float64()), + ("pep", pa.float64()), + ("quantity", pa.float64()), + ("engine", pa.string()), + ("engine_version", pa.string()), + ] +) + + +def resolve_engine_version(script_path: Path) -> str: + """Resolve the MuMDIA engine version from the Cargo manifests. + + mumdia-core declares ``version.workspace = true``, so the literal version + lives in the workspace root manifest. Falls back to "unknown". + """ + repo_root = script_path.resolve().parents[1] + core = repo_root / "rust" / "mumdia" / "crates" / "mumdia-core" / "Cargo.toml" + workspace = repo_root / "rust" / "mumdia" / "Cargo.toml" + try: + with core.open("rb") as fh: + core_cfg = tomllib.load(fh) + ver = core_cfg.get("package", {}).get("version") + if isinstance(ver, str): + return ver + with workspace.open("rb") as fh: + ws_cfg = tomllib.load(fh) + ver = ws_cfg.get("workspace", {}).get("package", {}).get("version") + if isinstance(ver, str): + return ver + except (OSError, tomllib.TOMLDecodeError): + pass + return "unknown" + + +def pick_column(columns: list[str], prefs: list[str]) -> str | None: + """Return the first preferred column that is present, else None.""" + present = set(columns) + for name in prefs: + if name in present: + return name + return None + + +def normalize( + scored: pd.DataFrame, + psms: pd.DataFrame | None, + pep_quant: pd.DataFrame | None, + run_id: str, + entrapment_substr: str, + score_col: str | None, + engine_version: str, +) -> pd.DataFrame: + """Map MuMDIA columns onto the common normalized schema.""" + n = len(scored) + out = pd.DataFrame(index=scored.index) + + out["run_id"] = run_id + out["precursor_id"] = scored["candidate_id"].astype("uint32") + + pforms = scored["peptidoform"].astype("string") + out["stripped_sequence"] = pforms.str.replace(_MOD_RE, "", regex=True).str.replace( + _DECOY_RE, "", regex=True + ) + out["modified_sequence"] = pforms + out["charge"] = scored["charge"].astype("int32") + out["protein_ids"] = scored["protein"].astype("string") + out["is_decoy"] = (scored["label"].astype("string") == "decoy").astype(bool) + out["is_entrapment"] = ( + scored["protein"].astype("string").str.contains(entrapment_substr, regex=False) + ).fillna(False).astype(bool) + + # precursor_mz and apex_rt live in the psms table (not in scored); take them + # from a join when available, otherwise from scored if it carries them, else + # leave null. + out["precursor_mz"] = _fill_from_join( + scored, psms, "precursor_mz", "candidate_id" + ) + out["apex_rt"] = _fill_from_join(scored, psms, "apex_rt", "candidate_id") + + if score_col is None: + score_col = pick_column(list(scored.columns), _SCORE_PREF) + if score_col is None: + out["score"] = np.nan + else: + if score_col not in scored.columns: + raise ValueError(f"score column {score_col!r} not in scored table") + out["score"] = pd.to_numeric(scored[score_col], errors="coerce") + + out["q_value"] = pd.to_numeric(scored["q_value"], errors="coerce") + + pep_col = pick_column(list(scored.columns), _PEP_PREF) + out["pep"] = ( + pd.to_numeric(scored[pep_col], errors="coerce") + if pep_col + else pd.Series(np.nan, index=scored.index) + ) + + out["quantity"] = _fill_from_join( + scored, pep_quant, "quantity", "candidate_id" + ) + + out["engine"] = "mumdia" + out["engine_version"] = engine_version + + assert len(out) == n + return out + + +def _fill_from_join( + scored: pd.DataFrame, + other: pd.DataFrame | None, + value_col: str, + key: str, +) -> pd.Series: + """Return value_col aligned to scored: from `other` via join, or from scored, + else all-null. The join collapses duplicate keys in `other` to the first.""" + if other is not None and value_col in other.columns and key in other.columns: + lut = other.drop_duplicates(subset=[key]).set_index(key)[value_col] + return pd.to_numeric(scored[key].map(lut), errors="coerce") + if value_col in scored.columns: + return pd.to_numeric(scored[value_col], errors="coerce") + return pd.Series(np.nan, index=scored.index) + + +def to_arrow(df: pd.DataFrame) -> pa.Table: + """Build an Arrow table with the fixed common schema (NaN -> null).""" + arrays = [] + for field in _OUT_SCHEMA: + arrays.append(pa.array(df[field.name], type=field.type, from_pandas=True)) + return pa.Table.from_arrays(arrays, schema=_OUT_SCHEMA) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Normalize a MuMDIA scored table to the common schema (P0.2)." + ) + p.add_argument("--scored", required=True, type=Path, help="MuMDIA scored Parquet.") + p.add_argument( + "--psms", type=Path, help="MuMDIA psms Parquet (for apex_rt, precursor_mz)." + ) + p.add_argument( + "--peptide-quant", + type=Path, + help="MuMDIA peptide-quant Parquet (for quantity).", + ) + p.add_argument("--out", type=Path, help="Output normalized Parquet.") + p.add_argument("--run-id", default="run", help="Run identifier (default 'run').") + p.add_argument( + "--entrapment-substr", + default="_HUMAN", + help="Substring flagging a protein as entrapment (default '_HUMAN').", + ) + p.add_argument( + "--score-col", + default=None, + help="Score column to use (default: first of 'score', 'prelim_score').", + ) + mode = p.add_mutually_exclusive_group() + mode.add_argument( + "--all-candidates", + action="store_true", + help="Keep all candidates (default).", + ) + mode.add_argument( + "--reported-only", + action="store_true", + help="Keep only rows with q_value <= --q.", + ) + p.add_argument( + "--q", type=float, default=0.01, help="q-value cutoff for --reported-only." + ) + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if not args.scored.exists(): + print(f"error: scored table not found: {args.scored}", file=sys.stderr) + return 1 + + scored = pq.read_table(args.scored).to_pandas() + psms = pq.read_table(args.psms).to_pandas() if args.psms else None + pep_quant = ( + pq.read_table(args.peptide_quant).to_pandas() if args.peptide_quant else None + ) + + engine_version = resolve_engine_version(Path(__file__)) + out = normalize( + scored, + psms, + pep_quant, + run_id=args.run_id, + entrapment_substr=args.entrapment_substr, + score_col=args.score_col, + engine_version=engine_version, + ) + + n_total = len(out) + if args.reported_only: + out = out[out["q_value"] <= args.q].copy() + + # Deterministic ordering: precursor_id is the stable candidate_id. + out = out.sort_values(["run_id", "precursor_id"], kind="stable").reset_index( + drop=True + ) + + table = to_arrow(out) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(table, args.out) + print(f"wrote normalized table: {args.out}") + + n_dec = int(out["is_decoy"].sum()) + n_ent = int(out["is_entrapment"].sum()) + mode = "reported-only" if args.reported_only else "all-candidates" + print( + f"rows: {len(out)} (of {n_total} scored; mode={mode}); " + f"targets={len(out) - n_dec}, decoys={n_dec}, entrapment={n_ent}" + ) + print(f"columns: {', '.join(f.name for f in _OUT_SCHEMA)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search_space_manifest.py b/scripts/search_space_manifest.py new file mode 100644 index 0000000..d85ee32 --- /dev/null +++ b/scripts/search_space_manifest.py @@ -0,0 +1,564 @@ +"""Derive an effective search-space manifest from a MuMDIA library. + +Sensitivity-plan backlog item P0.1 (search-space parity). The script is +non-invasive: it reads an existing MuMDIA library-precursors Parquet, derives a +machine-readable manifest in the shape of sensitivity_plan spec 02 section 3, +and optionally compares that manifest to a declared manifest (--compare) or a +DIA-NN report (--diann). With --fail-on-mismatch it exits nonzero when the +effective search spaces differ in a way that would invalidate a benchmark +comparison. + +Only fields that a precursor library actually encodes are derived. Fields that a +library cannot encode (enzyme, missed cleavages, fragment m/z window, FDR +thresholds, fixed-vs-variable mod classification) are written as the literal +"unknown_from_library" with an explanatory note, so a reader never mistakes an +absent value for a real setting. + +The output is deterministic: no wall-clock timestamps, sorted keys where order is +not semantically meaningful, and a SHA-256 content hash of the input library for +provenance. + +Interpreter: C:/Users/robbi/anaconda3/envs/py312_mumdia/python.exe +(pyarrow, pandas, numpy, pyyaml, stdlib tomllib/hashlib). + +Example +------- +python search_space_manifest.py \ + --library-precursors C:/proteobench/lib/lib_precursors_ft.parquet \ + --out C:/proteobench/accept/ecoli_search_space.yaml +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import sys +import tomllib +from pathlib import Path +from typing import Any + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq +import yaml + +UNKNOWN = "unknown_from_library" + +# Bracketed UniMod-style modification token, e.g. "[Carbamidomethyl]". +_MOD_RE = re.compile(r"\[[^\]]*\]") +_MOD_TOKEN_RE = re.compile(r"\[([^\]]*)\]") +_DECOY_RE = re.compile(r"^DECOY_") + + +# --------------------------------------------------------------------------- # +# Small shared helpers +# --------------------------------------------------------------------------- # +def sha256_file(path: Path, chunk: int = 1 << 20) -> str: + """Return the SHA-256 hex digest of a file, read in fixed-size chunks.""" + h = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(chunk), b""): + h.update(block) + return h.hexdigest() + + +def resolve_engine_version(script_path: Path) -> str: + """Resolve the MuMDIA engine version from the Cargo manifests. + + mumdia-core declares ``version.workspace = true``, so the literal version + lives in the workspace root manifest. Falls back to "unknown". + """ + repo_root = script_path.resolve().parents[1] + core = repo_root / "rust" / "mumdia" / "crates" / "mumdia-core" / "Cargo.toml" + workspace = repo_root / "rust" / "mumdia" / "Cargo.toml" + try: + with core.open("rb") as fh: + core_cfg = tomllib.load(fh) + ver = core_cfg.get("package", {}).get("version") + if isinstance(ver, str): + return ver + with workspace.open("rb") as fh: + ws_cfg = tomllib.load(fh) + ver = ws_cfg.get("workspace", {}).get("package", {}).get("version") + if isinstance(ver, str): + return ver + except (OSError, tomllib.TOMLDecodeError): + pass + return "unknown" + + +def _norm_col(name: str) -> str: + """Normalize a column name to lowercase alphanumerics for matching.""" + return re.sub(r"[^a-z0-9]", "", name.lower()) + + +def _find_col(columns: list[str], candidates: list[str]) -> str | None: + """Return the first column whose normalized name matches a candidate.""" + norm = {_norm_col(c): c for c in columns} + for cand in candidates: + if cand in norm: + return norm[cand] + return None + + +# --------------------------------------------------------------------------- # +# Manifest derivation +# --------------------------------------------------------------------------- # +def derive_manifest(lib_path: Path, engine_version: str) -> dict[str, Any]: + """Derive an effective search-space manifest from a library-precursors table.""" + needed = ["peptidoform", "charge", "precursor_mz", "label", "protein"] + schema_names = pq.read_schema(lib_path).names + read_cols = [c for c in needed if c in schema_names] + tbl = pq.read_table(lib_path, columns=read_cols) + n_rows = tbl.num_rows + + # Charge and m/z ranges are the same for targets and decoys, so compute + # them over all precursors for a faithful picture of the extraction space. + charge_arr = tbl.column("charge") + mz_arr = tbl.column("precursor_mz") + charge_mm = pc.min_max(charge_arr) + mz_mm = pc.min_max(mz_arr) + min_charge = charge_mm["min"].as_py() + max_charge = charge_mm["max"].as_py() + min_mz = mz_mm["min"].as_py() + max_mz = mz_mm["max"].as_py() + + # Charge histogram over all precursors, sorted by charge. + vc = pc.value_counts(charge_arr) + charge_hist = { + int(struct["values"].as_py()): int(struct["counts"].as_py()) for struct in vc + } + charge_hist = dict(sorted(charge_hist.items())) + + # Target / decoy split. + label_arr = tbl.column("label") + is_target = pc.equal(label_arr, "target") + n_target = int(pc.sum(pc.cast(is_target, pa.int64())).as_py()) + n_decoy = n_rows - n_target + + # String-level statistics are computed over the target subset (the real + # search space); decoys are artificial reverse/scramble sequences. + targets = tbl.filter(is_target).select( + [c for c in ["peptidoform", "protein"] if c in read_cols] + ) + tdf = targets.to_pandas() + pforms = tdf["peptidoform"].astype("string") + + stripped = pforms.str.replace(_MOD_RE, "", regex=True).str.replace( + _DECOY_RE, "", regex=True + ) + lengths = stripped.str.len().to_numpy(dtype="int64") + n_distinct_stripped = int(stripped.nunique()) + + has_mod = pforms.str.contains("[", regex=False) + n_with_mod = int(has_mod.sum()) + observed_mods: set[str] = set() + for pf in pforms[has_mod]: + observed_mods.update(_MOD_TOKEN_RE.findall(pf)) + observed_mods_sorted = sorted(observed_mods) + + # Distinct proteins over targets, splitting group strings on ';'. + prot_series = tdf["protein"].astype("string") if "protein" in tdf else None + if prot_series is not None: + prot_set: set[str] = set() + for grp in prot_series.dropna().unique(): + for pid in str(grp).split(";"): + pid = pid.strip() + if pid: + prot_set.add(pid) + n_distinct_proteins: int | str = len(prot_set) + else: + n_distinct_proteins = UNKNOWN + + length_note = ( + "peptide lengths derived from stripped target sequences; enzyme, " + "specificity, and missed cleavages are search-time settings not stored " + "in a precursor library" + ) + + manifest: dict[str, Any] = { + "schema_version": 1, + "provenance": { + "source_library": str(lib_path), + "source_sha256": sha256_file(lib_path), + "source_bytes": lib_path.stat().st_size, + "n_rows": n_rows, + "engine": "mumdia", + "engine_version": engine_version, + "generator": "search_space_manifest.py", + "manifest_schema_version": 1, + }, + "digestion": { + "enzyme": UNKNOWN, + "specificity": UNKNOWN, + "missed_cleavages": UNKNOWN, + "min_length": int(lengths.min()), + "max_length": int(lengths.max()), + "note": length_note, + }, + "precursors": { + "min_charge": int(min_charge), + "max_charge": int(max_charge), + "min_mz": round(float(min_mz), 4), + "max_mz": round(float(max_mz), 4), + }, + "fragments": { + "ion_series": UNKNOWN, + "min_mz": UNKNOWN, + "max_mz": UNKNOWN, + "note": "fragment ion series and m/z window require the fragment " + "table (lib_fragments), not provided here", + }, + "modifications": { + "fixed": UNKNOWN, + "variable": UNKNOWN, + "max_variable_modifications": UNKNOWN, + "observed_mod_tokens": observed_mods_sorted, + "n_peptidoforms_with_mod": n_with_mod, + "note": "fixed vs variable classification cannot be inferred from a " + "library; observed bracketed tokens are listed as-is", + }, + "fdr": { + "precursor": UNKNOWN, + "peptide": UNKNOWN, + "protein": UNKNOWN, + "note": "FDR thresholds are a search-time setting, not stored in the " + "library", + }, + "derived": { + "n_precursors": n_rows, + "n_target_precursors": n_target, + "n_decoy_precursors": n_decoy, + "n_distinct_stripped_sequences": n_distinct_stripped, + "n_distinct_proteins": n_distinct_proteins, + "charge_histogram": charge_hist, + "peptide_length": { + "min": int(lengths.min()), + "max": int(lengths.max()), + "median": float(np.median(lengths)), + }, + }, + } + return manifest + + +# --------------------------------------------------------------------------- # +# MuMDIA target precursor keys (for --diann overlap) +# --------------------------------------------------------------------------- # +def mumdia_target_keys(lib_path: Path) -> tuple[set[tuple[str, int]], dict[str, Any]]: + """Return {(stripped_seq_upper, charge)} for target precursors plus ranges.""" + tbl = pq.read_table( + lib_path, columns=["peptidoform", "charge", "precursor_mz", "label"] + ) + is_target = pc.equal(tbl.column("label"), "target") + tdf = tbl.filter(is_target).to_pandas() + stripped = ( + tdf["peptidoform"] + .astype("string") + .str.replace(_MOD_RE, "", regex=True) + .str.replace(_DECOY_RE, "", regex=True) + .str.upper() + ) + charges = tdf["charge"].astype("int64") + keys = set(zip(stripped.tolist(), charges.tolist())) + ranges = { + "charge_range": [int(charges.min()), int(charges.max())], + "mz_range": [ + round(float(tdf["precursor_mz"].min()), 4), + round(float(tdf["precursor_mz"].max()), 4), + ], + } + return keys, ranges + + +# --------------------------------------------------------------------------- # +# Reference (DIA-NN) loading +# --------------------------------------------------------------------------- # +def load_reference_keys(path: Path) -> tuple[set[tuple[str, int]], dict[str, Any]]: + """Load (stripped_seq_upper, charge) keys and ranges from a DIA-NN report. + + Accepts Parquet or a tab-separated report. Column names are matched + case-insensitively against the usual DIA-NN header set. + """ + suffix = path.suffix.lower() + if suffix in {".parquet", ".pq"}: + cols = pq.read_schema(path).names + else: + import pandas as pd + + cols = list(pd.read_csv(path, sep="\t", nrows=0).columns) + + stripped_col = _find_col( + cols, ["strippedsequence", "peptide", "sequence", "pepseq"] + ) + charge_col = _find_col(cols, ["precursorcharge", "charge", "z"]) + mz_col = _find_col(cols, ["precursormz", "mz"]) + if stripped_col is None or charge_col is None: + raise ValueError( + f"could not locate stripped-sequence and charge columns in {path}; " + f"available columns: {cols}" + ) + + read_cols = [stripped_col, charge_col] + ([mz_col] if mz_col else []) + if suffix in {".parquet", ".pq"}: + df = pq.read_table(path, columns=read_cols).to_pandas() + else: + import pandas as pd + + df = pd.read_csv(path, sep="\t", usecols=read_cols) + + df = df.dropna(subset=[stripped_col, charge_col]) + stripped = df[stripped_col].astype("string").str.upper() + charges = df[charge_col].astype("int64") + keys = set(zip(stripped.tolist(), charges.tolist())) + ranges: dict[str, Any] = { + "charge_range": [int(charges.min()), int(charges.max())], + } + if mz_col: + ranges["mz_range"] = [ + round(float(df[mz_col].min()), 4), + round(float(df[mz_col].max()), 4), + ] + return keys, ranges + + +# --------------------------------------------------------------------------- # +# Comparison logic +# --------------------------------------------------------------------------- # +def compare_overlap( + mumdia_keys: set[tuple[str, int]], + mumdia_ranges: dict[str, Any], + ref_keys: set[tuple[str, int]], + ref_ranges: dict[str, Any], + ref_label: str, + mz_tol: float, +) -> dict[str, Any]: + """Compute (stripped_seq, charge) precursor-key overlap and range mismatches.""" + shared = mumdia_keys & ref_keys + only_mum = mumdia_keys - ref_keys + only_ref = ref_keys - mumdia_keys + n_ref = len(ref_keys) + n_mum = len(mumdia_keys) + + charge_mismatch = mumdia_ranges["charge_range"] != ref_ranges.get("charge_range") + mz_mismatch = False + if "mz_range" in ref_ranges and "mz_range" in mumdia_ranges: + m = mumdia_ranges["mz_range"] + r = ref_ranges["mz_range"] + mz_mismatch = abs(m[0] - r[0]) > mz_tol or abs(m[1] - r[1]) > mz_tol + + return { + "reference": ref_label, + "n_mumdia_target_keys": n_mum, + "n_reference_keys": n_ref, + "n_shared": len(shared), + "n_only_in_mumdia": len(only_mum), + "n_only_in_reference": len(only_ref), + "shared_fraction_of_reference": (len(shared) / n_ref) if n_ref else 0.0, + "shared_fraction_of_mumdia": (len(shared) / n_mum) if n_mum else 0.0, + "charge_range_mumdia": mumdia_ranges["charge_range"], + "charge_range_reference": ref_ranges.get("charge_range"), + "mz_range_mumdia": mumdia_ranges.get("mz_range"), + "mz_range_reference": ref_ranges.get("mz_range"), + "charge_range_mismatch": charge_mismatch, + "mz_range_mismatch": mz_mismatch, + } + + +def compare_declared( + derived: dict[str, Any], other: dict[str, Any], mz_tol: float +) -> dict[str, Any]: + """Compare a derived manifest against a declared manifest, field by field.""" + diffs: list[dict[str, Any]] = [] + + def cmp(section: str, field: str, material_if_diff: bool, tol: float = 0.0) -> None: + a = derived.get(section, {}).get(field) + b = other.get(section, {}).get(field) + if a is None or b is None: + return + if a == UNKNOWN or b == UNKNOWN: + diffs.append( + { + "field": f"{section}.{field}", + "derived": a, + "declared": b, + "status": "not_comparable", + "material": False, + } + ) + return + if isinstance(a, (int, float)) and isinstance(b, (int, float)) and tol > 0.0: + differs = abs(float(a) - float(b)) > tol + else: + differs = a != b + diffs.append( + { + "field": f"{section}.{field}", + "derived": a, + "declared": b, + "status": "mismatch" if differs else "match", + "material": bool(differs and material_if_diff), + } + ) + + cmp("digestion", "enzyme", True) + cmp("digestion", "specificity", True) + cmp("digestion", "missed_cleavages", True) + cmp("digestion", "min_length", True) + cmp("digestion", "max_length", True) + cmp("precursors", "min_charge", True) + cmp("precursors", "max_charge", True) + cmp("precursors", "min_mz", True, tol=mz_tol) + cmp("precursors", "max_mz", True, tol=mz_tol) + cmp("modifications", "max_variable_modifications", True) + cmp("fdr", "precursor", True) + cmp("fdr", "peptide", True) + cmp("fdr", "protein", True) + + return { + "reference": "declared_manifest", + "fields": diffs, + "n_material_mismatches": sum(1 for d in diffs if d["material"]), + } + + +# --------------------------------------------------------------------------- # +# Reporting +# --------------------------------------------------------------------------- # +def print_overlap(cmp: dict[str, Any], min_shared_frac: float) -> bool: + """Print an overlap diff. Return True if it constitutes a material mismatch.""" + print(f"\n=== search-space overlap vs {cmp['reference']} ===") + print(f" MuMDIA target keys : {cmp['n_mumdia_target_keys']}") + print(f" reference keys : {cmp['n_reference_keys']}") + print(f" shared : {cmp['n_shared']}") + print(f" only in MuMDIA : {cmp['n_only_in_mumdia']}") + print(f" only in reference : {cmp['n_only_in_reference']}") + print( + f" shared / reference : {cmp['shared_fraction_of_reference']:.4f} " + f"(threshold {min_shared_frac:.4f})" + ) + print(f" shared / MuMDIA : {cmp['shared_fraction_of_mumdia']:.4f}") + print( + f" charge range : MuMDIA {cmp['charge_range_mumdia']} vs " + f"reference {cmp['charge_range_reference']}" + + (" MISMATCH" if cmp["charge_range_mismatch"] else "") + ) + print( + f" m/z range : MuMDIA {cmp['mz_range_mumdia']} vs " + f"reference {cmp['mz_range_reference']}" + + (" MISMATCH" if cmp["mz_range_mismatch"] else "") + ) + low_overlap = cmp["shared_fraction_of_reference"] < min_shared_frac + if low_overlap: + print(" -> shared fraction below threshold") + return low_overlap or cmp["charge_range_mismatch"] or cmp["mz_range_mismatch"] + + +def print_declared(cmp: dict[str, Any]) -> bool: + """Print a declared-manifest diff. Return True if any material mismatch.""" + print("\n=== declared-manifest comparison ===") + for d in cmp["fields"]: + flag = { + "match": "ok", + "mismatch": "MISMATCH", + "not_comparable": "n/a", + }[d["status"]] + material = " (material)" if d["material"] else "" + print( + f" {d['field']:<38} derived={d['derived']!r:<24} " + f"declared={d['declared']!r:<24} {flag}{material}" + ) + print(f" material mismatches: {cmp['n_material_mismatches']}") + return cmp["n_material_mismatches"] > 0 + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Derive and validate a MuMDIA search-space manifest (P0.1)." + ) + p.add_argument( + "--library-precursors", + required=True, + type=Path, + help="MuMDIA library-precursors Parquet.", + ) + p.add_argument("--out", type=Path, help="Output manifest YAML (default: stdout).") + p.add_argument( + "--compare", type=Path, help="Declared manifest YAML to compare against." + ) + p.add_argument( + "--diann", type=Path, help="DIA-NN report (Parquet or TSV) to compare against." + ) + p.add_argument( + "--fail-on-mismatch", + action="store_true", + help="Exit nonzero on a material search-space mismatch.", + ) + p.add_argument( + "--min-shared-frac", + type=float, + default=0.9, + help="Minimum shared fraction of reference precursors (default 0.9).", + ) + p.add_argument( + "--mz-range-tol", + type=float, + default=5.0, + help="Tolerance (Th) for treating m/z-range endpoints as equal (default 5.0).", + ) + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if not args.library_precursors.exists(): + print(f"error: library not found: {args.library_precursors}", file=sys.stderr) + return 1 + + engine_version = resolve_engine_version(Path(__file__)) + manifest = derive_manifest(args.library_precursors, engine_version) + + yaml_text = yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(yaml_text, encoding="utf-8") + print(f"wrote manifest: {args.out}") + else: + print(yaml_text) + + material_mismatch = False + + if args.compare or args.diann: + mum_keys, mum_ranges = mumdia_target_keys(args.library_precursors) + + if args.diann: + ref_keys, ref_ranges = load_reference_keys(args.diann) + ocmp = compare_overlap( + mum_keys, + mum_ranges, + ref_keys, + ref_ranges, + str(args.diann), + args.mz_range_tol, + ) + material_mismatch |= print_overlap(ocmp, args.min_shared_frac) + + if args.compare: + with args.compare.open("rb") as fh: + other = yaml.safe_load(fh) + dcmp = compare_declared(manifest, other, args.mz_range_tol) + material_mismatch |= print_declared(dcmp) + + if material_mismatch and args.fail_on_mismatch: + print("\nFAIL: material search-space mismatch detected.", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4ebf765bfd8a88155d0fd3164cd5177f7f902a48 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:46:33 +0200 Subject: [PATCH 20/40] feat(sensitivity): evidence-count apex selection option (extract.apex_evidence_rank) Wide-window DIA fragment intensity is chimeric, so the tallest scan is often a co-isolated interferent. When extract.apex_evidence_rank = true, the apex is the scan with the most DISTINCT co-eluting predicted fragments (breadth of evidence), with observed signature-ion intensity used only as a sub-integer tiebreak; the RT prior still applies. Default false keeps the legacy signature-intensity apex bit-for-bit. The rolling distinct-fragment count still gates qualifying scans in both modes. Directly expresses the "evidence, not intensity, picks the peak" principle; validate ID effect with the entrapment gate before enabling. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 10 ++++++++++ .../crates/mumdia/src/stages/extract.rs | 20 ++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 5fd56ff..06c989b 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -531,6 +531,15 @@ pub struct ExtractConfig { /// and writes `.audit.parquet` (spec 01 §4 / P0.3). Near-zero cost /// when false (no per-candidate audit allocation). Default false (production). pub emit_candidate_audit: bool, + /// Evidence-count apex selection: choose the apex scan by the NUMBER of distinct + /// co-eluting predicted fragments present (breadth of evidence), using observed + /// signature-ion intensity only as a sub-integer tiebreak. In wide-window DIA a + /// single fragment m/z channel is chimeric, so the tallest scan is often a + /// co-isolated interferent; the scan where the most of the peptide's own + /// predicted transitions co-elute is a more reliable apex. `false` (default) + /// keeps the legacy signature-intensity apex. The rolling distinct-fragment + /// count (`apex_count_window`) still gates which scans qualify in both modes. + pub apex_evidence_rank: bool, } impl Default for ExtractConfig { fn default() -> Self { @@ -565,6 +574,7 @@ impl Default for ExtractConfig { ms1_rescue: false, // opt-in; relaxes acceptance, validate FDR first retain_top_peaks: 1, // legacy single-apex behaviour (K=1) emit_candidate_audit: false, // diagnostic; off in production + apex_evidence_rank: false, // legacy signature-intensity apex } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index b72f793..62de8a9 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -720,10 +720,24 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { continue; } let sig_sum: f32 = sig.iter().map(|&o| map.get(&o).copied().unwrap_or(0.0)).sum(); - let score = if use_prior { - sig_sum * (-0.5 * ((*rt - rt_cal_c) / rt_prior_sigma).powi(2)).exp() as f32 + let prior = if use_prior { + (-0.5 * ((*rt - rt_cal_c) / rt_prior_sigma).powi(2)).exp() as f32 } else { - sig_sum + 1.0 + }; + let score = if p.cfg.apex_evidence_rank { + // Breadth-of-evidence apex: the count of distinct co-eluting + // predicted fragments at this scan dominates; observed signature + // intensity only breaks ties within [0,1). Interference-resistant + // in wide-window DIA (a chimeric-intensity spike cannot outvote a + // scan where more of the peptide's own transitions co-elute). + let n_frag = map.len() as f32; + let tie = sig_sum / (sig_sum + 1.0); + (n_frag + tie) * prior + } else { + // Legacy: signature-ion observed intensity (x RT prior). Bit-identical + // to the previous behaviour (prior = 1.0 when the RT prior is off). + sig_sum * prior }; if score > best_sig { best_sig = score; From a9e3df67a0b369d168716c710180d74507db5f66 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:49:51 +0200 Subject: [PATCH 21/40] feat(sensitivity): adaptive per-region RT extraction window (P3.2/P3.3) rt_im_train.adaptive_rt_window (default false) replaces the single global residual-percentile RT half-width with a LOCAL one: calibration anchors are binned by calibrated RT (adaptive_rt_bins, default 12) and each candidate gets its RT region's residual percentile, clamped to [rt_window_min_s, fallback_rt_window_s] and scaled by rt_window_multiplier. Well-calibrated regions get a tighter window (less interference), poorly-calibrated regions a wider one (more recall); empty/sparse bins fall back to the global width. Off by default (single global window unchanged). Validate ID effect with the entrapment gate. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 17 +++++++ .../crates/mumdia/src/stages/rt_im_train.rs | 51 ++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 06c989b..5652ad5 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -414,6 +414,20 @@ pub struct RtImTrainConfig { /// main use is library-input mode, where the base iRT comes from the imported /// library rather than a DeepLC prediction. pub finetune_deeplc: bool, + /// Adaptive RT window (sensitivity_plan spec 03 §3.5, backlog P3.2/P3.3): + /// instead of one global residual-percentile half-width for every candidate, + /// bin the calibration anchors by calibrated RT and give each candidate the + /// LOCAL residual percentile of its RT region, clamped to + /// `[rt_window_min_s, fallback_rt_window_s]` and scaled by + /// `rt_window_multiplier`. A fixed window is simultaneously too wide for + /// well-calibrated regions and too narrow for poorly-calibrated ones; this + /// tightens clean regions (less interference) and widens noisy ones (more + /// recall). Empty/sparse bins fall back to the global width. Default false. + pub adaptive_rt_window: bool, + /// Number of equal-width calibrated-RT bins for the adaptive window. + pub adaptive_rt_bins: usize, + /// Lower clamp (seconds) for any RT half-window (the existing 1 s floor). + pub rt_window_min_s: f64, } impl Default for RtImTrainConfig { fn default() -> Self { @@ -427,6 +441,9 @@ impl Default for RtImTrainConfig { loess_span: 0.3, fallback_rt_window_s: 120.0, finetune_deeplc: false, + adaptive_rt_window: false, + adaptive_rt_bins: 12, + rt_window_min_s: 1.0, } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs b/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs index 5facf8c..86be5f6 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rt_im_train.rs @@ -113,6 +113,45 @@ pub fn run(p: RtImTrainParams) -> Result { (p.cfg.fallback_rt_window_s, "fallback_fixed".to_string()) }; + // Optional adaptive window: local residual-percentile half-width per + // calibrated-RT bin, so well-calibrated regions get a tight window (less + // interference) and poorly-calibrated regions a wider one (more recall). + // `None` keeps the single global `w_rt`. Empty bins fall back to `w_rt`. + let adaptive: Option<(f64, f64, Vec)> = + if p.cfg.adaptive_rt_window && n_train >= min_anchors { + let cals: Vec = train_irt.iter().map(|x| predict(*x)).collect(); + let resid: Vec = cals.iter().zip(&train_rt).map(|(c, y)| (y - c).abs()).collect(); + let rt_min = cals.iter().cloned().fold(f64::INFINITY, f64::min); + let rt_max = cals.iter().cloned().fold(f64::NEG_INFINITY, f64::max); + let nb = p.cfg.adaptive_rt_bins.max(1); + if rt_max > rt_min { + let span = rt_max - rt_min; + let mut per_bin: Vec> = vec![Vec::new(); nb]; + for (c, r) in cals.iter().zip(&resid) { + let frac = ((c - rt_min) / span).clamp(0.0, 0.999_999); + per_bin[(frac * nb as f64) as usize].push(*r); + } + let lo_clamp = p.cfg.rt_window_min_s.max(0.0); + let hi_clamp = p.cfg.fallback_rt_window_s.max(lo_clamp); + let widths: Vec = per_bin + .iter() + .map(|rs| { + if rs.is_empty() { + w_rt + } else { + (percentile(rs, p.cfg.p_rt) * p.cfg.rt_window_multiplier) + .clamp(lo_clamp, hi_clamp) + } + }) + .collect(); + Some((rt_min, span, widths)) + } else { + None + } + } else { + None + }; + // Apply to every library candidate. let cid = lib_cid; let irt = lib_irt; @@ -123,10 +162,18 @@ pub fn run(p: RtImTrainParams) -> Result { (Vec::with_capacity(n), Vec::with_capacity(n), Vec::with_capacity(n)); for i in 0..n { let cal = predict(irt[i] as f64); + let width = match &adaptive { + Some((rt_min, span, widths)) => { + let nb = widths.len(); + let frac = ((cal - rt_min) / span).clamp(0.0, 0.999_999); + widths[(frac * nb as f64) as usize] + } + None => w_rt, + }; cid_c.push(cid[i]); cal_c.push(cal); - lo_c.push(cal - w_rt); - hi_c.push(cal + w_rt); + lo_c.push(cal - width); + hi_c.push(cal + width); im_c.push(None); imlo_c.push(None); imhi_c.push(None); From 6c5b479e0d89f45c4358377e4298bc884a555726 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:51:44 +0200 Subject: [PATCH 22/40] docs(sensitivity): register +23 new features (383 total) + session-2 backlog status feature_registry.yaml gains the apex_dispersion (13) and mass_uncertainty (10) families (383 features total). FEATURE_REGISTRY.md notes them; IMPLEMENTATION_STATUS.md records the full session-2 done/TODO backlog status and the acceptance-validation plan (entrapment gate on >=2 datasets before enabling any default-off knob). Co-Authored-By: Claude Opus 4.8 --- feature_registry.yaml | 3724 +++++++++++---------- sensitivity_plan/FEATURE_REGISTRY.md | 18 + sensitivity_plan/IMPLEMENTATION_STATUS.md | 35 + 3 files changed, 1968 insertions(+), 1809 deletions(-) diff --git a/feature_registry.yaml b/feature_registry.yaml index 32923c8..0e7ecf8 100644 --- a/feature_registry.yaml +++ b/feature_registry.yaml @@ -1,1816 +1,1922 @@ -# MuMDIA feature registry (machine-readable) -# Auto-generated from _feat_inventory.json (360 features, 17 families). -# Schema per feature: family, level, direction, source_file. -# direction: higher_better | lower_better | neutral | ? -# level: fragment | peak | precursor | candidate | run | ? -# NOTE: 4 names collide with reserved Minimal/Rich names and are dropped -# from the scored schema; the dropped variant is keyed '@' -# and carries 'dropped_collision: true'. The active (reserved) row keeps -# the plain name. total_features: 360 n_families: 17 features: - "rt_error_abs": - family: "minimal" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "rt_error_rel": - family: "minimal" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_matched_fragments": - family: "minimal" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "coelution_run": - family: "minimal" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "log_apex_intensity": - family: "minimal" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "frag_corr": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "frag_cosine": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "spectral_angle": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "coelution_mean": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "coelution_best": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_coelution_above": - family: "minimal" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "charge": - family: "minimal" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "peptide_length": - family: "minimal" - level: "candidate" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_proteins": - family: "minimal" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "library_norm_manhattan": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "library_rmsd": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "xcorr_coelution": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "xcorr_shape": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "sum_b_intensity": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "sum_y_intensity": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "diff_by_intensity": - family: "rich" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_b_ions": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_y_ions": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "weighted_mass_error": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "mean_mass_error": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "isotope_corr": - family: "rich" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "ms1_isom1_ratio": - family: "rich" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "log_mono_ms1": - family: "rich" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "has_ms1": - family: "rich" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "log_sn": - family: "rich" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_observations": - family: "rich" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "base_width_rt": - family: "rich" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "seed_score": - family: "rich" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "seed_identified": - family: "rich" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "matched_fraction": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "profile_cos": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "ref_corr": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "best_ref_corr": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "low_frag_coel": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "evidence": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "contrast_min": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "resid_corr": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "coel_clean": - family: "rich" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "shadow_frac": - family: "rich" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "peak_contested_frac": - family: "extended-extra (psms-derived)" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "n_charge_states": - family: "extended-extra (cross-candidate)" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "charge_multi_flag": - family: "extended-extra (cross-candidate)" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "cross_charge_intensity_log": - family: "extended-extra (cross-candidate)" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features.rs" - "spectrum_cosine_matched": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectrum_cosine_sqrt": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectrum_cosine_log": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_angle@similarity": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" + rt_error_abs: + family: minimal + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + rt_error_rel: + family: minimal + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_matched_fragments: + family: minimal + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + coelution_run: + family: minimal + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + log_apex_intensity: + family: minimal + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + frag_corr: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + frag_cosine: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + spectral_angle: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + coelution_mean: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + coelution_best: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_coelution_above: + family: minimal + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + charge: + family: minimal + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + peptide_length: + family: minimal + level: candidate + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_proteins: + family: minimal + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + library_norm_manhattan: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + library_rmsd: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + xcorr_coelution: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + xcorr_shape: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + sum_b_intensity: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + sum_y_intensity: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + diff_by_intensity: + family: rich + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_b_ions: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_y_ions: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + weighted_mass_error: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + mean_mass_error: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + isotope_corr: + family: rich + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + ms1_isom1_ratio: + family: rich + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + log_mono_ms1: + family: rich + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + has_ms1: + family: rich + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + log_sn: + family: rich + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_observations: + family: rich + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + base_width_rt: + family: rich + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + seed_score: + family: rich + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + seed_identified: + family: rich + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + matched_fraction: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + profile_cos: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + ref_corr: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + best_ref_corr: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + low_frag_coel: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + evidence: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + contrast_min: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + resid_corr: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + coel_clean: + family: rich + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + shadow_frac: + family: rich + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + peak_contested_frac: + family: extended-extra (psms-derived) + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + n_charge_states: + family: extended-extra (cross-candidate) + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + charge_multi_flag: + family: extended-extra (cross-candidate) + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + cross_charge_intensity_log: + family: extended-extra (cross-candidate) + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features.rs + spectrum_cosine_matched: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectrum_cosine_sqrt: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectrum_cosine_log: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_angle@similarity: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs dropped_collision: true - "spectral_angle_sqrt": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_angle_matched": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "pearson_intensity_matched": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "pearson_intensity_log": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spearman_intensity": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spearman_intensity_matched": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "kendall_tau_intensity": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "dot_product_raw": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "dot_product_norm": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "library_recall_intensity": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "manhattan_sim": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "manhattan_sqrt": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "rmsd_norm": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "mae_norm": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "mse_log": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "mae_weighted_pred": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "abs_diff_q3": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "max_positive_residual": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "chebyshev_dist": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "minkowski_p3": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "bray_curtis": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "bray_curtis_sqrt": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "canberra": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "canberra_matched": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "wave_hedges": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "chi_square_pearson": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "chi_square_symmetric": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "divergence_distance": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "bhattacharyya_coef": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "hellinger": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "squared_chord": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "harmonic_mean_sim": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "jaccard_presence": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "dice_presence": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "intensity_weighted_pearson": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "regression_slope": - family: "similarity" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "gini_diff": - family: "similarity" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "wasserstein_mz": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "footrule_norm": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "rank_overlap_top3": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "top1_frag_match": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "top1_predicted_observed": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "frac_top3_predicted_observed": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "count_strong_predicted_absent": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "frac_predicted_absent": - family: "similarity" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "pearson_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_angle_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_fullwindow": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "stein_scott_weighted_dot": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "log_dot_product": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_log_evidence": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "scribe_score": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "log_dot_product_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_log_evidence_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "scribe_score_area": - family: "similarity" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_high_ordinal": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_robust_trim1": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_robust_trim2": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "cosine_robust_trim3": - family: "similarity" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/similarity.rs" - "spectral_entropy_similarity": - family: "entropy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "weighted_spectral_entropy_similarity": - family: "entropy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "spectral_entropy_similarity_sqrt": - family: "entropy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "spectral_entropy_similarity_topk": - family: "entropy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "spectral_entropy_similarity_area": - family: "entropy" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "jensen_shannon_divergence": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "jeffreys_divergence": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "kl_obs_pred": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "kl_pred_obs": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "cross_entropy_obs_pred": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "obs_spectrum_entropy": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "pred_spectrum_entropy": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "entropy_diff": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "entropy_ratio": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "obs_normalized_entropy": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "normalized_entropy_diff": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "residual_spectrum_entropy": - family: "entropy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "entropy_weight_obs": - family: "entropy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/entropy.rs" - "frag_ref_corr_mean": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_obsweighted": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_min": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_std": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_sq_mean": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_topk_weighted": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "n_frag_ref_corr_above_0_9": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frac_frag_ref_corr_above_0_8": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_ref_corr_mean_full": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "full_vs_peak_corr_gain": - family: "coelution" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_weighted": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_min": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_median": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_std": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_frac_negative": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_hi": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "pairwise_coelution_lo": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "coelution_hi_lo_contrast": - family: "coelution" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "coelution_corr_entropy": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_shape_mean": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_shape_min": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_shape_std": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_mean_abs": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_std": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_iqr": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_frac_zero": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_max_abs": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "xcorr_lag_entropy": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "ref_xcorr_lag_mean": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "ref_xcorr_shape_mean": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "observed_sum_vs_template_corr": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_loo_ref_corr_mean": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frag_loo_ref_corr_min": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "frac_frags_apex_aligned": - family: "coelution" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "top3_frag_ref_corr": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "by_cross_coelution": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "by_cross_lag_mean": - family: "coelution" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "charge_cross_coelution": - family: "coelution" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/coelution.rs" - "explained_variance_ref": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "profile_residual_fraction": - family: "interference" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "n_interfered_fragments": - family: "interference" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "corrected_vs_raw_cos": - family: "interference" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "corrected_vs_raw_ratio": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "ifs_removed_count": - family: "interference" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "ifs_removed_intensity_frac": - family: "interference" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "ifs_corr_gain": - family: "interference" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "ifs_retained_frac": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "matched_frac_after_ifs": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "peak_to_full_area_ratio_profile": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "peak_to_full_area_ratio_frag_mean": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "peak_to_full_area_ratio_weighted": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "out_of_peak_intensity_frac": - family: "interference" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "profile_corr_full_vs_peak_delta": - family: "interference" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "frac_frag_ref_corr_below_0_5": - family: "interference" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "explained_apex_intensity_frac": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "apex_purity": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "interference_apex_residual_fraction": - family: "interference" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "dominant_frag_ref_corr": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "explained_variance_ratio": - family: "interference" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "second_component_fraction": - family: "interference" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "profile_second_peak_ratio": - family: "interference" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "n_competing_peaks_in_window": - family: "interference" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "matched_pred_intensity_fraction": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "top_pred_frag_matched": - family: "interference" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/interference.rs" - "gaussian_fit_r2": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "gaussian_cosine": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "emg_fit_improvement": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_prominence": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "profile_peak_snr": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "fwhm_seconds": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "fwhm_to_window_ratio": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "width_at_10pct": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "width_ratio_10_50": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "hwhm_asymmetry": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "tailing_factor_usp": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "asymmetry_factor_10pct": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_sharpness": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_curvature": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_to_boundary_ratio": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_dominance": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "zigzag_index": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "jaggedness": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "roughness_2nd_deriv": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "n_local_maxima": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "modality": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "rt_skewness": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "rt_excess_kurtosis": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "rt_std_seconds": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "mean_mode_offset": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "fraction_area_within_fwhm": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "triangle_area_similarity": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "baseline_fraction": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "peak_completeness": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "apex_centering_offset": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "intensity_score": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "total_xic_log": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_fwhm_cv": - family: "chromatographic" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_fwhm_mean": - family: "chromatographic" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_apex_rt_dispersion": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_apex_rt_dispersion_weighted": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_apex_offset_from_profile_mean": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_gaussianity_mean": - family: "chromatographic" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_gaussianity_weighted": - family: "chromatographic" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "frag_zigzag_mean": - family: "chromatographic" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "sumtrace_unweighted_gaussian_r2": - family: "chromatographic" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "reference_profile_rt_entropy_peak": - family: "chromatographic" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "reference_profile_rt_entropy_ratio": - family: "chromatographic" - level: "peak" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs" - "median_abs_frag_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "signed_mean_frag_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "ppm_std": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "ppm_iqr": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "ppm_range": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "max_abs_frag_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "intensity_weighted_abs_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "intensity_weighted_signed_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "intensity_weighted_ppm_std": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "lib_weighted_abs_ppm": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "frac_frag_within_half_tol": - family: "mass_accuracy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "high_ppm_intensity_frac": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "ppm_intensity_anticorr": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "mass_error_mz_trend": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "mean_abs_mz_error_da": - family: "mass_accuracy" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "mass_evidence_gauss": - family: "mass_accuracy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "mass_log_evidence": - family: "mass_accuracy" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs" - "n_matched_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "n_matched_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "frac_matched_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "frac_matched_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_count_balance": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_intensity_ratio": - family: "ion_series" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_ratio_agreement": - family: "ion_series" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_ratio_consistency": - family: "ion_series" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "longest_b_run": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "longest_y_run": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "longest_run_max": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "longest_run_frac_length": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "series_coverage_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "series_coverage_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "sequence_coverage": - family: "ion_series" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "series_gap_fraction": - family: "ion_series" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_complement_count": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_complement_mz_consistency": - family: "ion_series" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_complement_coelution": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "ordinal_intensity_concordance_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "ordinal_intensity_concordance_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "series_coelution_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "series_coelution_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "spectral_angle_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "spectral_angle_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "pearson_b": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "pearson_y": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "cosine_charge1": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "cosine_charge2": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "charge_corr_balance": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "mean_matched_ordinal_norm": - family: "ion_series" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_ion_contiguous_intensity": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "by_ion_contiguous_lib_frac": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "both_series_present": - family: "ion_series" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs" - "ms1_isotope_cosine_apex": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_spectral_angle_apex": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_chi2_apex": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_manhattan_apex": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_ratio_1_0": - family: "ms1" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_ratio_2_0": - family: "ms1" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_plus1_ratio_dev": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_plus2_ratio_dev": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_minus_one_fraction": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "iso_overlap_flag": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "log_ms1_mono": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_total_isotope_log": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "has_ms1_signal": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_apex_entropy_3": - family: "ms1" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_m1_entropy_contribution": - family: "ms1" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_ms2_time_corr": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_ms2_envelope_time_corr": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_iso_coelution": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_ms2_apex_rt_delta": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_iso_ratio_stability": - family: "ms1" - level: "precursor" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_mono_gaussianity": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_ms2_fwhm_ratio": - family: "ms1" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_corr_xic": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_envelope_over_time_corr": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "ms1_isotope_xic_shape_consistency": - family: "ms1" - level: "precursor" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/ms1.rs" - "rt_error_signed": - family: "rt" - level: "candidate" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_error_abs@rt": - family: "rt" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" + spectral_angle_sqrt: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_angle_matched: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + pearson_intensity_matched: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + pearson_intensity_log: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spearman_intensity: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spearman_intensity_matched: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + kendall_tau_intensity: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + dot_product_raw: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + dot_product_norm: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + library_recall_intensity: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + manhattan_sim: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + manhattan_sqrt: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + rmsd_norm: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + mae_norm: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + mse_log: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + mae_weighted_pred: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + abs_diff_q3: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + max_positive_residual: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + chebyshev_dist: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + minkowski_p3: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + bray_curtis: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + bray_curtis_sqrt: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + canberra: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + canberra_matched: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + wave_hedges: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + chi_square_pearson: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + chi_square_symmetric: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + divergence_distance: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + bhattacharyya_coef: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + hellinger: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + squared_chord: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + harmonic_mean_sim: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + jaccard_presence: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + dice_presence: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + intensity_weighted_pearson: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + regression_slope: + family: similarity + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + gini_diff: + family: similarity + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + wasserstein_mz: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + footrule_norm: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + rank_overlap_top3: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + top1_frag_match: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + top1_predicted_observed: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + frac_top3_predicted_observed: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + count_strong_predicted_absent: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + frac_predicted_absent: + family: similarity + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + pearson_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_angle_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_fullwindow: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + stein_scott_weighted_dot: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + log_dot_product: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_log_evidence: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + scribe_score: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + log_dot_product_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_log_evidence_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + scribe_score_area: + family: similarity + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_high_ordinal: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_robust_trim1: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_robust_trim2: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + cosine_robust_trim3: + family: similarity + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/similarity.rs + spectral_entropy_similarity: + family: entropy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + weighted_spectral_entropy_similarity: + family: entropy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + spectral_entropy_similarity_sqrt: + family: entropy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + spectral_entropy_similarity_topk: + family: entropy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + spectral_entropy_similarity_area: + family: entropy + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + jensen_shannon_divergence: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + jeffreys_divergence: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + kl_obs_pred: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + kl_pred_obs: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + cross_entropy_obs_pred: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + obs_spectrum_entropy: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + pred_spectrum_entropy: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + entropy_diff: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + entropy_ratio: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + obs_normalized_entropy: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + normalized_entropy_diff: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + residual_spectrum_entropy: + family: entropy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + entropy_weight_obs: + family: entropy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/entropy.rs + frag_ref_corr_mean: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_obsweighted: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_min: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_std: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_sq_mean: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_topk_weighted: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + n_frag_ref_corr_above_0_9: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frac_frag_ref_corr_above_0_8: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_ref_corr_mean_full: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + full_vs_peak_corr_gain: + family: coelution + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_weighted: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_min: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_median: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_std: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_frac_negative: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_hi: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + pairwise_coelution_lo: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + coelution_hi_lo_contrast: + family: coelution + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + coelution_corr_entropy: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_shape_mean: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_shape_min: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_shape_std: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_mean_abs: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_std: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_iqr: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_frac_zero: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_max_abs: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + xcorr_lag_entropy: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + ref_xcorr_lag_mean: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + ref_xcorr_shape_mean: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + observed_sum_vs_template_corr: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_loo_ref_corr_mean: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frag_loo_ref_corr_min: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + frac_frags_apex_aligned: + family: coelution + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + top3_frag_ref_corr: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + by_cross_coelution: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + by_cross_lag_mean: + family: coelution + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + charge_cross_coelution: + family: coelution + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/coelution.rs + explained_variance_ref: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + profile_residual_fraction: + family: interference + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + n_interfered_fragments: + family: interference + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + corrected_vs_raw_cos: + family: interference + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + corrected_vs_raw_ratio: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + ifs_removed_count: + family: interference + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + ifs_removed_intensity_frac: + family: interference + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + ifs_corr_gain: + family: interference + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + ifs_retained_frac: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + matched_frac_after_ifs: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + peak_to_full_area_ratio_profile: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + peak_to_full_area_ratio_frag_mean: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + peak_to_full_area_ratio_weighted: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + out_of_peak_intensity_frac: + family: interference + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + profile_corr_full_vs_peak_delta: + family: interference + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + frac_frag_ref_corr_below_0_5: + family: interference + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + explained_apex_intensity_frac: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + apex_purity: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + interference_apex_residual_fraction: + family: interference + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + dominant_frag_ref_corr: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + explained_variance_ratio: + family: interference + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + second_component_fraction: + family: interference + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + profile_second_peak_ratio: + family: interference + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + n_competing_peaks_in_window: + family: interference + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + matched_pred_intensity_fraction: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + top_pred_frag_matched: + family: interference + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/interference.rs + gaussian_fit_r2: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + gaussian_cosine: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + emg_fit_improvement: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_prominence: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + profile_peak_snr: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + fwhm_seconds: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + fwhm_to_window_ratio: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + width_at_10pct: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + width_ratio_10_50: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + hwhm_asymmetry: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + tailing_factor_usp: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + asymmetry_factor_10pct: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_sharpness: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_curvature: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_to_boundary_ratio: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_dominance: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + zigzag_index: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + jaggedness: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + roughness_2nd_deriv: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + n_local_maxima: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + modality: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + rt_skewness: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + rt_excess_kurtosis: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + rt_std_seconds: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + mean_mode_offset: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + fraction_area_within_fwhm: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + triangle_area_similarity: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + baseline_fraction: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + peak_completeness: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + apex_centering_offset: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + intensity_score: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + total_xic_log: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_fwhm_cv: + family: chromatographic + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_fwhm_mean: + family: chromatographic + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_apex_rt_dispersion: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_apex_rt_dispersion_weighted: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_apex_offset_from_profile_mean: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_gaussianity_mean: + family: chromatographic + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_gaussianity_weighted: + family: chromatographic + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + frag_zigzag_mean: + family: chromatographic + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + sumtrace_unweighted_gaussian_r2: + family: chromatographic + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + reference_profile_rt_entropy_peak: + family: chromatographic + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + reference_profile_rt_entropy_ratio: + family: chromatographic + level: peak + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/chromatographic.rs + median_abs_frag_ppm: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + signed_mean_frag_ppm: + family: mass_accuracy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + ppm_std: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + ppm_iqr: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + ppm_range: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + max_abs_frag_ppm: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + intensity_weighted_abs_ppm: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + intensity_weighted_signed_ppm: + family: mass_accuracy + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + intensity_weighted_ppm_std: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + lib_weighted_abs_ppm: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + frac_frag_within_half_tol: + family: mass_accuracy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + high_ppm_intensity_frac: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + ppm_intensity_anticorr: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + mass_error_mz_trend: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + mean_abs_mz_error_da: + family: mass_accuracy + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + mass_evidence_gauss: + family: mass_accuracy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + mass_log_evidence: + family: mass_accuracy + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_accuracy.rs + n_matched_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + n_matched_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + frac_matched_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + frac_matched_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_count_balance: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_intensity_ratio: + family: ion_series + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_ratio_agreement: + family: ion_series + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_ratio_consistency: + family: ion_series + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + longest_b_run: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + longest_y_run: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + longest_run_max: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + longest_run_frac_length: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + series_coverage_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + series_coverage_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + sequence_coverage: + family: ion_series + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + series_gap_fraction: + family: ion_series + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_complement_count: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_complement_mz_consistency: + family: ion_series + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_complement_coelution: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + ordinal_intensity_concordance_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + ordinal_intensity_concordance_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + series_coelution_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + series_coelution_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + spectral_angle_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + spectral_angle_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + pearson_b: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + pearson_y: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + cosine_charge1: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + cosine_charge2: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + charge_corr_balance: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + mean_matched_ordinal_norm: + family: ion_series + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_ion_contiguous_intensity: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + by_ion_contiguous_lib_frac: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + both_series_present: + family: ion_series + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ion_series.rs + ms1_isotope_cosine_apex: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_spectral_angle_apex: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_chi2_apex: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_manhattan_apex: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_ratio_1_0: + family: ms1 + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_ratio_2_0: + family: ms1 + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_plus1_ratio_dev: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_plus2_ratio_dev: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_minus_one_fraction: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + iso_overlap_flag: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + log_ms1_mono: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_total_isotope_log: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + has_ms1_signal: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_apex_entropy_3: + family: ms1 + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_m1_entropy_contribution: + family: ms1 + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_ms2_time_corr: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_ms2_envelope_time_corr: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_iso_coelution: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_ms2_apex_rt_delta: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_iso_ratio_stability: + family: ms1 + level: precursor + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_mono_gaussianity: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_ms2_fwhm_ratio: + family: ms1 + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_corr_xic: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_envelope_over_time_corr: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + ms1_isotope_xic_shape_consistency: + family: ms1 + level: precursor + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/ms1.rs + rt_error_signed: + family: rt + level: candidate + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_error_abs@rt: + family: rt + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs dropped_collision: true - "rt_error_squared": - family: "rt" - level: "candidate" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_error_signed_norm_gradient": - family: "rt" - level: "run" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_error_abs_norm_gradient": - family: "rt" - level: "run" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "observed_rt_raw": - family: "rt" - level: "candidate" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "predicted_rt_raw": - family: "rt" - level: "candidate" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "observed_rt_fraction": - family: "rt" - level: "run" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "predicted_rt_fraction": - family: "rt" - level: "run" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_error_over_peak_width": - family: "rt" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_error_over_fwhm": - family: "rt" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "rt_diff_profile_apex": - family: "rt" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "predicted_rt_in_gradient": - family: "rt" - level: "run" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/rt.rs" - "log_seed_hyperscore": - family: "novel" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "seed_hyperscore_per_matched": - family: "novel" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "seed_identified@novel": - family: "novel" - level: "candidate" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + rt_error_squared: + family: rt + level: candidate + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_error_signed_norm_gradient: + family: rt + level: run + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_error_abs_norm_gradient: + family: rt + level: run + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + observed_rt_raw: + family: rt + level: candidate + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + predicted_rt_raw: + family: rt + level: candidate + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + observed_rt_fraction: + family: rt + level: run + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + predicted_rt_fraction: + family: rt + level: run + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_error_over_peak_width: + family: rt + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_error_over_fwhm: + family: rt + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + rt_diff_profile_apex: + family: rt + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + predicted_rt_in_gradient: + family: rt + level: run + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/rt.rs + log_seed_hyperscore: + family: novel + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + seed_hyperscore_per_matched: + family: novel + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + seed_identified@novel: + family: novel + level: candidate + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs dropped_collision: true - "peptide_length@novel": - family: "novel" - level: "candidate" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" + peptide_length@novel: + family: novel + level: candidate + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs dropped_collision: true - "precursor_charge": - family: "novel" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "charge_is_2": - family: "novel" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "charge_is_3": - family: "novel" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "charge_is_4plus": - family: "novel" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "precursor_mass": - family: "novel" - level: "precursor" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "log_total_matched_intensity": - family: "novel" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "n_matched_frags": - family: "novel" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "n_predicted_frags": - family: "novel" - level: "fragment" - direction: "neutral" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/novel.rs" - "frag_corr_peakmax": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "frag_cosine_peakmax": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "spectral_angle_peakmax": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "frag_corr_matched_nz": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "frag_cosine_matched_nz": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "peakmax_apex_gain": - family: "nonzero" - level: "fragment" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "n_frag_present_inpeak": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "frac_frag_present_inpeak": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "coelution_mean_bothpos": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "coelution_mean_summpos": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "ref_corr_nz": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "profile_cos_nz": - family: "nonzero" - level: "fragment" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs" - "rank_corr_vs_apex_mean": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "rank_corr_vs_apex_std": - family: "order_consistency" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "rank_corr_adjacent_mean": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "kendall_vs_apex_mean": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "top1_frag_persistence": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "top2_order_persistence": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "argmax_frag_entropy": - family: "order_consistency" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "self_cosine_vs_apex_mean": - family: "order_consistency" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs" - "n_peak_scans": - family: "peak_scans" - level: "peak" - direction: "higher_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs" - "peak_window_degenerate": - family: "peak_scans" - level: "peak" - direction: "lower_better" - source_file: "rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs" + precursor_charge: + family: novel + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + charge_is_2: + family: novel + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + charge_is_3: + family: novel + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + charge_is_4plus: + family: novel + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + precursor_mass: + family: novel + level: precursor + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + log_total_matched_intensity: + family: novel + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + n_matched_frags: + family: novel + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + n_predicted_frags: + family: novel + level: fragment + direction: neutral + source_file: rust/mumdia/crates/mumdia/src/stages/features/novel.rs + frag_corr_peakmax: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + frag_cosine_peakmax: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + spectral_angle_peakmax: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + frag_corr_matched_nz: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + frag_cosine_matched_nz: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + peakmax_apex_gain: + family: nonzero + level: fragment + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + n_frag_present_inpeak: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + frac_frag_present_inpeak: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + coelution_mean_bothpos: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + coelution_mean_summpos: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + ref_corr_nz: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + profile_cos_nz: + family: nonzero + level: fragment + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/nonzero.rs + rank_corr_vs_apex_mean: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + rank_corr_vs_apex_std: + family: order_consistency + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + rank_corr_adjacent_mean: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + kendall_vs_apex_mean: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + top1_frag_persistence: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + top2_order_persistence: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + argmax_frag_entropy: + family: order_consistency + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + self_cosine_vs_apex_mean: + family: order_consistency + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/order_consistency.rs + n_peak_scans: + family: peak_scans + level: peak + direction: higher_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs + peak_window_degenerate: + family: peak_scans + level: peak + direction: lower_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/peak_scans.rs + frag_apex_rt_std: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + frag_apex_rt_mad: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + frag_apex_max_dev: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + frag_apex_mean_dev: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + frag_apex_agree_frac: + family: apex_dispersion + level: peak_group + direction: higher_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + precursor_frag_apex_delta: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_symmetry: + family: apex_dispersion + level: peak_group + direction: target_0.5 + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_tailing: + family: apex_dispersion + level: peak_group + direction: target_1.0 + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_n_local_maxima: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_shoulder_score: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_fwhm_scans: + family: apex_dispersion + level: peak_group + direction: ambiguous + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + peak_truncation: + family: apex_dispersion + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + apex_frac_of_window: + family: apex_dispersion + level: peak_group + direction: ambiguous + source_file: rust/mumdia/crates/mumdia/src/stages/features/apex_dispersion.rs + frag_mass_err_median: + family: mass_uncertainty + level: peak_group + direction: target_0 + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frag_mass_err_abs_median: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frag_mass_err_std: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frag_mass_err_iqr: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frag_mass_err_max_abs: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frag_mass_err_range: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + effective_frag_count: + family: mass_uncertainty + level: peak_group + direction: higher_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + evidence_concentration: + family: mass_uncertainty + level: peak_group + direction: lower_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frac_top3_pred_observed: + family: mass_uncertainty + level: peak_group + direction: higher_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs + frac_top5_pred_observed: + family: mass_uncertainty + level: peak_group + direction: higher_is_better + source_file: rust/mumdia/crates/mumdia/src/stages/features/mass_uncertainty.rs diff --git a/sensitivity_plan/FEATURE_REGISTRY.md b/sensitivity_plan/FEATURE_REGISTRY.md index c2c3db6..a0cda4c 100644 --- a/sensitivity_plan/FEATURE_REGISTRY.md +++ b/sensitivity_plan/FEATURE_REGISTRY.md @@ -11,6 +11,24 @@ family ablations (§4). A machine-readable copy of this table is `feature_registry.yaml` at the repository root (one entry per feature: `family`, `level`, `direction`, `source_file`). +> Update: the Extended battery now has **383** features (was 356 at first +> writing; the registry documents 383 including the Minimal/Rich tiers). Two +> families were added this session (append-only, so the Extended-count test is +> dynamic and stays green): +> - **apex_dispersion** (13, `stages/features/apex_dispersion.rs`, P5.3): per- +> fragment apex-RT scatter (std/mad/max/mean deviation, agreement fraction), +> precursor-fragment apex delta, and consensus peak shape (symmetry, tailing, +> local maxima, shoulder, FWHM, truncation, apex position in window). +> - **mass_uncertainty** (10, `stages/features/mass_uncertainty.rs`, P5.1/P5.2): +> fragment mass-error distribution (median/abs-median/std/IQR/max/range), +> effective fragment count, evidence concentration, and fraction of top-3/top-5 +> predicted ions observed (breadth of the strong ions). +> +> These are computed from the per-PSM `Evidence` (peak-bounded traces + mass +> errors) and are interference-resistant (shape and breadth, not absolute +> intensity). They are registered in `feature_registry.yaml`; the per-family +> tables below predate them. + ## 1. Feature-set tiers The active feature set is selected by `features.set` (`FeatureSet` enum, diff --git a/sensitivity_plan/IMPLEMENTATION_STATUS.md b/sensitivity_plan/IMPLEMENTATION_STATUS.md index fd650f6..8b8d705 100644 --- a/sensitivity_plan/IMPLEMENTATION_STATUS.md +++ b/sensitivity_plan/IMPLEMENTATION_STATUS.md @@ -115,3 +115,38 @@ before dropping any family. ## Recommended next steps - See `NEXT_STEPS.md` (written at end of session). + +--- + +## Session 2 (implement-more push) — full backlog status + +Branch `feat/sensitivity-improvements`. All Rust changes keep the suite green +(95 tests) and are default-off / behaviour-preserving unless noted. + +### Done this push +- P0.1 `scripts/search_space_manifest.py` (effective manifest + DIA-NN/declared parity, `--fail-on-mismatch`). +- P0.2 `scripts/normalize_output.py` (MuMDIA scored -> spec 02 §4 common schema; `--reported-only` = 9,540 on E. coli). +- P4.2 `scripts/feature_audit.py` (missingness/quantiles/constant, target-decoy-entrapment separation, redundancy clusters, leakage/intensity warnings). Finding on E. coli: 355 audited, 3 constant, 0 leakage-flagged (decoy-vs-entrapment gap <=0.043), 54 clusters. +- P4.3 already done (`feature_ablation.py`); full both-model out_ecoli run produced in `accept/`. +- P5.1 `mass_uncertainty` family (10) + P5.3 `apex_dispersion` family (13). Registry now 383 features. +- P5.2 partially covered by `mass_uncertainty` (fragment-evidence distributions). +- Apex evidence-count option `extract.apex_evidence_rank` (the interference insight: pick the apex by co-eluting fragment breadth, not intensity). Default off. +- P3.2/P3.3 `rt_im_train.adaptive_rt_window` (per-RT-region local residual window, clamped). Default off. +- P7.1 `scripts/benchmark_report.py` (self-contained HTML; smoke -> `accept/benchmark_report.html`). +- P7.2 `scripts/candidate_diagnostics.py` (per-candidate chromatogram + feature bundle). + +### Still TODO (large or non-additive; documented in NEXT_STEPS.md) +- P1.1/P1.2/P1.3 top-K peak RETENTION wired into scoring (enumerator + config + opportunity analysis done; the downstream rewire that carries multiple peaks per candidate through features/compete/rescore + a peak-selection model is the large remaining piece). The reference_apex_topk analysis already measures the opportunity (rank-1 47.9% full). +- P2.2 peak-group conflict graph + P2.3 full conflict-feature list (only `contested_frac` + the new evidence-breadth features exist; a cross-candidate claimant graph is not built). +- P3.1 two-pass mass calibration (RT adaptive window done; mass side not). +- P5.4 remaining MS1/isotope features; P5.5 candidate-ambiguity margins (needs cross-candidate neighborhood). +- P6.2 localization competition + P6.3 staged modification search (need site-determining-ion scoring). +- P0.1 parity check not yet run against a real DIA-NN report (needs the report). + +### Acceptance validation status +- Full out_ecoli ablation (logreg + hgb) produced in `accept/`. HYE + TTOF runs + were launched (`run_accept.sh`); TTOF sample identity is unconfirmed (see the + chat log). None of the new default-off knobs (apex_evidence_rank, + adaptive_rt_window, competition modes) has an engine-gain measurement yet: each + must pass the entrapment-holdout gate on >=2 datasets before being enabled by + default (spec 05 §6). That validation loop is the next action, not more code. From b55d3bae11a6d0f92e1d98cfc2ffa644f0315805 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 19:54:25 +0200 Subject: [PATCH 23/40] feat(sensitivity): --rank-by count (evidence-breadth) in reference_apex_topk Adds a peak-ranking mode that ranks each peak by the number of distinct predicted fragments co-eluting in its RT window (breadth of evidence), not integrated intensity. Intensity is chimeric in wide-window DIA, so area-ranking rewards interferents; evidence-count ranking is interference-resistant. Enables the area-vs-count comparison for the peak-selection question and the eventual --diann reference-apex adjudication. Co-Authored-By: Claude Opus 4.8 --- scripts/reference_apex_topk.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/reference_apex_topk.py b/scripts/reference_apex_topk.py index 00695db..d4f8d12 100644 --- a/scripts/reference_apex_topk.py +++ b/scripts/reference_apex_topk.py @@ -241,6 +241,10 @@ def main(): ap.add_argument("--bound-fraction", type=float, default=1.0 / 3.0) ap.add_argument("--min-prominence", type=float, default=0.05) ap.add_argument("--rt-tol-s", type=float, default=10.0) + ap.add_argument("--rank-by", choices=["area", "count"], default="area", + help="peak ranking: 'area' = integrated intensity (chimeric in DIA); " + "'count' = number of distinct predicted fragments co-eluting in the " + "peak window (breadth of evidence, interference-resistant)") ap.add_argument("--rt-round", type=int, default=3, help="RT rounding decimals for axis union") ap.add_argument("--batch-size", type=int, default=100000) args = ap.parse_args() @@ -306,6 +310,27 @@ def process_candidate(cid, rows): rt_axis, prof = build_consensus(rows, args.top_frags, args.rt_round) peaks = enumerate_peaks(prof, len(prof) if len(prof) else 0, args.bound_fraction, args.min_prominence) + # Evidence-count ranking (interference-resistant): rank each peak by the + # number of DISTINCT predicted fragments with any nonzero intensity inside + # its RT window, not by integrated intensity. In DIA the tallest peak is + # often a co-isolated interferent; the true peak is the one where the most + # of the peptide's own predicted transitions co-elute. Uses ALL fragment + # rows (breadth), not just the top-N summed into the detection profile. + if args.rank_by == "count" and peaks: + for p in peaks: + lo, hi = rt_axis[p["start_idx"]], rt_axis[p["end_idx"]] + ev = 0 + for _predi, rt_list, int_list in rows: + present = any( + (lo <= float(rt) <= hi) and (float(it) > 0.0) + for rt, it in zip(rt_list, int_list) + ) + if present: + ev += 1 + p["evidence"] = ev + peaks.sort(key=lambda p: (-p.get("evidence", 0), -p["area"], p["apex_idx"])) + for r, p in enumerate(peaks): + p["rank"] = r peaks_per_cand.append(len(peaks)) processed.add(cid) # SELF From b183fecdbd13efc1074efa19bf2c8efd0f7974e3 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 20:16:18 +0200 Subject: [PATCH 24/40] feat(sensitivity): retain top-K peak groups as evidence (P1.1/P1.2) When extract.retain_top_peaks > 1, extraction enumerates peak groups over each candidate's per-scan distinct-fragment COUNT profile (co-eluting breadth, not intensity) via mumdia::peaks::enumerate_peaks, and writes .peaks.parquet (candidate_id, peak_rank, apex_rt, start_rt, end_rt, evidence_count, area), one row per retained peak. The main psms/chrom flow still reports the single selected apex, so target-decoy FDR is unaffected; the retained peaks are candidate inputs for an offline peak-selection model (reference_apex_topk.py measures the recall). K=1 (default) writes no peaks table and is byte-identical to before. Co-Authored-By: Claude Opus 4.8 --- .../crates/mumdia/src/stages/extract.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index 62de8a9..2d4dd3d 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -588,6 +588,13 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ms1_i2: Option, /// (cid, frag_name, frag_mz, frag_obs_mz, predicted_intensity, rt, intensity) chrom: Vec<(u32, String, f64, f64, f32, Vec, Vec)>, + /// Top-K retained peak groups (sensitivity_plan P1.1/P1.2), populated only + /// when `retain_top_peaks > 1`. Each: (rank, apex_rt, start_rt, end_rt, + /// evidence_count, area). Ranked by co-eluting fragment breadth (not + /// intensity). The main PSM above still reports the single selected apex, + /// so FDR is unaffected; these are candidate peaks for an offline peak- + /// selection model. Empty for K=1. + peaks: Vec<(u8, f64, f64, f64, f64, f64)>, } let results: Vec> = cand_hits @@ -910,6 +917,32 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } } + // Top-K peak retention (opt-in; sensitivity_plan P1.1/P1.2). Enumerate peak + // groups over the per-scan distinct-fragment COUNT profile (co-eluting + // breadth, interference-resistant per the intensity-is-chimeric argument), + // ranked by breadth-area. The PSM above still carries the selected apex, so + // FDR is unchanged; these are extra candidate peaks for an offline peak- + // selection model. Empty for K=1 (the default). + let peaks: Vec<(u8, f64, f64, f64, f64, f64)> = + if p.cfg.retain_top_peaks > 1 && !groups.is_empty() { + let count_prof: Vec = groups.iter().map(|(_, m)| m.len() as f32).collect(); + crate::peaks::enumerate_peaks(&count_prof, p.cfg.retain_top_peaks, 1.0 / 3.0, 0.1) + .into_iter() + .map(|pk| { + ( + pk.rank as u8, + groups[pk.apex_idx].0, + groups[pk.start_idx].0, + groups[pk.end_idx].0, + groups[pk.apex_idx].1.len() as f64, + pk.area as f64, + ) + }) + .collect() + } else { + Vec::new() + }; + Some(CandOut { cid, apex_rt, @@ -931,6 +964,7 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ms1_i1: o_ms1_i1, ms1_i2: o_ms1_i2, chrom: chrom_rows, + peaks, }) }) .collect(); @@ -938,8 +972,23 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { // Append results in the deterministic cand_ids order (parallel work above was // order-preserving via `collect`), reproducing the serial push order exactly. let mut n_accepted = 0u64; + // Top-K retained peaks (opt-in; empty for K=1). + let (mut pk_cid, mut pk_rank): (Vec, Vec) = (Vec::new(), Vec::new()); + let (mut pk_apex, mut pk_start, mut pk_end): (Vec, Vec, Vec) = + (Vec::new(), Vec::new(), Vec::new()); + let (mut pk_ev, mut pk_area): (Vec, Vec) = (Vec::new(), Vec::new()); for r in results.into_iter().flatten() { n_accepted += 1; + let rcid = r.cid; + for (rank, apex, start, end, ev, area) in &r.peaks { + pk_cid.push(rcid); + pk_rank.push(*rank as i32); + pk_apex.push(*apex); + pk_start.push(*start); + pk_end.push(*end); + pk_ev.push(*ev); + pk_area.push(*area); + } cid_c.push(r.cid); apexrt_c.push(r.apex_rt); apexim_c.push(None); @@ -1013,6 +1062,25 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ], )?; + // Top-K retained peaks (opt-in, sensitivity_plan P1.1/P1.2). Written next to + // the psms table only when retain_top_peaks > 1; one row per (candidate, peak). + if !pk_cid.is_empty() { + let pk_path = format!("{}.peaks.parquet", p.out_psms); + let n_peaks = write_table( + &pk_path, + vec![ + Col::U32("candidate_id".into(), pk_cid), + Col::I32("peak_rank".into(), pk_rank), + Col::F64("apex_rt".into(), pk_apex), + Col::F64("start_rt".into(), pk_start), + Col::F64("end_rt".into(), pk_end), + Col::F64("evidence_count".into(), pk_ev), + Col::F64("area".into(), pk_area), + ], + )?; + info!(peaks = n_peaks, path = %pk_path, "extract: wrote top-K peak table"); + } + let elapsed = t0.elapsed().as_millis(); let mut stats = std::collections::BTreeMap::new(); stats.insert("accepted".to_string(), json!(n_accepted)); From d61e3fe0ada44a5b18a6425e0eda3f88cda68e6b Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 20:19:48 +0200 Subject: [PATCH 25/40] feat(sensitivity): robust two-pass fragment mass calibration (P3.1) search_seed.two_pass_mass_cal (default false): after the median-offset + 95th-percentile tolerance fit, re-fit on only the deviations inside the first-pass window (outlier rejection), giving a tighter offset + local mass-uncertainty estimate. Exports frag_ppm_sigma + cal_passes in masscal.json. Falls back to the single-pass result when too few in-window calibrants remain. Single-pass default unchanged. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 7 +++++ .../crates/mumdia/src/stages/search_seed.rs | 31 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 5652ad5..32d1013 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -379,6 +379,12 @@ pub struct SearchSeedConfig { pub top_n_peaks: usize, /// Fragment-matcher backend (fragindex_spec). Default `Fragindex`. pub matcher: MatcherKind, + /// Robust two-pass fragment mass calibration (sensitivity_plan P3.1). After the + /// first median-offset + tolerance fit, re-fit on only the deviations inside the + /// first-pass tolerance window (rejecting outliers), giving a tighter, more + /// robust offset + local uncertainty. Falls back to the single-pass result when + /// too few in-window calibrants remain. Default false (single pass unchanged). + pub two_pass_mass_cal: bool, } impl Default for SearchSeedConfig { fn default() -> Self { @@ -390,6 +396,7 @@ impl Default for SearchSeedConfig { min_matched_peaks: 4, top_n_peaks: 0, matcher: MatcherKind::Fragindex, + two_pass_mass_cal: false, } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/search_seed.rs b/rust/mumdia/crates/mumdia/src/stages/search_seed.rs index c34b51e..4b5943b 100644 --- a/rust/mumdia/crates/mumdia/src/stages/search_seed.rs +++ b/rust/mumdia/crates/mumdia/src/stages/search_seed.rs @@ -173,25 +173,46 @@ pub fn run(p: SearchSeedParams) -> Result { } } } - let (frag_ppm_offset, frag_tol_learned) = if devs.len() >= 20 { - let mut sorted = devs.clone(); + // Median offset + 95th-percentile-of-centered tolerance. `fit` is reused by the + // optional robust second pass on the outlier-trimmed calibrants. + let fit = |d: &[f64]| -> (f64, f64) { + let mut sorted = d.to_vec(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); let offset = sorted[sorted.len() / 2]; - let centered: Vec = devs.iter().map(|d| (d - offset).abs()).collect(); + let centered: Vec = d.iter().map(|x| (x - offset).abs()).collect(); let tol = (crate::calibrate::percentile(¢ered, 0.95) * 1.5).max(5.0); (offset, tol) + }; + let (frag_ppm_offset, frag_tol_learned, cal_passes) = if devs.len() >= 20 { + let (o1, t1) = fit(&devs); + if p.cfg.two_pass_mass_cal { + // Second pass: keep only deviations inside the first-pass window, so + // random-match outliers cannot bias the offset, then re-fit. + let inl: Vec = devs.iter().cloned().filter(|d| (d - o1).abs() <= t1).collect(); + if inl.len() >= 20 { + let (o2, t2) = fit(&inl); + (o2, t2, 2) + } else { + (o1, t1, 1) + } + } else { + (o1, t1, 1) + } } else { - (0.0, p.cfg.fragment_tol_ppm) + (0.0, p.cfg.fragment_tol_ppm, 0) }; mumdia_io::json::write_json( &format!("{}.masscal.json", p.out), &json!({ "frag_ppm_offset": frag_ppm_offset, "frag_tol_ppm": frag_tol_learned, + // The learned tolerance is the local mass-uncertainty estimate. + "frag_ppm_sigma": frag_tol_learned, "n_dev": devs.len(), + "cal_passes": cal_passes, }), )?; - info!(frag_ppm_offset, frag_tol_learned, "search-seed: mass recalibration"); + info!(frag_ppm_offset, frag_tol_learned, cal_passes, "search-seed: mass recalibration"); let n = write_table( p.out, From f6e6a6f22060927f58310652215cc75ba4f7b8c5 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 20:27:18 +0200 Subject: [PATCH 26/40] feat(sensitivity): conflict-graph, localization, peak-selection sidecars (P2.1/2.2/2.3/P5.5/P6.2/P1.3) Non-invasive first-pass sidecars over existing artifacts (join into rescoring experiments; in-core wiring is future work): - conflict_features.py (P2.1/P2.2/P2.3 + P5.5): bounded fragment-claimant index + peak-group conflict graph -> per-candidate contested/unique fragment counts, conflict_group_size, shared_intensity_frac, and candidate-ambiguity margins (to best alt peptide / best decoy, competitor entropy) from prelim_score. - localization.py (P6.2): groups localization variants (same stripped seq + mod multiset, different sites) and computes site-determining ion count/intensity + localization_confidence. On E. coli finds 0 ambiguity groups (Carbamidomethyl fixed); validated on a synthetic phospho pair. - peak_selection_model.py (P1.3): grouped-OOF ranker over the top-K peaks table; compares learned vs evidence-count vs area ranking; DIA-NN or weak self-label. Co-Authored-By: Claude Opus 4.8 --- scripts/conflict_features.py | 334 ++++++++++++++++++++++++++++++++ scripts/localization.py | 285 +++++++++++++++++++++++++++ scripts/peak_selection_model.py | 193 ++++++++++++++++++ 3 files changed, 812 insertions(+) create mode 100644 scripts/conflict_features.py create mode 100644 scripts/localization.py create mode 100644 scripts/peak_selection_model.py diff --git a/scripts/conflict_features.py b/scripts/conflict_features.py new file mode 100644 index 0000000..bacae3a --- /dev/null +++ b/scripts/conflict_features.py @@ -0,0 +1,334 @@ +"""Cross-candidate interference and ambiguity features (non-invasive pass). + +Implements sensitivity_plan backlog items P2.1 (fragment claimant index), +P2.2 (peak-group conflict graph), P2.3 (conflict features) and P5.5 (candidate +ambiguity) as a single pass over existing MuMDIA artifacts. It reads the +per-candidate PSM table and the per-transition chromatogram table, builds a +bounded fragment claimant index, and writes one row of conflict features per +candidate to `conflict.parquet`, joinable into rescoring by `candidate_id`. + +The engine is not modified. This is a first-pass ("fix later") implementation: +correctness, bounded memory, and a working smoke test are prioritised over +completeness. All computation is deterministic (stable sorts, sorted iteration +where floats are summed; no RNG is used). + +Two candidates "claim" the same fragment when + * their fragment m/z agree within `--frag-tol-ppm`, AND + * their candidate apex RT agree within `--rt-window-s`, AND + * their precursor m/z agree within `--mz-precursor-tol` (co-isolatable). + +Only the scalar chromatogram columns (candidate_id, frag_mz, +predicted_intensity) are read; the large list columns (rt, intensity) are never +loaded, which keeps memory bounded. + +House style: literal prose, no em-dashes, no hardcoded secrets. +""" + +import argparse +import math +import sys +import time + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + + +def log(msg): + print(msg, file=sys.stderr, flush=True) + + +def load_psms(path, max_candidates): + """Load per-candidate metadata. Returns a dict of numpy arrays keyed by a + compact 0..C-1 candidate index, plus the sorted original candidate ids.""" + cols = ["candidate_id", "apex_rt", "precursor_mz", "charge", "label", + "base_peptide_id"] + t = pq.read_table(path, columns=cols) + cid = t.column("candidate_id").to_numpy() + # Stable sort by candidate_id for reproducible compact indexing. + order = np.argsort(cid, kind="stable") + cid = cid[order] + apex_rt = t.column("apex_rt").to_numpy()[order] + prec_mz = t.column("precursor_mz").to_numpy()[order] + charge = t.column("charge").to_numpy()[order] + label = np.asarray(t.column("label").to_pylist(), dtype=object)[order] + base_pep = t.column("base_peptide_id").to_numpy()[order] + + if max_candidates is not None and max_candidates < len(cid): + cid = cid[:max_candidates] + apex_rt = apex_rt[:max_candidates] + prec_mz = prec_mz[:max_candidates] + charge = charge[:max_candidates] + label = label[:max_candidates] + base_pep = base_pep[:max_candidates] + + is_decoy = np.array([str(x).lower() == "decoy" for x in label], dtype=bool) + return { + "uids": cid, # sorted original candidate ids (== compact order) + "apex_rt": apex_rt.astype(np.float64), + "prec_mz": prec_mz.astype(np.float64), + "charge": charge.astype(np.int32), + "is_decoy": is_decoy, + "base_pep": base_pep.astype(np.int64), + } + + +def load_prelim(path, uids): + """Load prelim_score from the comp table, aligned to the compact index.""" + t = pq.read_table(path, columns=["candidate_id", "prelim_score"]) + cid = t.column("candidate_id").to_numpy() + score = t.column("prelim_score").to_numpy() + prelim = np.full(len(uids), np.nan, dtype=np.float64) + comp = np.searchsorted(uids, cid) + inb = comp < len(uids) + ok = np.zeros(len(cid), dtype=bool) + ok[inb] = uids[comp[inb]] == cid[inb] + prelim[comp[ok]] = score[ok] + return prelim + + +def load_fragments(path, uids, batch_size=1_000_000): + """Stream the scalar chromatogram columns and map each transition to the + compact candidate index. Fragments of candidates outside `uids` are dropped + (this happens when --max-candidates subsets the candidate set).""" + pf = pq.ParquetFile(path) + cols = ["candidate_id", "frag_mz", "predicted_intensity"] + comp_chunks, mz_chunks, pint_chunks = [], [], [] + for batch in pf.iter_batches(columns=cols, batch_size=batch_size): + cid = batch.column("candidate_id").to_numpy(zero_copy_only=False) + mz = batch.column("frag_mz").to_numpy(zero_copy_only=False) + pint = batch.column("predicted_intensity").to_numpy(zero_copy_only=False) + comp = np.searchsorted(uids, cid) + inb = comp < len(uids) + keep = np.zeros(len(cid), dtype=bool) + keep[inb] = uids[comp[inb]] == cid[inb] + if not keep.any(): + continue + comp_chunks.append(comp[keep].astype(np.int32)) + mz_chunks.append(mz[keep].astype(np.float64)) + pint_chunks.append(pint[keep].astype(np.float64)) + if not comp_chunks: + empty_i = np.zeros(0, dtype=np.int32) + empty_f = np.zeros(0, dtype=np.float64) + return empty_i, empty_f, empty_f + return (np.concatenate(comp_chunks), + np.concatenate(mz_chunks), + np.concatenate(pint_chunks)) + + +def build_bin_index(frag_bin, frag_rt): + """Group fragments by m/z bin, sorted by candidate apex RT within each bin. + Returns the sort order and a dict bin_value -> (start, end) into the sorted + arrays. Within [start, end) the RT array is ascending, so RT-window + neighbours are a range scan rather than a full scan.""" + order = np.lexsort((frag_rt, frag_bin)) # primary bin, secondary rt + sbin = frag_bin[order] + bin_index = {} + if len(sbin): + uniq, starts = np.unique(sbin, return_index=True) + ends = np.append(starts[1:], len(sbin)) + for b, s, e in zip(uniq.tolist(), starts.tolist(), ends.tolist()): + bin_index[b] = (s, e) + return order, bin_index + + +def softmax_entropy(scores): + """Shannon entropy (nats) of the softmax over `scores`. Deterministic given + a fixed input order. Returns 0.0 for a single element.""" + if len(scores) <= 1: + return 0.0 + s = np.asarray(scores, dtype=np.float64) + if not np.all(np.isfinite(s)): + s = np.nan_to_num(s, nan=np.nanmin(s) if np.any(np.isfinite(s)) else 0.0) + z = s - s.max() + w = np.exp(z) + tot = w.sum() + if tot <= 0: + return 0.0 + w = w / tot + nz = w > 0 + return float(-np.sum(w[nz] * np.log(w[nz]))) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--psms", required=True) + ap.add_argument("--chrom", required=True) + ap.add_argument("--comp", default=None, + help="optional comp.parquet with prelim_score for ambiguity features") + ap.add_argument("--out", default="conflict.parquet") + ap.add_argument("--frag-tol-ppm", type=float, default=20.0) + ap.add_argument("--rt-window-s", type=float, default=30.0) + ap.add_argument("--mz-precursor-tol", type=float, default=0.5) + ap.add_argument("--max-candidates", type=int, default=None, + help="cap the candidate set (both queries and claimant " + "universe); for fast smoke tests over the full chrom") + args = ap.parse_args() + + t0 = time.time() + log(f"[conflict] loading psms {args.psms}") + meta = load_psms(args.psms, args.max_candidates) + uids = meta["uids"] + C = len(uids) + log(f"[conflict] {C} candidates") + + prelim = None + if args.comp: + log(f"[conflict] loading prelim_score {args.comp}") + prelim = load_prelim(args.comp, uids) + + log(f"[conflict] streaming fragments {args.chrom}") + fc, fmz, fpint = load_fragments(args.chrom, uids) + nfrag = len(fc) + log(f"[conflict] {nfrag} transitions retained") + + # Per-fragment candidate-level RT and precursor m/z (co-isolation keys). + frt = meta["apex_rt"][fc] + fpmz = meta["prec_mz"][fc] + + # ppm-scaled log bins. Two m/z within frag_tol_ppm differ by ~one bin, so + # queries also scan neighbour bins (b-1, b+1). + log_step = math.log1p(args.frag_tol_ppm * 1e-6) + fbin = np.floor(np.log(fmz) / log_step).astype(np.int64) + + order, bin_index = build_bin_index(fbin, frt) + sbin = fbin[order] + srt = frt[order] + smz = fmz[order] + spmz = fpmz[order] + scid = fc[order] + spint = fpint[order] + + ppm = args.frag_tol_ppm * 1e-6 + rtw = args.rt_window_s + mztol = args.mz_precursor_tol + + n_frags = np.zeros(C, dtype=np.int64) + claim_sum = np.zeros(C, dtype=np.float64) + claim_max = np.zeros(C, dtype=np.int64) + contested_cnt = np.zeros(C, dtype=np.int64) + contested_int = np.zeros(C, dtype=np.float64) + total_int = np.zeros(C, dtype=np.float64) + group_sets = [set() for _ in range(C)] + + log("[conflict] scanning claimant index") + for p in range(len(sbin)): + b = sbin[p] + rt_i = srt[p] + mz_i = smz[p] + pmz_i = spmz[p] + c_i = scid[p] + tol_abs = mz_i * ppm + rt_lo = rt_i - rtw + rt_hi = rt_i + rtw + others = None + for nb in (b - 1, b, b + 1): + se = bin_index.get(nb) + if se is None: + continue + s, e = se + seg_rt = srt[s:e] + lo = s + int(np.searchsorted(seg_rt, rt_lo, side="left")) + hi = s + int(np.searchsorted(seg_rt, rt_hi, side="right")) + if hi <= lo: + continue + mzs = smz[lo:hi] + pmzs = spmz[lo:hi] + cids = scid[lo:hi] + m = (np.abs(mzs - mz_i) <= tol_abs) & \ + (np.abs(pmzs - pmz_i) <= mztol) & \ + (cids != c_i) + if m.any(): + if others is None: + others = set() + others.update(int(x) for x in cids[m]) + cc = 0 if others is None else len(others) + n_frags[c_i] += 1 + claim_sum[c_i] += cc + if cc > claim_max[c_i]: + claim_max[c_i] = cc + total_int[c_i] += spint[p] + if cc > 0: + contested_cnt[c_i] += 1 + contested_int[c_i] += spint[p] + group_sets[c_i].update(others) + + log("[conflict] reducing per-candidate features") + with np.errstate(invalid="ignore", divide="ignore"): + nf = n_frags.astype(np.float64) + safe_nf = np.where(nf > 0, nf, 1.0) + claimant_count_mean = claim_sum / safe_nf + contested_frac = contested_cnt / safe_nf + unique_cnt = n_frags - contested_cnt + unique_frac = unique_cnt / safe_nf + safe_int = np.where(total_int > 0, total_int, 1.0) + shared_intensity_frac = contested_int / safe_int + # Candidates with no fragments get zeroed features (documented default). + claimant_count_mean[nf == 0] = 0.0 + contested_frac[nf == 0] = 0.0 + unique_frac[nf == 0] = 0.0 + shared_intensity_frac[total_int == 0] = 0.0 + conflict_group_size = np.array([len(g) for g in group_sets], dtype=np.int64) + + out = { + "candidate_id": uids.astype(np.uint32), + "claimant_count_mean": claimant_count_mean, + "claimant_count_max": claim_max, + "contested_fragment_count": contested_cnt, + "contested_fragment_frac": contested_frac, + "unique_fragment_count": unique_cnt, + "unique_fragment_frac": unique_frac, + "conflict_group_size": conflict_group_size, + "shared_intensity_frac": shared_intensity_frac, + } + + if prelim is not None: + log("[conflict] computing candidate ambiguity (P5.5)") + base_pep = meta["base_pep"] + is_decoy = meta["is_decoy"] + margin_alt = np.full(C, np.nan, dtype=np.float64) + margin_dec = np.full(C, np.nan, dtype=np.float64) + n_comp = np.zeros(C, dtype=np.int64) + comp_entropy = np.zeros(C, dtype=np.float64) + for c in range(C): + grp = sorted(group_sets[c]) # deterministic order + n_comp[c] = len(grp) + if not grp: + continue + self_score = prelim[c] + alt = [prelim[o] for o in grp if base_pep[o] != base_pep[c] + and np.isfinite(prelim[o])] + dec = [prelim[o] for o in grp if is_decoy[o] and np.isfinite(prelim[o])] + if alt: + margin_alt[c] = self_score - max(alt) + if dec: + margin_dec[c] = self_score - max(dec) + scores = [self_score] + [prelim[o] for o in grp] + comp_entropy[c] = softmax_entropy(scores) + out["margin_to_best_alt_peptide"] = margin_alt + out["margin_to_best_decoy"] = margin_dec + out["n_competitors_within_group"] = n_comp + out["competitor_score_entropy"] = comp_entropy + + table = pa.table(out) + pq.write_table(table, args.out) + + # Summary + mean_group = float(conflict_group_size.mean()) if C else 0.0 + mean_contested = float(contested_frac.mean()) if C else 0.0 + dt = time.time() - t0 + log("[conflict] done") + print(f"conflict.parquet written to {args.out}") + print(f"candidates: {C}") + print(f"transitions scanned: {nfrag}") + print(f"mean conflict_group_size: {mean_group:.4f}") + print(f"mean contested_fragment_frac: {mean_contested:.4f}") + if args.max_candidates is not None: + print(f"NOTE: --max-candidates={args.max_candidates} limits both the " + f"query set and the claimant universe; counts are lower bounds.") + print(f"runtime_s: {dt:.1f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/localization.py b/scripts/localization.py new file mode 100644 index 0000000..81bf961 --- /dev/null +++ b/scripts/localization.py @@ -0,0 +1,285 @@ +"""Modification-localization competition features (first pass). + +Implements sensitivity_plan backlog item P6.2 as a non-invasive pass over +existing MuMDIA artifacts. Peptidoforms that share a stripped sequence, a +modification multiset, and a charge but place the modifications on different +sites are localization variants. When such variants co-elute they are a +localization-ambiguity group, and site-determining ions (fragments whose m/z is +unique to one variant) distinguish them. + +The engine is not modified. This is a first-pass ("fix later") implementation: +correctness, bounded memory, and a working smoke test are prioritised over +completeness. Computation is deterministic (stable sorts, sorted iteration; no +RNG). The large chromatogram list columns are read only for the small subset of +candidates that fall in an ambiguity group, which keeps memory bounded. + +Per candidate it writes to `localization.parquet`: + is_localization_ambiguous, n_localization_variants, + site_determining_ion_count, site_determining_ion_intensity, + localization_confidence. + +House style: literal prose, no em-dashes, no hardcoded secrets. +""" + +import argparse +import re +import sys +import time +from collections import defaultdict + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +MOD_RE = re.compile(r"\[[^\]]*\]") +SITE_TOL_PPM = 20.0 # tolerance for calling a fragment m/z "shared" with a sibling + + +def log(msg): + print(msg, file=sys.stderr, flush=True) + + +def parse_peptidoform(pep): + """Parse a ProForma-lite peptidoform into (is_decoy, stripped_sequence, + modification_multiset, variant_signature). + + The multiset is the sorted tuple of modification tokens ignoring position. + The signature is the sorted tuple of (residue_index, token) pairs, which + distinguishes localization variants that share a multiset.""" + is_decoy = pep.startswith("DECOY_") + core = pep[6:] if is_decoy else pep + stripped = [] + mods = [] # (residue_index, token) + i = 0 + res_idx = -1 # index of the most recent residue; -1 == N-terminal + n = len(core) + while i < n: + ch = core[i] + if ch == "[": + j = core.find("]", i) + if j == -1: + # Malformed; treat the rest as sequence and stop mod parsing. + stripped.append(core[i:]) + break + token = core[i + 1:j] + mods.append((res_idx, token)) + i = j + 1 + else: + stripped.append(ch) + res_idx += 1 + i += 1 + stripped_seq = "".join(stripped) + multiset = tuple(sorted(t for _, t in mods)) + signature = tuple(sorted(mods)) + return is_decoy, stripped_seq, multiset, signature + + +def load_fragment_subset(path, wanted_ids, batch_size=1_000_000): + """Stream chrom and collect (frag_mz, observed_apex_intensity) per candidate + for candidates in `wanted_ids`. Observed apex intensity is the maximum of the + per-fragment intensity trace. Bounded memory: only the wanted subset is held.""" + store = defaultdict(list) # candidate_id -> list[(frag_mz, obs_intensity)] + if len(wanted_ids) == 0: + return store + wanted = np.asarray(sorted(wanted_ids), dtype=np.uint32) + pf = pq.ParquetFile(path) + cols = ["candidate_id", "frag_mz", "intensity"] + for batch in pf.iter_batches(columns=cols, batch_size=batch_size): + cid = batch.column("candidate_id").to_numpy(zero_copy_only=False) + keep = np.isin(cid, wanted) + if not keep.any(): + continue + idx = np.nonzero(keep)[0] + frag_mz_col = batch.column("frag_mz") + int_col = batch.column("intensity") + for k in idx.tolist(): + c = int(cid[k]) + mz = float(frag_mz_col[k].as_py()) + ints = int_col[k].as_py() + obs = float(max(ints)) if ints else 0.0 + store[c].append((mz, obs)) + return store + + +def unique_ion_evidence(self_mz, self_int, sibling_mz): + """Count and sum-intensity of self fragments whose m/z does not match any + sibling fragment m/z within SITE_TOL_PPM (site-determining ions).""" + if len(self_mz) == 0: + return 0, 0.0 + if len(sibling_mz) == 0: + # All fragments are unique when there are no sibling fragments. + return len(self_mz), float(np.sum(self_int)) + sib = np.sort(np.asarray(sibling_mz, dtype=np.float64)) + count = 0 + inten = 0.0 + for mz, it in zip(self_mz, self_int): + tol = mz * SITE_TOL_PPM * 1e-6 + pos = np.searchsorted(sib, mz) + matched = False + if pos < len(sib) and abs(sib[pos] - mz) <= tol: + matched = True + elif pos > 0 and abs(sib[pos - 1] - mz) <= tol: + matched = True + if not matched: + count += 1 + inten += it + return count, float(inten) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--psms", required=True) + ap.add_argument("--chrom", required=True) + ap.add_argument("--out", default="localization.parquet") + ap.add_argument("--rt-window-s", type=float, default=30.0) + ap.add_argument("--max-candidates", type=int, default=None, + help="cap the candidate set (for fast smoke tests)") + args = ap.parse_args() + + t0 = time.time() + log(f"[loc] loading psms {args.psms}") + t = pq.read_table(args.psms, + columns=["candidate_id", "apex_rt", "charge", "peptidoform"]) + cid = t.column("candidate_id").to_numpy() + order = np.argsort(cid, kind="stable") + cid = cid[order] + apex_rt = t.column("apex_rt").to_numpy()[order] + charge = t.column("charge").to_numpy()[order] + pep = np.asarray(t.column("peptidoform").to_pylist(), dtype=object)[order] + + if args.max_candidates is not None and args.max_candidates < len(cid): + cid = cid[:args.max_candidates] + apex_rt = apex_rt[:args.max_candidates] + charge = charge[:args.max_candidates] + pep = pep[:args.max_candidates] + + C = len(cid) + log(f"[loc] {C} candidates; grouping by (decoy, stripped_seq, multiset, charge)") + + # Group candidates by (is_decoy, stripped_seq, multiset, charge). + groups = defaultdict(list) # key -> list of local indices + signatures = [None] * C + for i in range(C): + is_decoy, stripped, multiset, sig = parse_peptidoform(str(pep[i])) + signatures[i] = sig + key = (is_decoy, stripped, multiset, int(charge[i])) + groups[key].append(i) + + # Per-candidate outputs (defaults for non-ambiguous candidates). + is_amb = np.zeros(C, dtype=bool) + n_variants = np.ones(C, dtype=np.int64) + # cluster[i] = list of local indices co-eluting with i (including i) + clusters = [None] * C + + rtw = args.rt_window_s + n_amb_groups = 0 + for key, members in groups.items(): + if len(members) < 2: + continue + # Distinct signatures present in the whole key-group. + if len({signatures[m] for m in members}) < 2: + continue # only one localization variant; not ambiguous + m_arr = np.array(members) + rts = apex_rt[m_arr] + sord = np.argsort(rts, kind="stable") + m_sorted = m_arr[sord] + rts_sorted = rts[sord] + group_has_amb = False + for a in range(len(m_sorted)): + i = int(m_sorted[a]) + rt_i = rts_sorted[a] + lo = np.searchsorted(rts_sorted, rt_i - rtw, side="left") + hi = np.searchsorted(rts_sorted, rt_i + rtw, side="right") + cluster_local = [int(m_sorted[b]) for b in range(lo, hi)] + distinct_sigs = {signatures[j] for j in cluster_local} + if len(distinct_sigs) > 1: + is_amb[i] = True + n_variants[i] = len(distinct_sigs) + clusters[i] = cluster_local + group_has_amb = True + if group_has_amb: + n_amb_groups += 1 + + amb_ids = {int(cid[i]) for i in range(C) if is_amb[i]} + log(f"[loc] {len(amb_ids)} candidates in ambiguity groups " + f"across {n_amb_groups} groups") + + sd_count = np.zeros(C, dtype=np.int64) + sd_inten = np.zeros(C, dtype=np.float64) + loc_conf = np.ones(C, dtype=np.float64) # non-ambiguous default: fully localized + + if amb_ids: + log(f"[loc] streaming chrom for {len(amb_ids)} candidates") + frag_store = load_fragment_subset(args.chrom, amb_ids) + cid_to_local = {int(cid[i]): i for i in range(C) if is_amb[i]} + + # Per-candidate self fragment arrays. + self_mz = {} + self_int = {} + for c, lst in frag_store.items(): + if lst: + arr = np.array(lst, dtype=np.float64) + self_mz[c] = arr[:, 0] + self_int[c] = arr[:, 1] + else: + self_mz[c] = np.zeros(0) + self_int[c] = np.zeros(0) + + # Site-determining evidence per ambiguous candidate. + for c, i in cid_to_local.items(): + cluster_local = clusters[i] or [i] + siblings = [j for j in cluster_local + if signatures[j] != signatures[i]] + sib_mz_parts = [] + for j in siblings: + jc = int(cid[j]) + if jc in self_mz and len(self_mz[jc]): + sib_mz_parts.append(self_mz[jc]) + sib_mz = np.concatenate(sib_mz_parts) if sib_mz_parts else np.zeros(0) + smz = self_mz.get(c, np.zeros(0)) + sint = self_int.get(c, np.zeros(0)) + cnt, inten = unique_ion_evidence(smz, sint, sib_mz) + sd_count[i] = cnt + sd_inten[i] = inten + + # Localization confidence = self site-determining intensity over the + # cluster sum. First pass: reuse each member's own site-determining + # intensity (computed against its own siblings). + for c, i in cid_to_local.items(): + cluster_local = clusters[i] or [i] + denom = 0.0 + for j in cluster_local: + denom += sd_inten[j] + if denom > 0: + loc_conf[i] = sd_inten[i] / denom + else: + loc_conf[i] = float("nan") + + out = pa.table({ + "candidate_id": cid.astype(np.uint32), + "is_localization_ambiguous": is_amb, + "n_localization_variants": n_variants, + "site_determining_ion_count": sd_count, + "site_determining_ion_intensity": sd_inten, + "localization_confidence": loc_conf, + }) + pq.write_table(out, args.out) + + n_amb = int(is_amb.sum()) + mean_variants = float(n_variants[is_amb].mean()) if n_amb else 0.0 + dt = time.time() - t0 + log("[loc] done") + print(f"localization.parquet written to {args.out}") + print(f"candidates: {C}") + print(f"ambiguity groups: {n_amb_groups}") + print(f"ambiguous candidates: {n_amb}") + print(f"mean variants per ambiguous candidate: {mean_variants:.4f}") + if args.max_candidates is not None: + print(f"NOTE: --max-candidates={args.max_candidates} limits the " + f"candidate set; ambiguity detection is over the subset only.") + print(f"runtime_s: {dt:.1f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/peak_selection_model.py b/scripts/peak_selection_model.py new file mode 100644 index 0000000..496d97f --- /dev/null +++ b/scripts/peak_selection_model.py @@ -0,0 +1,193 @@ +"""Peak-selection model (sensitivity_plan backlog P1.3, spec 03 §6 peak-selection). + +First-pass, grouped out-of-fold peak scoring over the top-K peak table emitted by +`extract` when `retain_top_peaks > 1` (`.peaks.parquet`). The question is: +given several retained chromatographic peaks for one precursor, which one is the +right one? This trains a simple ranker on per-peak descriptors and reports how +often it puts the correct peak at rank 1 / in the top 3, versus the raw +evidence-count and area rankings. + +Correctness label: + - with --diann (a DIA-NN report giving a reference apex RT per precursor): the + correct peak is the retained peak whose [start_rt, end_rt] contains, or whose + apex is nearest (within --rt-tol-s) to, the reference apex. This is the honest + label. + - without --diann: a WEAK self-label is used (the peak nearest the apex the + engine currently selected, from psms.apex_rt). This only measures whether the + ranker reproduces the current heuristic, not correctness, and is printed with + that caveat. + +Grouping: all peaks of one candidate stay in the same CV fold (group = candidate). +Leakage guards: fit scaling inside the training fold; the label is never a feature. + +Caveat (P1.2 dependency): the peak table carries apex/boundaries/evidence/area but +not a full per-peak feature vector (the engine computes features only for the +selected apex). A production peak-selection model needs per-peak features; this +first pass uses the peak-shape descriptors that are available. + +Usage: + python peak_selection_model.py --peaks .peaks.parquet --psms psms.parquet + [--diann report] [--folds 3] [--rt-tol-s 10] [--out metrics.json] +Requires an env with scikit-learn + pyarrow (py312_mumdia). Deterministic. +""" +import argparse +import hashlib +import json +import re +import sys + +import numpy as np +import pyarrow.parquet as pq + +strip = lambda p: re.sub(r"\[[^\]]*\]", "", str(p)) + + +def fold_of(cid, folds): + h = hashlib.sha1(str(int(cid)).encode()).hexdigest() + return int(h, 16) % folds + + +def load_diann(path): + """Return dict (stripped_seq, charge) -> apex RT in seconds. Defensive columns.""" + t = pq.read_table(path).to_pandas() if path.endswith(".parquet") else None + if t is None: + import pandas as pd + sep = "\t" if path.endswith((".tsv", ".txt")) else "," + t = pd.read_csv(path, sep=sep) + + def pick(*cands): + for c in cands: + if c in t.columns: + return c + return None + + seqc = pick("Modified.Sequence", "ModifiedPeptide", "Stripped.Sequence", "Peptide") + zc = pick("Precursor.Charge", "Charge", "PrecursorCharge") + rtc = pick("RT", "iRT", "Retention.Time", "RT.Start", "Apex.RT") + if not (seqc and zc and rtc): + raise SystemExit(f"peak_selection_model: DIA-NN columns not found in {path}") + rt = t[rtc].to_numpy(dtype=float) + # detect minutes vs seconds by magnitude (gradients are usually > 20 min in s) + if np.nanmax(rt) < 300: + rt = rt * 60.0 + out = {} + for s, z, r in zip(t[seqc].astype(str), t[zc].astype(int), rt): + out[(strip(s), int(z))] = float(r) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--peaks", required=True) + ap.add_argument("--psms", required=True) + ap.add_argument("--diann", default=None) + ap.add_argument("--folds", type=int, default=3) + ap.add_argument("--rt-tol-s", type=float, default=10.0) + ap.add_argument("--out", default=None) + args = ap.parse_args() + + from sklearn.linear_model import LogisticRegression + + pk = pq.read_table(args.peaks).to_pandas() + ps = pq.read_table(args.psms, columns=["candidate_id", "apex_rt", "peptidoform", "charge", "label"]).to_pandas() + sel_rt = dict(zip(ps.candidate_id.astype(int), ps.apex_rt.astype(float))) + key = {int(c): (strip(p), int(z)) for c, p, z in zip(ps.candidate_id, ps.peptidoform, ps.charge)} + + diann = load_diann(args.diann) if args.diann else None + label_kind = "diann-reference" if diann else "weak-self (current apex)" + + # Per-candidate peak groups. + pk = pk.sort_values(["candidate_id", "peak_rank"]).reset_index(drop=True) + feats, labels, groups = [], [], [] + per_cand = {} + n_labeled = 0 + for cid, g in pk.groupby("candidate_id"): + cid = int(cid) + ev = g.evidence_count.to_numpy(float) + ar = g.area.to_numpy(float) + ap = g.apex_rt.to_numpy(float) + width = (g.end_rt.to_numpy(float) - g.start_rt.to_numpy(float)) + # reference RT for the label + ref = None + if diann is not None: + ref = diann.get(key.get(cid)) + else: + ref = sel_rt.get(cid) + if ref is None: + continue + # correct peak = nearest apex within tol (or containing the ref) + d = np.abs(ap - ref) + if d.min() > args.rt_tol_s: + continue # no retained peak matches the reference -> unlabelable here + correct = int(np.argmin(d)) + n_labeled += 1 + mx_ev = ev.max() if ev.max() > 0 else 1.0 + mx_ar = ar.max() if ar.max() > 0 else 1.0 + for j in range(len(g)): + feats.append([ev[j], ar[j], float(g.peak_rank.iloc[j]), width[j], ev[j] / mx_ev, ar[j] / mx_ar]) + labels.append(1 if j == correct else 0) + groups.append(cid) + per_cand[cid] = (ev, ar, correct) + + if n_labeled < 20: + raise SystemExit(f"peak_selection_model: too few labelable candidates ({n_labeled})") + X = np.asarray(feats, float) + y = np.asarray(labels, int) + grp = np.asarray(groups, int) + + # Out-of-fold scores. + oof = np.zeros(len(y)) + folds = np.array([fold_of(c, args.folds) for c in grp]) + for f in range(args.folds): + tr, te = folds != f, folds == f + if tr.sum() == 0 or te.sum() == 0 or len(np.unique(y[tr])) < 2: + continue + mu, sd = X[tr].mean(0), X[tr].std(0) + 1e-9 + m = LogisticRegression(max_iter=2000, C=1.0) + m.fit((X[tr] - mu) / sd, y[tr]) + oof[te] = m.predict_proba((X[te] - mu) / sd)[:, 1] + + # Recall: does the top-scored peak per candidate match the correct one? + def recall(scorer): + top1 = top3 = 0 + i = 0 + by = {} + for c in grp: + by.setdefault(c, []).append(i) + i += 1 + for c, idx in by.items(): + idx = np.array(idx) + corr = np.array([y[k] for k in idx]).argmax() + order = np.argsort(-scorer[idx], kind="stable") + if order[0] == corr: + top1 += 1 + if corr in order[:3]: + top3 += 1 + n = len(by) + return top1 / n, top3 / n + + r_model = recall(oof) + r_ev = recall(X[:, 0]) # evidence_count + r_area = recall(X[:, 1]) # area + res = { + "label_kind": label_kind, + "n_labeled_candidates": n_labeled, + "model_top1": r_model[0], "model_top3": r_model[1], + "evidence_rank_top1": r_ev[0], "evidence_rank_top3": r_ev[1], + "area_rank_top1": r_area[0], "area_rank_top3": r_area[1], + } + print(f"=== peak-selection model ({label_kind}) ===") + print(f" labelable candidates: {n_labeled}") + print(f" learned model : top1={r_model[0]:.3f} top3={r_model[1]:.3f}") + print(f" evidence-rank : top1={r_ev[0]:.3f} top3={r_ev[1]:.3f}") + print(f" area-rank : top1={r_area[0]:.3f} top3={r_area[1]:.3f}") + if not diann: + print(" NOTE: weak self-label (current apex); measures agreement with the " + "existing heuristic, not correctness. Supply --diann for the honest label.") + if args.out: + json.dump(res, open(args.out, "w"), indent=1) + print(f" wrote {args.out}") + + +if __name__ == "__main__": + main() From 58962369deddd734b7937aa328aacc7eba126b48 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 20:30:03 +0200 Subject: [PATCH 27/40] docs(sensitivity): session-3 large-items status (top-K validated, peak-selection, conflict, localization) Co-Authored-By: Claude Opus 4.8 --- sensitivity_plan/IMPLEMENTATION_STATUS.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/sensitivity_plan/IMPLEMENTATION_STATUS.md b/sensitivity_plan/IMPLEMENTATION_STATUS.md index 8b8d705..8fb82bf 100644 --- a/sensitivity_plan/IMPLEMENTATION_STATUS.md +++ b/sensitivity_plan/IMPLEMENTATION_STATUS.md @@ -150,3 +150,39 @@ Branch `feat/sensitivity-improvements`. All Rust changes keep the suite green adaptive_rt_window, competition modes) has an engine-gain measurement yet: each must pass the entrapment-holdout gate on >=2 datasets before being enabled by default (spec 05 §6). That validation loop is the next action, not more code. + +--- + +## Session 3 (large items) — status + +All default-off / sidecar; suite green (95 tests), release builds. Validated on the +real E. coli artifacts. + +### Done +- P1.1/P1.2 top-K peak retention: `extract.retain_top_peaks>1` writes + `.peaks.parquet` (evidence-breadth-ranked peak groups). Validated: K=5 on + E. coli emitted 1,699,995 peaks (mean 4.97/candidate); main psms/chrom flow and + FDR unchanged (accepted 341,754 as at K=1). +- P1.3 peak-selection model `scripts/peak_selection_model.py`: grouped-OOF ranker + over the peaks table. E. coli (weak self-label, 237,328 candidates): learned + top1 0.426 / top3 0.802; evidence-rank 0.418; area-rank 0.390. Evidence beats + area (consistent with the intensity-is-chimeric argument). Honest label needs + `--diann`. +- P3.1 two-pass mass calibration (`search_seed.two_pass_mass_cal`). +- P2.1/P2.2/P2.3/P5.5 `scripts/conflict_features.py`: fragment-claimant index + + peak-group conflict graph + contested/unique/ambiguity features -> conflict.parquet. +- P6.2 `scripts/localization.py`: localization-variant grouping + site-determining + ions -> localization.parquet (0 ambiguity groups on E. coli, as expected). + +### Genuinely remaining (fix later, per user) +- P6.3 staged modification search (calibration stage then extended-mod stage): not + started. +- In-core wiring of the sidecar features (conflict_features / localization) into the + rescorer feature schema: they currently land as joinable Parquet sidecars. Wiring + needs the cross-candidate pass inside `features.rs` (precedent: the cross_charge + extended-extras) + schema-count test updates. +- Top-K peak SELECTION fed back into the reported PSM (the peaks are retained + a + selection model exists, but the engine still reports the heuristic apex; closing + the loop needs per-peak features, P1.2-full, then re-selection before rescore). +- All new default-off knobs still require the entrapment-gate validation on >=2 + datasets before being enabled by default (spec 05 §6). From 6de0b2fd1756db0e033a00ee8071c5886e5f74e3 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Fri, 17 Jul 2026 21:29:37 +0200 Subject: [PATCH 28/40] docs: sync CLAUDE.md + sensitivity_plan derived docs with implemented state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLAUDE.md: pipeline lists `audit`; implementation-status table updated (audit row, top-K peaks / apex_evidence_rank / emit_candidate_audit on extract, two-pass mass cal, adaptive RT window, competition modes, 383-feature 14-family battery); new "Sensitivity program" section; what-needs-doing points at sensitivity_plan/NEXT_STEPS.md. - sensitivity_plan derived docs refreshed (spec files 01-06 untouched): ARCHITECTURE_MAP §1 no longer says top-K is unwired (it is wired: peaks.parquet, validated K=5); NEXT_STEPS reordered around closing the top-K loop + in-core wiring + P6.3 + entrapment-gate validation; BENCHMARK_GUIDE documents all 10 diagnostic scripts + --rank-by count + the TTOF-invalid note; README gains an implementation-status banner. Co-Authored-By: Claude Opus 4.8 --- sensitivity_plan/ARCHITECTURE_MAP.md | 124 ++++++++++++----- sensitivity_plan/BENCHMARK_GUIDE.md | 142 ++++++++++++++++++- sensitivity_plan/NEXT_STEPS.md | 195 +++++++++++++++------------ sensitivity_plan/README.md | 10 ++ 4 files changed, 351 insertions(+), 120 deletions(-) diff --git a/sensitivity_plan/ARCHITECTURE_MAP.md b/sensitivity_plan/ARCHITECTURE_MAP.md index f363ebd..91b979b 100644 --- a/sensitivity_plan/ARCHITECTURE_MAP.md +++ b/sensitivity_plan/ARCHITECTURE_MAP.md @@ -33,21 +33,42 @@ safe dense array index downstream. ## 1. Modules added by the sensitivity lead agent -Committed on `feat/sensitivity-improvements`: `2f46d6d` (rejection reasons, top-K -peak enumerator, config scaffolding), `eb9da89` (candidate `audit` stage + -subcommand), `de5ae2b` (competition modes wired into `compete`). +Committed on `feat/sensitivity-improvements` across 17 commits (`2f46d6d` +rejection reasons + top-K enumerator + config scaffolding, `eb9da89` candidate +`audit` stage + subcommand, `de5ae2b` competition modes, `048cccd` apex-dispersion ++ mass-uncertainty feature families, `4ebf765` evidence-count apex selection, +`a9e3df6` adaptive RT window, `b183fec` top-K peak retention wired into `extract`, +`d61e3fe` two-pass mass calibration, `f6e6a6f` conflict/localization/peak-selection +sidecars). All additions are default-off / K=1-compatible; production defaults are +unchanged. Status of each addition: - `mumdia audit` subcommand: WIRED and verified on real data (P0.3/P0.4). - `CompetitionMode` in `compete`: WIRED (`de5ae2b`); default `WinnerTakeAll` reproduces the legacy behaviour bit-for-bit; `none`/`features_only`/ `unique_evidence`/`margin_gated` are selectable and unit tested. -- `enumerate_peaks` (`mumdia::peaks`): a tested pure helper, NOT yet called from - `extract` (extract still emits one apex per candidate). This is the one - remaining destructive-stage change; the exact hook site is in §4. -- `ExtractConfig.retain_top_peaks` / `emit_candidate_audit`: parsed and validated, - NOT yet consumed by `extract` (the top-K wiring and the in-extract audit sidecar - are the primary next step; see `NEXT_STEPS.md`). +- `enumerate_peaks` (`mumdia::peaks`): WIRED into `extract` (`b183fec`). When + `extract.retain_top_peaks > 1` the extractor enumerates top-K chromatographic + peak groups per candidate and writes `.peaks.parquet`; the default + `retain_top_peaks = 1` bypasses the enumerator and keeps the single-apex path. + Validated: K=5 on E. coli emitted 1,699,995 peak rows (mean ~4.97/candidate) + with the main `psms`/`chrom` flow and target-decoy FDR unchanged. +- `ExtractConfig.retain_top_peaks` / `emit_candidate_audit`: both CONSUMED. + `retain_top_peaks > 1` triggers the top-K peak table above; `emit_candidate_audit` + is read by `run` to write `candidate_audit.parquet` after rescore. +- `ExtractConfig.apex_evidence_rank`: IMPLEMENTED (default off). Selects the apex + by co-eluting fragment breadth (evidence count) rather than signature-ion + intensity. The peak-selection analysis found evidence-rank beats area-rank. +- `RtImTrainConfig.adaptive_rt_window` (+ `adaptive_rt_bins`, `rt_window_min_s`): + IMPLEMENTED (default off). Per-RT-region local residual window, clamped. +- `SearchSeedConfig.two_pass_mass_cal`: IMPLEMENTED (default off). Robust two-pass + precursor + fragment mass calibration. +- Two new feature families `apex_dispersion` (13) and `mass_uncertainty` (10) are + registered in `FAMILIES` (`048cccd`); the registry is now 383 features. +- None of the default-off knobs (top-K, `apex_evidence_rank`, `adaptive_rt_window`, + `two_pass_mass_cal`, competition modes) has passed the entrapment-holdout gate on + >=2 datasets yet, so all remain off by default; that validation is the next + action (see `NEXT_STEPS.md`). ### `mumdia_core::rejection` (`rust/mumdia/crates/mumdia-core/src/rejection.rs`) @@ -90,18 +111,29 @@ Status of each addition: ### New config fields (`rust/mumdia/crates/mumdia-core/src/config.rs`) -- `ExtractConfig.retain_top_peaks: usize` (`config.rs:528`, default 1): K +- `ExtractConfig.retain_top_peaks: usize` (`config.rs:552`, default 1): K chromatographic peak groups per candidate; 1 = legacy single apex. Validated - `>= 1` (`config.rs:917`). -- `ExtractConfig.emit_candidate_audit: bool` (`config.rs:533`, default false): - when true, extraction is to write `.audit.parquet` per-candidate - survivor flags / earliest reason. Near-zero cost when false. -- `CompeteConfig.mode: CompetitionMode` (`config.rs:613`, default `WinnerTakeAll`), - `margin: f64` (`config.rs:616`), `unique_evidence_min_fragments: usize` - (`config.rs:620`), `emit_competition_audit: bool` (`config.rs:623`). -- `CompetitionMode` enum (`config.rs:646`): `WinnerTakeAll` (legacy) / `None` / + `>= 1` (`config.rs:951`). CONSUMED in `extract.rs` (`>1` calls `enumerate_peaks` + at `extract.rs:929` and writes `.peaks.parquet` at `extract.rs:1068`). +- `ExtractConfig.emit_candidate_audit: bool` (`config.rs:557`, default false): + when true, `run` writes `candidate_audit.parquet` after rescore (`run.rs:265`). + Near-zero cost when false. (The finer in-extract `.audit.parquet` sidecar + that `stages::audit::load_extract_reasons` can read is still future work; see + `NEXT_STEPS.md`.) +- `ExtractConfig.apex_evidence_rank: bool` (`config.rs:566`, default false): + when true, the apex loop scores by co-eluting fragment count rather than + signature-ion intensity (`extract.rs:735`). +- `SearchSeedConfig.two_pass_mass_cal: bool` (`config.rs:387`, default false): + robust two-pass precursor + fragment mass calibration in `search_seed.rs:188`. +- `RtImTrainConfig.adaptive_rt_window: bool` (`config.rs:433`, default false) with + `adaptive_rt_bins: usize` (`config.rs:435`, default 12) and `rt_window_min_s: f64` + (`config.rs:437`, default 1.0): per-RT-region local residual window in + `rt_im_train.rs:121`, clamped to `[rt_window_min_s, fallback_rt_window_s]`. +- `CompeteConfig.mode: CompetitionMode` (`config.rs:647`, default `WinnerTakeAll`), + plus `margin`, `unique_evidence_min_fragments`, and `emit_competition_audit`. +- `CompetitionMode` enum (`config.rs:680`): `WinnerTakeAll` (legacy) / `None` / `FeaturesOnly` / `UniqueEvidence` / `MarginGated`, with `from_token` - (`config.rs:667`). Maps to spec 04 §6 strategies A/B/C/D. CONSUMED in + (`config.rs:699`). Maps to spec 04 §6 strategies A/B/C/D. CONSUMED in `compete.rs` via the pure `resolve_competition()` (`de5ae2b`); `WinnerTakeAll` is bit-identical to the previous behaviour. @@ -172,6 +204,8 @@ config knobs that govern it, and existing tests. + `.masscal.json` `{frag_ppm_offset, frag_tol_ppm, n_dev}`. Native Sage-lite hyperscore for calibration only (not a library filter). Drives per-run mass recalibration; fallback `{0.0, cfg.fragment_tol_ppm}` when `n_dev < 20`. +- New opt-in knob (default off): `two_pass_mass_cal` runs a robust two-pass + precursor + fragment mass calibration (`search_seed.rs:188`). - Knobs: `search_seed.fdr_seed` (`config.rs:368`), `fragment_tol_ppm` (370), `min_matched_peaks` (373/374), `report_psms` (371), `matcher` (381). `precursor_tol_ppm` (369) is a dead knob (warns). @@ -187,6 +221,10 @@ config knobs that govern it, and existing tests. Fits predicted_irt -> observed RT (linear always, LOESS when configured), sets a single global RT window half-width `w_rt` applied uniformly to every candidate (no per-candidate uncertainty). Optional DeepLC multitask fine-tune first. +- New opt-in knob (default off): `adaptive_rt_window` (+ `adaptive_rt_bins`, + `rt_window_min_s`) replaces the single global `w_rt` with a per-RT-region local + residual window, clamped to `[rt_window_min_s, fallback_rt_window_s]` + (`rt_im_train.rs:121-134`). - Knobs: `rt_im_train.calibration_method` (`config.rs:401`, `None` rejected), `q_train` (402), `p_rt` / `rt_window_multiplier` (404-405), `min_seed_for_calibration` (406), `loess_span` (408), `fallback_rt_window_s` @@ -210,8 +248,15 @@ config knobs that govern it, and existing tests. `chromatograms.parquet` (7 cols, `LargeListF32` traces, keyed by candidate_id, write `extract.rs:986`). Peak-major over the SoA inverted index; a cheap-to- expensive cascade (distinct-fragment presence -> co-elution run -> matched - fraction -> Pearson gate) accepts candidates. Emits exactly one apex per - candidate (`extract.rs:718`); no peak dimension exists yet. + fraction -> Pearson gate) accepts candidates. The main `psms_extracted` table + still emits exactly one apex per candidate; the peak dimension now exists only in + the opt-in `.peaks.parquet` sidecar (7 cols: `candidate_id, peak_rank, + apex_rt, start_rt, end_rt, evidence_count, area`) written when + `retain_top_peaks > 1` (`extract.rs:1068`). +- New opt-in knobs (default off): `retain_top_peaks > 1` enumerates top-K peak + groups (`extract.rs:927-929`, via `peaks::enumerate_peaks`); `apex_evidence_rank` + scores the apex by fragment breadth instead of intensity (`extract.rs:735`); + `emit_candidate_audit` makes `run` write `candidate_audit.parquet`. - Knobs (all `ExtractConfig`, `config.rs:436-533`): `frag_tol_ppm` (440), `prec_tol_ppm` (441), `presence_min_matched` (443), `presence_min_fragments` (445), `presence_min_coelution` (447), `min_frag_corr` (452/453), @@ -219,8 +264,9 @@ config knobs that govern it, and existing tests. (439), `apex_top_fragments` (465), `apex_rt_prior_s` (469), `apex_count_tol` (474), `apex_count_window` (484), `emit_window_grid` (489), `peak_claim` (498), `emit_contested_features` (503), `peak_claim_margin` (507), `matcher` (509), - `ms1_rescue` (521), plus the new `retain_top_peaks` (528) and - `emit_candidate_audit` (533). Dead: `k_select` (491), `max_fragment_charge` + `ms1_rescue` (521), plus the new `retain_top_peaks` (552), + `emit_candidate_audit` (557) and `apex_evidence_rank` (566). Dead: `k_select` (491), + `max_fragment_charge` (495), `scan_scale` (438), `ScanWindowMode::PeakWidthDerived`. - Tests: none at stage level (exercised only via full `run`). @@ -456,11 +502,17 @@ Exact file:line sites for the six work items, from the stage maps. reason column (`Col::OptI32` exists at `table.rs:33`; only `opt_f64`/`opt_*` readers check `is_null` today), or use `Col::Str`/`Col::Bool`. -### 4.2 Top-K peaks (one apex/candidate today) - -- Primary hook: the single-argmax apex loop `extract.rs:715-733`. Replace with a - local-maxima detector over the `score`/`smoothed` series; `peaks::enumerate_peaks` - (`peaks.rs:52`) is the ready-made pure function. +### 4.2 Top-K peaks + +- DONE (`b183fec`): when `retain_top_peaks > 1`, `extract` enumerates top-K peak + groups with `peaks::enumerate_peaks` (`extract.rs:927-929`) and writes them to + `.peaks.parquet` (`extract.rs:1068`). This is a diagnostic sidecar; the + scored `psms_extracted` path still reports the single heuristic apex. +- Remaining hook (close the loop): the single-argmax apex selection still governs + the reported PSM. Feed a top-K peak back into the scored path via a re-selection + pass (the peak-selection model `scripts/peak_selection_model.py` is the offline + prototype); `peaks::enumerate_peaks` (`peaks.rs:52`) is the ready-made pure + function. - Emission: `CandOut` (`extract.rs:569`), single-row append (`extract.rs:926-958`), `psms_extracted` writer (`extract.rs:960`), chrom writer keyed by candidate_id (`extract.rs:986`, key `:989`). Add a `peak_index`/`peak_rank` column; candidate_id @@ -498,7 +550,9 @@ Exact file:line sites for the six work items, from the stage maps. prelim[*w]` (`compete.rs:95`) with a margin test; `UniqueEvidence` keeps a loser that carries enough independent fragment evidence (needs the claimant graph and a `unique_fragment_count` feature). `group_by` stays orthogonal (equivalence - class); `mode` is the removal policy. Not yet wired. + class); `mode` is the removal policy. WIRED (`de5ae2b`) via the pure + `resolve_competition()`; `UniqueEvidence` currently approximates unique evidence + from the existing features pending the claimant graph (§4.3). ### 4.5 Feature registry @@ -552,10 +606,14 @@ Exact file:line sites for the six work items, from the stage maps. modes now exist (`de5ae2b`): selecting them preserves every candidate so the rescorer, not `prelim_score`, arbitrates. Moving competition to AFTER an initial rescoring pass (spec 04 §11) remains future work. -- One apex per candidate. Spec 01 §3.1 hypothesizes that a single apex is chosen - too early; confirmed: `extract.rs:718` emits exactly one apex PSM per candidate, - no peak dimension. `retain_top_peaks` (`config.rs:528`) and `peaks.rs` are the - answer but are not yet wired into extract. +- One apex per candidate in the SCORED path. Spec 01 §3.1 hypothesizes that a + single apex is chosen too early; confirmed for the default path: the + `psms_extracted` table still emits one apex PSM per candidate. Top-K peak + retention is now wired (`retain_top_peaks > 1` writes `.peaks.parquet` via + `peaks::enumerate_peaks`), but the retained peaks are a diagnostic sidecar; the + loop is not yet closed, i.e. the engine still reports the heuristic apex and does + not re-select a peak from the top-K set before features/compete/rescore. Closing + it needs full per-peak features and a re-selection pass (see `NEXT_STEPS.md`). - Registry metadata is thinner in code than in spec. Spec 03 §2 wants `requires_calibration`, `uses_cross_run_information`, `missing_value_policy`, and `computational_cost` per feature; the code registry (`FAMILIES`) stores only diff --git a/sensitivity_plan/BENCHMARK_GUIDE.md b/sensitivity_plan/BENCHMARK_GUIDE.md index adbd4f1..a69947b 100644 --- a/sensitivity_plan/BENCHMARK_GUIDE.md +++ b/sensitivity_plan/BENCHMARK_GUIDE.md @@ -69,9 +69,15 @@ python scripts/reference_apex_topk.py \ --chrom out_ecoli/chrom.parquet \ [--diann diann_report.tsv] \ [--out out_ecoli/topk_metrics.json] \ - [--max-candidates 20000] [--bound-fraction 0.333] [--rt-tol-s 10] + [--max-candidates 20000] [--bound-fraction 0.333] [--rt-tol-s 10] \ + [--rank-by area|count] ``` +`--rank-by count` (default `area`) ranks the enumerated peaks by co-eluting +fragment breadth (evidence count) instead of integrated area. This mirrors the +engine's `extract.apex_evidence_rank` option and is the ranking the peak-selection +analysis found stronger than area. + Self analysis (no DIA-NN needed) on 20,000 E. coli candidates: | metric | value | @@ -133,7 +139,10 @@ python scripts/entrapment_holdout.py out_ecoli/comp.parquet --q 0.01 Use this as the accept/reject gate for every change (spec 05 §6): a change ships only if held-out entrapment identifications rise without FDP inflation, reproduced on a -second dataset. +second dataset. The on-disk SWATH TTOF file is NOT a valid second dataset here: it +produced 0 IDs against the Ox HYE library (the sample does not match the library). A +genuine second labelled run (a matching TTOF library, or a ProteoBench HYE run) is +still required for the held-out reproduction criterion. ## 5. End-to-end recipe for one experiment (spec 05) @@ -145,6 +154,135 @@ second dataset. 6. Change ONE component (e.g. `compete.mode`, `retain_top_peaks`, a feature family), rerun 1-5, and compare at matched empirical FDP. Keep raw candidate outputs. +## 6. Diagnostic and analysis tool reference + +Every diagnostic script in `scripts/` added by the sensitivity program, with its +one-line purpose and CLI. All are non-invasive (read artifacts, write new files, +never modify the engine or its inputs) and use the `py312_mumdia` interpreter. +The first three are covered in detail above; the rest are documented here. + +### `reference_apex_topk.py` (top-K peak recall) + +Detailed in §2. How often the selected apex is the strongest peak; with `--diann`, +whether the reference apex is within the top-K MuMDIA peaks. + +``` +python scripts/reference_apex_topk.py --psms psms.parquet --chrom chrom.parquet \ + [--diann report.tsv] [--out topk.json] [--rank-by area|count] [--max-candidates N] +``` + +### `feature_ablation.py` (feature-family ablation) + +Detailed in §3. Grouped cross-validated per-family ablation at empirical FDP. + +``` +python scripts/feature_ablation.py --features comp.parquet --registry feature_registry.yaml \ + --out ablation [--model both] [--folds 3] [--fdp 0.01] [--max-rows 60000] +``` + +### `feature_audit.py` (per-feature data-quality audit) + +Per-feature missingness / quantiles / constant detection, target-decoy-entrapment +separation, redundancy clusters, and leakage / intensity warnings (spec 03 §5). + +``` +python scripts/feature_audit.py --features comp.parquet --registry feature_registry.yaml \ + [--out DIR] [--max-rows N] [--entrapment-substr _HUMAN] [--real-substr _ECOLI] \ + [--redundancy-threshold 0.95] +``` + +Finding on E. coli: 355 features audited, 3 constant, 0 leakage-flagged +(decoy-vs-entrapment gap <= 0.043), 54 redundancy clusters. + +### `benchmark_report.py` (self-contained HTML report) + +Assembles the audit waterfall, top-K recall, ablation, and entrapment FDP into one +portable HTML file (spec 02 §8). All inputs optional; each present section renders. + +``` +python scripts/benchmark_report.py --out report.html \ + [--audit-metrics audit.metrics.json] [--audit candidate_audit.parquet] \ + [--topk topk.json] [--ablation ablation] [--entrapment holdout.txt] [--title "..."] +``` + +### `candidate_diagnostics.py` (per-candidate bundle) + +For selected `candidate_id`s, exports overlaid fragment chromatograms +(`fragments.png`), predicted-vs-observed intensities, and a `candidate.json` with +metadata + all features (spec 02 §9). + +``` +python scripts/candidate_diagnostics.py --chrom chrom.parquet --psms psms.parquet \ + [--comp comp.parquet] [--scored scored.parquet] \ + [--candidates 12,34,56 | --candidates-file ids.txt] [--out candidate_diag] +``` + +### `search_space_manifest.py` (P0.1 search-space parity) + +Derives an effective search-space manifest from a MuMDIA library and optionally +compares it to a declared manifest or a DIA-NN report; `--fail-on-mismatch` exits +nonzero on a benchmark-invalidating difference. + +``` +python scripts/search_space_manifest.py --library-precursors lib_precursors.parquet \ + [--out manifest.yaml] [--compare declared.yaml] [--diann report.parquet] \ + [--fail-on-mismatch] +``` + +### `normalize_output.py` (P0.2 common-schema converter) + +Converts a MuMDIA scored table to the spec 02 §4 common schema (`run_id, +precursor_id, stripped_sequence, modified_sequence, charge, ..., q_value, quantity`). + +``` +python scripts/normalize_output.py --scored scored.parquet [--out normalized.parquet] \ + [--run-id ecoli] [--reported-only | --all-candidates] [--q 0.01] +``` + +`--reported-only` on E. coli keeps 9,540 rows (q <= 0.01); default keeps all +candidates. + +### `conflict_features.py` (cross-candidate conflict features) + +Fragment-claimant index + peak-group conflict graph + contested / unique / ambiguity +features (spec 04 §5, P2.1-2.3, P5.5), one row per candidate to `conflict.parquet`, +joinable into rescoring by `candidate_id`. + +``` +python scripts/conflict_features.py --psms psms.parquet --chrom chrom.parquet \ + [--comp comp.parquet] [--out conflict.parquet] [--frag-tol-ppm 20] [--rt-window-s 30] \ + [--max-candidates N] +``` + +### `localization.py` (modification-localization competition) + +Groups localization variants (same stripped sequence + mod multiset + charge, +different sites) and scores site-determining ions (spec 06 P6.2) to +`localization.parquet`. + +``` +python scripts/localization.py --psms psms.parquet --chrom chrom.parquet \ + [--out localization.parquet] [--rt-window-s 30] [--max-candidates N] +``` + +Finding on E. coli: 0 ambiguity groups (as expected for an unmodified search). + +### `peak_selection_model.py` (top-K peak ranker) + +Grouped out-of-fold ranker over the `.peaks.parquet` table emitted by +`extract` when `retain_top_peaks > 1` (spec 03 §6): given several retained peaks for +one precursor, which is correct? Reports top-1 / top-3 accuracy vs raw +evidence-count and area rankings. Honest labels need `--diann`. + +``` +python scripts/peak_selection_model.py --peaks psms.parquet.peaks.parquet \ + --psms psms.parquet [--diann report.tsv] [--folds 3] [--rt-tol-s 10] [--out sel.json] +``` + +Finding on E. coli (weak self-label, 237,328 candidates): learned top1 0.426 / +top3 0.802; evidence-rank 0.418 beats area-rank 0.390, consistent with the argument +that peak intensity is chimeric. + ## Determinism and cost - All Rust stages are deterministic under a fixed seed; `mumdia audit` is a pure join. diff --git a/sensitivity_plan/NEXT_STEPS.md b/sensitivity_plan/NEXT_STEPS.md index f4a3a2c..b551d31 100644 --- a/sensitivity_plan/NEXT_STEPS.md +++ b/sensitivity_plan/NEXT_STEPS.md @@ -1,107 +1,132 @@ -# Sensitivity Program — Next Steps +# Sensitivity Program - Next Steps Prioritized, with exact hook sites (from `ARCHITECTURE_MAP.md`). Items are ordered by expected sensitivity value per unit risk. Everything below builds on the -`feat/sensitivity-improvements` branch. - -## 1. Wire top-K peak retention into extraction (highest value, highest risk) - -The enumerator (`mumdia::peaks::enumerate_peaks`), config (`ExtractConfig.retain_top_peaks`), -and tests exist; extraction still emits one apex per candidate. - -- **Hook:** `extract.rs:718` (single-argmax apex loop) and `CandOut` (`extract.rs:569`), - which today carries one apex + one `Vec`. Multiply it to `Vec` (one - per retained peak) when `retain_top_peaks > 1`. -- **Approach:** after the scan-group build (`extract.rs:608`) and rolling count - (`extract.rs:681`), build the signature-ion summed profile over the grid and call - `enumerate_peaks(profile, k, bound_peak_fraction, prominence)`. For each returned - `PeakGroup`, run the existing apex/chrom emission restricted to `[start_idx, end_idx]`, - stamping a new `peak_rank` column on `psms_extracted` and `chromatograms`. -- **Compatibility:** `K == 1` must bypass the enumerator and keep the current path - (a regression test must show byte-identical `psms.parquet` for K=1 on a fixed input). -- **Downstream:** the features stage and `compete` key currently assume one row per - `candidate_id`. Add `peak_rank` to the competition key or add a peak-selection pass - (below) so multiple peaks of one candidate do not all survive to the report. -- **Validation:** run `scripts/reference_apex_topk.py` before/after; the SELF top-K - distribution predicts the achievable gain. Confirm entrapment FDP is not inflated. - -## 2. In-extract candidate-audit emitter (precise extract-stage reasons) - -`mumdia audit` already reconstructs the ladder from artifacts, but extraction losses -collapse to `NO_PEAK_GROUP`. Emit the precise reason per candidate. +`feat/sensitivity-improvements` branch. The large scaffolding is now in place: +top-K peak retention, competition modes, two feature families, two-pass mass +calibration, and the adaptive RT window are all implemented and default-off. What +remains is closing loops and validating each knob, not building new plumbing. + +## 1. Close the top-K peak loop (highest value, highest risk) + +Peak RETENTION is done: `extract.retain_top_peaks > 1` enumerates top-K peak groups +with `peaks::enumerate_peaks` and writes `.peaks.parquet` +(`extract.rs:927-929`, `:1068`; 7 cols `candidate_id, peak_rank, apex_rt, start_rt, +end_rt, evidence_count, area`). Validated K=5 on E. coli = 1,699,995 peaks +(mean ~4.97/candidate) with the scored path unchanged. The offline peak-selection +prototype `scripts/peak_selection_model.py` learns to rank the retained peaks +(E. coli weak-self-label: learned top1 0.426 / top3 0.802; evidence-rank 0.418 beats +area-rank 0.390). What is NOT done: the engine still reports the single heuristic +apex; the retained peaks are a diagnostic sidecar, not fed back into scoring. + +- **Remaining hook (close the loop):** the retained peaks carry only coarse + descriptors today (apex/start/end RT, evidence count, area). To re-select a peak + before rescore, each retained `PeakGroup` needs the FULL per-peak feature vector. + Emit one `psms_extracted` + `chromatograms` row per `(candidate_id, peak_rank)` + (peak-restricted apex/chrom emission from `CandOut`, `extract.rs:569`), stamp a + `peak_rank` column, and let `features.rs` compute per-peak features (chrom grouping + must key by `(candidate_id, peak_rank)`). +- **Re-selection:** run the peak-selection ranker (port `peak_selection_model.py`) + or a first native pass, keep the winning peak, then collapse to one row before + `compete` (`compete.rs:72` key) and `rescore` (peptide/protein grouping) so + multiple peaks of one candidate do not all reach the report. +- **Compatibility:** `K == 1` must keep the current byte-identical path (regression + test on a fixed input). +- **Validation:** `scripts/reference_apex_topk.py` before/after; confirm the + entrapment FDP is not inflated (item 7). + +## 2. In-core wiring of the conflict / localization sidecar features + +The conflict graph and localization competition exist as non-invasive Python +sidecars: `scripts/conflict_features.py` -> `conflict.parquet` (fragment-claimant +index + peak-group conflict graph + contested/unique/ambiguity features, joinable by +`candidate_id`) and `scripts/localization.py` -> `localization.parquet` +(localization-variant grouping + site-determining ions; 0 ambiguity groups on +E. coli, as expected). They are not yet in the rescorer feature schema. + +- **Hook:** add the cross-candidate pass inside `features.rs` (precedent: the + cross_charge extended-extras block), add the new fields to `Evidence` + (`features.rs:269`), and register a new family module following the + `NAMES` + `values(&Evidence)` contract appended to `FAMILIES` (`features.rs:49`); + dedup / schema-id / PIN flow then handle it automatically. +- **Schema:** bump the family count and update the `feature_sets_sized` test + (`features.rs:1263`) and `feature_registry.yaml`. +- These directly feed `compete.mode = unique_evidence`, which currently approximates + unique evidence from the existing features. -- **Hook:** the per-candidate cascade at `extract.rs:566` returns `Option`; - every `return None` (lines 600, 750-753, 808) is a mapped reason (see the drop - table in `ARCHITECTURE_MAP.md` §3). Change it to return - `Result` (or a small `enum CandEval`), collect via rayon, - and when `extract.emit_candidate_audit` is set write `.audit.parquet` - (candidate_id, rejection_reason). Also diff the library candidate range against the - accumulator keys (`extract.rs:302`) to emit `NO_FRAGMENT_TRACES` for never-materialized - candidates. -- `stages::audit::load_extract_reasons` already reads this sidecar and refines the - waterfall, so no audit-stage change is needed. -- **Cost guard:** only allocate the audit vector when the flag is set. +## 3. P6.3 staged modification search -## 3. Competition after an initial rescoring pass (spec 04 §11) +Not started. A calibration stage (unmodified / common-mod search) followed by an +extended-mod stage that opens the search space only where the calibration stage +found evidence. -`compete` runs before `rescore` on the heuristic `prelim_score` (`run.rs:243` before -`:252`), so a candidate a trained model would keep can be removed early. +- **Prerequisite:** site-determining-ion scoring; `scripts/localization.py` is the + offline prototype for the localization half. +- **Hook:** a second `peptidoforms` -> `predict-frag` -> `extract` pass gated on the + first pass's confident precursors; needs a new orchestration branch in `run.rs` + and a config flag. This is genuinely new plumbing (unlike the items above) and is + the largest remaining feature. -- **Interim (already available):** run with `compete.mode = none` (or `features_only`) - so competition removes nothing and the rescorer arbitrates. Benchmark this against - `winner_take_all` at matched empirical FDP (experiment E14). -- **Full:** add a first-pass rescore (native `percolator_lite` is cheap) that writes an - out-of-fold score, then run `compete` on that score instead of `prelim_score`. Keep - the fold grouping by peptidoform+charge to avoid leakage. +## 4. Entrapment-gate validation of every default-off knob on >=2 datasets -## 4. Fragment claimant / conflict-graph features (spec 04 §5, P2.1-2.3) +Every knob shipped this program is OFF by default because none has passed the +acceptance gate. This is the next ACTION, not more code. -The nucleus exists: the per-peak `claimants` buffer (`extract.rs:304`) and the -two-pass `contested` map (`extract.rs:308`, feature `contested_frac` at -`extract.rs:814`). Extend to a candidate-level family: +- **Knobs to validate:** `extract.retain_top_peaks` (once item 1 closes the loop), + `extract.apex_evidence_rank`, `rt_im_train.adaptive_rt_window`, + `search_seed.two_pass_mass_cal`, and `compete.mode` + (`none`/`features_only`/`unique_evidence`/`margin_gated`). +- **Gate:** `scripts/entrapment_holdout.py` gives a leakage-free held-out entrapment + count on an unseen human null. A knob ships as a default only if held-out + entrapment identifications rise WITHOUT FDP inflation, reproduced on a SECOND + dataset (spec 05 §6). +- **Blocker:** only one valid dataset is loaded here (the E. coli / HYE file). The + on-disk SWATH TTOF file is NOT a valid second dataset: it produced 0 IDs against + the Ox HYE library (the sample does not match the library). A genuine second + labelled run (a matching TTOF library, or a ProteoBench HYE run) is required. -- `claimant_count` / `contested_fragment_count`, `unique_fragment_count`, - `unique_intensity_fraction`, `shared_trace_correlation`, `conflict_group_size`, - `strongest_competitor_score`, `score_margin`. -- **Hook:** compute in the two-pass arbitration (`extract.rs:456-511`) and emit as new - `psms_extracted` columns, or as a new `stages/features/*.rs` family reading the - chromatogram overlaps. Register in `feature_registry.yaml`. Keep target/decoy - computation symmetric (audit per spec 04 §9). -- These directly feed `compete.mode = unique_evidence`, which currently approximates - unique evidence from `n_matched_fragments * (1 - contested_frac)`. +## 5. In-extract candidate-audit emitter (precise extract-stage reasons) -## 5. New feature families with existing data (spec 03 §8, P5) +`emit_candidate_audit` makes `run` write `candidate_audit.parquet` after rescore, but +extraction losses still collapse to `NO_PEAK_GROUP` at artifact resolution. -Lowest risk, additive. Priorities by data availability (see `FEATURE_REGISTRY.md` -gap table): +- **Hook:** the per-candidate cascade at `extract.rs:566` returns `Option`; + every `return None` (the mapped reasons in `ARCHITECTURE_MAP.md` §3) plus the + never-materialized cohort (library candidate range minus accumulator keys, + `extract.rs:302`) should write a `.audit.parquet` (candidate_id, + rejection_reason). `stages::audit::load_extract_reasons` already reads that sidecar + and refines the waterfall, so no audit-stage change is needed. +- **Cost guard:** only allocate the audit vector when the flag is set. + +## 6. Competition after an initial rescoring pass (spec 04 §11) -- **Uncertainty-normalized residuals** (P5.1): `abs(rt_residual)/local_rt_sigma`, - `abs(mass_error)/local_mz_sigma`. Needs local uncertainty from calibration (item 6). -- **Apex dispersion** (P5.3): fragment-apex RT stddev/MAD, precursor-fragment apex - delta. Data already in the chromatograms; compute in a new features family. -- **Candidate ambiguity** (P5.5): margins to alternative peaks/peptides using - `prelim_score` (an earlier-stage score, not the final model, to avoid circularity). +`compete` runs before `rescore` on the heuristic `prelim_score` (`run.rs` order), so +a candidate a trained model would keep can be removed early. The modes are wired; +the reordering is not. -## 6. Two-pass calibration + local uncertainty (spec 03 §5, P3) +- **Interim (available now):** run with `compete.mode = none` (or `features_only`) so + competition removes nothing and the rescorer arbitrates. Benchmark against + `winner_take_all` at matched empirical FDP (validate per item 4). +- **Full:** add a first-pass native `percolator_lite` rescore that writes an + out-of-fold score, then run `compete` on that score instead of `prelim_score`. Keep + the fold grouping by peptidoform+charge to avoid leakage. -`rt_im_train` (`rt_im_train.rs`) fits per-run RT calibration; mass calibration is in -`search_seed` (`masscal.json`). Add: robust two-pass precursor+fragment mass -calibration, a monotonic nonlinear RT map, and a LOCAL uncertainty estimate exported -per region. The uncertainty unlocks item 5's normalized residuals and adaptive -extraction windows (`window = max(min, scale * local_sigma)`), spec P3.3. +## 7. Local calibration uncertainty (finish spec 03 §5 / P3) -## 7. Empirical-FDP-first evaluation loop +Two-pass mass calibration (`search_seed.two_pass_mass_cal`) and the adaptive RT +window (`rt_im_train.adaptive_rt_window`) are done. The remaining piece is exporting a +LOCAL per-region uncertainty estimate so features can normalize residuals +(`abs(rt_residual)/local_rt_sigma`, `abs(mass_error)/local_mz_sigma`). -`scripts/entrapment_holdout.py` already gives a leakage-free held-out entrapment count -on an unseen human null. Make it the acceptance gate for every change above (spec 05 -§6): a change ships only if held-out entrapment identifications rise without FDP -inflation, reproduced on a second dataset. The single E. coli/HYE file here is one -dataset; a second (the TTOF SWATH file, or a ProteoBench HYE run) is needed for the -spec's held-out reproduction criterion. +- **Hook:** capture the residual distribution into `cal.json` (`rt_im_train.rs`) and + `masscal.json` (`search_seed.rs`); add a `pred_sigma` field to `Evidence` + (`features.rs:269`) fed by a new library/chromatogram column. The `mass_uncertainty` + and `apex_dispersion` families are the consumers already in place. ## Cannot be completed in this environment -- Held-out reproduction on a second dataset (needs a second labelled run loaded). +- Held-out reproduction on a second dataset (needs a valid second labelled run; the + on-disk TTOF file is the wrong sample for the Ox HYE library). - Reference-apex recall vs DIA-NN (needs a DIA-NN report; `scripts/reference_apex_topk.py` - computes it once `--diann` is supplied). + and `scripts/peak_selection_model.py` compute it once `--diann` is supplied). - Ion-mobility / diaPASEF families (no 4D data). diff --git a/sensitivity_plan/README.md b/sensitivity_plan/README.md index dc2a29c..99b789d 100644 --- a/sensitivity_plan/README.md +++ b/sensitivity_plan/README.md @@ -8,6 +8,16 @@ The central principle is: A DIA-NN-only identification can disappear because it was absent from the search space, never generated as a candidate, extracted incorrectly, assigned to the wrong chromatographic peak, outcompeted by another peptide interpretation, ranked poorly, or removed by false-discovery-rate filtering. +## Implementation status + +The spec files in this directory (01-06) are the requirements and stay fixed. The +derived docs track what is built: `IMPLEMENTATION_STATUS.md` is the source of truth +for progress (candidate audit + `mumdia audit`, top-K peak retention, competition +modes, two-pass mass calibration, adaptive RT window, two new feature families, and +ten diagnostic scripts are all implemented, default-off). `ARCHITECTURE_MAP.md` maps +the code, `BENCHMARK_GUIDE.md` documents the diagnostics, and `NEXT_STEPS.md` lists +the remaining work with exact hook sites. + ## Documentation map 1. [`01_workflow_and_gap_analysis.md`](01_workflow_and_gap_analysis.md) From 10e80992720649654316e32dfac5c76074f11676 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Sat, 18 Jul 2026 14:58:55 +0200 Subject: [PATCH 29/40] feat(sensitivity): co-elution correlation as the extraction gate (extract.gate_coelution) The min_frag_corr gate compared observed-vs-predicted fragment intensities at the single apex scan, which chimeric DIA interference inflates. gate_coelution=true instead thresholds min_frag_corr on the predicted-intensity-weighted mean Pearson correlation of each matched fragment's XIC to the signature-ion reference profile over the elution window (the OpenSWATH/DIA-NN co-elution discriminator): an interferent that only coincides at the apex does not co-elute with the peptide's own fragments. Default false (legacy single-scan Pearson). 3 unit tests. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 11 ++ .../crates/mumdia/src/stages/extract.rs | 113 ++++++++++++++++-- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 32d1013..1c7ab2f 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -564,6 +564,16 @@ pub struct ExtractConfig { /// keeps the legacy signature-intensity apex. The rolling distinct-fragment /// count (`apex_count_window`) still gates which scans qualify in both modes. pub apex_evidence_rank: bool, + /// Gate on CO-ELUTION correlation instead of the single-scan apex intensity + /// Pearson. The `min_frag_corr` threshold is then applied to the + /// predicted-intensity-weighted mean correlation of each matched fragment's XIC + /// to the signature-ion reference profile over the elution window, rather than + /// to the observed-vs-predicted intensity pattern at the single apex scan. In + /// chimeric DIA an interferent inflates the apex-scan pattern but does not + /// co-elute with the peptide's own fragments, so co-elution is a stronger, + /// interference-robust acceptance signal (the OpenSWATH/DIA-NN discriminator). + /// `false` (default) keeps the legacy single-scan Pearson gate. + pub gate_coelution: bool, } impl Default for ExtractConfig { fn default() -> Self { @@ -599,6 +609,7 @@ impl Default for ExtractConfig { retain_top_peaks: 1, // legacy single-apex behaviour (K=1) emit_candidate_audit: false, // diagnostic; off in production apex_evidence_rank: false, // legacy signature-intensity apex + gate_coelution: false, // legacy single-scan intensity Pearson gate } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index 2d4dd3d..c2ceb72 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -117,6 +117,49 @@ fn sum_near(mz: &[f64], inten: &[f32], target: f64, tol_ppm: f64) -> f32 { acc } +/// Co-elution acceptance score (sensitivity program): predicted-intensity-weighted +/// mean Pearson correlation of each matched fragment's XIC to the signature-ion +/// reference profile, over the elution scan groups. High when the peptide's own +/// fragments co-elute (real); low when a matched fragment is a non-co-eluting +/// interferent that only coincides at the apex. More robust to chimeric DIA +/// interference than the single-scan apex intensity Pearson. Returns 1.0 (do not +/// reject) when there are too few scan groups or no reference signal. +fn coelution_gate_score( + groups: &[(f64, std::collections::BTreeMap)], + distinct: &[u16], + sig: &[u16], + fints0: &[f32], +) -> f64 { + if groups.len() < 3 { + return 1.0; + } + let refp: Vec = groups + .iter() + .map(|(_, m)| sig.iter().map(|o| *m.get(o).unwrap_or(&0.0) as f64).sum::()) + .collect(); + if refp.iter().all(|x| *x <= 0.0) { + return 1.0; + } + let (mut wsum, mut wtot) = (0.0f64, 0.0f64); + for &f in distinct { + let tr: Vec = groups + .iter() + .map(|(_, m)| *m.get(&f).unwrap_or(&0.0) as f64) + .collect(); + if tr.iter().any(|x| *x > 0.0) { + let c = crate::stats::pearson(&tr, &refp).max(0.0); + let w = *fints0.get(f as usize).unwrap_or(&0.0) as f64 + 1e-9; + wsum += c * w; + wtot += w; + } + } + if wtot > 0.0 { + wsum / wtot + } else { + 1.0 + } +} + /// fragindex non-two-pass accumulation over isolation-window groups, in parallel. /// Each scan belongs to exactly one window, so the groups are independent; the /// per-candidate hit lists are merged by concatenating in (window-sorted) group @@ -816,18 +859,26 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { .find(|(rt, _)| (*rt - apex_rt).abs() < 1e-9) .map(|(_, m)| m); if p.cfg.min_frag_corr > 0.0 { - if let Some(map) = apex_map { + // Acceptance score: either the CO-ELUTION correlation over the elution + // window (interference-robust) or the legacy single-scan intensity + // Pearson at the apex, per `gate_coelution`. + let gate_score = if p.cfg.gate_coelution { + coelution_gate_score(&groups, &distinct, &sig, &fints0) + } else if let Some(map) = apex_map { let obs: Vec = (0..fmzs0.len()) .map(|k| *map.get(&(k as u16)).unwrap_or(&0.0) as f64) .collect(); let pred: Vec = fints0.iter().map(|x| *x as f64).collect(); - if crate::stats::pearson(&obs, &pred) < p.cfg.min_frag_corr { - let rescued = p.cfg.ms1_rescue - && ms1_support - && distinct.len() >= p.cfg.presence_min_fragments.max(1); - if !rescued { - return None; - } + crate::stats::pearson(&obs, &pred) + } else { + 1.0 // no apex scan resolved; do not reject on spectral agreement + }; + if gate_score < p.cfg.min_frag_corr { + let rescued = p.cfg.ms1_rescue + && ms1_support + && distinct.len() >= p.cfg.presence_min_fragments.max(1); + if !rescued { + return None; } } } @@ -1117,3 +1168,49 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ); Ok((n_psms, n_chrom)) } + +#[cfg(test)] +mod coelution_tests { + use super::coelution_gate_score; + use std::collections::BTreeMap; + + fn g(rows: &[(f64, &[(u16, f32)])]) -> Vec<(f64, BTreeMap)> { + rows.iter() + .map(|(rt, fs)| (*rt, fs.iter().cloned().collect())) + .collect() + } + + #[test] + fn coeluting_fragments_score_high() { + // frags 0,1,2 all peak together at group index 2 + let groups = g(&[ + (0.0, &[(0, 1.0), (1, 1.0), (2, 1.0)]), + (1.0, &[(0, 4.0), (1, 3.0), (2, 2.0)]), + (2.0, &[(0, 9.0), (1, 8.0), (2, 5.0)]), + (3.0, &[(0, 4.0), (1, 3.0), (2, 2.0)]), + (4.0, &[(0, 1.0), (1, 1.0), (2, 1.0)]), + ]); + let s = coelution_gate_score(&groups, &[0, 1, 2], &[0, 1], &[10.0, 8.0, 5.0]); + assert!(s > 0.95, "co-eluting fragments should score high, got {s}"); + } + + #[test] + fn non_coeluting_interferent_drops_the_score() { + // frags 0,1 co-elute; frag 2 (a strong-predicted interferent) sits off-peak + let groups = g(&[ + (0.0, &[(0, 1.0), (1, 1.0), (2, 9.0)]), + (1.0, &[(0, 4.0), (1, 3.0), (2, 0.0)]), + (2.0, &[(0, 9.0), (1, 8.0), (2, 0.0)]), + (3.0, &[(0, 4.0), (1, 3.0), (2, 0.0)]), + (4.0, &[(0, 1.0), (1, 1.0), (2, 0.0)]), + ]); + let s = coelution_gate_score(&groups, &[0, 1, 2], &[0, 1], &[10.0, 8.0, 9.0]); + assert!(s < 0.8, "a strong non-co-eluting interferent should lower the score, got {s}"); + } + + #[test] + fn too_few_scans_does_not_reject() { + let groups = g(&[(0.0, &[(0, 5.0)]), (1.0, &[(0, 9.0)])]); + assert_eq!(coelution_gate_score(&groups, &[0], &[0], &[10.0]), 1.0); + } +} From b2fa749ae9189e076ba5a4353308aa170ade8f9f Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Sat, 18 Jul 2026 15:08:50 +0200 Subject: [PATCH 30/40] fix(sensitivity): restrict co-elution gate score to the elution peak, not the full window The first co-elution-gate cut computed the correlation over the entire (up to ~1177 s) extraction window, which is mostly zeros with a narrow peak -> noisy, unstable correlation that over-rejected real peptides (-2028 on E. coli). Now the score is computed only over the contiguous scans around the signature-reference apex above 10% of its height (the actual elution peak), where co-elution is meaningful. Co-Authored-By: Claude Opus 4.8 --- .../crates/mumdia/src/stages/extract.rs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index c2ceb72..0f9da54 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -133,21 +133,42 @@ fn coelution_gate_score( if groups.len() < 3 { return 1.0; } + // Signature-ion reference profile per scan group. let refp: Vec = groups .iter() .map(|(_, m)| sig.iter().map(|o| *m.get(o).unwrap_or(&0.0) as f64).sum::()) .collect(); - if refp.iter().all(|x| *x <= 0.0) { + let (apex, apex_v) = refp + .iter() + .enumerate() + .fold((0usize, 0.0f64), |(bi, bv), (i, v)| if *v > bv { (i, *v) } else { (bi, bv) }); + if apex_v <= 0.0 { return 1.0; } + // Restrict the correlation to the PEAK: the contiguous scans around the + // reference apex above 10% of its height. Over the full (wide) extraction + // window the traces are mostly zeros and the correlation is noise; co-elution + // is only meaningful across the elution peak itself. + let thr = 0.1 * apex_v; + let (mut lo, mut hi) = (apex, apex); + while lo > 0 && refp[lo - 1] >= thr { + lo -= 1; + } + while hi + 1 < refp.len() && refp[hi + 1] >= thr { + hi += 1; + } + if hi - lo + 1 < 3 { + return 1.0; // peak too narrow to assess co-elution; do not reject + } + let refw = &refp[lo..=hi]; let (mut wsum, mut wtot) = (0.0f64, 0.0f64); for &f in distinct { - let tr: Vec = groups + let tr: Vec = groups[lo..=hi] .iter() .map(|(_, m)| *m.get(&f).unwrap_or(&0.0) as f64) .collect(); if tr.iter().any(|x| *x > 0.0) { - let c = crate::stats::pearson(&tr, &refp).max(0.0); + let c = crate::stats::pearson(&tr, refw).max(0.0); let w = *fints0.get(f as usize).unwrap_or(&0.0) as f64 + 1e-9; wsum += c * w; wtot += w; From 6fa59bb6386ebd438c5426bb313824034c61e1aa Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Mon, 20 Jul 2026 14:39:14 +0200 Subject: [PATCH 31/40] feat(sensitivity): gate modes, PyTorch NN rescorer, parallel peak-claim, soft competition All additions default-off so the production chain stays byte-identical. Gate: - GateMode enum (ApexPearson default / PeakSpectral / SpectralEntropy / Coelution / Combined) replacing the gate_coelution bool; min_frag_corr thresholds the active mode. peak_window + peak_spectral_score helpers; spectral_entropy_similarity_sqrt exposed from features::entropy (shared kernel, no duplication). - emit_gate_diagnostics knob: the four gate_* diagnostic columns (and their extra per-candidate compute) only when set. Investigation found apex Pearson is the best single gate metric; peak-integrated/entropy/coelution regress at matched pool. Rescorer: - RescorerKind::NnTorch: PyTorch semi-supervised MLP sidecar (scripts/ nn_rescore_worker.py) over the shared positional-CLI PIN contract; run_mokapot refactored to run_pin_sidecar(script). doctor probes torch when selected. - Streaming memmap backend in the worker (auto >4GB PIN, or MUMDIA_NN_STREAM=1) so multi-run experiment-wide rescoring never loads the full PIN into RAM. - scripts/nn_semisupervised_rescore.ipynb: standalone PIN -> NN rescore notebook. Extraction: - extract_twopass_windows: parallelize the two-pass co-elution peak-claim across isolation windows (~4x faster, far lower memory; within-window arbitration). - --restrict-candidates allowlist for gate-first-then-compete; routes non-coelution claim strategies through the restrict-aware single-pass. - Soft competition features (default-off behind emit_contested_features): peak_contested_count_frac + peak_apportioned_frac from a Contested stats struct (won/lost intensity + counts + co-elution apportioned share). Extended 379 -> 381. 81 lib tests green. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 64 +- rust/mumdia/crates/mumdia/src/main.rs | 17 +- .../crates/mumdia/src/stages/extract.rs | 681 +++++++++++++----- .../crates/mumdia/src/stages/features.rs | 20 +- .../mumdia/src/stages/features/entropy.rs | 12 + .../crates/mumdia/src/stages/rescore.rs | 34 +- rust/mumdia/crates/mumdia/src/stages/run.rs | 1 + rust/mumdia/crates/mumdia/tests/pipeline.rs | 1 + scripts/nn_rescore_worker.py | 281 ++++++++ scripts/nn_semisupervised_rescore.ipynb | 567 +++++++++++++++ 10 files changed, 1469 insertions(+), 209 deletions(-) create mode 100644 scripts/nn_rescore_worker.py create mode 100644 scripts/nn_semisupervised_rescore.ipynb diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 1c7ab2f..26592a7 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -161,6 +161,13 @@ pub enum RescorerKind { NativeTda, /// Mokapot Python sidecar (PLAN.md Section 0). Mokapot, + /// PyTorch semi-supervised MLP sidecar (`nn_rescore_worker.py`): a nonlinear + /// Percolator/mokapot-style rescorer (CV folds + iterative positive + /// re-selection). On the E.coli benchmark it beats the linear mokapot model on + /// the same PIN, and — being robust to an unfiltered pool — gains further when + /// the extraction gate is opened. Same positional-CLI PIN contract as Mokapot; + /// requires `rescore.python` to point at an interpreter with torch. + NnTorch, /// External percolator.exe over the PIN file. Percolator, /// Spike-in (entrapment) negative rescorer: treat foreign-proteome PSMs @@ -564,16 +571,20 @@ pub struct ExtractConfig { /// keeps the legacy signature-intensity apex. The rolling distinct-fragment /// count (`apex_count_window`) still gates which scans qualify in both modes. pub apex_evidence_rank: bool, - /// Gate on CO-ELUTION correlation instead of the single-scan apex intensity - /// Pearson. The `min_frag_corr` threshold is then applied to the - /// predicted-intensity-weighted mean correlation of each matched fragment's XIC - /// to the signature-ion reference profile over the elution window, rather than - /// to the observed-vs-predicted intensity pattern at the single apex scan. In - /// chimeric DIA an interferent inflates the apex-scan pattern but does not - /// co-elute with the peptide's own fragments, so co-elution is a stronger, - /// interference-robust acceptance signal (the OpenSWATH/DIA-NN discriminator). - /// `false` (default) keeps the legacy single-scan Pearson gate. - pub gate_coelution: bool, + /// Emit the four gate-diagnostic scores (`gate_apex`, `gate_peak_spectral`, + /// `gate_coelution`, `gate_spectral_entropy`) as extra `psms.parquet` columns, + /// for the offline gate-metric comparison. Default `false` (diagnostic sidecar, + /// like `emit_candidate_audit`): when off, neither the columns nor the extra + /// per-candidate score computation happen, so the default chain is byte-identical. + pub emit_gate_diagnostics: bool, + /// Which spectral-agreement score the `min_frag_corr` gate thresholds + /// (sensitivity program). The legacy gate uses a single apex-scan intensity + /// Pearson, which one chimeric scan can dominate. See [`GateMode`]. + pub gate_mode: GateMode, + /// Second threshold for `GateMode::Combined`: the co-elution score must exceed + /// this while the peak-integrated spectral score exceeds `min_frag_corr`. + /// Requiring BOTH is more specific (rejects interferents that pass one axis). + pub gate_coelution_min: f64, } impl Default for ExtractConfig { fn default() -> Self { @@ -609,11 +620,42 @@ impl Default for ExtractConfig { retain_top_peaks: 1, // legacy single-apex behaviour (K=1) emit_candidate_audit: false, // diagnostic; off in production apex_evidence_rank: false, // legacy signature-intensity apex - gate_coelution: false, // legacy single-scan intensity Pearson gate + emit_gate_diagnostics: false, // diagnostic gate-score columns; off in production + gate_mode: GateMode::ApexPearson, // legacy single-scan intensity Pearson + gate_coelution_min: 0.5, // used only by GateMode::Combined } } } +/// Spectral-agreement score the extraction acceptance gate (`min_frag_corr`) +/// thresholds. All are computed at the gate from data already in hand. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum GateMode { + /// Legacy: Pearson of observed-vs-predicted fragment intensities at the single + /// apex scan. One chimeric scan can dominate it. + #[default] + ApexPearson, + /// Pearson of the PEAK-INTEGRATED observed spectrum (each fragment summed over + /// the elution-peak scans) vs predicted intensities. Averages out a single + /// interfered scan; the standard library-dot-product measure. + PeakSpectral, + /// Li spectral-entropy similarity of the sqrt-transformed apex-scan observed vs + /// predicted intensities (`spectral_entropy_similarity_sqrt`). The full-feature + /// gate search (all ~379 features, target-vs-decoy) found this the single best + /// gate discriminator: AUC 0.826 / matched-pool recall 69.8%, versus apex + /// Pearson's 0.781 / 64.5%. Same inputs as `ApexPearson`, better separation. + SpectralEntropy, + /// Predicted-intensity-weighted mean CO-ELUTION correlation of each matched + /// fragment's XIC to the signature reference over the elution peak (temporal + /// agreement, orthogonal to intensity agreement). + Coelution, + /// Require BOTH: peak-integrated spectral Pearson >= `min_frag_corr` AND the + /// co-elution score >= `gate_coelution_min`. More specific (an interferent + /// passing one axis is still rejected), for a cleaner FDR pool. + Combined, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct FeaturesConfig { diff --git a/rust/mumdia/crates/mumdia/src/main.rs b/rust/mumdia/crates/mumdia/src/main.rs index 0ad691b..fdfd265 100644 --- a/rust/mumdia/crates/mumdia/src/main.rs +++ b/rust/mumdia/crates/mumdia/src/main.rs @@ -109,6 +109,12 @@ enum Cmd { out_psms: String, #[arg(long)] out_chrom: String, + /// Optional candidate allowlist (a prior run's psms.parquet): restrict + /// extraction to these candidate_ids. For "gate first, then compete" - + /// re-extract with a peak_claim strategy over only the gate-accepted + /// survivors, keeping the two-pass profile map small. + #[arg(long)] + restrict_candidates: Option, #[arg(long)] config: Option, }, @@ -276,9 +282,16 @@ enum Cmd { /// Probe each configured sidecar interpreter for its required packages, so a /// broken or missing environment is reported clearly instead of failing mid-run. fn doctor(cfg: &Config) -> Result<()> { + use mumdia_core::config::RescorerKind; use std::process::Command; + // The rescore sidecar's required packages depend on the selected classifier: + // the PyTorch NN needs torch; mokapot/entrapment need mokapot + sklearn. + let (rescore_label, rescore_pkgs) = match cfg.rescore.classifier { + RescorerKind::NnTorch => ("rescore.python (nn_torch)", "torch,numpy,pandas,pyarrow"), + _ => ("rescore.python (mokapot)", "mokapot,sklearn,numpy,pandas,pyarrow"), + }; let checks = [ - ("rescore.python (mokapot)", cfg.rescore.python.as_deref(), "mokapot,sklearn,numpy,pandas,pyarrow"), + (rescore_label, cfg.rescore.python.as_deref(), rescore_pkgs), ("predict_frag.deeplc_python (DeepLC)", cfg.predict_frag.deeplc_python.as_deref(), "deeplc,numpy,pandas"), ("predict_frag.ms2pip_python (MS2PIP)", cfg.predict_frag.ms2pip_python.as_deref(), "ms2pip,numpy,pandas"), ]; @@ -437,6 +450,7 @@ fn main() -> Result<()> { mass_cal, out_psms, out_chrom, + restrict_candidates, config, } => { let cfg = load_config(&config)?; @@ -450,6 +464,7 @@ fn main() -> Result<()> { mass_cal: mass_cal.as_deref(), out_psms: &out_psms, out_chrom: &out_chrom, + restrict_candidates: restrict_candidates.as_deref(), cfg: &cfg.extract, config_hash: &ch, })?; diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index 0f9da54..a29e433 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -16,7 +16,7 @@ use std::collections::{BTreeMap, HashMap}; use std::time::Instant; use anyhow::Result; -use mumdia_core::config::{ExtractConfig, PeakClaim}; +use mumdia_core::config::{ExtractConfig, GateMode, PeakClaim}; use mumdia_core::schema::artifact; use mumdia_io::report::ArtifactReport; use mumdia_io::table::{write_table, Col, Table}; @@ -71,6 +71,12 @@ pub struct ExtractParams<'a> { pub mass_cal: Option<&'a str>, pub out_psms: &'a str, pub out_chrom: &'a str, + /// Optional candidate allowlist (a prior run's `psms.parquet`): restrict + /// extraction to these `candidate_id`s. Used for "gate first, then compete": + /// run a cheap gate-on pass, then re-extract with a peak-claim strategy over + /// only the accepted survivors, so the expensive two-pass profile map is built + /// over ~10^5 candidates instead of ~10^7. + pub restrict_candidates: Option<&'a str>, pub cfg: &'a ExtractConfig, pub config_hash: &'a str, } @@ -84,6 +90,23 @@ struct Hit { obs_mz: f64, } +/// Per-candidate contested-peak statistics from the co-elution arbitration +/// (two-pass path). `won`/`lost` are the summed observed intensity of shared peaks +/// this candidate won (was the most-eluting claimant) or lost to a better +/// co-eluter; `n_won`/`n_lost` are the corresponding peak-instance counts; and +/// `apportioned` is the candidate's co-elution-weighted proportional share of the +/// contested intensity (what it would keep under `CoelutionProportional`). These +/// feed the soft competition features (`contested_frac`, `contested_count_frac`, +/// `apportioned_frac`) without removing any candidate. +#[derive(Default, Clone, Copy)] +struct Contested { + won: f64, + lost: f64, + n_won: u32, + n_lost: u32, + apportioned: f64, +} + /// Index of the value in ascending `rts` nearest to `t` (binary search). fn nearest_index(rts: &[f64], t: f64) -> usize { if rts.is_empty() { @@ -124,16 +147,19 @@ fn sum_near(mz: &[f64], inten: &[f32], target: f64, tol_ppm: f64) -> f32 { /// interferent that only coincides at the apex. More robust to chimeric DIA /// interference than the single-scan apex intensity Pearson. Returns 1.0 (do not /// reject) when there are too few scan groups or no reference signal. -fn coelution_gate_score( +/// Contiguous elution-peak scan indices `[lo, hi]` around the signature-ion apex +/// (scans above 10% of the reference apex height) plus the reference profile. +/// `None` when there are too few scans, no reference signal, or a < 3-scan peak. +/// Over the full (wide) extraction window the traces are mostly zeros and any +/// correlation is noise; the spectral/co-elution gates are only meaningful across +/// the elution peak itself. +fn peak_window( groups: &[(f64, std::collections::BTreeMap)], - distinct: &[u16], sig: &[u16], - fints0: &[f32], -) -> f64 { +) -> Option<(usize, usize, Vec)> { if groups.len() < 3 { - return 1.0; + return None; } - // Signature-ion reference profile per scan group. let refp: Vec = groups .iter() .map(|(_, m)| sig.iter().map(|o| *m.get(o).unwrap_or(&0.0) as f64).sum::()) @@ -143,12 +169,8 @@ fn coelution_gate_score( .enumerate() .fold((0usize, 0.0f64), |(bi, bv), (i, v)| if *v > bv { (i, *v) } else { (bi, bv) }); if apex_v <= 0.0 { - return 1.0; + return None; } - // Restrict the correlation to the PEAK: the contiguous scans around the - // reference apex above 10% of its height. Over the full (wide) extraction - // window the traces are mostly zeros and the correlation is noise; co-elution - // is only meaningful across the elution peak itself. let thr = 0.1 * apex_v; let (mut lo, mut hi) = (apex, apex); while lo > 0 && refp[lo - 1] >= thr { @@ -158,8 +180,51 @@ fn coelution_gate_score( hi += 1; } if hi - lo + 1 < 3 { - return 1.0; // peak too narrow to assess co-elution; do not reject + return None; } + Some((lo, hi, refp)) +} + +/// Peak-integrated spectral Pearson: correlate the PEAK-SUMMED observed spectrum +/// (each predicted fragment integrated over the elution-peak scans) with the +/// predicted intensities. Averaging over the peak removes the single-interfered- +/// scan fragility of the apex-only Pearson. Returns 1.0 when no peak is resolved. +fn peak_spectral_score( + groups: &[(f64, std::collections::BTreeMap)], + sig: &[u16], + fints0: &[f32], +) -> f64 { + let (lo, hi, _refp) = match peak_window(groups, sig) { + Some(w) => w, + None => return 1.0, + }; + let obs: Vec = (0..fints0.len()) + .map(|f| { + groups[lo..=hi] + .iter() + .map(|(_, m)| *m.get(&(f as u16)).unwrap_or(&0.0) as f64) + .sum::() + }) + .collect(); + let pred: Vec = fints0.iter().map(|x| *x as f64).collect(); + crate::stats::pearson(&obs, &pred) +} + +/// Co-elution acceptance score (temporal): predicted-intensity-weighted mean +/// Pearson correlation of each matched fragment's XIC to the signature-ion +/// reference profile, over the elution peak. High when the peptide's own fragments +/// co-elute; low when a matched fragment only coincides at the apex. Orthogonal to +/// the intensity-agreement of `peak_spectral_score`. +fn coelution_gate_score( + groups: &[(f64, std::collections::BTreeMap)], + distinct: &[u16], + sig: &[u16], + fints0: &[f32], +) -> f64 { + let (lo, hi, refp) = match peak_window(groups, sig) { + Some(w) => w, + None => return 1.0, + }; let refw = &refp[lo..=hi]; let (mut wsum, mut wtot) = (0.0f64, 0.0f64); for &f in distinct { @@ -283,10 +348,222 @@ fn extract_accumulate_windows( acc } +/// Parallel two-pass co-elution peak-claim. Each isolation-window group is +/// processed independently (a candidate's precursor m/z places it in one window, +/// and a peak's claimants come only from that window via `candidate_range`), so +/// both the base accumulation (pass 1, for elution profiles) and the arbitration +/// (pass 2) fan out across the ~150 windows. Returns the (possibly reassigned) +/// accumulation and per-candidate (won, lost) contested intensity. Mirrors the +/// serial two-pass exactly, window-partitioned; merge is disjoint across windows +/// (extend/sum is overlap-safe if windows ever overlap in m/z). +#[allow(clippy::too_many_arguments)] +fn extract_twopass_windows( + idx: Option<&FragIndex>, + lib: &Library, + scans: &[Ms2Scan], + rt_lo: &[f64], + rt_hi: &[f64], + offset_factor: f64, + frag_tol: f64, + cfg: &ExtractConfig, + restrict: Option<&std::collections::HashSet>, + reassign: bool, + claim_margin: f32, +) -> (HashMap>, HashMap) { + use std::collections::BTreeMap; + let mut groups: BTreeMap<(u64, u64), Vec> = BTreeMap::new(); + for (si, scan) in scans.iter().enumerate() { + groups + .entry((scan.window.lower_mz.to_bits(), scan.window.upper_mz.to_bits())) + .or_default() + .push(si); + } + let group_vec: Vec> = groups.into_values().collect(); + + type Part = (Vec<(u32, Vec)>, Vec<(u32, Contested)>); + let partials: Vec = group_vec + .par_iter() + .map(|ids| { + if ids.is_empty() { + return (Vec::new(), Vec::new()); + } + let w = &scans[ids[0]].window; + let (lo, hi) = lib.candidate_range(w.lower_mz, w.upper_mz); + if hi <= lo { + return (Vec::new(), Vec::new()); + } + let mut claimants: Vec<(u32, u16, f32)> = Vec::new(); + // PASS 1: base accumulation (full peak intensity) for elution profiles. + let mut acc1: HashMap> = HashMap::new(); + for &si in ids { + let scan = &scans[si]; + let rt = scan.rt_seconds; + for peak in &scan.peaks { + let inten = peak.intensity; + let q_mz = peak.mz / offset_factor; + let obs_mz = peak.mz; + claimants.clear(); + { + let mut push = |cid: u32, frag: u16, pi: f32| { + let c = cid as usize; + if rt < rt_lo[c] || rt > rt_hi[c] { + return; + } + if let Some(s) = restrict { + if !s.contains(&cid) { + return; + } + } + claimants.push((cid, frag, pi)); + }; + probe_matched(idx, lib, frag_tol, q_mz, lo, hi, &mut push); + } + for &(cid, frag, _) in &claimants { + acc1.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); + } + } + } + let mut profile: HashMap> = HashMap::new(); + for (cid, hits) in &acc1 { + let m = profile.entry(*cid).or_default(); + for h in hits { + *m.entry(h.rt.to_bits()).or_insert(0.0) += h.inten; + } + } + // PASS 2: arbitrate each shared peak by which claimant is most eluting. + let mut acc2: HashMap> = HashMap::new(); + let mut contested: HashMap = HashMap::new(); + for &si in ids { + let scan = &scans[si]; + let rt = scan.rt_seconds; + let rtb = rt.to_bits(); + for peak in &scan.peaks { + let inten = peak.intensity; + let q_mz = peak.mz / offset_factor; + let obs_mz = peak.mz; + claimants.clear(); + { + let mut push = |cid: u32, frag: u16, pi: f32| { + let c = cid as usize; + if rt < rt_lo[c] || rt > rt_hi[c] { + return; + } + if let Some(s) = restrict { + if !s.contains(&cid) { + return; + } + } + claimants.push((cid, frag, pi)); + }; + probe_matched(idx, lib, frag_tol, q_mz, lo, hi, &mut push); + } + if claimants.is_empty() { + continue; + } + let ph = |cid: u32| -> f32 { + profile.get(&cid).and_then(|m| m.get(&rtb)).copied().unwrap_or(0.0) + }; + let mut best = 0usize; + for i in 1..claimants.len() { + let (ci, _, pii) = claimants[i]; + let (cb, _, pib) = claimants[best]; + let (hi_, hb) = (ph(ci), ph(cb)); + if hi_ > hb || (hi_ == hb && (pii > pib || (pii == pib && ci < cb))) { + best = i; + } + } + let win = claimants[best].0; + let sum_ph: f32 = claimants.iter().map(|c| ph(c.0)).sum(); + let top_ph = ph(win); + let second_ph = claimants + .iter() + .filter(|c| c.0 != win) + .map(|c| ph(c.0)) + .fold(0.0f32, f32::max); + let dominant = top_ph > 0.0 && (second_ph <= 0.0 || top_ph >= claim_margin * second_ph); + for &(cid, frag, _pi) in &claimants { + let e = contested.entry(cid).or_default(); + // Co-elution-weighted proportional share (retained intensity + // under CoelutionProportional), tracked for every claimant. + let share = if sum_ph > 0.0 { + inten * (ph(cid) / sum_ph) + } else { + inten / claimants.len() as f32 + }; + e.apportioned += share as f64; + if cid == win { + e.won += inten as f64; + e.n_won += 1; + } else { + e.lost += inten as f64; + e.n_lost += 1; + } + if reassign { + match cfg.peak_claim { + PeakClaim::CoelutionWinner => { + if cid == win { + acc2.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); + } + } + PeakClaim::CoelutionProportional => { + let share = if sum_ph > 0.0 { + inten * (ph(cid) / sum_ph) + } else { + inten / claimants.len() as f32 + }; + acc2.entry(cid).or_default().push(Hit { rt, frag, inten: share, obs_mz }); + } + PeakClaim::CoelutionWinnerMargin => { + if !dominant || cid == win { + acc2.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); + } + } + _ => {} + } + } + } + } + } + let out_acc = if reassign { acc2 } else { acc1 }; + (out_acc.into_iter().collect(), contested.into_iter().collect()) + }) + .collect(); + + let mut acc: HashMap> = HashMap::new(); + let mut contested: HashMap = HashMap::new(); + for (a, c) in partials { + for (cid, hits) in a { + acc.entry(cid).or_default().extend(hits); + } + for (cid, s) in c { + let e = contested.entry(cid).or_default(); + e.won += s.won; + e.lost += s.lost; + e.n_won += s.n_won; + e.n_lost += s.n_lost; + e.apportioned += s.apportioned; + } + } + (acc, contested) +} + pub fn run(p: ExtractParams) -> Result<(u64, u64)> { let t0 = Instant::now(); let lib = Library::load(p.library_precursors, p.library_fragments, p.cfg.bucket_size)?; + // Optional candidate allowlist (gate-first-then-compete): restrict extraction to + // the accepted survivors of a prior gate-on run so the two-pass peak-claim profile + // map stays small. + let restrict: Option> = match p.restrict_candidates { + Some(path) => { + let t = Table::read(path)?; + let s: std::collections::HashSet = t.u32("candidate_id")?.into_iter().collect(); + info!(restrict_candidates = s.len(), "extract: restricting to candidate allowlist"); + Some(s) + } + None => None, + }; + // run windows indexed by candidate_id let rw = Table::read(p.run_windows)?; let rw_cid = rw.u32("candidate_id")?; @@ -366,10 +643,9 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { let mut acc: HashMap> = HashMap::new(); // Reused per-peak buffer of (candidate_id, local_frag_index, predicted_intensity). let mut claimants: Vec<(u32, u16, f32)> = Vec::new(); - // Per-candidate (won, lost) peak intensity under the co-elution arbitration, - // for the non-destructive `contested_frac` feature. Populated only on the - // two-pass path. - let mut contested: HashMap = HashMap::new(); + // Per-candidate contested-peak stats under the co-elution arbitration, for the + // non-destructive soft competition features. Populated only on the two-pass path. + let mut contested: HashMap = HashMap::new(); // The two co-elution strategies and the contested feature need a first pass to // build per-candidate elution profiles before shared peaks can be arbitrated. let two_pass = matches!( @@ -381,9 +657,12 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { let claim_margin = p.cfg.peak_claim_margin as f32; if !two_pass { - if let Some(idx) = fidx.as_ref() { + if let (Some(idx), true) = (fidx.as_ref(), restrict.is_none()) { // Parallel across isolation-window groups (bit-identical to serial: the - // cascade rt-sorts each candidate's hits before summing). + // cascade rt-sorts each candidate's hits before summing). Only when there + // is no candidate allowlist; a `restrict` list routes to the serial path + // below, which applies the allowlist filter and honors every peak_claim + // strategy (Winner/Proportional/None). acc = extract_accumulate_windows(idx, &scans, &rt_lo, &rt_hi, offset_factor, p.cfg); } else { for scan in &scans { @@ -405,6 +684,11 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { if rt < rt_lo[c] || rt > rt_hi[c] { return; } + if let Some(s) = &restrict { + if !s.contains(&cid) { + return; + } + } claimants.push((cid, frag, pi)); }; probe_matched(fidx.as_ref(), &lib, frag_tol, q_mz, lo, hi, &mut push); @@ -448,136 +732,30 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } } } else { - // PASS 1: base (None) accumulation to build honest elution profiles. - for scan in &scans { - let (lo, hi) = lib.candidate_range(scan.window.lower_mz, scan.window.upper_mz); - if hi <= lo { - continue; - } - let rt = scan.rt_seconds; - for peak in &scan.peaks { - let inten = peak.intensity; - let q_mz = peak.mz / offset_factor; - let obs_mz = peak.mz; - claimants.clear(); - { - let mut push = |cid: u32, frag: u16, pi: f32| { - let c = cid as usize; - if rt < rt_lo[c] || rt > rt_hi[c] { - return; - } - claimants.push((cid, frag, pi)); - }; - probe_matched(fidx.as_ref(), &lib, frag_tol, q_mz, lo, hi, &mut push); - } - for &(cid, frag, _) in &claimants { - acc.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); - } - } - } - // Per-candidate per-scan elution profile: summed matched intensity at each RT. - let mut profile: HashMap> = HashMap::new(); - for (cid, hits) in &acc { - let m = profile.entry(*cid).or_default(); - for h in hits { - *m.entry(h.rt.to_bits()).or_insert(0.0) += h.inten; - } - } + // Two-pass co-elution peak-claim, parallelized across isolation windows + // (each window's candidates interact only within it, so the two expensive + // probing passes fan out over the ~150 windows). let reassign = matches!( p.cfg.peak_claim, PeakClaim::CoelutionWinner | PeakClaim::CoelutionProportional | PeakClaim::CoelutionWinnerMargin ); - // PASS 2: arbitrate each shared peak by which claimant is most eluting at - // this scan (profile height), and record won/lost intensity per candidate. - let mut acc2: HashMap> = HashMap::new(); - for scan in &scans { - let (lo, hi) = lib.candidate_range(scan.window.lower_mz, scan.window.upper_mz); - if hi <= lo { - continue; - } - let rt = scan.rt_seconds; - let rtb = rt.to_bits(); - for peak in &scan.peaks { - let inten = peak.intensity; - let q_mz = peak.mz / offset_factor; - let obs_mz = peak.mz; - claimants.clear(); - { - let mut push = |cid: u32, frag: u16, pi: f32| { - let c = cid as usize; - if rt < rt_lo[c] || rt > rt_hi[c] { - return; - } - claimants.push((cid, frag, pi)); - }; - probe_matched(fidx.as_ref(), &lib, frag_tol, q_mz, lo, hi, &mut push); - } - if claimants.is_empty() { - continue; - } - let ph = |cid: u32| -> f32 { - profile.get(&cid).and_then(|m| m.get(&rtb)).copied().unwrap_or(0.0) - }; - // winner: most eluting at this scan; ties -> higher predicted int -> lower cid. - let mut best = 0usize; - for i in 1..claimants.len() { - let (ci, _, pii) = claimants[i]; - let (cb, _, pib) = claimants[best]; - let (hi_, hb) = (ph(ci), ph(cb)); - if hi_ > hb || (hi_ == hb && (pii > pib || (pii == pib && ci < cb))) { - best = i; - } - } - let win = claimants[best].0; - let sum_ph: f32 = claimants.iter().map(|c| ph(c.0)).sum(); - // Margin gate: does the top eluter clearly dominate the runner-up? - let top_ph = ph(win); - let second_ph = claimants - .iter() - .filter(|c| c.0 != win) - .map(|c| ph(c.0)) - .fold(0.0f32, f32::max); - let dominant = top_ph > 0.0 && (second_ph <= 0.0 || top_ph >= claim_margin * second_ph); - for &(cid, frag, _pi) in &claimants { - let e = contested.entry(cid).or_insert((0.0, 0.0)); - if cid == win { - e.0 += inten as f64; - } else { - e.1 += inten as f64; - } - if reassign { - match p.cfg.peak_claim { - PeakClaim::CoelutionWinner => { - if cid == win { - acc2.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); - } - } - PeakClaim::CoelutionProportional => { - let share = if sum_ph > 0.0 { - inten * (ph(cid) / sum_ph) - } else { - inten / claimants.len() as f32 - }; - acc2.entry(cid).or_default().push(Hit { rt, frag, inten: share, obs_mz }); - } - PeakClaim::CoelutionWinnerMargin => { - // Claim only when the top eluter dominates; else keep - // the peak shared (give every claimant the full peak). - if !dominant || cid == win { - acc2.entry(cid).or_default().push(Hit { rt, frag, inten, obs_mz }); - } - } - _ => {} - } - } - } - } - } - if reassign { - acc = acc2; - } + let (a, c) = extract_twopass_windows( + fidx.as_ref(), + &lib, + &scans, + &rt_lo, + &rt_hi, + offset_factor, + frag_tol, + p.cfg, + restrict.as_ref(), + reassign, + claim_margin, + ); + acc = a; + contested = c; } info!(materialized = acc.len(), "extract: candidates with evidence"); @@ -599,6 +777,8 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { // Fraction of this candidate's matched intensity that a co-eluting competitor // claims more strongly (co-elution arbitration); 0 when the two-pass path is off. let mut contested_c: Vec = Vec::new(); + // Richer soft-competition columns, emitted only with emit_contested_features. + let (mut contested_count_c, mut apportioned_c): (Vec, Vec) = (Vec::new(), Vec::new()); // MS1 apex isotope intensities (null when no MS1 provided). let (mut ms1_m1, mut ms1_mono, mut ms1_i1, mut ms1_i2): ( Vec>, @@ -606,6 +786,13 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { Vec>, Vec>, ) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + // Gate diagnostic scores (per accepted candidate; see CandOut). + let (mut gate_apex_c, mut gate_peakspec_c, mut gate_coel_c, mut gate_se_c): ( + Vec, + Vec, + Vec, + Vec, + ) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); // chromatograms columns let (mut ch_cid, mut ch_name, mut ch_fmz, mut ch_pint) = @@ -640,6 +827,8 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { calrt: f64, mz: f64, contested: f64, + contested_count_frac: f64, + apportioned_frac: f64, z: i32, label: String, base: u32, @@ -650,6 +839,15 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ms1_mono: Option, ms1_i1: Option, ms1_i2: Option, + /// Gate diagnostic scores, computed for EVERY accepted candidate regardless + /// of `gate_mode` (sensitivity program): the single-apex-scan intensity + /// Pearson, the peak-integrated spectral Pearson, and the temporal co-elution + /// score. Emitted so an offline analysis can compare gate metrics (and their + /// combination) at matched pool size, without re-extraction. + gate_apex: f32, + gate_peak_spectral: f32, + gate_coelution: f32, + gate_spectral_entropy: f32, /// (cid, frag_name, frag_mz, frag_obs_mz, predicted_intensity, rt, intensity) chrom: Vec<(u32, String, f64, f64, f32, Vec, Vec)>, /// Top-K retained peak groups (sensitivity_plan P1.1/P1.2), populated only @@ -879,22 +1077,49 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { .iter() .find(|(rt, _)| (*rt - apex_rt).abs() < 1e-9) .map(|(_, m)| m); + // Spectral-agreement score closures, evaluated lazily: the acceptance gate + // needs only the ACTIVE `gate_mode`'s score, and the four diagnostic scores + // are computed only when `emit_gate_diagnostics` is set (see below), so the + // default chain pays the same per-candidate cost as before this feature. + let apex_obs: Option> = apex_map.map(|map| { + (0..fmzs0.len()) + .map(|k| *map.get(&(k as u16)).unwrap_or(&0.0) as f64) + .collect() + }); + let pred_f64: Vec = fints0.iter().map(|x| *x as f64).collect(); + // Single-apex-scan intensity Pearson (1.0 when no apex scan resolved -> do + // not reject on spectral agreement). + let apex_pearson = || match &apex_obs { + Some(obs) => crate::stats::pearson(obs, &pred_f64), + None => 1.0, + }; + // spectral_entropy_similarity_sqrt of the apex spectrum (shared kernel in + // features::entropy; best single target/decoy gate discriminator). + let apex_entropy = || match &apex_obs { + Some(obs) => { + crate::stages::features::entropy::spectral_entropy_similarity_sqrt(obs, &pred_f64) + } + None => 1.0, + }; + let peak_spec = || peak_spectral_score(&groups, &sig, &fints0); + let coel = || coelution_gate_score(&groups, &distinct, &sig, &fints0); + if p.cfg.min_frag_corr > 0.0 { - // Acceptance score: either the CO-ELUTION correlation over the elution - // window (interference-robust) or the legacy single-scan intensity - // Pearson at the apex, per `gate_coelution`. - let gate_score = if p.cfg.gate_coelution { - coelution_gate_score(&groups, &distinct, &sig, &fints0) - } else if let Some(map) = apex_map { - let obs: Vec = (0..fmzs0.len()) - .map(|k| *map.get(&(k as u16)).unwrap_or(&0.0) as f64) - .collect(); - let pred: Vec = fints0.iter().map(|x| *x as f64).collect(); - crate::stats::pearson(&obs, &pred) - } else { - 1.0 // no apex scan resolved; do not reject on spectral agreement + // Acceptance gate. `min_frag_corr` thresholds the ACTIVE gate_mode's + // spectral-agreement score (plan Section 9): the legacy single-apex-scan + // Pearson (one chimeric scan can dominate), the peak-integrated spectral + // Pearson, the apex spectral-entropy similarity, the temporal co-elution + // score, or Combined (both, more specific). Only the active score computes. + let rejected = match p.cfg.gate_mode { + GateMode::ApexPearson => apex_pearson() < p.cfg.min_frag_corr, + GateMode::PeakSpectral => peak_spec() < p.cfg.min_frag_corr, + GateMode::SpectralEntropy => apex_entropy() < p.cfg.min_frag_corr, + GateMode::Coelution => coel() < p.cfg.min_frag_corr, + GateMode::Combined => { + peak_spec() < p.cfg.min_frag_corr || coel() < p.cfg.gate_coelution_min + } }; - if gate_score < p.cfg.min_frag_corr { + if rejected { let rescued = p.cfg.ms1_rescue && ms1_support && distinct.len() >= p.cfg.presence_min_fragments.max(1); @@ -904,9 +1129,36 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } } + // Diagnostic scores (all four metrics, for the offline gate-metric + // comparison). Computed and emitted ONLY when `emit_gate_diagnostics` is set, + // so the default psms.parquet schema and per-candidate compute are unchanged + // (sensitivity-program: default-off, byte-identical). Zero when off (the four + // columns are not written). + let (gate_apex, gate_peak_spectral, gate_coelution, gate_spectral_entropy) = + if p.cfg.emit_gate_diagnostics { + (apex_pearson(), peak_spec(), coel(), apex_entropy()) + } else { + (0.0, 0.0, 0.0, 0.0) + }; + + // Soft competition features from the co-elution arbitration (all 0 when the + // two-pass path did not run). contested_frac: fraction of contested INTENSITY + // lost to better co-eluters. contested_count_frac: fraction of contested + // fragment-PEAKS lost. apportioned_frac: fraction of contested intensity the + // candidate retains under proportional apportionment (1 = keeps all, ~0 = a + // peak-borrower stripped by its co-eluting competitors). + let cst = contested.get(&cid).copied().unwrap_or_default(); let contested_val = { - let (w, l) = contested.get(&cid).copied().unwrap_or((0.0, 0.0)); - if w + l > 0.0 { l / (w + l) } else { 0.0 } + let t = cst.won + cst.lost; + if t > 0.0 { cst.lost / t } else { 0.0 } + }; + let contested_count_frac = { + let n = cst.n_won + cst.n_lost; + if n > 0 { cst.n_lost as f64 / n as f64 } else { 0.0 } + }; + let apportioned_frac = { + let t = cst.won + cst.lost; + if t > 0.0 { cst.apportioned / t } else { 0.0 } }; // Per-fragment intensity-weighted observed m/z (for mass accuracy). @@ -1025,6 +1277,8 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { calrt: rt_cal[cid as usize], mz: c.precursor_mz, contested: contested_val, + contested_count_frac, + apportioned_frac, z: c.charge, label: if c.is_decoy { "decoy" } else { "target" }.to_string(), base: c.base_peptide_id, @@ -1035,6 +1289,10 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ms1_mono: o_ms1_mono, ms1_i1: o_ms1_i1, ms1_i2: o_ms1_i2, + gate_apex: gate_apex as f32, + gate_peak_spectral: gate_peak_spectral as f32, + gate_coelution: gate_coelution as f32, + gate_spectral_entropy: gate_spectral_entropy as f32, chrom: chrom_rows, peaks, }) @@ -1071,6 +1329,10 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { calrt_c.push(r.calrt); mz_c.push(r.mz); contested_c.push(r.contested); + if p.cfg.emit_contested_features { + contested_count_c.push(r.contested_count_frac); + apportioned_c.push(r.apportioned_frac); + } z_c.push(r.z); label_c.push(r.label); base_c.push(r.base); @@ -1081,6 +1343,12 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { ms1_mono.push(r.ms1_mono); ms1_i1.push(r.ms1_i1); ms1_i2.push(r.ms1_i2); + if p.cfg.emit_gate_diagnostics { + gate_apex_c.push(r.gate_apex); + gate_peakspec_c.push(r.gate_peak_spectral); + gate_coel_c.push(r.gate_coelution); + gate_se_c.push(r.gate_spectral_entropy); + } for (cc, nm, fmz, omz, pint, rt, it) in r.chrom { ch_cid.push(cc); ch_name.push(nm); @@ -1092,31 +1360,43 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { } } - let n_psms = write_table( - p.out_psms, - vec![ - Col::U32("candidate_id".into(), cid_c), - Col::F64("apex_rt".into(), apexrt_c), - Col::OptF64("apex_im".into(), apexim_c), - Col::F32("apex_intensity".into(), apexint_c), - Col::I32("n_matched_fragments".into(), nmatch_c), - Col::I32("n_predicted_fragments".into(), npred_c), - Col::I32("coelution_run".into(), corun_c), - Col::F64("rt_pred_cal".into(), calrt_c), - Col::F64("precursor_mz".into(), mz_c), - Col::I32("charge".into(), z_c), - Col::Str("label".into(), label_c), - Col::U32("base_peptide_id".into(), base_c), - Col::Str("peptidoform".into(), pform_c), - Col::Str("protein".into(), prot_c), - Col::F32("predicted_irt".into(), irt_c), - Col::F64("contested_frac".into(), contested_c), - Col::OptF64("ms1_isom1".into(), ms1_m1), - Col::OptF64("ms1_mono".into(), ms1_mono), - Col::OptF64("ms1_iso1".into(), ms1_i1), - Col::OptF64("ms1_iso2".into(), ms1_i2), - ], - )?; + let mut psms_cols = vec![ + Col::U32("candidate_id".into(), cid_c), + Col::F64("apex_rt".into(), apexrt_c), + Col::OptF64("apex_im".into(), apexim_c), + Col::F32("apex_intensity".into(), apexint_c), + Col::I32("n_matched_fragments".into(), nmatch_c), + Col::I32("n_predicted_fragments".into(), npred_c), + Col::I32("coelution_run".into(), corun_c), + Col::F64("rt_pred_cal".into(), calrt_c), + Col::F64("precursor_mz".into(), mz_c), + Col::I32("charge".into(), z_c), + Col::Str("label".into(), label_c), + Col::U32("base_peptide_id".into(), base_c), + Col::Str("peptidoform".into(), pform_c), + Col::Str("protein".into(), prot_c), + Col::F32("predicted_irt".into(), irt_c), + Col::F64("contested_frac".into(), contested_c), + Col::OptF64("ms1_isom1".into(), ms1_m1), + Col::OptF64("ms1_mono".into(), ms1_mono), + Col::OptF64("ms1_iso1".into(), ms1_i1), + Col::OptF64("ms1_iso2".into(), ms1_i2), + ]; + // Richer soft-competition columns only when emit_contested_features (default-off + // keeps the schema byte-identical; contested_frac above is the pre-existing one). + if p.cfg.emit_contested_features { + psms_cols.push(Col::F64("contested_count_frac".into(), contested_count_c)); + psms_cols.push(Col::F64("apportioned_frac".into(), apportioned_c)); + } + // Diagnostic gate-score columns only when enabled (default-off keeps the schema + // byte-identical to the production chain). + if p.cfg.emit_gate_diagnostics { + psms_cols.push(Col::F32("gate_apex".into(), gate_apex_c)); + psms_cols.push(Col::F32("gate_peak_spectral".into(), gate_peakspec_c)); + psms_cols.push(Col::F32("gate_coelution".into(), gate_coel_c)); + psms_cols.push(Col::F32("gate_spectral_entropy".into(), gate_se_c)); + } + let n_psms = write_table(p.out_psms, psms_cols)?; let n_chrom = write_table( p.out_chrom, @@ -1192,7 +1472,7 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { #[cfg(test)] mod coelution_tests { - use super::coelution_gate_score; + use super::{coelution_gate_score, peak_spectral_score}; use std::collections::BTreeMap; fn g(rows: &[(f64, &[(u16, f32)])]) -> Vec<(f64, BTreeMap)> { @@ -1234,4 +1514,35 @@ mod coelution_tests { let groups = g(&[(0.0, &[(0, 5.0)]), (1.0, &[(0, 9.0)])]); assert_eq!(coelution_gate_score(&groups, &[0], &[0], &[10.0]), 1.0); } + + #[test] + fn peak_spectral_high_when_integrated_pattern_matches() { + // observed peak-summed spectrum (9:8:5 at apex, tails scale) matches predicted + let groups = g(&[ + (0.0, &[(0, 1.0), (1, 1.0), (2, 1.0)]), + (1.0, &[(0, 4.0), (1, 3.0), (2, 2.0)]), + (2.0, &[(0, 9.0), (1, 8.0), (2, 5.0)]), + (3.0, &[(0, 4.0), (1, 3.0), (2, 2.0)]), + (4.0, &[(0, 1.0), (1, 1.0), (2, 1.0)]), + ]); + let s = peak_spectral_score(&groups, &[0, 1], &[19.0, 16.0, 11.0]); + assert!(s > 0.99, "integrated pattern matches predicted, got {s}"); + } + + #[test] + fn peak_spectral_recovers_fragment_absent_at_apex_scan() { + // A real strong-predicted fragment (2) is momentarily unsampled at the apex + // scan (DIA scan gap) but present across the rest of the peak. The single-scan + // apex Pearson would see obs=0 for it and collapse; integrating over the peak + // recovers its true contribution and matches the predicted 19:16:16. + let groups = g(&[ + (0.0, &[(0, 1.0), (1, 1.0), (2, 2.0)]), + (1.0, &[(0, 4.0), (1, 3.0), (2, 6.0)]), + (2.0, &[(0, 9.0), (1, 8.0), (2, 0.0)]), // frag 2 unsampled at apex scan + (3.0, &[(0, 4.0), (1, 3.0), (2, 6.0)]), + (4.0, &[(0, 1.0), (1, 1.0), (2, 2.0)]), + ]); + let s = peak_spectral_score(&groups, &[0, 1], &[19.0, 16.0, 16.0]); + assert!(s > 0.99, "peak integration should recover the off-apex fragment, got {s}"); + } } diff --git a/rust/mumdia/crates/mumdia/src/stages/features.rs b/rust/mumdia/crates/mumdia/src/stages/features.rs index ca3509e..322dc53 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features.rs @@ -31,7 +31,7 @@ use rayon::prelude::*; mod apex_dispersion; mod chromatographic; mod coelution; -mod entropy; +pub(crate) mod entropy; mod interference; mod mass_uncertainty; mod ion_series; @@ -209,8 +209,12 @@ pub fn active_features(set: FeatureSet) -> Vec { } if matches!(set, FeatureSet::Extended) { v.extend(extended_names()); - // psms-derived (not an Evidence family): the co-elution peak-contest metric. + // psms-derived (not an Evidence family): the co-elution peak-contest metrics. + // A peak-borrowing decoy loses most contested intensity/fragments to the real + // co-eluting peptide, so these three separate borrowers from genuine IDs. v.push("peak_contested_frac".to_string()); + v.push("peak_contested_count_frac".to_string()); + v.push("peak_apportioned_frac".to_string()); // Cross-candidate charge-state corroboration (aggregated across the charge // states of one peptidoform, not visible to the per-PSM Evidence families): // a real peptide co-occurs at multiple charges more than a shift decoy. @@ -511,6 +515,10 @@ pub fn run(p: FeaturesParams) -> Result { let protein = ps.str("protein")?; let mz = ps.f64("precursor_mz")?; let contested = ps.f64("contested_frac").unwrap_or_else(|_| vec![0.0; ps.nrows]); + // Richer soft-competition columns (present only with emit_contested_features; + // default to 0 so the feature vector length is stable when absent). + let contested_count = ps.f64("contested_count_frac").unwrap_or_else(|_| vec![0.0; ps.nrows]); + let apportioned = ps.f64("apportioned_frac").unwrap_or_else(|_| vec![0.0; ps.nrows]); let ms1_m1 = ps.opt_f64("ms1_isom1").unwrap_or_else(|_| vec![None; ps.nrows]); let ms1_mono = ps.opt_f64("ms1_mono").unwrap_or_else(|_| vec![None; ps.nrows]); let ms1_i1 = ps.opt_f64("ms1_iso1").unwrap_or_else(|_| vec![None; ps.nrows]); @@ -713,6 +721,8 @@ pub fn run(p: FeaturesParams) -> Result { push(&mut fmap, "coel_clean", ff.coel_clean); push(&mut fmap, "shadow_frac", ff.shadow_frac); push(&mut fmap, "peak_contested_frac", contested[i]); + push(&mut fmap, "peak_contested_count_frac", contested_count[i]); + push(&mut fmap, "peak_apportioned_frac", apportioned[i]); // Extended battery (opt-in). Build the shared Evidence once per PSM and // fan it out to the family modules; push their values under the fixed @@ -1269,8 +1279,10 @@ mod tests { assert_eq!(active_features(FeatureSet::Rich).len(), 14 + 30); // Extended = minimal + rich + the family battery, and its names are unique. let ext = active_features(FeatureSet::Extended); - // +4 psms-derived extras: peak_contested_frac + 3 charge-corroboration features. - assert_eq!(ext.len(), 14 + 30 + extended_names().len() + 4); + // +6 psms-derived extras: 3 co-elution peak-contest metrics + // (peak_contested_frac + peak_contested_count_frac + peak_apportioned_frac) + // + 3 charge-corroboration features. + assert_eq!(ext.len(), 14 + 30 + extended_names().len() + 6); let uniq: std::collections::HashSet<&String> = ext.iter().collect(); assert_eq!(uniq.len(), ext.len(), "duplicate feature name in Extended set"); } diff --git a/rust/mumdia/crates/mumdia/src/stages/features/entropy.rs b/rust/mumdia/crates/mumdia/src/stages/features/entropy.rs index 4186279..2f14cb8 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/entropy.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/entropy.rs @@ -73,6 +73,18 @@ fn shannon(p: &[f64]) -> f64 { fin(h) } +/// Spectral-entropy similarity of the sqrt-transformed observed vs predicted +/// intensity vectors. This is exactly the `spectral_entropy_similarity_sqrt` +/// feature (see `values`, item 3), exposed so the extraction acceptance gate +/// (`GateMode::SpectralEntropy`) can threshold the single best target/decoy +/// discriminator found in the full-feature gate search, without duplicating the +/// entropy kernel. `obs` and `pred` are co-indexed by predicted fragment. +pub fn spectral_entropy_similarity_sqrt(obs: &[f64], pred: &[f64]) -> f64 { + let o: Vec = obs.iter().map(|x| if *x > 0.0 { x.sqrt() } else { 0.0 }).collect(); + let l: Vec = pred.iter().map(|x| if *x > 0.0 { x.sqrt() } else { 0.0 }).collect(); + entropy_sim(&o, &l) +} + /// Li spectral-entropy similarity: `1 - (2 H(m) - H(o) - H(l)) / ln 4`, with /// `m = (o + l) / 2` over sum-normalized `o` and `l`. Returns 0.0 if either /// input has non-positive mass or the lengths differ. Clamped to [0, 1]. diff --git a/rust/mumdia/crates/mumdia/src/stages/rescore.rs b/rust/mumdia/crates/mumdia/src/stages/rescore.rs index 07c5716..a19a475 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rescore.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rescore.rs @@ -101,7 +101,7 @@ pub fn run(p: RescoreParams) -> Result { Vec::new() } else { match p.cfg.classifier { - RescorerKind::Mokapot => match run_mokapot(&p, &feat_names, &cid, &label, &pform, &protein, &mz, &feats) { + RescorerKind::Mokapot => match run_pin_sidecar(&p, "mokapot_worker.py", &feat_names, &cid, &label, &pform, &protein, &mz, &feats) { Ok(s) => { info!("rescore: using Mokapot scores"); classifier_used = "mokapot"; @@ -116,6 +116,21 @@ pub fn run(p: RescoreParams) -> Result { native_scores(&p, &feats, &is_decoy, &base, &prelim) } }, + RescorerKind::NnTorch => match run_pin_sidecar(&p, "nn_rescore_worker.py", &feat_names, &cid, &label, &pform, &protein, &mz, &feats) { + Ok(s) => { + info!("rescore: using PyTorch NN sidecar scores"); + classifier_used = "nn_torch"; + model_identity = "nn-torch-semisup-sidecar-v1".to_string(); + s + } + Err(e) => { + if p.cfg.strict { + anyhow::bail!("rescore: NnTorch sidecar failed ({e}) and rescore.strict=true"); + } + warn!("rescore: NnTorch failed ({e}); falling back to native_tda"); + native_scores(&p, &feats, &is_decoy, &base, &prelim) + } + }, RescorerKind::Percolator => { if p.cfg.strict { anyhow::bail!("rescore: classifier=percolator but percolator.exe is not wired, and rescore.strict=true"); @@ -479,11 +494,14 @@ fn run_entrapment_gbm( .collect()) } -/// Run Mokapot over a PIN written from the competed set; return scores aligned -/// to the input candidate order (PLAN.md Section 3.2 file contract). +/// Run a PIN-contract Python rescorer sidecar (`mokapot_worker.py` or +/// `nn_rescore_worker.py`) over a PIN written from the competed set; return scores +/// aligned to the input candidate order (PLAN.md Section 3.2 file contract). Both +/// sidecars share this exact contract: PIN in, `candidate_id`+`score` parquet out. #[allow(clippy::too_many_arguments)] -fn run_mokapot( +fn run_pin_sidecar( p: &RescoreParams, + script_name: &str, feat_names: &[String], cid: &[u32], label: &[String], @@ -497,10 +515,10 @@ fn run_mokapot( .cfg .python .as_deref() - .ok_or_else(|| anyhow::anyhow!("classifier=mokapot requires rescore.python"))?; + .ok_or_else(|| anyhow::anyhow!("classifier sidecar {script_name} requires rescore.python"))?; std::fs::create_dir_all(p.work_dir).ok(); let pin = format!("{}/rescore.pin", p.work_dir); - let outp = format!("{}/mokapot_out.parquet", p.work_dir); + let outp = format!("{}/rescore_sidecar_out.parquet", p.work_dir); let mut s = String::new(); s.push_str("SpecId\tLabel\tScanNr\tExpMass\tCalcMass\t"); @@ -522,7 +540,7 @@ fn run_mokapot( } std::fs::write(&pin, s)?; - let script = crate::sidecar::resolve_script(p.script_dir, "mokapot_worker.py"); + let script = crate::sidecar::resolve_script(p.script_dir, script_name); let status = std::process::Command::new(python) .arg(&script) .arg(&pin) @@ -530,7 +548,7 @@ fn run_mokapot( .env("PYTHONUTF8", "1") .status()?; if !status.success() { - anyhow::bail!("mokapot worker exited with {status}"); + anyhow::bail!("{script_name} exited with {status}"); } // The worker echoes the PIN's SpecId tail as `candidate_id`, which here is the diff --git a/rust/mumdia/crates/mumdia/src/stages/run.rs b/rust/mumdia/crates/mumdia/src/stages/run.rs index 9fe04bf..c7b19da 100644 --- a/rust/mumdia/crates/mumdia/src/stages/run.rs +++ b/rust/mumdia/crates/mumdia/src/stages/run.rs @@ -220,6 +220,7 @@ pub fn run(p: RunParams) -> Result<()> { mass_cal: Some(&format!("{seed}.masscal.json")), out_psms: &psms, out_chrom: &chrom, + restrict_candidates: None, cfg: &cfg.extract, config_hash: &ch, })?; diff --git a/rust/mumdia/crates/mumdia/tests/pipeline.rs b/rust/mumdia/crates/mumdia/tests/pipeline.rs index e2c9ee6..ff6aa4a 100644 --- a/rust/mumdia/crates/mumdia/tests/pipeline.rs +++ b/rust/mumdia/crates/mumdia/tests/pipeline.rs @@ -132,6 +132,7 @@ fn run_extract(prec: &str, frag: &str, ms2: &str, win: &str, tag: &str) -> (Stri mass_cal: None, out_psms: &psms, out_chrom: &chrom, + restrict_candidates: None, cfg: &cfg.extract, config_hash: "test", }) diff --git a/scripts/nn_rescore_worker.py b/scripts/nn_rescore_worker.py new file mode 100644 index 0000000..6d5b6c1 --- /dev/null +++ b/scripts/nn_rescore_worker.py @@ -0,0 +1,281 @@ +"""PyTorch semi-supervised NN rescorer sidecar (Stage F, RescorerKind::NnTorch). + +Usage: + python nn_rescore_worker.py + +Reads a Percolator PIN, rescores it with a PyTorch MLP trained in the +Percolator/mokapot semi-supervised scheme, writes `candidate_id` (the SpecId tail) ++ `score` + `q_value`. Scores EVERY PSM (targets and decoys) so target-decoy FDR +downstream is intact. Same positional-CLI file contract as `mokapot_worker.py`; +select it with `rescore.classifier = "nn_torch"` and point `rescore.python` at an +interpreter with torch + pandas + pyarrow. + +Algorithm (per CV fold, so every PSM is scored out-of-fold by a model that never +trained on it): initialise from the best single feature+sign, then iterate +{recompute target-decoy q on the training folds -> targets at q<=train_fdr are +positives, all decoys negatives -> train the MLP from scratch -> rescore} for +`iters` rounds; score the held-out fold with the final model. + +MEMORY (multi-run / large PINs): two feature backends behind one accessor. + - in-memory (default for PINs <= MUMDIA_NN_STREAM_GB, 4 GB): the full standardised + feature matrix is held in RAM (median/IQR standardisation). + - streaming memmap (large PINs, or MUMDIA_NN_STREAM=1): the PIN is read ONCE in + chunks into a disk-backed float32 memmap (mean/std standardisation accumulated + in the same pass); training and scoring then draw MINIBATCHES indexed into the + memmap, so peak RAM is one batch + per-row metadata, NOT the whole matrix. + This is what makes combining many runs into one rescoring tractable: the full + PIN never lives in RAM at once. + +Determinism note (plan.md Section 7): NN training is only approximately reproducible. +Set MUMDIA_NN_SEEDS>1 to ensemble seeds and average out-of-fold scores. + +Env knobs (all optional): + MUMDIA_NN_FOLDS = 3 cross-validation folds + MUMDIA_NN_ITERS = 5 semi-supervised self-training iterations + MUMDIA_NN_EPOCHS = 25 NN epochs per iteration + MUMDIA_NN_HIDDEN = "128,64" comma-separated hidden layer sizes + MUMDIA_NN_DROPOUT = 0.3 + MUMDIA_NN_LR = 1e-3 + MUMDIA_NN_WD = 1e-4 weight decay + MUMDIA_NN_BATCH = 4096 + MUMDIA_NN_TRAIN_FDR = 0.01 positive-selection FDR during training + MUMDIA_NN_SEEDS = 1 seed models to ensemble (average OOF) + MUMDIA_NN_STREAM = auto auto|1|0 force the streaming memmap backend + MUMDIA_NN_STREAM_GB = 4 auto-stream when the PIN exceeds this many GB + MUMDIA_NN_CHUNK = 250000 PIN rows per read chunk (streaming backend) + MUMDIA_NN_INIT_SAMPLE = 300000 rows used to pick the init feature (streaming) +""" + +import hashlib +import os +import re +import sys + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +NON_FEATURE = {"SpecId", "Label", "ScanNr", "ExpMass", "CalcMass", "Peptide", "Proteins"} + + +def env_f(name, default): + return float(os.environ.get(name, default)) + + +def env_i(name, default): + return int(os.environ.get(name, default)) + + +def strip_pep(p): + s = re.sub(r"\[[^\]]*\]", "", str(p)) + s = re.sub(r"^[A-Z-]\.", "", s) + s = re.sub(r"\.[A-Z-]$", "", s) + return s + + +def tda_q(scores, is_target): + """Target-decoy q-values. scores desc; FDR=(decoys+1)/max(1,targets), q=running min.""" + order = np.argsort(-scores, kind="stable") + t = is_target[order].astype(float) + ct = np.cumsum(t) + cd = np.cumsum(1 - t) + fdr = (cd + 1) / np.maximum(ct, 1) + q = np.minimum.accumulate(fdr[::-1])[::-1] + out = np.empty_like(q) + out[order] = q + return out + + +def n_targets_at(scores, is_target, fdr): + q = tda_q(scores, is_target) + return int(((q <= fdr) & (is_target == 1)).sum()) + + +def main(): + pin_path, out_path = sys.argv[1], sys.argv[2] + + import torch + import torch.nn as nn + + FOLDS = env_i("MUMDIA_NN_FOLDS", 3) + ITERS = env_i("MUMDIA_NN_ITERS", 5) + EPOCHS = env_i("MUMDIA_NN_EPOCHS", 25) + HIDDEN = [int(x) for x in os.environ.get("MUMDIA_NN_HIDDEN", "128,64").split(",") if x] + DROPOUT = env_f("MUMDIA_NN_DROPOUT", 0.3) + LR = env_f("MUMDIA_NN_LR", 1e-3) + WD = env_f("MUMDIA_NN_WD", 1e-4) + BATCH = env_i("MUMDIA_NN_BATCH", 4096) + TRAIN_FDR = env_f("MUMDIA_NN_TRAIN_FDR", 0.01) + N_SEEDS = env_i("MUMDIA_NN_SEEDS", 1) + CHUNK = env_i("MUMDIA_NN_CHUNK", 250000) + DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + + stream_env = os.environ.get("MUMDIA_NN_STREAM", "auto").lower() + filesize = os.path.getsize(pin_path) + stream = stream_env in ("1", "on", "true") or ( + stream_env == "auto" and filesize > env_f("MUMDIA_NN_STREAM_GB", 4) * 1024 ** 3 + ) + + header = pd.read_csv(pin_path, sep="\t", nrows=0).columns.tolist() + feat_cols = [c for c in header if c not in NON_FEATURE] + nf = len(feat_cols) + fold_of = lambda p: int(hashlib.md5(strip_pep(p).encode()).hexdigest(), 16) % FOLDS + + mm_path = None + if not stream: + # ---- in-memory backend (median/IQR standardisation) ---- + pin = pd.read_csv(pin_path, sep="\t") + y = (pin["Label"].to_numpy() == 1).astype(np.float32) + cids = np.array([int(s.rsplit("_", 1)[-1]) for s in pin["SpecId"].astype(str)], np.int64) + fold = pin["Peptide"].map(fold_of).to_numpy() + X = np.nan_to_num(pin[feat_cols].to_numpy(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + del pin + med = np.median(X, axis=0) + iqr = np.subtract(*np.percentile(X, [75, 25], axis=0)) + iqr[iqr == 0] = 1.0 + Xs = np.clip((X - med) / iqr, -8, 8).astype(np.float32) + del X + n = len(y) + get = lambda idx: Xs[idx] + col = lambda j: Xs[:, j] + else: + # ---- streaming memmap backend (mean/std, one text pass) ---- + with open(pin_path, "rb") as fh: + n = sum(1 for _ in fh) - 1 + mm_path = out_path + ".feat.mm" + mm = np.memmap(mm_path, dtype=np.float32, mode="w+", shape=(n, nf)) + y = np.empty(n, np.float32) + cids = np.empty(n, np.int64) + fold = np.empty(n, np.int16) + s1 = np.zeros(nf, np.float64) + s2 = np.zeros(nf, np.float64) + keep = set(["SpecId", "Label", "Peptide"] + feat_cols) + off = 0 + for chunk in pd.read_csv(pin_path, sep="\t", usecols=lambda c: c in keep, chunksize=CHUNK): + k = len(chunk) + xf = np.nan_to_num(chunk[feat_cols].to_numpy(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + mm[off:off + k] = xf + s1 += xf.sum(axis=0, dtype=np.float64) + s2 += (xf.astype(np.float64) ** 2).sum(axis=0) + y[off:off + k] = (chunk["Label"].to_numpy() == 1).astype(np.float32) + cids[off:off + k] = [int(s.rsplit("_", 1)[-1]) for s in chunk["SpecId"].astype(str)] + fold[off:off + k] = chunk["Peptide"].map(fold_of).to_numpy() + off += k + mean = (s1 / n).astype(np.float32) + std = np.sqrt(np.maximum(s2 / n - (s1 / n) ** 2, 1e-12)).astype(np.float32) + std[std == 0] = 1.0 + # standardise the memmap in place, chunked (binary, sequential, low RAM) + for i in range(0, n, CHUNK): + mm[i:i + CHUNK] = np.clip((mm[i:i + CHUNK] - mean) / std, -8, 8) + mm.flush() + get = lambda idx: np.ascontiguousarray(mm[idx]) + col = lambda j: np.asarray(mm[:, j]) + + # initial direction: best single feature+sign by targets at TRAIN_FDR (on a sample) + SAMPLE = min(n, env_i("MUMDIA_NN_INIT_SAMPLE", 300000)) + samp = np.arange(SAMPLE) + Xsamp, ysamp = get(samp), y[samp] + best_j, best_sign, best_n = 0, 1, -1 + for j in range(nf): + for sign in (1, -1): + m_ = n_targets_at(sign * Xsamp[:, j], ysamp, TRAIN_FDR) + if m_ > best_n: + best_n, best_j, best_sign = m_, j, sign + init_score = (best_sign * col(best_j)).astype(np.float32) + print(f"nn_rescore_worker: device={DEVICE} backend={'stream' if stream else 'in-memory'} " + f"pool={n} feats={nf} init={feat_cols[best_j]} sign{best_sign:+d} " + f"({best_n}@{TRAIN_FDR:.0%} on {SAMPLE} sample)", flush=True) + + class MLP(nn.Module): + def __init__(self, d_in, hidden, p): + super().__init__() + layers, d = [], d_in + for h in hidden: + layers += [nn.Linear(d, h), nn.BatchNorm1d(h), nn.ReLU(), nn.Dropout(p)] + d = h + layers += [nn.Linear(d, 1)] + self.net = nn.Sequential(*layers) + + def forward(self, x): + return self.net(x).squeeze(-1) + + def train_model(train_idx, pos_weight, seed): + """Minibatch train, drawing each batch's features through `get` (memmap or RAM).""" + torch.manual_seed(seed) + m = MLP(nf, HIDDEN, DROPOUT).to(DEVICE) + opt = torch.optim.Adam(m.parameters(), lr=LR, weight_decay=WD) + lossf = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(pos_weight, device=DEVICE)) + idx = np.asarray(train_idx) + for _ in range(EPOCHS): + m.train() + perm = idx[np.random.permutation(len(idx))] + for i in range(0, len(perm), BATCH): + b = perm[i:i + BATCH] + Xb = torch.from_numpy(get(b)).to(DEVICE) + yb = torch.from_numpy(y[b]).to(DEVICE) + opt.zero_grad() + lossf(m(Xb), yb).backward() + opt.step() + return m + + @torch.no_grad() + def score_idx(m, idx): + m.eval() + idx = np.asarray(idx) + out = np.empty(len(idx), np.float32) + step = BATCH * 4 + for i in range(0, len(idx), step): + b = idx[i:i + step] + out[i:i + len(b)] = m(torch.from_numpy(get(b)).to(DEVICE)).cpu().numpy() + return out + + def one_pass(seed): + """One full CV pass -> out-of-fold scores for all PSMs.""" + oof = np.zeros(n, np.float32) + for f in range(FOLDS): + tr_idx = np.where(fold != f)[0] + te_idx = np.where(fold == f)[0] + ytr = y[tr_idx] + score_tr = init_score[tr_idx].copy() + model = None + for _ in range(ITERS): + q = tda_q(score_tr, ytr) + pos = (q <= TRAIN_FDR) & (ytr == 1) + neg = ytr == 0 + sel = tr_idx[pos | neg] + pw = float(neg.sum()) / max(1.0, float(pos.sum())) + model = train_model(sel, pw, seed) + score_tr = score_idx(model, tr_idx) + oof[te_idx] = score_idx(model, te_idx) + print(f" seed {seed} fold {f}: train targets@{TRAIN_FDR:.0%} = " + f"{n_targets_at(score_tr, ytr, TRAIN_FDR)}", flush=True) + return oof + + # seed ensemble: average rank-normalised out-of-fold scores across seeds + acc = np.zeros(n, np.float64) + for s in range(N_SEEDS): + np.random.seed(s) + torch.manual_seed(s) + oof = one_pass(s) + acc += pd.Series(oof).rank(method="average").to_numpy() / n + final = acc / N_SEEDS + + out = pa.table({ + "candidate_id": pa.array(cids.astype(np.uint32), pa.uint32()), + "score": pa.array(final.astype(np.float64), pa.float64()), + "q_value": pa.array(np.zeros(n, np.float64), pa.float64()), + }) + pq.write_table(out, out_path) + if mm_path and os.path.exists(mm_path): + try: + del mm + os.remove(mm_path) + except OSError: + pass + print(f"nn_rescore_worker: {n} PSMs rescored (targets+decoys), {N_SEEDS} seed(s), " + f"OOF at {FOLDS} folds, backend={'stream' if stream else 'in-memory'}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/nn_semisupervised_rescore.ipynb b/scripts/nn_semisupervised_rescore.ipynb new file mode 100644 index 0000000..dd30f63 --- /dev/null +++ b/scripts/nn_semisupervised_rescore.ipynb @@ -0,0 +1,567 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3dc93822", + "metadata": {}, + "source": [ + "# Semi-supervised NN rescoring of a MuMDIA PIN\n", + "\n", + "Ingests a Percolator `.pin` (MuMDIA `features`/`compete` output) and rescores it with a\n", + "**PyTorch MLP** trained in the **Percolator/mokapot semi-supervised** scheme, i.e. the\n", + "DIA-NN-style nonlinear classifier over the same feature set.\n", + "\n", + "Algorithm (per cross-validation fold, so every PSM is scored by a model that never saw it):\n", + "1. Initialise a score from the single best feature+sign (most targets at `train_fdr`).\n", + "2. Repeat `n_iter` times: recompute target-decoy q-values on the training folds, take\n", + " **target PSMs with q <= train_fdr as positives** and **all decoys as negatives**, train\n", + " the MLP from scratch on that labelled set, rescore the training folds.\n", + "3. Score the held-out fold with the final model.\n", + "\n", + "Final target-decoy q-values are computed at PSM and peptide level; the peptide count at\n", + "1% FDR is the number to compare against mokapot (linear) on the same PIN.\n", + "\n", + "**Note.** The NN introduces run-to-run nondeterminism even with seeds set (CPU thread\n", + "scheduling, cuDNN off here). Treat the peptide count as approximate (+/- a few tens)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b42eead6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:11:57.843746Z", + "iopub.status.busy": "2026-07-19T10:11:57.843746Z", + "iopub.status.idle": "2026-07-19T10:11:57.846628Z", + "shell.execute_reply": "2026-07-19T10:11:57.846628Z" + } + }, + "outputs": [], + "source": [ + "# ---- parameters ----\n", + "# PIN_PATH = r'C:/proteobench/out_ecoli_ox/run.pin' # apex-gated pool (107k): NN=10,195, mokapot=9,928\n", + "PIN_PATH = r'C:/proteobench/out_ox_gateoff/x.pin' # GATE-OFF pool (503k, no spectral gate)\n", + "N_FOLDS = 3 # cross-validation folds (mokapot default)\n", + "N_ITER = 5 # semi-supervised self-training iterations\n", + "TRAIN_FDR = 0.01 # positive-selection FDR during training\n", + "EPOCHS = 25 # NN epochs per iteration\n", + "HIDDEN = [128, 64]\n", + "DROPOUT = 0.3\n", + "LR = 1e-3\n", + "WEIGHT_DECAY = 1e-4\n", + "BATCH = 4096\n", + "SEED = 0\n", + "\n", + "# optional DIA-NN concordance (set to None to skip)\n", + "DIANN_REPORT = r'C:/Users/robbi/OneDrive - UGent/MuMDIA_NG/out_diann/report.tsv'\n", + "DIANN_TAG = 'ECOLI' # substring in Proteins marking the target proteome\n", + "MOKAPOT_BASELINE = 9928 # apex-gated mokapot @1% (the number to beat; gate-off has no mokapot ref)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9dddbace", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:11:57.847633Z", + "iopub.status.busy": "2026-07-19T10:11:57.847633Z", + "iopub.status.idle": "2026-07-19T10:12:04.084669Z", + "shell.execute_reply": "2026-07-19T10:12:04.084669Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "device: cuda | torch 2.5.1+cu121\n" + ] + } + ], + "source": [ + "import re\n", + "import numpy as np\n", + "import pandas as pd\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "np.random.seed(SEED)\n", + "torch.manual_seed(SEED)\n", + "torch.use_deterministic_algorithms(False)\n", + "DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n", + "print('device:', DEVICE, '| torch', torch.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "110cac1f", + "metadata": {}, + "source": [ + "## Parse the PIN\n", + "\n", + "Columns are `SpecId, Label, ScanNr, ExpMass, CalcMass, , Peptide, Proteins`.\n", + "`ExpMass`/`CalcMass` are Percolator bookkeeping, not features. Folds are assigned by a hash\n", + "of the **stripped peptide** so all PSMs of a peptide (and its paired decoy) stay together,\n", + "avoiding train/test leakage." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b96b880e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:12:04.085673Z", + "iopub.status.busy": "2026-07-19T10:12:04.085673Z", + "iopub.status.idle": "2026-07-19T10:12:23.146046Z", + "shell.execute_reply": "2026-07-19T10:12:23.145039Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "503,251 PSMs | 379 features\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\robbi\\AppData\\Local\\Temp\\ipykernel_44372\\3106957684.py:14: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n", + " pin['strip'] = pin['Peptide'].map(strip_pep)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\robbi\\AppData\\Local\\Temp\\ipykernel_44372\\3106957684.py:15: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n", + " pin['pep_mod'] = pin['Peptide'].map(lambda p: re.sub(r'^[A-Z-]\\.|\\.[A-Z-]$', '', str(p)))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold sizes: [167533 167862 167856]\n" + ] + } + ], + "source": [ + "pin = pd.read_csv(PIN_PATH, sep='\\t')\n", + "NON_FEATURE = {'SpecId', 'Label', 'ScanNr', 'ExpMass', 'CalcMass', 'Peptide', 'Proteins'}\n", + "feat_cols = [c for c in pin.columns if c not in NON_FEATURE]\n", + "print(f'{len(pin):,} PSMs | {len(feat_cols)} features')\n", + "\n", + "y = (pin['Label'].to_numpy() == 1).astype(np.float32) # 1 target, 0 decoy\n", + "X = np.nan_to_num(pin[feat_cols].to_numpy(np.float32), nan=0.0, posinf=0.0, neginf=0.0)\n", + "\n", + "STRIP = lambda p: re.sub(r'\\[[^\\]]*\\]', '', str(p)).strip('-.').split('.')[-1] if '.' in str(p) else str(p)\n", + "def strip_pep(p):\n", + " s = re.sub(r'\\[[^\\]]*\\]', '', str(p)) # drop mods\n", + " s = re.sub(r'^[A-Z-]\\.', '', s); s = re.sub(r'\\.[A-Z-]$', '', s) # drop flanks\n", + " return s\n", + "pin['strip'] = pin['Peptide'].map(strip_pep)\n", + "pin['pep_mod'] = pin['Peptide'].map(lambda p: re.sub(r'^[A-Z-]\\.|\\.[A-Z-]$', '', str(p)))\n", + "\n", + "# deterministic fold by peptide hash (keeps a peptide's PSMs in one fold)\n", + "import hashlib\n", + "def h(s):\n", + " return int(hashlib.md5(s.encode()).hexdigest(), 16)\n", + "fold = pin['strip'].map(lambda s: h(s) % N_FOLDS).to_numpy()\n", + "print('fold sizes:', np.bincount(fold))\n", + "\n", + "# robust standardisation (median / IQR, clip) for stable NN training\n", + "med = np.median(X, axis=0)\n", + "iqr = np.subtract(*np.percentile(X, [75, 25], axis=0)); iqr[iqr == 0] = 1.0\n", + "Xs = np.clip((X - med) / iqr, -8, 8).astype(np.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6028bf21", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:12:23.149047Z", + "iopub.status.busy": "2026-07-19T10:12:23.148046Z", + "iopub.status.idle": "2026-07-19T10:12:23.153557Z", + "shell.execute_reply": "2026-07-19T10:12:23.153049Z" + } + }, + "outputs": [], + "source": [ + "def tda_q(scores, is_target):\n", + " \"\"\"Target-decoy q-values. scores desc; FDR=(decoys+1)/max(1,targets), q=running min.\"\"\"\n", + " order = np.argsort(-scores, kind='stable')\n", + " t = is_target[order].astype(float)\n", + " ct = np.cumsum(t); cd = np.cumsum(1 - t)\n", + " fdr = (cd + 1) / np.maximum(ct, 1)\n", + " q = np.minimum.accumulate(fdr[::-1])[::-1]\n", + " out = np.empty_like(q); out[order] = q\n", + " return out\n", + "\n", + "def n_targets_at(scores, is_target, fdr=0.01):\n", + " q = tda_q(scores, is_target)\n", + " return int(((q <= fdr) & (is_target == 1)).sum())\n", + "\n", + "def peptide_count(scores, is_target, pep, fdr=0.01):\n", + " \"\"\"Best PSM per peptide, then target-decoy q at peptide level.\"\"\"\n", + " df = pd.DataFrame({'s': scores, 't': is_target, 'p': pep})\n", + " best = df.sort_values('s', ascending=False).drop_duplicates('p')\n", + " q = tda_q(best['s'].to_numpy(), best['t'].to_numpy())\n", + " keep = (q <= fdr) & (best['t'].to_numpy() == 1)\n", + " return int(keep.sum()), best.loc[keep, 'p']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4ca8b2ff", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:12:23.154561Z", + "iopub.status.busy": "2026-07-19T10:12:23.154561Z", + "iopub.status.idle": "2026-07-19T10:12:52.551170Z", + "shell.execute_reply": "2026-07-19T10:12:52.550654Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "init feature: spectral_entropy_similarity_topk sign +1 -> 8571 targets @ 1%\n" + ] + } + ], + "source": [ + "# initial direction: best single feature+sign by targets at TRAIN_FDR (mokapot-style)\n", + "best_feat, best_sign, best_n = None, 1, -1\n", + "for j in range(Xs.shape[1]):\n", + " for sign in (1, -1):\n", + " n = n_targets_at(sign * Xs[:, j], y, TRAIN_FDR)\n", + " if n > best_n:\n", + " best_n, best_feat, best_sign = n, j, sign\n", + "print(f'init feature: {feat_cols[best_feat]} sign {best_sign:+d} -> {best_n} targets @ {TRAIN_FDR:.0%}')\n", + "init_score = best_sign * Xs[:, best_feat]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fe435008", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:12:52.552178Z", + "iopub.status.busy": "2026-07-19T10:12:52.552178Z", + "iopub.status.idle": "2026-07-19T10:12:52.556224Z", + "shell.execute_reply": "2026-07-19T10:12:52.556224Z" + } + }, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " def __init__(self, d_in, hidden, p):\n", + " super().__init__()\n", + " layers, d = [], d_in\n", + " for hdim in hidden:\n", + " layers += [nn.Linear(d, hdim), nn.BatchNorm1d(hdim), nn.ReLU(), nn.Dropout(p)]\n", + " d = hdim\n", + " layers += [nn.Linear(d, 1)]\n", + " self.net = nn.Sequential(*layers)\n", + " def forward(self, x):\n", + " return self.net(x).squeeze(-1)\n", + "\n", + "def train_model(Xtr, ytr, epochs, pos_weight):\n", + " torch.manual_seed(SEED)\n", + " m = MLP(Xtr.shape[1], HIDDEN, DROPOUT).to(DEVICE)\n", + " opt = torch.optim.Adam(m.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n", + " lossf = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(pos_weight, device=DEVICE))\n", + " Xt = torch.from_numpy(Xtr).to(DEVICE); yt = torch.from_numpy(ytr).to(DEVICE)\n", + " n = len(Xt)\n", + " for _ in range(epochs):\n", + " m.train(); perm = torch.randperm(n, device=DEVICE)\n", + " for i in range(0, n, BATCH):\n", + " idx = perm[i:i + BATCH]\n", + " opt.zero_grad(); loss = lossf(m(Xt[idx]), yt[idx]); loss.backward(); opt.step()\n", + " return m\n", + "\n", + "@torch.no_grad()\n", + "def score_model(m, Xarr):\n", + " m.eval()\n", + " return m(torch.from_numpy(Xarr).to(DEVICE)).cpu().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "054398f3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:12:52.557230Z", + "iopub.status.busy": "2026-07-19T10:12:52.557230Z", + "iopub.status.idle": "2026-07-19T10:13:23.477226Z", + "shell.execute_reply": "2026-07-19T10:13:23.476219Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0 iter 0: 5701 positives, 163106 decoys, train targets@1% = 7431\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0 iter 1: 7431 positives, 163106 decoys, train targets@1% = 7732\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0 iter 2: 7732 positives, 163106 decoys, train targets@1% = 8022\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0 iter 3: 8022 positives, 163106 decoys, train targets@1% = 8138\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0 iter 4: 8138 positives, 163106 decoys, train targets@1% = 8176\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 1 iter 0: 5667 positives, 162038 decoys, train targets@1% = 7529\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 1 iter 1: 7529 positives, 162038 decoys, train targets@1% = 7770\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 1 iter 2: 7770 positives, 162038 decoys, train targets@1% = 7905\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 1 iter 3: 7905 positives, 162038 decoys, train targets@1% = 7983\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 1 iter 4: 7983 positives, 162038 decoys, train targets@1% = 8056\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 2 iter 0: 5766 positives, 162700 decoys, train targets@1% = 7168\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 2 iter 1: 7168 positives, 162700 decoys, train targets@1% = 7664\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 2 iter 2: 7664 positives, 162700 decoys, train targets@1% = 8034\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 2 iter 3: 8034 positives, 162700 decoys, train targets@1% = 8125\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 2 iter 4: 8125 positives, 162700 decoys, train targets@1% = 8143\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CV scoring done\n" + ] + } + ], + "source": [ + "# semi-supervised cross-validated training\n", + "final = np.zeros(len(Xs), np.float32)\n", + "for f in range(N_FOLDS):\n", + " tr = fold != f; te = fold == f\n", + " Xtr, ytr = Xs[tr], y[tr]\n", + " score_tr = init_score[tr].copy()\n", + " model = None\n", + " for it in range(N_ITER):\n", + " q = tda_q(score_tr, ytr)\n", + " pos = (q <= TRAIN_FDR) & (ytr == 1)\n", + " neg = ytr == 0\n", + " sel = pos | neg\n", + " pw = float(neg.sum()) / max(1.0, float(pos.sum())) # balance rare positives\n", + " model = train_model(Xtr[sel], ytr[sel], EPOCHS, pw)\n", + " score_tr = score_model(model, Xtr)\n", + " print(f'fold {f} iter {it}: {int(pos.sum())} positives, {int(neg.sum())} decoys, '\n", + " f'train targets@1% = {n_targets_at(score_tr, ytr, TRAIN_FDR)}')\n", + " final[te] = score_model(model, Xs[te])\n", + "print('CV scoring done')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c4d804cb", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:13:23.478225Z", + "iopub.status.busy": "2026-07-19T10:13:23.478225Z", + "iopub.status.idle": "2026-07-19T10:13:23.774638Z", + "shell.execute_reply": "2026-07-19T10:13:23.774638Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PSMs @1% FDR: 11819\n", + "peptides @1% FDR (NN): 10447 | mokapot baseline: 9928 | delta +519\n" + ] + } + ], + "source": [ + "# final target-decoy q-values\n", + "psm_n = n_targets_at(final, y, 0.01)\n", + "pep_n, pep_ids = peptide_count(final, y, pin['pep_mod'].to_numpy(), 0.01)\n", + "print(f'PSMs @1% FDR: {psm_n}')\n", + "print(f'peptides @1% FDR (NN): {pep_n} | mokapot baseline: {MOKAPOT_BASELINE} | delta {pep_n - MOKAPOT_BASELINE:+d}')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e39d8742", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-19T10:13:23.776643Z", + "iopub.status.busy": "2026-07-19T10:13:23.776643Z", + "iopub.status.idle": "2026-07-19T10:13:24.064412Z", + "shell.execute_reply": "2026-07-19T10:13:24.064412Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DIA-NN E.coli @1%: 11642\n", + "NN E.coli peptides: 10327 | recovered 9707 = 83.4% of DIA-NN | concordance 94.0%\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\robbi\\AppData\\Local\\Temp\\ipykernel_44372\\2592420269.py:8: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n", + " df = pin.assign(q=q, s=final)\n", + "C:\\Users\\robbi\\AppData\\Local\\Temp\\ipykernel_44372\\2592420269.py:8: PerformanceWarning: DataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n", + " df = pin.assign(q=q, s=final)\n" + ] + } + ], + "source": [ + "# optional: DIA-NN E.coli concordance\n", + "if DIANN_REPORT:\n", + " dn = pd.read_csv(DIANN_REPORT, sep='\\t', usecols=['Stripped.Sequence', 'Protein.Names', 'Q.Value'])\n", + " dn = dn[(dn['Q.Value'] <= 0.01) & (dn['Protein.Names'].astype(str).str.contains(DIANN_TAG, na=False))]\n", + " diann = set(dn['Stripped.Sequence'])\n", + " # NN target peptides that are E.coli (Proteins carries the tag)\n", + " q = tda_q(final, y)\n", + " df = pin.assign(q=q, s=final)\n", + " tgt = df[(df.Label == 1) & (df.q <= 0.01) & df.Proteins.astype(str).str.contains(DIANN_TAG, na=False)]\n", + " nn_strip = set(tgt['strip'])\n", + " rec = nn_strip & diann\n", + " print(f'DIA-NN E.coli @1%: {len(diann)}')\n", + " print(f'NN E.coli peptides: {len(nn_strip)} | recovered {len(rec)} = {100*len(rec)/len(diann):.1f}% of DIA-NN '\n", + " f'| concordance {100*len(rec)/max(1,len(nn_strip)):.1f}%')" + ] + }, + { + "cell_type": "markdown", + "id": "0bfa71cc", + "metadata": {}, + "source": [ + "## Notes\n", + "\n", + "- **Fair comparison:** run against the same PIN mokapot scored (`out_ecoli_ox/run.pin`, apex-gated pool,\n", + " mokapot = 9,928 peptides). Any lift is the nonlinear classifier extracting more from the 379 features.\n", + "- **q-value estimator:** `(decoys+1)/max(1,targets)` with running-min, standard TDA. mokapot's estimator\n", + " differs slightly, so treat the absolute count as comparable-not-identical; the delta is the signal.\n", + "- **Leakage:** folds are split by stripped peptide, so a target and its paired decoy never straddle\n", + " train/test. Standardisation is global (negligible leakage).\n", + "- **Levers to try:** deeper/wider `HIDDEN`, more `N_ITER`, ensembling several seeds and averaging scores\n", + " (reduces NN variance, closer to DIA-NN's NN ensemble), focal/ranking loss instead of BCE, or feeding a\n", + " gate-off PIN (larger pool) to test whether the NN tolerates the FDR-flood better than the linear model." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312_mumdia", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From efd63d7ab75e53a34a73b599d21d34373971e3e3 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Tue, 21 Jul 2026 13:55:57 +0200 Subject: [PATCH 32/40] feat(sensitivity): confident-seed global elution boundary (features.bound_from_confident, default on) The feature-stage elution peak that restricts trace-based feature calculation was detected per-candidate from each candidate's own top-3-predicted-fragment profile. That boundary is noisy and manipulable for chimeric decoys: their window can shrink onto a chance-aligned scan or balloon over noise (measured p10-p90 width 8.75-46.6s on HYE A_02, vs 11.6-43.9s for targets). Add features.bound_from_confident: learn a single pair of left/right elution half-widths once from the confident seed PSMs (spectrum_q <= 0.01, target-only, the same anchor set used for RT calibration / DeepLC fine-tune), at the features.bound_confident_pct percentile (default 50 = median), and apply it to every candidate around its own apex. Half-widths are asymmetric (8.71s left, 14.53s right on HYE A_02) so the real chromatographic tail is preserved. If the seed yields < 20 confident anchors the stage warns and falls back to per-candidate detection. On HYE A_02 (reverse decoys, single-seed NN rescore) this lifts 1% FDR peptides 57,991 -> 58,368 (+377) with decoy fraction unchanged at 0.99%. A percentile sweep is monotonic (pct 50 > 75 > 90): a tighter uniform window excludes more interference, so the median is the right default. Enabled by default; falls back safely and is deterministic. Also adds features.bound_peak_grace (default 0, legacy behaviour): number of consecutive sub-threshold scans to bridge when walking the per-candidate peak boundary. grace=1 was neutral (57,669) so the default stays 0. Helpers elution_peak_rt_bounds (shared per-candidate boundary logic) and global_bound_indices (map a global half-width onto the scan grid). A peak-shape plausibility feature family (native width vs the confident norm) was prototyped and dropped: hard decoys have target-identical peak widths (AUC 0.47), so the width axis cannot separate the decoys that survive to 1% FDR. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 23 ++ .../crates/mumdia/src/stages/features.rs | 204 ++++++++++++++++-- 2 files changed, 204 insertions(+), 23 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 26592a7..8f76b88 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -670,6 +670,26 @@ pub struct FeaturesConfig { /// to peak*fraction, or stop earlier at a valley below it). 1/3 matched DIA-NN's /// RT bounds best in the diagnostic-plot benchmark. pub bound_peak_fraction: f64, + /// Grace when walking the elution-peak boundary: number of consecutive + /// sub-threshold scans to BRIDGE before stopping. 0 (default) stops at the first + /// scan below `bound_peak_fraction` (brittle on jagged/gappy profiles); 1 bridges + /// a single-scan dip (DIA sampling gap / noise), giving steadier boundaries. + pub bound_peak_grace: usize, + /// Elution-peak boundary source. When true (default) a single set of left/right + /// half-widths (seconds) is learned once from the confident seed PSMs + /// (`spectrum_q <= 0.01`, target-only, the same set that anchors RT calibration / + /// DeepLC fine-tune) and applied to EVERY candidate around its own apex. This + /// removes per-candidate boundary manipulation so a decoy is scored over a real- + /// peptide-width window centred on its apex. When false, each candidate detects its + /// own peak boundary from its top-3-predicted-fragment profile (per-candidate, + /// but noisy/manipulable for chimeric decoys; the legacy behaviour). If the seed + /// yields < 20 confident anchors the stage logs a warning and falls back to + /// per-candidate detection for that run. + pub bound_from_confident: bool, + /// Percentile (0-100) of the confident-set half-widths taken as the global left/ + /// right elution half-width when `bound_from_confident` is true. 50 = median + /// (typical real peak width); higher percentiles widen the shared window. + pub bound_confident_pct: f64, } impl Default for FeaturesConfig { fn default() -> Self { @@ -679,6 +699,9 @@ impl Default for FeaturesConfig { prec_tol_ppm: 20.0, bound_features: true, bound_peak_fraction: 1.0 / 3.0, + bound_peak_grace: 0, // stop at first sub-threshold scan (legacy) + bound_from_confident: true, // fixed feature window from confident-seed norm + bound_confident_pct: 50.0, // median confident half-width } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/features.rs b/rust/mumdia/crates/mumdia/src/stages/features.rs index 322dc53..8dac336 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features.rs @@ -17,8 +17,9 @@ use mumdia_io::report::ArtifactReport; use mumdia_io::table::{write_table, Col, Table}; use serde::{Deserialize, Serialize}; use serde_json::json; -use tracing::info; +use tracing::{info, warn}; +use crate::calibrate::percentile; use crate::stats::{cosine, pearson, spectral_angle}; use rayon::prelude::*; @@ -341,7 +342,14 @@ fn parse_ion(name: &str) -> (bool, u32, u32) { /// rows (scalar fields default; the caller fills them). Mirrors the alignment /// and peak-bounding of [`fragment_features`] so the extended families see the /// same elution peak the legacy features use. -fn build_evidence(rows: &[ChromRow], ms1_rows: &[ChromRow], apex_rt: f64, frac: f64) -> Evidence { +fn build_evidence( + rows: &[ChromRow], + ms1_rows: &[ChromRow], + apex_rt: f64, + frac: f64, + grace: usize, + global_bounds: Option<(f64, f64)>, +) -> Evidence { let m = rows.len(); let mut obs_apex = Vec::with_capacity(m); let mut pred = Vec::with_capacity(m); @@ -385,13 +393,6 @@ fn build_evidence(rows: &[ChromRow], ms1_rows: &[ChromRow], apex_rt: f64, frac: .collect(); let (lo_i, hi_i) = if axis_full.len() >= 3 { - let mut ord: Vec = (0..pred.len()).collect(); - ord.sort_by(|&a, &b| pred[b].partial_cmp(&pred[a]).unwrap_or(std::cmp::Ordering::Equal)); - let k3: Vec = ord.into_iter().take(3).collect(); - let prof_raw: Vec = (0..axis_full.len()) - .map(|k| k3.iter().map(|&i| traces_full[i][k]).sum::()) - .collect(); - let prof = smooth3(&prof_raw); let ai = axis_full .iter() .enumerate() @@ -403,7 +404,21 @@ fn build_evidence(rows: &[ChromRow], ms1_rows: &[ChromRow], apex_rt: f64, frac: }) .map(|(i, _)| i) .unwrap_or(0); - peak_bounds(&prof, ai, frac, 0) + match global_bounds { + Some((l, r)) => global_bound_indices(&axis_full, apex_rt, ai, l, r), + None => { + let mut ord: Vec = (0..pred.len()).collect(); + ord.sort_by(|&a, &b| { + pred[b].partial_cmp(&pred[a]).unwrap_or(std::cmp::Ordering::Equal) + }); + let k3: Vec = ord.into_iter().take(3).collect(); + let prof_raw: Vec = (0..axis_full.len()) + .map(|k| k3.iter().map(|&i| traces_full[i][k]).sum::()) + .collect(); + let prof = smooth3(&prof_raw); + peak_bounds(&prof, ai, frac, grace) + } + } } else { (0, axis_full.len().saturating_sub(1)) }; @@ -554,22 +569,84 @@ pub fn run(p: FeaturesParams) -> Result { } } - // Seed corroboration maps (candidate_id -> seed score / identified flag). - let (seed_score_map, seed_id_map): (HashMap, HashMap) = match p.seed { + // Seed corroboration maps (candidate_id -> seed score / identified flag) plus the + // confident-target candidate set (spectrum_q <= 0.01, label == target) used to + // learn a global elution half-width when `bound_from_confident` is set. This + // mirrors the RT-calibration / DeepLC-fine-tune anchor set (rt_im_train.rs). + let (seed_score_map, seed_id_map, confident_cids): ( + HashMap, + HashMap, + std::collections::HashSet, + ) = match p.seed { Some(path) => { let s = Table::read(path)?; let scid = s.u32("candidate_id")?; let ssc = s.f64("score")?; let sq = s.f64("spectrum_q")?; + let slabel = s.str("label")?; let mut sm = HashMap::new(); let mut im = HashMap::new(); + let mut conf = std::collections::HashSet::new(); for i in 0..s.nrows { sm.insert(scid[i], ssc[i]); im.insert(scid[i], if sq[i] <= 0.01 { 1.0 } else { 0.0 }); + if sq[i] <= 0.01 && slabel[i] == "target" { + conf.insert(scid[i]); + } + } + (sm, im, conf) + } + None => (HashMap::new(), HashMap::new(), std::collections::HashSet::new()), + }; + + // Global elution half-widths learned once from the confident set. Some((L, R)) in + // seconds when `bound_from_confident` and >= 20 confident anchors have a resolvable + // peak; then every candidate's feature region is [apex - L, apex + R]. None keeps + // the per-candidate boundary detection (default). + let global_bounds: Option<(f64, f64)> = if p.cfg.bound_from_confident { + let mut lefts: Vec = Vec::new(); + let mut rights: Vec = Vec::new(); + for i in 0..ps.nrows { + if !confident_cids.contains(&cid[i]) { + continue; + } + if let Some(rows) = chrom.get(&cid[i]) { + if let Some((lo, hi)) = elution_peak_rt_bounds( + rows, + apex_rt[i], + p.cfg.bound_peak_fraction, + p.cfg.bound_peak_grace, + ) { + let l = apex_rt[i] - lo as f64; + let r = hi as f64 - apex_rt[i]; + if l >= 0.0 && r >= 0.0 { + lefts.push(l); + rights.push(r); + } + } } - (sm, im) } - None => (HashMap::new(), HashMap::new()), + if lefts.len() >= 20 { + let q = (p.cfg.bound_confident_pct / 100.0).clamp(0.0, 1.0); + let (l, r) = (percentile(&lefts, q), percentile(&rights, q)); + info!( + n_confident = lefts.len(), + left_hw_s = l, + right_hw_s = r, + pct = p.cfg.bound_confident_pct, + "features: global elution half-widths from confident set" + ); + Some((l, r)) + } else { + warn!( + n_confident = lefts.len(), + "features: bound_from_confident set but < 20 confident anchors; \ + falling back to per-candidate boundary" + ); + None + } + } else { + None }; let gradient = apex_rt.iter().cloned().fold(0.0f64, f64::max).max(1.0); @@ -627,6 +704,8 @@ pub fn run(p: FeaturesParams) -> Result { p.cfg.coelution_corr_threshold, p.cfg.bound_features, p.cfg.bound_peak_fraction, + p.cfg.bound_peak_grace, + global_bounds, ), _ => FragFeatures::default(), }; @@ -635,7 +714,7 @@ pub fn run(p: FeaturesParams) -> Result { Some(rows) if !rows.is_empty() => { let ms1_rows = ms1x.get(&cid[i]).map(|v| v.as_slice()).unwrap_or(&[]); let mut ev = - build_evidence(rows, ms1_rows, apex_rt[i], p.cfg.bound_peak_fraction); + build_evidence(rows, ms1_rows, apex_rt[i], p.cfg.bound_peak_fraction, p.cfg.bound_peak_grace, global_bounds); ev.rt_pred_cal = rt_cal[i]; ev.rt_err = (apex_rt[i] - rt_cal[i]).abs(); ev.gradient = gradient; @@ -926,12 +1005,84 @@ pub(crate) fn peak_bounds(prof: &[f64], ai: usize, frac: f64, grace: usize) -> ( (lo, hi) } +/// Map a global (left, right) elution half-width (seconds) around `apex_rt` onto +/// index bounds of `axis_full`, falling back to the apex-nearest scan `ai` if the +/// window collapses between scans (sparse grid, or half-width below one cycle). +fn global_bound_indices( + axis_full: &[f32], + apex_rt: f64, + ai: usize, + l: f64, + r: f64, +) -> (usize, usize) { + let lo_rt = (apex_rt - l) as f32; + let hi_rt = (apex_rt + r) as f32; + let li = axis_full.iter().position(|&t| t >= lo_rt).unwrap_or(0); + let hi = axis_full + .iter() + .rposition(|&t| t <= hi_rt) + .unwrap_or(axis_full.len().saturating_sub(1)); + if li <= hi { + (li, hi) + } else { + (ai, ai) + } +} + +/// Per-candidate elution-peak RT bounds (seconds): reference = smoothed sum of the +/// top-3 predicted-intensity fragments, walked from the apex-nearest scan while +/// >= `frac` x apex height, bridging <= `grace` sub-threshold scans. Returns +/// (lo_rt, hi_rt), or None when fewer than 3 distinct scans. Mirrors the boundary +/// logic inside `fragment_features`/`build_evidence` so the confident-set half-widths +/// match the per-candidate detector they replace when `bound_from_confident` is set. +fn elution_peak_rt_bounds( + rows: &[ChromRow], + apex_rt: f64, + frac: f64, + grace: usize, +) -> Option<(f32, f32)> { + let mut axis: Vec = rows.iter().flat_map(|r| r.rt.iter().cloned()).collect(); + axis.sort_by(|a, b| a.partial_cmp(b).unwrap()); + axis.dedup(); + if axis.len() < 3 { + return None; + } + let traces: Vec> = rows + .iter() + .map(|r| { + let map: HashMap = + r.rt.iter().zip(&r.inten).map(|(&t, &v)| (t.to_bits(), v)).collect(); + axis.iter().map(|t| *map.get(&t.to_bits()).unwrap_or(&0.0) as f64).collect() + }) + .collect(); + let mut ord: Vec = (0..rows.len()).collect(); + ord.sort_by(|&a, &b| { + rows[b].pred_int.partial_cmp(&rows[a].pred_int).unwrap_or(std::cmp::Ordering::Equal) + }); + let k3: Vec = ord.into_iter().take(3).collect(); + let prof_raw: Vec = + (0..axis.len()).map(|k| k3.iter().map(|&i| traces[i][k]).sum::()).collect(); + let prof = smooth3(&prof_raw); + let ai = axis + .iter() + .enumerate() + .min_by(|(_, a), (_, b)| { + (**a as f64 - apex_rt).abs().partial_cmp(&((**b as f64 - apex_rt).abs())).unwrap() + }) + .map(|(i, _)| i) + .unwrap_or(0); + let (lo, hi) = peak_bounds(&prof, ai, frac, grace); + Some((axis[lo], axis[hi])) +} + fn fragment_features( rows: &[ChromRow], apex_rt: f64, coel_thresh: f64, bound: bool, frac: f64, + grace: usize, + global_bounds: Option<(f64, f64)>, ) -> FragFeatures { let mut f = FragFeatures::default(); // Observed apex intensity per fragment (nearest scan to apex). @@ -1002,13 +1153,6 @@ fn fragment_features( .collect(); let (lo_i, hi_i) = if bound && axis_full.len() >= 3 { // boundary on the smoothed summed top-3-predicted-fragment profile, around apex - let mut ord: Vec = (0..pred.len()).collect(); - ord.sort_by(|&a, &b| pred[b].partial_cmp(&pred[a]).unwrap_or(std::cmp::Ordering::Equal)); - let k3: Vec = ord.into_iter().take(3).collect(); - let prof_raw: Vec = (0..axis_full.len()) - .map(|k| k3.iter().map(|&i| traces_full[i][k]).sum::()) - .collect(); - let prof = smooth3(&prof_raw); let ai = axis_full .iter() .enumerate() @@ -1020,7 +1164,21 @@ fn fragment_features( }) .map(|(i, _)| i) .unwrap_or(0); - peak_bounds(&prof, ai, frac, 0) + match global_bounds { + Some((l, r)) => global_bound_indices(&axis_full, apex_rt, ai, l, r), + None => { + let mut ord: Vec = (0..pred.len()).collect(); + ord.sort_by(|&a, &b| { + pred[b].partial_cmp(&pred[a]).unwrap_or(std::cmp::Ordering::Equal) + }); + let k3: Vec = ord.into_iter().take(3).collect(); + let prof_raw: Vec = (0..axis_full.len()) + .map(|k| k3.iter().map(|&i| traces_full[i][k]).sum::()) + .collect(); + let prof = smooth3(&prof_raw); + peak_bounds(&prof, ai, frac, grace) + } + } } else { (0, axis_full.len().saturating_sub(1)) }; From 9df1cc7d8eed10c2b54eef1c482fcda89248572a Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Wed, 22 Jul 2026 10:58:34 +0200 Subject: [PATCH 33/40] feat(rt): configurable DeepLC fine-tune hyperparameters (epochs, patience, q_train, auto-scaled batch) The DeepLC multitask fine-tune sidecar was invoked with positional args only, so its epochs (25), patience (10), confidence threshold (hardcoded spectrum_q <= 0.01), and batch size (512) were fixed. Two of those defaults are actively harmful: - Batch 512 underfits small confident seeds. An E.coli run with ~4k confident anchors gets ~8 gradient steps/epoch and never converges (held-out RT MAD 165s at 50 epochs, vs ~30s at batch 32). deeplc_finetune.py now takes `--batch 0` (default) which auto-scales to clamp(n_ref // 30, 16, 512) so every epoch runs >= ~30 steps regardless of seed size. - The fine-tune reference threshold was hardcoded, so rt_im_train.q_train never reached it. Added `--q-train`. Thread all four through as typed config fields on RtImTrainConfig (finetune_epochs=25, finetune_patience=10, finetune_batch=0=auto) and rt_im_train.q_train (already existed) via sidecar::run_deeplc_finetune into run.rs. Defaults reproduce the prior behaviour except the batch auto-scale, which only changes small-seed runs (previously broken). No change when finetune_deeplc is off. Also adds two diagnostic harnesses used to characterize the RT predictors: - ft_epoch_eval.py: fine-tune epoch sweep with train vs held-out RT MAD (random or q-band split), optional raw-lib baseline. Established that raw DIA-NN iRT is locally noisy (~110s MAD) and fine-tune tightens it ~8x (HYE) / ~4x (E.coli). - ft_calibrate_eval.py: DeepLC calibration vs fine-tune vs raw baseline on a common split. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 15 ++ rust/mumdia/crates/mumdia/src/sidecar.rs | 19 ++- rust/mumdia/crates/mumdia/src/stages/run.rs | 12 +- scripts/deeplc_finetune.py | 20 ++- scripts/ft_calibrate_eval.py | 124 +++++++++++++++++ scripts/ft_epoch_eval.py | 139 +++++++++++++++++++ 6 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 scripts/ft_calibrate_eval.py create mode 100644 scripts/ft_epoch_eval.py diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 8f76b88..28f5d19 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -428,6 +428,18 @@ pub struct RtImTrainConfig { /// main use is library-input mode, where the base iRT comes from the imported /// library rather than a DeepLC prediction. pub finetune_deeplc: bool, + /// DeepLC fine-tune training epochs (passed to `deeplc_finetune.py --epochs`). + /// Early stopping with `finetune_patience` usually halts before this cap, so it + /// is an upper bound rather than a fixed count. Only used when `finetune_deeplc`. + pub finetune_epochs: usize, + /// DeepLC fine-tune early-stopping patience (`--patience`): epochs without + /// validation-loss improvement before stopping. Only used when `finetune_deeplc`. + pub finetune_patience: usize, + /// DeepLC fine-tune batch size (`--batch`). 0 (default) auto-scales to the confident + /// seed size so each epoch has >= ~30 gradient steps; a fixed large batch underfits + /// small seeds (a ~4k-peptide reference at batch 512 is ~8 steps/epoch and never + /// converges). Only used when `finetune_deeplc`. + pub finetune_batch: usize, /// Adaptive RT window (sensitivity_plan spec 03 §3.5, backlog P3.2/P3.3): /// instead of one global residual-percentile half-width for every candidate, /// bin the calibration anchors by calibrated RT and give each candidate the @@ -455,6 +467,9 @@ impl Default for RtImTrainConfig { loess_span: 0.3, fallback_rt_window_s: 120.0, finetune_deeplc: false, + finetune_epochs: 25, // deeplc_finetune.py default + finetune_patience: 10, // deeplc_finetune.py default + finetune_batch: 0, // 0 = auto-scale to seed size adaptive_rt_window: false, adaptive_rt_bins: 12, rt_window_min_s: 1.0, diff --git a/rust/mumdia/crates/mumdia/src/sidecar.rs b/rust/mumdia/crates/mumdia/src/sidecar.rs index 715186a..61ebeac 100644 --- a/rust/mumdia/crates/mumdia/src/sidecar.rs +++ b/rust/mumdia/crates/mumdia/src/sidecar.rs @@ -109,10 +109,23 @@ pub fn run_deeplc_finetune( lib_in: &str, seed: &str, lib_out: &str, + epochs: usize, + patience: usize, + q_train: f64, + batch: usize, ) -> Result<()> { - info!(lib_in, seed, lib_out, "sidecar: running DeepLC multitask fine-tune"); - run_worker(python, script, &[lib_in, seed, lib_out], true) - .context("DeepLC fine-tune failed") + info!(lib_in, seed, lib_out, epochs, patience, q_train, batch, "sidecar: running DeepLC multitask fine-tune"); + let ep = epochs.to_string(); + let pa = patience.to_string(); + let qt = q_train.to_string(); + let ba = batch.to_string(); + run_worker( + python, + script, + &[lib_in, seed, lib_out, "--epochs", &ep, "--patience", &pa, "--q-train", &qt, "--batch", &ba], + true, + ) + .context("DeepLC fine-tune failed") } /// Invoke a Python worker: `python script arg...`. `utf8` forces UTF-8 I/O diff --git a/rust/mumdia/crates/mumdia/src/stages/run.rs b/rust/mumdia/crates/mumdia/src/stages/run.rs index c7b19da..d732cc0 100644 --- a/rust/mumdia/crates/mumdia/src/stages/run.rs +++ b/rust/mumdia/crates/mumdia/src/stages/run.rs @@ -191,7 +191,17 @@ pub fn run(p: RunParams) -> Result<()> { let script = crate::sidecar::resolve_script(&cfg.predict_frag.sidecar_script_dir, "deeplc_finetune.py"); let lib_p_ft = d("fragment_library_precursors_ft.parquet"); - crate::sidecar::run_deeplc_finetune(python, &script, &lib_p, &seed, &lib_p_ft)?; + crate::sidecar::run_deeplc_finetune( + python, + &script, + &lib_p, + &seed, + &lib_p_ft, + cfg.rt_im_train.finetune_epochs, + cfg.rt_im_train.finetune_patience, + cfg.rt_im_train.q_train, + cfg.rt_im_train.finetune_batch, + )?; lib_p_ft } else { lib_p diff --git a/scripts/deeplc_finetune.py b/scripts/deeplc_finetune.py index b514a41..0af33cd 100644 --- a/scripts/deeplc_finetune.py +++ b/scripts/deeplc_finetune.py @@ -60,8 +60,14 @@ def main(): ap.add_argument("--threads", type=int, default=int(_THREADS), help="torch CPU threads for training (bounded to avoid OpenMP oversubscription; cpu only)") ap.add_argument("--epochs", type=int, default=25) - ap.add_argument("--batch", type=int, default=512) + ap.add_argument("--batch", type=int, default=0, + help="fine-tune batch size; 0 (default) auto-scales to the reference " + "size so every epoch has >= ~30 gradient steps. A fixed large " + "batch (e.g. 512) underfits small seeds: a ~4k-peptide E.coli " + "reference gives only ~8 steps/epoch and never converges.") ap.add_argument("--patience", type=int, default=10) + ap.add_argument("--q-train", dest="q_train", type=float, default=0.01, + help="max spectrum_q for a seed PSM to enter the fine-tune reference set") ap.add_argument("--max-ref", type=int, default=0, help="cap reference PSMs (0 = all); use a small value for a smoke test") ap.add_argument("--predict-limit", type=int, default=0, @@ -93,7 +99,7 @@ def main(): ref = {} for i in range(len(seed["peptidoform"])): pf = seed["peptidoform"][i] - if seed["label"][i] == "target" and seed["spectrum_q"][i] <= 0.01 and is_std(pf): + if seed["label"][i] == "target" and seed["spectrum_q"][i] <= args.q_train and is_std(pf): ref[pf] = seed["observed_rt"][i] ref_items = list(ref.items()) if args.max_ref and len(ref_items) > args.max_ref: @@ -102,10 +108,18 @@ def main(): for k, (pf, rt) in enumerate(ref_items)]) print(f"fine-tune reference: {len(ref_psms)} confident seed peptides", flush=True) + # Batch size: 0 -> auto-scale so each epoch runs ~30+ gradient steps. A fixed 512 + # underfits small references (e.g. ~4k E.coli seed = ~8 steps/epoch, never + # converges); clamp to [16, 512]. + batch = args.batch + if batch <= 0: + batch = int(min(512, max(16, len(ref_psms) // 30))) + print(f"fine-tune batch_size={batch} (~{max(1, len(ref_psms) // max(1, batch))} steps/epoch)", flush=True) + train_kwargs = { "num_workers": 0, # no DataLoader subprocesses "epochs": args.epochs, - "batch_size": args.batch, + "batch_size": batch, "patience": args.patience, "device": args.device, } diff --git a/scripts/ft_calibrate_eval.py b/scripts/ft_calibrate_eval.py new file mode 100644 index 0000000..45702b3 --- /dev/null +++ b/scripts/ft_calibrate_eval.py @@ -0,0 +1,124 @@ +"""DeepLC CALIBRATION (no fine-tuning) vs fine-tune vs DIA-NN, on a common random +held-out split. Calibration fits a calibration curve + selects the best internal MT +model on the reference set WITHOUT changing weights, so it cannot overfit like +fine-tuning. Reports held-out MAD for: + (a) DeepLC native predict_and_calibrate (calibrated seconds, its own curve), + (b) DeepLC base predict + binned-median calibration (isolates base-model quality, + same calibrator used for DIA-NN), + (c) DIA-NN raw library iRT + binned-median calibration (baseline). + +Usage: python ft_calibrate_eval.py --q-train Q --held-frac F --seed S --raw-lib LIB +""" +import os +_T = os.environ.get("DEEPLC_FT_THREADS", "8") +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" +for k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"): + os.environ[k] = "1" + +import argparse +import re +import deeplc +import numpy as np +import pyarrow.parquet as pq +import torch +from psm_utils import PSM, PSMList + +STD = set("ACDEFGHIKLMNPQRSTVWY") +is_std = lambda pf: all(c in STD for c in re.sub(r"\[[^\]]*\]", "", pf)) + + +def calibrate_fit(x, y, nbins=80): + o = np.argsort(x); xs, ys = x[o], y[o] + e = np.linspace(0, len(xs), nbins + 1).astype(int) + cx, cy = [], [] + for b in range(nbins): + lo, hi = e[b], e[b + 1] + if hi > lo: + cx.append(np.median(xs[lo:hi])); cy.append(np.median(ys[lo:hi])) + cx, cy = np.array(cx), np.array(cy) + return lambda q: np.interp(q, cx, cy) + + +def mad(pred, obs): + r = np.asarray(pred, float) - np.asarray(obs, float) + return np.median(np.abs(r - np.median(r))) + + +def flat(a): + a = np.asarray(a, float) + return a.mean(axis=1) if a.ndim == 2 else a + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("seed_path") + ap.add_argument("--q-train", dest="q_train", type=float, default=0.01) + ap.add_argument("--held-frac", dest="held_frac", type=float, default=0.15) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--threads", type=int, default=int(_T)) + ap.add_argument("--raw-lib", dest="raw_lib", default=None) + a = ap.parse_args() + torch.set_num_threads(max(1, a.threads)) + try: + torch.set_num_interop_threads(1) + except RuntimeError: + pass + + s = pq.read_table(a.seed_path).to_pydict() + have_irt = "predicted_irt" in s + best = {} + for i in range(len(s["peptidoform"])): + if s["label"][i] != "target": + continue + pf = s["peptidoform"][i] + if not is_std(pf): + continue + b = s["base_peptide_id"][i] + if b not in best or s["score"][i] > best[b][0]: + irt = float(s["predicted_irt"][i]) if have_irt else float("nan") + best[b] = (s["score"][i], pf, s["observed_rt"][i], s["spectrum_q"][i], irt) + conf = [(pf, rt, q) for (_, pf, rt, q, _) in best.values() if q < 0.01] + # raw iRT per peptidoform straight from the seed (the library iRT that was used), + # so the DIA-NN baseline needs no external lib file. + seed_irt = {pf: irt for (_, pf, rt, q, irt) in best.values()} + rng = np.random.default_rng(a.seed) + mask = rng.random(len(conf)) < a.held_frac + held = [(pf, rt) for i, (pf, rt, q) in enumerate(conf) if mask[i]] + train = [(pf, rt) for i, (pf, rt, q) in enumerate(conf) if not mask[i] and q < a.q_train] + print(f"q_train={a.q_train} train={len(train)} held={len(held)}", flush=True) + + ref = PSMList(psm_list=[PSM(peptidoform=pf, retention_time=rt, spectrum_id=str(k)) + for k, (pf, rt) in enumerate(train)]) + tr_pf = [pf for pf, _ in train]; tr_obs = np.array([rt for _, rt in train], float) + hd_pf = [pf for pf, _ in held]; hd_obs = np.array([rt for _, rt in held], float) + + # (a) DeepLC native predict_and_calibrate (calibrated seconds) + tr_cal = flat(deeplc.predict_and_calibrate(tr_pf, psm_list_reference=ref)) + hd_cal = flat(deeplc.predict_and_calibrate(hd_pf, psm_list_reference=ref)) + print(f"CAL q_train={a.q_train} train_MAD={mad(tr_cal, tr_obs):.3f} held_MAD={mad(hd_cal, hd_obs):.3f}", flush=True) + + # (b) DeepLC base predict + binned-median calibration + tr_base = flat(deeplc.predict(tr_pf)); hd_base = flat(deeplc.predict(hd_pf)) + cb = calibrate_fit(tr_base, tr_obs) + print(f"BASE q_train={a.q_train} train_MAD={mad(cb(tr_base), tr_obs):.3f} held_MAD={mad(cb(hd_base), hd_obs):.3f}", flush=True) + + # (c) DIA-NN raw. Prefer an external lib; else use the seed's own predicted_irt. + rm = None + if a.raw_lib: + rl = pq.read_table(a.raw_lib, columns=["peptidoform", "predicted_irt"]).to_pydict() + rm = {} + for pf, irt in zip(rl["peptidoform"], rl["predicted_irt"]): + rm.setdefault(pf, float(irt)) + elif have_irt: + rm = {pf: irt for pf, irt in seed_irt.items() if np.isfinite(irt)} + if rm: + rtr = np.array([rm[pf] for pf in tr_pf if pf in rm], float) + rtro = np.array([rt for pf, rt in train if pf in rm], float) + rhd = np.array([rm[pf] for pf in hd_pf if pf in rm], float) + rhdo = np.array([rt for pf, rt in held if pf in rm], float) + cr = calibrate_fit(rtr, rtro) + print(f"DIANN q_train={a.q_train} train_MAD={mad(cr(rtr), rtro):.3f} held_MAD={mad(cr(rhd), rhdo):.3f}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/ft_epoch_eval.py b/scripts/ft_epoch_eval.py new file mode 100644 index 0000000..e71b4eb --- /dev/null +++ b/scripts/ft_epoch_eval.py @@ -0,0 +1,139 @@ +"""Fast DeepLC fine-tune epoch sweep for generalization. Fine-tune on the q detects overfitting. Skips the full +3.74M-peptide library prediction so each epoch setting costs only its training time. + +Usage: python ft_epoch_eval.py [--q-train Q] [--patience P] +""" +import os +_T = os.environ.get("DEEPLC_FT_THREADS", "8") +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" +for k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"): + os.environ[k] = "1" + +import argparse +import re +import deeplc +import numpy as np +import pyarrow.parquet as pq +import torch +from psm_utils import PSM, PSMList + +STD = set("ACDEFGHIKLMNPQRSTVWY") +strip_mods = lambda s: re.sub(r"\[[^\]]*\]", "", s) +is_std = lambda pf: all(c in STD for c in strip_mods(pf)) + + +def calibrate_fit(x, y, nbins=80): + """Robust monotone calibration curve (quantile-binned medians), numpy-only + stand-in for LOESS. Returns a predictor callable via np.interp.""" + o = np.argsort(x) + xs, ys = x[o], y[o] + edges = np.linspace(0, len(xs), nbins + 1).astype(int) + cx, cy = [], [] + for b in range(nbins): + lo, hi = edges[b], edges[b + 1] + if hi > lo: + cx.append(np.median(xs[lo:hi])) + cy.append(np.median(ys[lo:hi])) + cx, cy = np.array(cx), np.array(cy) + return lambda q: np.interp(q, cx, cy) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("seed_path") + ap.add_argument("epochs", type=int) + ap.add_argument("--q-train", dest="q_train", type=float, default=0.001) + ap.add_argument("--patience", type=int, default=10) + ap.add_argument("--batch", type=int, default=512, + help="fine-tune batch size; small sets need a small batch so each " + "epoch has enough gradient steps to converge") + ap.add_argument("--threads", type=int, default=int(_T)) + ap.add_argument("--held-frac", dest="held_frac", type=float, default=0.0, + help="if >0, hold out this random fraction of the confident (q<0.01) " + "set (excluded from training regardless of q_train); the rest, " + "subject to q_train, is the train set. Enables fair comparison " + "of q_train choices on a common held-out set. 0 = legacy q-band held-out.") + ap.add_argument("--seed", type=int, default=0, help="RNG seed for the random held-out split") + ap.add_argument("--raw-lib", dest="raw_lib", default=None, + help="optional library parquet; also report the raw (un-fine-tuned) " + "iRT held-out MAD on the SAME split, as the baseline to beat") + a = ap.parse_args() + torch.set_num_threads(max(1, a.threads)) + try: + torch.set_num_interop_threads(1) + except RuntimeError: + pass + + s = pq.read_table(a.seed_path).to_pydict() + # best-scoring target PSM per base_peptide_id + best = {} + for i in range(len(s["peptidoform"])): + if s["label"][i] != "target": + continue + pf = s["peptidoform"][i] + if not is_std(pf): + continue + b = s["base_peptide_id"][i] + if b not in best or s["score"][i] > best[b][0]: + best[b] = (s["score"][i], pf, s["observed_rt"][i], s["spectrum_q"][i]) + rows = list(best.values()) + conf = [(pf, rt, q) for (_, pf, rt, q) in rows if q < 0.01] # confident universe + if a.held_frac > 0: + # Random split of the confident set: `held` is fixed across q_train choices so + # different train-set sizes are compared on the SAME held-out peptides. + rng = np.random.default_rng(a.seed) + mask = rng.random(len(conf)) < a.held_frac + held = [(pf, rt) for i, (pf, rt, q) in enumerate(conf) if mask[i]] + train = [(pf, rt) for i, (pf, rt, q) in enumerate(conf) if not mask[i] and q < a.q_train] + else: + train = [(pf, rt) for (pf, rt, q) in conf if q < a.q_train] + held = [(pf, rt) for (pf, rt, q) in conf if a.q_train <= q < 0.01] + print(f"epochs={a.epochs} q_train={a.q_train} held_frac={a.held_frac} " + f"train={len(train)} held={len(held)}", flush=True) + + ref = PSMList(psm_list=[PSM(peptidoform=pf, retention_time=rt, spectrum_id=str(k)) + for k, (pf, rt) in enumerate(train)]) + tk = {"num_workers": 0, "epochs": a.epochs, "batch_size": a.batch, + "patience": a.patience, "device": "cpu", "num_threads": max(1, a.threads)} + model = deeplc.finetune(ref, train_kwargs=tk) + + def predict(items): + pf = [p for p, _ in items] + pr = deeplc.predict(pf, model=model) + pr = np.asarray(pr, float) + if pr.ndim == 2: + pr = pr.mean(axis=1) + return pr, np.array([rt for _, rt in items], float) + + tr_irt, tr_obs = predict(train) + hd_irt, hd_obs = predict(held) + cal = calibrate_fit(tr_irt, tr_obs) + + def mad(pred, obs): + r = pred - obs + return np.median(np.abs(r - np.median(r))) + + tr_mad = mad(cal(tr_irt), tr_obs) + hd_mad = mad(cal(hd_irt), hd_obs) + print(f"RESULT q_train={a.q_train} epochs={a.epochs} train_MAD={tr_mad:.3f} held_MAD={hd_mad:.3f}", flush=True) + + # Raw (un-fine-tuned) baseline on the identical train/held split. + if a.raw_lib: + rl = pq.read_table(a.raw_lib, columns=["peptidoform", "predicted_irt"]).to_pydict() + rawmap = {} + for pf, irt in zip(rl["peptidoform"], rl["predicted_irt"]): + rawmap.setdefault(pf, float(irt)) + rtr = np.array([rawmap[pf] for pf, _ in train if pf in rawmap], float) + rtr_o = np.array([rt for pf, rt in train if pf in rawmap], float) + rhd = np.array([rawmap[pf] for pf, _ in held if pf in rawmap], float) + rhd_o = np.array([rt for pf, rt in held if pf in rawmap], float) + rcal = calibrate_fit(rtr, rtr_o) + print(f"RAW q_train={a.q_train} train_MAD={mad(rcal(rtr), rtr_o):.3f} " + f"held_MAD={mad(rcal(rhd), rhd_o):.3f}", flush=True) + + +if __name__ == "__main__": + main() From 0202caae000788dbbb41396830980ea19393f9b0 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Wed, 22 Jul 2026 11:01:41 +0200 Subject: [PATCH 34/40] fix(quant): configurable q-value filter column (quant.q_filter) for cross-run quant quant filtered candidates on peptide_q_value. That is per-run for a single-run rescore, but under an experiment-wide rescore peptide_q_value is a GLOBAL value carried on the single best PSM per peptide across all runs. Splitting the pooled scored table by run and quantifying each run then keeps only the peptides whose global-best PSM lands in that run and drops the rest, so per-run quant sets come out disjoint and the cross-run intensity matrix is empty (0 complete-case, NaN size factors, blank ProteoBench submission). Add QuantQColumn { PeptideQ (default), PsmQ } and quant.q_filter. PsmQ filters on the per-PSM q_value (per run) and is the correct choice for a precursor-level cross-run quant such as a ProteoBench submission. Default PeptideQ preserves existing single-run behaviour. Recorded in the quant report params. Verified empirically: on the 6-run HYE set, PeptideQ gave 0 cross-run precursor overlap (empty matrix); the per-run q gives ~54k overlap and a fully populated 84k-precursor submission with on-target A/B species ratios. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 23 +++++++++++++++++++ rust/mumdia/crates/mumdia/src/stages/quant.rs | 13 ++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 28f5d19..098a51e 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -875,6 +875,25 @@ impl NormalizeMethod { } } +/// Which q-value column quant filters candidates on. Peptide-level q is correct +/// for a single-run quant (per-run peptide FDR). Under an experiment-wide rescore, +/// however, `peptide_q_value` is a GLOBAL per-peptide value carried on the single +/// best PSM across all runs, so a per-run quant over one run's slice keeps only the +/// peptides whose global-best PSM falls in that run and drops the rest, giving +/// disjoint per-run quant sets and an empty cross-run intensity matrix. `PsmQ` +/// filters on the per-PSM `q_value` instead (per run), the correct choice for a +/// precursor-level cross-run quant such as a ProteoBench submission. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum QuantQColumn { + /// Filter on `peptide_q_value` (per-run peptide FDR). Default. + #[default] + PeptideQ, + /// Filter on the per-PSM `q_value`. Use for cross-run quant off an + /// experiment-wide rescore, where the peptide q is global. + PsmQ, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct QuantConfig { @@ -900,6 +919,9 @@ pub struct QuantConfig { /// Peptide q-value cutoff defining the "confident" set that calibrates the /// consensus half-widths (Consensus mode only). Tighter than `q_threshold`. pub reliable_q: f64, + /// Which q-value column to filter candidates on (`peptide_q` default, or `psm_q` + /// for cross-run quant off an experiment-wide rescore). See [`QuantQColumn`]. + pub q_filter: QuantQColumn, } impl Default for QuantConfig { fn default() -> Self { @@ -913,6 +935,7 @@ impl Default for QuantConfig { peak_grace: 1, peak_window_mode: PeakWindowMode::PerCandidate, reliable_q: 0.001, + q_filter: QuantQColumn::PeptideQ, } } } diff --git a/rust/mumdia/crates/mumdia/src/stages/quant.rs b/rust/mumdia/crates/mumdia/src/stages/quant.rs index 598f180..20b951b 100644 --- a/rust/mumdia/crates/mumdia/src/stages/quant.rs +++ b/rust/mumdia/crates/mumdia/src/stages/quant.rs @@ -9,7 +9,7 @@ use std::collections::{BTreeMap, HashMap}; use std::time::Instant; use anyhow::Result; -use mumdia_core::config::{NormalizeMethod, PeakWindowMode, QuantConfig, RollupMethod}; +use mumdia_core::config::{NormalizeMethod, PeakWindowMode, QuantConfig, QuantQColumn, RollupMethod}; use mumdia_core::schema::artifact; use mumdia_io::report::ArtifactReport; use mumdia_io::table::{write_table, Col, Table}; @@ -154,7 +154,13 @@ pub fn run(p: QuantParams) -> Result<(u64, u64)> { let charge = ps.i32("charge")?; let label = ps.str("label")?; let pg = ps.str("protein_group")?; - let pep_q = ps.f64("peptide_q_value")?; + // q-value column to filter on. Peptide q is per-run in a single-run rescore, but + // GLOBAL (best PSM per peptide across all runs) under an experiment-wide rescore, + // where the per-PSM q_value is the correct per-run choice for cross-run quant. + let pep_q = match p.cfg.q_filter { + QuantQColumn::PeptideQ => ps.f64("peptide_q_value")?, + QuantQColumn::PsmQ => ps.f64("q_value")?, + }; // Chromatograms grouped by candidate. let ch = Table::read(p.chromatograms)?; @@ -389,7 +395,8 @@ pub fn run(p: QuantParams) -> Result<(u64, u64)> { content_hash: mumdia_io::hash::blake3_file(path)?, params: json!({"q_threshold": p.cfg.q_threshold, "top_n_fragments": p.cfg.top_n_fragments, "rollup": format!("{:?}", p.cfg.rollup), "bound_peak": p.cfg.bound_peak, - "peak_fraction": p.cfg.peak_fraction, "peak_grace": p.cfg.peak_grace}), + "peak_fraction": p.cfg.peak_fraction, "peak_grace": p.cfg.peak_grace, + "q_filter": format!("{:?}", p.cfg.q_filter)}), stats: stats.clone(), model_identity: None, elapsed_ms: elapsed, From b1a2fccec4d8a289d821f2af78a83b6285d86556 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Wed, 22 Jul 2026 11:21:29 +0200 Subject: [PATCH 35/40] feat(mbr): match-between-runs transfer, rescuable tier (Stage D3, M1-M4) First working match-between-runs, validated on the 6-run HYE set before build (see mbr_plan.md). Transfers identifications across runs to boost per-run sensitivity and fill the cross-run quant matrix, under a controlled false-transfer FDR. Scope of this commit: the RESCUABLE tier -- precursors confident (q<=q_anchor) in >= min_anchor_runs OTHER runs but sub-threshold in a target run where they WERE already extracted, so no re-extraction is needed. Steps M1-M4: - M1 anchors: per-run confident set from the experiment-wide scored table. - M2 expected RT: predict a precursor's RT in the target run from the median of the other runs' binned-median-aligned apex RTs. Leave-target-out residual ~17 s p95 (~15x tighter than the search window). - M3 transfer test: accept when the observed apex sits within the learned window. - M4 FDR: permuted-RT decoy-transfer null -> transfer q-value by target/decoy competition on the RT residual. Implemented as a Python sidecar (scripts/mbr_worker.py) over a positional contract, matching the mokapot/deeplc/entrapment sidecar pattern, plus a typed config block (MbrStrategy {None default / EmpiricalLibrary / RtTransfer / Full}, DecoyTransfer, MbrConfig) and a `mumdia mbr` CLI command (sidecar::run_mbr). Default strategy None is byte-identical; the command errors if strategy is None, < 2 runs, or mbr.python is unset. Validated on HYE (6 runs, q_transfer<=0.01): 25,278 accepted target transfers (+5.7-9.9%/run), empirical decoy fraction 0.00%, RT window 20.2 s. Complete-case precursors (present in all 6 runs) 33,740 -> 39,275 (+16.4%), the ProteoBench quant metric. `mumdia mbr` reproduces the worker exactly. Follow-ups (next commits): fold accepted transfers into the scored set + requant (M5); the re-extraction tier for precursors absent from a run (65,805 candidates) via extract --restrict-candidates at the transfer window; a fragment-consensus correlation guard against RT-concordant interference; multi-run orchestration (M6). Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/config.rs | 78 +++++++++ rust/mumdia/crates/mumdia/src/main.rs | 33 ++++ rust/mumdia/crates/mumdia/src/sidecar.rs | 33 ++++ scripts/mbr_worker.py | 175 +++++++++++++++++++ 4 files changed, 319 insertions(+) create mode 100644 scripts/mbr_worker.py diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index 098a51e..d6ffa8c 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -940,6 +940,81 @@ impl Default for QuantConfig { } } +/// Match-between-runs strategy (Stage D3, `mbr_plan.md`). Default `None` reproduces +/// the current chain byte-for-byte. Later variants transfer identification evidence +/// across a run set: `EmpiricalLibrary` builds the consensus anchor library only; +/// `RtTransfer` adds cross-run expected-RT transfer extraction; `Full` adds +/// requantification. All require >= 2 runs and a decoy-transfer FDR (see the plan). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum MbrStrategy { + /// No match-between-runs (default). + #[default] + None, + /// Build the cross-run consensus anchor library (M1) only; no transfer. + EmpiricalLibrary, + /// EmpiricalLibrary + cross-run expected-RT transfer extraction (M2/M3). + RtTransfer, + /// RtTransfer + requantification of accepted transfers (M5). + Full, +} + +/// Decoy-transfer null for the MBR false-transfer FDR (M4). `ReverseSequence` +/// transfers reverse/scramble decoys at the same expected RT; `PermutedRt` transfers +/// real precursors to a decoupled (wrong) expected RT; `Both` combines them. The +/// prototype's shuffled-RT null gave a ~0.6% in-window false rate vs 66.6% true +/// (113x separation), so the transfer q-value is well-calibrated. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum DecoyTransfer { + #[default] + PermutedRt, + ReverseSequence, + Both, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct MbrConfig { + pub strategy: MbrStrategy, + /// q-value for a precursor to become a cross-run anchor (validated at 0.01). + pub q_anchor: f64, + /// Minimum number of OTHER runs a precursor must be confident in to transfer. + pub min_anchor_runs: usize, + /// Accept threshold for a transferred identification's transfer q-value. + pub q_transfer: f64, + /// Transfer RT half-window (seconds) around the cross-run-predicted RT. The M2 + /// leave-target-out residual was ~17 s at p95, ~15x tighter than the search + /// window; this is the default so the false-transfer search space stays small. + pub rt_window_s: f64, + /// Which decoy-transfer null estimates the false-transfer rate (M4). + pub decoy_transfer: DecoyTransfer, + /// Minimum correlation of the observed fragment pattern to the empirical + /// consensus for a transfer to be accepted (interference guard; 0 disables). + pub consensus_corr_min: f64, + /// Requantify already-identified precursors too (fill the matrix), not only + /// transferred ones. Only used when `strategy = Full`. + pub requant_all: bool, + /// Python interpreter for the `mbr_worker.py` sidecar (pandas/pyarrow/numpy; + /// e.g. the `py312_mumdia` env). Required when `strategy != None`. + pub python: Option, +} +impl Default for MbrConfig { + fn default() -> Self { + Self { + strategy: MbrStrategy::None, + q_anchor: 0.01, + min_anchor_runs: 2, + q_transfer: 0.01, + rt_window_s: 20.0, // >= the p95 M2 residual (~17 s) + decoy_transfer: DecoyTransfer::PermutedRt, + consensus_corr_min: 0.0, + requant_all: false, + python: None, + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct RescoreConfig { @@ -1012,6 +1087,8 @@ pub struct Config { pub compete: CompeteConfig, pub rescore: RescoreConfig, pub quant: QuantConfig, + #[serde(default)] + pub mbr: MbrConfig, } impl Default for Config { fn default() -> Self { @@ -1028,6 +1105,7 @@ impl Default for Config { compete: t(), rescore: t(), quant: t(), + mbr: t(), } } } diff --git a/rust/mumdia/crates/mumdia/src/main.rs b/rust/mumdia/crates/mumdia/src/main.rs index fdfd265..e7128ad 100644 --- a/rust/mumdia/crates/mumdia/src/main.rs +++ b/rust/mumdia/crates/mumdia/src/main.rs @@ -226,6 +226,19 @@ enum Cmd { #[arg(long)] config: Option, }, + /// Match-between-runs identification transfer (Stage D3) -> transferred.parquet. + Mbr { + /// Experiment-wide scored_combined.parquet (has the `source` column). + #[arg(long)] + scored: String, + /// Per-run psms.parquet in `source` order (one per run). + #[arg(long, num_args = 1..)] + psms: Vec, + #[arg(long)] + out: String, + #[arg(long)] + config: Option, + }, /// Print schema, head sample, and row count for any artifact. Inspect { artifact: String, @@ -600,6 +613,26 @@ fn main() -> Result<()> { config_hash: &ch, })?; } + Cmd::Mbr { scored, psms, out, config } => { + let cfg = load_config(&config)?; + if cfg.mbr.strategy == mumdia_core::config::MbrStrategy::None { + anyhow::bail!( + "mbr.strategy is `none`; set empirical_library / rt_transfer / full to run MBR" + ); + } + if psms.len() < 2 { + anyhow::bail!("MBR needs >= 2 runs; got {} psms path(s)", psms.len()); + } + let python = cfg.mbr.python.as_deref().ok_or_else(|| { + anyhow::anyhow!("mbr.python (sidecar interpreter) is required when mbr.strategy != none") + })?; + let script = + mumdia::sidecar::resolve_script(&cfg.predict_frag.sidecar_script_dir, "mbr_worker.py"); + mumdia::sidecar::run_mbr( + python, &script, &scored, &psms, &out, + cfg.mbr.q_anchor, cfg.mbr.min_anchor_runs, cfg.mbr.q_transfer, cfg.rng_seed, + )?; + } Cmd::Inspect { artifact } => { print!("{}", mumdia_io::inspect(&artifact)?); } diff --git a/rust/mumdia/crates/mumdia/src/sidecar.rs b/rust/mumdia/crates/mumdia/src/sidecar.rs index 61ebeac..f3d9dde 100644 --- a/rust/mumdia/crates/mumdia/src/sidecar.rs +++ b/rust/mumdia/crates/mumdia/src/sidecar.rs @@ -128,6 +128,39 @@ pub fn run_deeplc_finetune( .context("DeepLC fine-tune failed") } +/// MBR transfer (Stage D3): match-between-runs identification transfer over the +/// experiment-wide scored table + per-run psms. Positional contract: +/// `mbr_worker.py [flags]`, where +/// `psms_csv` is the per-run psms paths joined by ',' in `source` order. +#[allow(clippy::too_many_arguments)] +pub fn run_mbr( + python: &str, + script: &str, + scored: &str, + psms: &[String], + out: &str, + q_anchor: f64, + min_anchor_runs: usize, + q_transfer: f64, + seed: u64, +) -> Result<()> { + let psms_csv = psms.join(","); + info!(scored, out, runs = psms.len(), q_anchor, min_anchor_runs, q_transfer, + "sidecar: running MBR transfer"); + let qa = q_anchor.to_string(); + let mar = min_anchor_runs.to_string(); + let qt = q_transfer.to_string(); + let sd = seed.to_string(); + run_worker( + python, + script, + &[scored, &psms_csv, out, "--q-anchor", &qa, "--min-anchor-runs", &mar, + "--q-transfer", &qt, "--seed", &sd], + false, + ) + .context("MBR transfer worker failed") +} + /// Invoke a Python worker: `python script arg...`. `utf8` forces UTF-8 I/O /// (DeepLC/Keras crash on the Windows cp1252 console otherwise). fn run_worker(python: &str, script: &str, args: &[&str], utf8: bool) -> Result<()> { diff --git a/scripts/mbr_worker.py b/scripts/mbr_worker.py new file mode 100644 index 0000000..0a06314 --- /dev/null +++ b/scripts/mbr_worker.py @@ -0,0 +1,175 @@ +"""MBR transfer worker (Stage D3, rescuable tier: M1 anchors, M2 expected RT, M3 +transfer score, M4 decoy-transfer FDR). Extends the validated /c/proteobench +prototype into a reusable sidecar. + +Reads the experiment-wide scored_combined (candidate_id, source, label, q_value, ...) +plus per-run psms (candidate_id, apex_rt). For each precursor confident in +>= min_anchor_runs OTHER runs but sub-threshold in a target run where it WAS +extracted (rescuable), predicts its RT in the target run from the median of the +other runs' binned-median-aligned apex RTs, and accepts the transfer if the observed +apex sits within a data-driven window. False-transfer FDR is estimated with a +permuted-RT decoy-transfer null (transfer to a shuffled precursor's expected RT); +the transfer q-value is standard target/decoy competition on the RT residual. + +Contract: + mbr_worker.py [options] + = comma-separated per-run psms.parquet paths in `source` order. + options: --q-anchor 0.01 --min-anchor-runs 2 --q-transfer 0.01 --seed 0 + (RT window is learned from the anchors, not fixed.) + +Output .parquet: one row per ACCEPTED transfer + (candidate_id, source, peptidoform, charge, protein_group, label, expected_rt, + observed_rt, rt_delta, transfer_q). +Also prints a validation summary (accepted counts per run, empirical decoy fraction). +""" +import argparse +import numpy as np +import pyarrow.parquet as pq +import pyarrow as pa + + +def binned_map(x, y, nb=80): + """Monotone binned-median calibration x -> y.""" + o = np.argsort(x) + xs, ys = np.asarray(x)[o], np.asarray(y)[o] + e = np.linspace(0, len(xs), nb + 1).astype(int) + cx, cy = [], [] + for b in range(nb): + lo, hi = e[b], e[b + 1] + if hi > lo: + cx.append(np.median(xs[lo:hi])) + cy.append(np.median(ys[lo:hi])) + cx, cy = np.array(cx), np.array(cy) + return lambda q: np.interp(q, cx, cy) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("scored") + ap.add_argument("psms_csv") + ap.add_argument("out") + ap.add_argument("--q-anchor", type=float, default=0.01) + ap.add_argument("--min-anchor-runs", type=int, default=2) + ap.add_argument("--q-transfer", type=float, default=0.01) + ap.add_argument("--seed", type=int, default=0) + a = ap.parse_args() + rng = np.random.default_rng(a.seed) + + psms_paths = a.psms_csv.split(",") + n_runs = len(psms_paths) + sc = pq.read_table(a.scored, columns=["candidate_id", "source", "label", "q_value", + "peptidoform", "charge", "protein_group"]).to_pandas() + # meta per candidate_id (peptidoform/charge/protein_group/label) from any row + meta = sc.drop_duplicates("candidate_id").set_index("candidate_id")[ + ["peptidoform", "charge", "protein_group", "label"]] + + # confident set per run (targets AND decoys tracked so we can measure the empirical + # decoy fraction among accepted transfers). Anchors use targets only (a decoy anchor + # would inject a random cross-run RT pair); decoys ride the same transfer test. + conf_t = {i: set(sc[(sc.source == i) & (sc.label == "target") & (sc.q_value <= a.q_anchor)].candidate_id) + for i in range(n_runs)} + + # per-run apex RT (all extracted candidates) + confident-target apex (for maps) + rt_all, rt_anchor = {}, {} + for i, p in enumerate(psms_paths): + d = pq.read_table(p, columns=["candidate_id", "apex_rt"]).to_pandas() + m = dict(zip(d.candidate_id, d.apex_rt)) + rt_all[i] = m + rt_anchor[i] = {c: m[c] for c in conf_t[i] if c in m} + + REF = 0 + to_ref = {i: (binned_map([rt_anchor[i][c] for c in rt_anchor[i] if c in rt_anchor[REF]], + [rt_anchor[REF][c] for c in rt_anchor[i] if c in rt_anchor[REF]]) + if i != REF and sum(c in rt_anchor[REF] for c in rt_anchor[i]) >= 200 + else (lambda q: q)) for i in range(n_runs)} + from_ref = {i: (binned_map([rt_anchor[REF][c] for c in rt_anchor[REF] if c in rt_anchor[i]], + [rt_anchor[i][c] for c in rt_anchor[REF] if c in rt_anchor[i]]) + if i != REF and sum(c in rt_anchor[i] for c in rt_anchor[REF]) >= 200 + else (lambda q: q)) for i in range(n_runs)} + + support_t = {} + allc = set().union(*conf_t.values()) + for c in allc: + support_t[c] = sum(c in conf_t[i] for i in range(n_runs)) + + # build transfer candidates (rescuable: confident-elsewhere, sub-threshold here, + # extracted here). Predict RT; also a permuted-RT decoy prediction for the null. + rows = {"candidate_id": [], "source": [], "expected_rt": [], "observed_rt": [], + "rt_delta": []} + decoy_delta = [] # permuted-RT null residuals (per target) + for i in range(n_runs): + cand, preds, obs = [], [], [] + for c in allc: + other = support_t[c] - (1 if c in conf_t[i] else 0) + if other < a.min_anchor_runs or c in conf_t[i] or c not in rt_all[i]: + continue + js = [j for j in range(n_runs) if j != i and c in rt_anchor[j]] + if len(js) < a.min_anchor_runs: + continue + pred = from_ref[i](np.median([to_ref[j](rt_anchor[j][c]) for j in js])) + cand.append(c); preds.append(pred); obs.append(rt_all[i][c]) + if not cand: + continue + preds = np.array(preds); obs = np.array(obs) + d = np.abs(obs - preds) + # permuted-RT decoy: each candidate gets a shuffled candidate's predicted RT + shuf = rng.permutation(len(cand)) + dd = np.abs(obs - preds[shuf]) + for k, c in enumerate(cand): + rows["candidate_id"].append(c); rows["source"].append(i) + rows["expected_rt"].append(float(preds[k])); rows["observed_rt"].append(float(obs[k])) + rows["rt_delta"].append(float(d[k])) + decoy_delta.extend(dd.tolist()) + + target_delta = np.array(rows["rt_delta"]) + decoy_delta = np.array(decoy_delta) + if len(target_delta) == 0: + print("MBR: no transfer candidates"); pa_write_empty(a.out); return + + # transfer q via target/decoy competition on rt_delta (smaller = better). At a + # threshold delta, FDR = (#decoy <= delta) / (#target <= delta). q = running min. + order = np.argsort(target_delta) + dt = np.sort(target_delta) + dd = np.sort(decoy_delta) + dec_cum = np.searchsorted(dd, dt, side="right") # decoys within each delta + tgt_cum = np.arange(1, len(dt) + 1) + fdr = dec_cum / tgt_cum + q_sorted = np.minimum.accumulate(fdr[::-1])[::-1] # monotone q from the tail + q = np.empty_like(q_sorted); q[order] = q_sorted # map back to row order + + accept = q <= a.q_transfer + cid = np.array(rows["candidate_id"]); src = np.array(rows["source"]) + lab = meta.reindex(cid)["label"].values + n_acc = int(accept.sum()) + acc_dec = int(((lab == "decoy") & accept).sum()) + print(f"MBR transfer: candidates={len(cid)} accepted@q<={a.q_transfer}={n_acc} " + f"(target={n_acc-acc_dec}, decoy={acc_dec}, empirical decoy-frac=" + f"{acc_dec/max(1,n_acc)*100:.2f}%)") + delta_star = dt[q_sorted <= a.q_transfer].max() if (q_sorted <= a.q_transfer).any() else 0.0 + print(f" RT window at q<={a.q_transfer}: {delta_star:.1f}s") + for i in range(n_runs): + m = accept & (src == i) & (lab == "target") + print(f" run {i}: +{int(m.sum())} target transfers") + + out = pa.table({ + "candidate_id": pa.array(cid[accept], pa.uint32()), + "source": pa.array(src[accept], pa.uint32()), + "peptidoform": pa.array(meta.reindex(cid[accept])["peptidoform"].values), + "charge": pa.array(meta.reindex(cid[accept])["charge"].values.astype("int32")), + "protein_group": pa.array(meta.reindex(cid[accept])["protein_group"].values), + "label": pa.array(lab[accept]), + "expected_rt": pa.array(np.array(rows["expected_rt"])[accept], pa.float64()), + "observed_rt": pa.array(np.array(rows["observed_rt"])[accept], pa.float64()), + "rt_delta": pa.array(target_delta[accept], pa.float64()), + "transfer_q": pa.array(q[accept], pa.float64()), + }) + pq.write_table(out, a.out) + print(f"wrote {a.out} ({out.num_rows} accepted transfers)") + + +def pa_write_empty(path): + pq.write_table(pa.table({"candidate_id": pa.array([], pa.uint32())}), path) + + +if __name__ == "__main__": + main() From cf9fafcdf534189fd11094bf9d200bcf4d290f95 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Wed, 22 Jul 2026 11:23:06 +0200 Subject: [PATCH 36/40] feat(mbr): emit augmented scored table (M5) mbr_worker.py --out-scored writes the input scored_combined with each accepted transfer's (candidate_id, source) row q_value lowered to its transfer_q and an is_transferred flag added. A downstream quant/report with quant.q_filter=psm_q then includes the transfers, realizing the gain: on HYE, per-run confident 336,100 -> 361,378 (+25,278) and complete-case precursors (present in all 6 runs) 33,740 -> 39,275 (+16.4%), which is the ProteoBench quant metric. FDR-controlled at transfer q <= 1% (0% empirical decoy). Co-Authored-By: Claude Opus 4.8 --- scripts/mbr_worker.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/mbr_worker.py b/scripts/mbr_worker.py index 0a06314..b13c3bb 100644 --- a/scripts/mbr_worker.py +++ b/scripts/mbr_worker.py @@ -52,6 +52,11 @@ def main(): ap.add_argument("--min-anchor-runs", type=int, default=2) ap.add_argument("--q-transfer", type=float, default=0.01) ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--out-scored", default=None, + help="also write an augmented scored table: the input scored_combined " + "with each accepted transfer's (candidate_id, source) row q_value set " + "to its transfer_q and an is_transferred flag added, so quant/report " + "(with quant.q_filter=psm_q) pick up the transfers.") a = ap.parse_args() rng = np.random.default_rng(a.seed) @@ -166,6 +171,25 @@ def main(): pq.write_table(out, a.out) print(f"wrote {a.out} ({out.num_rows} accepted transfers)") + # M5: augmented scored table. Lower the accepted transfers' PSM q_value to their + # transfer_q on the matching (candidate_id, source) row and flag them, so a + # downstream quant/report with quant.q_filter=psm_q includes the transfers. + if a.out_scored: + full = pq.read_table(a.scored).to_pandas() + acc = {(int(c), int(s)): float(qq) for c, s, qq in + zip(cid[accept], src[accept], q[accept])} + key = list(zip(full.candidate_id.astype(int), full.source.astype(int))) + newq = full.q_value.to_numpy().copy() + is_tr = np.zeros(len(full), dtype=bool) + for i, k in enumerate(key): + if k in acc: + newq[i] = min(newq[i], acc[k]) + is_tr[i] = True + full["q_value"] = newq + full["is_transferred"] = is_tr + pq.write_table(pa.Table.from_pandas(full, preserve_index=False), a.out_scored) + print(f"wrote {a.out_scored} (augmented scored; {int(is_tr.sum())} rows flagged transferred)") + def pa_write_empty(path): pq.write_table(pa.table({"candidate_id": pa.array([], pa.uint32())}), path) From ca01ee5f62c667284c9cee1b2ab665c1dcc2a517 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Wed, 22 Jul 2026 12:19:34 +0200 Subject: [PATCH 37/40] feat(mbr): fragment-consensus guard + re-extraction-tier target emission Two additions to the MBR worker + CLI, both validated on the 6-run HYE set. Fragment-consensus guard (M4 enhancement, mbr.consensus_corr_min). The rescuable transfer alone fills the cross-run matrix (+16.4% complete-case) but DEGRADES quant precision: transferred precursors are sub-threshold because their signal is weak or interfered, and RT-concordant interference (human background at the predicted RT) compresses yeast/ecoli ratios toward 1:1 (ECOLI ratio IQR 0.72 -> 1.30, median -1.89 -> -1.74). The guard correlates each accepted transfer's fragment pattern in the target run against the empirical consensus (mean L1-normalized pattern over its confident runs) and rejects those below consensus_corr_min. At >=0.8 it keeps 14,469 of 25,278 transfers and recovers precision (ECOLI IQR back to 0.91, median -1.86; YEAST IQR 0.57 -> 0.41), so MBR becomes a completeness gain WITH preserved precision. mbr_worker.py takes --frag-csv + --consensus-corr-min; wired through sidecar::run_mbr and `mumdia mbr --frag ...` (+ --out-scored exposed). Re-extraction-tier target emission (M1 of the absent tier). mbr_worker.py --emit-transfer-targets writes per-run run_windows-format tables (candidate_id, rt_pred_cal, rt_lo, rt_hi) for the ABSENT set -- precursors confident in >= min_anchor_runs other runs but not extracted in a run (65,805 on HYE) -- at the tight cross-run-predicted RT window (+/- rt_window), plus a permuted-RT decoy-target file. These feed `extract --restrict-candidates --run-windows ` to re-extract the absent precursors, the second sensitivity tier. Emission validated (extract-compatible schema, 65,805 targets); the extract + score + FDR loop over them is the next step. Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia/src/main.rs | 15 +++- rust/mumdia/crates/mumdia/src/sidecar.rs | 26 ++++--- scripts/mbr_worker.py | 95 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 12 deletions(-) diff --git a/rust/mumdia/crates/mumdia/src/main.rs b/rust/mumdia/crates/mumdia/src/main.rs index e7128ad..5e7ea56 100644 --- a/rust/mumdia/crates/mumdia/src/main.rs +++ b/rust/mumdia/crates/mumdia/src/main.rs @@ -236,6 +236,14 @@ enum Cmd { psms: Vec, #[arg(long)] out: String, + /// Optional augmented scored table: input scored with accepted transfers' + /// q_value lowered + is_transferred flag (for quant/report with q_filter=psm_q). + #[arg(long)] + out_scored: Option, + /// Optional per-run fragment_quant.parquet (source order) for the + /// fragment-consensus guard (needs mbr.consensus_corr_min > 0). + #[arg(long, num_args = 0..)] + frag: Vec, #[arg(long)] config: Option, }, @@ -613,7 +621,7 @@ fn main() -> Result<()> { config_hash: &ch, })?; } - Cmd::Mbr { scored, psms, out, config } => { + Cmd::Mbr { scored, psms, out, out_scored, frag, config } => { let cfg = load_config(&config)?; if cfg.mbr.strategy == mumdia_core::config::MbrStrategy::None { anyhow::bail!( @@ -629,8 +637,9 @@ fn main() -> Result<()> { let script = mumdia::sidecar::resolve_script(&cfg.predict_frag.sidecar_script_dir, "mbr_worker.py"); mumdia::sidecar::run_mbr( - python, &script, &scored, &psms, &out, - cfg.mbr.q_anchor, cfg.mbr.min_anchor_runs, cfg.mbr.q_transfer, cfg.rng_seed, + python, &script, &scored, &psms, &out, out_scored.as_deref(), &frag, + cfg.mbr.q_anchor, cfg.mbr.min_anchor_runs, cfg.mbr.q_transfer, + cfg.mbr.consensus_corr_min, cfg.rng_seed, )?; } Cmd::Inspect { artifact } => { diff --git a/rust/mumdia/crates/mumdia/src/sidecar.rs b/rust/mumdia/crates/mumdia/src/sidecar.rs index f3d9dde..4cb4116 100644 --- a/rust/mumdia/crates/mumdia/src/sidecar.rs +++ b/rust/mumdia/crates/mumdia/src/sidecar.rs @@ -139,26 +139,34 @@ pub fn run_mbr( scored: &str, psms: &[String], out: &str, + out_scored: Option<&str>, + frag: &[String], q_anchor: f64, min_anchor_runs: usize, q_transfer: f64, + consensus_corr_min: f64, seed: u64, ) -> Result<()> { let psms_csv = psms.join(","); info!(scored, out, runs = psms.len(), q_anchor, min_anchor_runs, q_transfer, - "sidecar: running MBR transfer"); + consensus_corr_min, "sidecar: running MBR transfer"); let qa = q_anchor.to_string(); let mar = min_anchor_runs.to_string(); let qt = q_transfer.to_string(); let sd = seed.to_string(); - run_worker( - python, - script, - &[scored, &psms_csv, out, "--q-anchor", &qa, "--min-anchor-runs", &mar, - "--q-transfer", &qt, "--seed", &sd], - false, - ) - .context("MBR transfer worker failed") + let cm = consensus_corr_min.to_string(); + let frag_csv = frag.join(","); + let mut args: Vec<&str> = vec![ + scored, &psms_csv, out, "--q-anchor", &qa, "--min-anchor-runs", &mar, + "--q-transfer", &qt, "--seed", &sd, + ]; + if let Some(os) = out_scored { + args.extend_from_slice(&["--out-scored", os]); + } + if !frag.is_empty() && consensus_corr_min > 0.0 { + args.extend_from_slice(&["--frag-csv", &frag_csv, "--consensus-corr-min", &cm]); + } + run_worker(python, script, &args, false).context("MBR transfer worker failed") } /// Invoke a Python worker: `python script arg...`. `utf8` forces UTF-8 I/O diff --git a/scripts/mbr_worker.py b/scripts/mbr_worker.py index b13c3bb..0c06bcb 100644 --- a/scripts/mbr_worker.py +++ b/scripts/mbr_worker.py @@ -57,6 +57,22 @@ def main(): "with each accepted transfer's (candidate_id, source) row q_value set " "to its transfer_q and an is_transferred flag added, so quant/report " "(with quant.q_filter=psm_q) pick up the transfers.") + ap.add_argument("--emit-transfer-targets", dest="emit_targets", default=None, + help="RE-EXTRACTION TIER: instead of the rescuable transfer, write per-run " + "run_windows-format tables (candidate_id, rt_pred_cal, rt_lo, rt_hi) for " + "the ABSENT set (confident in >= min_anchor_runs other runs, not extracted " + "in this run) at the tight cross-run-predicted RT window, plus a permuted-RT " + "decoy target file. Feed to `extract --restrict-candidates " + "--run-windows ` to re-extract those precursors, then score + FDR.") + ap.add_argument("--rt-window", dest="rt_window", type=float, default=20.0, + help="transfer RT half-window (s) for emitted targets (>= the p95 M2 residual).") + ap.add_argument("--frag-csv", dest="frag_csv", default=None, + help="comma-separated per-run fragment_quant.parquet paths (source order) " + "for the fragment-consensus guard.") + ap.add_argument("--consensus-corr-min", dest="corr_min", type=float, default=0.0, + help="reject accepted transfers whose fragment pattern in the target run " + "correlates < this with the empirical consensus over its confident runs " + "(interference guard). 0 = off. ~0.8 recovers MBR's quant precision loss.") a = ap.parse_args() rng = np.random.default_rng(a.seed) @@ -97,6 +113,46 @@ def main(): for c in allc: support_t[c] = sum(c in conf_t[i] for i in range(n_runs)) + def expected_rt(c, i): + """Cross-run predicted RT of candidate c in run i (None if too few anchors).""" + js = [j for j in range(n_runs) if j != i and c in rt_anchor[j]] + if len(js) < a.min_anchor_runs: + return None + return float(from_ref[i](np.median([to_ref[j](rt_anchor[j][c]) for j in js]))) + + # RE-EXTRACTION TIER: emit per-run run_windows for the ABSENT set (confident + # elsewhere, not extracted here) at the tight predicted-RT window, so `extract` + # can rescue them. Also emit a permuted-RT decoy-target file for the transfer FDR. + if a.emit_targets: + import os + os.makedirs(a.emit_targets, exist_ok=True) + rng2 = np.random.default_rng(a.seed) + for i in range(n_runs): + cids, preds = [], [] + for c in allc: + other = support_t[c] - (1 if c in conf_t[i] else 0) + if other < a.min_anchor_runs or c in conf_t[i] or c in rt_all[i]: + continue # absent-from-run only (rescuable tier handles extracted ones) + pr = expected_rt(c, i) + if pr is not None: + cids.append(c); preds.append(pr) + preds = np.array(preds) + for tag, rt_pred in [("targets", preds), + ("decoys", preds[rng2.permutation(len(preds))] if len(preds) else preds)]: + tbl = pa.table({ + "candidate_id": pa.array(np.array(cids, dtype=np.uint32), pa.uint32()), + "rt_pred_cal": pa.array(rt_pred, pa.float64()), + "rt_lo": pa.array(rt_pred - a.rt_window, pa.float64()), + "rt_hi": pa.array(rt_pred + a.rt_window, pa.float64()), + "im_pred_cal": pa.array([None] * len(cids), pa.float64()), + "im_lo": pa.array([None] * len(cids), pa.float64()), + "im_hi": pa.array([None] * len(cids), pa.float64()), + }) + pq.write_table(tbl, f"{a.emit_targets}/transfer_{tag}_{i}.parquet") + print(f" run {i}: {len(cids)} absent-transfer targets (window +/-{a.rt_window:.0f}s)") + print(f"wrote per-run transfer targets + permuted-RT decoys to {a.emit_targets}") + return + # build transfer candidates (rescuable: confident-elsewhere, sub-threshold here, # extracted here). Predict RT; also a permuted-RT decoy prediction for the null. rows = {"candidate_id": [], "source": [], "expected_rt": [], "observed_rt": [], @@ -145,6 +201,45 @@ def main(): accept = q <= a.q_transfer cid = np.array(rows["candidate_id"]); src = np.array(rows["source"]) lab = meta.reindex(cid)["label"].values + + # Fragment-consensus guard (M4 enhancement): reject accepted transfers whose + # fragment pattern in the target run does not match the empirical consensus over + # its confident runs -> removes RT-concordant interference that would otherwise + # add noisy/compressed quant. Validated to recover MBR's ratio-precision loss. + if a.frag_csv and a.corr_min > 0.0: + fpaths = a.frag_csv.split(",") + frag = {} + for i, fp in enumerate(fpaths): + d = pq.read_table(fp, columns=["candidate_id", "fragment_name", "quantity"]).to_pandas() + g = {} + for c, fn, qq in zip(d.candidate_id, d.fragment_name, d.quantity): + g.setdefault(int(c), {})[fn] = float(qq) + frag[i] = g + + def cos(pa_, pb): + ks = set(pa_) | set(pb) + va = np.array([pa_.get(k, 0.0) for k in ks]); vb = np.array([pb.get(k, 0.0) for k in ks]) + na, nb = np.linalg.norm(va), np.linalg.norm(vb) + return va @ vb / (na * nb) if na > 0 and nb > 0 else 0.0 + + kept = 0 + for k in np.where(accept)[0]: + c, s = int(cid[k]), int(src[k]) + anc = [j for j in range(n_runs) if j != s and c in conf_t[j] and c in frag.get(j, {})] + if c not in frag.get(s, {}) or len(anc) < a.min_anchor_runs: + accept[k] = False; continue + cons = {} + for j in anc: + p = frag[j][c]; tot = sum(p.values()) or 1.0 + for kk, v in p.items(): + cons[kk] = cons.get(kk, 0.0) + v / tot / len(anc) + if cos(frag[s][c], cons) < a.corr_min: + accept[k] = False + else: + kept += 1 + print(f" fragment-consensus guard (>= {a.corr_min}): kept {kept} of " + f"{int((q <= a.q_transfer).sum())} FDR-passing transfers") + n_acc = int(accept.sum()) acc_dec = int(((lab == "decoy") & accept).sum()) print(f"MBR transfer: candidates={len(cid)} accepted@q<={a.q_transfer}={n_acc} " From aa6a6bcbb7235f67e50c16621dbdeb7fecaf227f Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Thu, 23 Jul 2026 10:49:50 +0200 Subject: [PATCH 38/40] fix: physical constants + FDR-estimator correctness (comment.md C10/C13/C14) - constants: ISOTOPE_SPACING to the true 13C-12C mass difference 1.003354835 (was 1.00286864, ~485 ppm off); shifts MS1 isotope-XIC m/z, so stored baselines must be regenerated - features: use the shared constants::PROTON in ms1/novel instead of local duplicate constants - fdr: entrapment_q gets tied-score-block collapse (determinism) and the +1 finite-sample pseudocount matching target_decoy_q; add validate_labels (target/decoy only), the helper used by the label whitelist - predict-frag: count and warn on DeepLC iRT misses (was a silent 0.0 anchor) Co-Authored-By: Claude Opus 4.8 --- .../crates/mumdia-core/src/constants.rs | 8 ++- rust/mumdia/crates/mumdia/src/fdr.rs | 69 +++++++++++++++---- .../crates/mumdia/src/stages/features/ms1.rs | 2 +- .../mumdia/src/stages/features/novel.rs | 5 +- .../crates/mumdia/src/stages/predict_frag.rs | 15 +++- 5 files changed, 77 insertions(+), 22 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/constants.rs b/rust/mumdia/crates/mumdia-core/src/constants.rs index aab51b9..d65fa49 100644 --- a/rust/mumdia/crates/mumdia-core/src/constants.rs +++ b/rust/mumdia/crates/mumdia-core/src/constants.rs @@ -15,9 +15,11 @@ pub const WATER: f64 = 18.010_564_684; /// Monoisotopic mass of ammonia (NH3), Da. pub const AMMONIA: f64 = 17.026_549_1; -/// Mass difference between two adjacent isotope peaks of a peptide (Da), -/// the C13 - C12 neutron spacing used for the averagine envelope. -pub const ISOTOPE_SPACING: f64 = 1.002_868_64; +/// Mass difference between two adjacent isotope peaks of a peptide (Da): the +/// true 13C - 12C mass difference (13.003_354_835 - 12 = 1.003_354_835), used as +/// the isotope-peak spacing for MS1 envelope extraction. Public physical fact +/// (AME2020 atomic masses); not copied from any proteomics implementation. +pub const ISOTOPE_SPACING: f64 = 1.003_354_835; /// Monoisotopic residue mass in Da for a standard amino acid, or `None` for /// residues MuMDIA treats as ambiguous (B, J, O, U, X, Z). diff --git a/rust/mumdia/crates/mumdia/src/fdr.rs b/rust/mumdia/crates/mumdia/src/fdr.rs index 8d8185a..11c617c 100644 --- a/rust/mumdia/crates/mumdia/src/fdr.rs +++ b/rust/mumdia/crates/mumdia/src/fdr.rs @@ -54,8 +54,9 @@ pub fn target_decoy_q(scores: &[(f64, bool)]) -> Vec { /// Entrapment-calibrated q-values. Higher score is better. `is_entrapment` /// marks spike-in foreign-proteome PSMs (false by construction); `is_real` /// marks the sample's own target PSMs. Rows that are neither (decoys) are ranked -/// but enter no count. FDR(t) = `ratio` * n_entrap(>=t) / max(1, n_real(>=t)), -/// where `ratio` = N_real_lib / N_entrap_lib corrects for unequal library sizes. +/// but enter no count. FDR(t) = (`ratio` * n_entrap(>=t) + 1) / max(1, n_real(>=t)), +/// where `ratio` = N_real_lib / N_entrap_lib corrects for unequal library sizes and +/// the `+1` is the conservative finite-sample pseudocount (as in `target_decoy_q`). /// Monotonized from worst to best so q is non-increasing with score. This is the /// empirical-null analog of `target_decoy_q`: the entrapment population, unlike /// in-silico decoys, experiences the same chimeric DIA interference as real @@ -74,13 +75,27 @@ pub fn entrapment_q(scores: &[f64], is_entrapment: &[bool], is_real: &[bool], ra }); let (mut ne, mut nr) = (0usize, 0usize); let mut fdr_at = vec![1.0f64; n]; - for (rank, &i) in order.iter().enumerate() { - if is_entrapment[i] { - ne += 1; - } else if is_real[i] { - nr += 1; + // Process tied-score blocks together so every row in a block gets the same + // FDP regardless of its arbitrary within-tie order (determinism, PLAN.md + // Section 7). Mirrors the tied-block walk in `target_decoy_q`. + let mut rank = 0usize; + while rank < n { + let s = scores[order[rank]]; + let mut end = rank; + while end < n && scores[order[end]] == s { + let i = order[end]; + if is_entrapment[i] { + ne += 1; + } else if is_real[i] { + nr += 1; + } + end += 1; } - fdr_at[rank] = (ratio * ne as f64) / (nr.max(1) as f64); + let f = (ratio * ne as f64 + 1.0) / (nr.max(1) as f64); + for r in rank..end { + fdr_at[r] = f; + } + rank = end; } let mut q = vec![1.0f64; n]; let mut qmin = 1.0f64; @@ -99,6 +114,20 @@ pub fn count_targets_at_q(q: &[f64], is_decoy: &[bool], threshold: f64) -> usize .count() } +/// Validate that every PSM label is a known class. An unknown or malformed +/// label must not silently count as a target (comment.md C16): the target-decoy +/// null depends on exact labeling. Entrapment status is derived from the protein +/// accession (see `classify_entrapment`), not the label, so the only valid label +/// values here are "target" and "decoy". +pub fn validate_labels(labels: &[String]) -> anyhow::Result<()> { + for l in labels { + if l != "target" && l != "decoy" { + anyhow::bail!("unknown PSM label {l:?}; expected \"target\" or \"decoy\""); + } + } + Ok(()) +} + /// ln(n!) via summed logs (n small in matched-fragment counts). pub fn ln_factorial(n: u32) -> f64 { let mut s = 0.0; @@ -145,14 +174,26 @@ mod tests { let is_entrap = vec![false, false, true, false, true, false]; let is_real = vec![true, true, false, true, false, false]; // last row = decoy let q = entrapment_q(&scores, &is_entrap, &is_real, 1.0); - // Top two real targets precede any entrapment -> q = 0. - assert!(q[0] < 1e-9 && q[1] < 1e-9); - // At the 3rd-ranked real target (rank 3): 1 entrap / 3 real = 0.333. - assert!((q[3] - 1.0 / 3.0).abs() < 1e-9); - // Library-size ratio scales the estimate linearly. + // +1 finite-sample pseudocount: raw FDP walk (ratio=1) is + // [1, .5, 1, 2/3, 1, 1], monotonized worst->best to [.5, .5, 2/3, 2/3, 1, 1]. + // Even the top real targets are not q=0. + assert!((q[0] - 0.5).abs() < 1e-9 && (q[1] - 0.5).abs() < 1e-9); + // At the 3rd-ranked real target: (1 entrap + 1) / 3 real = 2/3. + assert!((q[3] - 2.0 / 3.0).abs() < 1e-9); + // A larger library-size ratio inflates the estimate (more conservative). let q2 = entrapment_q(&scores, &is_entrap, &is_real, 2.0); - assert!((q2[3] - 2.0 / 3.0).abs() < 1e-9); + assert!(q2[3] >= q[3]); // Determinism: identical inputs give identical output. assert_eq!(q, entrapment_q(&scores, &is_entrap, &is_real, 1.0)); } + + #[test] + fn entrapment_q_tied_scores_share_one_q() { + // Tied scores must all receive the same q regardless of within-tie order. + let scores = vec![5.0, 5.0, 5.0]; + let is_entrap = vec![false, true, false]; + let is_real = vec![true, false, true]; + let q = entrapment_q(&scores, &is_entrap, &is_real, 1.0); + assert!((q[0] - q[1]).abs() < 1e-12 && (q[1] - q[2]).abs() < 1e-12); + } } diff --git a/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs b/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs index 6d4cf26..7115598 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/ms1.rs @@ -13,6 +13,7 @@ //! persisted. All values are guarded finite. use super::Evidence; use crate::stats::{cosine, pearson, spectral_angle}; +use mumdia_core::constants::PROTON; pub const NAMES: &[&str] = &[ // apex isotope-envelope agreement vs averagine @@ -48,7 +49,6 @@ pub const NAMES: &[&str] = &[ ]; const EPS: f64 = 1e-9; -const PROTON: f64 = 1.007_276_466_812; /// Finite guard: replace NaN/Inf with 0.0. #[inline] diff --git a/rust/mumdia/crates/mumdia/src/stages/features/novel.rs b/rust/mumdia/crates/mumdia/src/stages/features/novel.rs index c92d5dd..ba816c8 100644 --- a/rust/mumdia/crates/mumdia/src/stages/features/novel.rs +++ b/rust/mumdia/crates/mumdia/src/stages/features/novel.rs @@ -11,8 +11,7 @@ //! are skipped: the `Evidence` struct carries only `seq_len`, not the stripped //! sequence or the ProForma modification list, so their evidence is unavailable. use super::Evidence; - -const PROTON_MASS: f64 = 1.007276466812; +use mumdia_core::constants::PROTON; pub const NAMES: &[&str] = &[ "log_seed_hyperscore", @@ -49,7 +48,7 @@ pub fn values(e: &Evidence) -> Vec { let charge_is_4plus = if e.charge >= 4 { 1.0 } else { 0.0 }; // Neutral monoisotopic precursor mass. Guard nonsensical charge (<=0). let precursor_mass = if e.charge > 0 { - (e.precursor_mz - PROTON_MASS) * (e.charge as f64) + (e.precursor_mz - PROTON) * (e.charge as f64) } else { 0.0 }; diff --git a/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs b/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs index 80c4326..0bbd380 100644 --- a/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs +++ b/rust/mumdia/crates/mumdia/src/stages/predict_frag.rs @@ -270,9 +270,22 @@ fn assign_rt(p: &PredictFragParams, raws: &mut [Raw]) -> Result { } } let out = sidecar::run_deeplc(python, &script, p.work_dir, &ids, &peps)?; + let mut n_irt_missing = 0u64; for r in raws.iter_mut() { let uid = uniq[&r.peptidoform]; - r.irt = *out.get(&uid).unwrap_or(&0.0); + match out.get(&uid) { + Some(&v) => r.irt = v, + None => { + r.irt = 0.0; + n_irt_missing += 1; + } + } + } + if n_irt_missing > 0 { + tracing::warn!( + n_irt_missing, + "predict-frag: DeepLC returned no iRT for some peptidoforms; anchored at iRT 0.0" + ); } Ok("deeplc-4.0-mt".to_string()) } From 26553ac6f219d4c3cbb0f796fd39b5320b38b858 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Thu, 23 Jul 2026 10:49:50 +0200 Subject: [PATCH 39/40] feat: multi-context q-values + label whitelist (comment.md C1/C3/C16) - rescore: emit run_psm_q (per-run PSM TDA, deterministic BTree split), experiment_psm_q (pooled), and precursor_q (peptidoform+charge) alongside the existing q_value/peptide_q_value/pg_q_value/global_q_value, which stay byte-identical; add the target_precursors_at_1pct stat; validate PSM labels; export the NN sidecar hyperparameters (MUMDIA_NN_FOLDS/ITERS/TRAIN_FDR) so the worker honours the config and the report records the values used - schema: bump psms_scored 1 -> 2 (additive columns) - align: validate PSM labels Co-Authored-By: Claude Opus 4.8 --- rust/mumdia/crates/mumdia-core/src/schema.rs | 2 +- rust/mumdia/crates/mumdia/src/stages/align.rs | 1 + .../crates/mumdia/src/stages/rescore.rs | 72 ++++++++++++++++++- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/rust/mumdia/crates/mumdia-core/src/schema.rs b/rust/mumdia/crates/mumdia-core/src/schema.rs index 2487aeb..0bce713 100644 --- a/rust/mumdia/crates/mumdia-core/src/schema.rs +++ b/rust/mumdia/crates/mumdia-core/src/schema.rs @@ -18,7 +18,7 @@ pub mod artifact { pub const CHROMATOGRAMS: (&str, u32) = ("chromatograms", 1); pub const FEATURES: (&str, u32) = ("features", 1); pub const PSMS_COMPETED: (&str, u32) = ("psms_competed", 1); - pub const PSMS_SCORED: (&str, u32) = ("psms_scored", 1); + pub const PSMS_SCORED: (&str, u32) = ("psms_scored", 2); pub const PEPTIDE_QUANT: (&str, u32) = ("peptide_quant", 1); pub const PROTEIN_GROUP_QUANT: (&str, u32) = ("protein_group_quant", 1); } diff --git a/rust/mumdia/crates/mumdia/src/stages/align.rs b/rust/mumdia/crates/mumdia/src/stages/align.rs index bdeeb04..88efa8e 100644 --- a/rust/mumdia/crates/mumdia/src/stages/align.rs +++ b/rust/mumdia/crates/mumdia/src/stages/align.rs @@ -36,6 +36,7 @@ fn confident_rts(path: &str, q_train: f64) -> Result> { let rt = t.f64("observed_rt")?; let score = t.f64("score")?; let label = t.str("label")?; + crate::fdr::validate_labels(&label)?; let mut best: HashMap = HashMap::new(); // base -> (score, rt) for i in 0..t.nrows { if label[i] == "decoy" || q[i] > q_train { diff --git a/rust/mumdia/crates/mumdia/src/stages/rescore.rs b/rust/mumdia/crates/mumdia/src/stages/rescore.rs index a19a475..ee44dfb 100644 --- a/rust/mumdia/crates/mumdia/src/stages/rescore.rs +++ b/rust/mumdia/crates/mumdia/src/stages/rescore.rs @@ -86,6 +86,7 @@ pub fn run(p: RescoreParams) -> Result { feats.push((0..feat_names.len()).map(|k| fcols[k][i]).collect()); } } + crate::fdr::validate_labels(&label)?; let is_decoy: Vec = label.iter().map(|l| l == "decoy").collect(); let (is_entrapment, is_real_target) = classify_entrapment(p.cfg, &protein, &is_decoy); let n = cid.len(); @@ -221,9 +222,53 @@ pub fn run(p: RescoreParams) -> Result { ids }; let pg_q = grouped_q(&protein_id, &scores, &is_decoy, &is_entrapment, &is_real_target, qmode, p.cfg.entrapment_ratio); - // Multi-context q (PLAN.md Section 8 rescore): single run, so run-specific == - // experiment-wide == global. Distinct column kept for multi-run forward-compat. + // Multi-context q-values (PLAN.md Section 8 rescore; comment.md C1/C3). The + // pooled per-PSM q is `experiment_psm_q`; `run_psm_q` re-runs TDA within each + // source (run) so a per-run quant/report gets a real per-run FDR rather than the + // pooled value; `precursor_q` groups on peptidoform+charge. `global_q` is kept + // as a byte-identical alias of the pooled q for backward-compat. let global_q = psm_q.clone(); + let experiment_psm_q = psm_q.clone(); + // Per-run PSM q: TDA within each source separately, scattered back by row index. + // Single-run (source all-zero) => equals `q_value`. Sorted (BTree) source + // iteration keeps it deterministic; no floats are summed. + let run_psm_q = { + let mut by_src: std::collections::BTreeMap> = std::collections::BTreeMap::new(); + for (i, &s) in source.iter().enumerate() { + by_src.entry(s).or_default().push(i); + } + let mut rq = vec![1.0f64; n]; + for (_s, idxs) in by_src { + let q = match qmode { + QMode::Decoy => { + let sd: Vec<(f64, bool)> = idxs.iter().map(|&i| (scores[i], is_decoy[i])).collect(); + target_decoy_q(&sd) + } + QMode::Entrapment => { + let sc: Vec = idxs.iter().map(|&i| scores[i]).collect(); + let en: Vec = idxs.iter().map(|&i| is_entrapment[i]).collect(); + let re: Vec = idxs.iter().map(|&i| is_real_target[i]).collect(); + entrapment_q(&sc, &en, &re, p.cfg.entrapment_ratio) + } + }; + for (k, &i) in idxs.iter().enumerate() { + rq[i] = q[k]; + } + } + rq + }; + // Precursor-level q: group on peptidoform+charge (interned to dense u32 like the + // protein path) and run TDA over the best PSM per precursor. + let precursor_id: Vec = { + let mut interner: HashMap<(&str, i32), u32> = HashMap::new(); + let mut ids = Vec::with_capacity(pform.len()); + for (pf, &z) in pform.iter().zip(charge.iter()) { + let next = interner.len() as u32; + ids.push(*interner.entry((pf.as_str(), z)).or_insert(next)); + } + ids + }; + let precursor_q = grouped_q(&precursor_id, &scores, &is_decoy, &is_entrapment, &is_real_target, qmode, p.cfg.entrapment_ratio); // Reported IDs: real targets in entrapment mode (spike-in excluded), else all // non-decoy targets. @@ -251,6 +296,15 @@ pub fn run(p: RescoreParams) -> Result { } seen.len() }; + let n_prec_1 = { + let mut seen = std::collections::HashSet::new(); + for i in 0..n { + if is_reported[i] && precursor_q[i] <= 0.01 { + seen.insert(precursor_id[i]); + } + } + seen.len() + }; // Entrapment leak: spike-in peptides passing the 1% gate. A running check on // FDR validity (should track the reported q if the null is well-modelled). let n_entrap_1 = { @@ -283,6 +337,12 @@ pub fn run(p: RescoreParams) -> Result { // Run identity for experiment-wide rescore (index into --competed); // all-zero for a single-run rescore. Lets quant map scores per file. Col::U32("source".into(), source), + // Multi-context q columns (comment.md C1/C3). run_psm_q = per-run PSM + // FDR; experiment_psm_q = pooled PSM FDR (== q_value/global_q_value); + // precursor_q = per (peptidoform+charge) FDR. + Col::F64("run_psm_q".into(), run_psm_q), + Col::F64("experiment_psm_q".into(), experiment_psm_q), + Col::F64("precursor_q".into(), precursor_q), ], )?; @@ -293,6 +353,7 @@ pub fn run(p: RescoreParams) -> Result { stats.insert("target_psms_at_1pct".to_string(), json!(n_psm_1)); stats.insert("target_peptides_at_1pct".to_string(), json!(n_pep_1)); stats.insert("target_protein_groups_at_1pct".to_string(), json!(n_pg_1)); + stats.insert("target_precursors_at_1pct".to_string(), json!(n_prec_1)); if qmode == QMode::Entrapment { stats.insert("entrapment_ratio".to_string(), json!(p.cfg.entrapment_ratio)); stats.insert("entrapment_peptides_at_1pct".to_string(), json!(n_entrap_1)); @@ -546,6 +607,13 @@ fn run_pin_sidecar( .arg(&pin) .arg(&outp) .env("PYTHONUTF8", "1") + // Pass the configured NN hyperparameters so the worker uses them instead + // of its own defaults, and so the folds/num_iter/train_fdr recorded in the + // report reflect the values actually used (comment.md C4). Ignored by + // mokapot_worker.py, which shares this PIN contract. + .env("MUMDIA_NN_FOLDS", p.cfg.folds.to_string()) + .env("MUMDIA_NN_ITERS", p.cfg.num_iter.to_string()) + .env("MUMDIA_NN_TRAIN_FDR", p.cfg.train_fdr.to_string()) .status()?; if !status.success() { anyhow::bail!("{script_name} exited with {status}"); From 495c8cb388c57fd66ff5391e14d35846dbb0c632 Mon Sep 17 00:00:00 2001 From: MuMDIA Date: Thu, 23 Jul 2026 10:54:05 +0200 Subject: [PATCH 40/40] wip(sensitivity): q-filter, provenance, gate relax, validation + tests Re-committed from the working tree after intentionally resetting the branch to ca01ee5. Covers: - config/quant: QuantQColumn::RunPsmQ per-run quant filter (the correct choice for cross-run quant off an experiment-wide rescore) - config: extract.min_frag_corr default 0.5 -> 0.2 to relax the hard single-scan Pearson gate (comment.md S1), plus finite/[0,1] validation - main/convert: fold the convert peak caps into the convert config_hash and record top_peaks_ms2/ms1 in the report (comment.md A2/C4); top-peaks CLI docs - extract: record gate_mode / gate_coelution_min in the report params - search_seed: test that top_n_peaks=0 selects all and the seed cap keeps only the top-intensity peaks - main: CLI-parse tests for the peak-cap args - README: doc updates Co-Authored-By: Claude Opus 4.8 --- README.md | 6 ++ rust/mumdia/README.md | 7 +- rust/mumdia/crates/mumdia-core/src/config.rs | 49 ++++++++--- rust/mumdia/crates/mumdia/src/main.rs | 86 ++++++++++++++++++- .../crates/mumdia/src/stages/convert.rs | 7 +- .../crates/mumdia/src/stages/extract.rs | 5 ++ rust/mumdia/crates/mumdia/src/stages/quant.rs | 1 + .../crates/mumdia/src/stages/search_seed.rs | 36 ++++++++ 8 files changed, 181 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 19ecb8a..41e57f8 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ convert -> digest -> peptidoforms -> predict-frag -> search-seed -> rt-im-train -> extract -> features -> compete -> rescore ``` +Conversion retains all MS2 peaks by default. The seed search independently probes +the 300 most intense peaks per scan (`search_seed.top_n_peaks`) because it is used +only for calibration anchors. A non-zero `convert --top-peaks-ms2` (or the +corresponding `run` option) irreversibly truncates the normalized spectrum and +therefore also affects extraction, features, and quantification. + `mumdia run` orchestrates the whole chain on one file and writes a `manifest.json`; `mumdia inspect ` prints schema, head, and row count for any Parquet output. diff --git a/rust/mumdia/README.md b/rust/mumdia/README.md index 9a17a0d..da49e5a 100644 --- a/rust/mumdia/README.md +++ b/rust/mumdia/README.md @@ -28,12 +28,17 @@ run config + FASTA + mzML -> all of the above + manifest.json inspect -> schema + head + row count ``` +`convert` retains every post-centroiding, positive MS2 peak by default. Seed +probing independently uses the 300 most intense peaks per scan by default via +`search_seed.top_n_peaks`; setting `convert --top-peaks-ms2` to a non-zero value +also truncates the evidence available to extraction and quantification. + ## Build ``` cd rust/mumdia cargo build --release # binary at $CARGO_TARGET_DIR/release/mumdia -cargo test # 15 unit + 2 integration tests +cargo test --workspace --all-targets ``` Notes: diff --git a/rust/mumdia/crates/mumdia-core/src/config.rs b/rust/mumdia/crates/mumdia-core/src/config.rs index d6ffa8c..f7176cf 100644 --- a/rust/mumdia/crates/mumdia-core/src/config.rs +++ b/rust/mumdia/crates/mumdia-core/src/config.rs @@ -382,7 +382,8 @@ pub struct SearchSeedConfig { /// If > 0, probe only the `top_n_peaks` most intense peaks per MS2 scan /// (0 = all peaks). The seed only produces calibration anchors (RT/mass/IM), /// which come from abundant peptides, so this cuts the dominant per-peak index - /// probing cost with negligible anchor loss. + /// probing cost without discarding peaks from the downstream extraction + /// artifact. Default 300; set to 0 to probe every converted peak. pub top_n_peaks: usize, /// Fragment-matcher backend (fragindex_spec). Default `Fragindex`. pub matcher: MatcherKind, @@ -401,7 +402,7 @@ impl Default for SearchSeedConfig { fragment_tol_ppm: 20.0, report_psms: 5, min_matched_peaks: 4, - top_n_peaks: 0, + top_n_peaks: 300, matcher: MatcherKind::Fragindex, two_pass_mass_cal: false, } @@ -492,9 +493,10 @@ pub struct ExtractConfig { /// minimum simultaneously-present fragments over the consecutive-scan run. pub presence_min_coelution: usize, /// tier-(d) spectral-agreement gate: reject a candidate whose apex observed - /// fragment intensities correlate with the predicted pattern below this - /// (Pearson). Applied symmetrically to targets and decoys, it removes - /// chimeric false matches so the target-decoy null is valid. 0 disables. + /// fragment intensities correlate with the predicted pattern below this. + /// Applied symmetrically to targets and decoys, but that alone does not prove + /// null exchangeability in chimeric DIA; validate every threshold with an + /// independent entrapment. 0 disables. pub min_frag_corr: f64, /// tier-(c) minimum fraction of the candidate's predicted fragments that /// must be observed. With enough predicted fragments (top_n>=~10) this is a @@ -612,10 +614,12 @@ impl Default for ExtractConfig { presence_min_matched: 3, presence_min_fragments: 3, presence_min_coelution: 2, - // Validated defensible regime (holds a ~valid target-decoy FDR): - // strict spectral gate on frag_corr. Loosening these raises raw - // counts but inflates FDR on chimeric DIA data (see COMPARISON.md). - min_frag_corr: 0.5, + // Loosening raises recall and candidate volume; external entrapment + // validation is required before treating any threshold as FDR-safe. + // Relaxed from the historical 0.5 to 0.2 to recover low-abundance + // candidates the hard single-scan Pearson gate was dropping + // (comment.md S1); still a hard gate, not the soft/budgeted redesign. + min_frag_corr: 0.2, min_matched_fraction: 0.0, apex_top_fragments: 0, // superseded by apex_count_tol; kept for compat apex_rt_prior_s: 0.0, // RT prior off by default @@ -892,6 +896,11 @@ pub enum QuantQColumn { /// Filter on the per-PSM `q_value`. Use for cross-run quant off an /// experiment-wide rescore, where the peptide q is global. PsmQ, + /// Filter on `run_psm_q` (per-run PSM FDR). The correct choice for cross-run + /// quant off an experiment-wide rescore: each run's PSMs are FDR-controlled + /// within their own run, so quant keeps the right per-run precursors without + /// the external `split_scored.py` peptide-q overwrite. + RunPsmQ, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -1147,6 +1156,15 @@ impl Config { .into(), )); } + if !self.extract.min_frag_corr.is_finite() + || !(0.0..=1.0).contains(&self.extract.min_frag_corr) + { + return Err(Invalid( + "extract.min_frag_corr must be finite and in [0, 1] (0 disables \ + the gate)." + .into(), + )); + } // Warn (not fail) when a declared-but-unimplemented knob is set away from // its default: it silently has no effect, which otherwise misleads tuning. let d = Self::default(); @@ -1183,7 +1201,7 @@ impl Config { /// Apply a named tuning profile on top of the current config. `dia` is the /// validated DIA preset (Extended features + rolling-window apex + RT prior); /// the other extraction defaults (emit_window_grid, reverse decoys, - /// min_frag_corr) are already the good values. Lets one command reach a + /// min_frag_corr) remain conservative baselines. Lets one command reach a /// respectable result without hand-authoring the full config JSON. pub fn apply_profile(&mut self, name: &str) -> Result<(), crate::error::ConfigError> { match name { @@ -1220,6 +1238,7 @@ mod tests { assert_eq!(back.digest.min_len, 5); assert_eq!(back.features.set, FeatureSet::Minimal); assert_eq!(back.rt_im_train.tolerance_regime, ToleranceRegime::Fixed); + assert_eq!(back.search_seed.top_n_peaks, 300); } #[test] @@ -1235,5 +1254,15 @@ mod tests { assert_eq!(c.digest.min_len, 7); assert_eq!(c.digest.max_len, 50); assert_eq!(c.peptidoforms.charge_max, 3); + assert_eq!(c.search_seed.top_n_peaks, 300); + } + + #[test] + fn explicit_uncapped_seed_and_invalid_gate_are_distinguished() { + let c = Config::from_json(r#"{"search_seed":{"top_n_peaks":0}}"#).unwrap(); + assert_eq!(c.search_seed.top_n_peaks, 0); + + assert!(Config::from_json(r#"{"extract":{"min_frag_corr":-0.1}}"#).is_err()); + assert!(Config::from_json(r#"{"extract":{"min_frag_corr":1.1}}"#).is_err()); } } diff --git a/rust/mumdia/crates/mumdia/src/main.rs b/rust/mumdia/crates/mumdia/src/main.rs index 5e7ea56..97bc085 100644 --- a/rust/mumdia/crates/mumdia/src/main.rs +++ b/rust/mumdia/crates/mumdia/src/main.rs @@ -24,8 +24,12 @@ enum Cmd { /// Limit spectra read (0 = all), for fast iteration. #[arg(long, default_value_t = 0)] max_spectra: usize, - /// Keep at most this many MS2 peaks per scan (0 = all). - #[arg(long, default_value_t = 300)] + /// Keep at most this many MS2 peaks in the normalized artifact (0 = all). + /// + /// This is an irreversible conversion-time cap that also affects extraction, + /// features, and quantification. Use `search_seed.top_n_peaks` for a + /// seed-only limit. + #[arg(long, default_value_t = 0)] top_peaks_ms2: usize, /// Keep at most this many MS1 peaks per scan (0 = all). #[arg(long, default_value_t = 0)] @@ -213,7 +217,9 @@ enum Cmd { profile: Option, #[arg(long, default_value_t = 0)] max_spectra: usize, - #[arg(long, default_value_t = 300)] + /// Irreversible conversion-time MS2 cap (0 = all). Seed-only peak limiting + /// is configured by `search_seed.top_n_peaks`. + #[arg(long, default_value_t = 0)] top_peaks_ms2: usize, }, /// Cross-run RT alignment (experiment-level) -> alignment.parquet. @@ -376,7 +382,14 @@ fn main() -> Result<()> { top_peaks_ms1, } => { let cfg = load_config(&None)?; - let config_hash = mumdia_io::hash::blake3_str(&cfg.canonical_json()); + // Fold the conversion CLI caps into the convert artifacts' provenance + // key: they change the spectra output but are not part of the config, so + // two different caps would otherwise produce an identical config_hash + // (comment.md A2/C4). The caps are also recorded in the convert report. + let config_hash = mumdia_io::hash::blake3_str(&format!( + "{}\u{1f}max_spectra={max_spectra}\u{1f}top_peaks_ms2={top_peaks_ms2}\u{1f}top_peaks_ms1={top_peaks_ms1}", + cfg.canonical_json() + )); stages::convert::run(stages::convert::ConvertParams { mzml: &mzml, out_dir: &out_dir, @@ -665,3 +678,68 @@ fn main() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod cli_tests { + use super::{Cli, Cmd}; + use clap::Parser; + + #[test] + fn conversion_caps_default_to_uncapped() { + let cli = Cli::try_parse_from([ + "mumdia", + "convert", + "--mzml", + "run.mzML", + "--out-dir", + "spectra", + ]) + .unwrap(); + match cli.cmd { + Cmd::Convert { + top_peaks_ms2, + top_peaks_ms1, + .. + } => { + assert_eq!(top_peaks_ms2, 0); + assert_eq!(top_peaks_ms1, 0); + } + _ => panic!("expected convert command"), + } + + let cli = Cli::try_parse_from([ + "mumdia", + "run", + "--fasta", + "proteome.fasta", + "--mzml", + "run.mzML", + "--out-dir", + "out", + ]) + .unwrap(); + match cli.cmd { + Cmd::Run { top_peaks_ms2, .. } => assert_eq!(top_peaks_ms2, 0), + _ => panic!("expected run command"), + } + } + + #[test] + fn explicit_conversion_cap_is_preserved() { + let cli = Cli::try_parse_from([ + "mumdia", + "convert", + "--mzml", + "run.mzML", + "--out-dir", + "spectra", + "--top-peaks-ms2", + "300", + ]) + .unwrap(); + match cli.cmd { + Cmd::Convert { top_peaks_ms2, .. } => assert_eq!(top_peaks_ms2, 300), + _ => panic!("expected convert command"), + } + } +} diff --git a/rust/mumdia/crates/mumdia/src/stages/convert.rs b/rust/mumdia/crates/mumdia/src/stages/convert.rs index 4ef224f..b699c68 100644 --- a/rust/mumdia/crates/mumdia/src/stages/convert.rs +++ b/rust/mumdia/crates/mumdia/src/stages/convert.rs @@ -246,7 +246,12 @@ pub fn run(p: ConvertParams) -> Result { ], p.config_hash, elapsed, - json!({"mzml": p.mzml, "max_spectra": p.max_spectra}), + json!({ + "mzml": p.mzml, + "max_spectra": p.max_spectra, + "top_peaks_ms2": p.top_peaks_ms2, + "top_peaks_ms1": p.top_peaks_ms1, + }), )?; info!( diff --git a/rust/mumdia/crates/mumdia/src/stages/extract.rs b/rust/mumdia/crates/mumdia/src/stages/extract.rs index a29e433..887a8d8 100644 --- a/rust/mumdia/crates/mumdia/src/stages/extract.rs +++ b/rust/mumdia/crates/mumdia/src/stages/extract.rs @@ -1450,8 +1450,13 @@ pub fn run(p: ExtractParams) -> Result<(u64, u64)> { content_hash: mumdia_io::hash::blake3_file(path)?, params: json!({ "frag_tol_ppm": p.cfg.frag_tol_ppm, + "effective_frag_tol_ppm": frag_tol, + "frag_ppm_offset": frag_offset, "presence_min_fragments": p.cfg.presence_min_fragments, "presence_min_coelution": p.cfg.presence_min_coelution, + "min_frag_corr": p.cfg.min_frag_corr, + "gate_mode": p.cfg.gate_mode, + "gate_coelution_min": p.cfg.gate_coelution_min, "scan_window": scan_window, }), stats: stats.clone(), diff --git a/rust/mumdia/crates/mumdia/src/stages/quant.rs b/rust/mumdia/crates/mumdia/src/stages/quant.rs index 20b951b..8354891 100644 --- a/rust/mumdia/crates/mumdia/src/stages/quant.rs +++ b/rust/mumdia/crates/mumdia/src/stages/quant.rs @@ -160,6 +160,7 @@ pub fn run(p: QuantParams) -> Result<(u64, u64)> { let pep_q = match p.cfg.q_filter { QuantQColumn::PeptideQ => ps.f64("peptide_q_value")?, QuantQColumn::PsmQ => ps.f64("q_value")?, + QuantQColumn::RunPsmQ => ps.f64("run_psm_q")?, }; // Chromatograms grouped by candidate. diff --git a/rust/mumdia/crates/mumdia/src/stages/search_seed.rs b/rust/mumdia/crates/mumdia/src/stages/search_seed.rs index 4b5943b..1362b44 100644 --- a/rust/mumdia/crates/mumdia/src/stages/search_seed.rs +++ b/rust/mumdia/crates/mumdia/src/stages/search_seed.rs @@ -248,6 +248,7 @@ pub fn run(p: SearchSeedParams) -> Result { "fragment_tol_ppm": p.cfg.fragment_tol_ppm, "report_psms": p.cfg.report_psms, "min_matched_peaks": p.cfg.min_matched_peaks, + "top_n_peaks": p.cfg.top_n_peaks, "fdr_seed": p.cfg.fdr_seed, }), stats, @@ -384,3 +385,38 @@ fn seed_fragindex_windows( fn hyperscore(matched: u32, sum_obs: f64) -> f64 { ln_factorial(matched) + (1.0 + sum_obs).ln() } + +#[cfg(test)] +mod peak_selection_tests { + use super::select_peaks; + use mumdia_core::types::{IsolationWindow, Ms2Scan, Peak}; + + fn scan(n: usize) -> Ms2Scan { + Ms2Scan { + scan_index: 0, + id: "scan=0".into(), + rt_seconds: 0.0, + window: IsolationWindow { + target_mz: 0.0, + lower_mz: 0.0, + upper_mz: 2_000.0, + im_lower: None, + im_upper: None, + }, + peaks: (0..n) + .map(|i| Peak { + mz: 100.0 + i as f64, + intensity: i as f32, + ion_mobility: None, + }) + .collect(), + } + } + + #[test] + fn zero_selects_all_and_seed_cap_keeps_only_top_intensity_peaks() { + let s = scan(305); + assert_eq!(select_peaks(&s, 0), (0..305).collect::>()); + assert_eq!(select_peaks(&s, 300), (5..305).collect::>()); + } +}