From 314993bf5544e46296e50eb7924f74e430b582b0 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:58:30 +0100 Subject: [PATCH 1/7] Let the temporal benchmark train and score on dated InterPro inputs The 2021 -> 2026 benchmark and the section 4 ablation train on GOA release 205 (2021-04-21) but read the 2026-07 protein2ipr, in training and in the evaluator's transfer step: look-ahead on the domain side (TODO P0). This is the code half of removing it; the InterPro 85.0 runs follow on this branch. - run_dcgo_human.py takes --gaf and --interpro (defaults unchanged). Both flow through RunRequest into the early missing-input checks and into the manifest with hashes and release headers like the defaults, but without the species' current-release URL label, which would name other bytes. This replaces the symlink staging (goa_human_t0_2021 / a linked default GAF path) in RESULTS.md, validation/mouse/PROVENANCE.md and the section 4 regeneration notes. - extract_human_interpro.py takes --source, repeatable --gaf (union) and --output, so a t0 subset can be cut from an archived protein2ipr and selected by the t0 and t1 GAFs. The current subset was selected by the t1 GAF's non-IEA proteins alone (19,089, exactly); t0 + t1 under any evidence is 19,871, a strict superset. - download_data.py --group interpro-85 fetches InterPro 85.0 (protein2ipr checked against the release's published md5, interpro2go, entry.list) into data/raw/interpro_archive/85.0/. - validation/ablation.py refuses to score a rung whose manifest records a different protein2ipr (by SHA-256) than its --interpro, and its provenance TSV records the protein2ipr path and hash. temporal_benchmark.py applies the same check against the manifest beside --predictions. - scripts/run_t0_interpro_ablation.py is the runbook for the follow-up: verify the archive, extract the subset, run the nine rungs exactly as their committed manifests record them plus --gaf/--interpro, evaluate. Co-Authored-By: Claude Opus 5.5 (1M context) --- RELATIVE_INFERENCE_REPORT.md | 5 +- REPRODUCIBILITY.md | 5 + RESULTS.md | 21 +- VALIDATION_PLAN.md | 28 +++ config/settings.py | 36 ++++ extract_human_interpro.py | 82 ++++++-- run_dcgo_human.py | 41 +++- scripts/download_data.py | 13 +- scripts/run_t0_interpro_ablation.py | 188 ++++++++++++++++++ src/runner.py | 15 +- .../test_extract_human_interpro.py | 129 ++++++++++++ tests/integration/test_input_overrides.py | 96 +++++++++ tests/integration/test_validation_clis.py | 159 +++++++++++++++ tests/unit/test_ablation.py | 44 +++- tests/unit/test_download_checksums.py | 35 ++++ tests/unit/test_generic_runner.py | 34 ++++ tests/unit/test_run_manifest_wiring.py | 39 ++++ tests/unit/test_run_t0_interpro_ablation.py | 110 ++++++++++ tests/unit/test_temporal_benchmark.py | 22 ++ validation/ablation.py | 44 +++- validation/mouse/PROVENANCE.md | 8 +- validation/temporal_benchmark.py | 44 +++- 22 files changed, 1146 insertions(+), 52 deletions(-) create mode 100644 scripts/run_t0_interpro_ablation.py create mode 100644 tests/integration/test_extract_human_interpro.py create mode 100644 tests/integration/test_input_overrides.py create mode 100644 tests/unit/test_run_t0_interpro_ablation.py diff --git a/RELATIVE_INFERENCE_REPORT.md b/RELATIVE_INFERENCE_REPORT.md index 3557149..706bd31 100644 --- a/RELATIVE_INFERENCE_REPORT.md +++ b/RELATIVE_INFERENCE_REPORT.md @@ -598,7 +598,10 @@ test with its measured cost rather than as a default. checkout of the branch (`78494f2`; `git.dirty: false` in every manifest), and an independent earlier run of the same 36 pipeline configurations from a working tree reproduced every association table - **byte-for-byte** (96/96 output TSVs). + **byte-for-byte** (96/96 output TSVs). Those runs read the 2021 GAF through + a link at the default GAF path; to regenerate, run each manifest's recorded + command with `--gaf data/raw/goa_archive/goa_human.gaf.205.gz` added + (`VALIDATION_PLAN.md` §4, "Regenerating a cell"). - **Robustness cells** (§4.2): the same nine-rung ladder rerun with `--domain-key ssf` and/or `--evidence-filter all`; eval tables and per-run manifests in `validation/ablation_cells/{ipr_all,ssf_manual,ssf_all}/`. diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md index 1e65132..8a42b62 100644 --- a/REPRODUCIBILITY.md +++ b/REPRODUCIBILITY.md @@ -52,6 +52,11 @@ Hashing is intentionally part of the run. For large inputs it adds sequential I/O, but prevents a mutable filename from silently referring to different data. The species-specific InterPro file is a derived input: its own hash identifies what was analyzed, while `derived_from` records the upstream mapping source. +Inputs passed with `--gaf`/`--interpro` (archived snapshots for the temporal +benchmark) are hashed and header-read the same way but carry no +`source_url`/`derived_from`: the species' current-release URL would name +different bytes. A subset's own `.provenance.json` sidecar names the archive it +was cut from. The inputs recorded are exactly those the selected ontology declares in `src/ontology_registry.py` (`needs`, plus `hierarchy_needs` when diff --git a/RESULTS.md b/RESULTS.md index 711277e..515ab72 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -202,25 +202,20 @@ Assumes the human InterPro subset is already extracted # 1. Fetch the 2021 t0 GOA snapshot uv run python scripts/download_data.py --goa-archive 205 -# 2. Stage t0 inputs under the --species name run_dcgo_human.py expects. -# Domains are fixed in time (only GOA moves), so symlink the current subset. -ln -sf "$PWD/data/raw/goa_archive/goa_human.gaf.205.gz" \ - data/raw/goa_annotations/goa_human_t0_2021.gaf.gz -ln -sf "$PWD/data/interim/protein2ipr_human.dat.gz" \ - data/interim/protein2ipr_human_t0_2021.dat.gz - -# 3. Train domain→GO associations on t0 -uv run python run_dcgo_human.py --species human_t0_2021 --num-cores 32 \ - --output-dir results_t0_2021 - -# 4. Protein-centric temporal benchmark (t1 = current GOA) +# 2. Train domain→GO associations on t0. Only GOA moves: the domains are the +# current subset (the --interpro default). VALIDATION_PLAN §4 has the rerun +# on InterPro 85.0 (t0) architectures. +uv run python run_dcgo_human.py --gaf data/raw/goa_archive/goa_human.gaf.205.gz \ + --num-cores 32 --output-dir results_t0_2021 + +# 3. Protein-centric temporal benchmark (t1 = current GOA) uv run python validation/temporal_benchmark.py \ --t0-gaf data/raw/goa_archive/goa_human.gaf.205.gz \ --t1-gaf data/raw/goa_annotations/goa_human.gaf.gz \ --predictions results_t0_2021/domain_go_associations_significant.tsv \ --min-ic 0 --min-ic 2 --min-ic 4 --min-ic 6 -# 5. Domain-centric eval — build the relative-filtered set, then compare both +# 4. Domain-centric eval — build the relative-filtered set, then compare both uv run python validation/apply_relative_inference.py \ --predictions results_t0_2021/domain_go_associations_significant.tsv \ --t0-gaf data/raw/goa_archive/goa_human.gaf.205.gz \ diff --git a/VALIDATION_PLAN.md b/VALIDATION_PLAN.md index 6f5c94b..b45aa65 100644 --- a/VALIDATION_PLAN.md +++ b/VALIDATION_PLAN.md @@ -1227,6 +1227,34 @@ uv run python validation/check_metric_conventions.py \ --run-dir results/ablation-replacedby/ipr_manual ``` +### Regenerating a cell, and the t0 InterPro rerun + +Each rung is the command its manifest records, with the training GAF passed +explicitly (the committed runs read it through a link at the default GAF path, +which is why their `gaf` input carries the current-release `source_url`; the +SHA-256 `69ae7d90…` is the identity): + +```bash +uv run python run_dcgo_human.py --num-cores 8 \ + --output-dir results/ablation-replacedby/ipr_manual/ \ + --gaf data/raw/goa_archive/goa_human.gaf.205.gz +uv run python validation/ablation.py \ + --t0-gaf data/raw/goa_archive/goa_human.gaf.205.gz \ + --t1-gaf data/raw/goa_annotations/goa_human.gaf.gz \ + --run-dir results/ablation-replacedby/ipr_manual --output-dir +``` + +These cells still read **current** domain architectures +(`protein2ipr_human.dat.gz`, InterPro of 2026-07-22) in training and in the +transfer step — temporal look-ahead on the domain side (TODO.md P0). +`scripts/run_t0_interpro_ablation.py` reruns the `ipr_manual` cell on InterPro +85.0 (2021-04-08, the last release before GOA 205): it cuts a human subset of +the archived `protein2ipr` selected by the t0 *and* t1 GAFs, runs the nine +recorded rung commands with `--gaf`/`--interpro` added, and evaluates into +`validation/ablation_cells/ipr_manual_t0interpro/`. `validation/ablation.py` +refuses to score a rung whose manifest records a different `protein2ipr` than +its `--interpro`. Results to follow. + ## 5. Decisions to settle before writing the paper ### Held-out validation of the surprise score (2026-07-28) — DONE diff --git a/config/settings.py b/config/settings.py index 57fa415..4f5f21d 100644 --- a/config/settings.py +++ b/config/settings.py @@ -723,6 +723,42 @@ class Config: description="Manually curated InterPro2GO mappings (validation reference)", required=False, ), + # ---- InterPro 85.0 (8 April 2021) -------------------------------- + # The last InterPro release before GOA release 205 (2021-04-21), + # the temporal benchmark's t0, so training and transfer can use + # domain architectures of the same date as the annotations + # (VALIDATION_PLAN §4, scripts/run_t0_interpro_ablation.sh). + # Opt-in: scripts/download_data.py --group interpro-85. + "interpro_85_protein2ipr": DataSource( + name="interpro_85_protein2ipr", + url="https://ftp.ebi.ac.uk/pub/databases/interpro/releases/85.0/protein2ipr.dat.gz", + description="InterPro 85.0 (2021-04-08) protein2ipr: t0 domain architectures (~15 GB)", + required=False, + # The release's own published protein2ipr.dat.gz.md5. + checksum="md5:ad41fd5c4a8dc83303683a19a34ec0f1", + size_bytes=15272180301, + subdir="interpro_archive/85.0", + ), + "interpro_85_interpro2go": DataSource( + name="interpro_85_interpro2go", + url="https://ftp.ebi.ac.uk/pub/databases/interpro/releases/85.0/interpro2go", + description="InterPro 85.0 InterPro2GO mapping (dated 2021-04-03)", + required=False, + # No published digest for the uncompressed files; these are the + # SHA-256s of the bytes served on 2026-10-06. + checksum="sha256:0757a8a2a12bf75ec34f881668ba445a41d64fcf67349f656e211d627ab93ab4", + size_bytes=3497883, + subdir="interpro_archive/85.0", + ), + "interpro_85_entry_list": DataSource( + name="interpro_85_entry_list", + url="https://ftp.ebi.ac.uk/pub/databases/interpro/releases/85.0/entry.list", + description="InterPro 85.0 entry list (accession, type, name)", + required=False, + checksum="sha256:b606ad13f8ea5a766cfe749ed38f4bfdbfe1a0bcd39bb9cd954f5dc4e72bd450", + size_bytes=2085328, + subdir="interpro_archive/85.0", + ), # ---- Published dcGO (Fang & Gough 2013) reference tables -------- # The comparator for VALIDATION_PLAN §3. Still served by SUPERFAMILY # at supfam.org; the /SUPERFAMILY/ path prefix is required (the diff --git a/extract_human_interpro.py b/extract_human_interpro.py index 926cca1..463b212 100644 --- a/extract_human_interpro.py +++ b/extract_human_interpro.py @@ -8,6 +8,17 @@ uv run python extract_human_interpro.py # human (default) uv run python extract_human_interpro.py --species mouse # any organism + +A subset for a temporal benchmark is cut from an archived protein2ipr and +selected by the union of several GAFs — the training snapshot *and* the later +one whose newly annotated proteins are scored, so both have architectures: + + uv run python extract_human_interpro.py \ + --source data/raw/interpro_archive/85.0/protein2ipr.dat.gz \ + --gaf data/raw/goa_archive/goa_human.gaf.205.gz \ + --gaf data/raw/goa_annotations/goa_human.gaf.gz \ + --evidence-filter all \ + --output data/interim/protein2ipr_human_t0_ipr85.dat.gz """ import argparse @@ -88,6 +99,27 @@ def main(): choices=["all", "manual", "experimental"], help="Evidence filter used to select the protein set (default: manual)", ) + parser.add_argument( + "--gaf", + type=Path, + action="append", + help="GAF whose proteins select the subset; repeat to take the union " + "(default: data/raw/goa_annotations/goa_.gaf.gz)", + ) + parser.add_argument( + "--source", + type=Path, + default=Path("data/raw/interpro_mappings/protein2ipr.dat.gz"), + help="protein2ipr file to filter, e.g. an archived release " + "(default: data/raw/interpro_mappings/protein2ipr.dat.gz)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output path (default: data/interim/protein2ipr_.dat.gz). " + "The selecting accession list is written beside it", + ) parser.add_argument( "--force", action="store_true", @@ -106,26 +138,41 @@ def main(): ) logger.info(f"Step 1: Parsing GOA to get {args.species} protein IDs...") - goa_file = Path(f"data/raw/goa_annotations/goa_{args.species}.gaf.gz") - if not goa_file.exists(): - logger.error(f"GOA file not found: {goa_file}") - logger.error( - f"Download it first, e.g. " - f"uv run python scripts/download_data.py --species {args.species}" + goa_files = args.gaf or [ + Path(f"data/raw/goa_annotations/goa_{args.species}.gaf.gz") + ] + for goa_file in goa_files: + if not goa_file.exists(): + logger.error(f"GOA file not found: {goa_file}") + logger.error( + f"Download it first, e.g. " + f"uv run python scripts/download_data.py --species {args.species}" + ) + return 1 + + protein_ids: set[str] = set() + for goa_file in goa_files: + protein_ids.update( + parse_goa( + goa_file, evidence_filter=args.evidence_filter, aspects={"P", "F", "C"} + ) ) - return 1 - protein_go_map = parse_goa( - goa_file, evidence_filter=args.evidence_filter, aspects={"P", "F", "C"} + output_file = args.output or Path(f"data/interim/protein2ipr_{args.species}.dat.gz") + # Beside a custom --output, so a second subset never overwrites the + # species' own list. + protein_list_file = ( + output_file.with_name( + output_file.name.removesuffix(".dat.gz") + "_proteins.txt" + ) + if args.output + else Path(f"data/interim/{args.species}_proteins.txt") ) - - # Write protein IDs to temp file - protein_list_file = Path(f"data/interim/{args.species}_proteins.txt") protein_list_file.parent.mkdir(parents=True, exist_ok=True) - logger.info(f"Writing {len(protein_go_map):,} protein IDs to {protein_list_file}") + logger.info(f"Writing {len(protein_ids):,} protein IDs to {protein_list_file}") with open(protein_list_file, "w") as f: - for protein_id in sorted(protein_go_map.keys()): + for protein_id in sorted(protein_ids): f.write(f"{protein_id}\n") # Extract species annotations @@ -133,7 +180,7 @@ def main(): logger.info( f"Step 2: Extracting {args.species} protein annotations from InterPro..." ) - interpro_file = Path("data/raw/interpro_mappings/protein2ipr.dat.gz") + interpro_file = args.source if not interpro_file.exists(): logger.error(f"InterPro mappings file not found: {interpro_file}") logger.error( @@ -141,7 +188,6 @@ def main(): "--datasets interpro_mappings" ) return 1 - output_file = Path(f"data/interim/protein2ipr_{args.species}.dat.gz") # An extract selected by a different rule (idmapping / accession list, via # scripts/extract_species_interpro.py) must not be silently clobbered. @@ -157,7 +203,7 @@ def main(): write_marker( output_file, selection_rule="goa", - selection_sources=[goa_file], + selection_sources=goa_files, interpro_source=interpro_file, n_accessions=n_accessions, n_matched_lines=n_matched, @@ -170,7 +216,7 @@ def main(): logger.info("=" * 60) logger.info(f"You can now use: {output_file}") logger.info( - f"This file contains only the {len(protein_go_map):,} " + f"This file contains only the {len(protein_ids):,} " f"{args.species} proteins from GOA" ) return 0 diff --git a/run_dcgo_human.py b/run_dcgo_human.py index aa3b763..9b32bbc 100644 --- a/run_dcgo_human.py +++ b/run_dcgo_human.py @@ -16,6 +16,8 @@ Options: --species STR Species to analyze: 'human', 'mouse', etc. (default: human) + --gaf PATH GO annotation file (default: data/raw/goa_annotations/goa_.gaf.gz) + --interpro PATH protein2ipr domain file (default: data/interim/protein2ipr_.dat.gz) --ontology STR Ontology to associate domains with (default: go). See src/ontology_registry.py, or --help, for the full list: go, ec, reactome, keyword, disease, doid, orphanet, @@ -94,6 +96,12 @@ # Key domains by SCOP superfamily instead of InterPro entry, to compare # against the published dcGO (VALIDATION_PLAN §3) uv run python run_dcgo_human.py --domain-key ssf + + # Train on dated snapshots (temporal benchmark, VALIDATION_PLAN §2/§4): + # archived GOA release 205 and a subset of InterPro 85.0, both April 2021 + uv run python run_dcgo_human.py \ + --gaf data/raw/goa_archive/goa_human.gaf.205.gz \ + --interpro data/interim/protein2ipr_human_t0_ipr85.dat.gz """ import argparse @@ -460,6 +468,14 @@ def start_run_manifest( "completed" one from a previous invocation. """ source_urls = input_source_urls(args.species) + # A file passed with --gaf/--interpro is not the one the species' upstream + # URL names (an archived GAF, a subset of an archived protein2ipr), so it + # gets no URL label rather than a wrong one. Its SHA-256 and release + # headers are recorded exactly as for the defaults. + if getattr(args, "gaf", None) is not None: + source_urls.pop("gaf", None) + if getattr(args, "interpro", None) is not None: + source_urls.pop("interpro_mappings", None) input_records = [ describe_file( interpro_file, @@ -648,6 +664,24 @@ def build_argument_parser() -> argparse.ArgumentParser: default="human", help="Species to analyze: 'human', 'mouse', or specific GOA file name (default: human)", ) + parser.add_argument( + "--gaf", + type=Path, + default=None, + help="GO annotation file, used when --ontology go (default: " + "data/raw/goa_annotations/goa_.gaf.gz). Point it at an archived " + "release (e.g. data/raw/goa_archive/goa_human.gaf.205.gz) to train on a " + "past snapshot", + ) + parser.add_argument( + "--interpro", + type=Path, + default=None, + help="protein2ipr file defining the domain architectures (default: " + "data/interim/protein2ipr_.dat.gz). Point it at a subset cut " + "from an archived InterPro release (extract_human_interpro.py --source " + "... --output ...) to train on past architectures", + ) parser.add_argument( "--ontology", default="go", @@ -1041,10 +1075,9 @@ def main(argv: list[str] | None = None) -> int: ) return 1 - # File paths - support different species - interpro_file = Path(f"data/interim/protein2ipr_{args.species}.dat.gz") - - # Check domain annotations exist (shared across ontologies) + # Domain annotations (shared across ontologies): --interpro, else the + # species' extract. + interpro_file = inputs.interpro_file if not interpro_file.exists(): logger.error(f"{args.species.title()} InterPro file not found: {interpro_file}") logger.error(f"Please extract {args.species} data from protein2ipr.dat.gz") diff --git a/scripts/download_data.py b/scripts/download_data.py index cbd4c89..54d7306 100644 --- a/scripts/download_data.py +++ b/scripts/download_data.py @@ -39,8 +39,9 @@ * Existing complete files are skipped. Use --force to re-download. * ``interpro_mappings`` (protein2ipr.dat.gz) is ~20 GB — this can take a while. * Sources pinned to an immutable release URL carry a ``checksum`` (and - sometimes a ``size_bytes``) in settings.py — ``disease_ontology``, and the - frozen published-dcGO and SCOP 1.75 archives. These are verified every time, + sometimes a ``size_bytes``) in settings.py — ``disease_ontology``, the + frozen published-dcGO and SCOP 1.75 archives, and the InterPro 85.0 release + (``--group interpro-85``). These are verified every time, including when an existing file is skipped, so a corrupted, truncated or swapped input fails the download step instead of quietly changing a run's results. @@ -138,6 +139,14 @@ "zfa_ontology", "zebrafish_idmapping", ], + # InterPro 85.0 (2021-04-08), the t0 domain architectures of the temporal + # benchmark (scripts/run_t0_interpro_ablation.sh). ~15 GB; the protein2ipr + # is checked against the release's published md5. + "interpro-85": [ + "interpro_85_protein2ipr", + "interpro_85_interpro2go", + "interpro_85_entry_list", + ], # fbcv and fbbt share the same three FlyBase tables; only the OBO differs. "flybase-phenotype": [ "flybase_genotype_phenotype", diff --git a/scripts/run_t0_interpro_ablation.py b/scripts/run_t0_interpro_ablation.py new file mode 100644 index 0000000..3c97d3d --- /dev/null +++ b/scripts/run_t0_interpro_ablation.py @@ -0,0 +1,188 @@ +"""Rerun the §4 ``ipr_manual`` ablation cell on InterPro 85.0 (t0) architectures. + +TODO.md P0 "Remove temporal look-ahead from domain architectures": the committed +cell trains on GOA release 205 (2021-04-21) but reads the 2026-07 protein2ipr, +both in training and in the evaluator's transfer step. InterPro 85.0 +(2021-04-08) is the last release before the t0 GAF. Three steps, each printed +before it runs and logged to ``/.log``: + +1. **Extract** the human subset of the archived protein2ipr. Proteins are + selected by the t0 *and* t1 GAFs under any evidence, so every training + protein and every held-out protein the evaluator transfers to has t0 + architectures. (The current subset was selected by the t1 GAF's non-IEA + proteins alone, a strict subset of this.) Skipped when the subset exists + and its provenance marker names the same source. +2. **Train** the nine rungs: exactly the commands recorded in + ``validation/ablation_manifests/.json``, with ``--output-dir`` moved + under ``--run-dir`` and ``--gaf``/``--interpro`` added. +3. **Evaluate** with ``validation/ablation.py`` under the committed + evaluation's arguments (all defaults besides the GAFs and run dir), plus + ``--interpro`` set to the subset. The rung manifests are copied to + ``/manifests/``, as for the other cells. + +Run it from the repository root; paths are relative to it, and ``data/`` must +hold the inputs. The md5 of the archived protein2ipr is verified first +(``scripts/download_data.py --group interpro-85``). + +Dry run of the whole chain without the archive: pass the current subset as +``--source`` (with scratch ``--subset``/``--run-dir``/``--eval-dir``). A wider +selection filtered out of a file that holds only selected proteins is that +file again, so it must reproduce the committed ``ipr_manual`` numbers exactly. + + uv run python scripts/run_t0_interpro_ablation.py + uv run python scripts/run_t0_interpro_ablation.py --dry-run # print only +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO)) + +from src.universe_provenance import read_marker # noqa: E402 + +MANIFESTS = REPO / "validation" / "ablation_manifests" +ARCHIVE = Path("data/raw/interpro_archive/85.0/protein2ipr.dat.gz") +T0_GAF = Path("data/raw/goa_archive/goa_human.gaf.205.gz") +T1_GAF = Path("data/raw/goa_annotations/goa_human.gaf.gz") +RUNGS = ( + "single", + "supra", + "supra_input", + "supra_relative", + "supra_output", + "supra_input_relative", + "supra_input_output", + "supra_relative_output", + "full", +) + + +def rung_command(manifest: Path, output_dir: Path, interpro: Path) -> list[str]: + """The command a rung's manifest records, re-pointed at t0 inputs. + + Only the interpreter (``command[0]``), the ``--output-dir`` value and the + two input flags change; every analysis flag is the recorded one. + """ + args = json.loads(manifest.read_text(encoding="utf-8"))["command"][1:] + args[args.index("--output-dir") + 1] = str(output_dir) + return [sys.executable, *args, "--gaf", str(T0_GAF), "--interpro", str(interpro)] + + +def build_steps( + source: Path, subset: Path, run_dir: Path, eval_dir: Path +) -> list[tuple[str, list[str]]]: + """(log name, command) for every step, in order.""" + steps = [] + if source == ARCHIVE: + steps.append( + ( + "verify", + [sys.executable, "scripts/download_data.py", "--group", "interpro-85"], + ) + ) + marker = read_marker(subset) + if not (subset.exists() and marker and marker.interpro_source == str(source)): + steps.append( + ( + "extract", + [ + sys.executable, + "extract_human_interpro.py", + "--source", + str(source), + "--gaf", + str(T0_GAF), + "--gaf", + str(T1_GAF), + "--evidence-filter", + "all", + "--output", + str(subset), + ], + ) + ) + for rung in RUNGS: + steps.append( + (rung, rung_command(MANIFESTS / f"{rung}.json", run_dir / rung, subset)) + ) + steps.append( + ( + "eval", + [ + sys.executable, + "validation/ablation.py", + "--t0-gaf", + str(T0_GAF), + "--t1-gaf", + str(T1_GAF), + "--run-dir", + str(run_dir), + "--interpro", + str(subset), + "--output-dir", + str(eval_dir), + ], + ) + ) + return steps + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Rerun the ipr_manual ablation cell on InterPro 85.0 (t0) " + "domain architectures (VALIDATION_PLAN §4)." + ) + parser.add_argument("--source", type=Path, default=ARCHIVE) + parser.add_argument( + "--subset", + type=Path, + default=Path("data/interim/protein2ipr_human_t0_ipr85.dat.gz"), + ) + parser.add_argument( + "--run-dir", type=Path, default=Path("results/ablation-t0interpro/ipr_manual") + ) + parser.add_argument( + "--eval-dir", + type=Path, + default=Path("validation/ablation_cells/ipr_manual_t0interpro"), + ) + parser.add_argument( + "--dry-run", action="store_true", help="print the commands and exit" + ) + args = parser.parse_args(argv) + + steps = build_steps(args.source, args.subset, args.run_dir, args.eval_dir) + if args.dry_run: + for name, cmd in steps: + print(f"# {name}\n{' '.join(cmd)}") + return 0 + + args.run_dir.mkdir(parents=True, exist_ok=True) + for name, cmd in steps: + log = args.run_dir / f"{name}.log" + print(f"[{name}] {' '.join(cmd)} > {log}", flush=True) + with open(log, "w") as handle: + result = subprocess.run(cmd, stdout=handle, stderr=subprocess.STDOUT) + if result.returncode != 0: + print(f"[{name}] FAILED rc={result.returncode} — see {log}", flush=True) + return 1 + + manifests = args.eval_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + for rung in RUNGS: + shutil.copy( + args.run_dir / rung / "run_manifest_go.json", manifests / f"{rung}.json" + ) + print(f"Done. Evaluation tables and rung manifests in {args.eval_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/runner.py b/src/runner.py index 7e6cdb9..d09e573 100644 --- a/src/runner.py +++ b/src/runner.py @@ -30,6 +30,9 @@ class RunRequest: propagate_annotations: bool enable_supra_domains: bool xref_db: str | None + #: ``--gaf`` / ``--interpro``; ``None`` means the species' default file. + gaf: Path | None + interpro: Path | None go_ontology: Path enzyme_dat: Path uniprot_dat: Path @@ -101,6 +104,8 @@ def from_namespace(cls, args: argparse.Namespace) -> "RunRequest": propagate_annotations=getattr(args, "propagate_annotations", False), enable_supra_domains=args.enable_supra_domains, xref_db=getattr(args, "xref_db", None), + gaf=getattr(args, "gaf", None), + interpro=getattr(args, "interpro", None), go_ontology=args.go_ontology, enzyme_dat=args.enzyme_dat, uniprot_dat=args.uniprot_dat, @@ -137,10 +142,16 @@ def from_namespace(cls, args: argparse.Namespace) -> "RunRequest": fbcv_obo=args.fbcv_obo, ) + @property + def interpro_file(self) -> Path: + """The protein2ipr file that defines the domain architectures.""" + return self.interpro or Path(f"data/interim/protein2ipr_{self.species}.dat.gz") + def ontology_paths(self) -> dict[str, Path]: """Return every registry-addressable ontology input path.""" return { - "gaf": Path(f"data/raw/goa_annotations/goa_{self.species}.gaf.gz"), + "gaf": self.gaf + or Path(f"data/raw/goa_annotations/goa_{self.species}.gaf.gz"), "go_obo": self.go_ontology, "enzyme_dat": self.enzyme_dat, "uniprot_dat": self.uniprot_dat, @@ -185,6 +196,7 @@ class InputResolution: ontology_entry: OntologyEntry ontology_label: str ontology_paths: dict[str, Path] + interpro_file: Path true_path_unsupported: bool missing_inputs: tuple[str, ...] @@ -201,6 +213,7 @@ def resolve_inputs(request: RunRequest) -> InputResolution: else request.ontology ), ontology_paths=paths, + interpro_file=request.interpro_file, true_path_unsupported=( request.enable_true_path and not entry.supports_true_path ), diff --git a/tests/integration/test_extract_human_interpro.py b/tests/integration/test_extract_human_interpro.py new file mode 100644 index 0000000..721beb5 --- /dev/null +++ b/tests/integration/test_extract_human_interpro.py @@ -0,0 +1,129 @@ +"""``extract_human_interpro.py`` cuts a subset of any protein2ipr by GAF selection. + +A temporal benchmark needs the t0 architectures of two protein sets: the +proteins training saw (t0 GAF) and the held-out ones annotated only later (t1 +GAF). So the selection is the union of several GAFs, and the source can be an +archived protein2ipr rather than the current one. +""" + +from __future__ import annotations + +import gzip +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +#: P1 only in the t0 GAF, P2 only in the t1 GAF (IEA), P3 in both; P9 in neither. +T0_ROWS = [("P1", "IDA"), ("P3", "IDA")] +T1_ROWS = [("P2", "IEA"), ("P3", "IDA")] +SOURCE_PROTEINS = ["P1", "P2", "P3", "P9"] + + +def _gaf(path: Path, rows: list[tuple[str, str]]) -> Path: + lines = ["!gaf-version: 2.2"] + [ + f"UniProtKB\t{protein}\t{protein}\t\tGO:0006811\tPMID:1\t{evidence}\t\tP" + f"\t{protein}\t\tprotein\ttaxon:9606\t20210101\tGOA" + for protein, evidence in rows + ] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(gzip.compress(("\n".join(lines) + "\n").encode())) + return path + + +def _protein2ipr(path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(path, "wt") as handle: + for protein in SOURCE_PROTEINS: + handle.write(f"{protein}\tIPR000001\tDomain\tPF00001\t10\t110\n") + return path + + +def _extract(cwd: Path, *args: object) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(REPO_ROOT / "extract_human_interpro.py"), *map(str, args)], + cwd=cwd, + capture_output=True, + text=True, + timeout=60, + ) + + +def _proteins(path: Path) -> set[str]: + with gzip.open(path, "rt") as handle: + return {line.split("\t", 1)[0] for line in handle} + + +@pytest.mark.parametrize( + "evidence,expected", [("all", {"P1", "P2", "P3"}), ("manual", {"P1", "P3"})] +) +def test_union_of_gafs_selects_from_an_archived_source( + tmp_path: Path, evidence: str, expected: set[str] +) -> None: + source = _protein2ipr(tmp_path / "archive" / "protein2ipr.dat.gz") + t0 = _gaf(tmp_path / "t0.gaf.gz", T0_ROWS) + t1 = _gaf(tmp_path / "t1.gaf.gz", T1_ROWS) + output = tmp_path / "subset" / "protein2ipr_human_t0.dat.gz" + + result = _extract( + tmp_path, + "--source", + source, + "--gaf", + t0, + "--gaf", + t1, + "--evidence-filter", + evidence, + "--output", + output, + ) + + assert result.returncode == 0, result.stderr + assert _proteins(output) == expected + # The selecting list sits beside the custom output, not over the species'. + listed = (output.parent / "protein2ipr_human_t0_proteins.txt").read_text().split() + assert set(listed) == expected + assert not (tmp_path / "data").exists() + marker = json.loads(Path(f"{output}.provenance.json").read_text()) + assert marker["selection_sources"] == [str(t0), str(t1)] + assert marker["interpro_source"] == str(source) + + +def test_defaults_are_the_species_files(tmp_path: Path) -> None: + _protein2ipr(tmp_path / "data/raw/interpro_mappings/protein2ipr.dat.gz") + _gaf(tmp_path / "data/raw/goa_annotations/goa_toy.gaf.gz", T0_ROWS) + + result = _extract(tmp_path, "--species", "toy") + + assert result.returncode == 0, result.stderr + assert _proteins(tmp_path / "data/interim/protein2ipr_toy.dat.gz") == {"P1", "P3"} + assert (tmp_path / "data/interim/toy_proteins.txt").read_text().split() == [ + "P1", + "P3", + ] + + +def test_a_missing_gaf_fails_before_scanning(tmp_path: Path) -> None: + source = _protein2ipr(tmp_path / "protein2ipr.dat.gz") + output = tmp_path / "subset.dat.gz" + + result = _extract( + tmp_path, + "--source", + source, + "--gaf", + _gaf(tmp_path / "t0.gaf.gz", T0_ROWS), + "--gaf", + tmp_path / "absent.gaf.gz", + "--output", + output, + ) + + assert result.returncode == 1 + assert "absent.gaf.gz" in result.stderr + assert not output.exists() diff --git a/tests/integration/test_input_overrides.py b/tests/integration/test_input_overrides.py new file mode 100644 index 0000000..4ca4a42 --- /dev/null +++ b/tests/integration/test_input_overrides.py @@ -0,0 +1,96 @@ +"""``--gaf``/``--interpro`` train on files other than the species' current ones. + +The temporal benchmark trains on an archived GOA release and a subset of an +archived InterPro release. These run the real pipeline from a directory that +holds *no* default inputs, so a run can only succeed by reading the overrides, +and check that the manifest identifies them exactly as it does the defaults. +""" + +from __future__ import annotations + +import gzip +import hashlib +import json +from pathlib import Path + +import pytest + +from run_dcgo_human import main + +DATE_GENERATED = "2021-04-21 13:14" + + +def gaf_line(protein: str, term: str) -> str: + return ( + f"UniProtKB\t{protein}\t{protein}\t\t{term}\tPMID:1\tIDA\t\tP" + f"\t{protein}\t\tprotein\ttaxon:9606\t20210101\tGOA" + ) + + +@pytest.fixture +def archived(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]: + """An archived GAF and protein2ipr subset outside the default layout.""" + monkeypatch.chdir(tmp_path) + gaf_lines = ["!gaf-version: 2.2", f"!date-generated: {DATE_GENERATED}"] + ipr_lines = [] + for i in range(40): + protein = f"P{i:05d}" + term, domain = ( + ("GO:0006811", "IPR000001") if i < 20 else ("GO:0051179", "IPR000002") + ) + gaf_lines.append(gaf_line(protein, term)) + ipr_lines.append(f"{protein}\t{domain}\tDomain\tPF00001\t10\t110") + + archive = tmp_path / "archive" + archive.mkdir() + gaf = archive / "goa_human.gaf.205.gz" + gaf.write_bytes(gzip.compress(("\n".join(gaf_lines) + "\n").encode())) + interpro = archive / "protein2ipr_human_t0.dat.gz" + interpro.write_bytes(gzip.compress(("\n".join(ipr_lines) + "\n").encode())) + return {"gaf": gaf, "interpro": interpro} + + +def run(archived: dict[str, Path], out: Path, *extra: str) -> int: + return main( + [ + "--gaf", + str(archived["gaf"]), + "--interpro", + str(archived["interpro"]), + "--output-dir", + str(out), + *extra, + ] + ) + + +def test_a_run_reads_and_records_the_overrides( + archived: dict[str, Path], tmp_path: Path +) -> None: + assert run(archived, tmp_path / "out") == 0 + + rows = (tmp_path / "out" / "domain_go_associations_significant.tsv").read_text() + assert "IPR000001\tGO:0006811" in rows + + manifest = json.loads((tmp_path / "out" / "run_manifest_go.json").read_text()) + by_role = {record["role"]: record for record in manifest["inputs"]} + for role, key in (("gaf", "gaf"), ("domain_annotations", "interpro")): + record = by_role[role] + assert record["path"] == str(archived[key]) + assert ( + record["sha256"] == hashlib.sha256(archived[key].read_bytes()).hexdigest() + ) + # Not the current-release URL the species name would have implied. + assert "source_url" not in record and "derived_from" not in record + assert by_role["gaf"]["release_metadata"]["date_generated"] == DATE_GENERATED + assert manifest["status"] == "completed" + + +@pytest.mark.parametrize("key", ["gaf", "interpro"]) +def test_a_missing_override_fails_before_the_run( + archived: dict[str, Path], tmp_path: Path, key: str +) -> None: + archived[key] = tmp_path / "absent.gz" + + assert run(archived, tmp_path / "out") == 1 + assert not (tmp_path / "out" / "run_manifest_go.json").exists() diff --git a/tests/integration/test_validation_clis.py b/tests/integration/test_validation_clis.py index b4ce271..4ced946 100644 --- a/tests/integration/test_validation_clis.py +++ b/tests/integration/test_validation_clis.py @@ -3,6 +3,8 @@ from __future__ import annotations import gzip +import hashlib +import json import subprocess import sys from pathlib import Path @@ -226,3 +228,160 @@ def test_validate_results_direct_script_uses_its_own_checkout(tmp_path: Path) -> assert result.returncode == 1 assert "Missing required inputs" in result.stderr assert "ModuleNotFoundError" not in result.stderr + + +@pytest.fixture +def t0_benchmark(tmp_path: Path) -> dict[str, Path]: + """A two-protein 2021→2026 split: P0 annotated at t0, P1 only at t1 (the + held-out cohort), both carrying IPR_A in the protein2ipr subset.""" + t0 = tmp_path / "t0.gaf.gz" + with gzip.open(t0, "wt") as handle: + handle.write("!gaf-version: 2.2\n" + _gaf_row("P0", CHILD) + "\n") + t1 = tmp_path / "t1.gaf.gz" + with gzip.open(t1, "wt") as handle: + handle.write("!gaf-version: 2.2\n") + for protein in ("P0", "P1"): + handle.write(_gaf_row(protein, CHILD) + "\n") + interpro = tmp_path / "protein2ipr_t0.dat.gz" + with gzip.open(interpro, "wt") as handle: + for protein in ("P0", "P1"): + handle.write(f"{protein}\tIPR_A\tA\tPF00001\t10\t100\n") + return {"t0": t0, "t1": t1, "interpro": interpro} + + +def _write_run(directory: Path, interpro_sha256: str) -> None: + """A rung's association tables and the manifest naming its protein2ipr.""" + directory.mkdir(parents=True, exist_ok=True) + table = pd.DataFrame( + [("IPR_A", CHILD, 0.001, 0.01, 0.01)], + columns=["domain", "go_term", "p_value", "adj_p_value", "q_value"], + ) + for name in ("associations_significant", "annotations_propagated"): + table.to_csv(directory / f"domain_go_{name}.tsv", sep="\t", index=False) + (directory / "run_manifest_go.json").write_text( + json.dumps( + { + "parameters": {"domain_key": "interpro", "evidence_filter": "manual"}, + "inputs": [{"role": "domain_annotations", "sha256": interpro_sha256}], + } + ), + encoding="utf-8", + ) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_temporal_benchmark_scores_through_the_architectures_training_used( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path] +) -> None: + run = tmp_path / "run" + _write_run(run, _sha256(t0_benchmark["interpro"])) + + result = _run( + "temporal_benchmark.py", + "--t0-gaf", + t0_benchmark["t0"], + "--t1-gaf", + t0_benchmark["t1"], + "--predictions", + run / "domain_go_associations_significant.tsv", + "--interpro", + t0_benchmark["interpro"], + "--go-ontology", + go_obo, + "--n-permutations", + 1, + "--min-ic", + 0, + "--output-dir", + tmp_path / "metrics", + ) + + assert result.returncode == 0, result.stderr + metrics = pd.read_csv( + tmp_path / "metrics" / "temporal_benchmark_metrics.tsv", sep="\t" + ) + dcgo = metrics[metrics["method"] == "dcGO"].iloc[0] + assert dcgo["n_eval_proteins"] == 1 # P1, transferred through IPR_A + assert dcgo["f_max"] == pytest.approx(1.0) + + +def test_temporal_benchmark_refuses_other_architectures( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path] +) -> None: + run = tmp_path / "run" + _write_run(run, "0" * 64) # trained on some other protein2ipr + + result = _run( + "temporal_benchmark.py", + "--t0-gaf", + t0_benchmark["t0"], + "--t1-gaf", + t0_benchmark["t1"], + "--predictions", + run / "domain_go_associations_significant.tsv", + "--interpro", + t0_benchmark["interpro"], + "--go-ontology", + go_obo, + "--output-dir", + tmp_path / "metrics", + ) + + assert result.returncode == 1 + assert "protein2ipr sha256" in result.stderr + assert not (tmp_path / "metrics").exists() + + +@pytest.mark.parametrize("matching", [True, False]) +def test_ablation_scores_only_runs_trained_on_its_protein2ipr( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path], matching: bool +) -> None: + sha256 = _sha256(t0_benchmark["interpro"]) + for rung in ( + "single", + "supra", + "supra_input", + "supra_relative", + "supra_output", + "supra_input_relative", + "supra_input_output", + "supra_relative_output", + "full", + ): + recorded = sha256 if matching or rung != "full" else "0" * 64 + _write_run(tmp_path / "runs" / rung, recorded) + + result = _run( + "ablation.py", + "--t0-gaf", + t0_benchmark["t0"], + "--t1-gaf", + t0_benchmark["t1"], + "--run-dir", + tmp_path / "runs", + "--interpro", + t0_benchmark["interpro"], + "--go-ontology", + go_obo, + "--min-ic", + 0, + "--n-bootstrap", + 2, + "--n-permutations", + 0, + "--output-dir", + tmp_path / "eval", + ) + + if not matching: + assert result.returncode == 1 + assert "full: run protein2ipr sha256=" in result.stderr + assert not (tmp_path / "eval" / "ablation_provenance.tsv").exists() + return + assert result.returncode == 0, result.stderr + provenance = pd.read_csv(tmp_path / "eval" / "ablation_provenance.tsv", sep="\t") + assert set(provenance["interpro_file"]) == {str(t0_benchmark["interpro"])} + assert set(provenance["interpro_sha256"]) == {sha256} diff --git a/tests/unit/test_ablation.py b/tests/unit/test_ablation.py index 0bf556f..23f5c60 100644 --- a/tests/unit/test_ablation.py +++ b/tests/unit/test_ablation.py @@ -178,12 +178,19 @@ def test_empty_file_is_zero(self, tmp_path): class TestRunSettingMismatches: - def _write(self, root, rung, **params): + def _write(self, root, rung, inputs=(), **params): (root / rung.run_dir).mkdir(parents=True, exist_ok=True) (root / rung.run_dir / "run_manifest_go.json").write_text( - json.dumps({"parameters": params}) + json.dumps({"parameters": params, "inputs": list(inputs)}) ) + @staticmethod + def _domains(sha256): + return [ + {"role": "domain_annotations", "sha256": sha256}, + {"role": "gaf", "sha256": "f" * 64}, + ] + def test_matching_runs_pass(self, tmp_path): for rung in ab.LADDER: self._write(tmp_path, rung, evidence_filter="all", domain_key="ssf") @@ -206,3 +213,36 @@ def test_runs_without_a_manifest_are_skipped(self, tmp_path): ) == [] ) + + def test_protein2ipr_mismatch_is_reported(self, tmp_path): + # The bug this guards: t0-trained associations transferred through + # today's architectures (or the reverse). + rung = ab.LADDER[0] + self._write(tmp_path, rung, inputs=self._domains("a" * 64)) + problems = ab.run_setting_mismatches( + tmp_path, [rung], {}, interpro_sha256="b" * 64 + ) + assert problems == [ + f"{rung.name}: run protein2ipr sha256={'a' * 64}, evaluator {'b' * 64}" + ] + + def test_matching_protein2ipr_passes(self, tmp_path): + for rung in ab.LADDER: + self._write(tmp_path, rung, inputs=self._domains("a" * 64)) + assert ( + ab.run_setting_mismatches(tmp_path, ab.LADDER, {}, interpro_sha256="a" * 64) + == [] + ) + + def test_protein2ipr_is_checked_only_when_given(self, tmp_path): + rung = ab.LADDER[0] + self._write(tmp_path, rung, inputs=self._domains("a" * 64)) + assert ab.run_setting_mismatches(tmp_path, [rung], {}) == [] + + def test_a_manifest_without_domain_annotations_is_not_checked(self, tmp_path): + rung = ab.LADDER[0] + self._write(tmp_path, rung) + assert ( + ab.run_setting_mismatches(tmp_path, [rung], {}, interpro_sha256="b" * 64) + == [] + ) diff --git a/tests/unit/test_download_checksums.py b/tests/unit/test_download_checksums.py index f27e0c0..c4da599 100644 --- a/tests/unit/test_download_checksums.py +++ b/tests/unit/test_download_checksums.py @@ -51,3 +51,38 @@ def test_streams_in_chunks(self, downloader, tmp_path): def test_other_algorithms_supported(self, downloader, payload): digest = hashlib.md5(payload.read_bytes()).hexdigest() downloader.verify_checksum(payload, ("md5", digest)) + + +class TestInterpro85Group: + """The archived t0 InterPro release is opt-in, pinned, and lands where the + temporal benchmark's runbook reads it.""" + + def test_members_are_pinned_to_the_85_0_release(self, downloader): + from config.settings import config + + for name in downloader.DATASET_GROUPS["interpro-85"]: + source = config.data_sources[name] + assert source.url.startswith( + "https://ftp.ebi.ac.uk/pub/databases/interpro/releases/85.0/" + ) + assert source.subdir == "interpro_archive/85.0" + assert source.checksum_parts() is not None + assert source.size_bytes is not None + assert not source.required + assert name not in downloader.DEFAULT_DATASETS + + def test_protein2ipr_is_checked_against_the_published_md5(self, downloader): + from config.settings import config + + source = config.data_sources["interpro_85_protein2ipr"] + # The content of releases/85.0/protein2ipr.dat.gz.md5. + assert source.checksum_parts() == ("md5", "ad41fd5c4a8dc83303683a19a34ec0f1") + + def test_files_keep_their_release_names(self, downloader): + from config.settings import config + + names = { + downloader._filename_for(config.data_sources[name].url, name) + for name in downloader.DATASET_GROUPS["interpro-85"] + } + assert names == {"protein2ipr.dat.gz", "interpro2go", "entry.list"} diff --git a/tests/unit/test_generic_runner.py b/tests/unit/test_generic_runner.py index 926a319..0704b99 100644 --- a/tests/unit/test_generic_runner.py +++ b/tests/unit/test_generic_runner.py @@ -64,3 +64,37 @@ def test_run_request_is_immutable() -> None: with pytest.raises(FrozenInstanceError): request.species = "mouse" # type: ignore[misc] + + +def test_gaf_and_interpro_default_to_the_species_files() -> None: + resolved = resolve_inputs(parse_run_request(["--species", "mouse"])) + + assert resolved.ontology_paths["gaf"] == Path( + "data/raw/goa_annotations/goa_mouse.gaf.gz" + ) + assert resolved.interpro_file == Path("data/interim/protein2ipr_mouse.dat.gz") + + +def test_gaf_and_interpro_overrides_replace_the_species_files(tmp_path: Path) -> None: + """The temporal benchmark trains on archived files, not the species' current ones.""" + gaf = tmp_path / "goa_human.gaf.205.gz" + gaf.write_text("!gaf-version: 2.2\n") + interpro = tmp_path / "protein2ipr_human_t0.dat.gz" + + resolved = resolve_inputs( + parse_run_request(["--gaf", str(gaf), "--interpro", str(interpro)]) + ) + + assert resolved.ontology_paths["gaf"] == gaf + assert resolved.interpro_file == interpro + assert resolved.missing_inputs == () + + +def test_a_missing_gaf_override_is_caught_by_the_early_input_check( + tmp_path: Path, +) -> None: + resolved = resolve_inputs( + parse_run_request(["--gaf", str(tmp_path / "absent.gaf.gz")]) + ) + + assert any("absent.gaf.gz" in missing for missing in resolved.missing_inputs) diff --git a/tests/unit/test_run_manifest_wiring.py b/tests/unit/test_run_manifest_wiring.py index 09e11cf..e872f18 100644 --- a/tests/unit/test_run_manifest_wiring.py +++ b/tests/unit/test_run_manifest_wiring.py @@ -14,6 +14,7 @@ import pytest +import run_dcgo_human from run_dcgo_human import build_ontology_paths, start_run_manifest from src.ontology_registry import ONTOLOGIES, get_ontology @@ -272,3 +273,41 @@ def test_hierarchy_stage_flags_make_the_ic_source_propagated(tmp_path, fake_inpu ) assert data["analysis"]["thresholds"]["ic_source"] == "propagated" assert data["analysis"]["thresholds"]["min_ic"] is None + + +UPSTREAM = { + "gaf": "https://example.org/goa_human.gaf.gz", + "interpro_mappings": "https://example.org/protein2ipr.dat.gz", +} + + +def test_default_inputs_are_labelled_with_their_upstream_source( + tmp_path, fake_inputs, monkeypatch +): + monkeypatch.setattr(run_dcgo_human, "input_source_urls", lambda _: dict(UPSTREAM)) + _, data = run_manifest_json(tmp_path, fake_inputs) + + by_role = {record["role"]: record for record in data["inputs"]} + assert by_role["gaf"]["source_url"] == UPSTREAM["gaf"] + assert ( + by_role["domain_annotations"]["derived_from"] == UPSTREAM["interpro_mappings"] + ) + + +def test_overridden_inputs_are_hashed_without_the_species_url( + tmp_path, fake_inputs, monkeypatch +): + """An archived GAF is not the file at the current-release URL; say nothing + rather than the wrong thing. The hash still identifies the bytes.""" + monkeypatch.setattr(run_dcgo_human, "input_source_urls", lambda _: dict(UPSTREAM)) + interpro = tmp_path / "protein2ipr_t0.dat.gz" + _, data = run_manifest_json( + tmp_path, fake_inputs, gaf=fake_inputs["gaf"], interpro=interpro + ) + + by_role = {record["role"]: record for record in data["inputs"]} + assert "source_url" not in by_role["gaf"] + assert "derived_from" not in by_role["domain_annotations"] + assert by_role["gaf"]["sha256"] and by_role["domain_annotations"]["sha256"] + assert data["parameters"]["gaf"] == str(fake_inputs["gaf"]) + assert data["parameters"]["interpro"] == str(interpro) diff --git a/tests/unit/test_run_t0_interpro_ablation.py b/tests/unit/test_run_t0_interpro_ablation.py new file mode 100644 index 0000000..e1dadc1 --- /dev/null +++ b/tests/unit/test_run_t0_interpro_ablation.py @@ -0,0 +1,110 @@ +"""The t0-InterPro ablation runbook re-runs exactly the committed rungs. + +Its contract is narrow: every rung command is the one the committed manifest +records, with only the inputs and the output directory changed, and the cell +covers the whole ablation ladder. +""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_ROOT = Path(__file__).resolve().parents[2] + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +runbook = _load( + "run_t0_interpro_ablation", _ROOT / "scripts" / "run_t0_interpro_ablation.py" +) +ab = _load("ablation", _ROOT / "validation" / "ablation.py") + +SUBSET = Path("data/interim/protein2ipr_human_t0_ipr85.dat.gz") + + +def test_the_cell_covers_the_ablation_ladder(): + assert set(runbook.RUNGS) == {rung.run_dir for rung in ab.LADDER} + + +@pytest.mark.parametrize("rung", runbook.RUNGS) +def test_rung_commands_change_only_inputs_and_output(rung): + manifest = runbook.MANIFESTS / f"{rung}.json" + recorded = json.loads(manifest.read_text())["command"][1:] + + command = runbook.rung_command(manifest, Path("out") / rung, SUBSET) + + assert command[0] == sys.executable + assert command[-4:] == ["--gaf", str(runbook.T0_GAF), "--interpro", str(SUBSET)] + out = command.index("--output-dir") + assert command[out + 1] == str(Path("out") / rung) + # Everything else, in order, is what the committed run recorded. + assert command[1:out] + command[out + 2 : -4] == ( + recorded[: out - 1] + recorded[out + 1 :] + ) + + +def test_the_archive_is_verified_and_the_subset_extracted(tmp_path): + steps = dict( + runbook.build_steps( + runbook.ARCHIVE, tmp_path / "subset.dat.gz", tmp_path, tmp_path + ) + ) + assert steps["verify"][-2:] == ["--group", "interpro-85"] + extract = steps["extract"] + assert extract[extract.index("--source") + 1] == str(runbook.ARCHIVE) + assert [extract[i + 1] for i, a in enumerate(extract) if a == "--gaf"] == [ + str(runbook.T0_GAF), + str(runbook.T1_GAF), + ] + assert extract[extract.index("--evidence-filter") + 1] == "all" + evaluate = steps["eval"] + assert evaluate[evaluate.index("--interpro") + 1] == str(tmp_path / "subset.dat.gz") + + +def test_a_subset_already_cut_from_the_same_source_is_reused(tmp_path): + from src.universe_provenance import write_marker + + source = tmp_path / "protein2ipr.dat.gz" + subset = tmp_path / "subset.dat.gz" + subset.write_bytes(b"") + write_marker( + subset, + selection_rule="goa", + selection_sources=[runbook.T0_GAF, runbook.T1_GAF], + interpro_source=source, + n_accessions=0, + n_matched_lines=0, + tool="extract_human_interpro.py", + ) + + names = [ + name for name, _ in runbook.build_steps(source, subset, tmp_path, tmp_path) + ] + assert "verify" not in names # not the archive: nothing to verify + assert "extract" not in names + other = [ + name + for name, _ in runbook.build_steps( + tmp_path / "other.dat.gz", subset, tmp_path, tmp_path + ) + ] + assert "extract" in other + + +def test_dry_run_prints_and_runs_nothing(tmp_path, capsys): + run_dir = tmp_path / "runs" + assert runbook.main(["--dry-run", "--run-dir", str(run_dir)]) == 0 + printed = capsys.readouterr().out + assert "# full" in printed and "validation/ablation.py" in printed + assert not run_dir.exists() diff --git a/tests/unit/test_temporal_benchmark.py b/tests/unit/test_temporal_benchmark.py index 1dcac31..fe06418 100644 --- a/tests/unit/test_temporal_benchmark.py +++ b/tests/unit/test_temporal_benchmark.py @@ -7,6 +7,7 @@ """ import importlib.util +import json import math import sys from pathlib import Path @@ -109,6 +110,27 @@ def test_merged_id_prediction_is_scored_and_propagated(self): assert pred["P1"] == {"GO:leaf": 1.0, "GO:mid": 1.0, "GO:other": 0.0} +class TestRecordedInterproSha256: + def test_reads_the_domain_annotations_input(self, tmp_path): + manifest = tmp_path / "run_manifest_go.json" + manifest.write_text( + json.dumps( + { + "inputs": [ + {"role": "domain_annotations", "sha256": "a" * 64}, + {"role": "gaf", "sha256": "b" * 64}, + ] + } + ) + ) + assert tb.recorded_interpro_sha256(manifest) == "a" * 64 + + def test_none_when_the_manifest_records_no_domain_file(self, tmp_path): + manifest = tmp_path / "run_manifest_go.json" + manifest.write_text(json.dumps({"inputs": [{"role": "gaf"}]})) + assert tb.recorded_interpro_sha256(manifest) is None + + class TestBuildNKBenchmark: def test_truth_is_full_t1_not_delta(self): # t0 experimental has nothing in BP; t1 adds leaf. Truth is the FULL diff --git a/validation/ablation.py b/validation/ablation.py index 2ff77e7..06f47d4 100644 --- a/validation/ablation.py +++ b/validation/ablation.py @@ -45,6 +45,14 @@ ``domain_go_annotations_propagated.tsv``; all other rungs use the significant association table. +Each rung is one ``run_dcgo_human.py --gaf --output-dir /`` +plus its stage flags (``validation/ablation_manifests/*.json`` record the exact +commands). ``scripts/run_t0_interpro_ablation.sh`` runs a whole cell — the nine +rungs and this evaluation — with ``--interpro`` set to t0 domain architectures. +``--interpro`` here must be the protein2ipr file the rungs were trained on, and +``--domain-key``/``--evidence-filter`` their settings: the driver refuses to +score a run whose manifest says otherwise. + Honest caveat, stated once here and again in ``VALIDATION_PLAN.md``: the True Path rungs are scored on ``q_value`` because the propagated output carries no ``p_value`` column. To keep the ladder comparable, **every** rung's primary score @@ -304,13 +312,19 @@ def rung_prediction_file( def run_setting_mismatches( - run_root: Path, rungs: Iterable[Rung], expected: Mapping[str, str] + run_root: Path, + rungs: Iterable[Rung], + expected: Mapping[str, str], + interpro_sha256: str | None = None, ) -> list[str]: """Rungs whose run manifest disagrees with the evaluator's settings. The no-knowledge cohort and IC are built from what training saw, so the evaluator's ``domain_key``/``evidence_filter`` must be the ones each run - used. Rungs without a manifest are not checked. + used. Likewise the transfer step must read the domain architectures the + run was trained on: ``interpro_sha256``, if given, is compared with the + run's recorded protein2ipr (see ``tb.recorded_interpro_sha256``). Rungs + without a manifest are not checked. """ problems = [] for rung in rungs: @@ -322,6 +336,13 @@ def run_setting_mismatches( got = params.get(key) if got is not None and got != want: problems.append(f"{rung.name}: run {key}={got}, evaluator {want}") + if interpro_sha256 is not None: + trained_on = tb.recorded_interpro_sha256(manifest) + if trained_on is not None and trained_on != interpro_sha256: + problems.append( + f"{rung.name}: run protein2ipr sha256={trained_on}, " + f"evaluator {interpro_sha256}" + ) return problems @@ -584,6 +605,10 @@ def main() -> int: # pragma: no cover - I/O wiring logger.remove() logger.add(sys.stderr, level="INFO") + if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + from src.run_manifest import sha256_file + parser = argparse.ArgumentParser( description="dcGO component ablation with bootstrap CIs and a permutation null " "(VALIDATION_PLAN §4)." @@ -597,7 +622,12 @@ def main() -> int: # pragma: no cover - I/O wiring help="Root holding one sub-directory per current-code factorial run", ) parser.add_argument( - "--interpro", type=Path, default=Path("data/interim/protein2ipr_human.dat.gz") + "--interpro", + type=Path, + default=Path("data/interim/protein2ipr_human.dat.gz"), + help="protein2ipr file for the transfer step and the cohort — must be " + "the one the runs under --run-dir were trained on " + "(default: data/interim/protein2ipr_human.dat.gz)", ) parser.add_argument( "--domain-key", @@ -658,10 +688,12 @@ def main() -> int: # pragma: no cover - I/O wiring if not path.exists(): logger.error(f"Missing required input: {path}") return 1 + interpro_sha256 = sha256_file(args.interpro) mismatches = run_setting_mismatches( args.run_dir, LADDER, {"domain_key": args.domain_key, "evidence_filter": args.evidence_filter}, + interpro_sha256=interpro_sha256, ) if mismatches: for problem in mismatches: @@ -703,6 +735,12 @@ def main() -> int: # pragma: no cover - I/O wiring except FileNotFoundError as missing: logger.error(str(missing)) return 1 + # Every method is scored on the cohort and architectures of this file. + architectures_used = { + "interpro_file": str(args.interpro), + "interpro_sha256": interpro_sha256, + } + provenance = [{**row, **architectures_used} for row in provenance] pd.DataFrame(provenance).to_csv( args.output_dir / "ablation_provenance.tsv", sep="\t", index=False ) diff --git a/validation/mouse/PROVENANCE.md b/validation/mouse/PROVENANCE.md index 462b485..5ec84e2 100644 --- a/validation/mouse/PROVENANCE.md +++ b/validation/mouse/PROVENANCE.md @@ -38,10 +38,10 @@ uv run python extract_human_interpro.py --species mouse curl -o data/raw/goa_archive/goa_mouse.gaf.191.gz \ https://ftp.ebi.ac.uk/pub/databases/GO/goa/old/MOUSE/goa_mouse.gaf.191.gz -# 2. Train on t0. `--species mouse_t0_2021` resolves to the symlinks -# goa_mouse_t0_2021.gaf.gz -> goa_mouse.gaf.191.gz and -# protein2ipr_mouse_t0_2021.dat.gz -> protein2ipr_mouse.dat.gz -uv run python run_dcgo_human.py --species mouse_t0_2021 --num-cores 8 \ +# 2. Train on t0: the archived GAF, the current mouse domain subset (the +# --interpro default for --species mouse) +uv run python run_dcgo_human.py --species mouse \ + --gaf data/raw/goa_archive/goa_mouse.gaf.191.gz --num-cores 8 \ --output-dir results_mouse_t0_2021 # 3. Score against t1 diff --git a/validation/temporal_benchmark.py b/validation/temporal_benchmark.py index 7383228..bb27e8a 100644 --- a/validation/temporal_benchmark.py +++ b/validation/temporal_benchmark.py @@ -20,9 +20,12 @@ experimental MF annotations) tests whether the naive baseline's F_max lead is just base-rate recovery of generic terms. -Because the domain architectures come from ``protein2ipr`` and are not -time-varying in our data, only the GOA annotations move in time — the split is -purely on the annotation side. +The domain architectures come from ``--interpro``. By default that is the +current ``protein2ipr`` subset, so only the GOA annotations move in time and the +split is purely on the annotation side. Passing a subset of an archived +InterPro release (``extract_human_interpro.py --source``) moves the +architectures to t0 too; it must be the file the predictions were trained on, +which is checked against the run manifest beside ``--predictions``. The metric maths lives in small pure functions (unit-tested in ``tests/unit/test_temporal_benchmark.py``); the ``main`` at the bottom wires them @@ -40,6 +43,7 @@ from __future__ import annotations +import json import math import sys from collections import defaultdict @@ -643,6 +647,23 @@ def _load_resampling(): # pragma: no cover - import plumbing return module +def recorded_interpro_sha256(manifest_path: Path) -> str | None: + """SHA-256 of the protein2ipr file a run manifest says training read. + + ``None`` when the manifest records no ``domain_annotations`` input. The + evaluator must transfer through the same architectures training learned + from: a different file is a different domain vocabulary (supra-domain ids + are built from it) and, for a temporal benchmark, a different date. The + digest is of the file's bytes, so a re-compressed copy of identical rows + also counts as different — pass the file the runs were made with. + """ + data = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + for record in data.get("inputs", []): + if record.get("role") == "domain_annotations": + return record.get("sha256") + return None + + def build_term_aspect(processor) -> dict[str, str]: """Map each GO term to BP/MF/CC using the ontology namespace.""" term_aspect: dict[str, str] = {} @@ -671,6 +692,7 @@ def main() -> int: from src.domain_annotation_parser import DomainAnnotationParser from src.goa_parser import EXPERIMENTAL_EVIDENCE, GOAParser, parse_goa_human from src.ontology_processor import OntologyProcessor + from src.run_manifest import sha256_file parser = argparse.ArgumentParser( description="Temporal held-out CAFA-style benchmark for dcGO (VALIDATION_PLAN §2)." @@ -691,7 +713,8 @@ def main() -> int: "--interpro", type=Path, default=Path("data/interim/protein2ipr_human.dat.gz"), - help="protein2ipr human subset (domain architectures)", + help="protein2ipr human subset (domain architectures) — must be the " + "file the predictions were trained on", ) parser.add_argument( "--go-ontology", @@ -764,6 +787,19 @@ def main() -> int: logger.error(f"Missing required input: {p}") return 1 + # Transfer through the architectures training learned from (see + # recorded_interpro_sha256). Predictions without a manifest are not checked. + manifest = args.predictions.parent / "run_manifest_go.json" + if manifest.exists(): + trained_on = recorded_interpro_sha256(manifest) + given = sha256_file(args.interpro) + if trained_on is not None and trained_on != given: + logger.error( + f"Settings mismatch — {manifest} records protein2ipr sha256 " + f"{trained_on}, --interpro {args.interpro} is {given}" + ) + return 1 + args.output_dir.mkdir(parents=True, exist_ok=True) logger.info("Loading GO ontology (for propagation + aspects)...") From 1c66e13f594e73be146d803bc0a986777d768e23 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 20:31:54 +0100 Subject: [PATCH 2/7] Resume the t0 runbook past rungs that already completed On a loaded machine one rung can outlive a job limit; rerunning the whole cell then repeats ~15 minutes of finished work. A rung whose manifest is finalised ("completed") is now skipped, as the extraction already was when its marker names the same source. Skipping is safe because the evaluator refuses any rung trained on a different protein2ipr than the subset. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/run_t0_interpro_ablation.py | 23 ++++++++++++++++----- tests/unit/test_run_t0_interpro_ablation.py | 16 ++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/scripts/run_t0_interpro_ablation.py b/scripts/run_t0_interpro_ablation.py index 3c97d3d..822dc98 100644 --- a/scripts/run_t0_interpro_ablation.py +++ b/scripts/run_t0_interpro_ablation.py @@ -22,7 +22,9 @@ Run it from the repository root; paths are relative to it, and ``data/`` must hold the inputs. The md5 of the archived protein2ipr is verified first -(``scripts/download_data.py --group interpro-85``). +(``scripts/download_data.py --group interpro-85``). Resumable: a rung whose +manifest says ``completed`` is not rerun — safe, because the evaluator refuses +a rung trained on a different protein2ipr than ``--subset``. Dry run of the whole chain without the archive: pass the current subset as ``--source`` (with scratch ``--subset``/``--run-dir``/``--eval-dir``). A wider @@ -45,6 +47,7 @@ REPO = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO)) +from src.run_manifest import manifest_filename # noqa: E402 from src.universe_provenance import read_marker # noqa: E402 MANIFESTS = REPO / "validation" / "ablation_manifests" @@ -75,10 +78,19 @@ def rung_command(manifest: Path, output_dir: Path, interpro: Path) -> list[str]: return [sys.executable, *args, "--gaf", str(T0_GAF), "--interpro", str(interpro)] +def completed(output_dir: Path) -> bool: + """Whether a rung's run finished (its manifest is finalised).""" + manifest = output_dir / manifest_filename("go") + return ( + manifest.exists() + and json.loads(manifest.read_text(encoding="utf-8"))["status"] == "completed" + ) + + def build_steps( source: Path, subset: Path, run_dir: Path, eval_dir: Path ) -> list[tuple[str, list[str]]]: - """(log name, command) for every step, in order.""" + """(log name, command) for every step still to run, in order.""" steps = [] if source == ARCHIVE: steps.append( @@ -109,9 +121,10 @@ def build_steps( ) ) for rung in RUNGS: - steps.append( - (rung, rung_command(MANIFESTS / f"{rung}.json", run_dir / rung, subset)) - ) + if not completed(run_dir / rung): + steps.append( + (rung, rung_command(MANIFESTS / f"{rung}.json", run_dir / rung, subset)) + ) steps.append( ( "eval", diff --git a/tests/unit/test_run_t0_interpro_ablation.py b/tests/unit/test_run_t0_interpro_ablation.py index e1dadc1..d2252e8 100644 --- a/tests/unit/test_run_t0_interpro_ablation.py +++ b/tests/unit/test_run_t0_interpro_ablation.py @@ -102,6 +102,22 @@ def test_a_subset_already_cut_from_the_same_source_is_reused(tmp_path): assert "extract" in other +def test_completed_rungs_are_not_rerun(tmp_path): + (tmp_path / "single").mkdir() + (tmp_path / "single" / "run_manifest_go.json").write_text('{"status": "completed"}') + (tmp_path / "supra").mkdir() # interrupted: the manifest never finalised + (tmp_path / "supra" / "run_manifest_go.json").write_text('{"status": "running"}') + + names = [ + name + for name, _ in runbook.build_steps( + tmp_path / "src.dat.gz", tmp_path / "subset.dat.gz", tmp_path, tmp_path + ) + ] + assert "single" not in names + assert names[names.index("supra") :] == [*runbook.RUNGS[1:], "eval"] + + def test_dry_run_prints_and_runs_nothing(tmp_path, capsys): run_dir = tmp_path / "runs" assert runbook.main(["--dry-run", "--run-dir", str(run_dir)]) == 0 From 14b9cd7f2b318c8f16027d47104ea05276f29682 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 7 Oct 2026 10:06:37 +0100 Subject: [PATCH 3/7] Check the training GAF and fix the t0 cohort in the evaluators Review of the t0 InterPro branch: - The evaluators now compare the run manifest's training GAF with --t0-gaf, as they already did protein2ipr with --interpro. The committed ablation manifests read GOA 205 through the default GAF path, which now holds the 2026 file: replaying one without --gaf trains on the held-out labels, and nothing refused it. The provenance TSV records the t0/t1 GAF and cohort file hashes too. apply_relative_inference.py, temporal_surprise.py and temporal_breadth.py apply the same check. - temporal_benchmark.py --allow-input-mismatch turns the check into a warning. The all-species held-out arm (scratch_allspecies/13, 14) trains on other protein2ipr and GAF files on purpose and was refused outright; both scripts now pass it. - ablation.py --cohort-interpro fixes the scored cohort to one file. The runbook passes the committed cell's, so the t0 arm is scored on the same no-knowledge proteins as the committed cell and the control: a protein with no InterPro 85.0 domain is a miss, not dropped from the denominator, and ablation_selection_counts.tsv counts them. Default unchanged. - The runbook reuses a subset only when its marker records this exact extraction (source, both GAFs, evidence filter "all"), reruns every rung when it cuts a new one, and copies the subset's sidecar into the eval manifests so the archive and selection are committed. Extract markers now record the evidence filter. - Docs: the runbook is .py, not .sh; the current protein2ipr was downloaded 2026-07-07, not "of 2026-07-22"; VALIDATION_PLAN states the rerun's remaining look-ahead (seven of nine rungs train on go-basic 2026-06-15). Co-Authored-By: Claude Opus 5.5 (1M context) --- RELATIVE_INFERENCE_REPORT.md | 6 +- REPRODUCIBILITY.md | 4 +- VALIDATION_PLAN.md | 33 +++- config/settings.py | 2 +- extract_human_interpro.py | 1 + scratch_allspecies/13_heldout_allspecies.sh | 3 + scratch_allspecies/14_heldout_experimental.sh | 7 +- scripts/download_data.py | 2 +- scripts/run_t0_interpro_ablation.py | 39 +++- src/universe_provenance.py | 6 + .../test_extract_human_interpro.py | 1 + tests/integration/test_validation_clis.py | 166 +++++++++++++++++- tests/unit/test_ablation.py | 55 +++++- tests/unit/test_run_t0_interpro_ablation.py | 107 ++++++++--- tests/unit/test_temporal_benchmark.py | 32 +++- tests/unit/test_universe_provenance.py | 20 +++ validation/ablation.py | 157 ++++++++++++----- validation/apply_relative_inference.py | 23 ++- validation/temporal_benchmark.py | 73 +++++--- validation/temporal_breadth.py | 14 ++ validation/temporal_surprise.py | 17 ++ 21 files changed, 633 insertions(+), 135 deletions(-) diff --git a/RELATIVE_INFERENCE_REPORT.md b/RELATIVE_INFERENCE_REPORT.md index 706bd31..86eda13 100644 --- a/RELATIVE_INFERENCE_REPORT.md +++ b/RELATIVE_INFERENCE_REPORT.md @@ -13,8 +13,8 @@ human-only run of that reimplementation: - **Training annotations:** human UniProtKB-GOA, archived release 205 (April 2021), manual (non-IEA) evidence codes — 18,735 annotated proteins. -- **Domains:** InterPro entries (InterPro `current_release` snapshot of - 2026-07-22), plus contiguous "supra-domain" combinations of up to three +- **Domains:** InterPro entries (InterPro `current_release` as downloaded + 2026-07-07), plus contiguous "supra-domain" combinations of up to three entries (as in the original method, which used SCOP superfamilies). - **Ontology:** GO, `go-basic.obo` release 2026-06-15, `is_a`/`part_of` edges. Input annotations are true-path propagated before testing, as the @@ -591,7 +591,7 @@ test with its measured cost rather than as a default. all of them clean their input. Inputs are unchanged: training GAF sha256 `69ae7d90…` (= archived GOA release 205), `data/interim/protein2ipr_human.dat.gz` sha256 `a932a515…` (InterPro - `current_release` snapshot of 2026-07-22), `go-basic.obo` sha256 + `current_release` as downloaded 2026-07-07), `go-basic.obo` sha256 `c72fc198…` (release 2026-06-15). Design and prose: `VALIDATION_PLAN.md` §4. - **Regeneration check**: the committed manifests come from a clean diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md index 8a42b62..dc5d94a 100644 --- a/REPRODUCIBILITY.md +++ b/REPRODUCIBILITY.md @@ -56,7 +56,9 @@ Inputs passed with `--gaf`/`--interpro` (archived snapshots for the temporal benchmark) are hashed and header-read the same way but carry no `source_url`/`derived_from`: the species' current-release URL would name different bytes. A subset's own `.provenance.json` sidecar names the archive it -was cut from. +was cut from, the GAFs and the evidence filter that selected it; +`scripts/run_t0_interpro_ablation.py` copies it into the evaluation's +`manifests/` so it is committed with the results. The inputs recorded are exactly those the selected ontology declares in `src/ontology_registry.py` (`needs`, plus `hierarchy_needs` when diff --git a/VALIDATION_PLAN.md b/VALIDATION_PLAN.md index b45aa65..2c4d7fd 100644 --- a/VALIDATION_PLAN.md +++ b/VALIDATION_PLAN.md @@ -1232,7 +1232,10 @@ uv run python validation/check_metric_conventions.py \ Each rung is the command its manifest records, with the training GAF passed explicitly (the committed runs read it through a link at the default GAF path, which is why their `gaf` input carries the current-release `source_url`; the -SHA-256 `69ae7d90…` is the identity): +SHA-256 `69ae7d90…` is the identity). That path now holds the 2026 GAF — the +held-out labels — so a replay without `--gaf` trains on them; the evaluator +refuses such a run, because it checks each manifest's GAF SHA-256 against +`--t0-gaf`: ```bash uv run python run_dcgo_human.py --num-cores 8 \ @@ -1245,15 +1248,27 @@ uv run python validation/ablation.py \ ``` These cells still read **current** domain architectures -(`protein2ipr_human.dat.gz`, InterPro of 2026-07-22) in training and in the -transfer step — temporal look-ahead on the domain side (TODO.md P0). -`scripts/run_t0_interpro_ablation.py` reruns the `ipr_manual` cell on InterPro -85.0 (2021-04-08, the last release before GOA 205): it cuts a human subset of -the archived `protein2ipr` selected by the t0 *and* t1 GAFs, runs the nine -recorded rung commands with `--gaf`/`--interpro` added, and evaluates into -`validation/ablation_cells/ipr_manual_t0interpro/`. `validation/ablation.py` +(`protein2ipr_human.dat.gz`, cut from InterPro `current_release` as downloaded +2026-07-07) in training and in the transfer step — temporal look-ahead on the +domain side (TODO.md P0). `scripts/run_t0_interpro_ablation.py` reruns the +`ipr_manual` cell on InterPro 85.0 (2021-04-08, the last release before GOA +205): it cuts a human subset of the archived `protein2ipr` selected by the t0 +*and* t1 GAFs, runs the nine recorded rung commands with `--gaf`/`--interpro` +added, and evaluates into `validation/ablation_cells/ipr_manual_t0interpro/`. +The evaluation keeps the committed cell's cohort (`--cohort-interpro`): a +no-knowledge protein with no InterPro 85.0 domain is scored as a miss, not +dropped, so the cell pays for coverage it lacked in 2021 +(`ablation_selection_counts.tsv` counts those proteins). `validation/ablation.py` refuses to score a rung whose manifest records a different `protein2ipr` than -its `--interpro`. Results to follow. +its `--interpro`, or a different GAF than its `--t0-gaf`. + +The rerun is domain- and annotation-dated, **not** free of look-ahead. Seven of +the nine rungs (all but `single` and `supra`) read `go-basic.obo` 2026-06-15 +during training — for input or output propagation and the parental +background — and the input-propagation rungs remap t0 terms obsoleted since +2021 to their 2026 successors (74 alt_id and 417 replaced_by terms in the +committed cell). The residual favours the hierarchy stages the ablation +compares, so the results must not be called prospective. Results to follow. ## 5. Decisions to settle before writing the paper diff --git a/config/settings.py b/config/settings.py index 4f5f21d..92d0f55 100644 --- a/config/settings.py +++ b/config/settings.py @@ -727,7 +727,7 @@ class Config: # The last InterPro release before GOA release 205 (2021-04-21), # the temporal benchmark's t0, so training and transfer can use # domain architectures of the same date as the annotations - # (VALIDATION_PLAN §4, scripts/run_t0_interpro_ablation.sh). + # (VALIDATION_PLAN §4, scripts/run_t0_interpro_ablation.py). # Opt-in: scripts/download_data.py --group interpro-85. "interpro_85_protein2ipr": DataSource( name="interpro_85_protein2ipr", diff --git a/extract_human_interpro.py b/extract_human_interpro.py index 463b212..e8f9c1b 100644 --- a/extract_human_interpro.py +++ b/extract_human_interpro.py @@ -205,6 +205,7 @@ def main(): selection_rule="goa", selection_sources=goa_files, interpro_source=interpro_file, + evidence_filter=args.evidence_filter, n_accessions=n_accessions, n_matched_lines=n_matched, tool="extract_human_interpro.py", diff --git a/scratch_allspecies/13_heldout_allspecies.sh b/scratch_allspecies/13_heldout_allspecies.sh index c8efbe6..4cdd4ac 100755 --- a/scratch_allspecies/13_heldout_allspecies.sh +++ b/scratch_allspecies/13_heldout_allspecies.sh @@ -9,6 +9,8 @@ # # --interpro stays the human domain file: it defines which proteins the # predictions are transferred onto for scoring, and the evaluation set is human. +# Training read the all-species protein2ipr and GAF, so the run manifest names +# other files than --interpro/--t0-gaf on purpose: --allow-input-mismatch. set -euo pipefail # Wait on the artefact 11_build_t0.sh actually produces, not on a log of it. @@ -28,6 +30,7 @@ uv run python validation/temporal_benchmark.py \ --t1-gaf data/raw/goa_annotations/goa_human.gaf.gz \ --predictions results_allspecies_t0_2021/domain_go_associations_significant.tsv \ --interpro data/interim/protein2ipr_human.dat.gz \ + --allow-input-mismatch \ --disable-supra-domains --n-permutations 20 \ --output-dir validation/heldout_allspecies_single \ > scratch_allspecies/bench_allspecies.log 2>&1 diff --git a/scratch_allspecies/14_heldout_experimental.sh b/scratch_allspecies/14_heldout_experimental.sh index ec5cdac..f4de5c9 100755 --- a/scratch_allspecies/14_heldout_experimental.sh +++ b/scratch_allspecies/14_heldout_experimental.sh @@ -25,8 +25,10 @@ run () { echo " done: $out" } +# Any further arguments go to temporal_benchmark.py. bench () { local preds=$1 out=$2 + shift 2 echo "=== benchmark: $out ===" uv run python validation/temporal_benchmark.py \ --t0-gaf data/raw/goa_annotations/goa_human_t0_2021.gaf.gz \ @@ -34,7 +36,7 @@ bench () { --predictions "$preds" \ --interpro data/interim/protein2ipr_human.dat.gz \ --disable-supra-domains --n-permutations 20 \ - --output-dir "$out" \ + --output-dir "$out" "$@" \ > "scratch_allspecies/bench_$(basename "$out").log" 2>&1 echo " done: $out" } @@ -44,8 +46,9 @@ run allspecies_t0_2021 results_allspecies_t0_exp bench results_human_t0_exp/domain_go_associations_significant.tsv \ validation/heldout_human_single_exp +# Trained on the all-species protein2ipr and GAF on purpose (see 13). bench results_allspecies_t0_exp/domain_go_associations_significant.tsv \ - validation/heldout_allspecies_single_exp + validation/heldout_allspecies_single_exp --allow-input-mismatch echo "EXPERIMENTAL HELD-OUT COMPLETE" uv run python scratch_allspecies/12_compare_heldout.py \ diff --git a/scripts/download_data.py b/scripts/download_data.py index 54d7306..c463c2e 100644 --- a/scripts/download_data.py +++ b/scripts/download_data.py @@ -140,7 +140,7 @@ "zebrafish_idmapping", ], # InterPro 85.0 (2021-04-08), the t0 domain architectures of the temporal - # benchmark (scripts/run_t0_interpro_ablation.sh). ~15 GB; the protein2ipr + # benchmark (scripts/run_t0_interpro_ablation.py). ~15 GB; the protein2ipr # is checked against the release's published md5. "interpro-85": [ "interpro_85_protein2ipr", diff --git a/scripts/run_t0_interpro_ablation.py b/scripts/run_t0_interpro_ablation.py index 822dc98..c34d497 100644 --- a/scripts/run_t0_interpro_ablation.py +++ b/scripts/run_t0_interpro_ablation.py @@ -11,20 +11,26 @@ protein and every held-out protein the evaluator transfers to has t0 architectures. (The current subset was selected by the t1 GAF's non-IEA proteins alone, a strict subset of this.) Skipped when the subset exists - and its provenance marker names the same source. + and its provenance marker records this exact extraction: the same source, + GAFs and evidence filter. 2. **Train** the nine rungs: exactly the commands recorded in ``validation/ablation_manifests/.json``, with ``--output-dir`` moved under ``--run-dir`` and ``--gaf``/``--interpro`` added. 3. **Evaluate** with ``validation/ablation.py`` under the committed evaluation's arguments (all defaults besides the GAFs and run dir), plus - ``--interpro`` set to the subset. The rung manifests are copied to - ``/manifests/``, as for the other cells. + ``--interpro`` set to the subset and ``--cohort-interpro`` to the committed + cell's file: every arm is scored on the committed cohort, and a cohort + protein with no domain in the subset counts as a miss instead of leaving + the denominator. The rung manifests are copied to ``/manifests/``, + as for the other cells, with the subset's provenance sidecar (the archive, + GAFs and evidence filter it was cut with). Run it from the repository root; paths are relative to it, and ``data/`` must hold the inputs. The md5 of the archived protein2ipr is verified first (``scripts/download_data.py --group interpro-85``). Resumable: a rung whose -manifest says ``completed`` is not rerun — safe, because the evaluator refuses -a rung trained on a different protein2ipr than ``--subset``. +manifest says ``completed`` is not rerun, unless the subset is being cut again +— and the evaluator refuses a rung trained on a different protein2ipr than +``--subset`` regardless. Dry run of the whole chain without the archive: pass the current subset as ``--source`` (with scratch ``--subset``/``--run-dir``/``--eval-dir``). A wider @@ -48,12 +54,16 @@ sys.path.insert(0, str(REPO)) from src.run_manifest import manifest_filename # noqa: E402 -from src.universe_provenance import read_marker # noqa: E402 +from src.universe_provenance import marker_path, read_marker # noqa: E402 MANIFESTS = REPO / "validation" / "ablation_manifests" ARCHIVE = Path("data/raw/interpro_archive/85.0/protein2ipr.dat.gz") T0_GAF = Path("data/raw/goa_archive/goa_human.gaf.205.gz") T1_GAF = Path("data/raw/goa_annotations/goa_human.gaf.gz") +#: Selects the subset: any protein either GAF annotates, under any evidence. +EVIDENCE_FILTER = "all" +#: The committed ipr_manual cell's protein2ipr, whose proteins are the cohort. +COHORT = Path("data/interim/protein2ipr_human.dat.gz") RUNGS = ( "single", "supra", @@ -100,7 +110,13 @@ def build_steps( ) ) marker = read_marker(subset) - if not (subset.exists() and marker and marker.interpro_source == str(source)): + extracting = not ( + subset.exists() + and marker is not None + and (marker.interpro_source, marker.selection_sources, marker.evidence_filter) + == (str(source), (str(T0_GAF), str(T1_GAF)), EVIDENCE_FILTER) + ) + if extracting: steps.append( ( "extract", @@ -114,14 +130,15 @@ def build_steps( "--gaf", str(T1_GAF), "--evidence-filter", - "all", + EVIDENCE_FILTER, "--output", str(subset), ], ) ) for rung in RUNGS: - if not completed(run_dir / rung): + # A new subset makes every finished rung stale. + if extracting or not completed(run_dir / rung): steps.append( (rung, rung_command(MANIFESTS / f"{rung}.json", run_dir / rung, subset)) ) @@ -139,6 +156,8 @@ def build_steps( str(run_dir), "--interpro", str(subset), + "--cohort-interpro", + str(COHORT), "--output-dir", str(eval_dir), ], @@ -193,6 +212,8 @@ def main(argv: list[str] | None = None) -> int: shutil.copy( args.run_dir / rung / "run_manifest_go.json", manifests / f"{rung}.json" ) + sidecar = marker_path(args.subset) + shutil.copy(sidecar, manifests / sidecar.name) print(f"Done. Evaluation tables and rung manifests in {args.eval_dir}") return 0 diff --git a/src/universe_provenance.py b/src/universe_provenance.py index 8a9e53e..316e370 100644 --- a/src/universe_provenance.py +++ b/src/universe_provenance.py @@ -48,6 +48,9 @@ class UniverseProvenance: n_matched_lines: protein2ipr lines written to the extract. tool: the script that wrote it. created: ISO-8601 UTC timestamp. + evidence_filter: the GAF evidence filter that selected the accessions + (``"goa"`` rule only); ``None`` for other rules and for markers + written before it was recorded. """ selection_rule: str @@ -57,6 +60,7 @@ class UniverseProvenance: n_matched_lines: int tool: str created: str + evidence_filter: Optional[str] = None def marker_path(extract_path: Path) -> Path: @@ -94,6 +98,7 @@ def write_marker( n_matched_lines: int, tool: str, created: Optional[str] = None, + evidence_filter: Optional[str] = None, ) -> Path: """Write the sidecar for a freshly written extract; returns its path.""" provenance = UniverseProvenance( @@ -104,6 +109,7 @@ def write_marker( n_matched_lines=n_matched_lines, tool=tool, created=created or datetime.now(timezone.utc).isoformat(timespec="seconds"), + evidence_filter=evidence_filter, ) path = marker_path(extract_path) path.write_text(json.dumps(asdict(provenance), indent=2) + "\n", encoding="utf-8") diff --git a/tests/integration/test_extract_human_interpro.py b/tests/integration/test_extract_human_interpro.py index 721beb5..5f67ae9 100644 --- a/tests/integration/test_extract_human_interpro.py +++ b/tests/integration/test_extract_human_interpro.py @@ -92,6 +92,7 @@ def test_union_of_gafs_selects_from_an_archived_source( marker = json.loads(Path(f"{output}.provenance.json").read_text()) assert marker["selection_sources"] == [str(t0), str(t1)] assert marker["interpro_source"] == str(source) + assert marker["evidence_filter"] == evidence def test_defaults_are_the_species_files(tmp_path: Path) -> None: diff --git a/tests/integration/test_validation_clis.py b/tests/integration/test_validation_clis.py index 4ced946..e038dda 100644 --- a/tests/integration/test_validation_clis.py +++ b/tests/integration/test_validation_clis.py @@ -249,8 +249,10 @@ def t0_benchmark(tmp_path: Path) -> dict[str, Path]: return {"t0": t0, "t1": t1, "interpro": interpro} -def _write_run(directory: Path, interpro_sha256: str) -> None: - """A rung's association tables and the manifest naming its protein2ipr.""" +def _write_run( + directory: Path, interpro_sha256: str, gaf_sha256: str | None = None +) -> None: + """A rung's association tables and the manifest naming its inputs.""" directory.mkdir(parents=True, exist_ok=True) table = pd.DataFrame( [("IPR_A", CHILD, 0.001, 0.01, 0.01)], @@ -262,7 +264,10 @@ def _write_run(directory: Path, interpro_sha256: str) -> None: json.dumps( { "parameters": {"domain_key": "interpro", "evidence_filter": "manual"}, - "inputs": [{"role": "domain_annotations", "sha256": interpro_sha256}], + "inputs": [ + {"role": "domain_annotations", "sha256": interpro_sha256}, + *([{"role": "gaf", "sha256": gaf_sha256}] if gaf_sha256 else []), + ], } ), encoding="utf-8", @@ -335,6 +340,89 @@ def test_temporal_benchmark_refuses_other_architectures( assert not (tmp_path / "metrics").exists() +def test_apply_relative_inference_refuses_other_architectures( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path] +) -> None: + # t0-trained predictions, but --interpro left at another (e.g. today's) file. + run = tmp_path / "run" + _write_run(run, "0" * 64, gaf_sha256=_sha256(t0_benchmark["t0"])) + + result = _run( + "apply_relative_inference.py", + "--predictions", + run / "domain_go_associations_significant.tsv", + "--t0-gaf", + t0_benchmark["t0"], + "--interpro", + t0_benchmark["interpro"], + "--go-ontology", + go_obo, + "--output", + tmp_path / "relative.tsv", + ) + + assert result.returncode == 1 + assert "protein2ipr sha256" in result.stderr + assert not (tmp_path / "relative.tsv").exists() + + +def _benchmark( + tmp_path: Path, go_obo: Path, inputs: dict[str, Path], *extra: object +) -> subprocess.CompletedProcess[str]: + return _run( + "temporal_benchmark.py", + "--t0-gaf", + inputs["t0"], + "--t1-gaf", + inputs["t1"], + "--predictions", + tmp_path / "run" / "domain_go_associations_significant.tsv", + "--interpro", + inputs["interpro"], + "--go-ontology", + go_obo, + "--n-permutations", + 1, + "--min-ic", + 0, + "--output-dir", + tmp_path / "metrics", + *extra, + ) + + +def test_temporal_benchmark_refuses_a_run_trained_on_another_gaf( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path] +) -> None: + # A replayed command without --gaf trains on the current GAF: the t1 labels. + _write_run( + tmp_path / "run", + _sha256(t0_benchmark["interpro"]), + gaf_sha256=_sha256(t0_benchmark["t1"]), + ) + + result = _benchmark(tmp_path, go_obo, t0_benchmark) + + assert result.returncode == 1 + assert "GAF sha256" in result.stderr + assert not (tmp_path / "metrics").exists() + + +def test_temporal_benchmark_scores_a_wider_universe_when_told_to( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path] +) -> None: + # The all-species arm: trained on other protein2ipr and GAF files on + # purpose, transferred onto the human subset and scored on the human GAFs. + _write_run(tmp_path / "run", "0" * 64, gaf_sha256="1" * 64) + + result = _benchmark(tmp_path, go_obo, t0_benchmark, "--allow-input-mismatch") + + assert result.returncode == 0, result.stderr + assert "protein2ipr sha256" in result.stderr # still said, as a warning + assert "GAF sha256" in result.stderr + assert (tmp_path / "metrics" / "temporal_benchmark_metrics.tsv").exists() + + @pytest.mark.parametrize("matching", [True, False]) def test_ablation_scores_only_runs_trained_on_its_protein2ipr( tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path], matching: bool @@ -385,3 +473,75 @@ def test_ablation_scores_only_runs_trained_on_its_protein2ipr( provenance = pd.read_csv(tmp_path / "eval" / "ablation_provenance.tsv", sep="\t") assert set(provenance["interpro_file"]) == {str(t0_benchmark["interpro"])} assert set(provenance["interpro_sha256"]) == {sha256} + assert set(provenance["t0_gaf_sha256"]) == {_sha256(t0_benchmark["t0"])} + assert set(provenance["t1_gaf_sha256"]) == {_sha256(t0_benchmark["t1"])} + + +LADDER_RUNS = ( + "single", + "supra", + "supra_input", + "supra_relative", + "supra_output", + "supra_input_relative", + "supra_input_output", + "supra_relative_output", + "full", +) + + +@pytest.mark.parametrize("fixed_cohort", [True, False]) +def test_ablation_scores_a_fixed_cohort_through_other_architectures( + tmp_path: Path, go_obo: Path, t0_benchmark: dict[str, Path], fixed_cohort: bool +) -> None: + """P2 joins the held-out cohort but has no domain in the t0 subset: with + the cohort fixed by the current file it is a miss, not a dropped protein.""" + t1 = tmp_path / "t1_p2.gaf.gz" + with gzip.open(t1, "wt") as handle: + handle.write("!gaf-version: 2.2\n") + for protein in ("P0", "P1", "P2"): + handle.write(_gaf_row(protein, CHILD) + "\n") + current = tmp_path / "protein2ipr_current.dat.gz" + with gzip.open(current, "wt") as handle: + for protein in ("P0", "P1", "P2"): + handle.write(f"{protein}\tIPR_A\tA\tPF00001\t10\t100\n") + for rung in LADDER_RUNS: + _write_run(tmp_path / "runs" / rung, _sha256(t0_benchmark["interpro"])) + + cohort = ["--cohort-interpro", current] if fixed_cohort else [] + result = _run( + "ablation.py", + "--t0-gaf", + t0_benchmark["t0"], + "--t1-gaf", + t1, + "--run-dir", + tmp_path / "runs", + "--interpro", + t0_benchmark["interpro"], + *cohort, + "--go-ontology", + go_obo, + "--min-ic", + 0, + "--n-bootstrap", + 2, + "--n-permutations", + 0, + "--output-dir", + tmp_path / "eval", + ) + + assert result.returncode == 0, result.stderr + metrics = pd.read_csv(tmp_path / "eval" / "ablation_metrics.tsv", sep="\t") + single = metrics[metrics["method"] == "single"].iloc[0] + counts = pd.read_csv(tmp_path / "eval" / "ablation_selection_counts.tsv", sep="\t") + missing = counts[counts["stage"] == "no_knowledge_without_transfer_architectures"] + if fixed_cohort: + assert single["n_eval_proteins"] == 2 # P1 hit, P2 missed + assert single["f_max"] == pytest.approx(2 / 3) # precision 1, recall 1/2 + assert dict(zip(missing["aspect"], missing["n_proteins"]))["BP"] == 1 + else: + assert single["n_eval_proteins"] == 1 # P2 silently left the cohort + assert single["f_max"] == pytest.approx(1.0) + assert missing.empty diff --git a/tests/unit/test_ablation.py b/tests/unit/test_ablation.py index 23f5c60..251ecea 100644 --- a/tests/unit/test_ablation.py +++ b/tests/unit/test_ablation.py @@ -52,7 +52,14 @@ def _load(name): def _rows(**kwargs): return ab.selection_stage_counts( - T0, T1, ["P1", "P2", "P3", "P4"], BENCH_ALL, BENCH_DOM, IC, [0.0, 2.0, 4.0] + T0, + T1, + ["P1", "P2", "P3", "P4"], + BENCH_ALL, + BENCH_DOM, + IC, + [0.0, 2.0, 4.0], + **kwargs, ) @@ -151,6 +158,21 @@ def test_one_row_per_aspect_and_floor(self): rows = [r for r in _rows() if r["stage"] == "cohort_at_ic_floor"] assert len(rows) == 3 * 3 # BP/MF/CC x three floors + def test_cohort_proteins_without_transfer_architectures_are_counted(self): + # A cohort fixed by another protein2ipr: P4 has no domain in the one + # the transfer step reads, so it is scored with no prediction. + rows = _rows(transfer_proteins=["P1", "P2", "P3"]) + missing = { + r["aspect"]: r["n_proteins"] + for r in rows + if r["stage"] == "no_knowledge_without_transfer_architectures" + } + assert missing == {"BP": 1, "MF": 0, "CC": 0} + + def test_no_transfer_row_when_the_cohort_is_the_transfer_file(self): + stages = {r["stage"] for r in _rows()} + assert "no_knowledge_without_transfer_architectures" not in stages + def test_empty_aspects_are_still_reported_as_zero(self): rows = [ r @@ -185,10 +207,10 @@ def _write(self, root, rung, inputs=(), **params): ) @staticmethod - def _domains(sha256): + def _domains(sha256, gaf_sha256="f" * 64): return [ {"role": "domain_annotations", "sha256": sha256}, - {"role": "gaf", "sha256": "f" * 64}, + {"role": "gaf", "sha256": gaf_sha256}, ] def test_matching_runs_pass(self, tmp_path): @@ -220,18 +242,33 @@ def test_protein2ipr_mismatch_is_reported(self, tmp_path): rung = ab.LADDER[0] self._write(tmp_path, rung, inputs=self._domains("a" * 64)) problems = ab.run_setting_mismatches( - tmp_path, [rung], {}, interpro_sha256="b" * 64 + tmp_path, [rung], {}, input_sha256={"domain_annotations": "b" * 64} ) assert problems == [ f"{rung.name}: run protein2ipr sha256={'a' * 64}, evaluator {'b' * 64}" ] - def test_matching_protein2ipr_passes(self, tmp_path): + def test_training_gaf_mismatch_is_reported(self, tmp_path): + # The bug this guards: a committed command replayed without --gaf reads + # the current GAF at the default path — the held-out labels. + rung = ab.LADDER[0] + self._write(tmp_path, rung, inputs=self._domains("a" * 64, "c" * 64)) + problems = ab.run_setting_mismatches( + tmp_path, + [rung], + {}, + input_sha256={"domain_annotations": "a" * 64, "gaf": "d" * 64}, + ) + assert problems == [ + f"{rung.name}: run GAF sha256={'c' * 64}, evaluator {'d' * 64}" + ] + + def test_matching_inputs_pass(self, tmp_path): for rung in ab.LADDER: self._write(tmp_path, rung, inputs=self._domains("a" * 64)) + given = {"domain_annotations": "a" * 64, "gaf": "f" * 64} assert ( - ab.run_setting_mismatches(tmp_path, ab.LADDER, {}, interpro_sha256="a" * 64) - == [] + ab.run_setting_mismatches(tmp_path, ab.LADDER, {}, input_sha256=given) == [] ) def test_protein2ipr_is_checked_only_when_given(self, tmp_path): @@ -243,6 +280,8 @@ def test_a_manifest_without_domain_annotations_is_not_checked(self, tmp_path): rung = ab.LADDER[0] self._write(tmp_path, rung) assert ( - ab.run_setting_mismatches(tmp_path, [rung], {}, interpro_sha256="b" * 64) + ab.run_setting_mismatches( + tmp_path, [rung], {}, input_sha256={"domain_annotations": "b" * 64} + ) == [] ) diff --git a/tests/unit/test_run_t0_interpro_ablation.py b/tests/unit/test_run_t0_interpro_ablation.py index d2252e8..ed1a3f7 100644 --- a/tests/unit/test_run_t0_interpro_ablation.py +++ b/tests/unit/test_run_t0_interpro_ablation.py @@ -70,57 +70,114 @@ def test_the_archive_is_verified_and_the_subset_extracted(tmp_path): assert extract[extract.index("--evidence-filter") + 1] == "all" evaluate = steps["eval"] assert evaluate[evaluate.index("--interpro") + 1] == str(tmp_path / "subset.dat.gz") + # Every arm is scored on the committed cell's cohort. + assert evaluate[evaluate.index("--cohort-interpro") + 1] == str(runbook.COHORT) -def test_a_subset_already_cut_from_the_same_source_is_reused(tmp_path): +def _cut(subset, source, gafs=(runbook.T0_GAF, runbook.T1_GAF), evidence="all"): + """A subset on disk whose marker records how it was extracted.""" from src.universe_provenance import write_marker - source = tmp_path / "protein2ipr.dat.gz" - subset = tmp_path / "subset.dat.gz" subset.write_bytes(b"") write_marker( subset, selection_rule="goa", - selection_sources=[runbook.T0_GAF, runbook.T1_GAF], + selection_sources=list(gafs), interpro_source=source, n_accessions=0, n_matched_lines=0, tool="extract_human_interpro.py", + evidence_filter=evidence, ) - names = [ - name for name, _ in runbook.build_steps(source, subset, tmp_path, tmp_path) - ] + +def _names(source, subset, run_dir): + return [name for name, _ in runbook.build_steps(source, subset, run_dir, run_dir)] + + +def test_a_subset_cut_by_this_exact_extraction_is_reused(tmp_path): + source = tmp_path / "protein2ipr.dat.gz" + subset = tmp_path / "subset.dat.gz" + _cut(subset, source) + + names = _names(source, subset, tmp_path) assert "verify" not in names # not the archive: nothing to verify assert "extract" not in names - other = [ - name - for name, _ in runbook.build_steps( - tmp_path / "other.dat.gz", subset, tmp_path, tmp_path - ) - ] - assert "extract" in other + assert "extract" in _names(tmp_path / "other.dat.gz", subset, tmp_path) + + +@pytest.mark.parametrize( + "gafs,evidence", + [ + ((runbook.T0_GAF,), "all"), # the t1-only no-knowledge proteins missing + ((runbook.T0_GAF, runbook.T1_GAF), "manual"), # the extract default + ((runbook.T0_GAF, runbook.T1_GAF), None), # marker predates the field + ], +) +def test_a_subset_selected_differently_is_cut_again(tmp_path, gafs, evidence): + source = tmp_path / "protein2ipr.dat.gz" + subset = tmp_path / "subset.dat.gz" + _cut(subset, source, gafs, evidence) + + assert "extract" in _names(source, subset, tmp_path) + + +def _finish(run_dir, rung, status="completed"): + (run_dir / rung).mkdir() + (run_dir / rung / "run_manifest_go.json").write_text(json.dumps({"status": status})) def test_completed_rungs_are_not_rerun(tmp_path): - (tmp_path / "single").mkdir() - (tmp_path / "single" / "run_manifest_go.json").write_text('{"status": "completed"}') - (tmp_path / "supra").mkdir() # interrupted: the manifest never finalised - (tmp_path / "supra" / "run_manifest_go.json").write_text('{"status": "running"}') - - names = [ - name - for name, _ in runbook.build_steps( - tmp_path / "src.dat.gz", tmp_path / "subset.dat.gz", tmp_path, tmp_path - ) - ] + source = tmp_path / "src.dat.gz" + subset = tmp_path / "subset.dat.gz" + _cut(subset, source) + _finish(tmp_path, "single") + _finish(tmp_path, "supra", "running") # interrupted: never finalised + + names = _names(source, subset, tmp_path) assert "single" not in names assert names[names.index("supra") :] == [*runbook.RUNGS[1:], "eval"] +def test_a_new_subset_reruns_completed_rungs(tmp_path): + _finish(tmp_path, "single") # trained on whatever subset was there before + + names = _names(tmp_path / "src.dat.gz", tmp_path / "subset.dat.gz", tmp_path) + assert names == ["extract", *runbook.RUNGS, "eval"] + + def test_dry_run_prints_and_runs_nothing(tmp_path, capsys): run_dir = tmp_path / "runs" assert runbook.main(["--dry-run", "--run-dir", str(run_dir)]) == 0 printed = capsys.readouterr().out assert "# full" in printed and "validation/ablation.py" in printed assert not run_dir.exists() + + +def test_the_subset_sidecar_is_kept_with_the_rung_manifests(tmp_path, monkeypatch): + # The subset sits in gitignored data/interim; its sidecar (archive, GAFs, + # evidence filter) must travel with the committed evaluation. + source = tmp_path / "src.dat.gz" + subset = tmp_path / "subset.dat.gz" + _cut(subset, source) + run_dir = tmp_path / "runs" + run_dir.mkdir() + for rung in runbook.RUNGS: + _finish(run_dir, rung) + monkeypatch.setattr( + runbook.subprocess, + "run", + lambda *a, **k: runbook.subprocess.CompletedProcess(a, 0), + ) + + eval_dir = tmp_path / "eval" + argv = ["--source", str(source), "--subset", str(subset)] + argv += ["--run-dir", str(run_dir), "--eval-dir", str(eval_dir)] + assert runbook.main(argv) == 0 + + copied = eval_dir / "manifests" / "subset.dat.gz.provenance.json" + assert json.loads(copied.read_text())["evidence_filter"] == "all" + assert {p.name for p in (eval_dir / "manifests").glob("*.json")} == { + copied.name, + *(f"{rung}.json" for rung in runbook.RUNGS), + } diff --git a/tests/unit/test_temporal_benchmark.py b/tests/unit/test_temporal_benchmark.py index fe06418..8a2eca3 100644 --- a/tests/unit/test_temporal_benchmark.py +++ b/tests/unit/test_temporal_benchmark.py @@ -110,25 +110,43 @@ def test_merged_id_prediction_is_scored_and_propagated(self): assert pred["P1"] == {"GO:leaf": 1.0, "GO:mid": 1.0, "GO:other": 0.0} -class TestRecordedInterproSha256: - def test_reads_the_domain_annotations_input(self, tmp_path): - manifest = tmp_path / "run_manifest_go.json" - manifest.write_text( +class TestTrainingInputMismatches: + @pytest.fixture + def manifest(self, tmp_path): + path = tmp_path / "run_manifest_go.json" + path.write_text( json.dumps( { "inputs": [ {"role": "domain_annotations", "sha256": "a" * 64}, {"role": "gaf", "sha256": "b" * 64}, + {"role": "go_obo", "sha256": "c" * 64}, ] } ) ) - assert tb.recorded_interpro_sha256(manifest) == "a" * 64 + return path + + def test_matching_inputs_pass(self, manifest): + given = {"domain_annotations": "a" * 64, "gaf": "b" * 64} + assert tb.training_input_mismatches(manifest, given) == [] + + def test_each_differing_input_is_named(self, manifest): + given = {"domain_annotations": "x" * 64, "gaf": "y" * 64} + assert tb.training_input_mismatches(manifest, given) == [ + f"run protein2ipr sha256={'a' * 64}, evaluator {'x' * 64}", + f"run GAF sha256={'b' * 64}, evaluator {'y' * 64}", + ] + + def test_only_the_roles_given_are_checked(self, manifest): + # go_obo differs from nothing: the evaluator did not say what it reads. + assert tb.training_input_mismatches(manifest, {"gaf": "b" * 64}) == [] - def test_none_when_the_manifest_records_no_domain_file(self, tmp_path): + def test_a_role_the_manifest_does_not_record_is_not_checked(self, tmp_path): manifest = tmp_path / "run_manifest_go.json" manifest.write_text(json.dumps({"inputs": [{"role": "gaf"}]})) - assert tb.recorded_interpro_sha256(manifest) is None + given = {"domain_annotations": "x" * 64, "gaf": "y" * 64} + assert tb.training_input_mismatches(manifest, given) == [] class TestBuildNKBenchmark: diff --git a/tests/unit/test_universe_provenance.py b/tests/unit/test_universe_provenance.py index 1d6f33f..c638d34 100644 --- a/tests/unit/test_universe_provenance.py +++ b/tests/unit/test_universe_provenance.py @@ -44,6 +44,26 @@ def test_written_marker_reads_back(self, extract): assert marker.n_matched_lines == 4200 assert marker.created == "2026-08-17T00:00:00+00:00" + def test_evidence_filter_reads_back(self, extract): + write_marker( + extract, + selection_rule="goa", + selection_sources=["t0.gaf.gz", "t1.gaf.gz"], + interpro_source="protein2ipr.dat.gz", + n_accessions=2, + n_matched_lines=2, + tool="extract_human_interpro.py", + evidence_filter="all", + ) + assert read_marker(extract).evidence_filter == "all" + + def test_a_marker_from_before_the_evidence_filter_reads_as_unknown(self, extract): + write_goa_marker(extract) + payload = json.loads(marker_path(extract).read_text()) + del payload["evidence_filter"] + marker_path(extract).write_text(json.dumps(payload)) + assert read_marker(extract).evidence_filter is None + def test_missing_marker_reads_as_none(self, extract): assert read_marker(extract) is None diff --git a/validation/ablation.py b/validation/ablation.py index 06f47d4..998c4b5 100644 --- a/validation/ablation.py +++ b/validation/ablation.py @@ -47,11 +47,17 @@ Each rung is one ``run_dcgo_human.py --gaf --output-dir /`` plus its stage flags (``validation/ablation_manifests/*.json`` record the exact -commands). ``scripts/run_t0_interpro_ablation.sh`` runs a whole cell — the nine +commands). ``scripts/run_t0_interpro_ablation.py`` runs a whole cell — the nine rungs and this evaluation — with ``--interpro`` set to t0 domain architectures. -``--interpro`` here must be the protein2ipr file the rungs were trained on, and -``--domain-key``/``--evidence-filter`` their settings: the driver refuses to -score a run whose manifest says otherwise. +``--interpro`` here must be the protein2ipr file the rungs were trained on, +``--t0-gaf`` their GAF and ``--domain-key``/``--evidence-filter`` their +settings: the driver refuses to score a run whose manifest says otherwise. + +The scored cohort is the no-knowledge proteins with a domain in +``--cohort-interpro`` (default: ``--interpro``). Fixing it to one file lets +cells trained on different architectures be scored on the same proteins: a +cohort protein with no domain in ``--interpro`` gets no dcGO prediction and +counts as a miss, rather than leaving the denominator. Honest caveat, stated once here and again in ``VALIDATION_PLAN.md``: the True Path rungs are scored on ``q_value`` because the propagated output carries no @@ -201,6 +207,7 @@ def selection_stage_counts( benchmark_with_domains: Mapping[str, Mapping[str, set]], ic: Mapping[str, float], ic_floors: Sequence[float], + transfer_proteins: Iterable[str] | None = None, ) -> list[dict]: """Every filter between "the input files" and "the scored cohort", counted. @@ -214,7 +221,10 @@ def selection_stage_counts( ``benchmark_all`` is the no-knowledge benchmark built **without** the has-a-domain restriction; ``benchmark_with_domains`` is the one actually - scored. Both are ``{aspect: {protein: truth}}``. + scored. Both are ``{aspect: {protein: truth}}``. ``transfer_proteins``, + given when the cohort comes from another protein2ipr than the transfer + step, adds how many cohort proteins have no architecture to transfer + through (scored as misses). """ rows: list[dict] = [ { @@ -269,6 +279,22 @@ def selection_stage_counts( "note": "the scored cohort before any IC floor", } ) + if transfer_proteins is not None: + transfer = set(transfer_proteins) + rows.append( + { + "stage": "no_knowledge_without_transfer_architectures", + "aspect": aspect, + "min_ic": "-", + "n_proteins": len( + set(benchmark_with_domains.get(aspect, {})) - transfer + ), + "n_dropped_vs_ic0": "-", + "pct_of_ic0": "-", + "note": "cohort proteins with no domain in --interpro: no " + "dcGO prediction, scored as misses", + } + ) for aspect in ("BP", "MF", "CC"): base = set(benchmark_with_domains.get(aspect, {})) for min_ic in ic_floors: @@ -315,16 +341,16 @@ def run_setting_mismatches( run_root: Path, rungs: Iterable[Rung], expected: Mapping[str, str], - interpro_sha256: str | None = None, + input_sha256: Mapping[str, str] | None = None, ) -> list[str]: """Rungs whose run manifest disagrees with the evaluator's settings. The no-knowledge cohort and IC are built from what training saw, so the evaluator's ``domain_key``/``evidence_filter`` must be the ones each run - used. Likewise the transfer step must read the domain architectures the - run was trained on: ``interpro_sha256``, if given, is compared with the - run's recorded protein2ipr (see ``tb.recorded_interpro_sha256``). Rungs - without a manifest are not checked. + used. Likewise the evaluator must read the inputs the run was trained on: + ``input_sha256``, if given, maps manifest input roles (the protein2ipr and + the t0 GAF) to the evaluator's digests — see + ``tb.training_input_mismatches``. Rungs without a manifest are not checked. """ problems = [] for rung in rungs: @@ -336,13 +362,11 @@ def run_setting_mismatches( got = params.get(key) if got is not None and got != want: problems.append(f"{rung.name}: run {key}={got}, evaluator {want}") - if interpro_sha256 is not None: - trained_on = tb.recorded_interpro_sha256(manifest) - if trained_on is not None and trained_on != interpro_sha256: - problems.append( - f"{rung.name}: run protein2ipr sha256={trained_on}, " - f"evaluator {interpro_sha256}" - ) + if input_sha256: + problems.extend( + f"{rung.name}: {problem}" + for problem in tb.training_input_mismatches(manifest, input_sha256) + ) return problems @@ -366,7 +390,11 @@ class BenchmarkInputs: replaced_by_map: dict[str, str] t0_map: dict[str, set[str]] t1_exp_map: dict[str, set[str]] + #: protein -> features in ``--interpro`` (the transfer step) protein_domains: dict[str, list[str]] + #: proteins with a feature in the cohort protein2ipr (``--cohort-interpro``, + #: else ``--interpro``): the has-a-domain restriction on the cohort + cohort_proteins: set[str] #: ``{aspect: {protein: truth}}`` without / with the has-a-domain restriction benchmark_all: dict[str, dict[str, set[str]]] benchmark: dict[str, dict[str, set[str]]] @@ -383,8 +411,14 @@ def load_benchmark_inputs( go_ontology: Path, domain_key: str, evidence_filter: str, + cohort_interpro: Path | None = None, ) -> BenchmarkInputs: # pragma: no cover - I/O wiring - """Parse the GAFs, ontology and architectures into the scored benchmark.""" + """Parse the GAFs, ontology and architectures into the scored benchmark. + + The cohort is the no-knowledge proteins with a feature in + ``cohort_interpro`` (default: ``interpro``); a cohort protein with none in + ``interpro`` gets no dcGO prediction and is scored as a miss. + """ from loguru import logger if str(_ROOT) not in sys.path: @@ -413,18 +447,24 @@ def load_benchmark_inputs( t1_exp_map, processor.alt_id_map, processor.replaced_by_map ) + def feature_map(protein2ipr: Path) -> dict[str, list[str]]: + architectures = DomainAnnotationParser( + max_supra_domain_length=3, + min_domain_length=10, + domain_key=domain_key, + ).parse_protein2ipr_file(protein2ipr) + out: dict[str, list[str]] = {} + for protein, arch in architectures.items(): + domains = list(arch.single_domains) + list(arch.supra_domains) + if domains: + out[protein] = domains + return out + logger.info("Parsing domain architectures...") - dom_parser = DomainAnnotationParser( - max_supra_domain_length=3, - min_domain_length=10, - domain_key=domain_key, + protein_domains = feature_map(interpro) + cohort_proteins = ( + set(feature_map(cohort_interpro)) if cohort_interpro else set(protein_domains) ) - architectures = dom_parser.parse_protein2ipr_file(interpro) - protein_domains: dict[str, list[str]] = {} - for protein, arch in architectures.items(): - domains = list(arch.single_domains) + list(arch.supra_domains) - if domains: - protein_domains[protein] = domains # One cohort for every rung. Supra-domains are built out of single domains, # so "has >=1 feature" is the same set either way — which is what makes the @@ -438,7 +478,7 @@ def load_benchmark_inputs( t1_exp_map, term_aspect, get_ancestors, - predictable_proteins=set(protein_domains), + predictable_proteins=cohort_proteins, ) eval_proteins = {p for aspect in benchmark.values() for p in aspect} for aspect, truths in benchmark.items(): @@ -461,6 +501,7 @@ def load_benchmark_inputs( t0_map=t0_map, t1_exp_map=t1_exp_map, protein_domains=protein_domains, + cohort_proteins=cohort_proteins, benchmark_all=benchmark_all, benchmark=benchmark, eval_proteins=eval_proteins, @@ -491,7 +532,9 @@ def build_methods( rung's predictions are missing. """ transfer = TRANSFERS[transfer_name] - eval_domains = {p: inputs.protein_domains[p] for p in inputs.eval_proteins} + # A cohort protein with no architecture to transfer through gets no dcGO + # prediction (only possible with a separate cohort protein2ipr). + eval_domains = {p: inputs.protein_domains.get(p, []) for p in inputs.eval_proteins} def on_scoring_vocabulary(scores: dict) -> dict: return tb.remap_retired_scores( @@ -625,10 +668,19 @@ def main() -> int: # pragma: no cover - I/O wiring "--interpro", type=Path, default=Path("data/interim/protein2ipr_human.dat.gz"), - help="protein2ipr file for the transfer step and the cohort — must be " - "the one the runs under --run-dir were trained on " + help="protein2ipr file for the transfer step — must be the one the " + "runs under --run-dir were trained on " "(default: data/interim/protein2ipr_human.dat.gz)", ) + parser.add_argument( + "--cohort-interpro", + type=Path, + default=None, + help="protein2ipr file whose proteins define the scored cohort " + "(default: --interpro). Pass one file to score cells trained on " + "different architectures on the same proteins; a cohort protein with " + "no domain in --interpro is scored with no dcGO predictions", + ) parser.add_argument( "--domain-key", choices=["interpro", "ssf"], @@ -684,16 +736,36 @@ def main() -> int: # pragma: no cover - I/O wiring ic_floors = sorted(set(args.min_ic)) if args.min_ic else [0.0, 2.0, 4.0] args.output_dir.mkdir(parents=True, exist_ok=True) - for path in (args.t0_gaf, args.t1_gaf, args.interpro, args.go_ontology): + cohort_interpro = args.cohort_interpro or args.interpro + for path in ( + args.t0_gaf, + args.t1_gaf, + args.interpro, + cohort_interpro, + args.go_ontology, + ): if not path.exists(): logger.error(f"Missing required input: {path}") return 1 - interpro_sha256 = sha256_file(args.interpro) + # Every method is scored with these inputs; recorded per provenance row. + inputs_used = { + "interpro_file": str(args.interpro), + "interpro_sha256": sha256_file(args.interpro), + "cohort_interpro_file": str(cohort_interpro), + "cohort_interpro_sha256": sha256_file(cohort_interpro), + "t0_gaf_file": str(args.t0_gaf), + "t0_gaf_sha256": sha256_file(args.t0_gaf), + "t1_gaf_file": str(args.t1_gaf), + "t1_gaf_sha256": sha256_file(args.t1_gaf), + } mismatches = run_setting_mismatches( args.run_dir, LADDER, {"domain_key": args.domain_key, "evidence_filter": args.evidence_filter}, - interpro_sha256=interpro_sha256, + input_sha256={ + "domain_annotations": inputs_used["interpro_sha256"], + "gaf": inputs_used["t0_gaf_sha256"], + }, ) if mismatches: for problem in mismatches: @@ -708,6 +780,7 @@ def main() -> int: # pragma: no cover - I/O wiring args.go_ontology, args.domain_key, args.evidence_filter, + cohort_interpro=args.cohort_interpro, ) benchmark, ic, term_aspect = inputs.benchmark, inputs.ic, inputs.term_aspect if not inputs.eval_proteins: @@ -717,11 +790,12 @@ def main() -> int: # pragma: no cover - I/O wiring counts = selection_stage_counts( inputs.t0_map, inputs.t1_exp_map, - inputs.protein_domains, + inputs.cohort_proteins, inputs.benchmark_all, benchmark, ic, ic_floors, + transfer_proteins=set(inputs.protein_domains) if args.cohort_interpro else None, ) counts_file = args.output_dir / "ablation_selection_counts.tsv" pd.DataFrame(counts).to_csv(counts_file, sep="\t", index=False) @@ -735,12 +809,7 @@ def main() -> int: # pragma: no cover - I/O wiring except FileNotFoundError as missing: logger.error(str(missing)) return 1 - # Every method is scored on the cohort and architectures of this file. - architectures_used = { - "interpro_file": str(args.interpro), - "interpro_sha256": interpro_sha256, - } - provenance = [{**row, **architectures_used} for row in provenance] + provenance = [{**row, **inputs_used} for row in provenance] pd.DataFrame(provenance).to_csv( args.output_dir / "ablation_provenance.tsv", sep="\t", index=False ) @@ -851,7 +920,9 @@ def main() -> int: # pragma: no cover - I/O wiring ) base_scores = rung_scores[null_rung] transfer = TRANSFERS[args.transfer] - eval_domains = {p: inputs.protein_domains[p] for p in inputs.eval_proteins} + eval_domains = { + p: inputs.protein_domains.get(p, []) for p in inputs.eval_proteins + } null_samples: dict[tuple[str, float, str], list[float]] = defaultdict(list) for i in range(args.n_permutations): seed = args.seed * 1_000_003 + i diff --git a/validation/apply_relative_inference.py b/validation/apply_relative_inference.py index fe87ba7..591c68f 100644 --- a/validation/apply_relative_inference.py +++ b/validation/apply_relative_inference.py @@ -48,7 +48,9 @@ from src.domain_annotation_parser import DomainAnnotationParser # noqa: E402 from src.goa_parser import parse_goa_human # noqa: E402 from src.ontology_processor import OntologyProcessor # noqa: E402 +from src.run_manifest import sha256_file # noqa: E402 from validation.association_io import load_associations # noqa: E402 +from validation.temporal_benchmark import training_input_mismatches # noqa: E402 def build_protein_domain_map( @@ -71,7 +73,10 @@ def main() -> int: ap.add_argument("--predictions", type=Path, required=True) ap.add_argument("--t0-gaf", type=Path, required=True) ap.add_argument( - "--interpro", type=Path, default=Path("data/interim/protein2ipr_human.dat.gz") + "--interpro", + type=Path, + default=Path("data/interim/protein2ipr_human.dat.gz"), + help="protein2ipr file — must be the one the predictions were trained on", ) ap.add_argument( "--go-ontology", type=Path, default=Path("data/raw/go_ontology/go-basic.obo") @@ -85,6 +90,22 @@ def main() -> int: ) args = ap.parse_args() + # The backgrounds must be built from the inputs training read (see + # training_input_mismatches); predictions without a manifest are not checked. + manifest = args.predictions.parent / "run_manifest_go.json" + if manifest.exists(): + mismatches = training_input_mismatches( + manifest, + { + "domain_annotations": sha256_file(args.interpro), + "gaf": sha256_file(args.t0_gaf), + }, + ) + for problem in mismatches: + logger.error(f"Settings mismatch — {manifest}: {problem}") + if mismatches: + return 1 + logger.info("Loading GO ontology...") processor = OntologyProcessor(args.go_ontology) diff --git a/validation/temporal_benchmark.py b/validation/temporal_benchmark.py index bb27e8a..ab9bc48 100644 --- a/validation/temporal_benchmark.py +++ b/validation/temporal_benchmark.py @@ -647,21 +647,38 @@ def _load_resampling(): # pragma: no cover - import plumbing return module -def recorded_interpro_sha256(manifest_path: Path) -> str | None: - """SHA-256 of the protein2ipr file a run manifest says training read. - - ``None`` when the manifest records no ``domain_annotations`` input. The - evaluator must transfer through the same architectures training learned - from: a different file is a different domain vocabulary (supra-domain ids - are built from it) and, for a temporal benchmark, a different date. The - digest is of the file's bytes, so a re-compressed copy of identical rows - also counts as different — pass the file the runs were made with. +#: Run-manifest input roles an evaluator reads too, and how to name them. +TRAINING_INPUT_LABELS = {"domain_annotations": "protein2ipr", "gaf": "GAF"} + + +def training_input_mismatches( + manifest_path: Path, evaluator_sha256: Mapping[str, str] +) -> list[str]: + """Training inputs a run manifest records with other bytes than the evaluator's. + + ``evaluator_sha256`` maps a manifest input role to the SHA-256 of the file + the evaluator reads for it: + + * ``domain_annotations`` (``--interpro``) — the transfer step must read the + architectures training learned from: a different file is a different + domain vocabulary (supra-domain ids are built from it) and, for a + temporal benchmark, a different date. + * ``gaf`` (``--t0-gaf``) — the cohort, IC and naive baseline must come from + the snapshot training saw. A run trained on a later GAF has seen the + held-out labels. + + A role the manifest does not record is not checked. The digest is of the + file's bytes, so a re-compressed copy of identical rows also counts as + different — pass the files the runs were made with. """ data = json.loads(Path(manifest_path).read_text(encoding="utf-8")) - for record in data.get("inputs", []): - if record.get("role") == "domain_annotations": - return record.get("sha256") - return None + recorded = {r.get("role"): r.get("sha256") for r in data.get("inputs", [])} + return [ + f"run {TRAINING_INPUT_LABELS.get(role, role)} sha256={recorded[role]}, " + f"evaluator {sha256}" + for role, sha256 in evaluator_sha256.items() + if recorded.get(role) is not None and recorded[role] != sha256 + ] def build_term_aspect(processor) -> dict[str, str]: @@ -716,6 +733,14 @@ def main() -> int: help="protein2ipr human subset (domain architectures) — must be the " "file the predictions were trained on", ) + parser.add_argument( + "--allow-input-mismatch", + action="store_true", + help="Score predictions whose run manifest records another protein2ipr " + "or GAF than --interpro/--t0-gaf, with a warning instead of an error. " + "For a deliberately wider training universe transferred onto the human " + "subset (scratch_allspecies/13, 14)", + ) parser.add_argument( "--go-ontology", type=Path, @@ -787,17 +812,21 @@ def main() -> int: logger.error(f"Missing required input: {p}") return 1 - # Transfer through the architectures training learned from (see - # recorded_interpro_sha256). Predictions without a manifest are not checked. + # Score through the architectures and t0 snapshot training read (see + # training_input_mismatches). Predictions without a manifest are not checked. manifest = args.predictions.parent / "run_manifest_go.json" if manifest.exists(): - trained_on = recorded_interpro_sha256(manifest) - given = sha256_file(args.interpro) - if trained_on is not None and trained_on != given: - logger.error( - f"Settings mismatch — {manifest} records protein2ipr sha256 " - f"{trained_on}, --interpro {args.interpro} is {given}" - ) + mismatches = training_input_mismatches( + manifest, + { + "domain_annotations": sha256_file(args.interpro), + "gaf": sha256_file(args.t0_gaf), + }, + ) + log = logger.warning if args.allow_input_mismatch else logger.error + for problem in mismatches: + log(f"Settings mismatch — {manifest}: {problem}") + if mismatches and not args.allow_input_mismatch: return 1 args.output_dir.mkdir(parents=True, exist_ok=True) diff --git a/validation/temporal_breadth.py b/validation/temporal_breadth.py index 670d77b..ac660d7 100644 --- a/validation/temporal_breadth.py +++ b/validation/temporal_breadth.py @@ -197,7 +197,9 @@ def main() -> int: # pragma: no cover - I/O wiring from src.domain_annotation_parser import DomainAnnotationParser from src.ontology_registry import get_ontology + from src.run_manifest import manifest_filename, sha256_file from src.uniprot_annotation_source import parse_uniprot_accessions + from validation.temporal_benchmark import training_input_mismatches logger.remove() logger.add(sys.stderr, level="INFO") @@ -327,6 +329,18 @@ def main() -> int: # pragma: no cover - I/O wiring # Before the architecture pass: every one of these checks is cheap, and # every one of them used to fail only after that pass had spent its time. problems = snapshot_problems(args.ontologies, paths_t0, paths_t1, args.propagate) + # Carriers come from --interpro: it must be what each t0 run was trained on + # (see training_input_mismatches). Runs without a manifest are not checked. + interpro_sha256 = sha256_file(args.interpro) + for ontology in args.ontologies: + manifest = args.t0_results / manifest_filename(ontology) + if manifest.exists(): + problems += [ + f"Settings mismatch — {manifest}: {problem}" + for problem in training_input_mismatches( + manifest, {"domain_annotations": interpro_sha256} + ) + ] if problems: for problem in problems: logger.error(problem) diff --git a/validation/temporal_surprise.py b/validation/temporal_surprise.py index aaedb61..0d0a5c9 100644 --- a/validation/temporal_surprise.py +++ b/validation/temporal_surprise.py @@ -1288,6 +1288,8 @@ def main() -> int: # pragma: no cover - I/O wiring, exercised by running it from src.domain_annotation_parser import DomainAnnotationParser from src.goa_parser import EXPERIMENTAL_EVIDENCE, GOAParser, parse_goa_human from src.ontology_processor import OntologyProcessor + from src.run_manifest import sha256_file + from validation.temporal_benchmark import training_input_mismatches logger.remove() logger.add(sys.stderr, level="INFO") @@ -1382,6 +1384,21 @@ def main() -> int: # pragma: no cover - I/O wiring, exercised by running it if not path.exists(): logger.error(f"Missing required input: {path}") return 1 + # Score through the inputs training read (see + # training_input_mismatches); a ranking without a manifest is not checked. + manifest = args.surprising.parent / "run_manifest_go.json" + if manifest.exists(): + mismatches = training_input_mismatches( + manifest, + { + "domain_annotations": sha256_file(args.interpro), + "gaf": sha256_file(args.t0_gaf), + }, + ) + for problem in mismatches: + logger.error(f"Settings mismatch — {manifest}: {problem}") + if mismatches: + return 1 if args.replay: logger.info(f"Replaying counted outcomes from {args.replay}...") From 344c806463600d17128405bde34d969f9ab00ff2 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 7 Oct 2026 13:37:13 +0100 Subject: [PATCH 4/7] Pair the section 4 cell across InterPro releases on one cohort validation/compare_t0_interpro.py scores three arms of the ipr_manual cell on the committed cohort: the committed cell (current protein2ipr, t1 non-IEA selection), a control (current protein2ipr, t0+t1 selection) and InterPro 85.0 with the same selection. Per aspect x IC floor it runs one resampling.paired_bootstrap over every arm x method panel, on the resamples the per-arm cells used (ablation.bootstrap_seed), and writes control - baseline (selection), t0 - control (InterPro release) and t0 - baseline for the report's four configurations and naive. It writes nothing unless every arm reproduces its committed ablation_metrics.tsv, the arms share one cohort and IC table, and the naive baseline is identical across them. Runs trained on another protein2ipr or GAF are refused as in ablation.py. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/unit/test_compare_t0_interpro.py | 80 ++++++ validation/compare_t0_interpro.py | 324 +++++++++++++++++++++++++ 2 files changed, 404 insertions(+) create mode 100644 tests/unit/test_compare_t0_interpro.py create mode 100644 validation/compare_t0_interpro.py diff --git a/tests/unit/test_compare_t0_interpro.py b/tests/unit/test_compare_t0_interpro.py new file mode 100644 index 0000000..51d2c34 --- /dev/null +++ b/tests/unit/test_compare_t0_interpro.py @@ -0,0 +1,80 @@ +"""Unit tests for the cross-arm comparison of the §4 cell (VALIDATION_PLAN §4). + +The arms are only comparable as a paired difference if every arm x method is +bootstrapped on the same resamples of the same proteins, so that is what these +pin: an arm against itself differs by exactly zero in every replicate, a real +difference keeps its observed sign, and arms scored on different cohorts are +refused rather than compared. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_ROOT = Path(__file__).resolve().parents[2] + + +def _load(name): + spec = importlib.util.spec_from_file_location( + name, _ROOT / "validation" / f"{name}.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +tb = _load("temporal_benchmark") +rs = _load("resampling") +ct = _load("compare_t0_interpro") + +TRUTH = {"P1": {"GO:a", "GO:b"}, "P2": {"GO:b", "GO:c"}, "P3": {"GO:a"}} +IC = {"GO:a": 1.0, "GO:b": 2.0, "GO:c": 3.0, "GO:x": 5.0} +GOOD = { + "P1": {"GO:a": 1.0, "GO:b": 0.8}, + "P2": {"GO:b": 0.9, "GO:c": 0.7}, + "P3": {"GO:a": 0.9, "GO:x": 0.1}, +} +POOR = {"P1": {"GO:x": 1.0, "GO:a": 0.2}, "P2": {"GO:x": 0.9}, "P3": {}} + + +def _panel(pred, truth=TRUTH): + return rs.build_panel(pred, truth, IC, tb._candidate_thresholds(pred)) + + +def _rows(panels, contrast): + return ct.compare_cell(panels, [contrast], n_replicates=50, seed=3) + + +def test_an_arm_against_itself_differs_by_exactly_zero(): + panels = {"x": {"m": _panel(GOOD)}, "y": {"m": _panel(GOOD)}} + for row in _rows(panels, ("x", "y")): + assert row["observed_diff"] == 0.0 + assert (row["diff_ci_lo"], row["diff_ci_hi"]) == (0.0, 0.0) + assert not row["significant"] + + +def test_rows_name_the_contrast_and_keep_the_observed_values(): + panels = {"t0": {"m": _panel(GOOD)}, "ctrl": {"m": _panel(POOR)}} + rows = _rows(panels, ("t0", "ctrl")) + assert [(r["arm_a"], r["arm_b"], r["method"], r["metric"]) for r in rows] == [ + ("t0", "ctrl", "m", "f_max"), + ("t0", "ctrl", "m", "auprc"), + ] + good, poor = rs.panel_metrics(_panel(GOOD)), rs.panel_metrics(_panel(POOR)) + for row in rows: + assert row["observed_a"] == good[row["metric"]] + assert row["observed_b"] == poor[row["metric"]] + assert row["observed_diff"] > 0 + assert "method_a" not in row + + +def test_arms_scored_on_different_cohorts_are_refused(): + other = {p: TRUTH[p] for p in ("P1", "P2")} + panels = {"x": {"m": _panel(GOOD)}, "y": {"m": _panel(GOOD, other)}} + with pytest.raises(ValueError, match="different cohort"): + _rows(panels, ("x", "y")) diff --git a/validation/compare_t0_interpro.py b/validation/compare_t0_interpro.py new file mode 100644 index 0000000..a7f95b5 --- /dev/null +++ b/validation/compare_t0_interpro.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Pair the §4 ``ipr_manual`` cell across training arms on the same proteins. + +Three arms, each the nine committed rung commands trained on GOA release 205, +differ only in the protein2ipr subset they read (training and transfer): + +* ``baseline`` — the committed cell: current protein2ipr, proteins selected by + the t1 GAF's non-IEA annotations (``protein2ipr_human.dat.gz``). +* ``selctrl`` — current protein2ipr, proteins selected by the t0 and t1 GAFs + under any evidence: the t0 arm's selection on today's architectures. +* ``t0`` — InterPro 85.0 (2021-04-08) with that same selection. + +``selctrl - baseline`` isolates the protein selection, ``t0 - selctrl`` the +InterPro release (the domain-architecture look-ahead) and ``t0 - baseline`` +is the whole move. Every arm is scored on the committed cell's cohort +(``--cohort-interpro``; a cohort protein with no domain in an arm's subset is a +miss), so per aspect x IC floor one ``resampling.paired_bootstrap`` runs over +every arm x method panel, on the resamples the per-arm cells were bootstrapped +on (``ablation.bootstrap_seed``). Nothing is written unless every arm +reproduces its committed ``ablation_metrics.tsv``, the arms share one cohort +and IC, and the naive baseline is identical across them. + +Regeneration (from the repository root):: + + uv run python validation/compare_t0_interpro.py \\ + --t0-gaf data/raw/goa_archive/goa_human.gaf.205.gz \\ + --t1-gaf data/raw/goa_annotations/goa_human.gaf.gz +""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +_HERE = Path(__file__).resolve().parent + + +def _load_sibling(name: str): + """Import a sibling `validation/*.py` file (validation/ is not a package).""" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, _HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +abl = _load_sibling("ablation") +tb = abl.tb +rs = abl.rs + +ASPECTS = ("BP", "MF", "CC") +METRICS = ("f_max", "auprc") +#: the report's four configurations (Base = supra_input) and the naive baseline +METHODS = ("supra_input", "supra_input_relative", "supra_input_output", "full", "naive") +#: arm -> (run dir, protein2ipr it was trained on, committed evaluation dir) +ARMS = { + "baseline": ( + Path("results/ablation-replacedby/ipr_manual"), + Path("data/interim/protein2ipr_human.dat.gz"), + _HERE, + ), + "selctrl": ( + Path("results/ablation-t0interpro/ipr_manual_selctrl"), + Path("data/interim/protein2ipr_human_t0t1sel_current.dat.gz"), + _HERE / "ablation_cells" / "ipr_manual_selctrl", + ), + "t0": ( + Path("results/ablation-t0interpro/ipr_manual"), + Path("data/interim/protein2ipr_human_t0_ipr85.dat.gz"), + _HERE / "ablation_cells" / "ipr_manual_t0interpro", + ), +} +CONTRASTS = (("selctrl", "baseline"), ("t0", "selctrl"), ("t0", "baseline")) + + +def compare_cell( + panels: Mapping[str, Mapping[str, rs.EvaluationPanel]], + contrasts: Sequence[tuple[str, str]], + n_replicates: int, + seed: int, + level: float = 0.95, +) -> list[dict]: + """Paired arm-vs-arm differences of every method, on one set of resamples. + + ``panels`` is ``{arm: {method: panel}}``; every panel must hold the same + cohort in the same order (``resampling.paired_bootstrap`` checks). Returns + one row per contrast x method x metric: ``arm_a - arm_b`` on that method. + """ + flat = { + f"{arm}:{method}": panel + for arm, by_method in panels.items() + for method, panel in by_method.items() + } + observed = {name: rs.panel_metrics(panel) for name, panel in flat.items()} + reps = rs.paired_bootstrap( + flat, metrics=METRICS, n_replicates=n_replicates, seed=seed, level=level + ) + rows = [] + for arm_a, arm_b in contrasts: + for method in panels[arm_a]: + a, b = f"{arm_a}:{method}", f"{arm_b}:{method}" + for metric in METRICS: + summary = rs.summarise_paired( + reps, a, b, metric, observed[a][metric], observed[b][metric], level + ) + del summary["method_a"], summary["method_b"] + rows.append( + {"arm_a": arm_a, "arm_b": arm_b, "method": method, **summary} + ) + return rows + + +# --------------------------------------------------------------------------- # +# One aspect x IC-floor cell (one worker each) # +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class Job: + """Shared state the forked workers inherit.""" + + benchmark: dict[str, dict[str, set[str]]] + ic: dict[str, float] + #: arm -> method -> aspect -> protein -> term -> score + aspect_preds: dict[str, dict[str, dict[str, dict]]] + #: (arm, aspect, min_ic, method) -> the arm's committed metrics row + committed: dict[tuple[str, str, float, str], dict] + n_bootstrap: int + seed: int + ci_level: float + + +_JOB: Job | None = None + + +def _score_cell(task: tuple[str, float]) -> tuple[list[dict], list[str]]: + """Every contrast in one cell; problems if an arm misses its committed row.""" + aspect, min_ic = task + job = _JOB + assert job is not None + problems = [] + panels: dict[str, dict[str, rs.EvaluationPanel]] = {} + for arm, methods in job.aspect_preds.items(): + panels[arm] = {} + for method, preds in methods.items(): + truth_f, pred_f, taus = abl.floor_cell( + job.benchmark[aspect], preds[aspect], job.ic, min_ic + ) + panel = rs.build_panel(pred_f, truth_f, job.ic, taus) + panels[arm][method] = panel + ours = rs.panel_metrics(panel) + committed = job.committed[(arm, aspect, min_ic, method)] + for key in METRICS: + if abs(ours[key] - committed[key]) > 1e-9: + problems.append( + f"[{arm} {aspect} IC>={min_ic:g}] {method}: {key} " + f"{ours[key]!r} != committed {committed[key]!r}" + ) + rows = compare_cell( + panels, + CONTRASTS, + job.n_bootstrap, + abl.bootstrap_seed(job.seed, aspect, min_ic), + job.ci_level, + ) + n = next(iter(panels.values()))[METHODS[0]].n_proteins + return [ + {"aspect": aspect, "min_ic": min_ic, "n_eval_proteins": n, **row} + for row in rows + ], problems + + +# --------------------------------------------------------------------------- # +# Driver # +# --------------------------------------------------------------------------- # +def main() -> int: # pragma: no cover - I/O wiring + import argparse + import multiprocessing as mp + + import pandas as pd + from loguru import logger + + global _JOB + + logger.remove() + logger.add(sys.stderr, level="INFO") + + parser = argparse.ArgumentParser( + description="Paired cross-arm comparison of the §4 ipr_manual cell " + "(current vs InterPro 85.0 domain architectures)." + ) + parser.add_argument("--t0-gaf", type=Path, required=True) + parser.add_argument("--t1-gaf", type=Path, required=True) + parser.add_argument( + "--cohort-interpro", + type=Path, + default=Path("data/interim/protein2ipr_human.dat.gz"), + help="protein2ipr whose proteins define the cohort every arm is scored on", + ) + parser.add_argument( + "--arm", + nargs=4, + action="append", + metavar=("NAME", "RUN_DIR", "INTERPRO", "COMMITTED_DIR"), + help=f"Override one arm ({', '.join(ARMS)}) — e.g. to point at run " + "directories outside the checkout", + ) + parser.add_argument( + "--go-ontology", type=Path, default=Path("data/raw/go_ontology/go-basic.obo") + ) + parser.add_argument( + "--output", + type=Path, + default=_HERE / "ablation_cells" / "t0_interpro_comparison.tsv", + ) + parser.add_argument("--n-bootstrap", type=int, default=1000) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--ci-level", type=float, default=0.95) + parser.add_argument("--workers", type=int, default=9) + args = parser.parse_args() + ic_floors = [0.0, 2.0, 4.0] + + if str(abl._ROOT) not in sys.path: + sys.path.insert(0, str(abl._ROOT)) + from src.run_manifest import sha256_file + + arms = {name: tuple(spec) for name, spec in ARMS.items()} + for name, *paths in args.arm or []: + if name not in arms: + parser.error(f"--arm {name}: not one of {', '.join(ARMS)}") + arms[name] = tuple(Path(p) for p in paths) + + t0_gaf_sha256 = sha256_file(args.t0_gaf) + loaded = {} + aspect_preds: dict[str, dict[str, dict[str, dict]]] = {} + committed: dict[tuple[str, str, float, str], dict] = {} + for arm, (run_dir, interpro, committed_dir) in arms.items(): + problems = abl.run_setting_mismatches( + run_dir, + abl.LADDER, + {"domain_key": "interpro", "evidence_filter": "manual"}, + input_sha256={ + "domain_annotations": sha256_file(interpro), + "gaf": t0_gaf_sha256, + }, + ) + if problems: + for problem in problems: + logger.error(f"[{arm}] Settings mismatch — {problem}") + return 1 + logger.info(f"[{arm}] {run_dir} on {interpro}") + inputs = abl.load_benchmark_inputs( + args.t0_gaf, + args.t1_gaf, + interpro, + args.go_ontology, + "interpro", + "manual", + cohort_interpro=args.cohort_interpro, + ) + methods, _, _ = abl.build_methods(run_dir, inputs, "pscore", "interpro") + loaded[arm] = inputs + aspect_preds[arm] = { + m: { + a: tb.restrict_to_aspect(methods[m], a, inputs.term_aspect) + for a in ASPECTS + } + for m in METHODS + } + for row in pd.read_csv( + committed_dir / "ablation_metrics.tsv", sep="\t" + ).to_dict("records"): + committed[(arm, row["aspect"], float(row["min_ic"]), row["method"])] = row + + first, *rest = arms + for arm in rest: + for field in ("benchmark", "ic"): + if getattr(loaded[arm], field) != getattr(loaded[first], field): + logger.error(f"{arm} and {first} differ in {field}: not paired") + return 1 + if aspect_preds[arm]["naive"] != aspect_preds[first]["naive"]: + logger.error(f"{arm} and {first} differ in the naive baseline") + return 1 + logger.info(f"✓ One cohort, one IC and one naive baseline across {len(arms)} arms") + + _JOB = Job( + benchmark=loaded[first].benchmark, + ic=loaded[first].ic, + aspect_preds=aspect_preds, + committed=committed, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + ci_level=args.ci_level, + ) + tasks = [(a, m) for a in ASPECTS for m in ic_floors] + rows, problems = [], [] + with mp.get_context("fork").Pool(args.workers) as pool: + for cell_rows, cell_problems in pool.imap(_score_cell, tasks): + rows += cell_rows + problems += cell_problems + if problems: + for problem in problems: + logger.error(f"Does not reproduce the committed arm — {problem}") + return 1 + logger.info("✓ Every arm reproduces its committed ablation_metrics.tsv") + + df = pd.DataFrame(rows) + args.output.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(args.output, sep="\t", index=False) + logger.info(f"✓ {args.output}") + for row in df[df.method == "supra_input"].to_dict("records"): + logger.info( + f" [{row['aspect']} IC>={row['min_ic']:g}] Base {row['metric']:5s} " + f"{row['arm_a']} - {row['arm_b']} = {row['observed_diff']:+.4f} " + f"[{row['diff_ci_lo']:+.4f}, {row['diff_ci_hi']:+.4f}]" + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) From 1c60ca6acb2c2f5436e16a4d7cd06ad4f9e7ea3f Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 7 Oct 2026 13:37:14 +0100 Subject: [PATCH 5/7] Land the section 4 ablation on InterPro 85.0 (t0) domain architectures The ipr_manual cell retrained on InterPro 85.0 (2021-04-08) and a selection-control arm (current protein2ipr, the same t0+t1 protein selection) were scored with the post-#74 evaluator. Both use the committed cell's cohort (--cohort-interpro): 298 BP / 406 MF / 436 CC, of which 19 / 16 / 14 proteins have no InterPro 85.0 domain and count as misses. The runs predate this rebase. run_dcgo_human.py and the analysis code are unchanged since, so they were scored directly with validation/ablation.py rather than through the runbook, whose evidence-filter check would re-extract and retrain them. - Selection (control - baseline): within -0.003..+0.008 F_max and -0.001..+0.005 AUPRC across the four configurations. - InterPro release (t0 - control): lower in 32/36 F_max (11 significant) and 36/36 AUPRC (18 significant) comparisons, at most 0.023 / 0.018. Base loses most in MF: 0.357 -> 0.335 F_max at IC 2. - No conclusion changes. Base -> Base + relative keeps the same significant cells in the same direction (F_max 7/9 lower, 5 sig; AUPRC 8/9 lower, 5 sig, MF IC 0 a significant gain). The permutation null is 18/18 at p = 1/201 in every arm. Tables: validation/ablation_cells/{ipr_manual_t0interpro, ipr_manual_selctrl}/ with rung manifests and the subset provenance sidecars, the own-cohort sensitivity in ipr_manual_t0interpro_owncohort/, and the paired comparison in t0_interpro_comparison.tsv. VALIDATION_PLAN section 4 has the write-up, including what is not removed: seven of nine rungs still train with go-basic.obo 2026-06-15. RELATIVE_INFERENCE_REPORT section 4.2 gets the t0 row, and TODO P0 lists the analyses that still read current protein2ipr. Co-Authored-By: Claude Opus 5.5 (1M context) --- RELATIVE_INFERENCE_REPORT.md | 18 +- TODO.md | 11 + VALIDATION_PLAN.md | 114 ++++- .../ipr_manual_selctrl/ablation_metrics.tsv | 136 ++++++ .../ablation_paired_bootstrap.tsv | 415 ++++++++++++++++++ .../ablation_permutation_null.tsv | 19 + .../ablation_provenance.tsv | 16 + .../ablation_selection_counts.tsv | 22 + .../ipr_manual_selctrl/manifests/full.json | 216 +++++++++ ...man_t0t1sel_current.dat.gz.provenance.json | 12 + .../ipr_manual_selctrl/manifests/single.json | 188 ++++++++ .../ipr_manual_selctrl/manifests/supra.json | 188 ++++++++ .../manifests/supra_input.json | 207 +++++++++ .../manifests/supra_input_output.json | 215 +++++++++ .../manifests/supra_input_relative.json | 215 +++++++++ .../manifests/supra_output.json | 210 +++++++++ .../manifests/supra_relative.json | 210 +++++++++ .../manifests/supra_relative_output.json | 211 +++++++++ .../ablation_metrics.tsv | 136 ++++++ .../ablation_paired_bootstrap.tsv | 415 ++++++++++++++++++ .../ablation_permutation_null.tsv | 19 + .../ablation_provenance.tsv | 16 + .../ablation_selection_counts.tsv | 22 + .../ipr_manual_t0interpro/manifests/full.json | 216 +++++++++ ...2ipr_human_t0_ipr85.dat.gz.provenance.json | 12 + .../manifests/single.json | 188 ++++++++ .../manifests/supra.json | 188 ++++++++ .../manifests/supra_input.json | 207 +++++++++ .../manifests/supra_input_output.json | 215 +++++++++ .../manifests/supra_input_relative.json | 215 +++++++++ .../manifests/supra_output.json | 210 +++++++++ .../manifests/supra_relative.json | 210 +++++++++ .../manifests/supra_relative_output.json | 211 +++++++++ .../ablation_metrics.tsv | 136 ++++++ .../ablation_paired_bootstrap.tsv | 415 ++++++++++++++++++ .../ablation_provenance.tsv | 16 + .../ablation_selection_counts.tsv | 19 + .../ablation_cells/t0_interpro_comparison.tsv | 271 ++++++++++++ 38 files changed, 5956 insertions(+), 4 deletions(-) create mode 100644 validation/ablation_cells/ipr_manual_selctrl/ablation_metrics.tsv create mode 100644 validation/ablation_cells/ipr_manual_selctrl/ablation_paired_bootstrap.tsv create mode 100644 validation/ablation_cells/ipr_manual_selctrl/ablation_permutation_null.tsv create mode 100644 validation/ablation_cells/ipr_manual_selctrl/ablation_provenance.tsv create mode 100644 validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/full.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/protein2ipr_human_t0t1sel_current.dat.gz.provenance.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/single.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_output.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_relative.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_output.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative.json create mode 100644 validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative_output.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/ablation_metrics.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/ablation_paired_bootstrap.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/ablation_permutation_null.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/ablation_provenance.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/full.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/protein2ipr_human_t0_ipr85.dat.gz.provenance.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/single.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_output.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_relative.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_output.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative_output.json create mode 100644 validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_metrics.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_paired_bootstrap.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_provenance.tsv create mode 100644 validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_selection_counts.tsv create mode 100644 validation/ablation_cells/t0_interpro_comparison.tsv diff --git a/RELATIVE_INFERENCE_REPORT.md b/RELATIVE_INFERENCE_REPORT.md index 86eda13..6773196 100644 --- a/RELATIVE_INFERENCE_REPORT.md +++ b/RELATIVE_INFERENCE_REPORT.md @@ -385,8 +385,9 @@ the same cells either way. **Robustness across the design axes.** The same four configurations were rerun with the domain key switched to SCOP superfamilies (`ssf`, the published dcGO's domain universe) and with IEA annotations included in -training — four cells in all. The Base → Base + relative contrast, paired -bootstrap per cell: +training — four cells in all — and, in the last row, on 2021 domain +architectures. The Base → Base + relative contrast, paired bootstrap per +cell: | cell | cohort BP / MF / CC | F_max: relative worse | AUPRC: relative worse | |---|---|---|---| @@ -394,6 +395,7 @@ bootstrap per cell: | InterPro, + IEA | 187 / 242 / 161 | 8/9 (3 sig; 0 sig better) | 8/9 (5 sig; 1 sig better) | | SCOP SSF, manual | 140 / 221 / 195 | 7/9 (6 sig; 2 sig better) | 7/9 (6 sig; 1 sig better) | | SCOP SSF, + IEA | 72 / 104 / 51 | 8/9 (2 sig; 1 sig better) | 8/9 (6 sig; 1 sig better) | +| InterPro 85.0 (t0), manual | 298 / 406 / 436 | 7/9 (5 sig; 0 sig better) | 8/9 (5 sig; 1 sig better) | The direction is the same in every cell: the relative test is worse in 7–8 of 9 aspect × IC cells on both metrics, with at most two significant gains. @@ -407,6 +409,12 @@ is always built from exactly the evidence training saw. The SSF cells keep only proteins carrying SSF domains. So absolute numbers are not comparable across cells; the within-cell contrasts are.) +On 2021 architectures (last row: the InterPro/manual cell retrained on +InterPro 85.0, released 2021-04-08, and scored on the same cohort), Base is +slightly lower, by up to 0.022 F_max in MF. The same cells are significant, +in the same direction, so domain-side look-ahead does not explain the result +either (`VALIDATION_PLAN.md` §4 "t0 InterPro architectures"). + ### 4.3 Sensitivity / PPV decomposition at the operating point *(Recall = sensitivity. Precision = positive predictive value, PPV — the @@ -604,7 +612,11 @@ test with its measured cost rather than as a default. (`VALIDATION_PLAN.md` §4, "Regenerating a cell"). - **Robustness cells** (§4.2): the same nine-rung ladder rerun with `--domain-key ssf` and/or `--evidence-filter all`; eval tables and - per-run manifests in `validation/ablation_cells/{ipr_all,ssf_manual,ssf_all}/`. + per-run manifests in `validation/ablation_cells/{ipr_all,ssf_manual,ssf_all}/`; + the InterPro 85.0 (t0) rerun and its selection control in + `validation/ablation_cells/{ipr_manual_t0interpro,ipr_manual_selctrl}/`, + paired against the committed cell in + `validation/ablation_cells/t0_interpro_comparison.tsv`. - **Scoring conventions** (§4.0, §4.2): `validation/check_metric_conventions.py` re-scores the cell with F_max only at τ ≥ 0.001 and AUPRC without its point at the origin, on the committed bootstrap resamples; output diff --git a/TODO.md b/TODO.md index cdc30cc..ccd2e14 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,17 @@ implementation. archived t0 InterPro/protein2ipr snapshot, or explicitly limit every claim to an annotation-temporal benchmark. This also closes the 31.9% universe loss in the all-species held-out arm. + **§4 ablation done on t0 architectures (2026-10-07).** The `ipr_manual` + cell was retrained and rescored on InterPro 85.0 (2021-04-08), with a + selection-control arm, and paired on the committed cohort + (`VALIDATION_PLAN.md` §4 "t0 InterPro architectures"). Base drops by up to + 0.022 F_max (MF). No conclusion changes. Still open: + - the §2 headline benchmark, the multi-ontology breadth test, the + surprise-score held-out test, and the all-species and mouse arms still read + current `protein2ipr` (the code path exists: `--interpro`, + `extract_human_interpro.py --source`); + - GO-vocabulary look-ahead: seven of nine rungs train with `go-basic.obo` + 2026-06-15 (propagation, parental background, retired-id remap). - [ ] **Correct phylogenetic non-independence (gene / UniRef50 collapse).** Protein-level pooling inflates all-species support by about 2.44× and half of associations rest on at most three UniRef50 clusters; the model-organism and diff --git a/VALIDATION_PLAN.md b/VALIDATION_PLAN.md index 2c4d7fd..18d503b 100644 --- a/VALIDATION_PLAN.md +++ b/VALIDATION_PLAN.md @@ -1268,7 +1268,119 @@ during training — for input or output propagation and the parental background — and the input-propagation rungs remap t0 terms obsoleted since 2021 to their 2026 successors (74 alt_id and 417 replaced_by terms in the committed cell). The residual favours the hierarchy stages the ablation -compares, so the results must not be called prospective. Results to follow. +compares, so the results must not be called prospective. Results in the next +subsection. + +### t0 InterPro architectures: three arms (2026-10-07) + +**Design.** Three arms of the `ipr_manual` cell. Each is the nine committed +rung commands trained on GOA release 205 (`--gaf`); they differ only in the +protein2ipr subset read in training and in the transfer step: + +| arm | protein2ipr | proteins selected by | subset: proteins / InterPro entries | eval tables | +|---|---|---|---|---| +| baseline (committed) | current (downloaded 2026-07-07) | t1 GAF, non-IEA | 18,909 / 19,534 | `validation/ablation_*.tsv` | +| control | current | t0 ∪ t1 GAFs, any evidence | 19,523 / 19,771 | `validation/ablation_cells/ipr_manual_selctrl/` | +| t0 | InterPro 85.0 (2021-04-08) | t0 ∪ t1 GAFs, any evidence | 19,394 / 17,061 | `validation/ablation_cells/ipr_manual_t0interpro/` | + +So control − baseline isolates the protein selection, and t0 − control +isolates the InterPro release: the domain-architecture look-ahead. The t0 +arm learns fewer associations (`supra_input`: 436,620 over 23,379 features, +against 491,823 over 27,457 for control). All three are scored on the +committed cell's cohort (`--cohort-interpro data/interim/protein2ipr_human.dat.gz`): +298 BP, 406 MF and 436 CC proteins. Of these, 19 / 16 / 14 have no +InterPro 85.0 domain. They get no t0 prediction and count as misses (0 in +the control arm). The naive baseline is therefore identical in every arm. +`validation/compare_t0_interpro.py` pairs the arms on those proteins: one +1,000-replicate paired bootstrap per aspect × IC cell, over the +four configurations of `RELATIVE_INFERENCE_REPORT.md` and naive, drawn from +the same resamples as the per-arm cells. Output: +`validation/ablation_cells/t0_interpro_comparison.tsv`. It refuses to write +unless every arm reproduces its committed `ablation_metrics.tsv` and the arms +share one cohort, one IC table and one naive baseline. Both checks passed. + +**Base (`supra_input`) per arm**, and the InterPro-release contrast +(\* = paired 95% CI excludes 0): + +| cell | F_max: baseline / control / t0 | t0 − control (95% CI) | AUPRC: baseline / control / t0 | t0 − control (95% CI) | +|---|---|---:|---|---:| +| BP 0 | 0.243 / 0.244 / 0.238 | −0.006 (−0.018, +0.005) | 0.120 / 0.120 / 0.113 | −0.007 (−0.017, +0.002) | +| BP 2 | 0.193 / 0.193 / 0.187 | −0.006 (−0.015, +0.006) | 0.076 / 0.076 / 0.069 | −0.007 (−0.016, +0.001) | +| BP 4 | 0.135 / 0.138 / 0.138 | +0.000 (−0.006, +0.007) | 0.036 / 0.036 / 0.035 | −0.001 (−0.005, +0.002) | +| MF 0 | 0.363 / 0.366 / 0.360 | −0.005 (−0.014, +0.003) | 0.175 / 0.176 / 0.172 | −0.004\* (−0.008, −0.001) | +| MF 2 | 0.356 / 0.357 / 0.335 | −0.022\* (−0.038, −0.004) | 0.218 / 0.218 / 0.208 | −0.010\* (−0.019, −0.002) | +| MF 4 | 0.338 / 0.338 / 0.325 | −0.013\* (−0.027, −0.000) | 0.200 / 0.199 / 0.193 | −0.007 (−0.015, +0.000) | +| CC 0 | 0.371 / 0.370 / 0.366 | −0.004 (−0.013, +0.002) | 0.207 / 0.207 / 0.200 | −0.007\* (−0.013, −0.003) | +| CC 2 | 0.194 / 0.193 / 0.190 | −0.004 (−0.010, +0.002) | 0.058 / 0.058 / 0.054 | −0.004\* (−0.008, −0.001) | +| CC 4 | 0.128 / 0.128 / 0.126 | −0.001 (−0.012, +0.005) | 0.036 / 0.036 / 0.031 | −0.005\* (−0.010, −0.001) | + +- **Protein selection moves nothing material.** Control − baseline is + within ±0.003 for Base, and within −0.003 to +0.008 (F_max) and −0.001 to + +0.005 (AUPRC) across all four configurations. A few of these differences + are significant only because the two arms are almost identical, so the + intervals are very narrow. +- **Dating the architectures costs a little, mostly in MF.** Across the four + configurations, t0 − control is lower in 32 of 36 F_max comparisons + (11 significant, none significantly higher). It is lower in 36 of 36 AUPRC + comparisons (18 significant). The largest drops are 0.023 F_max and 0.018 + AUPRC. Some of the cost is coverage: 14–19 cohort proteins per aspect had no + 2021 domain. Scored instead on its own cohort (no-knowledge proteins with an + InterPro 85.0 domain: 280 / 392 / 422), t0 Base is within 0.011 of the + committed cell. That comparison is not paired, because the cohorts differ + (`validation/ablation_cells/ipr_manual_t0interpro_owncohort/`). In short, + current architectures inflated the committed absolute numbers by up to + about 0.02. +- **No conclusion changes.** In the t0 arm, the supra model beats all 200 + shuffled mappings in 18/18 cell × metric tests (`p = 1/201`), as in the + other two arms. Supra-domains do not change F_max significantly in any cell. + Relative inference, added directly to supra, lowers both metrics in all nine cells + (eight significant on each). Added to input + output propagation it lowers + F_max in 8 of 9 (5 significant) and AUPRC in 8 of 9 (7 significant; the MF + IC 0 gain stays). Input propagation rescues the relative rungs: `full` vs + `supra_relative_output` is higher in 9/9 cells on both metrics (6 and 8 + significant). Base → Base + relative, the report's spine, keeps its + significance pattern. F_max is lower in 7 of 9 cells (5 significant, none + significantly higher); AUPRC is lower in 8 of 9 (5 significant, with MF IC 0 + significantly higher). These are the same significant cells, in the same + direction, as the committed cell. The only sign change is CC 0 F_max (−0.004 → + +0.002, neither significant). The best rung still varies by cell. Input + + output is best on both metrics for BP 0, BP 2 and CC 2. Input alone is best + on both for BP 4, MF 2 and MF 4. `full` is best for MF 0, output alone for + CC 0. At CC 4, single domains have the best F_max and output alone the best + AUPRC. + +**What this does and does not remove.** It removes the domain-side look-ahead +from this one cell: training and transfer both read InterPro 85.0, released +13 days before GOA 205. It does **not** remove the vocabulary look-ahead +described above. Seven of the nine rungs still train with `go-basic.obo` +2026-06-15 for propagation and the parental background, and retired t0 ids are +remapped to their 2026 successors. The evaluator also uses that ontology for +truth propagation, IC and aspects, as CAFA does. The other temporal analyses +(§2 headline benchmark, multi-ontology breadth, surprise-score held-out test, +all-species and mouse arms) still read current `protein2ipr`. + +Regeneration. The two new arms' runs (`run_dcgo_human.py` unchanged since) +are the runbook's: + +```bash +uv run python scripts/run_t0_interpro_ablation.py # t0 arm +uv run python scripts/run_t0_interpro_ablation.py \ + --source data/raw/interpro_mappings/protein2ipr.dat.gz \ + --subset data/interim/protein2ipr_human_t0t1sel_current.dat.gz \ + --run-dir results/ablation-t0interpro/ipr_manual_selctrl \ + --eval-dir validation/ablation_cells/ipr_manual_selctrl # control arm +uv run python validation/compare_t0_interpro.py \ + --t0-gaf data/raw/goa_archive/goa_human.gaf.205.gz \ + --t1-gaf data/raw/goa_annotations/goa_human.gaf.gz +``` + +The committed tables were scored by running the runbook's `eval` step +(`validation/ablation.py ... --interpro --cohort-interpro +data/interim/protein2ipr_human.dat.gz`) on the existing runs. Their subset +sidecars predate the marker's `evidence_filter` field: the extraction logs +record `Evidence codes: ALL`. A runbook rerun would therefore cut the subsets +and retrain again. The own-cohort sensitivity is the same command +without `--cohort-interpro` and with `--n-permutations 0`. ## 5. Decisions to settle before writing the paper diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_metrics.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_metrics.tsv new file mode 100644 index 0000000..b974459 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_metrics.tsv @@ -0,0 +1,136 @@ +aspect min_ic method n_eval_proteins f_max f_max_tau s_min s_min_tau auprc coverage_at_fmax coverage_any f_max_ci_lo f_max_ci_hi auprc_ci_lo auprc_ci_hi +BP 0.0 single 298 0.23462050056636874 0.1009090320663031 87.03683820364283 0.26 0.10658278695025916 0.540268456375839 0.5503355704697986 0.20319979162320806 0.2645765106055159 0.0831351889995671 0.12993497576491062 +BP 0.0 single__p 298 0.23073374344603279 0.07 86.70800255469075 0.28 0.10847321232742393 0.540268456375839 0.5503355704697986 0.20075673065919994 0.2602962814349992 0.08519718104631915 0.1317281553147522 +BP 0.0 supra 298 0.23389853138263522 0.06 86.85056157423494 0.26 0.10862153438760377 0.540268456375839 0.5503355704697986 0.20377288232888746 0.2641130370431208 0.08408942559276783 0.13249041762412506 +BP 0.0 supra__p 298 0.23231176680570664 0.07 86.63988909394779 0.28 0.11054932640901875 0.5335570469798657 0.5503355704697986 0.20322208420209156 0.26255000427158637 0.08631594479252865 0.13430332572720097 +BP 0.0 supra_input 298 0.24391948612898903 0.08519748975544367 86.84296844596588 0.25 0.12031130066802086 0.5536912751677853 0.5570469798657718 0.21384526150867808 0.27406594269408935 0.09478031588216461 0.14441746460181215 +BP 0.0 supra_input__p 298 0.24644244957948366 0.07743856453961838 86.6678794344737 0.22 0.12330518679401269 0.5536912751677853 0.5570469798657718 0.21541120271822914 0.2772419924216772 0.09703191523562338 0.1477531459685087 +BP 0.0 supra_relative 298 0.1952356755425039 0.0 88.5112830296655 0.1643236840785566 0.061404243604493335 0.47315436241610737 0.47315436241610737 0.16605742996233827 0.22419147779808313 0.044815276641618725 0.07928220526923001 +BP 0.0 supra_relative__p 298 0.1952356755425039 0.0 88.49839816183966 0.25 0.06358656436002388 0.47315436241610737 0.47315436241610737 0.16600265334553302 0.2237093419960962 0.04650487085342438 0.08105476434670895 +BP 0.0 supra_output 298 0.239828172030419 0.05541015188537278 86.93461141510437 0.15 0.11621098601018635 0.5503355704697986 0.5503355704697986 0.20945593262965945 0.269214372588119 0.09069487479130439 0.1409887047409187 +BP 0.0 supra_input_relative 298 0.21161694252967744 0.001322985433736653 87.7509866674077 0.14 0.09092431931044631 0.5033557046979866 0.5033557046979866 0.1858373287938453 0.24059472639228505 0.069892948531383 0.11224311719161875 +BP 0.0 supra_input_relative__p 298 0.2122085268270943 0.01576712673355739 87.56722254321323 0.1402209607691329 0.0928719308971931 0.4966442953020134 0.5033557046979866 0.186603602768624 0.24175512055571746 0.07114275424705438 0.11416228597120374 +BP 0.0 supra_input_output 298 0.24970556786024375 0.04 86.45329757963833 0.15 0.12990810902139693 0.5570469798657718 0.5570469798657718 0.21785049156121408 0.27851401907659484 0.10283634932252131 0.15461204924369393 +BP 0.0 supra_relative_output 298 0.1991796963716254 0.01082777524355583 88.44131705046676 0.12 0.07435490655374574 0.47315436241610737 0.47315436241610737 0.16880043520349933 0.22757664343542042 0.05528998450256985 0.0947177860825932 +BP 0.0 full 298 0.22006836976335684 0.03485027367672731 87.38647679574991 0.11775806303808652 0.09730776842727502 0.5033557046979866 0.5033557046979866 0.19194370833427105 0.24860485039885313 0.07469101066765939 0.11958924846795306 +BP 0.0 naive 298 0.24522800676034753 0.246047504670403 87.81969159551754 0.246047504670403 0.1548729085557252 1.0 1.0 0.23210521828645966 0.26143307712405467 0.14126086596529053 0.168859588434164 +BP 2.0 single 298 0.1734134360870587 0.11 82.74236711373015 0.31 0.068340971805973 0.5302013422818792 0.5503355704697986 0.14398834898894922 0.20761205209519207 0.046704784844398606 0.09435659979652498 +BP 2.0 single__p 298 0.172988886807484 0.07275212072519098 82.47892942715958 0.28 0.06907175494481486 0.5335570469798657 0.5503355704697986 0.14325508022869216 0.20644871802394255 0.04745088560116152 0.09486986885858088 +BP 2.0 supra 298 0.1761580707594129 0.061085531575059894 82.55132612864877 0.26 0.07180857194540718 0.5369127516778524 0.5503355704697986 0.14560584834738957 0.20974944055560021 0.049582016846640736 0.09836741589998367 +BP 2.0 supra__p 298 0.1727929767731314 0.046434375336935285 82.3849354358462 0.28 0.07247674932876655 0.5335570469798657 0.5503355704697986 0.14549770014166946 0.20874621972539711 0.05063315332906785 0.09885895726870621 +BP 2.0 supra_input 298 0.19299661151865918 0.027400846629243348 82.57744386852903 0.25 0.07592523255095349 0.5536912751677853 0.5570469798657718 0.1619955213426049 0.2263578212138313 0.05391627394003992 0.10076508519086178 +BP 2.0 supra_input__p 298 0.1933879786174145 0.05 82.52575518194544 0.25 0.07823221780944516 0.5536912751677853 0.5570469798657718 0.16262928380313085 0.22724931071648216 0.056197143484103984 0.10374629645187702 +BP 2.0 supra_relative 298 0.13995283133552622 0.0 83.81855040720536 0.23 0.032853065552072104 0.47315436241610737 0.47315436241610737 0.11088289858991524 0.16998322754828868 0.020330110090449462 0.049452705024262356 +BP 2.0 supra_relative__p 298 0.13995283133552622 0.0 83.85558836804906 0.25 0.03435211839570298 0.47315436241610737 0.47315436241610737 0.11083220832160809 0.17042107624843017 0.021117010455580423 0.05190691173121019 +BP 2.0 supra_output 298 0.1809761204212188 0.032346508835104995 82.86538014609015 0.1449820575910462 0.06817650955107903 0.5503355704697986 0.5503355704697986 0.15077742461752794 0.21284531693539044 0.04787706538071174 0.09239178932340207 +BP 2.0 supra_input_relative 298 0.1590048692983248 0.0011987910383646685 83.28231211906387 0.14 0.053680719886573844 0.5033557046979866 0.5033557046979866 0.13064609740234695 0.18715167844137684 0.03687794514951261 0.07491663804456254 +BP 2.0 supra_input_relative__p 298 0.15925563467279807 0.0031417876717293727 83.21997584779145 0.14 0.055258849008942644 0.5033557046979866 0.5033557046979866 0.13120440170332062 0.1884604972993399 0.03806852200999812 0.07700346893984514 +BP 2.0 supra_input_output 298 0.19290019525631774 0.024763687401881746 82.48190777737437 0.15 0.08150147032223701 0.5536912751677853 0.5570469798657718 0.1626722421646621 0.22438260934354684 0.05903725906908419 0.10819218192481433 +BP 2.0 supra_relative_output 298 0.14267615352407817 0.002232977183183446 83.88003859614916 0.12 0.036862518281928124 0.47315436241610737 0.47315436241610737 0.11407857856992037 0.17355633619417554 0.023031469083321945 0.055410558831756704 +BP 2.0 full 298 0.15848590904470627 0.002593740892774367 83.20068931648949 0.12024988322513586 0.054411779329926364 0.5033557046979866 0.5033557046979866 0.13166214039035032 0.1872008578168793 0.03728641279703247 0.07588002827697626 +BP 2.0 naive 298 0.1393754232163564 0.12271470509741127 84.8347549558933 0.23094528956498533 0.05246757199300765 1.0 1.0 0.12932984580650728 0.14944692205357168 0.04676691808057873 0.05836943398200638 +BP 4.0 single 292 0.1131630709975688 0.1 68.1987987513269 0.26 0.029164069258372262 0.5136986301369864 0.5513698630136986 0.08807274893358608 0.1500155268501458 0.01712688570656073 0.04857638185090283 +BP 4.0 single__p 292 0.11118722545959618 0.0547736523835257 68.05357563993098 0.27 0.02966421515910911 0.5376712328767124 0.5513698630136986 0.08734006708594935 0.14570291519186412 0.017898381728832576 0.048661959470960484 +BP 4.0 supra 292 0.11805494385249392 0.06 68.08682422041258 0.26 0.03275518908553014 0.5376712328767124 0.5513698630136986 0.09102038376686608 0.15365122715477042 0.01913900929529108 0.05391737279348348 +BP 4.0 supra__p 292 0.117186105167848 0.07 68.0045215004124 0.28 0.03319294624547443 0.5308219178082192 0.5513698630136986 0.09013873696120957 0.15100062770411884 0.020031488864619054 0.05345971267346148 +BP 4.0 supra_input 292 0.13785042940917663 0.011228959418977055 68.23141207818566 0.25 0.036198295212282634 0.5547945205479452 0.5547945205479452 0.10826857084483672 0.1695354417788305 0.02287336732513037 0.0542382164455364 +BP 4.0 supra_input__p 292 0.1351424334013583 0.01760933185886738 68.2291143213808 0.21 0.036962890528700866 0.5513698630136986 0.5547945205479452 0.10721512428656961 0.16838495841529552 0.023623591854009394 0.05524344800477482 +BP 4.0 supra_relative 292 0.08498026914465588 0.0 68.77063771342449 0.22915089829619675 0.01356999959646461 0.4554794520547945 0.4554794520547945 0.06172770712082376 0.11507339597565616 0.00676371131749096 0.024250822357977524 +BP 4.0 supra_relative__p 292 0.08498026914465588 0.0 68.86142207391424 0.25 0.01344092082906797 0.4554794520547945 0.4554794520547945 0.061934319846540596 0.1141210606077235 0.006805019578679928 0.02420714112942443 +BP 4.0 supra_output 292 0.1187927440433954 0.01577815069359943 68.57507042561157 0.15 0.026632418405861672 0.5513698630136986 0.5513698630136986 0.09047218538690849 0.1512233094266759 0.015803577970112257 0.0425809234562346 +BP 4.0 supra_input_relative 292 0.093217666191155 0.0013493762493373104 68.61069890700612 0.14 0.019850976304688597 0.4897260273972603 0.4897260273972603 0.07094842164634468 0.11809855550232483 0.010719939312756874 0.033533256500594924 +BP 4.0 supra_input_relative__p 292 0.09284181828228551 0.0006765690529224352 68.58346527663453 0.14 0.02050166008751001 0.4897260273972603 0.4897260273972603 0.07120124767354227 0.1187044217280471 0.01128165879669148 0.03466381889341036 +BP 4.0 supra_input_output 292 0.131901838297952 0.01529801395707868 68.31985357458497 0.14512327561564375 0.03637360789031955 0.5513698630136986 0.5547945205479452 0.10678262081681095 0.16735208957455938 0.0229575675360861 0.054459019008515026 +BP 4.0 supra_relative_output 292 0.08779686723124253 0.002085852403460882 68.96742941058933 0.12 0.01300225721216053 0.4486301369863014 0.4554794520547945 0.06370669081338284 0.11467818029569771 0.006033237161714707 0.023945267381077082 +BP 4.0 full 292 0.09090782814931803 0.00026682375680492376 68.64942555246917 0.09298307580016145 0.02187984746106082 0.4897260273972603 0.4897260273972603 0.06999442378219012 0.11636374117855347 0.011885537276441677 0.03706728490155736 +BP 4.0 naive 292 0.04686917580427525 0.04859781158259942 69.96559437929291 0.06228983186549239 0.013722538414023608 1.0 1.0 0.04072911691354331 0.05358136988666269 0.012021347236526566 0.015701235045600684 +MF 0.0 single 406 0.36126219819528144 0.1754938446857179 12.3085773037057 0.51 0.18039683004408205 0.5738916256157636 0.6305418719211823 0.321615939513396 0.39878211621554277 0.1490635927386993 0.2134828010855519 +MF 0.0 single__p 406 0.3742401179055157 0.15 12.285709269361362 0.65 0.19443328533025786 0.5911330049261084 0.6305418719211823 0.3389600014421806 0.41132140353432406 0.160945340417446 0.2294892325099976 +MF 0.0 supra 406 0.34917244942038655 0.17 12.342529152967504 0.76 0.17863188024464952 0.5738916256157636 0.6305418719211823 0.3128847369199524 0.38490437885728795 0.14669907248092673 0.2123719891957057 +MF 0.0 supra__p 406 0.3640464808322841 0.2 12.270805160024688 0.68 0.19107449600914383 0.5714285714285714 0.6305418719211823 0.33092237999156626 0.40361064504411637 0.15837092803715422 0.225699559845707 +MF 0.0 supra_input 406 0.3655787927448233 0.07 12.511112836705179 0.46 0.17602961321469285 0.6157635467980296 0.6379310344827587 0.3308571960832066 0.40090962330503604 0.14565959602745185 0.208222826772925 +MF 0.0 supra_input__p 406 0.3667883375569147 0.07 12.481269178109429 0.37 0.17931440548182404 0.6182266009852216 0.6379310344827587 0.33364640319854716 0.402388407805116 0.14875243277820602 0.21141853113964978 +MF 0.0 supra_relative 406 0.34023526115113495 0.08 13.106709124144658 0.67 0.1595380439620823 0.5344827586206896 0.5665024630541872 0.3099167641022738 0.375386059917939 0.13132294643202172 0.1922521506092058 +MF 0.0 supra_relative__p 406 0.3456609710778204 0.09759930279899799 13.054037329099932 0.63 0.1703137512428886 0.5369458128078818 0.5665024630541872 0.3149722613614337 0.3809951935304286 0.14056019012009008 0.20430821342237887 +MF 0.0 supra_output 406 0.358212635059794 0.07 12.57154309737589 0.33 0.18606858135827942 0.603448275862069 0.6305418719211823 0.32411622416977126 0.39526164145578696 0.15460978454597243 0.22016850175307803 +MF 0.0 supra_input_relative 406 0.36075231398177715 0.12 12.99554269490813 0.39 0.19859411226663576 0.5738916256157636 0.6157635467980296 0.326939458067137 0.39812121291703917 0.1651156865188801 0.23401501905721764 +MF 0.0 supra_input_relative__p 406 0.37330712419946654 0.1310318423751731 12.970049793916994 0.35 0.20944674218159992 0.5788177339901478 0.6157635467980296 0.33773677299349314 0.4107395983029553 0.17534300418584056 0.2468748372406117 +MF 0.0 supra_input_output 406 0.3712904130952929 0.07 12.385995385032464 0.3 0.19631964007936464 0.6182266009852216 0.6379310344827587 0.3352478498280658 0.4074896092023632 0.1634686206841008 0.23078802952254204 +MF 0.0 supra_relative_output 406 0.33622861513771435 0.13 13.255693732853132 0.31 0.18553875154271193 0.4753694581280788 0.5665024630541872 0.31047970228110755 0.3785743330237947 0.15351666978952444 0.22262856635437883 +MF 0.0 full 406 0.38274701944288786 0.09 12.872103995889306 0.17 0.21842592506441955 0.5714285714285714 0.6157635467980296 0.3464221228524778 0.4195626144076439 0.18342649065015934 0.25466937540009293 +MF 0.0 naive 406 0.7189989169653068 0.2616402455297571 13.241152459749525 0.15066399786495863 0.4190445836593093 1.0 1.0 0.6771715184983654 0.7610189683025428 0.378372572506467 0.4606028266213177 +MF 2.0 single 159 0.3624206773738544 0.47286241979678767 26.480981406998445 0.47286241979678767 0.2009778910151442 0.5157232704402516 0.6981132075471698 0.29644152500421117 0.4298872783304056 0.14849590126866086 0.25800967564803895 +MF 2.0 single__p 159 0.3605503119804898 0.36 26.79618654347556 0.44 0.21187236763689593 0.5220125786163522 0.6981132075471698 0.2978394803246209 0.42654716558980327 0.15755582538759522 0.26920746049827826 +MF 2.0 supra 159 0.3575134302039365 0.38 26.734222900726657 0.47 0.20961207022311829 0.5283018867924528 0.6981132075471698 0.2964337702908799 0.4246561715739057 0.15319298307044116 0.2694412815874959 +MF 2.0 supra__p 159 0.35682263131926306 0.2780995767712847 26.931248572006435 0.42 0.21934785252779934 0.5408805031446541 0.6981132075471698 0.2945299914242254 0.42499997442499615 0.16232583668770073 0.27981841109470196 +MF 2.0 supra_input 159 0.35670234664936507 0.13 27.874194223296684 0.26 0.21789011079771334 0.5911949685534591 0.710691823899371 0.2941721069459892 0.4208347319157996 0.16316126380178067 0.2779145863882162 +MF 2.0 supra_input__p 159 0.35697768076494346 0.13 27.594740841188674 0.24 0.22064495087851968 0.5911949685534591 0.710691823899371 0.29576868591387157 0.4212269174135605 0.16541686463199815 0.2811105916485548 +MF 2.0 supra_relative 159 0.26640481629856816 0.020534264333664485 31.094114894704393 0.34222427668025623 0.09810165326207468 0.5786163522012578 0.6037735849056604 0.214868144866196 0.32708586658443034 0.06078507421644852 0.14435698126651716 +MF 2.0 supra_relative__p 159 0.2666590517853343 0.027334150613863722 30.84340329080371 0.487363673262399 0.10207056575657446 0.5786163522012578 0.6037735849056604 0.21401652986233302 0.3235172422241372 0.06349685477710774 0.14859197696018436 +MF 2.0 supra_output 159 0.3366341487828252 0.0818227575271097 28.434497550714642 0.15 0.2042822117811935 0.5911949685534591 0.6981132075471698 0.280217254259789 0.3981521551588211 0.14959876563257132 0.26474226761480324 +MF 2.0 supra_input_relative 159 0.30838965519130074 0.03381540462696795 29.602271607348808 0.19 0.1712713104376412 0.660377358490566 0.6792452830188679 0.25551204899541413 0.37173956793521257 0.12051990079194118 0.23123185813449587 +MF 2.0 supra_input_relative__p 159 0.31128499609980237 0.07 29.450454760226503 0.17 0.17667423632432544 0.6352201257861635 0.6792452830188679 0.2578685306380418 0.37263617791618736 0.12534294014673938 0.23459106456111412 +MF 2.0 supra_input_output 159 0.3509584986613415 0.09 27.67704672541936 0.17 0.2220098622274635 0.5911949685534591 0.710691823899371 0.29064310442012525 0.4120867116137788 0.16486949477912194 0.2851667407616529 +MF 2.0 supra_relative_output 159 0.28055796784500686 0.019860943178177208 31.068642378977287 0.13 0.11226843435692951 0.5849056603773585 0.6037735849056604 0.22407259294189286 0.3373479150813377 0.07031285553032794 0.16481327094801454 +MF 2.0 full 159 0.31749948602842004 0.03612165347120001 29.44228894253694 0.11 0.16934947196131758 0.6540880503144654 0.6792452830188679 0.2637617725166517 0.37517360486873574 0.1208896752847239 0.2260077401608347 +MF 2.0 naive 159 0.07561162501367721 0.050395516413130496 33.409141271799434 0.1096215639178009 0.026528609897190324 1.0 1.0 0.06416366594110963 0.08958879742758147 0.021516874326196884 0.03240999023166993 +MF 4.0 single 151 0.3298667018196884 0.4591638975666064 26.086899978861858 0.47 0.1939744433720369 0.5033112582781457 0.6887417218543046 0.26380829363602737 0.4030570825235187 0.13859767675436357 0.25757571022425385 +MF 4.0 single__p 151 0.3289766947217402 0.44 26.288034066631138 0.44 0.20428682965956058 0.4966887417218543 0.6887417218543046 0.2653668404733776 0.4047994647320423 0.14635159069171683 0.2686693330276674 +MF 4.0 supra 151 0.3262836462924187 0.38 26.30535332223921 0.47 0.1942229401483132 0.5231788079470199 0.6887417218543046 0.2642249398621763 0.3978319574399733 0.13734765852039083 0.25755600197917417 +MF 4.0 supra__p 151 0.3286146850150951 0.28 26.42321441788833 0.42 0.20289262627364152 0.5298013245033113 0.6887417218543046 0.26752966672206857 0.40111832455659974 0.14425366316896365 0.26803883381559407 +MF 4.0 supra_input 151 0.33803499974566886 0.13 27.130752032707125 0.26215841315965954 0.19917894898248917 0.5629139072847682 0.7019867549668874 0.2721073441573473 0.4039989783387203 0.14463525980584296 0.2617790020990403 +MF 4.0 supra_input__p 151 0.33871101728753 0.1281687752317547 26.98813371307144 0.24 0.20237148001995317 0.5629139072847682 0.7019867549668874 0.27336620528508176 0.40506670881872786 0.1473567190429198 0.2657289272111604 +MF 4.0 supra_relative 151 0.23537269599888028 0.041297139009963686 29.80086802553966 0.64 0.0817924273099161 0.543046357615894 0.5695364238410596 0.18825599439866184 0.2967448702621361 0.049724737877703186 0.12421861618094356 +MF 4.0 supra_relative__p 151 0.23554111291251084 0.02861202758184105 29.641415981082684 0.48 0.08576847103219928 0.5496688741721855 0.5695364238410596 0.18593219218541807 0.2923117425141783 0.052169973664631865 0.12742102948851997 +MF 4.0 supra_output 151 0.3157306071081526 0.08 27.524972697823983 0.14 0.18613794150147983 0.5562913907284768 0.6887417218543046 0.25602375546398315 0.38194506637323805 0.13264070944578443 0.2469382610191286 +MF 4.0 supra_input_relative 151 0.28129841286364055 0.07470456811259686 28.917258203848437 0.12 0.13470318550435026 0.6225165562913907 0.6688741721854304 0.22472191889353882 0.347728836607328 0.08988410053074097 0.18971713943762913 +MF 4.0 supra_input_relative__p 151 0.2839419642515468 0.08 28.78902010869179 0.17 0.13771409948073624 0.609271523178808 0.6688741721854304 0.22617747816316605 0.34811003053258965 0.0917943241793722 0.19339843405314003 +MF 4.0 supra_input_output 151 0.33136710383897316 0.08 26.954858658748964 0.16 0.20407333923335857 0.5629139072847682 0.7019867549668874 0.26902105284505545 0.3949221212802167 0.14794007474389362 0.2682481505148165 +MF 4.0 supra_relative_output 151 0.2403644107271082 0.019773743358133103 30.198171735766195 0.13 0.08662340309592595 0.5496688741721855 0.5695364238410596 0.1908581654459627 0.2963138023085404 0.05318775127530632 0.13019362014205413 +MF 4.0 full 151 0.2910231966668611 0.039260560897590474 28.78446322570643 0.11 0.13886787332506306 0.6225165562913907 0.6688741721854304 0.23129879715760607 0.3529148155348506 0.093039872973868 0.1925614229596739 +MF 4.0 naive 151 0.07531254466979251 0.05290899386175607 31.782400402647035 0.12447291166266346 0.02307128915639914 1.0 1.0 0.060367845875674626 0.09666103185263615 0.018552059417728722 0.028392641107576045 +CC 0.0 single 436 0.37992957089151047 0.016598538322808373 21.715232609121255 0.19 0.2206137897500839 0.5091743119266054 0.5206422018348624 0.3489206677713311 0.4084434207415569 0.18962814568865652 0.2527579544494135 +CC 0.0 single__p 436 0.3810739622864283 0.02 21.702538967879352 0.1737885287674954 0.22906493678232098 0.5091743119266054 0.5206422018348624 0.3500663821906321 0.410498679937061 0.19718680661119337 0.26139196170024187 +CC 0.0 supra 436 0.37497903858073917 0.017684217753851574 21.73476356469245 0.18 0.21886015071674497 0.5045871559633027 0.5206422018348624 0.34546383943041215 0.4054326360304714 0.18777120803705882 0.2497061281830789 +CC 0.0 supra__p 436 0.3784969129077132 0.024456093149169227 21.65606197681325 0.18 0.22710008251437833 0.5045871559633027 0.5206422018348624 0.34943178781174944 0.40877398786087576 0.19489467838215493 0.2586242298968998 +CC 0.0 supra_input 436 0.3702482776163785 0.015278904682258648 22.173006426458223 0.12 0.20742636150879065 0.4793577981651376 0.518348623853211 0.3406913896695123 0.4001758521738263 0.1768334711136948 0.2371333207994467 +CC 0.0 supra_input__p 436 0.3738629171139879 0.01 22.139171186331662 0.11994641896605546 0.21302617375856195 0.5068807339449541 0.518348623853211 0.34460802029582716 0.40192294316336613 0.18222365868351081 0.24287970031909598 +CC 0.0 supra_relative 436 0.347468249316779 0.01 21.942066006627844 0.27 0.17980251378717382 0.44036697247706424 0.463302752293578 0.3180704835059526 0.3800604975649636 0.15425822457488791 0.20717499398165157 +CC 0.0 supra_relative__p 436 0.35013192197074033 0.007241128877133074 21.859985335081156 0.29 0.18165566080061937 0.44954128440366975 0.463302752293578 0.3190669400743222 0.38235113016305083 0.15605849896167281 0.20942370122754173 +CC 0.0 supra_output 436 0.3865627580544418 0.0177268432807619 21.716355375621575 0.08237629357912948 0.24871874271560107 0.5160550458715596 0.5206422018348624 0.3571549914817983 0.41622651383559595 0.21693120183820377 0.2803661757241776 +CC 0.0 supra_input_relative 436 0.37268360539863454 0.004765863311814793 21.99844492229525 0.17 0.20558766875108328 0.49770642201834864 0.5068807339449541 0.3413427257068616 0.4028641261150094 0.1750513829872175 0.23652788740253738 +CC 0.0 supra_input_relative__p 436 0.37141883837088846 0.007444502635312336 21.943803690644376 0.17 0.20952030005470812 0.49770642201834864 0.5068807339449541 0.34192765246012813 0.4023819522584599 0.17936896809757275 0.24048175102901914 +CC 0.0 supra_input_output 436 0.3869795489823925 0.011653615791635137 21.788839389549995 0.0698719716008464 0.24806639386107782 0.5160550458715596 0.518348623853211 0.35692933855017095 0.415595050429171 0.21548769846129198 0.27959982223531604 +CC 0.0 supra_relative_output 436 0.35844968314631653 0.00578444153032776 21.89463601405919 0.12876747229348928 0.21315413830976376 0.463302752293578 0.463302752293578 0.32852105177120644 0.3888274846650605 0.18585086760583813 0.24256028753984693 +CC 0.0 full 436 0.38155247242054213 0.01 21.795740303747177 0.1 0.23398195570502178 0.5068807339449541 0.5068807339449541 0.3522813323556057 0.41206625782064343 0.20346951737646382 0.2649655091898008 +CC 0.0 naive 436 0.593050572267429 0.2848678943154524 19.830858368031098 0.2848678943154524 0.5414744252479067 1.0 1.0 0.5760583007709689 0.6133359283683919 0.5183452559067593 0.5650951654931785 +CC 2.0 single 298 0.2034339979081705 0.002813894309287136 24.498128882010988 0.16 0.057802355414117046 0.5033557046979866 0.5134228187919463 0.17122460832655492 0.23681801853788212 0.0373916501344801 0.08331310683914535 +CC 2.0 single__p 298 0.20281131242724537 0.0010405713097839284 24.539476080771465 0.16 0.05857576653466249 0.5033557046979866 0.5134228187919463 0.17066069528684527 0.23642491040190408 0.038373187173376705 0.08464292915840621 +CC 2.0 supra 298 0.19924497775877514 0.001197786287886931 24.451451407919333 0.14164956688710786 0.05712819334284143 0.5067114093959731 0.5167785234899329 0.16752476058419488 0.2312419308174136 0.03768964560379098 0.08253997048605428 +CC 2.0 supra__p 298 0.19623132529934278 0.0007100684178886583 24.41906906131458 0.19 0.05715582037085817 0.5033557046979866 0.5167785234899329 0.16417737842143437 0.2279438294779606 0.0383670994242635 0.08269292012085298 +CC 2.0 supra_input 298 0.19343405375604655 6.92678649240084e-05 24.712483394427217 0.15 0.05806421570392059 0.5067114093959731 0.5134228187919463 0.162135516511177 0.22338100898627253 0.038922442057912406 0.08352450534070248 +CC 2.0 supra_input__p 298 0.18927099396430336 8.965170586376601e-06 24.72681367973203 0.12 0.05822272432075315 0.5100671140939598 0.5134228187919463 0.15972108036663873 0.21970000927503977 0.03977519648617143 0.08348523967382958 +CC 2.0 supra_relative 298 0.17952044684106486 0.0 25.00654716930475 0.26 0.04473430249200807 0.4429530201342282 0.4429530201342282 0.14633376911597626 0.2125418054972493 0.028536906297276114 0.06492437043821017 +CC 2.0 supra_relative__p 298 0.17952044684106486 0.0 24.92367266192378 0.29 0.04450590883627963 0.4429530201342282 0.4429530201342282 0.14633376911597626 0.21272326817474285 0.028588695945565554 0.06454190477944306 +CC 2.0 supra_output 298 0.20995311289051635 0.001194206774132004 24.527909977390614 0.08249999999999998 0.059741954000379696 0.5134228187919463 0.5167785234899329 0.1757773599854713 0.24346689948952382 0.04045286611914171 0.08374203365253612 +CC 2.0 supra_input_relative 298 0.20495063609979508 0.0014689591793895945 24.60861890455283 0.12 0.052436747996529906 0.49328859060402686 0.5 0.1733624965682074 0.24049701619451533 0.03495261132401608 0.07525988040664223 +CC 2.0 supra_input_relative__p 298 0.20568134456605464 0.0020250072924527175 24.61178593111546 0.17 0.0521049587961016 0.4899328859060403 0.5 0.17386923486062647 0.24026644340554598 0.03470062782381399 0.07480133595939091 +CC 2.0 supra_input_output 298 0.21768741321397259 0.0010642121869099549 24.490668408161447 0.075 0.0653755442200359 0.5100671140939598 0.5134228187919463 0.18505262090189972 0.2514192082717889 0.045799795743892736 0.09042984298553164 +CC 2.0 supra_relative_output 298 0.18824252482325185 0.0010049380233196363 24.989376346174225 0.12368215761915297 0.04363441672467662 0.436241610738255 0.4429530201342282 0.15478130607441354 0.2236629004258581 0.028714315487776285 0.0618676988264316 +CC 2.0 full 298 0.20783250955677182 0.002056778929699232 24.71425794340687 0.09876291641531358 0.052007881921068415 0.5 0.5 0.1772394491632952 0.24346106923640293 0.03646181545927734 0.07340273893857163 +CC 2.0 naive 298 0.2720885241633347 0.10965999466239659 24.37542448179979 0.20930237523352016 0.1451197420848838 1.0 1.0 0.2500562969208015 0.30046153488752814 0.12937238422026592 0.16212109267301172 +CC 4.0 single 210 0.12966758302364031 0.002786322342480885 24.307847963745786 0.13 0.03379172626930409 0.44761904761904764 0.5 0.09143638090612322 0.17506690984298465 0.014518504433942144 0.06020563399503935 +CC 4.0 single__p 210 0.13107465296161744 0.0023316773336466683 24.368462401594606 0.17 0.03221100238499726 0.44285714285714284 0.5 0.09350300572958109 0.17704619894479126 0.013932738957785148 0.05695583400900875 +CC 4.0 supra 210 0.12630996046336843 0.04 24.34205843820473 0.14 0.031794658809648026 0.34285714285714286 0.5095238095238095 0.09250689104313772 0.1744905923738879 0.014032639685604618 0.056602273279831736 +CC 4.0 supra__p 210 0.126523132747442 0.05 24.33736898215918 0.19 0.03240961650919067 0.3523809523809524 0.5095238095238095 0.0934633532484225 0.17573276152276743 0.01444088224144998 0.05745745691486759 +CC 4.0 supra_input 210 0.12782363409703573 0.03185049154920013 24.538942625487348 0.09 0.03586106852445638 0.319047619047619 0.5095238095238095 0.094443989496628 0.17537715952279415 0.017209133353990755 0.0611190238099087 +CC 4.0 supra_input__p 210 0.12977022492817375 0.035787583408580015 24.562770076438376 0.07795623847428472 0.034433036289172284 0.319047619047619 0.5095238095238095 0.09489023471161136 0.17622262868730487 0.016452721598062135 0.05901557143838375 +CC 4.0 supra_relative 210 0.06785583012869659 0.0009674170938456294 24.928504612829254 0.26 0.01344602376340749 0.3476190476190476 0.36666666666666664 0.04025137141390231 0.10707800174888532 0.0038503656819588836 0.032583130167620974 +CC 4.0 supra_relative__p 210 0.0684185071917123 0.06028977636693802 24.893008183597868 0.29 0.013990111662375106 0.2523809523809524 0.36666666666666664 0.040320949639201575 0.10812952537215391 0.004186918397213848 0.033386046782404824 +CC 4.0 supra_output 210 0.12223317857868368 0.0004938892773757117 24.39828005793547 0.05999999999999999 0.03901494516963492 0.4714285714285714 0.5095238095238095 0.08971015656211787 0.1672317743696897 0.016504639836976618 0.06509306023762562 +CC 4.0 supra_input_relative 210 0.12554626887725337 0.0015037014705301516 24.605341307666503 0.09 0.030681186035091658 0.41904761904761906 0.42857142857142855 0.08584425499158228 0.16610680523912486 0.014113560999353543 0.05414953401443045 +CC 4.0 supra_input_relative__p 210 0.12680597575808103 0.001980351129936771 24.71682921516877 0.09 0.028372839620044246 0.4142857142857143 0.42857142857142855 0.08626645731543375 0.16693119539917578 0.013163081796440112 0.05061092874224047 +CC 4.0 supra_input_output 210 0.12547653290885707 0.0011615720578312403 24.44647330904609 0.0525 0.038374445533779236 0.44761904761904764 0.5095238095238095 0.09045625377421482 0.16965635023521292 0.01828398781853716 0.06402326946122651 +CC 4.0 supra_relative_output 210 0.06615289967671167 0.006121635663113865 24.999231656187952 0.07 0.013628547043723002 0.29523809523809524 0.36666666666666664 0.03819562541434294 0.10432237513639145 0.0034338796933065007 0.033471313719779155 +CC 4.0 full 210 0.12476406186624613 0.001981551246197991 24.81055295311823 0.061399151489476715 0.02865341327493268 0.4142857142857143 0.42857142857142855 0.08442651149901527 0.16414875196678616 0.012405889839620628 0.051011006767936344 +CC 4.0 naive 210 0.10440217265761789 0.04063624232719509 25.485077592733788 0.12308513477448625 0.04494942594240296 1.0 1.0 0.09374817360504709 0.12093250379485652 0.039116125355083756 0.051429210681231546 diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_paired_bootstrap.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_paired_bootstrap.tsv new file mode 100644 index 0000000..c1de83c --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_paired_bootstrap.tsv @@ -0,0 +1,415 @@ +aspect min_ic n_eval_proteins metric method_a method_b observed_a observed_b observed_diff diff_mean diff_ci_lo diff_ci_hi p_value significant n_replicates +BP 0.0 298 f_max supra single 0.23389853138263522 0.23462050056636874 -0.0007219691837335174 0.0003435112393617126 -0.0068553270442143 0.006284124305681499 0.904 False 1000 +BP 0.0 298 auprc supra single 0.10862153438760377 0.10658278695025916 0.0020387474373446124 0.0019902531146603186 -0.0010827559397496036 0.005702975700283 0.226 False 1000 +BP 0.0 298 f_max supra_input supra 0.24391948612898903 0.23389853138263522 0.01002095474635381 0.010526219197302964 -0.0010838288825089889 0.022925224015861733 0.074 False 1000 +BP 0.0 298 auprc supra_input supra 0.12031130066802086 0.10862153438760377 0.011689766280417083 0.01176518637680639 0.002213624002937504 0.02106583781161568 0.008 True 1000 +BP 0.0 298 f_max supra_relative supra 0.1952356755425039 0.23389853138263522 -0.03866285584013132 -0.03899723167516172 -0.057813742422220475 -0.020884793332037164 0.0 True 1000 +BP 0.0 298 auprc supra_relative supra 0.061404243604493335 0.10862153438760377 -0.04721729078311044 -0.04649438276274645 -0.061313082938638214 -0.03205975148914765 0.0 True 1000 +BP 0.0 298 f_max supra_output supra 0.239828172030419 0.23389853138263522 0.005929640647783785 0.0047359413763972584 -0.005312460837867111 0.015703625589523693 0.364 False 1000 +BP 0.0 298 auprc supra_output supra 0.11621098601018635 0.10862153438760377 0.00758945162258258 0.007391832670200894 0.000657353250503592 0.01455313424841462 0.036 True 1000 +BP 0.0 298 f_max supra_input_relative supra_input 0.21161694252967744 0.24391948612898903 -0.03230254359931159 -0.03153290519211576 -0.04841350762418164 -0.014506249795217448 0.0 True 1000 +BP 0.0 298 auprc supra_input_relative supra_input 0.09092431931044631 0.12031130066802086 -0.029386981357574546 -0.029021201628399468 -0.04135141928107224 -0.017437962651527264 0.0 True 1000 +BP 0.0 298 f_max supra_input_relative supra_relative 0.21161694252967744 0.1952356755425039 0.016381266987173543 0.017990545680348927 0.0055790707824915926 0.03199897828305377 0.008 True 1000 +BP 0.0 298 auprc supra_input_relative supra_relative 0.09092431931044631 0.061404243604493335 0.029520075705952976 0.02923836751115337 0.01950930714643755 0.03997633455732852 0.0 True 1000 +BP 0.0 298 f_max supra_input_output supra_input 0.24970556786024375 0.24391948612898903 0.005786081731254722 0.004633932047548863 -0.001951190093350666 0.011600681393492868 0.194 False 1000 +BP 0.0 298 auprc supra_input_output supra_input 0.12990810902139693 0.12031130066802086 0.009596808353376077 0.009291533984064352 0.004877340109483636 0.013871451333605386 0.0 True 1000 +BP 0.0 298 f_max supra_input_output supra_output 0.24970556786024375 0.239828172030419 0.009877395829824748 0.01042420986845457 0.0026634387621715257 0.019425607901721178 0.006 True 1000 +BP 0.0 298 auprc supra_input_output supra_output 0.12990810902139693 0.11621098601018635 0.01369712301121058 0.013664887690669848 0.00696176312081587 0.021482147832405648 0.0 True 1000 +BP 0.0 298 f_max supra_relative_output supra_relative 0.1991796963716254 0.1952356755425039 0.003944020829121486 0.0038344375836087048 -0.0021592790621820933 0.011459589703048022 0.254 False 1000 +BP 0.0 298 auprc supra_relative_output supra_relative 0.07435490655374574 0.061404243604493335 0.012950662949252408 0.012771719839159229 0.006864833891437334 0.019343914527648498 0.0 True 1000 +BP 0.0 298 f_max supra_relative_output supra_output 0.1991796963716254 0.239828172030419 -0.04064847565879362 -0.03989873546795027 -0.061119585597414795 -0.020761635199629314 0.0 True 1000 +BP 0.0 298 auprc supra_relative_output supra_output 0.07435490655374574 0.11621098601018635 -0.04185607945644061 -0.04111449559378811 -0.057691557693185805 -0.026085100033117054 0.0 True 1000 +BP 0.0 298 f_max full supra_input_relative 0.22006836976335684 0.21161694252967744 0.008451427233679393 0.006582734661019749 -0.000571319204928687 0.014161086132424266 0.084 False 1000 +BP 0.0 298 auprc full supra_input_relative 0.09730776842727502 0.09092431931044631 0.006383449116828713 0.006254845517784153 0.002319267531654941 0.010510179366012187 0.002 True 1000 +BP 0.0 298 f_max full supra_input_output 0.22006836976335684 0.24970556786024375 -0.02963719809688692 -0.029584102578644873 -0.046224120410455816 -0.012527315186327173 0.0 True 1000 +BP 0.0 298 auprc full supra_input_output 0.09730776842727502 0.12990810902139693 -0.03260034059412191 -0.032057890094679665 -0.04512549180850035 -0.019696079960702693 0.0 True 1000 +BP 0.0 298 f_max full supra_relative_output 0.22006836976335684 0.1991796963716254 0.02088867339173145 0.020738842757759977 0.008038911124643413 0.03379155472404789 0.0 True 1000 +BP 0.0 298 auprc full supra_relative_output 0.09730776842727502 0.07435490655374574 0.02295286187352928 0.022721493189778296 0.013827031255636467 0.03282277955393836 0.0 True 1000 +BP 0.0 298 f_max single naive 0.23462050056636874 0.24522800676034753 -0.010607506193978794 -0.01207334966119381 -0.045248082916899454 0.019672070148278632 0.458 False 1000 +BP 0.0 298 auprc single naive 0.10658278695025916 0.1548729085557252 -0.04829012160546603 -0.04869182932730692 -0.07390144099097183 -0.022964877873828966 0.002 True 1000 +BP 0.0 298 f_max supra naive 0.23389853138263522 0.24522800676034753 -0.011329475377712311 -0.0117298384218321 -0.04415140296522039 0.020166227408619612 0.468 False 1000 +BP 0.0 298 auprc supra naive 0.10862153438760377 0.1548729085557252 -0.04625137416812142 -0.0467015762126466 -0.0723785974080122 -0.01960161811215393 0.002 True 1000 +BP 0.0 298 f_max supra_input naive 0.24391948612898903 0.24522800676034753 -0.0013085206313585007 -0.0012036192245291342 -0.033982719531569096 0.029188704891475417 0.97 False 1000 +BP 0.0 298 auprc supra_input naive 0.12031130066802086 0.1548729085557252 -0.034561607887704335 -0.03493638983584021 -0.062260484289648026 -0.006935563080108801 0.02 True 1000 +BP 0.0 298 f_max supra_relative naive 0.1952356755425039 0.24522800676034753 -0.04999233121784363 -0.05072707009699382 -0.08269954828791473 -0.018918081512506284 0.004 True 1000 +BP 0.0 298 auprc supra_relative naive 0.061404243604493335 0.1548729085557252 -0.09346866495123185 -0.09319595897539304 -0.11250671216330309 -0.07268250606626617 0.0 True 1000 +BP 0.0 298 f_max supra_output naive 0.239828172030419 0.24522800676034753 -0.0053998347299285265 -0.006993897045434841 -0.03850614066400598 0.02215150435458913 0.64 False 1000 +BP 0.0 298 auprc supra_output naive 0.11621098601018635 0.1548729085557252 -0.03866192254553884 -0.03930974354244571 -0.0649278028386922 -0.012076870086416515 0.006 True 1000 +BP 0.0 298 f_max supra_input_relative naive 0.21161694252967744 0.24522800676034753 -0.03361106423067009 -0.032736524416644895 -0.06389273652597405 -0.0038032450089797235 0.036 True 1000 +BP 0.0 298 auprc supra_input_relative naive 0.09092431931044631 0.1548729085557252 -0.06394858924527888 -0.06395759146423967 -0.08720468157476276 -0.03999886782200505 0.0 True 1000 +BP 0.0 298 f_max supra_input_output naive 0.24970556786024375 0.24522800676034753 0.004477561099896221 0.003430312823019729 -0.02887964408104752 0.0316404904891881 0.786 False 1000 +BP 0.0 298 auprc supra_input_output naive 0.12990810902139693 0.1548729085557252 -0.024964799534328258 -0.02564485585177586 -0.052735995158223005 0.0018452048731169987 0.072 False 1000 +BP 0.0 298 f_max supra_relative_output naive 0.1991796963716254 0.24522800676034753 -0.04604831038872215 -0.04689263251338512 -0.07963004253463953 -0.015850488139141777 0.008 True 1000 +BP 0.0 298 auprc supra_relative_output naive 0.07435490655374574 0.1548729085557252 -0.08051800200197945 -0.08042423913623381 -0.10223806254646237 -0.05769135670281216 0.0 True 1000 +BP 0.0 298 f_max full naive 0.22006836976335684 0.24522800676034753 -0.025159636996990697 -0.026153789755625146 -0.0579481952148181 0.004524319884199825 0.084 False 1000 +BP 0.0 298 auprc full naive 0.09730776842727502 0.1548729085557252 -0.05756514012845017 -0.057702745946455525 -0.08132968497386052 -0.0334796812543227 0.0 True 1000 +BP 0.0 298 f_max full single 0.22006836976335684 0.23462050056636874 -0.014552130803011903 -0.014080440094431332 -0.030474256358575928 0.0006239209902096847 0.06 False 1000 +BP 0.0 298 auprc full single 0.09730776842727502 0.10658278695025916 -0.009275018522984138 -0.009010916619148601 -0.02051057590211293 0.002060249684491816 0.118 False 1000 +BP 2.0 298 f_max supra single 0.1761580707594129 0.1734134360870587 0.002744634672354196 0.002058030074584976 -0.0030026075177524117 0.0068038907664457506 0.402 False 1000 +BP 2.0 298 auprc supra single 0.07180857194540718 0.068340971805973 0.0034676001394341877 0.003430995129523339 0.0003261733643463293 0.007090847010860049 0.028 True 1000 +BP 2.0 298 f_max supra_input supra 0.19299661151865918 0.1761580707594129 0.01683854075924629 0.017223183647897506 0.0025789874590079523 0.03418359147181683 0.018 True 1000 +BP 2.0 298 auprc supra_input supra 0.07592523255095349 0.07180857194540718 0.004116660605546302 0.0038765056922207513 -0.0068130136731560835 0.014660690005480106 0.528 False 1000 +BP 2.0 298 f_max supra_relative supra 0.13995283133552622 0.1761580707594129 -0.036205239423886676 -0.037148601193855296 -0.05573726351993497 -0.018731022715412483 0.0 True 1000 +BP 2.0 298 auprc supra_relative supra 0.032853065552072104 0.07180857194540718 -0.03895550639333508 -0.03911677851306379 -0.05563389113823287 -0.024291086257482507 0.0 True 1000 +BP 2.0 298 f_max supra_output supra 0.1809761204212188 0.1761580707594129 0.0048180496618059065 0.004309894013264439 -0.005983770225026678 0.015309323970500194 0.424 False 1000 +BP 2.0 298 auprc supra_output supra 0.06817650955107903 0.07180857194540718 -0.003632062394328159 -0.00390573735343587 -0.013490728312356306 0.005085158018965634 0.418 False 1000 +BP 2.0 298 f_max supra_input_relative supra_input 0.1590048692983248 0.19299661151865918 -0.03399174222033438 -0.03460741098495101 -0.05480525390773876 -0.01681975380459589 0.0 True 1000 +BP 2.0 298 auprc supra_input_relative supra_input 0.053680719886573844 0.07592523255095349 -0.022244512664379643 -0.022194678267559893 -0.03472398273138919 -0.010637183181845674 0.002 True 1000 +BP 2.0 298 f_max supra_input_relative supra_relative 0.1590048692983248 0.13995283133552622 0.019052037962798585 0.019764373856801794 0.007144995166742613 0.03327557765727536 0.002 True 1000 +BP 2.0 298 auprc supra_input_relative supra_relative 0.053680719886573844 0.032853065552072104 0.02082765433450174 0.020798605937724646 0.01160166326057508 0.031597723809944245 0.0 True 1000 +BP 2.0 298 f_max supra_input_output supra_input 0.19290019525631774 0.19299661151865918 -9.641626234144218e-05 -0.0007128144976238885 -0.007995223212353929 0.005761273139390601 0.864 False 1000 +BP 2.0 298 auprc supra_input_output supra_input 0.08150147032223701 0.07592523255095349 0.005576237771283521 0.005472880027713661 0.0010986937118644304 0.010079470580361788 0.012 True 1000 +BP 2.0 298 f_max supra_input_output supra_output 0.19290019525631774 0.1809761204212188 0.011924074835098941 0.012200475137009177 0.0033369363537440115 0.02283589986729254 0.006 True 1000 +BP 2.0 298 auprc supra_input_output supra_output 0.08150147032223701 0.06817650955107903 0.013324960771157981 0.013255123073370281 0.007018451033404622 0.020600460414056637 0.0 True 1000 +BP 2.0 298 f_max supra_relative_output supra_relative 0.14267615352407817 0.13995283133552622 0.002723322188551952 0.0032970760154833467 -3.530927798193592e-06 0.007916828693808968 0.06 False 1000 +BP 2.0 298 auprc supra_relative_output supra_relative 0.036862518281928124 0.032853065552072104 0.0040094527298560195 0.004085843465542958 -0.0015990170479268156 0.010848267238882534 0.186 False 1000 +BP 2.0 298 f_max supra_relative_output supra_output 0.14267615352407817 0.1809761204212188 -0.03829996689714063 -0.03816141919163639 -0.05920547624660199 -0.018044791455781098 0.0 True 1000 +BP 2.0 298 auprc supra_relative_output supra_output 0.036862518281928124 0.06817650955107903 -0.0313139912691509 -0.031125197694084963 -0.0453801268874299 -0.017913125005490298 0.0 True 1000 +BP 2.0 298 f_max full supra_input_relative 0.15848590904470627 0.1590048692983248 -0.0005189602536185278 0.00011138601785305076 -0.0039043646122028327 0.0045218716637194884 0.998 False 1000 +BP 2.0 298 auprc full supra_input_relative 0.054411779329926364 0.053680719886573844 0.0007310594433525205 0.0007011706526396966 -0.003686833074450748 0.005374165171289749 0.786 False 1000 +BP 2.0 298 f_max full supra_input_output 0.15848590904470627 0.19290019525631774 -0.03441428621161147 -0.03378321046947407 -0.051931067614967755 -0.016860050984098332 0.0 True 1000 +BP 2.0 298 auprc full supra_input_output 0.054411779329926364 0.08150147032223701 -0.027089690992310643 -0.02696638764263386 -0.03973913734766223 -0.015257377359431784 0.0 True 1000 +BP 2.0 298 f_max full supra_relative_output 0.15848590904470627 0.14267615352407817 0.015809755520628105 0.016578683859171498 0.003063413247895666 0.029930622270298576 0.012 True 1000 +BP 2.0 298 auprc full supra_relative_output 0.054411779329926364 0.036862518281928124 0.01754926104799824 0.017413933124821382 0.010542219900037085 0.026153660401111185 0.0 True 1000 +BP 2.0 298 f_max single naive 0.1734134360870587 0.1393754232163564 0.034038012870702294 0.03543625393825642 0.0020558852524578597 0.06944325547527005 0.04 True 1000 +BP 2.0 298 auprc single naive 0.068340971805973 0.05246757199300765 0.015873399812965344 0.016584728780819718 -0.006185742085294511 0.04181264020515145 0.158 False 1000 +BP 2.0 298 f_max supra naive 0.1761580707594129 0.1393754232163564 0.03678264754305649 0.0374942840128414 0.003572279441906474 0.07038811689656488 0.016 True 1000 +BP 2.0 298 auprc supra naive 0.07180857194540718 0.05246757199300765 0.019340999952399532 0.02001572391034306 -0.002866718331589628 0.045626606085612656 0.088 False 1000 +BP 2.0 298 f_max supra_input naive 0.19299661151865918 0.1393754232163564 0.05362118830230278 0.05471746766073891 0.022135031885265725 0.08627169133734161 0.0 True 1000 +BP 2.0 298 auprc supra_input naive 0.07592523255095349 0.05246757199300765 0.023457660557945834 0.02389222960256381 0.0009757096995311865 0.048571600648820476 0.038 True 1000 +BP 2.0 298 f_max supra_relative naive 0.13995283133552622 0.1393754232163564 0.0005774081191698144 0.00034568281898610353 -0.02925261865653464 0.032004355313295436 0.994 False 1000 +BP 2.0 298 auprc supra_relative naive 0.032853065552072104 0.05246757199300765 -0.01961450644093555 -0.019101054602720734 -0.032640643301136814 -0.0026276115458591007 0.026 True 1000 +BP 2.0 298 f_max supra_output naive 0.1809761204212188 0.1393754232163564 0.0416006972048624 0.04180417802610584 0.010632458771203314 0.07334940448382683 0.006 True 1000 +BP 2.0 298 auprc supra_output naive 0.06817650955107903 0.05246757199300765 0.015708937558071373 0.016109986556907187 -0.004868246329802568 0.039765206829826344 0.138 False 1000 +BP 2.0 298 f_max supra_input_relative naive 0.1590048692983248 0.1393754232163564 0.0196294460819684 0.020110056675787896 -0.008078115045903682 0.04777211365537507 0.174 False 1000 +BP 2.0 298 auprc supra_input_relative naive 0.053680719886573844 0.05246757199300765 0.001213147893566191 0.0016975513350039116 -0.016485197413204224 0.022387674714999306 0.898 False 1000 +BP 2.0 298 f_max supra_input_output naive 0.19290019525631774 0.1393754232163564 0.05352477203996134 0.054004653163115014 0.022652118704430547 0.08508493040191945 0.0 True 1000 +BP 2.0 298 auprc supra_input_output naive 0.08150147032223701 0.05246757199300765 0.029033898329229355 0.029365109630277473 0.006736279065748549 0.05489584105628164 0.012 True 1000 +BP 2.0 298 f_max supra_relative_output naive 0.14267615352407817 0.1393754232163564 0.0033007303077217665 0.003642758834469451 -0.02582764942456395 0.035315463624804956 0.834 False 1000 +BP 2.0 298 auprc supra_relative_output naive 0.036862518281928124 0.05246757199300765 -0.01560505371107953 -0.015015211137177775 -0.03039116445508186 0.0036191769298985084 0.094 False 1000 +BP 2.0 298 f_max full naive 0.15848590904470627 0.1393754232163564 0.01911048582834987 0.020221442693640946 -0.007475561179603967 0.0477790698638569 0.166 False 1000 +BP 2.0 298 auprc full naive 0.054411779329926364 0.05246757199300765 0.0019442073369187116 0.0023987219876436083 -0.015880062867324893 0.022755599715098957 0.836 False 1000 +BP 2.0 298 f_max full single 0.15848590904470627 0.1734134360870587 -0.014927527042352423 -0.015214811244615475 -0.03166057030890665 0.0007507233750318926 0.072 False 1000 +BP 2.0 298 auprc full single 0.054411779329926364 0.068340971805973 -0.013929192476046633 -0.01418600679317611 -0.02921786894290298 -0.0007561701231903708 0.044 True 1000 +BP 4.0 292 f_max supra single 0.11805494385249392 0.1131630709975688 0.004891872854925122 0.004043935480742157 -0.0014643856018696012 0.01010120275513805 0.15 False 1000 +BP 4.0 292 auprc supra single 0.03275518908553014 0.029164069258372262 0.0035911198271578806 0.0035971380179923404 0.0009492849137604391 0.007312780343891462 0.006 True 1000 +BP 4.0 292 f_max supra_input supra 0.13785042940917663 0.11805494385249392 0.019795485556682715 0.018409772336432057 -0.002302600755659344 0.039304154789247814 0.078 False 1000 +BP 4.0 292 auprc supra_input supra 0.036198295212282634 0.03275518908553014 0.0034431061267524918 0.0031976088703763324 -0.005310448556871802 0.011893520407599785 0.442 False 1000 +BP 4.0 292 f_max supra_relative supra 0.08498026914465588 0.11805494385249392 -0.03307467470783804 -0.033706862124407114 -0.05567106740881214 -0.011483338145295938 0.002 True 1000 +BP 4.0 292 auprc supra_relative supra 0.01356999959646461 0.03275518908553014 -0.019185189489065533 -0.019704656298723377 -0.03381072255178606 -0.008559522645506495 0.0 True 1000 +BP 4.0 292 f_max supra_output supra 0.1187927440433954 0.11805494385249392 0.0007378001909014859 -0.0008000554063709318 -0.014014387075907384 0.01188503537618609 0.906 False 1000 +BP 4.0 292 auprc supra_output supra 0.026632418405861672 0.03275518908553014 -0.00612277067966847 -0.006488467756926049 -0.014331856211883109 -0.0005026668094502149 0.022 True 1000 +BP 4.0 292 f_max supra_input_relative supra_input 0.093217666191155 0.13785042940917663 -0.044632763218021634 -0.04466993870339097 -0.0715546282544661 -0.02164547898135812 0.0 True 1000 +BP 4.0 292 auprc supra_input_relative supra_input 0.019850976304688597 0.036198295212282634 -0.016347318907594038 -0.016505233806502075 -0.028556564135929042 -0.006044400471244032 0.0 True 1000 +BP 4.0 292 f_max supra_input_relative supra_relative 0.093217666191155 0.08498026914465588 0.008237397046499118 0.007446695757448202 -0.004937688004248916 0.0217489760841748 0.248 False 1000 +BP 4.0 292 auprc supra_input_relative supra_relative 0.019850976304688597 0.01356999959646461 0.006280976708223987 0.006397031362597638 0.0007178653836974833 0.013472014075057133 0.026 True 1000 +BP 4.0 292 f_max supra_input_output supra_input 0.131901838297952 0.13785042940917663 -0.005948591111224644 -0.002432526293421466 -0.011844251358373905 0.0048410382294467825 0.608 False 1000 +BP 4.0 292 auprc supra_input_output supra_input 0.03637360789031955 0.036198295212282634 0.0001753126780369152 7.13344615847374e-05 -0.003134744695512378 0.0032354463368269057 0.962 False 1000 +BP 4.0 292 f_max supra_input_output supra_output 0.131901838297952 0.1187927440433954 0.013109094254556586 0.016777301449381523 0.0030367924806562253 0.03509101368290395 0.01 True 1000 +BP 4.0 292 auprc supra_input_output supra_output 0.03637360789031955 0.026632418405861672 0.009741189484457877 0.009757411088887119 0.005000327237619708 0.015910221227788113 0.0 True 1000 +BP 4.0 292 f_max supra_relative_output supra_relative 0.08779686723124253 0.08498026914465588 0.002816598086586644 0.0013023420487635695 -0.003789631534842868 0.004803928161130728 0.444 False 1000 +BP 4.0 292 auprc supra_relative_output supra_relative 0.01300225721216053 0.01356999959646461 -0.0005677423843040794 -0.0005834315746593714 -0.004993774061889593 0.0032553849527059576 0.75 False 1000 +BP 4.0 292 f_max supra_relative_output supra_output 0.08779686723124253 0.1187927440433954 -0.03099587681215288 -0.031604464669272615 -0.057109016963001226 -0.00811136997899744 0.012 True 1000 +BP 4.0 292 auprc supra_relative_output supra_output 0.01300225721216053 0.026632418405861672 -0.013630161193701142 -0.0137996201164567 -0.024117352421973264 -0.004169160887231303 0.002 True 1000 +BP 4.0 292 f_max full supra_input_relative 0.09090782814931803 0.093217666191155 -0.002309838041836973 -0.0016015532833554762 -0.004465857174607178 0.0020825038503366686 0.294 False 1000 +BP 4.0 292 auprc full supra_input_relative 0.02187984746106082 0.019850976304688597 0.0020288711563722223 0.0019076836544542005 -0.002255236405264613 0.006354903086669125 0.36 False 1000 +BP 4.0 292 f_max full supra_input_output 0.09090782814931803 0.131901838297952 -0.040994010148633964 -0.04383896569332498 -0.07118524313365686 -0.02160927889136258 0.0 True 1000 +BP 4.0 292 auprc full supra_input_output 0.02187984746106082 0.03637360789031955 -0.01449376042925873 -0.014668884613632612 -0.02661922695803201 -0.004765628991718173 0.006 True 1000 +BP 4.0 292 f_max full supra_relative_output 0.09090782814931803 0.08779686723124253 0.0031109609180755005 0.004542800425329157 -0.007638788294187234 0.018279891177072304 0.474 False 1000 +BP 4.0 292 auprc full supra_relative_output 0.02187984746106082 0.01300225721216053 0.008877590248900289 0.008888146591711209 0.003153909827663134 0.01639224254507381 0.0 True 1000 +BP 4.0 292 f_max single naive 0.1131630709975688 0.04686917580427525 0.06629389519329354 0.06906891089459102 0.039662025505015866 0.10249065012445327 0.0 True 1000 +BP 4.0 292 auprc single naive 0.029164069258372262 0.013722538414023608 0.015441530844348654 0.01659135162806297 0.0031350557176041344 0.03466653295996757 0.01 True 1000 +BP 4.0 292 f_max supra naive 0.11805494385249392 0.04686917580427525 0.07118576804821866 0.07311284637533316 0.04271751560065043 0.10681551737763803 0.0 True 1000 +BP 4.0 292 auprc supra naive 0.03275518908553014 0.013722538414023608 0.019032650671506536 0.020188489646055314 0.005794764280039898 0.040062999424890826 0.002 True 1000 +BP 4.0 292 f_max supra_input naive 0.13785042940917663 0.04686917580427525 0.09098125360490139 0.09152261871176523 0.06161285316992385 0.12284024859566449 0.0 True 1000 +BP 4.0 292 auprc supra_input naive 0.036198295212282634 0.013722538414023608 0.022475756798259028 0.023386098516431645 0.009519127954514154 0.040325221675897666 0.0 True 1000 +BP 4.0 292 f_max supra_relative naive 0.08498026914465588 0.04686917580427525 0.03811109334038063 0.039405984250926056 0.013490899055842503 0.06764546542060777 0.002 True 1000 +BP 4.0 292 auprc supra_relative naive 0.01356999959646461 0.013722538414023608 -0.0001525388175589986 0.00048383334733193077 -0.006961529330375357 0.010689520533492924 0.99 False 1000 +BP 4.0 292 f_max supra_output naive 0.1187927440433954 0.04686917580427525 0.07192356823912016 0.07231279096896225 0.042993290440377516 0.10421039719908823 0.0 True 1000 +BP 4.0 292 auprc supra_output naive 0.026632418405861672 0.013722538414023608 0.012909879991838064 0.013700021889129261 0.0020469843309990922 0.028914292887716343 0.02 True 1000 +BP 4.0 292 f_max supra_input_relative naive 0.093217666191155 0.04686917580427525 0.04634849038687975 0.04685268000837427 0.02298360202542272 0.07360302202307137 0.0 True 1000 +BP 4.0 292 auprc supra_input_relative naive 0.019850976304688597 0.013722538414023608 0.006128437890664988 0.006880864709929569 -0.0029820997752110496 0.02013269591157942 0.244 False 1000 +BP 4.0 292 f_max supra_input_output naive 0.131901838297952 0.04686917580427525 0.08503266249367675 0.08909009241834376 0.05995215769479169 0.12111771386818812 0.0 True 1000 +BP 4.0 292 auprc supra_input_output naive 0.03637360789031955 0.013722538414023608 0.022651069476295943 0.023457432978016377 0.009537753823335274 0.04019339108578706 0.0 True 1000 +BP 4.0 292 f_max supra_relative_output naive 0.08779686723124253 0.04686917580427525 0.040927691426967276 0.04070832629968963 0.015501498959211131 0.0686266479637726 0.0 True 1000 +BP 4.0 292 auprc supra_relative_output naive 0.01300225721216053 0.013722538414023608 -0.000720281201863078 -9.959822732744068e-05 -0.008004705228691338 0.010226727177042287 0.892 False 1000 +BP 4.0 292 f_max full naive 0.09090782814931803 0.04686917580427525 0.04403865234504278 0.04525112672501878 0.021966240615700884 0.07133861950880124 0.0 True 1000 +BP 4.0 292 auprc full naive 0.02187984746106082 0.013722538414023608 0.00815730904703721 0.008788548364383768 -0.001982865318055957 0.02329396560109466 0.158 False 1000 +BP 4.0 292 f_max full single 0.09090782814931803 0.1131630709975688 -0.02225524284825077 -0.023817784169572233 -0.04582144459873754 -0.0034312417966460883 0.022 True 1000 +BP 4.0 292 auprc full single 0.02187984746106082 0.029164069258372262 -0.007284221797311443 -0.007802803263679202 -0.018916509413035496 0.0029481943345696384 0.138 False 1000 +MF 0.0 406 f_max supra single 0.34917244942038655 0.36126219819528144 -0.012089748774894882 -0.011024020007172872 -0.0223399784747439 -0.0009776604017022004 0.036 True 1000 +MF 0.0 406 auprc supra single 0.17863188024464952 0.18039683004408205 -0.0017649497994325358 -0.0016919980791168102 -0.006294938667759356 0.0027334152442445012 0.472 False 1000 +MF 0.0 406 f_max supra_input supra 0.3655787927448233 0.34917244942038655 0.016406343324436745 0.016225480200265204 -0.004045362088432439 0.036247630292812735 0.086 False 1000 +MF 0.0 406 auprc supra_input supra 0.17602961321469285 0.17863188024464952 -0.0026022670299566675 -0.002682031369815454 -0.01355996166701836 0.007960781644288772 0.604 False 1000 +MF 0.0 406 f_max supra_relative supra 0.34023526115113495 0.34917244942038655 -0.008937188269251606 -0.007887597645711732 -0.03316308274618768 0.016997587622973154 0.556 False 1000 +MF 0.0 406 auprc supra_relative supra 0.1595380439620823 0.17863188024464952 -0.019093836282567223 -0.019475435525263277 -0.040293498905759846 0.0021286377157843867 0.068 False 1000 +MF 0.0 406 f_max supra_output supra 0.358212635059794 0.34917244942038655 0.00904018563940745 0.00930678877351723 -0.008232362783033034 0.026922914321791513 0.296 False 1000 +MF 0.0 406 auprc supra_output supra 0.18606858135827942 0.17863188024464952 0.007436701113629901 0.007399058956728037 -0.00224230854086337 0.017187552094308396 0.164 False 1000 +MF 0.0 406 f_max supra_input_relative supra_input 0.36075231398177715 0.3655787927448233 -0.004826478763046149 -0.0030100912136301564 -0.028228656444542127 0.019838050928130152 0.828 False 1000 +MF 0.0 406 auprc supra_input_relative supra_input 0.19859411226663576 0.17602961321469285 0.02256449905194291 0.02231283097974575 0.008226482172920636 0.03648342380688801 0.0 True 1000 +MF 0.0 406 f_max supra_input_relative supra_relative 0.36075231398177715 0.34023526115113495 0.020517052830642202 0.02110298663234678 -0.005889301159445742 0.04612671881093128 0.142 False 1000 +MF 0.0 406 auprc supra_input_relative supra_relative 0.19859411226663576 0.1595380439620823 0.039056068304553465 0.039106235135193575 0.01836676196556481 0.05988908387208697 0.0 True 1000 +MF 0.0 406 f_max supra_input_output supra_input 0.3712904130952929 0.3655787927448233 0.005711620350469626 0.006094560187802294 -0.0066542453516503595 0.01802808030274884 0.324 False 1000 +MF 0.0 406 auprc supra_input_output supra_input 0.19631964007936464 0.17602961321469285 0.020290026864671795 0.02026476140009038 0.01435455248648081 0.026026844072638528 0.0 True 1000 +MF 0.0 406 f_max supra_input_output supra_output 0.3712904130952929 0.358212635059794 0.013077778035498921 0.013013251614550268 0.0010983924853001802 0.025549856411920185 0.024 True 1000 +MF 0.0 406 auprc supra_input_output supra_output 0.19631964007936464 0.18606858135827942 0.010251058721085227 0.010183671073546886 0.002769229186503847 0.01783002399853372 0.014 True 1000 +MF 0.0 406 f_max supra_relative_output supra_relative 0.33622861513771435 0.34023526115113495 -0.0040066460134206006 0.0010603502823384732 -0.015156131552054303 0.024172740940650635 0.988 False 1000 +MF 0.0 406 auprc supra_relative_output supra_relative 0.18553875154271193 0.1595380439620823 0.02600070758062964 0.026320345539209245 0.015474325519255702 0.03749536833861547 0.0 True 1000 +MF 0.0 406 f_max supra_relative_output supra_output 0.33622861513771435 0.358212635059794 -0.021984019922079656 -0.01613403613689049 -0.042258036446856446 0.009480307101133064 0.232 False 1000 +MF 0.0 406 auprc supra_relative_output supra_output 0.18553875154271193 0.18606858135827942 -0.0005298298155674841 -0.0005541489427820683 -0.0221942925785212 0.022672353076614617 0.942 False 1000 +MF 0.0 406 f_max full supra_input_relative 0.38274701944288786 0.36075231398177715 0.02199470546111071 0.02076456046062245 0.009421524224084734 0.03268165867969507 0.0 True 1000 +MF 0.0 406 auprc full supra_input_relative 0.21842592506441955 0.19859411226663576 0.019831812797783788 0.01987323774334714 0.012178080351527603 0.027476216005071406 0.0 True 1000 +MF 0.0 406 f_max full supra_input_output 0.38274701944288786 0.3712904130952929 0.011456606347594933 0.011659909059190003 -0.009173196000782983 0.03186282486146224 0.276 False 1000 +MF 0.0 406 auprc full supra_input_output 0.21842592506441955 0.19631964007936464 0.022106284985054903 0.021921307323002515 0.0074921028373116216 0.037123464134071894 0.002 True 1000 +MF 0.0 406 f_max full supra_relative_output 0.38274701944288786 0.33622861513771435 0.04651840430517351 0.04080719681063076 0.01889748060846154 0.06193743717909492 0.0 True 1000 +MF 0.0 406 auprc full supra_relative_output 0.21842592506441955 0.18553875154271193 0.03288717352170761 0.03265912733933147 0.015227889350660426 0.051137180738186884 0.0 True 1000 +MF 0.0 406 f_max single naive 0.36126219819528144 0.7189989169653068 -0.3577367187700254 -0.35803413301163173 -0.41524212414383 -0.3015192015866498 0.0 True 1000 +MF 0.0 406 auprc single naive 0.18039683004408205 0.4190445836593093 -0.23864775361522725 -0.23781991898198132 -0.29422729084282406 -0.18679147093122503 0.0 True 1000 +MF 0.0 406 f_max supra naive 0.34917244942038655 0.7189989169653068 -0.36982646754492027 -0.36905815301880457 -0.42388794990895273 -0.3131307375622247 0.0 True 1000 +MF 0.0 406 auprc supra naive 0.17863188024464952 0.4190445836593093 -0.24041270341465978 -0.23951191706109815 -0.2957903930983861 -0.1866890685742241 0.0 True 1000 +MF 0.0 406 f_max supra_input naive 0.3655787927448233 0.7189989169653068 -0.3534201242204835 -0.3528326728185394 -0.4095635267191234 -0.30028044088884426 0.0 True 1000 +MF 0.0 406 auprc supra_input naive 0.17602961321469285 0.4190445836593093 -0.24301497044461645 -0.24219394843091357 -0.2982143991656506 -0.18904165023770206 0.0 True 1000 +MF 0.0 406 f_max supra_relative naive 0.34023526115113495 0.7189989169653068 -0.3787636558141719 -0.37694575066451635 -0.42608919368187376 -0.32343574814730475 0.0 True 1000 +MF 0.0 406 auprc supra_relative naive 0.1595380439620823 0.4190445836593093 -0.25950653969722703 -0.2589873525863614 -0.3050714548144409 -0.21303561367481666 0.0 True 1000 +MF 0.0 406 f_max supra_output naive 0.358212635059794 0.7189989169653068 -0.3607862819055128 -0.35975136424528736 -0.4126021467855058 -0.30652917950726183 0.0 True 1000 +MF 0.0 406 auprc supra_output naive 0.18606858135827942 0.4190445836593093 -0.23297600230102988 -0.2321128581043701 -0.2858990634783965 -0.18181405057819344 0.0 True 1000 +MF 0.0 406 f_max supra_input_relative naive 0.36075231398177715 0.7189989169653068 -0.35824660298352967 -0.35584276403216963 -0.4087929706246481 -0.3031495273714159 0.0 True 1000 +MF 0.0 406 auprc supra_input_relative naive 0.19859411226663576 0.4190445836593093 -0.22045047139267354 -0.2198811174511678 -0.2741408431959568 -0.16731566743603227 0.0 True 1000 +MF 0.0 406 f_max supra_input_output naive 0.3712904130952929 0.7189989169653068 -0.3477085038700139 -0.34673811263073706 -0.40127915051394836 -0.2926561073418046 0.0 True 1000 +MF 0.0 406 auprc supra_input_output naive 0.19631964007936464 0.4190445836593093 -0.22272494357994466 -0.22192918703082318 -0.2774423072658909 -0.1678343015489985 0.0 True 1000 +MF 0.0 406 f_max supra_relative_output naive 0.33622861513771435 0.7189989169653068 -0.3827703018275925 -0.3758854003821779 -0.42525711826549994 -0.3246626726785401 0.0 True 1000 +MF 0.0 406 auprc supra_relative_output naive 0.18553875154271193 0.4190445836593093 -0.23350583211659737 -0.23266700704715215 -0.2812576050841544 -0.18451173701570922 0.0 True 1000 +MF 0.0 406 f_max full naive 0.38274701944288786 0.7189989169653068 -0.33625189752241896 -0.33507820357154705 -0.3882533377789176 -0.2852670689569053 0.0 True 1000 +MF 0.0 406 auprc full naive 0.21842592506441955 0.4190445836593093 -0.20061865859488975 -0.20000787970782072 -0.2527427821167058 -0.14764680947666078 0.0 True 1000 +MF 0.0 406 f_max full single 0.38274701944288786 0.36126219819528144 0.021484821247606423 0.022955929440084632 -0.0010660936112724866 0.04921172838624425 0.056 False 1000 +MF 0.0 406 auprc full single 0.21842592506441955 0.18039683004408205 0.038029095020337494 0.03781203927416062 0.022259791384393542 0.053982835982070244 0.0 True 1000 +MF 2.0 159 f_max supra single 0.3575134302039365 0.3624206773738544 -0.0049072471699179165 -0.003146467756524326 -0.010823017480037493 0.0038774451603761665 0.404 False 1000 +MF 2.0 159 auprc supra single 0.20961207022311829 0.2009778910151442 0.008634179207974074 0.008871246268385678 2.458137758324666e-06 0.019853974574351712 0.05 True 1000 +MF 2.0 159 f_max supra_input supra 0.35670234664936507 0.3575134302039365 -0.0008110835545714346 -0.0029227012831920126 -0.024816907599871995 0.018356137164855398 0.806 False 1000 +MF 2.0 159 auprc supra_input supra 0.21789011079771334 0.20961207022311829 0.008278040574595052 0.008439418969589664 -0.010934020944791121 0.03081539742461346 0.432 False 1000 +MF 2.0 159 f_max supra_relative supra 0.26640481629856816 0.3575134302039365 -0.09110861390536834 -0.0891178728108523 -0.12412393254305235 -0.05200464463883508 0.0 True 1000 +MF 2.0 159 auprc supra_relative supra 0.09810165326207468 0.20961207022311829 -0.1115104169610436 -0.11014417853749049 -0.14573104492176275 -0.07781252296714124 0.0 True 1000 +MF 2.0 159 f_max supra_output supra 0.3366341487828252 0.3575134302039365 -0.020879281421111284 -0.02121215849773531 -0.04298020579118116 -0.00013137254848641873 0.046 True 1000 +MF 2.0 159 auprc supra_output supra 0.2042822117811935 0.20961207022311829 -0.005329858441924795 -0.004683196915552008 -0.023223340094816088 0.014879011475610892 0.644 False 1000 +MF 2.0 159 f_max supra_input_relative supra_input 0.30838965519130074 0.35670234664936507 -0.04831269145806433 -0.04423823323513094 -0.06658506175330159 -0.021379957166021278 0.0 True 1000 +MF 2.0 159 auprc supra_input_relative supra_input 0.1712713104376412 0.21789011079771334 -0.04661880036007213 -0.04651354015810965 -0.07151372686624685 -0.020435136555937955 0.0 True 1000 +MF 2.0 159 f_max supra_input_relative supra_relative 0.30838965519130074 0.26640481629856816 0.04198483889273258 0.04195693829252935 0.0115837754113257 0.07584671863389951 0.01 True 1000 +MF 2.0 159 auprc supra_input_relative supra_relative 0.1712713104376412 0.09810165326207468 0.07316965717556653 0.07207005734897053 0.04329017727091993 0.1044073075168474 0.0 True 1000 +MF 2.0 159 f_max supra_input_output supra_input 0.3509584986613415 0.35670234664936507 -0.005743847988023576 -0.005567428049498963 -0.015300067595330702 0.003579273230036474 0.228 False 1000 +MF 2.0 159 auprc supra_input_output supra_input 0.2220098622274635 0.21789011079771334 0.004119751429750157 0.004290315349679475 -0.0053107819018041605 0.01580696473967421 0.43 False 1000 +MF 2.0 159 f_max supra_input_output supra_output 0.3509584986613415 0.3366341487828252 0.014324349878516274 0.012722029165044333 -0.0005224156463774607 0.025461322380217824 0.062 False 1000 +MF 2.0 159 auprc supra_input_output supra_output 0.2220098622274635 0.2042822117811935 0.017727650446270005 0.017412931234821148 0.007535784282048407 0.028958120072461763 0.0 True 1000 +MF 2.0 159 f_max supra_relative_output supra_relative 0.28055796784500686 0.26640481629856816 0.014153151546438703 0.00929127261059117 -0.0033669030692162994 0.022758729811605 0.154 False 1000 +MF 2.0 159 auprc supra_relative_output supra_relative 0.11226843435692951 0.09810165326207468 0.014166781094854836 0.014227222597380513 0.0008733775313473317 0.02959013245715563 0.046 True 1000 +MF 2.0 159 f_max supra_relative_output supra_output 0.28055796784500686 0.3366341487828252 -0.056076180937818354 -0.05861444170252581 -0.08790635359795904 -0.027145292581537278 0.002 True 1000 +MF 2.0 159 auprc supra_relative_output supra_output 0.11226843435692951 0.2042822117811935 -0.09201377742426398 -0.09123375902455796 -0.12349113479753845 -0.05943527885289197 0.0 True 1000 +MF 2.0 159 f_max full supra_input_relative 0.31749948602842004 0.30838965519130074 0.009109830837119304 0.006183085836361725 -0.007263240009623845 0.01867462642659407 0.344 False 1000 +MF 2.0 159 auprc full supra_input_relative 0.16934947196131758 0.1712713104376412 -0.0019218384763236285 -0.0014591693981855299 -0.01471579063905024 0.01130645296589727 0.818 False 1000 +MF 2.0 159 f_max full supra_input_output 0.31749948602842004 0.3509584986613415 -0.03345901263292145 -0.03248771934927025 -0.05594954910023687 -0.008020466220922801 0.006 True 1000 +MF 2.0 159 auprc full supra_input_output 0.16934947196131758 0.2220098622274635 -0.05266039026614591 -0.05226302490597465 -0.07336513775770617 -0.03222741692158324 0.0 True 1000 +MF 2.0 159 f_max full supra_relative_output 0.31749948602842004 0.28055796784500686 0.03694151818341318 0.03884875151829989 0.00857743583453706 0.06881012659021303 0.01 True 1000 +MF 2.0 159 auprc full supra_relative_output 0.16934947196131758 0.11226843435692951 0.05708103760438807 0.056383665353404465 0.030288886492469837 0.08339675665844627 0.0 True 1000 +MF 2.0 159 f_max single naive 0.3624206773738544 0.07561162501367721 0.2868090523601772 0.28640137492382656 0.22293562963000751 0.3507641092182427 0.0 True 1000 +MF 2.0 159 auprc single naive 0.2009778910151442 0.026528609897190324 0.17444928111795388 0.17440752958795697 0.12234591466911383 0.23078903558683264 0.0 True 1000 +MF 2.0 159 f_max supra naive 0.3575134302039365 0.07561162501367721 0.2819018051902593 0.2832549071673022 0.22100546397849447 0.34474864269193417 0.0 True 1000 +MF 2.0 159 auprc supra naive 0.20961207022311829 0.026528609897190324 0.18308346032592795 0.18327877585634264 0.129592697862371 0.24296934364419934 0.0 True 1000 +MF 2.0 159 f_max supra_input naive 0.35670234664936507 0.07561162501367721 0.2810907216356878 0.2803322058841102 0.22013501843075176 0.3420735738749051 0.0 True 1000 +MF 2.0 159 auprc supra_input naive 0.21789011079771334 0.026528609897190324 0.191361500900523 0.1917181948259323 0.13778865290195466 0.25029171594777955 0.0 True 1000 +MF 2.0 159 f_max supra_relative naive 0.26640481629856816 0.07561162501367721 0.19079319128489095 0.1941370343564499 0.14473060493397438 0.24623761389374163 0.0 True 1000 +MF 2.0 159 auprc supra_relative naive 0.09810165326207468 0.026528609897190324 0.07157304336488435 0.07313459731885215 0.03762032598845119 0.11532107347483399 0.0 True 1000 +MF 2.0 159 f_max supra_output naive 0.3366341487828252 0.07561162501367721 0.26102252376914803 0.2620427486695669 0.20732597694598892 0.32093047672550373 0.0 True 1000 +MF 2.0 159 auprc supra_output naive 0.2042822117811935 0.026528609897190324 0.17775360188400316 0.17859557894079064 0.12402529504765586 0.23764651544768733 0.0 True 1000 +MF 2.0 159 f_max supra_input_relative naive 0.30838965519130074 0.07561162501367721 0.23277803017762352 0.23609397264897924 0.18170193094439918 0.2921486340067721 0.0 True 1000 +MF 2.0 159 auprc supra_input_relative naive 0.1712713104376412 0.026528609897190324 0.14474270054045088 0.14520465466782265 0.09502199924850384 0.2028668194601218 0.0 True 1000 +MF 2.0 159 f_max supra_input_output naive 0.3509584986613415 0.07561162501367721 0.2753468736476643 0.27476477783461123 0.21604043140392137 0.33426220596838124 0.0 True 1000 +MF 2.0 159 auprc supra_input_output naive 0.2220098622274635 0.026528609897190324 0.19548125233027316 0.19600851017561177 0.14039708117719665 0.25897608828884744 0.0 True 1000 +MF 2.0 159 f_max supra_relative_output naive 0.28055796784500686 0.07561162501367721 0.20494634283132965 0.20342830696704106 0.15421949573661137 0.25564572470655184 0.0 True 1000 +MF 2.0 159 auprc supra_relative_output naive 0.11226843435692951 0.026528609897190324 0.08573982445973918 0.08736181991623265 0.04697130303524595 0.13433520959928366 0.0 True 1000 +MF 2.0 159 f_max full naive 0.31749948602842004 0.07561162501367721 0.24188786101474283 0.24227705848534098 0.19031859940015644 0.29529005609317494 0.0 True 1000 +MF 2.0 159 auprc full naive 0.16934947196131758 0.026528609897190324 0.14282086206412725 0.14374548526963712 0.09484484093342498 0.1977387548006009 0.0 True 1000 +MF 2.0 159 f_max full single 0.31749948602842004 0.3624206773738544 -0.044921191345434375 -0.04412431643848555 -0.0695053830426973 -0.019780517750405277 0.0 True 1000 +MF 2.0 159 auprc full single 0.16934947196131758 0.2009778910151442 -0.03162841905382663 -0.03066204431831983 -0.05366087535448409 -0.005944355415651474 0.012 True 1000 +MF 4.0 151 f_max supra single 0.3262836462924187 0.3298667018196884 -0.0035830555272697295 -0.0007847662031552737 -0.009705495825479755 0.009650244829548234 0.826 False 1000 +MF 4.0 151 auprc supra single 0.1942229401483132 0.1939744433720369 0.00024849677627630795 0.00027114959598139763 -0.0070765210619580105 0.008495550179366183 0.972 False 1000 +MF 4.0 151 f_max supra_input supra 0.33803499974566886 0.3262836462924187 0.01175135345325018 0.006812340886800629 -0.016741253477953924 0.03156267199747428 0.626 False 1000 +MF 4.0 151 auprc supra_input supra 0.19917894898248917 0.1942229401483132 0.004956008834175962 0.005277655310428722 -0.015096574162548696 0.02720068398682286 0.632 False 1000 +MF 4.0 151 f_max supra_relative supra 0.23537269599888028 0.3262836462924187 -0.0909109502935384 -0.09116350790125552 -0.13282207685226763 -0.050802263549678095 0.0 True 1000 +MF 4.0 151 auprc supra_relative supra 0.0817924273099161 0.1942229401483132 -0.11243051283839711 -0.11205371661667651 -0.1538219527098475 -0.0765701735189879 0.0 True 1000 +MF 4.0 151 f_max supra_output supra 0.3157306071081526 0.3262836462924187 -0.010553039184266089 -0.01187899967431217 -0.03312357558246973 0.009234305291975489 0.272 False 1000 +MF 4.0 151 auprc supra_output supra 0.18613794150147983 0.1942229401483132 -0.008084998646833375 -0.007666473561851157 -0.02607932469898928 0.010213896798473746 0.386 False 1000 +MF 4.0 151 f_max supra_input_relative supra_input 0.28129841286364055 0.33803499974566886 -0.05673658688202832 -0.052841980399365454 -0.0783850713677795 -0.027987854290924694 0.0 True 1000 +MF 4.0 151 auprc supra_input_relative supra_input 0.13470318550435026 0.19917894898248917 -0.06447576347813891 -0.06450457076760525 -0.09756444960932395 -0.03528370898053858 0.0 True 1000 +MF 4.0 151 f_max supra_input_relative supra_relative 0.28129841286364055 0.23537269599888028 0.04592571686476027 0.04513386838869069 0.012695700003461813 0.08274983462937753 0.008 True 1000 +MF 4.0 151 auprc supra_input_relative supra_relative 0.13470318550435026 0.0817924273099161 0.052910758194434165 0.052826801159499975 0.024505173583118545 0.08602074541015418 0.0 True 1000 +MF 4.0 151 f_max supra_input_output supra_input 0.33136710383897316 0.33803499974566886 -0.006667895906695698 -0.006444203073645971 -0.01648672830182641 0.003974981904483681 0.226 False 1000 +MF 4.0 151 auprc supra_input_output supra_input 0.20407333923335857 0.19917894898248917 0.004894390250869407 0.0049665076378181 -0.00522073178529848 0.01845156603563587 0.434 False 1000 +MF 4.0 151 f_max supra_input_output supra_output 0.33136710383897316 0.3157306071081526 0.01563649673082057 0.012247137487466826 -0.0018373058310825732 0.028848749478132447 0.094 False 1000 +MF 4.0 151 auprc supra_input_output supra_output 0.20407333923335857 0.18613794150147983 0.017935397731878744 0.01791063651009798 0.007807840832714195 0.030758359117829575 0.0 True 1000 +MF 4.0 151 f_max supra_relative_output supra_relative 0.2403644107271082 0.23537269599888028 0.004991714728227931 0.004225250413807815 -0.010602094035589602 0.016440623993670866 0.494 False 1000 +MF 4.0 151 auprc supra_relative_output supra_relative 0.08662340309592595 0.0817924273099161 0.004830975786009856 0.005570540596625648 -0.0073048940489242944 0.019008526441030068 0.4 False 1000 +MF 4.0 151 f_max supra_relative_output supra_output 0.2403644107271082 0.3157306071081526 -0.07536619638104439 -0.07505925781313554 -0.11056341693960463 -0.03714950801798993 0.0 True 1000 +MF 4.0 151 auprc supra_relative_output supra_output 0.08662340309592595 0.18613794150147983 -0.09951453840555388 -0.09881670245819972 -0.13731121867259796 -0.06649582637213469 0.0 True 1000 +MF 4.0 151 f_max full supra_input_relative 0.2910231966668611 0.28129841286364055 0.00972478380322056 0.006613037226229102 -0.004258660991694415 0.016890217948060733 0.208 False 1000 +MF 4.0 151 auprc full supra_input_relative 0.13886787332506306 0.13470318550435026 0.0041646878207128035 0.0044982785434623 -0.006866201737688444 0.01671054169067485 0.394 False 1000 +MF 4.0 151 f_max full supra_input_output 0.2910231966668611 0.33136710383897316 -0.04034390717211206 -0.03978474009949038 -0.06707237451233042 -0.015548780363578653 0.0 True 1000 +MF 4.0 151 auprc full supra_input_output 0.13886787332506306 0.20407333923335857 -0.06520546590829551 -0.06497279986196107 -0.08977822553150747 -0.04252379650433484 0.0 True 1000 +MF 4.0 151 f_max full supra_relative_output 0.2910231966668611 0.2403644107271082 0.0506587859397529 0.047521655201111984 0.014162039260260075 0.08361299339311515 0.008 True 1000 +MF 4.0 151 auprc full supra_relative_output 0.13886787332506306 0.08662340309592595 0.05224447022913711 0.05175453910633662 0.023981253513879704 0.08117027059608033 0.0 True 1000 +MF 4.0 151 f_max single naive 0.3298667018196884 0.07531254466979251 0.25455415714989593 0.2547519412873005 0.1927819080128679 0.31549415557273586 0.0 True 1000 +MF 4.0 151 auprc single naive 0.1939744433720369 0.02307128915639914 0.17090315421563776 0.1716129765949135 0.11697496089087112 0.2317083342285098 0.0 True 1000 +MF 4.0 151 f_max supra naive 0.3262836462924187 0.07531254466979251 0.25097110162262615 0.2539671750841452 0.19248765893459577 0.3120092975903396 0.0 True 1000 +MF 4.0 151 auprc supra naive 0.1942229401483132 0.02307128915639914 0.17115165099191407 0.1718841261908949 0.11709735942654326 0.23320209801776412 0.0 True 1000 +MF 4.0 151 f_max supra_input naive 0.33803499974566886 0.07531254466979251 0.2627224550758763 0.26077951597094584 0.19806493994039254 0.3228509840847253 0.0 True 1000 +MF 4.0 151 auprc supra_input naive 0.19917894898248917 0.02307128915639914 0.17610765982609003 0.1771617815013236 0.12223378364558002 0.23683066631025376 0.0 True 1000 +MF 4.0 151 f_max supra_relative naive 0.23537269599888028 0.07531254466979251 0.16006015132908777 0.1628036671828897 0.11302647632951263 0.215025673094283 0.0 True 1000 +MF 4.0 151 auprc supra_relative naive 0.0817924273099161 0.02307128915639914 0.058721138153516955 0.05983040957421839 0.027584340085272666 0.09895486399900541 0.0 True 1000 +MF 4.0 151 f_max supra_output naive 0.3157306071081526 0.07531254466979251 0.2404180624383601 0.24208817540983305 0.18479091557828994 0.29914202409509344 0.0 True 1000 +MF 4.0 151 auprc supra_output naive 0.18613794150147983 0.02307128915639914 0.1630666523450807 0.16421765262904375 0.11144283662096927 0.22166562302064263 0.0 True 1000 +MF 4.0 151 f_max supra_input_relative naive 0.28129841286364055 0.07531254466979251 0.20598586819384804 0.20793753557158037 0.15422723095276256 0.2617609730659185 0.0 True 1000 +MF 4.0 151 auprc supra_input_relative naive 0.13470318550435026 0.02307128915639914 0.11163189634795112 0.11265721073371836 0.06702101193778365 0.16446423568241933 0.0 True 1000 +MF 4.0 151 f_max supra_input_output naive 0.33136710383897316 0.07531254466979251 0.25605455916918063 0.2543353128972998 0.19521050383609237 0.31139683654903283 0.0 True 1000 +MF 4.0 151 auprc supra_input_output naive 0.20407333923335857 0.02307128915639914 0.18100205007695944 0.18212828913914175 0.12572832654239932 0.241967696673697 0.0 True 1000 +MF 4.0 151 f_max supra_relative_output naive 0.2403644107271082 0.07531254466979251 0.1650518660573157 0.1670289175966975 0.11632789472149818 0.21739005484172777 0.0 True 1000 +MF 4.0 151 auprc supra_relative_output naive 0.08662340309592595 0.02307128915639914 0.06355211393952681 0.06540095017084403 0.031099422418688953 0.10631771507067131 0.0 True 1000 +MF 4.0 151 f_max full naive 0.2910231966668611 0.07531254466979251 0.2157106519970686 0.21455057279780948 0.16270794516549594 0.26629699214120595 0.0 True 1000 +MF 4.0 151 auprc full naive 0.13886787332506306 0.02307128915639914 0.11579658416866392 0.11715548927718067 0.07074800396301886 0.16787043562232795 0.0 True 1000 +MF 4.0 151 f_max full single 0.2910231966668611 0.3298667018196884 -0.03884350515282731 -0.040201368489491004 -0.06641622385319271 -0.014217799062085581 0.006 True 1000 +MF 4.0 151 auprc full single 0.13886787332506306 0.1939744433720369 -0.055106570046973835 -0.05445748731773284 -0.0842246069594949 -0.026804091995896968 0.0 True 1000 +CC 0.0 436 f_max supra single 0.37497903858073917 0.37992957089151047 -0.004950532310771305 -0.003344140483175271 -0.010178436196991521 0.004437729414253629 0.38 False 1000 +CC 0.0 436 auprc supra single 0.21886015071674497 0.2206137897500839 -0.0017536390333389162 -0.0017549232399628916 -0.0047685350270656795 0.0011753166699765714 0.222 False 1000 +CC 0.0 436 f_max supra_input supra 0.3702482776163785 0.37497903858073917 -0.004730760964360647 -0.0037994837426477126 -0.015605281171035745 0.007613737554956411 0.534 False 1000 +CC 0.0 436 auprc supra_input supra 0.20742636150879065 0.21886015071674497 -0.01143378920795432 -0.011285107142410091 -0.01928483681372252 -0.0029757296772636756 0.006 True 1000 +CC 0.0 436 f_max supra_relative supra 0.347468249316779 0.37497903858073917 -0.027510789263960167 -0.02760885892658264 -0.042337766437984585 -0.013574470642025795 0.0 True 1000 +CC 0.0 436 auprc supra_relative supra 0.17980251378717382 0.21886015071674497 -0.03905763692957115 -0.0394144216285335 -0.05542700008242801 -0.024505081583982763 0.0 True 1000 +CC 0.0 436 f_max supra_output supra 0.3865627580544418 0.37497903858073917 0.011583719473702625 0.011019255305254862 0.0020070522099594097 0.020278731989048115 0.018 True 1000 +CC 0.0 436 auprc supra_output supra 0.24871874271560107 0.21886015071674497 0.0298585919988561 0.029694413638271782 0.022153583853427583 0.037920559077676046 0.0 True 1000 +CC 0.0 436 f_max supra_input_relative supra_input 0.37268360539863454 0.3702482776163785 0.002435327782256025 0.00020329969038879785 -0.013991118483594966 0.013201626466775014 0.974 False 1000 +CC 0.0 436 auprc supra_input_relative supra_input 0.20558766875108328 0.20742636150879065 -0.001838692757707372 -0.0019447121025568898 -0.01062646947561398 0.005765104198449098 0.698 False 1000 +CC 0.0 436 f_max supra_input_relative supra_relative 0.37268360539863454 0.347468249316779 0.025215356081855544 0.02401267487432373 0.008462229918465975 0.04007390808965908 0.006 True 1000 +CC 0.0 436 auprc supra_input_relative supra_relative 0.20558766875108328 0.17980251378717382 0.025785154963909457 0.02618460238356652 0.012958702388974785 0.04028872783864186 0.0 True 1000 +CC 0.0 436 f_max supra_input_output supra_input 0.3869795489823925 0.3702482776163785 0.01673127136601399 0.014598876199885598 0.005495602816750372 0.023836977677974443 0.004 True 1000 +CC 0.0 436 auprc supra_input_output supra_input 0.24806639386107782 0.20742636150879065 0.040640032352287164 0.04034350896273278 0.03281799528980506 0.0478653240095373 0.0 True 1000 +CC 0.0 436 f_max supra_input_output supra_output 0.3869795489823925 0.3865627580544418 0.0004167909279507165 -0.00021986284801697693 -0.007518458832008603 0.007399768062193091 0.966 False 1000 +CC 0.0 436 auprc supra_input_output supra_output 0.24806639386107782 0.24871874271560107 -0.0006523488545232548 -0.0006360118179490939 -0.005548842868897641 0.0041648128272393494 0.782 False 1000 +CC 0.0 436 f_max supra_relative_output supra_relative 0.35844968314631653 0.347468249316779 0.010981433829537535 0.010039032748967615 0.0006288449487214754 0.018398427517660618 0.03 True 1000 +CC 0.0 436 auprc supra_relative_output supra_relative 0.21315413830976376 0.17980251378717382 0.03335162452258994 0.033300157488848875 0.025556592278690452 0.040988503429864695 0.0 True 1000 +CC 0.0 436 f_max supra_relative_output supra_output 0.35844968314631653 0.3865627580544418 -0.028113074908125257 -0.02858908148286989 -0.04460482347948633 -0.01224889847936367 0.0 True 1000 +CC 0.0 436 auprc supra_relative_output supra_output 0.21315413830976376 0.24871874271560107 -0.03556460440583731 -0.03580867777795642 -0.05140540696412077 -0.020104480056038473 0.0 True 1000 +CC 0.0 436 f_max full supra_input_relative 0.38155247242054213 0.37268360539863454 0.00886886702190759 0.009542295381823091 0.0022292646376124267 0.017500159791989284 0.014 True 1000 +CC 0.0 436 auprc full supra_input_relative 0.23398195570502178 0.20558766875108328 0.028394286953938502 0.028178629266183284 0.020576948351153757 0.03569613340916653 0.0 True 1000 +CC 0.0 436 f_max full supra_input_output 0.38155247242054213 0.3869795489823925 -0.005427076561850375 -0.004853281127673711 -0.016857330049355242 0.006354688694793616 0.382 False 1000 +CC 0.0 436 auprc full supra_input_output 0.23398195570502178 0.24806639386107782 -0.014084438156056034 -0.014109591799106386 -0.024480929991732486 -0.005183438738048892 0.002 True 1000 +CC 0.0 436 f_max full supra_relative_output 0.38155247242054213 0.35844968314631653 0.0231027892742256 0.023515937507179202 0.009728634608479135 0.037713502330987705 0.0 True 1000 +CC 0.0 436 auprc full supra_relative_output 0.23398195570502178 0.21315413830976376 0.02082781739525802 0.021063074160900933 0.009110629196828191 0.03379111589335754 0.0 True 1000 +CC 0.0 436 f_max single naive 0.37992957089151047 0.593050572267429 -0.2131210013759185 -0.2148454567786269 -0.24974981910930036 -0.17970956983939426 0.0 True 1000 +CC 0.0 436 auprc single naive 0.2206137897500839 0.5414744252479067 -0.3208606354978228 -0.32124929204302416 -0.35831490489910883 -0.28165513254534896 0.0 True 1000 +CC 0.0 436 f_max supra naive 0.37497903858073917 0.593050572267429 -0.2180715336866898 -0.2181895972618022 -0.2522502950613899 -0.1835643304477737 0.0 True 1000 +CC 0.0 436 auprc supra naive 0.21886015071674497 0.5414744252479067 -0.3226142745311617 -0.32300421528298706 -0.36029561780938807 -0.2850486492001403 0.0 True 1000 +CC 0.0 436 f_max supra_input naive 0.3702482776163785 0.593050572267429 -0.22280229465105045 -0.22198908100444986 -0.25566298118706826 -0.18540562729786794 0.0 True 1000 +CC 0.0 436 auprc supra_input naive 0.20742636150879065 0.5414744252479067 -0.334048063739116 -0.3342893224253972 -0.37217803211533124 -0.2967778437766746 0.0 True 1000 +CC 0.0 436 f_max supra_relative naive 0.347468249316779 0.593050572267429 -0.24558232295064997 -0.2457984561883848 -0.2807545225964079 -0.20893790597187056 0.0 True 1000 +CC 0.0 436 auprc supra_relative naive 0.17980251378717382 0.5414744252479067 -0.36167191146073285 -0.3624186369115206 -0.3970495411409802 -0.32670043023110396 0.0 True 1000 +CC 0.0 436 f_max supra_output naive 0.3865627580544418 0.593050572267429 -0.20648781421298718 -0.2071703419565473 -0.2414608052494114 -0.17247744687862754 0.0 True 1000 +CC 0.0 436 auprc supra_output naive 0.24871874271560107 0.5414744252479067 -0.2927556825323056 -0.2933098016447153 -0.3310489314803494 -0.2556755003435467 0.0 True 1000 +CC 0.0 436 f_max supra_input_relative naive 0.37268360539863454 0.593050572267429 -0.22036696686879442 -0.22178578131406107 -0.25724675371950323 -0.18577845393548187 0.0 True 1000 +CC 0.0 436 auprc supra_input_relative naive 0.20558766875108328 0.5414744252479067 -0.3358867564968234 -0.33623403452795414 -0.37328664317343924 -0.29847683317082185 0.0 True 1000 +CC 0.0 436 f_max supra_input_output naive 0.3869795489823925 0.593050572267429 -0.20607102328503646 -0.2073902048045643 -0.2404381581318101 -0.17314621668330396 0.0 True 1000 +CC 0.0 436 auprc supra_input_output naive 0.24806639386107782 0.5414744252479067 -0.29340803138682886 -0.2939458134626644 -0.33160976946540394 -0.25631356950939066 0.0 True 1000 +CC 0.0 436 f_max supra_relative_output naive 0.35844968314631653 0.593050572267429 -0.23460088912111243 -0.2357594234394172 -0.2703581739771676 -0.19963004557882402 0.0 True 1000 +CC 0.0 436 auprc supra_relative_output naive 0.21315413830976376 0.5414744252479067 -0.3283202869381429 -0.32911847942267175 -0.36506683765789794 -0.2907542140938589 0.0 True 1000 +CC 0.0 436 f_max full naive 0.38155247242054213 0.593050572267429 -0.21149809984688683 -0.212243485932238 -0.24747659236492328 -0.17760135623714748 0.0 True 1000 +CC 0.0 436 auprc full naive 0.23398195570502178 0.5414744252479067 -0.3074924695428849 -0.30805540526177083 -0.34352727906159797 -0.2694441489383527 0.0 True 1000 +CC 0.0 436 f_max full single 0.38155247242054213 0.37992957089151047 0.0016229015290316617 0.0026019708463889055 -0.009123525968066202 0.013718682225976418 0.634 False 1000 +CC 0.0 436 auprc full single 0.23398195570502178 0.2206137897500839 0.013368165954937894 0.013193886781253411 0.001040062267808702 0.024685453779326225 0.03 True 1000 +CC 2.0 298 f_max supra single 0.19924497775877514 0.2034339979081705 -0.004189020149395345 -0.004782062320762792 -0.009984737060886475 -0.0007384702405397602 0.012 True 1000 +CC 2.0 298 auprc supra single 0.05712819334284143 0.057802355414117046 -0.0006741620712756163 -0.0006582342809177854 -0.0038658129100056443 0.0018599246054736508 0.676 False 1000 +CC 2.0 298 f_max supra_input supra 0.19343405375604655 0.19924497775877514 -0.00581092400272859 -0.006019232271042667 -0.018542553284690445 0.00665557138697495 0.322 False 1000 +CC 2.0 298 auprc supra_input supra 0.05806421570392059 0.05712819334284143 0.0009360223610791613 0.0012352495217668325 -0.005692443336563635 0.007475749992696844 0.706 False 1000 +CC 2.0 298 f_max supra_relative supra 0.17952044684106486 0.19924497775877514 -0.019724530917710287 -0.020420701972397693 -0.04688302253382443 0.003229225228106482 0.09 False 1000 +CC 2.0 298 auprc supra_relative supra 0.04473430249200807 0.05712819334284143 -0.01239389085083336 -0.012711293981365206 -0.02642613115798128 -0.0011891065952991577 0.038 True 1000 +CC 2.0 298 f_max supra_output supra 0.20995311289051635 0.19924497775877514 0.010708135131741209 0.010698284386166199 -0.00521066199132662 0.025393613531179835 0.218 False 1000 +CC 2.0 298 auprc supra_output supra 0.059741954000379696 0.05712819334284143 0.0026137606575382666 0.002470963073044075 -0.004820774713072507 0.008923112723637145 0.442 False 1000 +CC 2.0 298 f_max supra_input_relative supra_input 0.20495063609979508 0.19343405375604655 0.011516582343748527 0.012393242692646265 -0.005841644568114813 0.030596605607956708 0.202 False 1000 +CC 2.0 298 auprc supra_input_relative supra_input 0.052436747996529906 0.05806421570392059 -0.005627467707390685 -0.006075353710624318 -0.015427137337579092 0.0025517844293176228 0.15 False 1000 +CC 2.0 298 f_max supra_input_relative supra_relative 0.20495063609979508 0.17952044684106486 0.025430189258730224 0.026794712394001295 0.00786964405072725 0.04762428640824899 0.004 True 1000 +CC 2.0 298 auprc supra_input_relative supra_relative 0.052436747996529906 0.04473430249200807 0.007702445504521836 0.007871189792507722 -0.003714579141531301 0.019401816281681923 0.18 False 1000 +CC 2.0 298 f_max supra_input_output supra_input 0.21768741321397259 0.19343405375604655 0.024253359457926033 0.024779051630425163 0.01209856383016925 0.037893424461804 0.0 True 1000 +CC 2.0 298 auprc supra_input_output supra_input 0.0653755442200359 0.05806421570392059 0.007311328516115309 0.007222715117887107 0.0013942161956267418 0.013430923115419475 0.008 True 1000 +CC 2.0 298 f_max supra_input_output supra_output 0.21768741321397259 0.20995311289051635 0.0077343003234562335 0.008061534973216298 -0.0014345328310016088 0.018987511991300457 0.108 False 1000 +CC 2.0 298 auprc supra_input_output supra_output 0.0653755442200359 0.059741954000379696 0.005633590219656204 0.0059870015666098645 0.0007817280445427117 0.012533055745905665 0.032 True 1000 +CC 2.0 298 f_max supra_relative_output supra_relative 0.18824252482325185 0.17952044684106486 0.008722077982186993 0.010061436533400008 0.0036020612263905436 0.018420140091917593 0.002 True 1000 +CC 2.0 298 auprc supra_relative_output supra_relative 0.04363441672467662 0.04473430249200807 -0.0010998857673314502 -0.0010642084457144552 -0.008648765636173254 0.005855815050215109 0.788 False 1000 +CC 2.0 298 f_max supra_relative_output supra_output 0.18824252482325185 0.20995311289051635 -0.021710588067264502 -0.021057549825163887 -0.04581973895496082 0.0038933550525664185 0.1 False 1000 +CC 2.0 298 auprc supra_relative_output supra_output 0.04363441672467662 0.059741954000379696 -0.016107537275703077 -0.016246465500123736 -0.03166990663395124 -0.0022950563880833317 0.026 True 1000 +CC 2.0 298 f_max full supra_input_relative 0.20783250955677182 0.20495063609979508 0.0028818734569767412 0.003625669994664845 -0.004209222702895035 0.012574855714928173 0.388 False 1000 +CC 2.0 298 auprc full supra_input_relative 0.052007881921068415 0.052436747996529906 -0.0004288660754614912 -0.0002146869766702304 -0.006223205357651527 0.005056801381719468 0.988 False 1000 +CC 2.0 298 f_max full supra_input_output 0.20783250955677182 0.21768741321397259 -0.009854903657200764 -0.008760138943114055 -0.02628343235850469 0.008588647106863793 0.318 False 1000 +CC 2.0 298 auprc full supra_input_output 0.052007881921068415 0.0653755442200359 -0.013367662298967485 -0.013512755805181657 -0.022794700144752638 -0.004365746807633377 0.004 True 1000 +CC 2.0 298 f_max full supra_relative_output 0.20783250955677182 0.18824252482325185 0.01958998473351997 0.02035894585526613 0.0014720080831470165 0.04115829194582372 0.036 True 1000 +CC 2.0 298 auprc full supra_relative_output 0.052007881921068415 0.04363441672467662 0.008373465196391795 0.008720711261551946 -0.0012641167174459772 0.019435307952341187 0.094 False 1000 +CC 2.0 298 f_max single naive 0.2034339979081705 0.2720885241633347 -0.06865452625516419 -0.06988059925677106 -0.11039374862898366 -0.02829559682395133 0.0 True 1000 +CC 2.0 298 auprc single naive 0.057802355414117046 0.1451197420848838 -0.08731738667076674 -0.08642474222462188 -0.1142927916255458 -0.05576681610932688 0.0 True 1000 +CC 2.0 298 f_max supra naive 0.19924497775877514 0.2720885241633347 -0.07284354640455953 -0.07466266157753386 -0.11486243278191671 -0.034265542448681484 0.0 True 1000 +CC 2.0 298 auprc supra naive 0.05712819334284143 0.1451197420848838 -0.08799154874204236 -0.08708297650553966 -0.1139895442143255 -0.057274950798521754 0.0 True 1000 +CC 2.0 298 f_max supra_input naive 0.19343405375604655 0.2720885241633347 -0.07865447040728812 -0.0806818938485765 -0.11998760507071622 -0.04301682076247021 0.0 True 1000 +CC 2.0 298 auprc supra_input naive 0.05806421570392059 0.1451197420848838 -0.0870555263809632 -0.08584772698377281 -0.11232355668267092 -0.05645749918157046 0.0 True 1000 +CC 2.0 298 f_max supra_relative naive 0.17952044684106486 0.2720885241633347 -0.09256807732226982 -0.09508336354993156 -0.13358805121079753 -0.05678865303130964 0.0 True 1000 +CC 2.0 298 auprc supra_relative naive 0.04473430249200807 0.1451197420848838 -0.10038543959287571 -0.09979427048690488 -0.12347399644884319 -0.07436440247958194 0.0 True 1000 +CC 2.0 298 f_max supra_output naive 0.20995311289051635 0.2720885241633347 -0.062135411272818325 -0.06396437719136766 -0.10273104086291737 -0.024225184278138037 0.004 True 1000 +CC 2.0 298 auprc supra_output naive 0.059741954000379696 0.1451197420848838 -0.0853777880845041 -0.08461201343249558 -0.11126765050371834 -0.056389935654182155 0.0 True 1000 +CC 2.0 298 f_max supra_input_relative naive 0.20495063609979508 0.2720885241633347 -0.0671378880635396 -0.06828865115593027 -0.10672012605720263 -0.027568211577047794 0.006 True 1000 +CC 2.0 298 auprc supra_input_relative naive 0.052436747996529906 0.1451197420848838 -0.09268299408835388 -0.09192308069439714 -0.11838920002004444 -0.06470729518617287 0.0 True 1000 +CC 2.0 298 f_max supra_input_output naive 0.21768741321397259 0.2720885241633347 -0.05440111094936209 -0.05590284221815135 -0.0946732714478457 -0.014858242642985731 0.006 True 1000 +CC 2.0 298 auprc supra_input_output naive 0.0653755442200359 0.1451197420848838 -0.07974419786484789 -0.07862501186588572 -0.10601974104574416 -0.04962273017758079 0.0 True 1000 +CC 2.0 298 f_max supra_relative_output naive 0.18824252482325185 0.2720885241633347 -0.08384599934008283 -0.08502192701653154 -0.12400566246276054 -0.04628676395966807 0.0 True 1000 +CC 2.0 298 auprc supra_relative_output naive 0.04363441672467662 0.1451197420848838 -0.10148532536020717 -0.10085847893261932 -0.12242937337249513 -0.07748211697673466 0.0 True 1000 +CC 2.0 298 f_max full naive 0.20783250955677182 0.2720885241633347 -0.06425601460656286 -0.06466298116126541 -0.10320790216241282 -0.024293415552098976 0.006 True 1000 +CC 2.0 298 auprc full naive 0.052007881921068415 0.1451197420848838 -0.09311186016381537 -0.09213776767106738 -0.11644250451630991 -0.06696748898089515 0.0 True 1000 +CC 2.0 298 f_max full single 0.20783250955677182 0.2034339979081705 0.004398511648601333 0.00521761809550565 -0.015982471529097035 0.02489518886454284 0.604 False 1000 +CC 2.0 298 auprc full single 0.052007881921068415 0.057802355414117046 -0.005794473493048631 -0.005713025446445501 -0.018555882213107267 0.005931590097314167 0.36 False 1000 +CC 4.0 210 f_max supra single 0.12630996046336843 0.12966758302364031 -0.003357622560271889 -0.000826389590146164 -0.011242392933487924 0.011717184844359924 0.816 False 1000 +CC 4.0 210 auprc supra single 0.031794658809648026 0.03379172626930409 -0.0019970674596560614 -0.0018477481890242454 -0.005260044875182496 0.0005109228970353268 0.156 False 1000 +CC 4.0 210 f_max supra_input supra 0.12782363409703573 0.12630996046336843 0.0015136736336673062 0.0007398058715013769 -0.014488686373730955 0.017962695450062884 0.952 False 1000 +CC 4.0 210 auprc supra_input supra 0.03586106852445638 0.031794658809648026 0.004066409714808351 0.00397261661712192 -0.0010375378194814905 0.010322650051535277 0.12 False 1000 +CC 4.0 210 f_max supra_relative supra 0.06785583012869659 0.12630996046336843 -0.05845413033467184 -0.06106738540778024 -0.09849959160635319 -0.024401389547409044 0.0 True 1000 +CC 4.0 210 auprc supra_relative supra 0.01344602376340749 0.031794658809648026 -0.018348635046240534 -0.018923573307166472 -0.037102493745455316 -0.003953222716881324 0.012 True 1000 +CC 4.0 210 f_max supra_output supra 0.12223317857868368 0.12630996046336843 -0.004076781884684744 -0.005147902969785992 -0.01928794041206996 0.0037598862184696656 0.36 False 1000 +CC 4.0 210 auprc supra_output supra 0.03901494516963492 0.031794658809648026 0.007220286359986894 0.005776463467353679 -0.0010968523471862044 0.014606964621145168 0.116 False 1000 +CC 4.0 210 f_max supra_input_relative supra_input 0.12554626887725337 0.12782363409703573 -0.0022773652197823624 -0.00735065949732352 -0.026070698624237738 0.00810071640042338 0.412 False 1000 +CC 4.0 210 auprc supra_input_relative supra_input 0.030681186035091658 0.03586106852445638 -0.005179882489364719 -0.005579470329872813 -0.01693932123986697 0.0039209843064311085 0.254 False 1000 +CC 4.0 210 f_max supra_input_relative supra_relative 0.12554626887725337 0.06785583012869659 0.05769043874855678 0.054456531781958105 0.021895704259405662 0.09047940949239393 0.0 True 1000 +CC 4.0 210 auprc supra_input_relative supra_relative 0.030681186035091658 0.01344602376340749 0.017235162271684167 0.017316719594415577 0.005419142062542676 0.031908332620526406 0.002 True 1000 +CC 4.0 210 f_max supra_input_output supra_input 0.12547653290885707 0.12782363409703573 -0.0023471011881786574 -0.0036492438339777813 -0.012175335251256298 0.003627050609039953 0.468 False 1000 +CC 4.0 210 auprc supra_input_output supra_input 0.038374445533779236 0.03586106852445638 0.002513377009322859 0.0019411592222720305 -0.0038302245017993505 0.009648599371777046 0.56 False 1000 +CC 4.0 210 f_max supra_input_output supra_output 0.12547653290885707 0.12223317857868368 0.0032433543301733925 0.002238465007309588 -0.01836291815862723 0.022533085766973864 0.798 False 1000 +CC 4.0 210 auprc supra_input_output supra_output 0.038374445533779236 0.03901494516963492 -0.0006404996358556833 0.0001373123720402718 -0.006154770015342401 0.006429834868654985 0.938 False 1000 +CC 4.0 210 f_max supra_relative_output supra_relative 0.06615289967671167 0.06785583012869659 -0.001702930451984913 -0.002273051364608477 -0.009957765826277054 0.0035669737711938932 0.552 False 1000 +CC 4.0 210 auprc supra_relative_output supra_relative 0.013628547043723002 0.01344602376340749 0.00018252328031551253 4.7538618295312665e-05 -0.006694052472186392 0.0073438143624410795 0.978 False 1000 +CC 4.0 210 f_max supra_relative_output supra_output 0.06615289967671167 0.12223317857868368 -0.05608027890197201 -0.05819253380260274 -0.09632373433168921 -0.02140769569003083 0.0 True 1000 +CC 4.0 210 auprc supra_relative_output supra_output 0.013628547043723002 0.03901494516963492 -0.025386398125911917 -0.024652498156224837 -0.046494322890099846 -0.005110362011964313 0.012 True 1000 +CC 4.0 210 f_max full supra_input_relative 0.12476406186624613 0.12554626887725337 -0.0007822070110072438 -0.0017457565770090922 -0.008224407864971963 0.0020570027800080374 0.428 False 1000 +CC 4.0 210 auprc full supra_input_relative 0.02865341327493268 0.030681186035091658 -0.002027772760158978 -0.002386217195475975 -0.00862866974357257 0.003783584782722233 0.424 False 1000 +CC 4.0 210 f_max full supra_input_output 0.12476406186624613 0.12547653290885707 -0.0007124710426109487 -0.00544717224035483 -0.022648116270191095 0.009404180843368574 0.526 False 1000 +CC 4.0 210 auprc full supra_input_output 0.02865341327493268 0.038374445533779236 -0.009721032258846556 -0.009906846747620818 -0.021044186027498468 -0.000991199039484456 0.024 True 1000 +CC 4.0 210 f_max full supra_relative_output 0.12476406186624613 0.06615289967671167 0.05861116218953445 0.05498382656955749 0.022186099503939655 0.09097872671158563 0.0 True 1000 +CC 4.0 210 auprc full supra_relative_output 0.02865341327493268 0.013628547043723002 0.015024866231209678 0.01488296378064429 0.001523237062611553 0.028959601695131256 0.022 True 1000 +CC 4.0 210 f_max single naive 0.12966758302364031 0.10440217265761789 0.025265410366022423 0.02757170191229148 -0.01728525719232114 0.07090681872034521 0.194 False 1000 +CC 4.0 210 auprc single naive 0.03379172626930409 0.04494942594240296 -0.011157699673098875 -0.009521449653920575 -0.03096736392337584 0.01577681484918615 0.444 False 1000 +CC 4.0 210 f_max supra naive 0.12630996046336843 0.10440217265761789 0.021907787805750534 0.026745312322145318 -0.016239546561927862 0.07209637401291345 0.232 False 1000 +CC 4.0 210 auprc supra naive 0.031794658809648026 0.04494942594240296 -0.013154767132754937 -0.011369197842944821 -0.03113264434516956 0.012073997071567163 0.322 False 1000 +CC 4.0 210 f_max supra_input naive 0.12782363409703573 0.10440217265761789 0.02342146143941784 0.027485118193646692 -0.013045910152674275 0.07124481654586881 0.206 False 1000 +CC 4.0 210 auprc supra_input naive 0.03586106852445638 0.04494942594240296 -0.009088357417946585 -0.007396581225822901 -0.028320621411571628 0.01796020058737023 0.536 False 1000 +CC 4.0 210 f_max supra_relative naive 0.06785583012869659 0.10440217265761789 -0.036546342528921305 -0.034322073085634924 -0.06989447212726765 0.006369892151659862 0.104 False 1000 +CC 4.0 210 auprc supra_relative naive 0.01344602376340749 0.04494942594240296 -0.03150340217899547 -0.03029277115011129 -0.04424473525771525 -0.0110570407982897 0.004 True 1000 +CC 4.0 210 f_max supra_output naive 0.12223317857868368 0.10440217265761789 0.01783100592106579 0.021597409352359324 -0.01897394187257916 0.06113796760235773 0.334 False 1000 +CC 4.0 210 auprc supra_output naive 0.03901494516963492 0.04494942594240296 -0.005934480772768043 -0.005592734375591143 -0.028387280187366148 0.022572743533626354 0.676 False 1000 +CC 4.0 210 f_max supra_input_relative naive 0.12554626887725337 0.10440217265761789 0.021144096219635478 0.020134458696323174 -0.02220486843164876 0.06218257672050374 0.36 False 1000 +CC 4.0 210 auprc supra_input_relative naive 0.030681186035091658 0.04494942594240296 -0.014268239907311304 -0.012976051555695716 -0.03256766620452514 0.011756864095378992 0.276 False 1000 +CC 4.0 210 f_max supra_input_output naive 0.12547653290885707 0.10440217265761789 0.021074360251239183 0.023835874359668913 -0.017190452897917662 0.06660520042175122 0.27 False 1000 +CC 4.0 210 auprc supra_input_output naive 0.038374445533779236 0.04494942594240296 -0.006574980408623726 -0.00545542200355087 -0.026830294309838958 0.02083683658127076 0.668 False 1000 +CC 4.0 210 f_max supra_relative_output naive 0.06615289967671167 0.10440217265761789 -0.03824927298090622 -0.036595124450243405 -0.07201953303190065 0.002728274251132941 0.072 False 1000 +CC 4.0 210 auprc supra_relative_output naive 0.013628547043723002 0.04494942594240296 -0.03132087889867996 -0.030245232531815977 -0.044774791831301 -0.009816614278676762 0.002 True 1000 +CC 4.0 210 f_max full naive 0.12476406186624613 0.10440217265761789 0.020361889208628234 0.018388702119314083 -0.022786941620980272 0.06091832758862439 0.39 False 1000 +CC 4.0 210 auprc full naive 0.02865341327493268 0.04494942594240296 -0.016296012667470283 -0.01536226875117169 -0.033837876694593795 0.007735017281053418 0.178 False 1000 +CC 4.0 210 f_max full single 0.12476406186624613 0.12966758302364031 -0.004903521157394189 -0.009182999792977398 -0.0360623892536633 0.015640334909424795 0.504 False 1000 +CC 4.0 210 auprc full single 0.02865341327493268 0.03379172626930409 -0.005138312994371407 -0.005840819097251113 -0.01877025374876898 0.005928485281274633 0.328 False 1000 diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_permutation_null.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_permutation_null.tsv new file mode 100644 index 0000000..a8b81fa --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_permutation_null.tsv @@ -0,0 +1,19 @@ +aspect min_ic metric rung n_eval_proteins observed null_n null_mean null_sd null_min null_max null_ci_lo null_ci_hi ratio_observed_over_null_mean z_score empirical_p +BP 0.0 auprc supra 298 0.10862153438760377 200 0.043277176627405246 0.0041702667707538225 0.032921225546087826 0.05927958693681451 0.0366785263144021 0.05272120542389563 2.5099034376198017 15.66910736225798 0.004975124378109453 +BP 0.0 f_max supra 298 0.23389853138263522 200 0.12563732305956332 0.007730446528144377 0.10914385075722287 0.15599305002204733 0.11210012544967221 0.14208572614265824 1.8616962355346143 14.004521980576847 0.004975124378109453 +BP 2.0 auprc supra 298 0.07180857194540718 200 0.005829322130350874 0.0014874959855654798 0.003114624055019575 0.011175456647358912 0.003688246399563204 0.009147044224129267 12.318511542110462 44.3559179018382 0.004975124378109453 +BP 2.0 f_max supra 298 0.1761580707594129 200 0.05065620783218593 0.006516751656272109 0.036963420210952225 0.0703362339845386 0.04068066370310103 0.06486863293266157 3.4775218733899305 19.258346726537678 0.004975124378109453 +BP 4.0 auprc supra 292 0.03275518908553014 200 0.0006070807758487667 0.00030720075244804835 0.00018265482965827733 0.0019622314768695187 0.0002128608208329987 0.0013324897285560144 53.955240206271945 104.64853374705856 0.004975124378109453 +BP 4.0 f_max supra 292 0.11805494385249392 200 0.016124427097293895 0.004719422559349019 0.00887408799340424 0.03282368590127366 0.009586210223758709 0.025892690731011057 7.3214969524285705 21.598090756522545 0.004975124378109453 +CC 0.0 auprc supra 436 0.21886015071674497 200 0.1558271557875937 0.013488039630611098 0.12216070606978227 0.19331478213575842 0.13072532104694443 0.17963987886146426 1.4045058424545132 4.673251017597688 0.004975124378109453 +CC 0.0 f_max supra 436 0.37497903858073917 200 0.27471594594293824 0.015131735259083658 0.24090715570559126 0.31472684696289366 0.2458330140340072 0.3036867955542579 1.3649700504048161 6.626014196066011 0.004975124378109453 +CC 2.0 auprc supra 298 0.05712819334284143 200 0.008502933281715228 0.0034330247972979354 0.003258141899852521 0.027504597753334022 0.004225740846114551 0.016476062542949494 6.718645372143555 14.163969948425123 0.004975124378109453 +CC 2.0 f_max supra 298 0.19924497775877514 200 0.06963909276993041 0.012701263531465794 0.04235070403785552 0.10538118985006786 0.048937273158652546 0.09723528367770429 2.861108176940632 10.204172574465709 0.004975124378109453 +CC 4.0 auprc supra 210 0.031794658809648026 200 0.0013233282693889401 0.0007256119826320617 0.00018460940354031382 0.005044955562820352 0.00036031644993850346 0.003225855850765522 24.026282476629568 41.993973734734034 0.004975124378109453 +CC 4.0 f_max supra 210 0.12630996046336843 200 0.028292632239276427 0.007700428074563765 0.010793062459717099 0.052767229007312094 0.015273759316540111 0.04378211068573934 4.4644117731832065 12.72881549895455 0.004975124378109453 +MF 0.0 auprc supra 406 0.17863188024464952 200 0.059218410221143714 0.00950467637978714 0.03963445192024161 0.0874957305073261 0.043615759641876875 0.08161963512006114 3.016492330300175 12.563654484591732 0.004975124378109453 +MF 0.0 f_max supra 406 0.34917244942038655 200 0.17973234831136986 0.01884639845830099 0.1358027344917275 0.2267453043813563 0.14718302708051875 0.2222217276916974 1.9427356995051175 8.990582549971819 0.004975124378109453 +MF 2.0 auprc supra 159 0.20961207022311829 200 0.005982506517606556 0.003103049007214942 0.001114789902503866 0.020545573288671033 0.0020478722259371245 0.013842898373610852 35.03749968449321 65.62241306278109 0.004975124378109453 +MF 2.0 f_max supra 159 0.3575134302039365 200 0.050394066515827285 0.015382281028789961 0.01880951484303916 0.12105527031221945 0.029022668170684645 0.08978065617103284 7.094355643866369 19.96578811122323 0.004975124378109453 +MF 4.0 auprc supra 151 0.1942229401483132 200 0.0033226607014584996 0.002107098959999484 0.0006264511366081648 0.01599146833587484 0.0009497520009690592 0.00819192255736219 58.454039578298804 90.59863018816233 0.004975124378109453 +MF 4.0 f_max supra 151 0.3262836462924187 200 0.036462226032316976 0.012578240112093454 0.014622349533126524 0.10293207304117295 0.01819548808700298 0.06686266977420262 8.948538852324292 23.04149210678929 0.004975124378109453 diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_provenance.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_provenance.tsv new file mode 100644 index 0000000..6360c4b --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_provenance.tsv @@ -0,0 +1,16 @@ +rung label adds predictions_file n_rows n_domains score interpro_file interpro_sha256 cohort_interpro_file cohort_interpro_sha256 t0_gaf_file t0_gaf_sha256 t1_gaf_file t1_gaf_sha256 +single single domains - /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single/domain_go_associations_significant.tsv 43810 6335 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +single__p single domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single/domain_go_associations_significant.tsv 43810 6335 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra + supra-domains supra-domains /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra/domain_go_associations_significant.tsv 164183 27328 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra__p + supra-domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra/domain_go_associations_significant.tsv 164183 27328 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input + input propagation input propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input/domain_go_associations_significant.tsv 491823 27457 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input__p + input propagation (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input/domain_go_associations_significant.tsv 491823 27457 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative + relative inference relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative/domain_go_associations_significant.tsv 30795 10568 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative__p + relative inference (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative/domain_go_associations_significant.tsv 30795 10568 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_output + output propagation output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output/domain_go_annotations_propagated.tsv 1123671 27328 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative input + relative input propagation + relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative/domain_go_associations_significant.tsv 92774 11802 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative__p input + relative (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative/domain_go_associations_significant.tsv 92774 11802 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_output input + output input + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output/domain_go_annotations_propagated.tsv 1210884 27457 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative_output relative + output relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output/domain_go_annotations_propagated.tsv 167103 10568 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +full full current method input propagation + relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full/domain_go_annotations_propagated.tsv 281950 11802 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +naive CAFA naive baseline - - 19299 0 propagated t0 term frequency /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz 6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041 data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv new file mode 100644 index 0000000..e43cd36 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv @@ -0,0 +1,22 @@ +stage aspect min_ic n_proteins n_dropped_vs_ic0 pct_of_ic0 note +t0_annotated_proteins - - 18735 - - proteins with >=1 GO annotation at t0 under the training evidence filter (the training universe) +t1_experimental_proteins - - 16362 - - proteins with >=1 experimental GO annotation at t1 (the gold standard) +proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain (predictable at all by a domain method) +no_knowledge_candidates BP - 310 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains BP - 298 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures BP - 0 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +no_knowledge_candidates MF - 418 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains MF - 406 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures MF - 0 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +no_knowledge_candidates CC - 453 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains CC - 436 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures CC - 0 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +cohort_at_ic_floor BP 0 298 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 2 298 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 4 292 6 98.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 0 406 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 2 159 247 39.2 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 4 151 255 37.2 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 0 436 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 2 298 138 68.3 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 4 210 226 48.2 proteins whose truth is non-empty at this IC floor diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/full.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/full.json new file mode 100644 index 0000000..29817db --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/full.json @@ -0,0 +1,216 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full", + "--propagate-annotations", + "--enable-relative-inference", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:48:47.594035+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:48:45.000350+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "cd9dd1fd0e1abe20dadd169f59be47912b70f45f199295b033c2f72937bff97a", + "size_bytes": 13519691 + }, + { + "modified_at": "2026-10-07T09:48:45.125355+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "079d5d65381c25ce44dfabe2c7c463450299e459860614fe9ec15c57e68fe6aa", + "size_bytes": 11727 + }, + { + "modified_at": "2026-10-07T09:48:47.322442+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "9a765dc4266ee7b2d20dbe0a13313203aac8e4bdd9f8af7c6d2a259231b74578", + "size_bytes": 23555465 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/full", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:41:06.486585+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 8.345192721392234e-07, + "supra": 4.137618898264933e-07 + }, + "domains": 102375, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4703, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 925, + "ontology": "go", + "propagated_annotations": 281950, + "proteins": 18549, + "runtime_seconds": 85.59, + "significant_associations": 92774, + "terms": 18371, + "tests": 1880731125, + "tests_evaluated": 13567117 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/protein2ipr_human_t0t1sel_current.dat.gz.provenance.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/protein2ipr_human_t0t1sel_current.dat.gz.provenance.json new file mode 100644 index 0000000..0c4d4c6 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/protein2ipr_human_t0t1sel_current.dat.gz.provenance.json @@ -0,0 +1,12 @@ +{ + "selection_rule": "goa", + "selection_sources": [ + "data/raw/goa_archive/goa_human.gaf.205.gz", + "data/raw/goa_annotations/goa_human.gaf.gz" + ], + "interpro_source": "data/raw/interpro_mappings/protein2ipr.dat.gz", + "n_accessions": 19871, + "n_matched_lines": 241143, + "tool": "extract_human_interpro.py", + "created": "2026-10-07T09:02:09+00:00" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/single.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/single.json new file mode 100644 index 0000000..6825adb --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/single.json @@ -0,0 +1,188 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": false, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "direct", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single", + "--disable-supra-domains", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:03:01.294899+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:03:01.215663+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "0a8c9cf9585ee1ae47593e68e8febc7431f598a8ece38d7563fce0d3bb2e51a8", + "size_bytes": 5102192 + }, + { + "modified_at": "2026-10-07T09:03:01.262665+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "a30569fd0256009b8c3de5a0c38f49ec623653eaeb7d71a949f95b744ee490a4", + "size_bytes": 9274 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": false, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/single", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:02:13.669906+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4154122248063175e-06 + }, + "domains": 19260, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18549, + "runtime_seconds": 6.97, + "significant_associations": 43810, + "terms": 16057, + "tests": 309257820, + "tests_evaluated": 641223 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra.json new file mode 100644 index 0000000..d4272b0 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra.json @@ -0,0 +1,188 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "direct", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:05:22.646892+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:05:22.415785+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "75d13454beecb6eb295cfbac90019a4ad39f0d88dca5b7c462a58e948707b997", + "size_bytes": 24268295 + }, + { + "modified_at": "2026-10-07T09:05:22.495789+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "fb3869bdb6c6fbbecc08ddec2f6073e625d0ca4a10c0a3c7627fb6ac329891ea", + "size_bytes": 12030 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:03:05.261894+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4154122248063175e-06, + "supra": 9.009319480931458e-07 + }, + "domains": 102375, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18549, + "runtime_seconds": 30.37, + "significant_associations": 164183, + "terms": 16057, + "tests": 1643835375, + "tests_evaluated": 2956096 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input.json new file mode 100644 index 0000000..77bac25 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input.json @@ -0,0 +1,207 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input", + "--propagate-annotations", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:12:06.791195+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:12:06.391160+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "d9a4839d17f7d2ded50b50174e432fd3d5efa2b2f1660c27339fe626e62f1098", + "size_bytes": 72223270 + }, + { + "modified_at": "2026-10-07T09:12:06.498164+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "613ad9c8915058383934f56a018913d4a79df55c9404dcba774ba3542f96a3dc", + "size_bytes": 12452 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:05:29.991004+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.814556384017671e-06, + "supra": 2.3364892845331883e-06 + }, + "domains": 102375, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4703, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 925, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18549, + "runtime_seconds": 106.68, + "significant_associations": 491823, + "terms": 18371, + "tests": 1880731125, + "tests_evaluated": 13567117 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_output.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_output.json new file mode 100644 index 0000000..2d4d3f7 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_output.json @@ -0,0 +1,215 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output", + "--propagate-annotations", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:38:06.848161+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:37:52.016610+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "d9a4839d17f7d2ded50b50174e432fd3d5efa2b2f1660c27339fe626e62f1098", + "size_bytes": 72223270 + }, + { + "modified_at": "2026-10-07T09:37:52.303621+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "613ad9c8915058383934f56a018913d4a79df55c9404dcba774ba3542f96a3dc", + "size_bytes": 12452 + }, + { + "modified_at": "2026-10-07T09:38:05.398088+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "0349b741ba8940359b49661f113fe64068dd3eec788abcebfdf67bce88ac8289", + "size_bytes": 101635906 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_output", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:25:32.034778+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.814556384017671e-06, + "supra": 2.3364892845331883e-06 + }, + "domains": 102375, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4703, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 925, + "ontology": "go", + "propagated_annotations": 1210884, + "proteins": 18549, + "runtime_seconds": 77.04, + "significant_associations": 491823, + "terms": 18371, + "tests": 1880731125, + "tests_evaluated": 13567117 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_relative.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_relative.json new file mode 100644 index 0000000..b2fe583 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_input_relative.json @@ -0,0 +1,215 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative", + "--propagate-annotations", + "--enable-relative-inference", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:25:24.029576+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:25:22.899575+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "cd9dd1fd0e1abe20dadd169f59be47912b70f45f199295b033c2f72937bff97a", + "size_bytes": 13519691 + }, + { + "modified_at": "2026-10-07T09:25:23.005579+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "079d5d65381c25ce44dfabe2c7c463450299e459860614fe9ec15c57e68fe6aa", + "size_bytes": 11727 + }, + { + "modified_at": "2026-10-07T09:25:23.878616+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "7f648077991815e4e61f059ed4f9fcdfbeb4f9d9034bca2396660c1674854ddc", + "size_bytes": 7454531 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_input_relative", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:18:37.636046+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 8.345192721392234e-07, + "supra": 4.137618898264933e-07 + }, + "domains": 102375, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4703, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 925, + "ontology": "go", + "propagated_annotations": 92774, + "proteins": 18549, + "runtime_seconds": 78.43, + "significant_associations": 92774, + "terms": 18371, + "tests": 1880731125, + "tests_evaluated": 13567117 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_output.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_output.json new file mode 100644 index 0000000..cb370c4 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_output.json @@ -0,0 +1,210 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:18:30.381157+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:18:23.101042+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "c7690040f2b0374ffd195dd08cc12ce93f42a740cea356a7507c5fd44ada7337", + "size_bytes": 24163148 + }, + { + "modified_at": "2026-10-07T09:18:23.194046+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "d5c33f1641d731c0e0ca62842d1338304738d4a7981aec3545f3ec01eb459ca0", + "size_bytes": 12032 + }, + { + "modified_at": "2026-10-07T09:18:29.593314+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "9436195f7768c3daf55b2da8333a8e3abc63ff77c3dd7c5431126a72b9d72e3b", + "size_bytes": 95442696 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_output", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:13:58.167803+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4154122248063175e-06, + "supra": 9.009319480931458e-07 + }, + "domains": 102375, + "ontology": "go", + "propagated_annotations": 1123671, + "proteins": 18549, + "runtime_seconds": 19.8, + "significant_associations": 164183, + "terms": 16057, + "tests": 1643835375, + "tests_evaluated": 2956096 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative.json new file mode 100644 index 0000000..73cb3f9 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative.json @@ -0,0 +1,210 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative", + "--enable-relative-inference", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:13:51.618921+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:13:51.296604+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "3f13341934cddfaf7ad484129a262bdea725f51019359e41a0fd00305a2b8d29", + "size_bytes": 4432375 + }, + { + "modified_at": "2026-10-07T09:13:51.399609+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "1416341deab91a0b72e58d689dadd5cee89c5d83e330525222958d324950f349", + "size_bytes": 12491 + }, + { + "modified_at": "2026-10-07T09:13:51.566616+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "19dd796347cd0c8d8827b1cd58f629b4cae2698a46df7bc1a6d8d360fb03e851", + "size_bytes": 2396461 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:12:09.885368+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.0779145917124034e-07, + "supra": 1.5936006275213104e-07 + }, + "domains": 102375, + "ontology": "go", + "propagated_annotations": 30795, + "proteins": 18549, + "runtime_seconds": 17.13, + "significant_associations": 30795, + "terms": 16057, + "tests": 1643835375, + "tests_evaluated": 2956096 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative_output.json b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative_output.json new file mode 100644 index 0000000..bb143ce --- /dev/null +++ b/validation/ablation_cells/ipr_manual_selctrl/manifests/supra_relative_output.json @@ -0,0 +1,211 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output", + "--enable-relative-inference", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz" + ], + "completed_at": "2026-10-07T09:40:57.691269+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:02:09.360543+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "role": "domain_annotations", + "sha256": "6b5f6f6453d1d87ffc25a1dc946868829cf93b9c00700ec069dad554f11cf041", + "size_bytes": 2910276 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:40:56.180329+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "3f13341934cddfaf7ad484129a262bdea725f51019359e41a0fd00305a2b8d29", + "size_bytes": 4432375 + }, + { + "modified_at": "2026-10-07T09:40:56.332335+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "1416341deab91a0b72e58d689dadd5cee89c5d83e330525222958d324950f349", + "size_bytes": 12491 + }, + { + "modified_at": "2026-10-07T09:40:57.545380+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "d9f0a7313f3935d913de0aa5506f02f78a7a4df899052d8d7e528145354b27c4", + "size_bytes": 13958102 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0t1sel_current.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual_selctrl/supra_relative_output", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:38:17.054450+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.0779145917124034e-07, + "supra": 1.5936006275213104e-07 + }, + "domains": 102375, + "ontology": "go", + "propagated_annotations": 167103, + "proteins": 18549, + "runtime_seconds": 23.26, + "significant_associations": 30795, + "terms": 16057, + "tests": 1643835375, + "tests_evaluated": 2956096 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_metrics.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_metrics.tsv new file mode 100644 index 0000000..efb2d6b --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_metrics.tsv @@ -0,0 +1,136 @@ +aspect min_ic method n_eval_proteins f_max f_max_tau s_min s_min_tau auprc coverage_at_fmax coverage_any f_max_ci_lo f_max_ci_hi auprc_ci_lo auprc_ci_hi +BP 0.0 single 298 0.22249798683037006 0.09989841446510787 86.8877513785436 0.35 0.10170480274958267 0.5100671140939598 0.5302013422818792 0.19281310503777815 0.2536666407353266 0.07798492807073945 0.12518479273298483 +BP 0.0 single__p 298 0.2213460100771406 0.14 86.59407298683541 0.27 0.10434530952954481 0.5100671140939598 0.5302013422818792 0.19234406291149592 0.25407194900644237 0.08072054253027136 0.12820153252139183 +BP 0.0 supra 298 0.22413162677065615 0.06715348172452834 86.79372672974917 0.32 0.10296371456335705 0.5100671140939598 0.5302013422818792 0.19331149515267135 0.25435838743734096 0.0790022906596768 0.12760346574111103 +BP 0.0 supra__p 298 0.22488492396071466 0.08 86.37825024640405 0.28 0.10491744186630761 0.5100671140939598 0.5302013422818792 0.19506420201858216 0.2562577543004452 0.08097900053830145 0.12882372423833582 +BP 0.0 supra_input 298 0.23795056705598774 0.048014609166444455 86.9911742141396 0.23 0.11319817605149689 0.5302013422818792 0.5369127516778524 0.20717497222996434 0.26964991479108413 0.08774184502894283 0.13802711362136155 +BP 0.0 supra_input__p 298 0.24243041872375237 0.054446696960895956 86.7740535890496 0.25 0.11579888439103121 0.5335570469798657 0.5369127516778524 0.2101350177506764 0.27269872223066227 0.0898277183987879 0.14094814315527496 +BP 0.0 supra_relative 298 0.1828820274820355 0.004614477378065632 88.48902089476411 0.25095821213628483 0.05601452492028774 0.4429530201342282 0.4463087248322148 0.15601682827261773 0.21188619737322653 0.03998433089235242 0.07182148847742129 +BP 0.0 supra_relative__p 298 0.18306278848767743 0.004805935671786969 88.3414392138553 0.28 0.05829950183777443 0.4429530201342282 0.4463087248322148 0.15648297824090956 0.21220702574898723 0.0420117173783903 0.0749717762913416 +BP 0.0 supra_output 298 0.22798887668423184 0.05 86.972276962279 0.13760903904735539 0.1066322320296524 0.5302013422818792 0.5302013422818792 0.198214442187476 0.2591555436885633 0.08277519820577127 0.13134506896750628 +BP 0.0 supra_input_relative 298 0.20953063532557034 0.0014585005278235255 88.01178414065564 0.13 0.08534064572151016 0.47651006711409394 0.47651006711409394 0.18326500102547913 0.23843174348572277 0.0648953465672673 0.10557952902306057 +BP 0.0 supra_input_relative__p 298 0.20903225329323197 0.0006107839301861984 87.85898803401392 0.14 0.0870134399634125 0.47651006711409394 0.47651006711409394 0.18330137302660302 0.23813616578958735 0.06613503595177023 0.10709399260183425 +BP 0.0 supra_input_output 298 0.24542121830129587 0.04 86.53789845885028 0.15 0.12250816012290822 0.5369127516778524 0.5369127516778524 0.21454979295974222 0.27426969981025406 0.09555741291254163 0.14795402966542653 +BP 0.0 supra_relative_output 298 0.18286793068745932 0.0021558582847676982 88.61737860529333 0.12 0.06504998231071517 0.4463087248322148 0.4463087248322148 0.1558116549034719 0.21091113775372936 0.04743097127644941 0.08314448495059065 +BP 0.0 full 298 0.21359025032890977 0.0200742174151682 87.55828498254478 0.11625170921940697 0.09146101838127958 0.47651006711409394 0.47651006711409394 0.18553915691279624 0.24152541730995888 0.07106302604445466 0.11211285456368059 +BP 0.0 naive 298 0.24522800676034753 0.246047504670403 87.81969159551754 0.246047504670403 0.1548729085557252 1.0 1.0 0.23210521828645966 0.26143307712405467 0.14126086596529053 0.168859588434164 +BP 2.0 single 298 0.16336579304170887 0.1 82.52011203836051 0.39 0.06435395211501753 0.5 0.5302013422818792 0.13497718415419427 0.19787694279006626 0.044041511363681236 0.09024409683976678 +BP 2.0 single__p 298 0.16434699946601633 0.05 82.3471132243319 0.27 0.06529027478234027 0.5067114093959731 0.5302013422818792 0.13643697845664096 0.19951615147175503 0.04499702028480657 0.09051912045700655 +BP 2.0 supra 298 0.16719947495645657 0.06119420622202558 82.40433429891804 0.32 0.06741104829568473 0.5067114093959731 0.5302013422818792 0.13756278461536184 0.201341625848625 0.0464052235482989 0.09278609027626185 +BP 2.0 supra__p 298 0.1686628053438091 0.08 82.11797374883251 0.28 0.06834573888098451 0.5067114093959731 0.5302013422818792 0.13921437934131398 0.20317069006845045 0.047209793240518376 0.09361184344861953 +BP 2.0 supra_input 298 0.18709980332277934 0.011115711485338527 82.7462847375373 0.23 0.06923837639082926 0.5369127516778524 0.5369127516778524 0.15791719537707755 0.2229779667156213 0.04811561185120662 0.09273828320577139 +BP 2.0 supra_input__p 298 0.1882513969773453 0.05 82.58212469265462 0.26 0.07073201537069143 0.5335570469798657 0.5369127516778524 0.15877389139856135 0.22421060360534154 0.04972007696969571 0.0941729480381909 +BP 2.0 supra_relative 298 0.1269083869719604 0.0 83.68706649147336 0.2552626080848716 0.029511278976423923 0.4463087248322148 0.4463087248322148 0.10059393779325806 0.15745248241020138 0.017542324199990734 0.04486414546718423 +BP 2.0 supra_relative__p 298 0.1269083869719604 0.0 83.66417309276632 0.28 0.031143489326776932 0.4463087248322148 0.4463087248322148 0.10059071153921433 0.15739863557555708 0.018842264256786267 0.04725431406808008 +BP 2.0 supra_output 298 0.17163715719644868 0.03679362677169721 82.87071239199304 0.14 0.0614848737185092 0.5302013422818792 0.5302013422818792 0.14359268898255764 0.20469021838796567 0.04353402911536712 0.08373200016068356 +BP 2.0 supra_input_relative 298 0.15601748308402721 0.0008052536629037432 83.50967607775333 0.12572133473224617 0.048512655192598404 0.47651006711409394 0.47651006711409394 0.12865760211560692 0.18648598087628968 0.03231921249919139 0.06852589490162443 +BP 2.0 supra_input_relative__p 298 0.15583973647073532 0.0007340701711616174 83.50003689351526 0.12 0.04913403198990984 0.47651006711409394 0.47651006711409394 0.12809637742161634 0.1861585171301051 0.03316602992502998 0.06937892527716746 +BP 2.0 supra_input_output 298 0.18832348586599781 0.028366288037686878 82.53978680143769 0.15 0.07413412064090152 0.5335570469798657 0.5369127516778524 0.15765363083727477 0.22234313586650423 0.05290543830910317 0.09863242424030925 +BP 2.0 supra_relative_output 298 0.12861775648663223 0.001696834680185576 84.04228080186851 0.12 0.029342555105922298 0.4463087248322148 0.4463087248322148 0.10066783892187485 0.1593960495776171 0.01766646449067979 0.04561408262215075 +BP 2.0 full 298 0.15384527790027153 0.00019134673692246167 83.39466338873828 0.09703421221556015 0.04745326817925877 0.47651006711409394 0.47651006711409394 0.12801080307211224 0.18366070145647173 0.0327907215182455 0.06646717830342067 +BP 2.0 naive 298 0.1393754232163564 0.12271470509741127 84.8347549558933 0.23094528956498533 0.05246757199300765 1.0 1.0 0.12932984580650728 0.14944692205357168 0.04676691808057873 0.05836943398200638 +BP 4.0 single 292 0.11394706576576076 0.03678762602680692 68.08871833821073 0.39 0.030207105451247915 0.5171232876712328 0.5376712328767124 0.08926809159440079 0.14954584362586532 0.017245686332366044 0.05072183306398461 +BP 4.0 single__p 292 0.1146597753989986 0.05 67.94000372308821 0.27 0.030808780789037673 0.5136986301369864 0.5376712328767124 0.08941670774564192 0.14953898053782716 0.018089751022990347 0.05051980161517959 +BP 4.0 supra 292 0.11828896656791534 0.06937479845266326 68.01247104546681 0.32 0.03405078849746584 0.5136986301369864 0.5376712328767124 0.09158276910051681 0.15479630852422055 0.01924315471709185 0.05633426304887034 +BP 4.0 supra__p 292 0.11899167821855004 0.08 67.76226916181743 0.28 0.03389564448456917 0.5136986301369864 0.5376712328767124 0.09155961965820825 0.1549471543375489 0.019879939448249714 0.05542305018270697 +BP 4.0 supra_input 292 0.13808913314840468 0.011250748370743965 68.33031002075263 0.23 0.03501200013295161 0.541095890410959 0.541095890410959 0.10770777031971389 0.17028173795124893 0.022043488116138828 0.052507952141066426 +BP 4.0 supra_input__p 292 0.13569836083654202 0.017569237147694487 68.26726873162998 0.26 0.03541735308524031 0.5376712328767124 0.541095890410959 0.10712995893139966 0.16890802231675442 0.022550451689563333 0.053189139088881655 +BP 4.0 supra_relative 292 0.0810288367233145 0.0 68.72696194405496 0.23 0.012185748885143198 0.4349315068493151 0.4349315068493151 0.05920076763828952 0.10763401050739496 0.0060454288350649515 0.02222579100507133 +BP 4.0 supra_relative__p 292 0.0810288367233145 0.0 68.74188172647207 0.28 0.012708398072293887 0.4349315068493151 0.4349315068493151 0.059509808992487895 0.10762919676277348 0.006470991013580163 0.023181576210058993 +BP 4.0 supra_output 292 0.11843834179404938 0.015571128617361969 68.59221151201046 0.15 0.027289542126364027 0.5376712328767124 0.5376712328767124 0.0906179617636832 0.1499382819693824 0.015974218261927593 0.04344893400929384 +BP 4.0 supra_input_relative 292 0.09412487118656662 0.0007794381399490565 68.79504130932712 0.16984127210668754 0.01917880743576971 0.4691780821917808 0.4691780821917808 0.07122290448604758 0.1208769533338649 0.010514631789564742 0.03291853630818295 +BP 4.0 supra_input_relative__p 292 0.09404999356906034 0.0007419381017406207 68.76623870885497 0.14 0.019434037854674985 0.4691780821917808 0.4691780821917808 0.07090257861959025 0.12026096061373409 0.010548147124307372 0.032859739029442614 +BP 4.0 supra_input_output 292 0.13286708464632122 0.012047035630113471 68.35193716452137 0.12675168821234928 0.034378134445179405 0.5376712328767124 0.541095890410959 0.10647686431964361 0.16594711086215846 0.0213468221754356 0.05110420591230628 +BP 4.0 supra_relative_output 292 0.08423567672572724 0.001800685947732063 69.02530338955407 0.08491738230215237 0.011990176873338164 0.4280821917808219 0.4349315068493151 0.0608189863049037 0.11004534910288762 0.005553105380766333 0.022297141106864336 +BP 4.0 full 292 0.09256562477377679 0.00022387914847785188 68.78454696141097 0.11827194318903653 0.019870938340269942 0.4691780821917808 0.4691780821917808 0.0698059421377737 0.11835817388841272 0.010702271852960623 0.034190210964174406 +BP 4.0 naive 292 0.04686917580427525 0.04859781158259942 69.96559437929291 0.06228983186549239 0.013722538414023608 1.0 1.0 0.04072911691354331 0.05358136988666269 0.012021347236526566 0.015701235045600684 +MF 0.0 single 406 0.34492218474153336 0.17716892876838156 12.339436295154627 0.75 0.1712192350176612 0.5541871921182266 0.6059113300492611 0.3084687265183643 0.38313143978206177 0.14074423718855322 0.2055940421273997 +MF 0.0 single__p 406 0.36221195395658357 0.15 12.376613562535065 0.58 0.18385640323308675 0.5738916256157636 0.6059113300492611 0.32622830742492714 0.40016324783304424 0.15289723154831197 0.22017231494185918 +MF 0.0 supra 406 0.34357928540123506 0.18 12.349404097946495 0.75 0.171545113956216 0.5541871921182266 0.6059113300492611 0.30764557953667326 0.380533384928944 0.14187225930135494 0.20671518056292762 +MF 0.0 supra__p 406 0.3621944577671508 0.17 12.34727206072705 0.59 0.18410152537732397 0.5665024630541872 0.6059113300492611 0.32701004701783176 0.40067485850644985 0.1535176476986988 0.2206526919130847 +MF 0.0 supra_input 406 0.36037094536457454 0.08518363139817497 12.497419153859703 0.44 0.17171632950127838 0.5935960591133005 0.6157635467980296 0.3259534003020335 0.3966926622669428 0.1415692991074623 0.20449695710952695 +MF 0.0 supra_input__p 406 0.3589762551084863 0.07088187919187379 12.433153753337507 0.34654431806202607 0.17556770998996815 0.5985221674876847 0.6157635467980296 0.32633975607835536 0.3949652525102139 0.145616860041627 0.2085776992949368 +MF 0.0 supra_relative 406 0.33460122141587784 0.08412158499526236 13.108054371589407 0.61 0.15555521573877742 0.5197044334975369 0.5541871921182266 0.3015517526200004 0.370010460422406 0.12664219893279027 0.18611525841560167 +MF 0.0 supra_relative__p 406 0.33457589953079847 0.11 13.034727197586157 0.64 0.16522418261293156 0.5123152709359606 0.5541871921182266 0.30344387685316687 0.37061591021883444 0.1360473512048916 0.19831677670993111 +MF 0.0 supra_output 406 0.35133717588529867 0.06704141095279345 12.568333135265066 0.29 0.18005597834700826 0.5911330049261084 0.6059113300492611 0.3160187518111587 0.38837217556284825 0.14931950349409082 0.21413209049349288 +MF 0.0 supra_input_relative 406 0.3569474047131703 0.1069149455335412 13.092528303731115 0.37 0.1908100853598182 0.5541871921182266 0.5935960591133005 0.3219377291415968 0.39449766256088564 0.1596097406634191 0.22661959473217833 +MF 0.0 supra_input_relative__p 406 0.3645089435605144 0.13 13.060555360521892 0.36 0.200339825934053 0.5541871921182266 0.5935960591133005 0.3292951356683228 0.4041742860877765 0.16767878016751975 0.23741929390040273 +MF 0.0 supra_input_output 406 0.36076776543325934 0.07 12.40420389692 0.29 0.1899294765403534 0.5960591133004927 0.6157635467980296 0.32737642297956243 0.3970705698869258 0.15789374151563906 0.2248019747488702 +MF 0.0 supra_relative_output 406 0.3378752801234797 0.13 13.266442923824512 0.24 0.1846921135015052 0.47783251231527096 0.5541871921182266 0.3072835674659054 0.3817010760333571 0.15234177481545744 0.2221319190691977 +MF 0.0 full 406 0.37612656480454426 0.07 12.931245174449371 0.17 0.21123115567352646 0.5591133004926109 0.5935960591133005 0.34036037419236675 0.41307852223400165 0.17632975885253693 0.24892456078060854 +MF 0.0 naive 406 0.7189989169653068 0.2616402455297571 13.241152459749525 0.15066399786495863 0.4190445836593093 1.0 1.0 0.6771715184983654 0.7610189683025428 0.378372572506467 0.4606028266213177 +MF 2.0 single 159 0.33028872962052275 0.47 26.78666924042098 0.47 0.17950175005084912 0.4779874213836478 0.6415094339622641 0.26731261070442597 0.39985008818031886 0.1284843438418251 0.23506173461823807 +MF 2.0 single__p 159 0.3305578146526893 0.41 26.888797593228542 0.41 0.1886904292314876 0.48427672955974843 0.6415094339622641 0.26663341834843285 0.3997229451430598 0.13767431880726466 0.24394401144350777 +MF 2.0 supra 159 0.32980140754537307 0.43 26.964676743457456 0.47 0.18918327277822655 0.4779874213836478 0.6415094339622641 0.26654419543512764 0.39937083343368535 0.13436021900800318 0.24869617699983987 +MF 2.0 supra__p 159 0.3319855271740378 0.33 26.94842505314254 0.44 0.2014569737304722 0.48427672955974843 0.6415094339622641 0.2672355154866301 0.40292095289722213 0.14490322925397475 0.2637143255057515 +MF 2.0 supra_input 159 0.3351947698693985 0.12 27.357003255536508 0.26 0.2080034865452522 0.5534591194968553 0.660377358490566 0.2736210667638443 0.40154534225856503 0.15350292245958572 0.2701195740453789 +MF 2.0 supra_input__p 159 0.3333475397042083 0.12 27.118355039149538 0.25 0.21157893509884837 0.5534591194968553 0.660377358490566 0.27436179312762415 0.40064083771524656 0.15526092673814043 0.2741257196509607 +MF 2.0 supra_relative 159 0.25945157755716214 0.020598564717750692 30.811274330362174 0.33 0.09626807382427224 0.5471698113207547 0.5786163522012578 0.20912476900185442 0.32196086158055087 0.05916898382208687 0.14339065881727875 +MF 2.0 supra_relative__p 159 0.2582599234110892 0.013937133494698222 30.595791206589045 0.31 0.10003529885628203 0.5471698113207547 0.5786163522012578 0.20582979350993094 0.31916045777376634 0.06210555121366203 0.1482932812790203 +MF 2.0 supra_output 159 0.31817620684333403 0.08 27.966711726170153 0.14 0.186367547422579 0.5786163522012578 0.6415094339622641 0.26064697790389885 0.38235325720887114 0.13289349788080257 0.250213652498006 +MF 2.0 supra_input_relative 159 0.28809539534087936 0.044021693677736465 29.879270705468386 0.18970843340853166 0.153532150824977 0.610062893081761 0.6289308176100629 0.23414491981481042 0.35148210427065607 0.10599302731321379 0.21074995374820624 +MF 2.0 supra_input_relative__p 159 0.2898908419810066 0.05521095168554714 29.718591262887312 0.1550445696485765 0.15566383964945713 0.5974842767295597 0.6289308176100629 0.23526970703105443 0.3513941859064825 0.10690519014013093 0.2131085491943383 +MF 2.0 supra_input_output 159 0.3279453161295411 0.09 27.333467650496292 0.15 0.20413914043745163 0.559748427672956 0.660377358490566 0.2684133242058567 0.39322748569425225 0.14843001991030544 0.26698284101716674 +MF 2.0 supra_relative_output 159 0.2755198263008129 0.01 30.73731771409061 0.13 0.11189982515833893 0.5660377358490566 0.5786163522012578 0.2211531907399026 0.3356348736877796 0.07062969303185054 0.16489693851770146 +MF 2.0 full 159 0.2973310593460379 0.03425040187938083 29.623342429358463 0.11 0.15168797716709995 0.6037735849056604 0.6289308176100629 0.24495241646522972 0.3529993636442451 0.10346327738650063 0.2084846266654447 +MF 2.0 naive 159 0.07561162501367721 0.050395516413130496 33.409141271799434 0.1096215639178009 0.026528609897190324 1.0 1.0 0.06416366594110963 0.08958879742758147 0.021516874326196884 0.03240999023166993 +MF 4.0 single 151 0.3074692728343345 0.47 26.281427111840422 0.47 0.17600599998991664 0.4768211920529801 0.6423841059602649 0.2467693779171546 0.37915332597676477 0.12410000301313609 0.23798475548900797 +MF 4.0 single__p 151 0.3106511933673489 0.41 26.35326623351266 0.41 0.18609783948253975 0.4768211920529801 0.6423841059602649 0.24870934509687417 0.38218038295609463 0.1319257487925015 0.2480780885998367 +MF 4.0 supra 151 0.30712672755850184 0.43 26.433855423737835 0.47 0.17827778650251944 0.48344370860927155 0.6423841059602649 0.24744885578910095 0.38032206607700625 0.12602274276448922 0.2422356676866348 +MF 4.0 supra__p 151 0.31203881441833375 0.33 26.404060805411763 0.44 0.19012058590611525 0.48344370860927155 0.6423841059602649 0.2499790887795101 0.38506060406724385 0.13514096899041114 0.2541827117161742 +MF 4.0 supra_input 151 0.32507657135803186 0.12466307578617696 26.735562003654014 0.21 0.19253720279059416 0.5364238410596026 0.6622516556291391 0.2585365881869558 0.39554258743432946 0.13698239588158956 0.2559968725023434 +MF 4.0 supra_input__p 151 0.32104340384136604 0.12 26.52967263065358 0.25 0.1979483914608095 0.5364238410596026 0.6622516556291391 0.2583620886053564 0.3911589269323172 0.1423108082459032 0.2632995082420783 +MF 4.0 supra_relative 151 0.23755767703676015 0.08 29.731016459747003 0.61 0.08319590331958229 0.5231788079470199 0.5562913907284768 0.19048541841786779 0.3010691010514384 0.05024733560507399 0.12458275240044743 +MF 4.0 supra_relative__p 151 0.2342221558233697 0.013663116899985225 29.61224420113873 0.48 0.08628828700813716 0.5298013245033113 0.5562913907284768 0.18812822166513665 0.29086876447430826 0.052670545864608787 0.12955831286900857 +MF 4.0 supra_output 151 0.3059033984033558 0.08 27.203793885655614 0.13 0.1734068803259186 0.5562913907284768 0.6423841059602649 0.24625478629027392 0.3724643187488862 0.12200191690183916 0.23723762170601537 +MF 4.0 supra_input_relative 151 0.26256167023394233 0.08 29.167587356354584 0.19 0.12106071224226855 0.5827814569536424 0.6291390728476821 0.21246093747873138 0.33034102027550116 0.0796407988980092 0.17002087932413046 +MF 4.0 supra_input_relative__p 151 0.26121843793527255 0.06813144646794143 29.04083795789984 0.1542971336325714 0.12186587415524539 0.5761589403973509 0.6291390728476821 0.21135479120395917 0.3253880837970298 0.0808223515005416 0.17065602284487763 +MF 4.0 supra_input_output 151 0.3140200277746999 0.09 26.670624237021972 0.15 0.18749109239613948 0.543046357615894 0.6622516556291391 0.2505285355175019 0.3812181298598036 0.13468090791308698 0.2514218979174193 +MF 4.0 supra_relative_output 151 0.24801508792251012 0.003975484111015619 29.872060694256437 0.13 0.08971683359636826 0.5496688741721855 0.5562913907284768 0.19621287638712429 0.3035665253357036 0.05564968230995859 0.13416309322411207 +MF 4.0 full 151 0.2697141121356497 0.03286668011344428 28.951125165496304 0.11 0.12442969869416227 0.5827814569536424 0.6291390728476821 0.21671656930335412 0.3315260215158943 0.08237446109231714 0.1756640909003498 +MF 4.0 naive 151 0.07531254466979251 0.05290899386175607 31.782400402647035 0.12447291166266346 0.02307128915639914 1.0 1.0 0.060367845875674626 0.09666103185263615 0.018552059417728722 0.028392641107576045 +CC 0.0 single 436 0.37266279839932215 0.01718796417807132 21.723271480427258 0.19 0.2138930026796082 0.49770642201834864 0.5091743119266054 0.3421806724816551 0.40289896061924285 0.18243504493470347 0.24603899509545693 +CC 0.0 single__p 436 0.3780926705265978 0.02 21.627290848243998 0.17 0.2236545551302106 0.49770642201834864 0.5091743119266054 0.3475109334916413 0.40695404178592987 0.19087489565178095 0.2559596412664913 +CC 0.0 supra 436 0.37369500099331754 0.02024917060432352 21.740891751806338 0.18 0.21425352431010342 0.48853211009174313 0.5091743119266054 0.34416807400636723 0.40370961166967595 0.182677053257544 0.2446591353173193 +CC 0.0 supra__p 436 0.37355221596083305 0.02 21.637490975634254 0.17230075402994743 0.22298511934140713 0.5 0.5091743119266054 0.3438547700903447 0.40326618423392624 0.19112329507821435 0.253763420142237 +CC 0.0 supra_input 436 0.3657545116537211 0.004050468792212149 22.181549799758105 0.14 0.2001142781223428 0.5022935779816514 0.5068807339449541 0.3372488598582261 0.3943031612665119 0.17155938268540283 0.22906074070079602 +CC 0.0 supra_input__p 436 0.3677546963798026 0.01 22.23395940730139 0.15 0.20601254429057944 0.5 0.5068807339449541 0.33877980939786845 0.3955964808984464 0.1770672829742306 0.23562191692186582 +CC 0.0 supra_relative 436 0.3441068514445836 0.010584885172989004 21.97356942844991 0.26 0.17407450637713057 0.4426605504587156 0.4518348623853211 0.31297931709183874 0.3759395868677993 0.14788157867214474 0.20105776667283387 +CC 0.0 supra_relative__p 436 0.3455932923700789 0.01656610049576198 21.952572541962887 0.29 0.1762350349703178 0.4426605504587156 0.4518348623853211 0.31415276584816304 0.37773084710685206 0.1505797172034272 0.2034824624490701 +CC 0.0 supra_output 436 0.38167092847103545 0.017733046241645255 21.75402738285492 0.08184804960912312 0.24331519913843716 0.5022935779816514 0.5091743119266054 0.3514143098996769 0.4111871968056527 0.21099380024620323 0.27457928483805705 +CC 0.0 supra_input_relative 436 0.36749458141809443 0.0025277787512022654 22.05529850582212 0.13 0.1990108702115158 0.48623853211009177 0.49770642201834864 0.3370950181862267 0.3972176754328753 0.16919711923699154 0.23035202702616941 +CC 0.0 supra_input_relative__p 436 0.3695861315205082 0.007488904844379206 21.99341937113148 0.17 0.2038803910130888 0.48394495412844035 0.49770642201834864 0.3390728548427332 0.39859723639361044 0.17329002583396064 0.23433561417797155 +CC 0.0 supra_input_output 436 0.3808275731115995 0.012731292007165976 21.847933167905186 0.08 0.23984939219421486 0.5045871559633027 0.5068807339449541 0.3506748640181658 0.4096877644585293 0.20891414432528743 0.2714313310658922 +CC 0.0 supra_relative_output 436 0.34872847111799476 0.004196835802375299 21.851359298745262 0.11 0.20876423335308655 0.4518348623853211 0.4518348623853211 0.32002748201052184 0.3800322827418281 0.1814113836090548 0.2381566588189041 +CC 0.0 full 436 0.37665530089167504 0.00597060951020819 21.79983886316594 0.09 0.22855037107450735 0.49770642201834864 0.49770642201834864 0.3452513681508351 0.40572986842131703 0.19726272105912362 0.2592607018624296 +CC 0.0 naive 436 0.593050572267429 0.2848678943154524 19.830858368031098 0.2848678943154524 0.5414744252479067 1.0 1.0 0.5760583007709689 0.6133359283683919 0.5183452559067593 0.5650951654931785 +CC 2.0 single 298 0.2010995525310512 0.00376532540329356 24.554428803996856 0.14 0.055619578943533254 0.4966442953020134 0.5067114093959731 0.16769556017003176 0.23594128521861632 0.03562707078258947 0.0814483460046235 +CC 2.0 single__p 298 0.2011396385774765 0.0025851608181386063 24.436346124460737 0.1778576746805374 0.05692494402359471 0.4966442953020134 0.5067114093959731 0.16779131378746093 0.2355373117687655 0.03702200344288895 0.083458131478828 +CC 2.0 supra 298 0.2001711050330419 0.0011466507008032524 24.409917958156385 0.14365111289642082 0.05474025260411656 0.4966442953020134 0.5100671140939598 0.16758793463792 0.23466333977876558 0.035309537799945355 0.07964331860489772 +CC 2.0 supra__p 298 0.20187137589279572 0.0010655360924254 24.400774884951876 0.17560888396657337 0.055655761468655265 0.4966442953020134 0.5100671140939598 0.16823053208332478 0.23487674450989665 0.03659628949290166 0.08073232156183999 +CC 2.0 supra_input 298 0.18984512084059801 6.084350222272058e-05 24.75648857292668 0.14 0.05384834822245379 0.5067114093959731 0.5067114093959731 0.15896300591472218 0.22163459716673842 0.0355902569205766 0.07797961424456311 +CC 2.0 supra_input__p 298 0.18211639274675243 1.7268932979630467e-05 24.81226435090463 0.12 0.05398611825551895 0.5 0.5067114093959731 0.15187820144073247 0.21313245438252823 0.036367528612407225 0.077158733092604 +CC 2.0 supra_relative 298 0.17064440645913162 0.0 25.031556811376692 0.25 0.04053061546270086 0.4395973154362416 0.4395973154362416 0.13791528537421585 0.20201514219950303 0.025939533791448373 0.060315719464642634 +CC 2.0 supra_relative__p 298 0.17064440645913162 0.0 25.01702329221099 0.26 0.039936813345497056 0.4395973154362416 0.4395973154362416 0.13791528537421585 0.20201514219950303 0.0257915647728008 0.05900693965828432 +CC 2.0 supra_output 298 0.20541777466459651 0.0012358841271541334 24.555591496006492 0.10499999999999997 0.05734132917056218 0.5033557046979866 0.5100671140939598 0.17072006007482773 0.23879797834315666 0.03899292851401399 0.08040706076284534 +CC 2.0 supra_input_relative 298 0.19658370250400492 0.0014291514996182756 24.720855841034414 0.13 0.048846805703567135 0.46308724832214765 0.4966442953020134 0.1622319782315144 0.23077897410929377 0.032238947396599736 0.07109249630582179 +CC 2.0 supra_input_relative__p 298 0.19802534564826085 0.0020317301017947534 24.66845726676502 0.18 0.04894810966648133 0.4597315436241611 0.4966442953020134 0.1636542575584525 0.23267582960000263 0.03248516841137566 0.07012797880545765 +CC 2.0 supra_input_output 298 0.2093688753175775 0.0011977705194741986 24.651667948353246 0.08250000000000002 0.06107642913396808 0.5067114093959731 0.5067114093959731 0.17522913472064208 0.2437189954299575 0.04281782986603757 0.08557204091305896 +CC 2.0 supra_relative_output 298 0.1798405527981575 0.002309104381263972 24.967150506894004 0.11098916268723019 0.041418922970986816 0.41946308724832215 0.4395973154362416 0.14546547235755838 0.21555030604804676 0.02720644518396849 0.05987913275490189 +CC 2.0 full 298 0.20442953930504945 0.0025477051258693995 24.773580030816984 0.09133844359913025 0.04934663831207692 0.4865771812080537 0.4966442953020134 0.17146931232493912 0.240310034624961 0.033849531314033655 0.07057517856644982 +CC 2.0 naive 298 0.2720885241633347 0.10965999466239659 24.37542448179979 0.20930237523352016 0.1451197420848838 1.0 1.0 0.2500562969208015 0.30046153488752814 0.12937238422026592 0.16212109267301172 +CC 4.0 single 210 0.12866270049406495 0.002914903657190658 24.364293879262732 0.14 0.031321891529791215 0.4380952380952381 0.49047619047619045 0.0907029477675837 0.17545006640967137 0.01365090553259718 0.05688411210415567 +CC 4.0 single__p 210 0.13186903808317027 0.06 24.298914610215764 0.17619820570548286 0.03138378817607565 0.3380952380952381 0.49047619047619045 0.0932898750746756 0.1810028341378874 0.013721690915177274 0.05706365651197253 +CC 4.0 supra 210 0.12814077896255177 0.00342973289383412 24.337124287820338 0.1417303749005237 0.030098649903147593 0.44761904761904764 0.5 0.09223009641111476 0.1775940720403129 0.013555499652804231 0.05484195295502918 +CC 4.0 supra__p 210 0.12870728549428606 0.07 24.381478266202013 0.18 0.031027841690027574 0.3047619047619048 0.5 0.09200808491115726 0.1768103309729788 0.013751085737817087 0.05589872425662921 +CC 4.0 supra_input 210 0.1263849974354251 0.027018288150707056 24.75307167757782 0.14 0.03115695182023453 0.3476190476190476 0.5 0.09186469060647362 0.17142619560060005 0.014663143488830573 0.05542990004290054 +CC 4.0 supra_input__p 210 0.12646422272303945 0.03 24.812526727727626 0.12 0.03039523169691337 0.3476190476190476 0.5 0.09096459268585258 0.17143432039006978 0.014319616671334636 0.05374297462763919 +CC 4.0 supra_relative 210 0.06727985486494541 2.9593598308423522e-05 24.946055020013787 0.25 0.011264545566505739 0.3476190476190476 0.35714285714285715 0.03893113839518423 0.10351397479923638 0.003193237252378926 0.02725706681802822 +CC 4.0 supra_relative__p 210 0.06727985486494541 6.284760489552718e-05 24.92513764426102 0.27 0.011148637625545797 0.3476190476190476 0.35714285714285715 0.03893113839518423 0.10432319339323724 0.003321929366555105 0.028211637984659962 +CC 4.0 supra_output 210 0.11733779836398164 0.000672999694539203 24.557840216149255 0.059999999999999984 0.03523412972085431 0.44761904761904764 0.5 0.0858596957814457 0.16422966937122987 0.01553710075290023 0.06226906287420557 +CC 4.0 supra_input_relative 210 0.11463965887981685 0.0 24.721460840395665 0.09 0.026881474129342588 0.42857142857142855 0.42857142857142855 0.07730170118870792 0.15458358034425704 0.011613819499487055 0.050188529348180876 +CC 4.0 supra_input_relative__p 210 0.11463965887981685 0.0 24.86179984197727 0.14 0.025868292428221272 0.42857142857142855 0.42857142857142855 0.077696482884192 0.1549188978666504 0.011183955084288077 0.047806813563431104 +CC 4.0 supra_input_output 210 0.12239568929514537 0.016449130429118147 24.644317680959723 0.0675 0.03416233160249707 0.35714285714285715 0.5 0.08763438803973077 0.16751191621356495 0.015649996011760377 0.059255797709503864 +CC 4.0 supra_relative_output 210 0.06563727515811953 0.0005330076270346414 25.001579240447086 0.11 0.013385795583400913 0.32857142857142857 0.35714285714285715 0.03830528481871662 0.10298075369487947 0.0034006213273407673 0.03327655134650106 +CC 4.0 full 210 0.12055865495429378 0.0009548591988179063 24.89667738592882 0.10043225463152751 0.026574800701653457 0.4095238095238095 0.42857142857142855 0.07951713662947629 0.1583595364913902 0.011462280988070898 0.04852016780044499 +CC 4.0 naive 210 0.10440217265761789 0.04063624232719509 25.485077592733788 0.12308513477448625 0.04494942594240296 1.0 1.0 0.09374817360504709 0.12093250379485652 0.039116125355083756 0.051429210681231546 diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_paired_bootstrap.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_paired_bootstrap.tsv new file mode 100644 index 0000000..6da0e57 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_paired_bootstrap.tsv @@ -0,0 +1,415 @@ +aspect min_ic n_eval_proteins metric method_a method_b observed_a observed_b observed_diff diff_mean diff_ci_lo diff_ci_hi p_value significant n_replicates +BP 0.0 298 f_max supra single 0.22413162677065615 0.22249798683037006 0.0016336399402860913 0.0016459531098986498 -0.004138930848447055 0.006448904331797602 0.5 False 1000 +BP 0.0 298 auprc supra single 0.10296371456335705 0.10170480274958267 0.0012589118137743827 0.0012459091994407537 -0.001336634911401756 0.004122234679168353 0.36 False 1000 +BP 0.0 298 f_max supra_input supra 0.23795056705598774 0.22413162677065615 0.013818940285331593 0.014346839099495586 0.0009422077757458584 0.030882158782408953 0.034 True 1000 +BP 0.0 298 auprc supra_input supra 0.11319817605149689 0.10296371456335705 0.010234461488139834 0.010344406351778471 0.001163491940947902 0.019659896337491822 0.026 True 1000 +BP 0.0 298 f_max supra_relative supra 0.1828820274820355 0.22413162677065615 -0.04124959928862065 -0.04026538412184658 -0.0602692345489202 -0.021177423344938526 0.0 True 1000 +BP 0.0 298 auprc supra_relative supra 0.05601452492028774 0.10296371456335705 -0.046949189643069315 -0.046415473035051925 -0.06150099585644608 -0.03200667635499007 0.0 True 1000 +BP 0.0 298 f_max supra_output supra 0.22798887668423184 0.22413162677065615 0.0038572499135756944 0.003670314714812589 -0.005725559379001808 0.014812013120821995 0.516 False 1000 +BP 0.0 298 auprc supra_output supra 0.1066322320296524 0.10296371456335705 0.0036685174662953457 0.003465775817876727 -0.0036954442719152577 0.010771335669492826 0.342 False 1000 +BP 0.0 298 f_max supra_input_relative supra_input 0.20953063532557034 0.23795056705598774 -0.028419931730417403 -0.028437734761832444 -0.04638271619133204 -0.010802574891881602 0.002 True 1000 +BP 0.0 298 auprc supra_input_relative supra_input 0.08534064572151016 0.11319817605149689 -0.027857530329986724 -0.027669490608419157 -0.0397501067158603 -0.01670939675211852 0.0 True 1000 +BP 0.0 298 f_max supra_input_relative supra_relative 0.20953063532557034 0.1828820274820355 0.026648607843534844 0.026174488459509714 0.010914371037784263 0.042421996074178514 0.002 True 1000 +BP 0.0 298 auprc supra_input_relative supra_relative 0.08534064572151016 0.05601452492028774 0.029326120801222426 0.029090388778411243 0.01901661422705688 0.04008757950848356 0.0 True 1000 +BP 0.0 298 f_max supra_input_output supra_input 0.24542121830129587 0.23795056705598774 0.00747065124530813 0.0058202532960267665 -0.0007806262898861009 0.012762909582664413 0.082 False 1000 +BP 0.0 298 auprc supra_input_output supra_input 0.12250816012290822 0.11319817605149689 0.009309984071411329 0.00904822653379702 0.005008115327774226 0.013447061122054023 0.0 True 1000 +BP 0.0 298 f_max supra_input_output supra_output 0.24542121830129587 0.22798887668423184 0.01743234161706403 0.016496777680709764 0.006510715082528744 0.02811432089020222 0.0 True 1000 +BP 0.0 298 auprc supra_input_output supra_output 0.12250816012290822 0.1066322320296524 0.015875928093255817 0.015926857067698762 0.00845880956454483 0.024210783001729144 0.0 True 1000 +BP 0.0 298 f_max supra_relative_output supra_relative 0.18286793068745932 0.1828820274820355 -1.4096794576173854e-05 0.00012870371947618195 -0.005735185964147876 0.005368182832991535 1.0 False 1000 +BP 0.0 298 auprc supra_relative_output supra_relative 0.06504998231071517 0.05601452492028774 0.00903545739042743 0.00895528854979106 0.003911491109383008 0.014370967006863425 0.0 True 1000 +BP 0.0 298 f_max supra_relative_output supra_output 0.18286793068745932 0.22798887668423184 -0.04512094599677252 -0.04380699511718298 -0.06548698575261086 -0.024359447184578612 0.0 True 1000 +BP 0.0 298 auprc supra_relative_output supra_output 0.06504998231071517 0.1066322320296524 -0.04158224971893723 -0.040925960303137596 -0.056272876540454035 -0.026081168321321502 0.0 True 1000 +BP 0.0 298 f_max full supra_input_relative 0.21359025032890977 0.20953063532557034 0.0040596150033394285 0.0032017065132501524 -0.002500878384919257 0.010828960968758173 0.392 False 1000 +BP 0.0 298 auprc full supra_input_relative 0.09146101838127958 0.08534064572151016 0.006120372659769419 0.0059876521978983275 0.0023106465885821873 0.009828920275697343 0.0 True 1000 +BP 0.0 298 f_max full supra_input_output 0.21359025032890977 0.24542121830129587 -0.031830967972386104 -0.031056281544609057 -0.04848733384504038 -0.014305492744508935 0.0 True 1000 +BP 0.0 298 auprc full supra_input_output 0.09146101838127958 0.12250816012290822 -0.031047141741628634 -0.030730064944317844 -0.04390329696817864 -0.018865585888635315 0.0 True 1000 +BP 0.0 298 f_max full supra_relative_output 0.21359025032890977 0.18286793068745932 0.030722319641450446 0.029247491253283683 0.015237939325843585 0.04434061890788873 0.0 True 1000 +BP 0.0 298 auprc full supra_relative_output 0.09146101838127958 0.06504998231071517 0.026411036070564414 0.02612275242651851 0.016561702851054527 0.03694457405719588 0.0 True 1000 +BP 0.0 298 f_max single naive 0.22249798683037006 0.24522800676034753 -0.022730019929977474 -0.0233018611517397 -0.056485034339365144 0.009045853409657894 0.152 False 1000 +BP 0.0 298 auprc single naive 0.10170480274958267 0.1548729085557252 -0.05316810580614252 -0.05340211043776147 -0.07763743602640584 -0.028041396002968587 0.0 True 1000 +BP 0.0 298 f_max supra naive 0.22413162677065615 0.24522800676034753 -0.021096379989691383 -0.021655908041841053 -0.05519438494167542 0.01141323918303006 0.178 False 1000 +BP 0.0 298 auprc supra naive 0.10296371456335705 0.1548729085557252 -0.05190919399236814 -0.05215620123832072 -0.07793097316769296 -0.02540572641370805 0.0 True 1000 +BP 0.0 298 f_max supra_input naive 0.23795056705598774 0.24522800676034753 -0.007277439704359789 -0.007309068942345465 -0.04003035422913827 0.02372020959433673 0.654 False 1000 +BP 0.0 298 auprc supra_input naive 0.11319817605149689 0.1548729085557252 -0.041674732504228304 -0.04181179488654224 -0.0685937903117417 -0.014897000890456605 0.006 True 1000 +BP 0.0 298 f_max supra_relative naive 0.1828820274820355 0.24522800676034753 -0.062345979278312036 -0.06192129216368762 -0.09247204601641691 -0.032514421412841436 0.002 True 1000 +BP 0.0 298 auprc supra_relative naive 0.05601452492028774 0.1548729085557252 -0.09885838363543745 -0.09857167427337266 -0.11675928324659164 -0.08096893555525142 0.0 True 1000 +BP 0.0 298 f_max supra_output naive 0.22798887668423184 0.24522800676034753 -0.01723913007611569 -0.01798559332702846 -0.048028626431536346 0.0121531513899185 0.242 False 1000 +BP 0.0 298 auprc supra_output naive 0.1066322320296524 0.1548729085557252 -0.04824067652607279 -0.04869042542044399 -0.07311329281215069 -0.024014212907129878 0.002 True 1000 +BP 0.0 298 f_max supra_input_relative naive 0.20953063532557034 0.24522800676034753 -0.03569737143477719 -0.035746803704177904 -0.06543179574782601 -0.005754629794364819 0.024 True 1000 +BP 0.0 298 auprc supra_input_relative naive 0.08534064572151016 0.1548729085557252 -0.06953226283421503 -0.06948128549496141 -0.09152721676950173 -0.046354512356470805 0.0 True 1000 +BP 0.0 298 f_max supra_input_output naive 0.24542121830129587 0.24522800676034753 0.00019321154094834037 -0.0014888156463186979 -0.03248989845493351 0.02810031402953173 0.934 False 1000 +BP 0.0 298 auprc supra_input_output naive 0.12250816012290822 0.1548729085557252 -0.032364748432816975 -0.03276356835274522 -0.05936712216129517 -0.006411309107455392 0.016 True 1000 +BP 0.0 298 f_max supra_relative_output naive 0.18286793068745932 0.24522800676034753 -0.06236007607288821 -0.06179258844421145 -0.09217992675457705 -0.03218110884599477 0.002 True 1000 +BP 0.0 298 auprc supra_relative_output naive 0.06504998231071517 0.1548729085557252 -0.08982292624501002 -0.08961638572358159 -0.10835879839983145 -0.06888635739756882 0.0 True 1000 +BP 0.0 298 f_max full naive 0.21359025032890977 0.24522800676034753 -0.031637756431437764 -0.03254509719092775 -0.06153439520962624 -0.0022529111262894596 0.032 True 1000 +BP 0.0 298 auprc full naive 0.09146101838127958 0.1548729085557252 -0.06341189017444561 -0.06349363329706308 -0.08562207921207737 -0.039943806253720335 0.0 True 1000 +BP 0.0 298 f_max full single 0.21359025032890977 0.22249798683037006 -0.00890773650146029 -0.00924323603918806 -0.022781735211248464 0.005079462971681821 0.214 False 1000 +BP 0.0 298 auprc full single 0.09146101838127958 0.10170480274958267 -0.010243784368303088 -0.010091522859301604 -0.020850913207516246 0.0003479965947825402 0.054 False 1000 +BP 2.0 298 f_max supra single 0.16719947495645657 0.16336579304170887 0.0038336819147476997 0.003202546168903143 -0.0009288543663407704 0.00790993589290752 0.124 False 1000 +BP 2.0 298 auprc supra single 0.06741104829568473 0.06435395211501753 0.0030570961806671987 0.0030668506913033174 0.0005149848141013887 0.005810969466460309 0.014 True 1000 +BP 2.0 298 f_max supra_input supra 0.18709980332277934 0.16719947495645657 0.019900328366322773 0.021047292142818155 0.00569121377286149 0.038963701265028454 0.002 True 1000 +BP 2.0 298 auprc supra_input supra 0.06923837639082926 0.06741104829568473 0.0018273280951445342 0.0015649239823222656 -0.009685034490327012 0.012016942911509851 0.772 False 1000 +BP 2.0 298 f_max supra_relative supra 0.1269083869719604 0.16719947495645657 -0.040291087984496154 -0.04078698355592705 -0.061629610719671456 -0.021235093531238543 0.0 True 1000 +BP 2.0 298 auprc supra_relative supra 0.029511278976423923 0.06741104829568473 -0.037899769319260805 -0.03819017163885117 -0.05582162509549128 -0.02312704529377975 0.0 True 1000 +BP 2.0 298 f_max supra_output supra 0.17163715719644868 0.16719947495645657 0.004437682239992108 0.0037666145553696493 -0.006053809039961117 0.014150056069352317 0.466 False 1000 +BP 2.0 298 auprc supra_output supra 0.0614848737185092 0.06741104829568473 -0.005926174577175529 -0.006226852258996011 -0.01565141133681055 0.0021641269055908486 0.156 False 1000 +BP 2.0 298 f_max supra_input_relative supra_input 0.15601748308402721 0.18709980332277934 -0.031082320238752126 -0.03292624262941415 -0.054758216316987074 -0.013669260613703478 0.0 True 1000 +BP 2.0 298 auprc supra_input_relative supra_input 0.048512655192598404 0.06923837639082926 -0.020725721198230858 -0.020658136211031027 -0.032969478247984976 -0.009721569304978896 0.002 True 1000 +BP 2.0 298 f_max supra_input_relative supra_relative 0.15601748308402721 0.1269083869719604 0.0291090961120668 0.028908033069331058 0.014488168272393513 0.04395796359745345 0.0 True 1000 +BP 2.0 298 auprc supra_input_relative supra_relative 0.048512655192598404 0.029511278976423923 0.01900137621617448 0.019096959410142406 0.010271066623110033 0.029300557201668055 0.0 True 1000 +BP 2.0 298 f_max supra_input_output supra_input 0.18832348586599781 0.18709980332277934 0.001223682543218474 -0.0007010510581341391 -0.008744424886583517 0.005707656125441784 0.898 False 1000 +BP 2.0 298 auprc supra_input_output supra_input 0.07413412064090152 0.06923837639082926 0.004895744250072262 0.004908335794365002 0.0013851571298708868 0.008709189300497544 0.006 True 1000 +BP 2.0 298 f_max supra_input_output supra_output 0.18832348586599781 0.17163715719644868 0.01668632866954914 0.016579626529314363 0.006125818661938092 0.02908125029563868 0.0 True 1000 +BP 2.0 298 auprc supra_input_output supra_output 0.07413412064090152 0.0614848737185092 0.012649246922392325 0.012700112035683279 0.006224162439436016 0.02014427561744787 0.0 True 1000 +BP 2.0 298 f_max supra_relative_output supra_relative 0.12861775648663223 0.1269083869719604 0.0017093695146718213 0.0011886426477649753 -0.002931511202273342 0.004361365556886051 0.394 False 1000 +BP 2.0 298 auprc supra_relative_output supra_relative 0.029342555105922298 0.029511278976423923 -0.00016872387050162474 -4.807072870164922e-06 -0.004581942109310801 0.004506244522303269 0.956 False 1000 +BP 2.0 298 f_max supra_relative_output supra_output 0.12861775648663223 0.17163715719644868 -0.04301940070981644 -0.04336495546353173 -0.06423085596986536 -0.022932082429505667 0.0 True 1000 +BP 2.0 298 auprc supra_relative_output supra_output 0.029342555105922298 0.0614848737185092 -0.0321423186125869 -0.031968126452725315 -0.04656720310971485 -0.019325472238423156 0.0 True 1000 +BP 2.0 298 f_max full supra_input_relative 0.15384527790027153 0.15601748308402721 -0.0021722051837556877 -0.0014243745276665866 -0.004964943055380539 0.002594311886837907 0.384 False 1000 +BP 2.0 298 auprc full supra_input_relative 0.04745326817925877 0.048512655192598404 -0.0010593870133396369 -0.0011353227078789922 -0.005742221398661911 0.0030021919357103547 0.604 False 1000 +BP 2.0 298 f_max full supra_input_output 0.15384527790027153 0.18832348586599781 -0.03447820796572629 -0.03364956609894659 -0.052685967439753414 -0.016105588307831497 0.0 True 1000 +BP 2.0 298 auprc full supra_input_output 0.04745326817925877 0.07413412064090152 -0.026680852461642757 -0.02670179471327502 -0.038870380318567835 -0.015736964442968564 0.0 True 1000 +BP 2.0 298 f_max full supra_relative_output 0.15384527790027153 0.12861775648663223 0.025227521413639292 0.026295015893899498 0.011943388633153343 0.04041951535083449 0.0 True 1000 +BP 2.0 298 auprc full supra_relative_output 0.04745326817925877 0.029342555105922298 0.01811071307333647 0.017966443775133577 0.010956669551967745 0.02636967225356905 0.0 True 1000 +BP 2.0 298 f_max single naive 0.16336579304170887 0.1393754232163564 0.023990369825352464 0.02598927206088505 -0.005755757951761807 0.05870870782997046 0.122 False 1000 +BP 2.0 298 auprc single naive 0.06435395211501753 0.05246757199300765 0.011886380122009876 0.012733929143653129 -0.009082106899634283 0.03726259969189916 0.248 False 1000 +BP 2.0 298 f_max supra naive 0.16719947495645657 0.1393754232163564 0.027824051740100164 0.029191818229788193 -0.003890669883319664 0.06223537258366627 0.094 False 1000 +BP 2.0 298 auprc supra naive 0.06741104829568473 0.05246757199300765 0.014943476302677075 0.015800779834956445 -0.006459070005237635 0.041239788525575874 0.188 False 1000 +BP 2.0 298 f_max supra_input naive 0.18709980332277934 0.1393754232163564 0.04772438010642294 0.050239110372606345 0.01861676277162812 0.08347068513678152 0.0 True 1000 +BP 2.0 298 auprc supra_input naive 0.06923837639082926 0.05246757199300765 0.01677080439782161 0.017365703817278713 -0.0029485146359240675 0.04012145045922827 0.118 False 1000 +BP 2.0 298 f_max supra_relative naive 0.1269083869719604 0.1393754232163564 -0.01246703624439599 -0.011595165326138855 -0.040989653930390856 0.018879468904500773 0.456 False 1000 +BP 2.0 298 auprc supra_relative naive 0.029511278976423923 0.05246757199300765 -0.02295629301658373 -0.02238939180389472 -0.03553281262438744 -0.006686117509551256 0.004 True 1000 +BP 2.0 298 f_max supra_output naive 0.17163715719644868 0.1393754232163564 0.03226173398009227 0.03295843278515784 0.002668988293517534 0.06435681705488057 0.032 True 1000 +BP 2.0 298 auprc supra_output naive 0.0614848737185092 0.05246757199300765 0.009017301725501546 0.009573927575960437 -0.009569752452768487 0.0316540395283011 0.364 False 1000 +BP 2.0 298 f_max supra_input_relative naive 0.15601748308402721 0.1393754232163564 0.01664205986767081 0.017312867743192205 -0.01021955707772767 0.046649177359198525 0.236 False 1000 +BP 2.0 298 auprc supra_input_relative naive 0.048512655192598404 0.05246757199300765 -0.003954916800409249 -0.0032924323937523164 -0.020150076307462562 0.015847791991657187 0.702 False 1000 +BP 2.0 298 f_max supra_input_output naive 0.18832348586599781 0.1393754232163564 0.04894806264964141 0.04953805931447221 0.017414100877776388 0.08344356716022736 0.0 True 1000 +BP 2.0 298 auprc supra_input_output naive 0.07413412064090152 0.05246757199300765 0.02166654864789387 0.022274039611643714 0.0011109771566241973 0.045377571684149395 0.042 True 1000 +BP 2.0 298 f_max supra_relative_output naive 0.12861775648663223 0.1393754232163564 -0.010757666729724169 -0.01040652267837388 -0.04003903740771764 0.020241932865470675 0.476 False 1000 +BP 2.0 298 auprc supra_relative_output naive 0.029342555105922298 0.05246757199300765 -0.023125016887085355 -0.022394198876764887 -0.03595137727512802 -0.00661212086729948 0.004 True 1000 +BP 2.0 298 f_max full naive 0.15384527790027153 0.1393754232163564 0.014469854683915123 0.015888493215525618 -0.01190260615976834 0.0440726520838405 0.28 False 1000 +BP 2.0 298 auprc full naive 0.04745326817925877 0.05246757199300765 -0.0050143038137488855 -0.004427755101631308 -0.020809735403791104 0.013790587424338493 0.616 False 1000 +BP 2.0 298 f_max full single 0.15384527790027153 0.16336579304170887 -0.009520515141437341 -0.010100778845359435 -0.02454639846910398 0.004896868581382745 0.17 False 1000 +BP 2.0 298 auprc full single 0.04745326817925877 0.06435395211501753 -0.016900683935758762 -0.01716168424528444 -0.03175457883936905 -0.004243133917035483 0.006 True 1000 +BP 4.0 292 f_max supra single 0.11828896656791534 0.11394706576576076 0.004341900802154575 0.0033628108790055337 -0.0016285859832317945 0.009128574388714193 0.232 False 1000 +BP 4.0 292 auprc supra single 0.03405078849746584 0.030207105451247915 0.0038436830462179224 0.0037638372997061613 0.0011353248700282082 0.007638894246862735 0.002 True 1000 +BP 4.0 292 f_max supra_input supra 0.13808913314840468 0.11828896656791534 0.019800166580489345 0.018178534562120503 -0.0011785077998928575 0.03864421142896552 0.078 False 1000 +BP 4.0 292 auprc supra_input supra 0.03501200013295161 0.03405078849746584 0.0009612116354857717 0.0008699634405118419 -0.008466053028924293 0.009104839602794174 0.792 False 1000 +BP 4.0 292 f_max supra_relative supra 0.0810288367233145 0.11828896656791534 -0.03726012984460084 -0.03871883191372256 -0.06135001125707621 -0.016440121054130848 0.0 True 1000 +BP 4.0 292 auprc supra_relative supra 0.012185748885143198 0.03405078849746584 -0.02186503961232264 -0.02237205594985973 -0.03869531461574346 -0.010016023921209516 0.0 True 1000 +BP 4.0 292 f_max supra_output supra 0.11843834179404938 0.11828896656791534 0.0001493752261340492 -0.0009149692367410316 -0.013818375047343344 0.011623162422522393 0.896 False 1000 +BP 4.0 292 auprc supra_output supra 0.027289542126364027 0.03405078849746584 -0.006761246371101811 -0.0071069367902355665 -0.016662255672752785 -0.0006426582122780895 0.03 True 1000 +BP 4.0 292 f_max supra_input_relative supra_input 0.09412487118656662 0.13808913314840468 -0.04396426196183806 -0.044203144498694114 -0.07116058698684861 -0.021510966067428368 0.0 True 1000 +BP 4.0 292 auprc supra_input_relative supra_input 0.01917880743576971 0.03501200013295161 -0.0158331926971819 -0.016047062583819136 -0.027866549356773514 -0.005504542952213868 0.0 True 1000 +BP 4.0 292 f_max supra_input_relative supra_relative 0.09412487118656662 0.0810288367233145 0.013096034463252124 0.012694221977148945 -0.00017460302839816724 0.0259054324693954 0.054 False 1000 +BP 4.0 292 auprc supra_input_relative supra_relative 0.01917880743576971 0.012185748885143198 0.006993058550626513 0.007194956806552435 0.0010917136485192958 0.01469904726149451 0.012 True 1000 +BP 4.0 292 f_max supra_input_output supra_input 0.13286708464632122 0.13808913314840468 -0.005222048502083465 -0.00322343647684135 -0.010702010665878503 0.0027402550278509148 0.362 False 1000 +BP 4.0 292 auprc supra_input_output supra_input 0.034378134445179405 0.03501200013295161 -0.0006338656877722046 -0.0008863718774107892 -0.003990045674515975 0.002033945539401751 0.528 False 1000 +BP 4.0 292 f_max supra_input_output supra_output 0.13286708464632122 0.11843834179404938 0.01442874285227183 0.015870067322020186 0.0018500608080550412 0.03419559489383765 0.026 True 1000 +BP 4.0 292 auprc supra_input_output supra_output 0.034378134445179405 0.027289542126364027 0.007088592318815378 0.007090528353336619 0.0026524194456362692 0.012725558527580427 0.0 True 1000 +BP 4.0 292 f_max supra_relative_output supra_relative 0.08423567672572724 0.0810288367233145 0.003206840002412739 0.002091206594357935 -0.0015616886042885206 0.0056950527808390045 0.22 False 1000 +BP 4.0 292 auprc supra_relative_output supra_relative 0.011990176873338164 0.012185748885143198 -0.00019557201180503414 -0.00013116337409424484 -0.003979706938469413 0.003495469636825979 0.964 False 1000 +BP 4.0 292 f_max supra_relative_output supra_output 0.08423567672572724 0.11843834179404938 -0.03420266506832215 -0.0357126560826236 -0.0606210227194047 -0.011584548600964324 0.004 True 1000 +BP 4.0 292 auprc supra_relative_output supra_output 0.011990176873338164 0.027289542126364027 -0.015299365253025863 -0.015396282533718407 -0.02582070400804626 -0.005713665648791885 0.0 True 1000 +BP 4.0 292 f_max full supra_input_relative 0.09256562477377679 0.09412487118656662 -0.0015592464127898303 -0.00182894027655846 -0.00690271134250378 0.0006943973473770754 0.204 False 1000 +BP 4.0 292 auprc full supra_input_relative 0.019870938340269942 0.01917880743576971 0.0006921309045002308 0.000520854929162053 -0.0032646476067112935 0.0043305121196925605 0.772 False 1000 +BP 4.0 292 f_max full supra_input_output 0.09256562477377679 0.13286708464632122 -0.04030145987254442 -0.04280864829841123 -0.06913640019692603 -0.020195089994118114 0.0 True 1000 +BP 4.0 292 auprc full supra_input_output 0.019870938340269942 0.034378134445179405 -0.014507196104909463 -0.014639835777246294 -0.025725395019148046 -0.005132286625714214 0.004 True 1000 +BP 4.0 292 f_max full supra_relative_output 0.09256562477377679 0.08423567672572724 0.008329948048049554 0.008774075106232552 -0.004456367955871268 0.02194464357978949 0.186 False 1000 +BP 4.0 292 auprc full supra_relative_output 0.019870938340269942 0.011990176873338164 0.007880761466931778 0.007846975109808734 0.0027085651009395983 0.014401265187878298 0.002 True 1000 +BP 4.0 292 f_max single naive 0.11394706576576076 0.04686917580427525 0.0670778899614855 0.07042020476457775 0.04133204322512161 0.10374873334325473 0.0 True 1000 +BP 4.0 292 auprc single naive 0.030207105451247915 0.013722538414023608 0.016484567037224306 0.017690709961870076 0.0034200604390907086 0.03732929477627652 0.006 True 1000 +BP 4.0 292 f_max supra naive 0.11828896656791534 0.04686917580427525 0.07141979076364008 0.07378301564358328 0.04358002340425051 0.10869873024726112 0.0 True 1000 +BP 4.0 292 auprc supra naive 0.03405078849746584 0.013722538414023608 0.02032825008344223 0.021454547261576237 0.00565304683462294 0.04208362417453337 0.002 True 1000 +BP 4.0 292 f_max supra_input naive 0.13808913314840468 0.04686917580427525 0.09121995734412944 0.09196155020570379 0.0604264707012165 0.12288461323788405 0.0 True 1000 +BP 4.0 292 auprc supra_input naive 0.03501200013295161 0.013722538414023608 0.021289461718928003 0.02232451070208808 0.00833045267500264 0.03910702029943001 0.0 True 1000 +BP 4.0 292 f_max supra_relative naive 0.0810288367233145 0.04686917580427525 0.03415966091903925 0.03506418372986071 0.01110562166834975 0.06211823925728514 0.002 True 1000 +BP 4.0 292 auprc supra_relative naive 0.012185748885143198 0.013722538414023608 -0.0015367895288804102 -0.0009175086882834902 -0.007541946760976803 0.00855167021753404 0.762 False 1000 +BP 4.0 292 f_max supra_output naive 0.11843834179404938 0.04686917580427525 0.07156916598977414 0.07286804640684225 0.04345751953599959 0.10402175105502616 0.0 True 1000 +BP 4.0 292 auprc supra_output naive 0.027289542126364027 0.013722538414023608 0.013567003712340419 0.014347610471340673 0.0023904084257608776 0.029453032907884008 0.014 True 1000 +BP 4.0 292 f_max supra_input_relative naive 0.09412487118656662 0.04686917580427525 0.04725569538229137 0.047758405707009656 0.024061427910270064 0.07507999890315059 0.0 True 1000 +BP 4.0 292 auprc supra_input_relative naive 0.01917880743576971 0.013722538414023608 0.005456269021746103 0.006277448118268945 -0.0032973538148604645 0.01875196237930152 0.28 False 1000 +BP 4.0 292 f_max supra_input_output naive 0.13286708464632122 0.04686917580427525 0.08599790884204597 0.08873811372886244 0.0592400734042904 0.12008951313114473 0.0 True 1000 +BP 4.0 292 auprc supra_input_output naive 0.034378134445179405 0.013722538414023608 0.0206555960311558 0.021438138824677294 0.0075488117435396655 0.03701536238585178 0.0 True 1000 +BP 4.0 292 f_max supra_relative_output naive 0.08423567672572724 0.04686917580427525 0.03736650092145199 0.03715539032421865 0.012572770917806083 0.06473147992468767 0.002 True 1000 +BP 4.0 292 auprc supra_relative_output naive 0.011990176873338164 0.013722538414023608 -0.0017323615406854443 -0.001048672062377735 -0.008177148277966144 0.008637034232674897 0.764 False 1000 +BP 4.0 292 f_max full naive 0.09256562477377679 0.04686917580427525 0.04569644896950154 0.045929465430451205 0.02183920497406639 0.07271462149895656 0.0 True 1000 +BP 4.0 292 auprc full naive 0.019870938340269942 0.013722538414023608 0.006148399926246334 0.006798303047430999 -0.0032719853143860765 0.01986332644565891 0.258 False 1000 +BP 4.0 292 f_max full single 0.09256562477377679 0.11394706576576076 -0.02138144099198397 -0.02449073933412654 -0.04516997425098661 -0.005596378665912499 0.004 True 1000 +BP 4.0 292 auprc full single 0.019870938340269942 0.030207105451247915 -0.010336167110977974 -0.010892406914439079 -0.022306717429265533 -0.001142298689545454 0.03 True 1000 +MF 0.0 406 f_max supra single 0.34357928540123506 0.34492218474153336 -0.001342899340298298 -0.0012655394231294737 -0.011717529647944402 0.009367675868277768 0.778 False 1000 +MF 0.0 406 auprc supra single 0.171545113956216 0.1712192350176612 0.0003258789385547911 0.00046508445695062984 -0.003517924123439435 0.004497432200698657 0.83 False 1000 +MF 0.0 406 f_max supra_input supra 0.36037094536457454 0.34357928540123506 0.016791659963339478 0.01690710881154977 -0.001985693734803936 0.035857807361945354 0.082 False 1000 +MF 0.0 406 auprc supra_input supra 0.17171632950127838 0.171545113956216 0.00017121554506238645 3.874961482008053e-05 -0.010355048470327338 0.010174481290468558 0.998 False 1000 +MF 0.0 406 f_max supra_relative supra 0.33460122141587784 0.34357928540123506 -0.00897806398535722 -0.009276411159123941 -0.03365694143897909 0.01330601164382822 0.464 False 1000 +MF 0.0 406 auprc supra_relative supra 0.15555521573877742 0.171545113956216 -0.01598989821743857 -0.016518752217902395 -0.03598408038842676 0.004554965320236074 0.1 False 1000 +MF 0.0 406 f_max supra_output supra 0.35133717588529867 0.34357928540123506 0.007757890484063612 0.008237475157890028 -0.0073433593294350554 0.023709953100268605 0.34 False 1000 +MF 0.0 406 auprc supra_output supra 0.18005597834700826 0.171545113956216 0.008510864390792267 0.008400273503808222 -0.0011660571101540656 0.017899102291087047 0.102 False 1000 +MF 0.0 406 f_max supra_input_relative supra_input 0.3569474047131703 0.36037094536457454 -0.003423540651404211 -0.003146224549144132 -0.025797589955581 0.019073115316878574 0.788 False 1000 +MF 0.0 406 auprc supra_input_relative supra_input 0.1908100853598182 0.17171632950127838 0.019093755858539813 0.018891154702057527 0.006004201522966123 0.0327332508984218 0.002 True 1000 +MF 0.0 406 f_max supra_input_relative supra_relative 0.3569474047131703 0.33460122141587784 0.022346183297292488 0.02303729542152958 -0.002514563997382102 0.04913379324442609 0.088 False 1000 +MF 0.0 406 auprc supra_input_relative supra_relative 0.1908100853598182 0.15555521573877742 0.03525486962104077 0.03544865653478 0.01573739599595973 0.054056727338963 0.0 True 1000 +MF 0.0 406 f_max supra_input_output supra_input 0.36076776543325934 0.36037094536457454 0.0003968200686848 0.0010324096303106244 -0.009750978093061196 0.012322363039718582 0.848 False 1000 +MF 0.0 406 auprc supra_input_output supra_input 0.1899294765403534 0.17171632950127838 0.018213147039075006 0.018167950639536853 0.012814575503818659 0.02413583852135522 0.0 True 1000 +MF 0.0 406 f_max supra_input_output supra_output 0.36076776543325934 0.35133717588529867 0.009430589547960666 0.009702043283970369 -0.0008993725875607912 0.020661024646444977 0.088 False 1000 +MF 0.0 406 auprc supra_input_output supra_output 0.1899294765403534 0.18005597834700826 0.009873498193345126 0.009806426750548708 0.0016824003257013835 0.017319780480117307 0.028 True 1000 +MF 0.0 406 f_max supra_relative_output supra_relative 0.3378752801234797 0.33460122141587784 0.00327405870760189 0.007280336817903123 -0.010295086867463651 0.029014997969672683 0.464 False 1000 +MF 0.0 406 auprc supra_relative_output supra_relative 0.1846921135015052 0.15555521573877742 0.029136897762727765 0.02944384719891963 0.01838960636624384 0.04104457117405206 0.0 True 1000 +MF 0.0 406 f_max supra_relative_output supra_output 0.3378752801234797 0.35133717588529867 -0.013461895761818943 -0.010233549499110846 -0.03577204517473315 0.01377467592138651 0.438 False 1000 +MF 0.0 406 auprc supra_relative_output supra_output 0.1846921135015052 0.18005597834700826 0.004636135154496929 0.004524821477209013 -0.017102327358553514 0.026790712486188327 0.714 False 1000 +MF 0.0 406 f_max full supra_input_relative 0.37612656480454426 0.3569474047131703 0.019179160091373937 0.019238509617420546 0.00818105308225174 0.03183117189149554 0.0 True 1000 +MF 0.0 406 auprc full supra_input_relative 0.21123115567352646 0.1908100853598182 0.020421070313708267 0.02042934406931485 0.013092320190898591 0.027867259407074046 0.0 True 1000 +MF 0.0 406 f_max full supra_input_output 0.37612656480454426 0.36076776543325934 0.015358799371284926 0.015059875437965787 -0.004352460372793715 0.034101225701103716 0.122 False 1000 +MF 0.0 406 auprc full supra_input_output 0.21123115567352646 0.1899294765403534 0.021301679133173074 0.02115254813183553 0.006504394731232566 0.03598385264389075 0.006 True 1000 +MF 0.0 406 f_max full supra_relative_output 0.37612656480454426 0.3378752801234797 0.038251284681064535 0.034995468221047 0.014505955927994233 0.05513778096384294 0.0 True 1000 +MF 0.0 406 auprc full supra_relative_output 0.21123115567352646 0.1846921135015052 0.02653904217202127 0.02643415340517523 0.010937469727044851 0.04334210977362592 0.004 True 1000 +MF 0.0 406 f_max single naive 0.34492218474153336 0.7189989169653068 -0.37407673222377347 -0.3732176678915972 -0.4312213531018543 -0.3206574650326765 0.0 True 1000 +MF 0.0 406 auprc single naive 0.1712192350176612 0.4190445836593093 -0.2478253486416481 -0.24701721423785375 -0.30055771092242817 -0.19841125296875478 0.0 True 1000 +MF 0.0 406 f_max supra naive 0.34357928540123506 0.7189989169653068 -0.37541963156407177 -0.37448320731472673 -0.42927683846294046 -0.31901496283786884 0.0 True 1000 +MF 0.0 406 auprc supra naive 0.171545113956216 0.4190445836593093 -0.2474994697030933 -0.24655212978090313 -0.30042233973668203 -0.19617923588583785 0.0 True 1000 +MF 0.0 406 f_max supra_input naive 0.36037094536457454 0.7189989169653068 -0.3586279716007323 -0.3575760985031769 -0.41243680295089147 -0.3060516837609218 0.0 True 1000 +MF 0.0 406 auprc supra_input naive 0.17171632950127838 0.4190445836593093 -0.24732825415803092 -0.24651338016608304 -0.30259378905952544 -0.1944154189910412 0.0 True 1000 +MF 0.0 406 f_max supra_relative naive 0.33460122141587784 0.7189989169653068 -0.384397695549429 -0.38375961847385065 -0.4356315499804442 -0.32931462037433124 0.0 True 1000 +MF 0.0 406 auprc supra_relative naive 0.15555521573877742 0.4190445836593093 -0.2634893679205319 -0.2630708819988055 -0.3094912700725503 -0.21683089894961993 0.0 True 1000 +MF 0.0 406 f_max supra_output naive 0.35133717588529867 0.7189989169653068 -0.36766174108000815 -0.36624573215683665 -0.41867166115345694 -0.3146744300701625 0.0 True 1000 +MF 0.0 406 auprc supra_output naive 0.18005597834700826 0.4190445836593093 -0.23898860531230104 -0.2381518562770949 -0.29052672470134905 -0.18798361001315267 0.0 True 1000 +MF 0.0 406 f_max supra_input_relative naive 0.3569474047131703 0.7189989169653068 -0.3620515122521365 -0.36072232305232105 -0.41293830703869955 -0.3096356675740661 0.0 True 1000 +MF 0.0 406 auprc supra_input_relative naive 0.1908100853598182 0.4190445836593093 -0.2282344982994911 -0.22762222546402552 -0.28256040095993307 -0.1779936739286883 0.0 True 1000 +MF 0.0 406 f_max supra_input_output naive 0.36076776543325934 0.7189989169653068 -0.3582311515320475 -0.3565436888728663 -0.4103262901675488 -0.3032090571496054 0.0 True 1000 +MF 0.0 406 auprc supra_input_output naive 0.1899294765403534 0.4190445836593093 -0.2291151071189559 -0.22834542952654618 -0.2832155696250841 -0.175947418321142 0.0 True 1000 +MF 0.0 406 f_max supra_relative_output naive 0.3378752801234797 0.7189989169653068 -0.3811236368418271 -0.37647928165594746 -0.4243525674799245 -0.3234775608256933 0.0 True 1000 +MF 0.0 406 auprc supra_relative_output naive 0.1846921135015052 0.4190445836593093 -0.2343524701578041 -0.23362703479988586 -0.28249973409902307 -0.18470778635360274 0.0 True 1000 +MF 0.0 406 f_max full naive 0.37612656480454426 0.7189989169653068 -0.34287235216076256 -0.34148381343490053 -0.39239026790692605 -0.29173065773735385 0.0 True 1000 +MF 0.0 406 auprc full naive 0.21123115567352646 0.4190445836593093 -0.20781342798578284 -0.20719288139471068 -0.26115181576190294 -0.1573954199366047 0.0 True 1000 +MF 0.0 406 f_max full single 0.37612656480454426 0.34492218474153336 0.031204380063010906 0.031733854456696714 0.008658271038185056 0.05674239090689813 0.004 True 1000 +MF 0.0 406 auprc full single 0.21123115567352646 0.1712192350176612 0.04001192065586526 0.03982433284314309 0.02478611254282521 0.056191265719185333 0.0 True 1000 +MF 2.0 159 f_max supra single 0.32980140754537307 0.33028872962052275 -0.0004873220751496765 -0.000977739060492151 -0.008348196022522705 0.005942614408201822 0.788 False 1000 +MF 2.0 159 auprc supra single 0.18918327277822655 0.17950175005084912 0.009681522727377434 0.00985771778922635 0.003431152567851182 0.0178264580405881 0.002 True 1000 +MF 2.0 159 f_max supra_input supra 0.3351947698693985 0.32980140754537307 0.00539336232402543 0.005179282561996811 -0.015475241623821455 0.025815758950670807 0.62 False 1000 +MF 2.0 159 auprc supra_input supra 0.2080034865452522 0.18918327277822655 0.01882021376702564 0.018515792748502062 -5.4079206020527564e-05 0.03787138070901343 0.054 False 1000 +MF 2.0 159 f_max supra_relative supra 0.25945157755716214 0.32980140754537307 -0.07034982998821093 -0.0663160883643952 -0.10108068481422158 -0.03297123185837594 0.0 True 1000 +MF 2.0 159 auprc supra_relative supra 0.09626807382427224 0.18918327277822655 -0.09291519895395431 -0.09182303184188854 -0.12720904989537893 -0.05901380152465784 0.0 True 1000 +MF 2.0 159 f_max supra_output supra 0.31817620684333403 0.32980140754537307 -0.01162520070203904 -0.01099495798961366 -0.030674289949713805 0.00843563047476984 0.282 False 1000 +MF 2.0 159 auprc supra_output supra 0.186367547422579 0.18918327277822655 -0.002815725355647558 -0.002212140833671891 -0.02106628423925099 0.01828350006893456 0.812 False 1000 +MF 2.0 159 f_max supra_input_relative supra_input 0.28809539534087936 0.3351947698693985 -0.047099374528519145 -0.04488451177673089 -0.06708040616641044 -0.02401247861205919 0.0 True 1000 +MF 2.0 159 auprc supra_input_relative supra_input 0.153532150824977 0.2080034865452522 -0.05447133572027518 -0.05402264809682749 -0.07556003310541279 -0.03322894495298698 0.0 True 1000 +MF 2.0 159 f_max supra_input_relative supra_relative 0.28809539534087936 0.25945157755716214 0.028643817783717218 0.02661085914966112 0.0004157300437909904 0.05540258210177648 0.046 True 1000 +MF 2.0 159 auprc supra_input_relative supra_relative 0.153532150824977 0.09626807382427224 0.057264077000704766 0.056316176493563117 0.030913470214204524 0.08315804025893618 0.0 True 1000 +MF 2.0 159 f_max supra_input_output supra_input 0.3279453161295411 0.3351947698693985 -0.007249453739857414 -0.007522619362365075 -0.018299536713177435 0.0012163424603823135 0.086 False 1000 +MF 2.0 159 auprc supra_input_output supra_input 0.20413914043745163 0.2080034865452522 -0.0038643461078005648 -0.0034921580832488817 -0.010032782314935373 0.0029059531515914544 0.292 False 1000 +MF 2.0 159 f_max supra_input_output supra_output 0.3279453161295411 0.31817620684333403 0.009769109286207056 0.008651621189245397 -0.0042195231921833935 0.0201703699692643 0.158 False 1000 +MF 2.0 159 auprc supra_input_output supra_output 0.20413914043745163 0.186367547422579 0.017771593014872633 0.017235775498925073 0.00830233954785271 0.02751316672445792 0.0 True 1000 +MF 2.0 159 f_max supra_relative_output supra_relative 0.2755198263008129 0.25945157755716214 0.01606824874365076 0.011672279282653306 -0.003448760961976317 0.02473172412252112 0.136 False 1000 +MF 2.0 159 auprc supra_relative_output supra_relative 0.11189982515833893 0.09626807382427224 0.01563175133406669 0.015804081813206864 0.0027861367196246297 0.03157678094587273 0.028 True 1000 +MF 2.0 159 f_max supra_relative_output supra_output 0.2755198263008129 0.31817620684333403 -0.04265638054252113 -0.04364885109212823 -0.07290411980117269 -0.012771924025128223 0.004 True 1000 +MF 2.0 159 auprc supra_relative_output supra_output 0.11189982515833893 0.186367547422579 -0.07446772226424006 -0.07380680919500979 -0.10674482617017743 -0.044391958757834145 0.0 True 1000 +MF 2.0 159 f_max full supra_input_relative 0.2973310593460379 0.28809539534087936 0.009235664005158517 0.006766350955694824 -0.005632671098617696 0.019547578614814397 0.252 False 1000 +MF 2.0 159 auprc full supra_input_relative 0.15168797716709995 0.153532150824977 -0.0018441736578770629 -0.0014465456786338037 -0.013456867985509413 0.011126183343974343 0.782 False 1000 +MF 2.0 159 f_max full supra_input_output 0.2973310593460379 0.3279453161295411 -0.030614256783503213 -0.030595541458670994 -0.051058810160064604 -0.010805027446229389 0.002 True 1000 +MF 2.0 159 auprc full supra_input_output 0.15168797716709995 0.20413914043745163 -0.05245116327035168 -0.05197703569221241 -0.07154598471581101 -0.032388743313778945 0.0 True 1000 +MF 2.0 159 f_max full supra_relative_output 0.2973310593460379 0.2755198263008129 0.021811233045224976 0.021704930822702636 -0.0036724234620504758 0.04872392362458265 0.08 False 1000 +MF 2.0 159 auprc full supra_relative_output 0.15168797716709995 0.11189982515833893 0.03978815200876101 0.03906554900172246 0.017684375054174622 0.06268352636085561 0.0 True 1000 +MF 2.0 159 f_max single naive 0.33028872962052275 0.07561162501367721 0.25467710460684556 0.2558694235297399 0.1925916592142083 0.3193607047965262 0.0 True 1000 +MF 2.0 159 auprc single naive 0.17950175005084912 0.026528609897190324 0.1529731401536588 0.15340816500093837 0.10361185003545065 0.20697513276868373 0.0 True 1000 +MF 2.0 159 f_max supra naive 0.32980140754537307 0.07561162501367721 0.2541897825316959 0.25489168446924776 0.1907956024151922 0.3200259717924231 0.0 True 1000 +MF 2.0 159 auprc supra naive 0.18918327277822655 0.026528609897190324 0.16265466288103622 0.1632658827901647 0.10992696813900645 0.22149536470272907 0.0 True 1000 +MF 2.0 159 f_max supra_input naive 0.3351947698693985 0.07561162501367721 0.2595831448557213 0.26007096703124455 0.1996764063554409 0.3224064588774297 0.0 True 1000 +MF 2.0 159 auprc supra_input naive 0.2080034865452522 0.026528609897190324 0.18147487664806186 0.18178167553866678 0.12716065210320715 0.2420653218533099 0.0 True 1000 +MF 2.0 159 f_max supra_relative naive 0.25945157755716214 0.07561162501367721 0.18383995254348492 0.18857559610485253 0.13811458121214396 0.2414840425578814 0.0 True 1000 +MF 2.0 159 auprc supra_relative naive 0.09626807382427224 0.026528609897190324 0.06973946392708191 0.07144285094827614 0.03517514394750039 0.11523369052080568 0.0 True 1000 +MF 2.0 159 f_max supra_output naive 0.31817620684333403 0.07561162501367721 0.24256458182965682 0.2438967264796341 0.18551164857925556 0.3026499937258442 0.0 True 1000 +MF 2.0 159 auprc supra_output naive 0.186367547422579 0.026528609897190324 0.15983893752538866 0.16105374195649283 0.10811646573803677 0.22328794963634796 0.0 True 1000 +MF 2.0 159 f_max supra_input_relative naive 0.28809539534087936 0.07561162501367721 0.21248377032720214 0.21518645525451366 0.1620987382977213 0.27197819899624676 0.0 True 1000 +MF 2.0 159 auprc supra_input_relative naive 0.153532150824977 0.026528609897190324 0.12700354092778668 0.12775902744183928 0.08310570646595293 0.18367143099343117 0.0 True 1000 +MF 2.0 159 f_max supra_input_output naive 0.3279453161295411 0.07561162501367721 0.2523336911158639 0.25254834766887946 0.19505674455166908 0.3136764982029598 0.0 True 1000 +MF 2.0 159 auprc supra_input_output naive 0.20413914043745163 0.026528609897190324 0.1776105305402613 0.1782895174554179 0.1229894449571576 0.2402229140204972 0.0 True 1000 +MF 2.0 159 f_max supra_relative_output naive 0.2755198263008129 0.07561162501367721 0.19990820128713568 0.20024787538750588 0.149355542637825 0.2556677182919704 0.0 True 1000 +MF 2.0 159 auprc supra_relative_output naive 0.11189982515833893 0.026528609897190324 0.0853712152611486 0.08724693276148301 0.04744108705327983 0.13619091907632305 0.0 True 1000 +MF 2.0 159 f_max full naive 0.2973310593460379 0.07561162501367721 0.22171943433236066 0.2219528062102085 0.1703408492149486 0.2740218941542594 0.0 True 1000 +MF 2.0 159 auprc full naive 0.15168797716709995 0.026528609897190324 0.12515936726990962 0.12631248176320548 0.08036296857698119 0.1805060429029765 0.0 True 1000 +MF 2.0 159 f_max full single 0.2973310593460379 0.33028872962052275 -0.032957670274484874 -0.03391661731953141 -0.05672709236071984 -0.00932746503106637 0.004 True 1000 +MF 2.0 159 auprc full single 0.15168797716709995 0.17950175005084912 -0.02781377288374917 -0.027095683237732884 -0.048174207627707036 -0.004647848303613276 0.016 True 1000 +MF 4.0 151 f_max supra single 0.30712672755850184 0.3074692728343345 -0.0003425452758326619 0.0005681925092870043 -0.00653322486112876 0.009039796105298792 0.972 False 1000 +MF 4.0 151 auprc supra single 0.17827778650251944 0.17600599998991664 0.0022717865126027914 0.00239221812505047 -0.0020843075441918744 0.007238037428223863 0.288 False 1000 +MF 4.0 151 f_max supra_input supra 0.32507657135803186 0.30712672755850184 0.01794984379953002 0.014124504262300462 -0.007092361972108312 0.03623426186670683 0.228 False 1000 +MF 4.0 151 auprc supra_input supra 0.19253720279059416 0.17827778650251944 0.014259416288074728 0.01438553109713694 -0.0045104722905532834 0.03480523658232791 0.14 False 1000 +MF 4.0 151 f_max supra_relative supra 0.23755767703676015 0.30712672755850184 -0.0695690505217417 -0.06891118849429431 -0.10542041142881876 -0.03296240536170282 0.002 True 1000 +MF 4.0 151 auprc supra_relative supra 0.08319590331958229 0.17827778650251944 -0.09508188318293714 -0.09473671940886379 -0.1355648383911234 -0.0621418411781151 0.0 True 1000 +MF 4.0 151 f_max supra_output supra 0.3059033984033558 0.30712672755850184 -0.0012233291551460579 -0.0029304501325341297 -0.022772179682789318 0.018704803638994127 0.756 False 1000 +MF 4.0 151 auprc supra_output supra 0.1734068803259186 0.17827778650251944 -0.0048709061766008255 -0.004366128401720543 -0.02212928294903329 0.013246490519332941 0.634 False 1000 +MF 4.0 151 f_max supra_input_relative supra_input 0.26256167023394233 0.32507657135803186 -0.06251490112408953 -0.05804643998762411 -0.08207538030057149 -0.032635981527824695 0.0 True 1000 +MF 4.0 151 auprc supra_input_relative supra_input 0.12106071224226855 0.19253720279059416 -0.07147649054832561 -0.07153521777830883 -0.09926141771189911 -0.04753226631444715 0.0 True 1000 +MF 4.0 151 f_max supra_input_relative supra_relative 0.26256167023394233 0.23755767703676015 0.02500399319718219 0.024989252768970658 -0.0019498242048943929 0.055947304766250226 0.08 False 1000 +MF 4.0 151 auprc supra_input_relative supra_relative 0.12106071224226855 0.08319590331958229 0.03786480892268626 0.03758703272769188 0.013076468985222434 0.06457418967399242 0.0 True 1000 +MF 4.0 151 f_max supra_input_output supra_input 0.3140200277746999 0.32507657135803186 -0.011056543583331968 -0.009846669860640993 -0.01996943395872996 0.0006529547110214262 0.062 False 1000 +MF 4.0 151 auprc supra_input_output supra_input 0.18749109239613948 0.19253720279059416 -0.005046110394454684 -0.0048234734510031315 -0.011740150400089328 0.0013255904920202958 0.118 False 1000 +MF 4.0 151 f_max supra_input_output supra_output 0.3140200277746999 0.3059033984033558 0.008116629371344108 0.0072082845341936005 -0.005917370612326765 0.021643930550324385 0.312 False 1000 +MF 4.0 151 auprc supra_input_output supra_output 0.18749109239613948 0.1734068803259186 0.01408421207022087 0.013928186047854353 0.004280696119585524 0.02527868534611661 0.002 True 1000 +MF 4.0 151 f_max supra_relative_output supra_relative 0.24801508792251012 0.23755767703676015 0.010457410885749974 0.007355448089313037 -0.01298887399720402 0.02343748210082215 0.408 False 1000 +MF 4.0 151 auprc supra_relative_output supra_relative 0.08971683359636826 0.08319590331958229 0.006520930276785963 0.007240982159584296 -0.0065517806236031735 0.021255885426183896 0.296 False 1000 +MF 4.0 151 f_max supra_relative_output supra_output 0.24801508792251012 0.3059033984033558 -0.05788831048084567 -0.05862529027244714 -0.0964082443227068 -0.01865962016502012 0.006 True 1000 +MF 4.0 151 auprc supra_relative_output supra_output 0.08971683359636826 0.1734068803259186 -0.08369004672955035 -0.08312960884755896 -0.12297009928700518 -0.05348732184174397 0.0 True 1000 +MF 4.0 151 f_max full supra_input_relative 0.2697141121356497 0.26256167023394233 0.007152441901707385 0.0037937599853863817 -0.007397701161990044 0.014430545938432418 0.474 False 1000 +MF 4.0 151 auprc full supra_input_relative 0.12442969869416227 0.12106071224226855 0.0033689864518937213 0.003513308896190601 -0.005983892180881009 0.014058631873042609 0.486 False 1000 +MF 4.0 151 f_max full supra_input_output 0.2697141121356497 0.3140200277746999 -0.044305915639050175 -0.04440601014159674 -0.06595243183209003 -0.022734767935651386 0.0 True 1000 +MF 4.0 151 auprc full supra_input_output 0.12442969869416227 0.18749109239613948 -0.0630613937019772 -0.06319843543111511 -0.08839698690371924 -0.0408505349374322 0.0 True 1000 +MF 4.0 151 f_max full supra_relative_output 0.2697141121356497 0.24801508792251012 0.0216990242131396 0.021427564665044002 -0.008288125573964456 0.05271590582951217 0.164 False 1000 +MF 4.0 151 auprc full supra_relative_output 0.12442969869416227 0.08971683359636826 0.03471286509779402 0.03385935946429818 0.011877055988322303 0.0582973631909678 0.004 True 1000 +MF 4.0 151 f_max single naive 0.3074692728343345 0.07531254466979251 0.232156728164542 0.23344754750326052 0.17294440447040066 0.2968445551474107 0.0 True 1000 +MF 4.0 151 auprc single naive 0.17600599998991664 0.02307128915639914 0.1529347108335175 0.15365939685639043 0.10242463557014646 0.2121592747063015 0.0 True 1000 +MF 4.0 151 f_max supra naive 0.30712672755850184 0.07531254466979251 0.23181418288870934 0.2340157400125475 0.17323587821283531 0.29813071108595707 0.0 True 1000 +MF 4.0 151 auprc supra naive 0.17827778650251944 0.02307128915639914 0.1552064973461203 0.1560516149814409 0.10492255647055547 0.21702665935705762 0.0 True 1000 +MF 4.0 151 f_max supra_input naive 0.32507657135803186 0.07531254466979251 0.24976402668823935 0.248140244274848 0.18581694327641224 0.3133354847202147 0.0 True 1000 +MF 4.0 151 auprc supra_input naive 0.19253720279059416 0.02307128915639914 0.16946591363419503 0.17043714607857785 0.11585633600732057 0.23270208746954288 0.0 True 1000 +MF 4.0 151 f_max supra_relative naive 0.23755767703676015 0.07531254466979251 0.16224513236696764 0.16510455151825318 0.1150710526757828 0.22063940468372928 0.0 True 1000 +MF 4.0 151 auprc supra_relative naive 0.08319590331958229 0.02307128915639914 0.060124614163183154 0.06131489557257713 0.028411193686133613 0.1006821042830268 0.0 True 1000 +MF 4.0 151 f_max supra_output naive 0.3059033984033558 0.07531254466979251 0.23059085373356328 0.2310852898800134 0.17465354682786893 0.2936257428757428 0.0 True 1000 +MF 4.0 151 auprc supra_output naive 0.1734068803259186 0.02307128915639914 0.15033559116951947 0.15168548657972036 0.09898539781606422 0.20950143187990464 0.0 True 1000 +MF 4.0 151 f_max supra_input_relative naive 0.26256167023394233 0.07531254466979251 0.18724912556414983 0.19009380428722386 0.1387279267045777 0.2464613095766511 0.0 True 1000 +MF 4.0 151 auprc supra_input_relative naive 0.12106071224226855 0.02307128915639914 0.09798942308586941 0.098901928300269 0.05728976958125406 0.14575512266148002 0.0 True 1000 +MF 4.0 151 f_max supra_input_output naive 0.3140200277746999 0.07531254466979251 0.2387074831049074 0.238293574414207 0.18071937004747576 0.2998390273556997 0.0 True 1000 +MF 4.0 151 auprc supra_input_output naive 0.18749109239613948 0.02307128915639914 0.16441980323974034 0.1656136726275747 0.11264716032816617 0.22568424722050034 0.0 True 1000 +MF 4.0 151 f_max supra_relative_output naive 0.24801508792251012 0.07531254466979251 0.1727025432527176 0.17245999960756625 0.12216652201801294 0.2236209197035951 0.0 True 1000 +MF 4.0 151 auprc supra_relative_output naive 0.08971683359636826 0.02307128915639914 0.06664554443996912 0.06855587773216143 0.03349181968326084 0.11004157263191783 0.0 True 1000 +MF 4.0 151 f_max full naive 0.2697141121356497 0.07531254466979251 0.1944015674658572 0.19388756427261025 0.14337982181470196 0.2492103539396635 0.0 True 1000 +MF 4.0 151 auprc full naive 0.12442969869416227 0.02307128915639914 0.10135840953776314 0.1024152371964596 0.05956680775934648 0.15098790561608955 0.0 True 1000 +MF 4.0 151 f_max full single 0.2697141121356497 0.3074692728343345 -0.037755160698684787 -0.03955998323065026 -0.06475801288264475 -0.013100650832259959 0.006 True 1000 +MF 4.0 151 auprc full single 0.12442969869416227 0.17600599998991664 -0.05157630129575437 -0.051244159659930844 -0.07988829100723843 -0.026409548552637236 0.0 True 1000 +CC 0.0 436 f_max supra single 0.37369500099331754 0.37266279839932215 0.0010322025939953905 0.0007693176339514218 -0.006377062167846422 0.007212108072692925 0.77 False 1000 +CC 0.0 436 auprc supra single 0.21425352431010342 0.2138930026796082 0.00036052163049521413 0.00033477733462119975 -0.0028938394160536412 0.004222002494459863 0.89 False 1000 +CC 0.0 436 f_max supra_input supra 0.3657545116537211 0.37369500099331754 -0.007940489339596457 -0.006196017714563072 -0.018554820254248674 0.005043900614132831 0.326 False 1000 +CC 0.0 436 auprc supra_input supra 0.2001142781223428 0.21425352431010342 -0.014139246187760607 -0.013866462884726566 -0.02243599833948821 -0.005214211150022268 0.0 True 1000 +CC 0.0 436 f_max supra_relative supra 0.3441068514445836 0.37369500099331754 -0.029588149548733955 -0.029033230555690744 -0.04591046021905214 -0.013251861981195334 0.0 True 1000 +CC 0.0 436 auprc supra_relative supra 0.17407450637713057 0.21425352431010342 -0.04017901793297285 -0.04048260047417139 -0.05504203631766998 -0.026266533896088923 0.0 True 1000 +CC 0.0 436 f_max supra_output supra 0.38167092847103545 0.37369500099331754 0.007975927477717915 0.008444369239257378 -0.0018763413730351088 0.0190370694901021 0.104 False 1000 +CC 0.0 436 auprc supra_output supra 0.24331519913843716 0.21425352431010342 0.029061674828333744 0.028984966765373096 0.021243413079166985 0.03734574921291777 0.0 True 1000 +CC 0.0 436 f_max supra_input_relative supra_input 0.36749458141809443 0.3657545116537211 0.0017400697643733531 0.00034378348168236756 -0.013161850099688203 0.012786832875947386 0.946 False 1000 +CC 0.0 436 auprc supra_input_relative supra_input 0.1990108702115158 0.2001142781223428 -0.0011034079108269979 -0.0012697393668369896 -0.010626886581558359 0.006721798465548614 0.774 False 1000 +CC 0.0 436 f_max supra_input_relative supra_relative 0.36749458141809443 0.3441068514445836 0.02338772997351085 0.023180996322810038 0.006450368571705336 0.03979807134833992 0.006 True 1000 +CC 0.0 436 auprc supra_input_relative supra_relative 0.1990108702115158 0.17407450637713057 0.024936363834385245 0.025346398222607838 0.01180546374038223 0.0396363560070921 0.0 True 1000 +CC 0.0 436 f_max supra_input_output supra_input 0.3808275731115995 0.3657545116537211 0.0150730614578784 0.01325007720315042 0.004407940458976637 0.021615649157118094 0.002 True 1000 +CC 0.0 436 auprc supra_input_output supra_input 0.23984939219421486 0.2001142781223428 0.03973511407187205 0.03939048458617534 0.0320533173474209 0.04676105330964169 0.0 True 1000 +CC 0.0 436 f_max supra_input_output supra_output 0.3808275731115995 0.38167092847103545 -0.0008433553594359711 -0.0013903097506700296 -0.009116211764389539 0.0065791863567076685 0.714 False 1000 +CC 0.0 436 auprc supra_input_output supra_output 0.23984939219421486 0.24331519913843716 -0.0034658069442222994 -0.0034609450639243224 -0.00817254174519036 0.001075937263766146 0.154 False 1000 +CC 0.0 436 f_max supra_relative_output supra_relative 0.34872847111799476 0.3441068514445836 0.004621619673411181 0.005303855095587073 -0.0030253861521123138 0.013984734899772753 0.244 False 1000 +CC 0.0 436 auprc supra_relative_output supra_relative 0.20876423335308655 0.17407450637713057 0.03468972697595599 0.03471689402177083 0.027015892040705095 0.042676742057124295 0.0 True 1000 +CC 0.0 436 f_max supra_relative_output supra_output 0.34872847111799476 0.38167092847103545 -0.03294245735304069 -0.032173744699361045 -0.04849079369778929 -0.016419441560441988 0.0 True 1000 +CC 0.0 436 auprc supra_relative_output supra_output 0.20876423335308655 0.24331519913843716 -0.03455096578535061 -0.03475067321777365 -0.04965092957831626 -0.01888470707056917 0.0 True 1000 +CC 0.0 436 f_max full supra_input_relative 0.37665530089167504 0.36749458141809443 0.009160719473580603 0.009079294940857964 0.0005314538044045028 0.017214012450495645 0.03 True 1000 +CC 0.0 436 auprc full supra_input_relative 0.22855037107450735 0.1990108702115158 0.029539500862991536 0.0293601327654084 0.02161812741852219 0.03643595279981731 0.0 True 1000 +CC 0.0 436 f_max full supra_input_output 0.37665530089167504 0.3808275731115995 -0.004172272219924444 -0.003826998780610088 -0.01621395353863158 0.007710646723017979 0.524 False 1000 +CC 0.0 436 auprc full supra_input_output 0.22855037107450735 0.23984939219421486 -0.011299021119707514 -0.011300091187603929 -0.021412851322363115 -0.0022790653683612155 0.012 True 1000 +CC 0.0 436 f_max full supra_relative_output 0.37665530089167504 0.34872847111799476 0.027926829773680273 0.026956436168080928 0.013081071726426415 0.04258543448214786 0.0 True 1000 +CC 0.0 436 auprc full supra_relative_output 0.22855037107450735 0.20876423335308655 0.019786137721420793 0.019989636966245405 0.007830026890763432 0.03304567701448772 0.0 True 1000 +CC 0.0 436 f_max single naive 0.37266279839932215 0.593050572267429 -0.22038777386810682 -0.22166688731819092 -0.25617584126010634 -0.1889366883163326 0.0 True 1000 +CC 0.0 436 auprc single naive 0.2138930026796082 0.5414744252479067 -0.3275814225682985 -0.3280586622129851 -0.36503167132135095 -0.28940479620442494 0.0 True 1000 +CC 0.0 436 f_max supra naive 0.37369500099331754 0.593050572267429 -0.21935557127411143 -0.2208975696842395 -0.25508102374425884 -0.18646788011853513 0.0 True 1000 +CC 0.0 436 auprc supra naive 0.21425352431010342 0.5414744252479067 -0.32722090093780326 -0.3277238848783639 -0.36439720431596506 -0.29040082279443064 0.0 True 1000 +CC 0.0 436 f_max supra_input naive 0.3657545116537211 0.593050572267429 -0.22729606061370788 -0.2270935873988026 -0.26008768277920397 -0.19285590162657057 0.0 True 1000 +CC 0.0 436 auprc supra_input naive 0.2001142781223428 0.5414744252479067 -0.34136014712556384 -0.3415903477630905 -0.37957126969416793 -0.3052439269281793 0.0 True 1000 +CC 0.0 436 f_max supra_relative naive 0.3441068514445836 0.593050572267429 -0.24894372082284538 -0.24993080023993025 -0.28657866753530187 -0.21377375604859303 0.0 True 1000 +CC 0.0 436 auprc supra_relative naive 0.17407450637713057 0.5414744252479067 -0.3673999188707761 -0.3682064853525353 -0.40313576300479464 -0.33184429461571985 0.0 True 1000 +CC 0.0 436 f_max supra_output naive 0.38167092847103545 0.593050572267429 -0.2113796437963935 -0.21245320044498212 -0.2473811849668355 -0.17861641454694416 0.0 True 1000 +CC 0.0 436 auprc supra_output naive 0.24331519913843716 0.5414744252479067 -0.29815922610946954 -0.2987389181129908 -0.33719337470361577 -0.26103199790958226 0.0 True 1000 +CC 0.0 436 f_max supra_input_relative naive 0.36749458141809443 0.593050572267429 -0.22555599084933453 -0.2267498039171202 -0.26151887059707835 -0.19275797460111568 0.0 True 1000 +CC 0.0 436 auprc supra_input_relative naive 0.1990108702115158 0.5414744252479067 -0.34246355503639087 -0.3428600871299275 -0.380039536547612 -0.30608165734594595 0.0 True 1000 +CC 0.0 436 f_max supra_input_output naive 0.3808275731115995 0.593050572267429 -0.21222299915582948 -0.21384351019565215 -0.2472994722736411 -0.17929420357954834 0.0 True 1000 +CC 0.0 436 auprc supra_input_output naive 0.23984939219421486 0.5414744252479067 -0.3016250330536918 -0.3021998631769151 -0.34090443378055396 -0.26435414457400713 0.0 True 1000 +CC 0.0 436 f_max supra_relative_output naive 0.34872847111799476 0.593050572267429 -0.2443221011494342 -0.24462694514434316 -0.28077966280620953 -0.20915530922828776 0.0 True 1000 +CC 0.0 436 auprc supra_relative_output naive 0.20876423335308655 0.5414744252479067 -0.3327101918948201 -0.3334895913307645 -0.3698285281720581 -0.2958344732039987 0.0 True 1000 +CC 0.0 436 f_max full naive 0.37665530089167504 0.593050572267429 -0.21639527137575393 -0.21767050897626222 -0.251806398551941 -0.1834383963410053 0.0 True 1000 +CC 0.0 436 auprc full naive 0.22855037107450735 0.5414744252479067 -0.3129240541733993 -0.3134999543645191 -0.3504248347044004 -0.2762078368540404 0.0 True 1000 +CC 0.0 436 f_max full single 0.37665530089167504 0.37266279839932215 0.0039925024923528896 0.003996378341928682 -0.00794276235902157 0.014972704518157106 0.474 False 1000 +CC 0.0 436 auprc full single 0.22855037107450735 0.2138930026796082 0.014657368394899145 0.014558707848466046 0.0026702495395733374 0.026013776420932166 0.018 True 1000 +CC 2.0 298 f_max supra single 0.2001711050330419 0.2010995525310512 -0.0009284474980092994 -0.0012696792566870357 -0.006425032783756005 0.003295993099931607 0.564 False 1000 +CC 2.0 298 auprc supra single 0.05474025260411656 0.055619578943533254 -0.0008793263394166959 -0.0008171509430065924 -0.004662264957919884 0.0019235214408674477 0.632 False 1000 +CC 2.0 298 f_max supra_input supra 0.18984512084059801 0.2001711050330419 -0.010325984192443877 -0.010630034100699718 -0.02575626407724032 0.0039049752303668282 0.148 False 1000 +CC 2.0 298 auprc supra_input supra 0.05384834822245379 0.05474025260411656 -0.0008919043816627698 -0.0008290163739503591 -0.007429437058561408 0.00530436177894736 0.816 False 1000 +CC 2.0 298 f_max supra_relative supra 0.17064440645913162 0.2001711050330419 -0.02952669857391027 -0.030246868145574517 -0.056478791183192156 -0.005798975545026582 0.018 True 1000 +CC 2.0 298 auprc supra_relative supra 0.04053061546270086 0.05474025260411656 -0.014209637141415696 -0.014638679303072874 -0.02804456046624593 -0.0024769646683679837 0.018 True 1000 +CC 2.0 298 f_max supra_output supra 0.20541777466459651 0.2001711050330419 0.0052466696315546235 0.005343782275744 -0.01385038266708692 0.022367903395429822 0.54 False 1000 +CC 2.0 298 auprc supra_output supra 0.05734132917056218 0.05474025260411656 0.0026010765664456204 0.0024232165231713554 -0.005170832740526746 0.009113758554666817 0.45 False 1000 +CC 2.0 298 f_max supra_input_relative supra_input 0.19658370250400492 0.18984512084059801 0.006738581663406906 0.007184988809019402 -0.011236017560743104 0.024380349660002774 0.456 False 1000 +CC 2.0 298 auprc supra_input_relative supra_input 0.048846805703567135 0.05384834822245379 -0.005001542518886654 -0.00525740316387153 -0.014074732103340612 0.002659404026880535 0.18 False 1000 +CC 2.0 298 f_max supra_input_relative supra_relative 0.19658370250400492 0.17064440645913162 0.0259392960448733 0.0268018228538942 0.009275708318245828 0.04596120229339355 0.002 True 1000 +CC 2.0 298 auprc supra_input_relative supra_relative 0.048846805703567135 0.04053061546270086 0.008316190240866272 0.008552259765250984 -0.0020443742814139393 0.019713957178272058 0.124 False 1000 +CC 2.0 298 f_max supra_input_output supra_input 0.2093688753175775 0.18984512084059801 0.019523754476979488 0.01992780226632906 0.006724899431226003 0.03358246106256974 0.008 True 1000 +CC 2.0 298 auprc supra_input_output supra_input 0.06107642913396808 0.05384834822245379 0.0072280809115142924 0.007426195580641484 0.002451934127591168 0.01302138762602383 0.0 True 1000 +CC 2.0 298 f_max supra_input_output supra_output 0.2093688753175775 0.20541777466459651 0.003951100652980988 0.0039539858898853435 -0.004220954660876197 0.013360138397150974 0.35 False 1000 +CC 2.0 298 auprc supra_input_output supra_output 0.06107642913396808 0.05734132917056218 0.0037350999634059023 0.00417396268351977 -0.0007275063657214685 0.010654441908725625 0.124 False 1000 +CC 2.0 298 f_max supra_relative_output supra_relative 0.1798405527981575 0.17064440645913162 0.009196146339025879 0.011030068463913953 0.003957344668956393 0.020208464536448974 0.004 True 1000 +CC 2.0 298 auprc supra_relative_output supra_relative 0.041418922970986816 0.04053061546270086 0.0008883075082859529 0.0009480651140916234 -0.007597938831723134 0.00799865833120388 0.766 False 1000 +CC 2.0 298 f_max supra_relative_output supra_output 0.1798405527981575 0.20541777466459651 -0.025577221866439015 -0.024560581957404567 -0.048515319238420784 -0.0012994726592806454 0.04 True 1000 +CC 2.0 298 auprc supra_relative_output supra_output 0.041418922970986816 0.05734132917056218 -0.015922406199575363 -0.016113830712152607 -0.03170133000422757 -0.0017964931824621345 0.032 True 1000 +CC 2.0 298 f_max full supra_input_relative 0.20442953930504945 0.19658370250400492 0.007845836801044531 0.007767239186282652 -0.0006922119644634264 0.017764148862395572 0.072 False 1000 +CC 2.0 298 auprc full supra_input_relative 0.04934663831207692 0.048846805703567135 0.0004998326085097829 0.0007229459316720335 -0.004868015787438324 0.006167640822074933 0.78 False 1000 +CC 2.0 298 f_max full supra_input_output 0.20442953930504945 0.2093688753175775 -0.0049393360125280505 -0.0049755742710270055 -0.021758660606533788 0.011643483495306866 0.55 False 1000 +CC 2.0 298 auprc full supra_input_output 0.04934663831207692 0.06107642913396808 -0.011729790821891163 -0.01196065281284098 -0.021098115873999103 -0.003534646139830322 0.004 True 1000 +CC 2.0 298 f_max full supra_relative_output 0.20442953930504945 0.1798405527981575 0.024588986506891952 0.023538993576262903 0.005386064516945151 0.043727588583110266 0.01 True 1000 +CC 2.0 298 auprc full supra_relative_output 0.04934663831207692 0.041418922970986816 0.007927715341090102 0.008327140582831394 -0.001650448659599913 0.01931690624171908 0.124 False 1000 +CC 2.0 298 f_max single naive 0.2010995525310512 0.2720885241633347 -0.07098897163228349 -0.07229415366398433 -0.11449926481410454 -0.030550732767724128 0.0 True 1000 +CC 2.0 298 auprc single naive 0.055619578943533254 0.1451197420848838 -0.08950016314135054 -0.08860376053383189 -0.11555780382125548 -0.057288234055071094 0.0 True 1000 +CC 2.0 298 f_max supra naive 0.2001711050330419 0.2720885241633347 -0.07191741913029279 -0.07356383292067138 -0.1158043796092431 -0.03139946864827564 0.0 True 1000 +CC 2.0 298 auprc supra naive 0.05474025260411656 0.1451197420848838 -0.09037948948076724 -0.08942091147683848 -0.11622651080917232 -0.06005881288425477 0.0 True 1000 +CC 2.0 298 f_max supra_input naive 0.18984512084059801 0.2720885241633347 -0.08224340332273666 -0.0841938670213711 -0.12331884646748366 -0.0457600787248451 0.0 True 1000 +CC 2.0 298 auprc supra_input naive 0.05384834822245379 0.1451197420848838 -0.09127139386243 -0.09024992785078884 -0.11552665625815932 -0.0618939258786257 0.0 True 1000 +CC 2.0 298 f_max supra_relative naive 0.17064440645913162 0.2720885241633347 -0.10144411770420306 -0.1038107010662459 -0.14096266592232642 -0.0659900048571656 0.0 True 1000 +CC 2.0 298 auprc supra_relative naive 0.04053061546270086 0.1451197420848838 -0.10458912662218292 -0.10405959077991137 -0.12747164385010246 -0.08027455780084128 0.0 True 1000 +CC 2.0 298 f_max supra_output naive 0.20541777466459651 0.2720885241633347 -0.06667074949873816 -0.06822005064492738 -0.10759796344993118 -0.027971343993855036 0.004 True 1000 +CC 2.0 298 auprc supra_output naive 0.05734132917056218 0.1451197420848838 -0.08777841291432162 -0.08699769495366712 -0.1133657167149178 -0.05815731597084263 0.0 True 1000 +CC 2.0 298 f_max supra_input_relative naive 0.19658370250400492 0.2720885241633347 -0.07550482165932976 -0.07700887821235171 -0.11623699957060177 -0.03511469458858323 0.002 True 1000 +CC 2.0 298 auprc supra_input_relative naive 0.048846805703567135 0.1451197420848838 -0.09627293638131665 -0.09550733101466037 -0.12075774750985441 -0.06902551680707548 0.0 True 1000 +CC 2.0 298 f_max supra_input_output naive 0.2093688753175775 0.2720885241633347 -0.06271964884575718 -0.06426606475504204 -0.10318817498318242 -0.024433271991500232 0.006 True 1000 +CC 2.0 298 auprc supra_input_output naive 0.06107642913396808 0.1451197420848838 -0.08404331295091572 -0.08282373227014736 -0.10893311328594388 -0.054377723266779344 0.0 True 1000 +CC 2.0 298 f_max supra_relative_output naive 0.1798405527981575 0.2720885241633347 -0.09224797136517718 -0.09278063260233194 -0.13011539576554568 -0.055109975926851996 0.0 True 1000 +CC 2.0 298 auprc supra_relative_output naive 0.041418922970986816 0.1451197420848838 -0.10370081911389697 -0.10311152566581973 -0.12455608968595752 -0.0807730621721488 0.0 True 1000 +CC 2.0 298 f_max full naive 0.20442953930504945 0.2720885241633347 -0.06765898485828523 -0.06924163902606906 -0.10697566224417526 -0.02960347546220344 0.004 True 1000 +CC 2.0 298 auprc full naive 0.04934663831207692 0.1451197420848838 -0.09577310377280687 -0.09478438508298834 -0.11826558191815217 -0.06964335947164657 0.0 True 1000 +CC 2.0 298 f_max full single 0.20442953930504945 0.2010995525310512 0.0033299867739982614 0.0030525146379153023 -0.019370248432472172 0.02368597812870509 0.766 False 1000 +CC 2.0 298 auprc full single 0.04934663831207692 0.055619578943533254 -0.006272940631456336 -0.006180624549156449 -0.019223883621406965 0.005112810994309226 0.322 False 1000 +CC 4.0 210 f_max supra single 0.12814077896255177 0.12866270049406495 -0.0005219215315131753 0.0006597403274266857 -0.009096521511082884 0.011279322633682481 0.89 False 1000 +CC 4.0 210 auprc supra single 0.030098649903147593 0.031321891529791215 -0.0012232416266436223 -0.0014138247656030258 -0.0049021405064871475 0.0009577446699582367 0.334 False 1000 +CC 4.0 210 f_max supra_input supra 0.1263849974354251 0.12814077896255177 -0.0017557815271266597 -0.004165247810447397 -0.02533357480761 0.012440036178935365 0.68 False 1000 +CC 4.0 210 auprc supra_input supra 0.03115695182023453 0.030098649903147593 0.0010583019170869352 0.0011240366683441976 -0.0041230995675745116 0.006150248060673097 0.604 False 1000 +CC 4.0 210 f_max supra_relative supra 0.06727985486494541 0.12814077896255177 -0.060860924097606364 -0.06504716338367249 -0.10239052723674105 -0.028106320137072836 0.0 True 1000 +CC 4.0 210 auprc supra_relative supra 0.011264545566505739 0.030098649903147593 -0.018834104336641854 -0.019605847984558906 -0.03605830617235659 -0.005506854612364194 0.004 True 1000 +CC 4.0 210 f_max supra_output supra 0.11733779836398164 0.12814077896255177 -0.010802980598570128 -0.010466225712264394 -0.021252605826222414 5.138622560106162e-05 0.054 False 1000 +CC 4.0 210 auprc supra_output supra 0.03523412972085431 0.030098649903147593 0.005135479817706717 0.004593332476647589 -0.0022331773211001936 0.012968244555799672 0.184 False 1000 +CC 4.0 210 f_max supra_input_relative supra_input 0.11463965887981685 0.1263849974354251 -0.011745338555608262 -0.013669847504231761 -0.032813396930014435 0.0011842056057927283 0.068 False 1000 +CC 4.0 210 auprc supra_input_relative supra_input 0.026881474129342588 0.03115695182023453 -0.004275477690891941 -0.004593664227107958 -0.015334144867977252 0.004934468883427645 0.352 False 1000 +CC 4.0 210 f_max supra_input_relative supra_relative 0.11463965887981685 0.06727985486494541 0.04735980401487144 0.047212068068993335 0.014481346614879194 0.08088477517901775 0.0 True 1000 +CC 4.0 210 auprc supra_input_relative supra_relative 0.026881474129342588 0.011264545566505739 0.015616928562836849 0.016136220425795145 0.005298760419137329 0.029671791524312218 0.002 True 1000 +CC 4.0 210 f_max supra_input_output supra_input 0.12239568929514537 0.1263849974354251 -0.003989308140279743 -0.0034299012895581417 -0.010557076693303878 0.002860464948688329 0.37 False 1000 +CC 4.0 210 auprc supra_input_output supra_input 0.03416233160249707 0.03115695182023453 0.003005379782262542 0.002601055259175447 -0.0020212712831946004 0.00921306719944254 0.402 False 1000 +CC 4.0 210 f_max supra_input_output supra_output 0.12239568929514537 0.11733779836398164 0.005057890931163725 0.0028710766122588558 -0.019697789284459144 0.021313131695142574 0.714 False 1000 +CC 4.0 210 auprc supra_input_output supra_output 0.03416233160249707 0.03523412972085431 -0.0010717981183572395 -0.0008682405491279443 -0.007810715034177995 0.005681362699041299 0.824 False 1000 +CC 4.0 210 f_max supra_relative_output supra_relative 0.06563727515811953 0.06727985486494541 -0.0016425797068258807 -0.0015278271121728926 -0.008078471231620917 0.0036936116996285724 0.63 False 1000 +CC 4.0 210 auprc supra_relative_output supra_relative 0.013385795583400913 0.011264545566505739 0.0021212500168951744 0.002168044314117692 -0.0038817715106782007 0.0099968470856998 0.516 False 1000 +CC 4.0 210 f_max supra_relative_output supra_output 0.06563727515811953 0.11733779836398164 -0.05170052320586212 -0.056108764783581 -0.09307950484045725 -0.02180926388385634 0.0 True 1000 +CC 4.0 210 auprc supra_relative_output supra_output 0.013385795583400913 0.03523412972085431 -0.021848334137453396 -0.022031136147088803 -0.04395528521798049 -0.0023691998277416077 0.028 True 1000 +CC 4.0 210 f_max full supra_input_relative 0.12055865495429378 0.11463965887981685 0.005918996074476934 0.0036486702937316073 -0.005204953575564666 0.013348585782939862 0.422 False 1000 +CC 4.0 210 auprc full supra_input_relative 0.026574800701653457 0.026881474129342588 -0.000306673427689131 -0.00041913802949048915 -0.006778187758946827 0.006128046197465842 0.868 False 1000 +CC 4.0 210 f_max full supra_input_output 0.12055865495429378 0.12239568929514537 -0.0018370343408515843 -0.006591275920942012 -0.0258346724200935 0.008869884659719126 0.508 False 1000 +CC 4.0 210 auprc full supra_input_output 0.026574800701653457 0.03416233160249707 -0.0075875309008436136 -0.007613857515773895 -0.01776945644816896 0.0008099544042016503 0.074 False 1000 +CC 4.0 210 f_max full supra_relative_output 0.12055865495429378 0.06563727515811953 0.05492137979617426 0.052388565474897834 0.02042196908719534 0.0862134957274378 0.002 True 1000 +CC 4.0 210 auprc full supra_relative_output 0.026574800701653457 0.013385795583400913 0.013189005118252543 0.013549038082186961 0.0016651776475756956 0.026474993044300996 0.03 True 1000 +CC 4.0 210 f_max single naive 0.12866270049406495 0.10440217265761789 0.024260527836447054 0.027633327101810334 -0.016104144881318245 0.07128575046017728 0.214 False 1000 +CC 4.0 210 auprc single naive 0.031321891529791215 0.04494942594240296 -0.013627534412611747 -0.011668829387998512 -0.03114584427596266 0.013563083653009867 0.31 False 1000 +CC 4.0 210 f_max supra naive 0.12814077896255177 0.10440217265761789 0.02373860630493388 0.02829306742923702 -0.015392086256605406 0.07385342192299574 0.198 False 1000 +CC 4.0 210 auprc supra naive 0.030098649903147593 0.04494942594240296 -0.01485077603925537 -0.013082654153601539 -0.03213906712088271 0.009808570061206815 0.232 False 1000 +CC 4.0 210 f_max supra_input naive 0.1263849974354251 0.10440217265761789 0.02198282477780722 0.024127819618789622 -0.015571069731667317 0.06856847174121249 0.268 False 1000 +CC 4.0 210 auprc supra_input naive 0.03115695182023453 0.04494942594240296 -0.013792474122168434 -0.01195861748525734 -0.030364013754128456 0.011511691688457499 0.286 False 1000 +CC 4.0 210 f_max supra_relative naive 0.06727985486494541 0.10440217265761789 -0.037122317792672485 -0.03675409595443547 -0.07151261058420084 0.002253622363397643 0.074 False 1000 +CC 4.0 210 auprc supra_relative naive 0.011264545566505739 0.04494942594240296 -0.03368488037589722 -0.03268850213816044 -0.04536327888508021 -0.01552342324074404 0.002 True 1000 +CC 4.0 210 f_max supra_output naive 0.11733779836398164 0.10440217265761789 0.012935625706363751 0.017826841716972623 -0.021297494620056902 0.059681263865493495 0.42 False 1000 +CC 4.0 210 auprc supra_output naive 0.03523412972085431 0.04494942594240296 -0.009715296221548653 -0.008489321676953949 -0.029671537875034827 0.0172989529003151 0.5 False 1000 +CC 4.0 210 f_max supra_input_relative naive 0.11463965887981685 0.10440217265761789 0.010237486222198958 0.010457972114557861 -0.030992376292890093 0.051509107880363884 0.632 False 1000 +CC 4.0 210 auprc supra_input_relative naive 0.026881474129342588 0.04494942594240296 -0.018067951813060375 -0.0165522817123653 -0.03464679119064451 0.006039362555465282 0.15 False 1000 +CC 4.0 210 f_max supra_input_output naive 0.12239568929514537 0.10440217265761789 0.017993516637527476 0.02069791832923148 -0.01998198002422297 0.06423761586674694 0.342 False 1000 +CC 4.0 210 auprc supra_input_output naive 0.03416233160249707 0.04494942594240296 -0.010787094339905892 -0.009357562226081893 -0.029061133165358223 0.01586801409856701 0.422 False 1000 +CC 4.0 210 f_max supra_relative_output naive 0.06563727515811953 0.10440217265761789 -0.038764897499498366 -0.03828192306660836 -0.07204350659777255 0.000720704029956439 0.056 False 1000 +CC 4.0 210 auprc supra_relative_output naive 0.013385795583400913 0.04494942594240296 -0.031563630359002046 -0.03052045782404275 -0.045440537254651484 -0.010298178089673045 0.002 True 1000 +CC 4.0 210 f_max full naive 0.12055865495429378 0.10440217265761789 0.016156482296675892 0.014106642408289469 -0.026504048385465964 0.05501217505139191 0.514 False 1000 +CC 4.0 210 auprc full naive 0.026574800701653457 0.04494942594240296 -0.018374625240749506 -0.016971419741855787 -0.03503178739686338 0.004819235858373889 0.128 False 1000 +CC 4.0 210 f_max full single 0.12055865495429378 0.12866270049406495 -0.008104045539771162 -0.013526684693520865 -0.04105701444784193 0.008991731161521656 0.286 False 1000 +CC 4.0 210 auprc full single 0.026574800701653457 0.031321891529791215 -0.004747090828137759 -0.005302590353857276 -0.017415581572057925 0.005515962621526646 0.376 False 1000 diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_permutation_null.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_permutation_null.tsv new file mode 100644 index 0000000..3f3a34a --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_permutation_null.tsv @@ -0,0 +1,19 @@ +aspect min_ic metric rung n_eval_proteins observed null_n null_mean null_sd null_min null_max null_ci_lo null_ci_hi ratio_observed_over_null_mean z_score empirical_p +BP 0.0 auprc supra 298 0.10296371456335705 200 0.04110811260441004 0.004310890838282247 0.030500481389928343 0.05812228647610969 0.033555192793106245 0.04979344736064999 2.504705471501292 14.34868204261802 0.004975124378109453 +BP 0.0 f_max supra 298 0.22413162677065615 200 0.1212830903961599 0.0084707322850641 0.10243940537335518 0.14863263993090034 0.10507902816750196 0.13736798015932858 1.8480039223815214 12.1416346206387 0.004975124378109453 +BP 2.0 auprc supra 298 0.06741104829568473 200 0.005447461156336378 0.0013292702156578156 0.002943506896158224 0.011952847673048763 0.003265828152672613 0.008441345574586189 12.374764383087623 46.61474123881159 0.004975124378109453 +BP 2.0 f_max supra 298 0.16719947495645657 200 0.04998728911797779 0.006413145694972749 0.03505384891600458 0.07193001545825739 0.038771484960256695 0.06288037116883011 3.344839816414924 18.27686308925761 0.004975124378109453 +BP 4.0 auprc supra 292 0.03405078849746584 200 0.0005818930084330975 0.0003126567050956244 0.00013843393471513665 0.0020103917990846394 0.0001924810300494663 0.0015048627259179603 58.51726692705363 107.0467862788884 0.004975124378109453 +BP 4.0 f_max supra 292 0.11828896656791534 200 0.015647566263096456 0.004452947391048114 0.00737430320349715 0.030829012467119304 0.009076123041265666 0.02735400219813553 7.559576011950847 23.050216248043217 0.004975124378109453 +CC 0.0 auprc supra 436 0.21425352431010342 200 0.15036236795252805 0.011955844856021944 0.11069036664131138 0.18648815163550753 0.1276758703324013 0.17320161898264516 1.42491453963898 5.343926516861293 0.004975124378109453 +CC 0.0 f_max supra 436 0.37369500099331754 200 0.2700373986429829 0.014849917188290743 0.23332617834326896 0.31021515509821723 0.23918844083137827 0.2977184076852366 1.3838638754159407 6.980348848818454 0.004975124378109453 +CC 2.0 auprc supra 298 0.05474025260411656 200 0.008512256194817321 0.0033678828311570143 0.0034465723523367966 0.0212760073957525 0.004342345909096564 0.01807789526453829 6.430757175453096 13.726129656778443 0.004975124378109453 +CC 2.0 f_max supra 298 0.2001711050330419 200 0.06905990257217472 0.012662070517299706 0.04419120721845692 0.1073085639328589 0.04957905177011041 0.09737333965289174 2.8985141533300376 10.354641626875708 0.004975124378109453 +CC 4.0 auprc supra 210 0.030098649903147593 200 0.001291981390897895 0.0007406537655915228 0.00019140392764687304 0.004526612216647574 0.0003230395920669119 0.0030069158783171948 23.296504202920275 38.89356923642084 0.004975124378109453 +CC 4.0 f_max supra 210 0.12814077896255177 200 0.027704186363063984 0.00809365257337959 0.011076889824898475 0.052480110826607966 0.013650845264166133 0.04565619924592361 4.625321865918168 12.40930367209343 0.004975124378109453 +MF 0.0 auprc supra 406 0.171545113956216 200 0.05873768105661184 0.008522398458459791 0.0327126033208327 0.0839625646985584 0.04480822865369315 0.07552636373262862 2.9205292219636565 13.236582805820987 0.004975124378109453 +MF 0.0 f_max supra 406 0.34357928540123506 200 0.18150006302835991 0.018685002850936973 0.1283576211930075 0.24338134777701984 0.14582138347174653 0.2187574571048149 1.8929981602681305 8.67429476280479 0.004975124378109453 +MF 2.0 auprc supra 159 0.18918327277822655 200 0.005215926227973121 0.00279774935660825 0.0009402975860487409 0.020864894030750692 0.0019139445089873092 0.012736440634415086 36.27031221485318 65.75547810089735 0.004975124378109453 +MF 2.0 f_max supra 159 0.32980140754537307 200 0.04853795828098417 0.01465228997334194 0.018499166617731 0.10319629342001008 0.026544960941971365 0.079013352682614 6.7947111750388585 19.195869708838245 0.004975124378109453 +MF 4.0 auprc supra 151 0.17827778650251944 200 0.00275643765644404 0.0018078817249921576 0.00011521453387309605 0.013145740868420774 0.0008116971023235028 0.006864301170674047 64.67687962604163 97.08674324192131 0.004975124378109453 +MF 4.0 f_max supra 151 0.30712672755850184 200 0.03445935046138527 0.012250695061031774 0.006149932260881419 0.09693684804905771 0.017162530864132726 0.06413256115522621 8.912725383569384 22.257298523774704 0.004975124378109453 diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_provenance.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_provenance.tsv new file mode 100644 index 0000000..8f3689a --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_provenance.tsv @@ -0,0 +1,16 @@ +rung label adds predictions_file n_rows n_domains score interpro_file interpro_sha256 cohort_interpro_file cohort_interpro_sha256 t0_gaf_file t0_gaf_sha256 t1_gaf_file t1_gaf_sha256 +single single domains - /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_significant.tsv 37536 5114 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +single__p single domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_significant.tsv 37536 5114 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra + supra-domains supra-domains /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_significant.tsv 144823 23113 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra__p + supra-domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_significant.tsv 144823 23113 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input + input propagation input propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_significant.tsv 436620 23379 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input__p + input propagation (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_significant.tsv 436620 23379 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative + relative inference relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_significant.tsv 28282 9440 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative__p + relative inference (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_significant.tsv 28282 9440 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_output + output propagation output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output/domain_go_annotations_propagated.tsv 979440 23113 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative input + relative input propagation + relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_significant.tsv 85285 10392 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative__p input + relative (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_significant.tsv 85285 10392 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_output input + output input + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output/domain_go_annotations_propagated.tsv 1064167 23379 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative_output relative + output relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output/domain_go_annotations_propagated.tsv 152781 9440 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +full full current method input propagation + relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full/domain_go_annotations_propagated.tsv 255986 10392 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +naive CAFA naive baseline - - 19299 0 propagated t0 term frequency /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/interim/protein2ipr_human.dat.gz a932a51500cdee203b6235c93b3ed4a26db993c45439484c19aefe804e3e8611 data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv new file mode 100644 index 0000000..dfc3c60 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv @@ -0,0 +1,22 @@ +stage aspect min_ic n_proteins n_dropped_vs_ic0 pct_of_ic0 note +t0_annotated_proteins - - 18735 - - proteins with >=1 GO annotation at t0 under the training evidence filter (the training universe) +t1_experimental_proteins - - 16362 - - proteins with >=1 experimental GO annotation at t1 (the gold standard) +proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain (predictable at all by a domain method) +no_knowledge_candidates BP - 310 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains BP - 298 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures BP - 19 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +no_knowledge_candidates MF - 418 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains MF - 406 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures MF - 16 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +no_knowledge_candidates CC - 453 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains CC - 436 - - the scored cohort before any IC floor +no_knowledge_without_transfer_architectures CC - 14 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses +cohort_at_ic_floor BP 0 298 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 2 298 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 4 292 6 98.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 0 406 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 2 159 247 39.2 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 4 151 255 37.2 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 0 436 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 2 298 138 68.3 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 4 210 226 48.2 proteins whose truth is non-empty at this IC floor diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/full.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/full.json new file mode 100644 index 0000000..d32333c --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/full.json @@ -0,0 +1,216 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full", + "--propagate-annotations", + "--enable-relative-inference", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:54:21.733116+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:54:20.036660+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "4c32c3ebcb84c44c78b13e450bab66c89cf51d5f42965014ba222993d596f0ec", + "size_bytes": 12489205 + }, + { + "modified_at": "2026-10-07T09:54:20.106663+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "5de3ae13172882527c98832d626563bbc259593444940079037b9b66764e0bc9", + "size_bytes": 11805 + }, + { + "modified_at": "2026-10-07T09:54:21.492718+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "ab016a81862189fad01c841be012f3a898b8f207e39472bb63f15e1bef503fbd", + "size_bytes": 21440223 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:47:40.159698+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 8.576804544037081e-07, + "supra": 4.4757657941358196e-07 + }, + "domains": 88522, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4699, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 926, + "ontology": "go", + "propagated_annotations": 255986, + "proteins": 18507, + "runtime_seconds": 71.59, + "significant_associations": 85285, + "terms": 18359, + "tests": 1625175398, + "tests_evaluated": 12190154 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/protein2ipr_human_t0_ipr85.dat.gz.provenance.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/protein2ipr_human_t0_ipr85.dat.gz.provenance.json new file mode 100644 index 0000000..68f9c17 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/protein2ipr_human_t0_ipr85.dat.gz.provenance.json @@ -0,0 +1,12 @@ +{ + "selection_rule": "goa", + "selection_sources": [ + "data/raw/goa_archive/goa_human.gaf.205.gz", + "data/raw/goa_annotations/goa_human.gaf.gz" + ], + "interpro_source": "data/raw/interpro_archive/85.0/protein2ipr.dat.gz", + "n_accessions": 19871, + "n_matched_lines": 232836, + "tool": "extract_human_interpro.py", + "created": "2026-10-07T09:11:26+00:00" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/single.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/single.json new file mode 100644 index 0000000..95f04d8 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/single.json @@ -0,0 +1,188 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": false, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "direct", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single", + "--disable-supra-domains", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:11:59.827700+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:11:59.768879+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "07aae6baa8227958de94cb56a39d3b5225c02b0f7745935b98e5590ce5f8b5ca", + "size_bytes": 4377819 + }, + { + "modified_at": "2026-10-07T09:11:59.807880+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "e2eed33409fe1dd5f2dfa385974588e10671e05a4801878fad9c8ae230efe80a", + "size_bytes": 9334 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": false, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:11:30.792766+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4015190222725488e-06 + }, + "domains": 16659, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18507, + "runtime_seconds": 3.33, + "significant_associations": 37536, + "terms": 16040, + "tests": 267210360, + "tests_evaluated": 575091 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra.json new file mode 100644 index 0000000..b1a44c8 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra.json @@ -0,0 +1,188 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "direct", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:13:22.380647+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:13:22.220374+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "04717940a481e722fa459f5487934e5a19b59121356566ea9432b572d05bed0f", + "size_bytes": 21489331 + }, + { + "modified_at": "2026-10-07T09:13:22.279377+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "976f9fac4232733a7840d6e2429c8a6b90008b26f894b359937a7cab815c70b8", + "size_bytes": 12026 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:12:02.559685+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4015190222725488e-06, + "supra": 9.307380316674168e-07 + }, + "domains": 88522, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18507, + "runtime_seconds": 9.75, + "significant_associations": 144823, + "terms": 16040, + "tests": 1419892880, + "tests_evaluated": 2694075 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input.json new file mode 100644 index 0000000..695424e --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input.json @@ -0,0 +1,207 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input", + "--propagate-annotations", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:19:37.221611+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:19:36.579120+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "c669fa4dd7dbcf1d375dd40fb061bbead071b35b50c9a70024d122565a463cac", + "size_bytes": 64337940 + }, + { + "modified_at": "2026-10-07T09:19:36.843131+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "9232fd9f0b3931c33bfb93ae89aa6390bfdeabcc5c83b3269b158fb823f0f6cd", + "size_bytes": 11831 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:13:26.929335+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.8130838449298743e-06, + "supra": 2.4246489122437422e-06 + }, + "domains": 88522, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4699, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 926, + "ontology": "go", + "propagated_annotations": 0, + "proteins": 18507, + "runtime_seconds": 87.24, + "significant_associations": 436620, + "terms": 18359, + "tests": 1625175398, + "tests_evaluated": 12190154 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_output.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_output.json new file mode 100644 index 0000000..d2ff4d6 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_output.json @@ -0,0 +1,215 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output", + "--propagate-annotations", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:44:49.312129+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:44:37.081697+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "c669fa4dd7dbcf1d375dd40fb061bbead071b35b50c9a70024d122565a463cac", + "size_bytes": 64337940 + }, + { + "modified_at": "2026-10-07T09:44:37.377708+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "9232fd9f0b3931c33bfb93ae89aa6390bfdeabcc5c83b3269b158fb823f0f6cd", + "size_bytes": 11831 + }, + { + "modified_at": "2026-10-07T09:44:48.329130+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "a9966fc0d5e3edb1d03290b3c983877fe4aeba1f2122dd7539441bd7f0218a96", + "size_bytes": 89503999 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:32:42.420201+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.8130838449298743e-06, + "supra": 2.4246489122437422e-06 + }, + "domains": 88522, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4699, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 926, + "ontology": "go", + "propagated_annotations": 1064167, + "proteins": 18507, + "runtime_seconds": 68.29, + "significant_associations": 436620, + "terms": 18359, + "tests": 1625175398, + "tests_evaluated": 12190154 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_relative.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_relative.json new file mode 100644 index 0000000..f092a73 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_input_relative.json @@ -0,0 +1,215 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": true, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative", + "--propagate-annotations", + "--enable-relative-inference", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:32:34.265736+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:32:33.431473+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "4c32c3ebcb84c44c78b13e450bab66c89cf51d5f42965014ba222993d596f0ec", + "size_bytes": 12489205 + }, + { + "modified_at": "2026-10-07T09:32:33.544478+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "5de3ae13172882527c98832d626563bbc259593444940079037b9b66764e0bc9", + "size_bytes": 11805 + }, + { + "modified_at": "2026-10-07T09:32:34.110502+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "a2fd27174615db669354356a3f6cae52b3643ebd489cc3b0a2a13c8fd792d36f", + "size_bytes": 6874163 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative", + "permute_annotations": null, + "propagate_annotations": true, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:26:19.142591+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 8.576804544037081e-07, + "supra": 4.4757657941358196e-07 + }, + "domains": 88522, + "input_alt_ids_remapped": 74, + "input_pairs_dropped": 4699, + "input_replaced_by_remapped": 417, + "input_terms_not_in_hierarchy": 926, + "ontology": "go", + "propagated_annotations": 85285, + "proteins": 18507, + "runtime_seconds": 71.37, + "significant_associations": 85285, + "terms": 18359, + "tests": 1625175398, + "tests_evaluated": 12190154 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_output.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_output.json new file mode 100644 index 0000000..49767f2 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_output.json @@ -0,0 +1,210 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": false, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": null + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:26:11.522842+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:26:05.860364+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "ba84336441ff2a0ce9fe8d8c0fdbb141d6cfb565a81239de66f1a75315e2fe27", + "size_bytes": 21395599 + }, + { + "modified_at": "2026-10-07T09:26:05.942367+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "72cfdfbea66aeb4759bba85c7efb9c5b6ebb31dee638151a9f32246dd98af7d3", + "size_bytes": 12016 + }, + { + "modified_at": "2026-10-07T09:26:10.778568+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "6dfea77e1f74466ea680f345ab87861931e995d3d65f785ad6640253344687e3", + "size_bytes": 83366594 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": false, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:22:01.471274+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 1.4015190222725488e-06, + "supra": 9.307380316674168e-07 + }, + "domains": 88522, + "ontology": "go", + "propagated_annotations": 979440, + "proteins": 18507, + "runtime_seconds": 20.1, + "significant_associations": 144823, + "terms": 16040, + "tests": 1419892880, + "tests_evaluated": 2694075 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative.json new file mode 100644 index 0000000..c399677 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative.json @@ -0,0 +1,210 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": false + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative", + "--enable-relative-inference", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:21:53.459318+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:21:53.062826+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "c3517e3ba80edf4ecdfcbc247759555f547503db912fd9626b17f33c2b81eaec", + "size_bytes": 4093312 + }, + { + "modified_at": "2026-10-07T09:21:53.158830+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "f280a93c0e4569836f391ed230d7555b7bae95ce4be50ed29c563b9c5b8dfa66", + "size_bytes": 12555 + }, + { + "modified_at": "2026-10-07T09:21:53.402840+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "42034df9acf9d1613bcd7b60bac3868b1f6ae361de66ef9b5614f6701f8d4969", + "size_bytes": 2210144 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": false, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:19:44.806994+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.1457048492334887e-07, + "supra": 1.7228151621835345e-07 + }, + "domains": 88522, + "ontology": "go", + "propagated_annotations": 28282, + "proteins": 18507, + "runtime_seconds": 20.21, + "significant_associations": 28282, + "terms": 16040, + "tests": 1419892880, + "tests_evaluated": 2694075 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative_output.json b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative_output.json new file mode 100644 index 0000000..c8d7ac1 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro/manifests/supra_relative_output.json @@ -0,0 +1,211 @@ +{ + "analysis": { + "ontology": { + "annotation_inputs": [ + "gaf" + ], + "hierarchy_inputs": [ + "go_obo" + ], + "key": "go", + "label": "go", + "name": "Gene Ontology", + "ontology_id": "GO", + "propagation": "go_dag", + "propagation_relations": [ + "is_a", + "part_of" + ], + "relative_inference_enabled": true, + "supports_true_path": true, + "term_prefix": "GO:", + "true_path_enabled": true + }, + "thresholds": { + "enable_supra_domains": true, + "evidence_filter": "manual", + "fdr_families": [ + "single", + "supra" + ], + "fdr_method": "benjamini_hochberg", + "fdr_threshold": 0.01, + "fisher_alternative": "greater", + "ic_source": "propagated", + "input_annotations_propagated": false, + "max_supra_domain_length": 3, + "min_domain_length": 10, + "min_ic": null, + "min_proteins_per_association": null, + "parental_background_min_size": 3, + "relative_inference_combination": "max_overall_relative_then_bh" + } + }, + "command": [ + "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "run_dcgo_human.py", + "--num-cores", + "8", + "--output-dir", + "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output", + "--enable-relative-inference", + "--enable-true-path", + "--gaf", + "data/raw/goa_archive/goa_human.gaf.205.gz", + "--interpro", + "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz" + ], + "completed_at": "2026-10-07T09:47:32.683067+00:00", + "git": { + "available": true, + "branch": "t0-interpro-inputs", + "commit": "ac7485a81097fae331880065c994632cfa6742c5", + "dirty": false, + "untracked_files": 0 + }, + "inputs": [ + { + "modified_at": "2026-10-07T09:11:26.842479+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "role": "domain_annotations", + "sha256": "b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c", + "size_bytes": 2668416 + }, + { + "modified_at": "2026-07-09T09:44:17.040277+00:00", + "path": "data/raw/goa_archive/goa_human.gaf.205.gz", + "release_metadata": { + "date_generated": "2021-04-21 13:14", + "gaf_version": "2.2", + "generated_by": "UniProt", + "go_version": "http://purl.obolibrary.org/obo/go/releases/2021-04-18/extensions/go-plus.owl" + }, + "role": "gaf", + "sha256": "69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86", + "size_bytes": 8577421 + }, + { + "modified_at": "2026-07-07T19:03:02.551052+00:00", + "path": "data/raw/go_ontology/go-basic.obo", + "release_metadata": { + "data_version": "releases/2026-06-15", + "format_version": "1.2" + }, + "role": "go_obo", + "sha256": "c72fc198a86983d55e43aac585d1ffdbeb6e3601475b3f18b6045acdc0a0734c", + "size_bytes": 32215811, + "source_url": "https://current.geneontology.org/ontology/go-basic.obo" + } + ], + "outputs": [ + { + "modified_at": "2026-10-07T09:47:31.478465+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output/domain_go_associations_significant.tsv", + "role": "significant_associations", + "sha256": "c3517e3ba80edf4ecdfcbc247759555f547503db912fd9626b17f33c2b81eaec", + "size_bytes": 4093312 + }, + { + "modified_at": "2026-10-07T09:47:31.608470+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output/domain_go_associations_top100.tsv", + "role": "top100", + "sha256": "f280a93c0e4569836f391ed230d7555b7bae95ce4be50ed29c563b9c5b8dfa66", + "size_bytes": 12555 + }, + { + "modified_at": "2026-10-07T09:47:32.546507+00:00", + "path": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output/domain_go_annotations_propagated.tsv", + "role": "propagated_annotations", + "sha256": "4304450978c1f9c673407ec3ba17f113a7ef0d6e827d022c43a460aed10b4ae7", + "size_bytes": 12798248 + } + ], + "parameters": { + "batch_size": 50000, + "chebi_obo": "data/raw/chebi/chebi_lite.obo", + "civic_evidence": "data/raw/civic/nightly-ClinicalEvidenceSummaries.tsv", + "doid_obo": "data/raw/disease_ontology/doid.obo", + "domain_key": "interpro", + "efo_obo": "data/raw/efo/efo.obo", + "enable_relative_inference": true, + "enable_supra_domains": true, + "enable_true_path": true, + "enzyme_dat": "data/raw/enzyme/enzyme.dat", + "evidence_filter": "manual", + "fb_genotype_phenotype": "data/raw/flybase/genotype_phenotype_data_fb_2026_02.tsv.gz", + "fbal_to_fbgn": "data/raw/flybase/fbal_to_fbgn_fb_2026_02.tsv.gz", + "fbbt_obo": "data/raw/flybase_ontology/fbbt.obo", + "fbcv_obo": "data/raw/flybase_ontology/fbcv.obo", + "fbgn_uniprot": "data/raw/flybase/fbgn_NAseq_Uniprot_fb_2026_02.tsv.gz", + "fdr_threshold": 0.01, + "gaf": "data/raw/goa_archive/goa_human.gaf.205.gz", + "go_ontology": "data/raw/go_ontology/go-basic.obo", + "gwas_associations": "data/raw/gwas_catalog/gwas-catalog-associations_ontology-annotated-full.zip", + "hpa_single_cell": "data/raw/hpa/rna_single_cell_type.tsv.zip", + "hpo_genes_to_phenotype": "data/raw/hpo/genes_to_phenotype.txt", + "hpo_obo": "data/raw/hpo/hp.obo", + "interpro": "/home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz", + "keyword_list": "data/raw/uniprot_keywlist/keywlist.txt", + "mgi_genepheno": "data/raw/mgi_reports/MGI_GenePheno.rpt", + "mgi_marker_swissprot": "data/raw/mgi_reports/MRK_SwissProt_TrEMBL.rpt", + "min_ic": 0.0, + "min_support": 0, + "mondo_obo": "data/raw/mondo/mondo.obo", + "mp_obo": "data/raw/mp_ontology/mp.obo", + "ncit_obo": "data/raw/ncit/ncit.obo", + "num_cores": 8, + "oncotree_json": "data/raw/oncotree/oncotree_tumortypes.json", + "ontology": "go", + "output_dir": "/home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output", + "permute_annotations": null, + "propagate_annotations": false, + "reactome_relations": "data/raw/reactome_relations/ReactomePathwaysRelation.txt", + "species": "human", + "subcell": "data/raw/uniprot_subcell/subcell.txt", + "syngo_zip": "data/raw/syngo/syngo1.3_complete_data.zip", + "uniprot_dat": "data/raw/uniprot_sprot_dat/uniprot_sprot.dat.gz", + "wb_anatomy": "data/raw/wormbase/anatomy_association.WS298.wb.gz", + "wb_phenotype": "data/raw/wormbase/phenotype_association.WS298.wb.gz", + "wbbt_obo": "data/raw/wormbase_ontology/wbbt.obo", + "wbphenotype_obo": "data/raw/wormbase_ontology/wbphenotype.obo", + "worm_idmapping": "data/raw/uniprot_idmapping/CAEEL_6239_idmapping.dat.gz", + "xref_db": null, + "xref_term_from_type": false, + "xref_type": null, + "zfa_obo": "data/raw/zfin_ontology/zfa.obo", + "zfin_phenotype": "data/raw/zfin/phenoGeneCleanData_fish.txt", + "zfin_uniprot": "data/raw/zfin/uniprot.txt" + }, + "schema_version": "1.0", + "software": { + "dcgo_pipeline_version": "0.1.0", + "dependency_lock": { + "available": true, + "path": "uv.lock", + "sha256": "57e605b4882c7d51b2cd9000afa999c1c7ab56177e5c11a50d8d36b32bfedc9c", + "size_bytes": 386547 + }, + "executable": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro/.venv/bin/python3", + "platform": "Linux-6.8.0-100-generic-x86_64-with-glibc2.39", + "python_implementation": "CPython", + "python_version": "3.12.3" + }, + "started_at": "2026-10-07T09:44:59.531566+00:00", + "status": "completed", + "summary": { + "bh_threshold_pvalue": { + "single": 3.1457048492334887e-07, + "supra": 1.7228151621835345e-07 + }, + "domains": 88522, + "ontology": "go", + "propagated_annotations": 152781, + "proteins": 18507, + "runtime_seconds": 21.1, + "significant_associations": 28282, + "terms": 16040, + "tests": 1419892880, + "tests_evaluated": 2694075 + }, + "working_directory": "/home/danbolser/Build/dcGO-2.0-wt/t0-interpro" +} diff --git a/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_metrics.tsv b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_metrics.tsv new file mode 100644 index 0000000..4387974 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_metrics.tsv @@ -0,0 +1,136 @@ +aspect min_ic method n_eval_proteins f_max f_max_tau s_min s_min_tau auprc coverage_at_fmax coverage_any f_max_ci_lo f_max_ci_hi auprc_ci_lo auprc_ci_hi +BP 0.0 single 280 0.23118805206728577 0.09989841446510787 85.57080435080275 0.35 0.10824296864062724 0.5428571428571428 0.5642857142857143 0.20248823591526396 0.263582668338866 0.08597539737953488 0.13546732563828412 +BP 0.0 single__p 280 0.23043980477579162 0.14 85.30718823098844 0.27 0.11105322228501563 0.5428571428571428 0.5642857142857143 0.20366623850826726 0.26319474686764527 0.08865583896736724 0.13886476804206638 +BP 0.0 supra 280 0.23222735729077865 0.06715348172452834 85.48273360717454 0.32 0.10958281049957286 0.5428571428571428 0.5642857142857143 0.20364519776021783 0.26581491503312066 0.08709713457809332 0.13706780961828052 +BP 0.0 supra__p 280 0.23314481519350078 0.08 85.05635127005289 0.28 0.11166213455771308 0.5428571428571428 0.5642857142857143 0.20510398137984495 0.2669880913295213 0.08913171024273422 0.13972086181190826 +BP 0.0 supra_input 280 0.24634532325070788 0.048014609166444455 85.71529807782333 0.23 0.12047520165480738 0.5642857142857143 0.5714285714285714 0.21853517850223117 0.2799587908128485 0.09619462845957227 0.14913127571414533 +BP 0.0 supra_input__p 280 0.25097362892057856 0.054446696960895956 85.45812371052955 0.25 0.12324309838759742 0.5678571428571428 0.5714285714285714 0.22156794623998824 0.2828641096658727 0.0976600842929239 0.15158326189858556 +BP 0.0 supra_relative 280 0.19000131037077253 0.004614477378065632 87.27247422125312 0.25095821213628483 0.059615458665163386 0.4714285714285714 0.475 0.16354359018463843 0.22073939176914986 0.04361534443935364 0.07930017421023564 +BP 0.0 supra_relative__p 280 0.190192678766517 0.004805935671786969 87.09983762568847 0.28 0.06204732695591705 0.4714285714285714 0.475 0.16380858617796165 0.2215053036949405 0.04538186461655342 0.08248351083544565 +BP 0.0 supra_output 280 0.2362869730289279 0.05 85.75972421116703 0.14 0.11348716123155862 0.5642857142857143 0.5642857142857143 0.2080616064073026 0.2678002284942588 0.09040147817397845 0.13941027976249498 +BP 0.0 supra_input_relative 280 0.21619341675931053 0.0014585005278235255 86.78609791754069 0.17 0.09082683008932148 0.5071428571428571 0.5071428571428571 0.1909176944099623 0.24762085491772404 0.0702696316972305 0.11438287879818289 +BP 0.0 supra_input_relative__p 280 0.21575661563419163 0.021558235487832728 86.62647138437231 0.14 0.0926071611039176 0.5 0.5071428571428571 0.19119866165130522 0.2470570316494451 0.07219648367453192 0.11713627358819687 +BP 0.0 supra_input_output 280 0.2539255203757175 0.04 85.24508806765817 0.15 0.13038368470223805 0.5714285714285714 0.5714285714285714 0.22578235640989108 0.2845595855935583 0.1048452795552865 0.15892182804784777 +BP 0.0 supra_relative_output 280 0.19018741404788822 0.007094603778977181 87.40710520333587 0.15 0.06923176688783254 0.475 0.475 0.1630493990639586 0.22190662684349538 0.050996933552749764 0.08986880339914391 +BP 0.0 full 280 0.22172328957619325 0.0200742174151682 86.28793536260744 0.11625170921940697 0.09734065527721904 0.5071428571428571 0.5071428571428571 0.19339213914822456 0.25152417096797103 0.07632451108819858 0.1219021593325882 +BP 0.0 naive 280 0.2448261022375888 0.246047504670403 86.76022959545665 0.246047504670403 0.15353860455094878 1.0 1.0 0.2302057790133413 0.2618584399290958 0.13950430658270693 0.16716170719136111 +BP 2.0 single 280 0.16961760380902954 0.1 81.25656397986037 0.39 0.06849099189384009 0.5321428571428571 0.5642857142857143 0.13933650111507442 0.20486429734475664 0.04841074579746624 0.09514685924666551 +BP 2.0 single__p 280 0.1696272628265281 0.05227939123254674 81.12590993229685 0.27 0.06948750673263362 0.5392857142857143 0.5642857142857143 0.14169690200496227 0.20430285259080397 0.04917639029572054 0.09608316721336094 +BP 2.0 supra 280 0.17306362498704847 0.06119420622202558 81.1555898951714 0.32 0.07174461568612162 0.5392857142857143 0.5642857142857143 0.1422204980728778 0.2079843304696377 0.05065691785207953 0.10040167996200657 +BP 2.0 supra__p 280 0.1748213996237048 0.08 80.8627846676829 0.28 0.07273939352333356 0.5392857142857143 0.5642857142857143 0.1437033322016041 0.21173750636442681 0.05196484539491627 0.100922627234309 +BP 2.0 supra_input 280 0.19298127282714725 0.02740688798013765 81.53110640044713 0.27 0.07368941487309685 0.5678571428571428 0.5714285714285714 0.1654146480646142 0.22858808806038985 0.05337072820347833 0.10070095206649407 +BP 2.0 supra_input__p 280 0.1947859410897507 0.05 81.33035965340346 0.26 0.07527907350166443 0.5678571428571428 0.5714285714285714 0.16566904700351018 0.2288577381634202 0.05442362116637424 0.10199767542477596 +BP 2.0 supra_relative 280 0.13151832103319872 0.0 82.50315663270413 0.2552626080848716 0.03140843262490831 0.475 0.475 0.10391443625297273 0.16064567778635863 0.018819673041339007 0.04812369013891398 +BP 2.0 supra_relative__p 280 0.13151832103319872 0.0 82.47029831840788 0.28 0.03314557078349831 0.475 0.475 0.10390529943613562 0.16024266183029193 0.019820913352854505 0.049942832996360896 +BP 2.0 supra_output 280 0.17754422727003788 0.03679362677169721 81.70776479878013 0.14 0.06543747274327048 0.5642857142857143 0.5642857142857143 0.14845505178922885 0.21045962956991218 0.04752198080411408 0.08974805302684151 +BP 2.0 supra_input_relative 280 0.16090319819984983 0.0008052536629037432 82.33233046972069 0.17 0.05163132588355115 0.5071428571428571 0.5071428571428571 0.13336488798057533 0.19263308361862638 0.03516208099393428 0.07224211648187773 +BP 2.0 supra_input_relative__p 280 0.16071415169223746 0.0007340701711616174 82.32965971464662 0.14 0.052292648332118316 0.5071428571428571 0.5071428571428571 0.13322010775364243 0.19235924058843845 0.03586388179954004 0.07297259838103898 +BP 2.0 supra_input_output 280 0.19442959339214827 0.028366288037686878 81.31914272585662 0.15 0.07889988553924517 0.5678571428571428 0.5714285714285714 0.16467351251424509 0.2280975230156303 0.057992225624703034 0.10542067665507338 +BP 2.0 supra_relative_output 280 0.1334460693994402 0.001696834680185576 82.88450617543926 0.15 0.031228862219874462 0.475 0.475 0.10491691382574635 0.16207738821928327 0.01833014341859493 0.04783006141845034 +BP 2.0 full 280 0.1586593787825127 0.005318300757607169 82.19321476922717 0.11855140938374391 0.050503835419353965 0.5071428571428571 0.5071428571428571 0.13264998447191664 0.19086467056751286 0.034651846537463594 0.07065261802608962 +BP 2.0 naive 280 0.13750683947562561 0.12271470509741127 83.78417637278721 0.23094528956498533 0.05141761288677794 1.0 1.0 0.12769992627532048 0.14838901022214973 0.046169789904482575 0.05811558518249507 +BP 4.0 single 276 0.11708071265233112 0.03678762602680692 66.54898174564893 0.39 0.031958241999146354 0.5471014492753623 0.5688405797101449 0.09252766634292933 0.15408927758172655 0.019171364554772103 0.05321159359061663 +BP 4.0 single__p 276 0.11796134912667214 0.05 66.42112394918524 0.27 0.032594797066663024 0.5434782608695652 0.5688405797101449 0.09357016537029954 0.1513301026079041 0.019857289279069396 0.052633171597645335 +BP 4.0 supra 276 0.12225491312943826 0.06937479845266326 66.481454547869 0.32 0.036024747250942145 0.5434782608695652 0.5688405797101449 0.09599094408485975 0.15796345357587044 0.021410469899348385 0.05936109313111709 +BP 4.0 supra__p 276 0.12303010701263231 0.08 66.22129415126733 0.28 0.03586060938222534 0.5434782608695652 0.5688405797101449 0.09614223404993419 0.15884953170178095 0.021764031483692523 0.05848175402082753 +BP 4.0 supra_input 276 0.1415351523410215 0.011250748370743965 66.8205929051255 0.25 0.03704168130007926 0.572463768115942 0.572463768115942 0.11379524285431826 0.1747841436551425 0.024438597816897653 0.0559354951349084 +BP 4.0 supra_input__p 276 0.13917271927670194 0.017569237147694487 66.73693517291588 0.26 0.037470532974239766 0.5688405797101449 0.572463768115942 0.11310515440241788 0.17342122781203892 0.024358256081608427 0.05615929174366952 +BP 4.0 supra_relative 276 0.08373433234879325 0.0 67.23239263939112 0.25 0.01289216911036889 0.4601449275362319 0.4601449275362319 0.0611278450270288 0.11301448049000076 0.006362867908422226 0.023298237001336743 +BP 4.0 supra_relative__p 276 0.08373433234879325 0.0 67.23633361395518 0.28 0.013445116801122519 0.4601449275362319 0.4601449275362319 0.06121403029806832 0.11290698669625658 0.006727327193363489 0.02437307760700002 +BP 4.0 supra_output 276 0.12170029235321507 0.015571128617361969 67.10264036962651 0.15 0.028871544568472095 0.5688405797101449 0.5688405797101449 0.09437348095103076 0.15373567928829107 0.017478080106398956 0.045899663836068964 +BP 4.0 supra_input_relative 276 0.0966619842666126 0.0007794381399490565 67.29261462844985 0.16984127210668754 0.020290622359582438 0.4963768115942029 0.4963768115942029 0.07397096745362888 0.12540071896751992 0.011889611535212912 0.0354681157146903 +BP 4.0 supra_input_relative__p 276 0.0965830173233333 0.0007419381017406207 67.27191380469606 0.14 0.020560648744801083 0.4963768115942029 0.4963768115942029 0.07372264956600293 0.12478517088372153 0.012144241355099348 0.03563536677583882 +BP 4.0 supra_input_output 276 0.13624185783526765 0.012047035630113471 66.86671352647461 0.12675168821234928 0.03637106977533471 0.5688405797101449 0.572463768115942 0.11150702636719237 0.16877025559854517 0.023412244768385028 0.05429258944206879 +BP 4.0 supra_relative_output 276 0.08724122315327773 0.001800685947732063 67.56689436785595 0.08491738230215237 0.012685259590633123 0.4528985507246377 0.4601449275362319 0.06320516723602043 0.11367817253493694 0.00585431560935667 0.02416094936157986 +BP 4.0 full 276 0.09498655717961088 0.00022387914847785188 67.27631520123894 0.11827194318903653 0.021022876794778367 0.4963768115942029 0.4963768115942029 0.07271659063070599 0.1230785233855846 0.011460477355858895 0.03681310800665676 +BP 4.0 naive 276 0.046529709699944764 0.04859781158259942 68.52924755318239 0.06228983186549239 0.013590508225684453 1.0 1.0 0.04086373861665196 0.05360161136150896 0.011861404048472845 0.015505245077537054 +MF 0.0 single 392 0.35184878665395036 0.17716892876838156 12.425974476934792 0.75 0.1773342076968634 0.5739795918367347 0.6275510204081632 0.3160784904856867 0.3886656111013574 0.14678623537592286 0.20803988076896743 +MF 0.0 single__p 392 0.36899778114812976 0.15 12.472893972280206 0.58 0.19042270334855405 0.5943877551020408 0.6275510204081632 0.33512905693266504 0.40422831520815367 0.15854105454926207 0.22245198148513107 +MF 0.0 supra 392 0.3504597362612787 0.18 12.437509146704457 0.75 0.1776717251689381 0.5739795918367347 0.6275510204081632 0.3138379298883269 0.38707531325384603 0.14728060225029388 0.20943648628033518 +MF 0.0 supra__p 392 0.3693355074347025 0.17 12.443914049388997 0.59 0.19067657985508568 0.5867346938775511 0.6275510204081632 0.33406049932321163 0.40494983270094664 0.15923762819347062 0.22280370695943244 +MF 0.0 supra_input 392 0.36737678045727645 0.08518363139817497 12.592959984743631 0.44 0.17784905555489533 0.6147959183673469 0.6377551020408163 0.3339676410510388 0.403472036129526 0.14659533966073457 0.20968572806898933 +MF 0.0 supra_input__p 392 0.3652744653604423 0.07088187919187379 12.54011063364604 0.34654431806202607 0.1818379853467528 0.6198979591836735 0.6377551020408163 0.3314300826083878 0.39925308051687763 0.15075139338631968 0.21381503043780778 +MF 0.0 supra_relative 392 0.34065815985623116 0.08412158499526236 13.225536780928747 0.61 0.16111075915801953 0.5382653061224489 0.5739795918367347 0.30941654466955515 0.37512300574475094 0.1352887805686354 0.1914181696728179 +MF 0.0 supra_relative__p 392 0.3407532275340199 0.11 13.14685136791067 0.64 0.17112504627767916 0.5306122448979592 0.5739795918367347 0.31111875347981743 0.3758441194657525 0.14415352201693804 0.20216424704640992 +MF 0.0 supra_output 392 0.3576744070429309 0.06704141095279345 12.668942827545463 0.29 0.18648654900225867 0.6122448979591837 0.6275510204081632 0.3246382122764533 0.391712170668104 0.15412965240016138 0.2195949365522949 +MF 0.0 supra_input_relative 392 0.36420255420510744 0.1069149455335412 13.207932809335723 0.37 0.197624731265526 0.5739795918367347 0.6147959183673469 0.3303051142243926 0.3997188252228665 0.16551836071709758 0.2315031465144115 +MF 0.0 supra_input_relative__p 392 0.3724001925230976 0.13 13.173713028804087 0.36 0.207494819717412 0.5739795918367347 0.6147959183673469 0.336944842165138 0.4096139957751124 0.17436161054535432 0.2434126329073035 +MF 0.0 supra_input_output 392 0.36763063939325596 0.07 12.498427536784668 0.29 0.19671267213108037 0.6173469387755102 0.6377551020408163 0.3349572275107421 0.4022403394389338 0.1643013031401914 0.23045633797870085 +MF 0.0 supra_relative_output 392 0.34592202257534416 0.13 13.389972818345514 0.24 0.19128826041227334 0.49489795918367346 0.5739795918367347 0.31575254549614706 0.3863728359366606 0.16052913547765507 0.22753867400009786 +MF 0.0 full 392 0.3839210840186379 0.07 13.052281606495331 0.17 0.2187751255190095 0.5790816326530612 0.6147959183673469 0.34861311355433866 0.4212503826725544 0.18519283587632046 0.25639130951561623 +MF 0.0 naive 392 0.7203660688206672 0.2616402455297571 13.368464805535808 0.15066399786495863 0.4203418528418113 1.0 1.0 0.6786283190432112 0.759990617491232 0.3801695137089621 0.4619987587415928 +MF 2.0 single 151 0.3427980848534931 0.47 27.351067358913948 0.47 0.1890117765436093 0.5033112582781457 0.6754966887417219 0.2815506577533877 0.4100642070201364 0.1362923624646703 0.24603741590211692 +MF 2.0 single__p 151 0.3429562216890259 0.41 27.465772759136488 0.41 0.19868727316428175 0.5099337748344371 0.6754966887417219 0.2785279389754373 0.41098658590134574 0.14303239221757372 0.25649624009539534 +MF 2.0 supra 151 0.3420345280007133 0.43 27.538143899891136 0.47 0.19920622762740417 0.5033112582781457 0.6754966887417219 0.2801212889873945 0.4095351319513372 0.144377956350148 0.26103407190413547 +MF 2.0 supra__p 151 0.34379498976859074 0.33 27.523988470370433 0.44 0.21213019088175558 0.5099337748344371 0.6754966887417219 0.2796011118051042 0.41224526352734914 0.15439374267045805 0.2767105247726323 +MF 2.0 supra_input 151 0.34634258566256804 0.12 27.96312698476403 0.26 0.21902353881255027 0.5827814569536424 0.695364238410596 0.2866847001683329 0.4108105204292498 0.1626638815559968 0.2806404505435021 +MF 2.0 supra_input__p 151 0.34437577713869405 0.12 27.708421780343006 0.25 0.22278841510408542 0.5827814569536424 0.695364238410596 0.28729111914240046 0.41079916665667426 0.1664465584256112 0.28503859476295224 +MF 2.0 supra_relative 151 0.2668190504161003 0.08 31.60531305625909 0.33 0.10136836912622044 0.5695364238410596 0.609271523178808 0.22055990780783094 0.3291781703841761 0.06570640164413603 0.1490704082757768 +MF 2.0 supra_relative__p 151 0.26505872059355545 0.013937133494698222 31.375079101748472 0.31 0.10533518223939631 0.5761589403973509 0.609271523178808 0.21802457820661286 0.325126063418273 0.06858792085958394 0.15623878573555888 +MF 2.0 supra_output 151 0.3283248203031117 0.08 28.62484690872823 0.15 0.19624132476947065 0.609271523178808 0.6754966887417219 0.26897335233166475 0.391723093767253 0.14214972511205248 0.2570509230120993 +MF 2.0 supra_input_relative 151 0.29759678021539177 0.08 30.612248830271636 0.18970843340853166 0.16166630451106856 0.6291390728476821 0.6622516556291391 0.24694803047171163 0.358294969863535 0.11522541122749602 0.21799617624013568 +MF 2.0 supra_input_relative__p 151 0.2995118480414281 0.05521095168554714 30.45050834627663 0.1550445696485765 0.16391093049181252 0.6291390728476821 0.6622516556291391 0.24628541815203736 0.3572004101653959 0.11578112055471398 0.2216307356622887 +MF 2.0 supra_input_output 151 0.3387240186494688 0.09 27.953967945731947 0.15 0.21495445913612463 0.5894039735099338 0.695364238410596 0.2815071349820404 0.4009015915244533 0.15827161052752392 0.2777703808707377 +MF 2.0 supra_relative_output 151 0.2833439796045256 0.01 31.519974620078223 0.13 0.11782829271639662 0.5960264900662252 0.609271523178808 0.23168009345016985 0.33918882214354235 0.07786297440052867 0.17058539338378229 +MF 2.0 full 151 0.3071719115131589 0.03425040187938083 30.34641715505658 0.11 0.15972442628853586 0.6357615894039735 0.6622516556291391 0.25476354158417736 0.36209995310851495 0.11283230639683657 0.2148637099486271 +MF 2.0 naive 151 0.0747337284956894 0.04595356285028022 34.29281021115755 0.1096215639178009 0.026582066823136727 1.0 1.0 0.06424740220720905 0.08886629565678979 0.021685496255507277 0.03259954121427821 +MF 4.0 single 145 0.3167811928767344 0.47 26.6884837205874 0.47 0.18328900688605118 0.496551724137931 0.6689655172413793 0.2520063338844916 0.389882426786158 0.1264390777432008 0.24827536431537184 +MF 4.0 single__p 145 0.32000408368648553 0.41 26.768418501954827 0.41 0.19379843973698957 0.496551724137931 0.6689655172413793 0.25350865620097635 0.39425416656785667 0.13527240444489763 0.2576122960141982 +MF 4.0 supra 145 0.31620643824473516 0.43 26.846701049744965 0.47 0.18565479835779616 0.503448275862069 0.6689655172413793 0.25213776091067475 0.39006652911861084 0.12869402986954742 0.2510844769644634 +MF 4.0 supra__p 145 0.3208673876004453 0.33 26.818446793274333 0.44 0.19798764463326496 0.503448275862069 0.6689655172413793 0.25418139075319296 0.3944816766514441 0.13974838991625227 0.26480371979665107 +MF 4.0 supra_input 145 0.33381836824773203 0.12466307578617696 27.173154552744656 0.26 0.20050425945779105 0.5586206896551724 0.6896551724137931 0.26546055399044816 0.4021494869952328 0.14263073879302815 0.2673582828982574 +MF 4.0 supra_input__p 145 0.32956322840825886 0.12 26.95407802348229 0.25 0.20613935938332584 0.5586206896551724 0.6896551724137931 0.26370307975637103 0.39814858407824544 0.146484703341934 0.2718454149943285 +MF 4.0 supra_relative 145 0.24360821271781707 0.08 30.274435658776273 0.61 0.0866384924224616 0.5448275862068965 0.5793103448275863 0.19327066519268415 0.3034641355911909 0.05182454042423647 0.13252821694595288 +MF 4.0 supra_relative__p 145 0.23915965460269903 0.013663116899985225 30.15451615413676 0.48 0.0898588368153704 0.5517241379310345 0.5793103448275863 0.18882699085213714 0.2998115702291197 0.054482809402245994 0.13661401277759216 +MF 4.0 supra_output 145 0.3138096336503749 0.08 27.680833199091065 0.13 0.18058233744285324 0.5793103448275863 0.6689655172413793 0.24892353350640053 0.3817988270487252 0.1254764459301561 0.24787702701394448 +MF 4.0 supra_input_relative 145 0.26976449185413537 0.08 29.69936001522525 0.19 0.1260701210247072 0.6068965517241379 0.6551724137931034 0.21588378402957403 0.33372734757879324 0.0824622312400202 0.17749221671501714 +MF 4.0 supra_input_relative__p 145 0.2682315161130025 0.06813144646794143 29.572324817990477 0.1542971336325714 0.12690859998235904 0.6 0.6551724137931034 0.21389226263222602 0.3296308601155988 0.08302310869529547 0.1799676465107686 +MF 4.0 supra_input_output 145 0.3223521734024969 0.09 27.111750743408077 0.15 0.19524934449529005 0.5655172413793104 0.6896551724137931 0.25811970892891584 0.39011236288552265 0.1383509409588113 0.26209167986682064 +MF 4.0 supra_relative_output 145 0.25331622046992747 0.003975484111015619 30.435556303009694 0.13 0.09342925429690763 0.5724137931034483 0.5793103448275863 0.20044041137529617 0.31274340502985704 0.057288835754992176 0.14142895841265732 +MF 4.0 full 145 0.2768034568720887 0.03286668011344428 29.473649204028547 0.11 0.12957851381254135 0.6068965517241379 0.6551724137931034 0.21967475163003897 0.3370643187666539 0.08326049058374722 0.18296360681208046 +MF 4.0 naive 145 0.07527512705860413 0.05290899386175607 32.4047016544147 0.12447291166266346 0.023373921180975445 1.0 1.0 0.05851470520083728 0.09797897072812506 0.01849452039724683 0.028824412008815914 +CC 0.0 single 422 0.37939213745252304 0.01718796417807132 21.574399107234623 0.19 0.22098897907182266 0.514218009478673 0.5260663507109005 0.3510263117041664 0.40896100122205364 0.19250690885136354 0.2528444730867814 +CC 0.0 single__p 422 0.3849667758610253 0.02 21.48240502554639 0.17 0.23107437449471982 0.514218009478673 0.5260663507109005 0.3546864706023537 0.4158275358771159 0.20170970296383262 0.26246357750081745 +CC 0.0 supra 422 0.38066936485600744 0.02024917060432352 21.59301574510195 0.18 0.22136146113555719 0.504739336492891 0.5260663507109005 0.35191948032902287 0.40880259742084085 0.19188383654433722 0.25292851571143643 +CC 0.0 supra__p 422 0.38030312193514554 0.02 21.489292897420444 0.17230075402994743 0.23038272993567185 0.5165876777251185 0.5260663507109005 0.3531648361155315 0.41020284998919265 0.20120734337060017 0.26205939888492996 +CC 0.0 supra_input 422 0.37210747432733937 0.004050468792212149 22.03927784339081 0.14 0.20675314042971912 0.518957345971564 0.523696682464455 0.3470355399317577 0.4021851308794834 0.17974002436735417 0.23478780741540978 +CC 0.0 supra_input__p 422 0.3744675597865901 0.01 22.090793937124 0.15 0.21284708367462715 0.5165876777251185 0.523696682464455 0.34850201736835773 0.4042213238417158 0.18554744818272786 0.24102054587962413 +CC 0.0 supra_relative 422 0.35128104782911407 0.010584885172989004 21.82104600272686 0.27 0.17984949000101644 0.45734597156398105 0.466824644549763 0.32020866215053395 0.3821809032542541 0.15370730394453747 0.20664481804623644 +CC 0.0 supra_relative__p 422 0.35279494543315154 0.01656610049576198 21.796061979282054 0.29 0.18208169489824305 0.45734597156398105 0.466824644549763 0.31960061625778285 0.38176168606490757 0.1556578842255527 0.20999953516642955 +CC 0.0 supra_output 422 0.3892766795082576 0.017733046241645255 21.604449470313725 0.08184804960912312 0.2513872673562999 0.518957345971564 0.5260663507109005 0.3595550944638023 0.4198510415063025 0.221075852474153 0.28351034990174917 +CC 0.0 supra_input_relative 422 0.37429348759444325 0.0025277787512022654 21.90836221676096 0.19 0.2056131265692438 0.5023696682464455 0.514218009478673 0.3437926485418262 0.4034148544399681 0.17672467693947957 0.23499048799918928 +CC 0.0 supra_input_relative__p 422 0.37661711244496815 0.007488904844379206 21.842954093829583 0.17 0.21064419545428117 0.5 0.514218009478673 0.3443677391719537 0.4060503481960217 0.18156634208429914 0.2392470756154323 +CC 0.0 supra_input_output 422 0.38818181751479036 0.012731292007165976 21.700562850168176 0.08 0.2478064810347812 0.5213270142180095 0.523696682464455 0.3596979072235482 0.4168651004610047 0.21723081550230364 0.27890694277759737 +CC 0.0 supra_relative_output 422 0.3557514236403979 0.006353538741326298 21.698168401778076 0.11 0.2156900609998714 0.466824644549763 0.466824644549763 0.3258168331764418 0.38627048824423305 0.1862319240133203 0.2443201873236023 +CC 0.0 full 422 0.3837827299538696 0.00597060951020819 21.647647910417074 0.09 0.23613261087318776 0.514218009478673 0.514218009478673 0.35391559002540446 0.4139973874208273 0.20492438830028523 0.26743948590079847 +CC 0.0 naive 422 0.5918263432357462 0.2848678943154524 19.7265485923231 0.2848678943154524 0.5411543162750881 1.0 1.0 0.5757709479789495 0.6120033525103181 0.5197495296223517 0.5649137884123872 +CC 2.0 single 288 0.20418705801361683 0.00376532540329356 24.361987976228598 0.24 0.057550814323517045 0.5138888888888888 0.5243055555555556 0.1667984333946201 0.24180274016307995 0.03683829044097003 0.08496778852842804 +CC 2.0 single__p 288 0.2041998293060549 0.0025851608181386063 24.24887872268412 0.1778576746805374 0.058901504579969496 0.5138888888888888 0.5243055555555556 0.16666927829051956 0.2414822992516005 0.03852588499907847 0.08666841854486652 +CC 2.0 supra 288 0.20315757312646796 0.0011466507008032524 24.2276573544052 0.14365111289642082 0.05664095581953725 0.5138888888888888 0.5277777777777778 0.16624323170970295 0.24046858834791562 0.03701213326215563 0.08456422312901898 +CC 2.0 supra__p 288 0.20489234316843735 0.0010655360924254 24.209078043953898 0.18 0.057588253186316926 0.5138888888888888 0.5277777777777778 0.16720685361438703 0.24267471974856875 0.03894104226264106 0.08367503259877138 +CC 2.0 supra_input 288 0.19227286600693066 6.084350222272058e-05 24.57329808442205 0.14 0.055718082535733425 0.5243055555555556 0.5243055555555556 0.15732821839269723 0.22797141331515652 0.038254592104967 0.08155668727134552 +CC 2.0 supra_input__p 288 0.18438669162644808 1.7268932979630467e-05 24.63942050282569 0.12 0.05586063625050221 0.5173611111111112 0.5243055555555556 0.15297113119934613 0.2187982055670667 0.03888708235047651 0.08170623226406852 +CC 2.0 supra_relative 288 0.17406137853913475 0.0 24.853174300226136 0.25 0.041937928499600215 0.4548611111111111 0.4548611111111111 0.14155698005177106 0.20840843927104435 0.02672269814067963 0.0634861512406246 +CC 2.0 supra_relative__p 288 0.17406137853913475 0.0 24.837489672839194 0.26 0.04132350825332682 0.4548611111111111 0.4548611111111111 0.14184902345434153 0.20840843927104435 0.026481757965412318 0.0626321794074949 +CC 2.0 supra_output 288 0.20872491324071593 0.0012358841271541334 24.36001899437003 0.10499999999999997 0.05933234754454002 0.5208333333333334 0.5277777777777778 0.17116869608165736 0.24600714009919306 0.03974235196674098 0.08463018097072864 +CC 2.0 supra_input_relative 288 0.20014085427621897 0.0014291514996182756 24.540161680359912 0.13 0.0505428753460521 0.4791666666666667 0.5138888888888888 0.16583843167322834 0.23847153010589422 0.03350741761429259 0.07354242497209669 +CC 2.0 supra_input_relative__p 288 0.20161175748869592 0.0020317301017947534 24.476149887013975 0.18 0.05064769680767862 0.4756944444444444 0.5138888888888888 0.16699110784995544 0.23989912349810702 0.03363328457120633 0.07232710082501227 +CC 2.0 supra_input_output 288 0.21260013711238593 0.0011977705194741986 24.466425564304668 0.08250000000000002 0.0631971384788975 0.5243055555555556 0.5243055555555556 0.17504658678253265 0.24835131652623949 0.04392410393580273 0.08856674632742323 +CC 2.0 supra_relative_output 288 0.1838313800517334 0.002309104381263972 24.785341328238488 0.11098916268723019 0.04285708001859051 0.4340277777777778 0.4548611111111111 0.1511614758376443 0.22269059498843063 0.02811717588558786 0.06242517292330145 +CC 2.0 full 288 0.20841152876548033 0.0025477051258693995 24.588345470866273 0.09133844359913025 0.05106006325346844 0.5034722222222222 0.5138888888888888 0.17338846798206706 0.24448414833383475 0.035342675899770123 0.07424582220679013 +CC 2.0 naive 288 0.27592110996968483 0.10965999466239659 24.18824745800539 0.20930237523352016 0.14642831318147828 1.0 1.0 0.25432382387365987 0.3046949033176323 0.12979919201683468 0.16468359482236128 +CC 4.0 single 202 0.1310356728406018 0.002914903657190658 24.25410192227604 0.14 0.03256236248146611 0.45544554455445546 0.5099009900990099 0.09366868027309404 0.18275465552991635 0.014872975905898015 0.059759853677986856 +CC 4.0 single__p 202 0.13526073718318715 0.06 24.177474879482936 0.17619820570548286 0.03262671048007864 0.35148514851485146 0.5099009900990099 0.09668641323470327 0.1881791450775185 0.014993892889590927 0.05895536278978009 +CC 4.0 supra 202 0.13060876370294153 0.051264025211109475 24.226294136620158 0.1417303749005237 0.03129067564188612 0.32673267326732675 0.5198019801980198 0.09487084911106837 0.1841411836745516 0.014570101145346227 0.057548220595773275 +CC 4.0 supra__p 202 0.13213981302374483 0.07 24.265297540216118 0.18 0.032256667103494015 0.31683168316831684 0.5198019801980198 0.09391738738138164 0.18416772998056294 0.014859694456716175 0.05885835237996996 +CC 4.0 supra_input 202 0.1294870590779316 0.027018288150707056 24.64521622435574 0.14 0.03239089050618442 0.3613861386138614 0.5198019801980198 0.0916591057820535 0.17923459304346517 0.015268951596837362 0.0597018195896372 +CC 4.0 supra_input__p 202 0.1295930087369059 0.03 24.71336801890562 0.12 0.03159900324926638 0.3613861386138614 0.5198019801980198 0.09120224106166455 0.17860759004509238 0.014532261125143438 0.05829525864814854 +CC 4.0 supra_relative 202 0.06872836842386244 2.9593598308423522e-05 24.843670870928783 0.25 0.011710666183001015 0.3613861386138614 0.3712871287128713 0.03734675969553141 0.10773541307504796 0.0029055316775932825 0.02800667574577148 +CC 4.0 supra_relative__p 202 0.06872836842386244 6.284760489552718e-05 24.821089348880264 0.27 0.011590167828537709 0.3613861386138614 0.3712871287128713 0.03745477974083522 0.1087928711051437 0.0029469173526879082 0.028325080245772946 +CC 4.0 supra_output 202 0.11938378883792156 0.01955929790611426 24.455586793183404 0.059999999999999984 0.036629540798907956 0.3465346534653465 0.5198019801980198 0.08715357349041317 0.16771384443611517 0.01507834382252466 0.06505564132753046 +CC 4.0 supra_input_relative 202 0.11704074740568092 0.0 24.625727214366883 0.09 0.027946086966148243 0.44554455445544555 0.44554455445544555 0.08114081832877335 0.16247290065521394 0.01197278101596126 0.05303368655154523 +CC 4.0 supra_input_relative__p 202 0.11704074740568092 0.0 24.759214888617127 0.14 0.02689277925706172 0.44554455445544555 0.44554455445544555 0.08088334512827178 0.16309861537780748 0.011372849377635687 0.05093408273483661 +CC 4.0 supra_input_output 202 0.12533124073004 0.016449130429118147 24.538699633643553 0.0675 0.035515295230318725 0.3712871287128713 0.5198019801980198 0.0881954836000088 0.1749891708670859 0.015387818581733723 0.0638297843499772 +CC 4.0 supra_relative_output 202 0.06701519940569019 0.0005330076270346414 24.90034551919701 0.11 0.013915926101555409 0.3415841584158416 0.3712871287128713 0.03686041425469422 0.10473204686362447 0.00302313082928578 0.03408255089994772 +CC 4.0 full 202 0.12338995602996741 0.0013531997846224217 24.790986405023645 0.10043225463152751 0.027627268056174378 0.4207920792079208 0.44554455445544555 0.08265078988693852 0.16646252838858716 0.011560154630481836 0.05045488741811373 +CC 4.0 naive 202 0.10149307194327688 0.04063624232719509 25.40016779905635 0.12308513477448625 0.04397862691300466 1.0 1.0 0.09151555159838287 0.11784109368372173 0.03816446461587501 0.050777134757557234 diff --git a/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_paired_bootstrap.tsv b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_paired_bootstrap.tsv new file mode 100644 index 0000000..351220d --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_paired_bootstrap.tsv @@ -0,0 +1,415 @@ +aspect min_ic n_eval_proteins metric method_a method_b observed_a observed_b observed_diff diff_mean diff_ci_lo diff_ci_hi p_value significant n_replicates +BP 0.0 280 f_max supra single 0.23222735729077865 0.23118805206728577 0.0010393052234928812 0.0017462775010556725 -0.004395993376781115 0.007873358106891496 0.528 False 1000 +BP 0.0 280 auprc supra single 0.10958281049957286 0.10824296864062724 0.0013398418589456174 0.001353733160221323 -0.0012524656992081054 0.004335564122549573 0.368 False 1000 +BP 0.0 280 f_max supra_input supra 0.24634532325070788 0.23222735729077865 0.01411796595992923 0.014142555021873978 0.0004818430446284094 0.029778815524542743 0.048 True 1000 +BP 0.0 280 auprc supra_input supra 0.12047520165480738 0.10958281049957286 0.010892391155234521 0.010985629232382742 0.0017954310797491319 0.020897732032896323 0.012 True 1000 +BP 0.0 280 f_max supra_relative supra 0.19000131037077253 0.23222735729077865 -0.042226046920006116 -0.04234452404488921 -0.06356126206978097 -0.021695394902080715 0.0 True 1000 +BP 0.0 280 auprc supra_relative supra 0.059615458665163386 0.10958281049957286 -0.04996735183440947 -0.05010188962832567 -0.06582677684813185 -0.035968989282143454 0.0 True 1000 +BP 0.0 280 f_max supra_output supra 0.2362869730289279 0.23222735729077865 0.004059615738149253 0.0036116338335957206 -0.006952828513607854 0.015390525071562145 0.514 False 1000 +BP 0.0 280 auprc supra_output supra 0.11348716123155862 0.10958281049957286 0.0039043507319857645 0.0038972600413686278 -0.0035896606046693404 0.011137666912637722 0.31 False 1000 +BP 0.0 280 f_max supra_input_relative supra_input 0.21619341675931053 0.24634532325070788 -0.030151906491397346 -0.029724844622051286 -0.047787845076163085 -0.01168340576692331 0.0 True 1000 +BP 0.0 280 auprc supra_input_relative supra_input 0.09082683008932148 0.12047520165480738 -0.0296483715654859 -0.029889012839408508 -0.04290226289045497 -0.01820912642017843 0.0 True 1000 +BP 0.0 280 f_max supra_input_relative supra_relative 0.21619341675931053 0.19000131037077253 0.026192106388538 0.026762234444711912 0.009084807701926444 0.04296134791286667 0.0 True 1000 +BP 0.0 280 auprc supra_input_relative supra_relative 0.09082683008932148 0.059615458665163386 0.031211371424158092 0.0311985060212999 0.021391061032329812 0.042367890204754594 0.0 True 1000 +BP 0.0 280 f_max supra_input_output supra_input 0.2539255203757175 0.24634532325070788 0.007580197125009597 0.00621033244858066 -0.0005485341945221723 0.013414580789950779 0.072 False 1000 +BP 0.0 280 auprc supra_input_output supra_input 0.13038368470223805 0.12047520165480738 0.009908483047430666 0.00986646542167737 0.005729537682577928 0.014285348706624817 0.0 True 1000 +BP 0.0 280 f_max supra_input_output supra_output 0.2539255203757175 0.2362869730289279 0.017638547346789574 0.016741253636858917 0.006523843760383151 0.029004264144002864 0.0 True 1000 +BP 0.0 280 auprc supra_input_output supra_output 0.13038368470223805 0.11348716123155862 0.016896523470679423 0.016954834612691486 0.009760152519277872 0.02577991917119618 0.0 True 1000 +BP 0.0 280 f_max supra_relative_output supra_relative 0.19018741404788822 0.19000131037077253 0.00018610367711568987 0.00011466732673808217 -0.006426679243448687 0.0065247399617348765 1.0 False 1000 +BP 0.0 280 auprc supra_relative_output supra_relative 0.06923176688783254 0.059615458665163386 0.009616308222669158 0.009551327267638157 0.004383563095123749 0.01502698559572948 0.0 True 1000 +BP 0.0 280 f_max supra_relative_output supra_output 0.19018741404788822 0.2362869730289279 -0.04609955898103968 -0.045841490551746855 -0.06647926924717805 -0.024918095550445257 0.0 True 1000 +BP 0.0 280 auprc supra_relative_output supra_output 0.06923176688783254 0.11348716123155862 -0.04425539434372608 -0.04444782240205615 -0.06003486775409885 -0.0300110523130266 0.0 True 1000 +BP 0.0 280 f_max full supra_input_relative 0.22172328957619325 0.21619341675931053 0.005529872816882719 0.004309002014288395 -0.0023992979224042015 0.012306916094512996 0.258 False 1000 +BP 0.0 280 auprc full supra_input_relative 0.09734065527721904 0.09082683008932148 0.0065138251878975595 0.0065398676212836335 0.002542375935647362 0.011010038828812175 0.0 True 1000 +BP 0.0 280 f_max full supra_input_output 0.22172328957619325 0.2539255203757175 -0.03220223079952422 -0.03162617505634355 -0.04947150927951454 -0.01616240686026998 0.0 True 1000 +BP 0.0 280 auprc full supra_input_output 0.09734065527721904 0.13038368470223805 -0.03304302942501901 -0.033215610639802245 -0.04691288272298502 -0.021177457011494576 0.0 True 1000 +BP 0.0 280 f_max full supra_relative_output 0.22172328957619325 0.19018741404788822 0.03153587552830503 0.030956569132262225 0.014940350626954252 0.047044619181247374 0.0 True 1000 +BP 0.0 280 auprc full supra_relative_output 0.09734065527721904 0.06923176688783254 0.028108888389386494 0.028187046374945377 0.01817231240693274 0.039278623888322664 0.0 True 1000 +BP 0.0 280 f_max single naive 0.23118805206728577 0.2448261022375888 -0.013638050170303034 -0.013153041325116598 -0.046583173711681504 0.019559927789948323 0.42 False 1000 +BP 0.0 280 auprc single naive 0.10824296864062724 0.15353860455094878 -0.04529563591032154 -0.044859721083766305 -0.06964373700216742 -0.018123933812459945 0.0 True 1000 +BP 0.0 280 f_max supra naive 0.23222735729077865 0.2448261022375888 -0.012598744946810153 -0.011406763824060922 -0.04502279526164959 0.021571198517350593 0.488 False 1000 +BP 0.0 280 auprc supra naive 0.10958281049957286 0.15353860455094878 -0.043955794051375924 -0.04350598792354498 -0.06849704355939075 -0.015930082321123765 0.0 True 1000 +BP 0.0 280 f_max supra_input naive 0.24634532325070788 0.2448261022375888 0.0015192210131190764 0.0027357911978130567 -0.02923810684814609 0.037654229899696925 0.874 False 1000 +BP 0.0 280 auprc supra_input naive 0.12047520165480738 0.15353860455094878 -0.0330634028961414 -0.032520358691162245 -0.0592037281562923 -0.004794480305440932 0.028 True 1000 +BP 0.0 280 f_max supra_relative naive 0.19000131037077253 0.2448261022375888 -0.05482479186681627 -0.053751287868950136 -0.08428392078330911 -0.023535247380957424 0.0 True 1000 +BP 0.0 280 auprc supra_relative naive 0.059615458665163386 0.15353860455094878 -0.0939231458857854 -0.09360787755187064 -0.11282433700951149 -0.07416159192842514 0.0 True 1000 +BP 0.0 280 f_max supra_output naive 0.2362869730289279 0.2448261022375888 -0.0085391292086609 -0.007795129990465202 -0.03792675291932563 0.023652733037312735 0.59 False 1000 +BP 0.0 280 auprc supra_output naive 0.11348716123155862 0.15353860455094878 -0.04005144331939016 -0.03960872788217636 -0.06458493396475068 -0.013110239627523937 0.0 True 1000 +BP 0.0 280 f_max supra_input_relative naive 0.21619341675931053 0.2448261022375888 -0.02863268547827827 -0.02698905342423823 -0.05712133669988209 0.0035194184951035812 0.08 False 1000 +BP 0.0 280 auprc supra_input_relative naive 0.09082683008932148 0.15353860455094878 -0.0627117744616273 -0.06240937153057075 -0.08512036514791571 -0.03916856275042083 0.0 True 1000 +BP 0.0 280 f_max supra_input_output naive 0.2539255203757175 0.2448261022375888 0.009099418138128673 0.008946123646393717 -0.02231944699711045 0.041543775447676894 0.57 False 1000 +BP 0.0 280 auprc supra_input_output naive 0.13038368470223805 0.15353860455094878 -0.023154919848710737 -0.02265389326948487 -0.04981347246281995 0.0057689022045569254 0.114 False 1000 +BP 0.0 280 f_max supra_relative_output naive 0.19018741404788822 0.2448261022375888 -0.05463868818970058 -0.053636620542212056 -0.08354014630955259 -0.023699615355379372 0.0 True 1000 +BP 0.0 280 auprc supra_relative_output naive 0.06923176688783254 0.15353860455094878 -0.08430683766311624 -0.0840565502842325 -0.10514442253339715 -0.06330672121331302 0.0 True 1000 +BP 0.0 280 f_max full naive 0.22172328957619325 0.2448261022375888 -0.02310281266139555 -0.022680051409949834 -0.05296230679613516 0.007665047769600401 0.15 False 1000 +BP 0.0 280 auprc full naive 0.09734065527721904 0.15353860455094878 -0.056197949273729744 -0.05586950390928712 -0.07824154867725922 -0.031928187510326514 0.0 True 1000 +BP 0.0 280 f_max full single 0.22172328957619325 0.23118805206728577 -0.009464762491092515 -0.00952701008483324 -0.0248959410685953 0.005237046230510847 0.214 False 1000 +BP 0.0 280 auprc full single 0.09734065527721904 0.10824296864062724 -0.010902313363408203 -0.011009782825520812 -0.023049748693628826 -0.0006351144466498244 0.038 True 1000 +BP 2.0 280 f_max supra single 0.17306362498704847 0.16961760380902954 0.0034460211780189265 0.0034107231999206283 -0.0007510549324127146 0.00837273306341775 0.116 False 1000 +BP 2.0 280 auprc supra single 0.07174461568612162 0.06849099189384009 0.003253623792281529 0.003246997597152768 0.0003878315966146915 0.006313293816363391 0.026 True 1000 +BP 2.0 280 f_max supra_input supra 0.19298127282714725 0.17306362498704847 0.01991764784009878 0.021347786507025127 0.006226584473214239 0.0432603287268462 0.002 True 1000 +BP 2.0 280 auprc supra_input supra 0.07368941487309685 0.07174461568612162 0.001944799186975238 0.0017558027553350377 -0.010572098603433113 0.013919378847747596 0.778 False 1000 +BP 2.0 280 f_max supra_relative supra 0.13151832103319872 0.17306362498704847 -0.041545303953849755 -0.04215624783624564 -0.06438387621552086 -0.021610050300843314 0.0 True 1000 +BP 2.0 280 auprc supra_relative supra 0.03140843262490831 0.07174461568612162 -0.0403361830612133 -0.04076194836005893 -0.05900993928239558 -0.02495066522282944 0.0 True 1000 +BP 2.0 280 f_max supra_output supra 0.17754422727003788 0.17306362498704847 0.00448060228298941 0.004125948037745166 -0.006901298443608957 0.016236079306431127 0.466 False 1000 +BP 2.0 280 auprc supra_output supra 0.06543747274327048 0.07174461568612162 -0.006307142942851132 -0.006518012441452411 -0.015978928882221917 0.002527505747544476 0.178 False 1000 +BP 2.0 280 f_max supra_input_relative supra_input 0.16090319819984983 0.19298127282714725 -0.03207807462729742 -0.03384110539483642 -0.05591205090544008 -0.015791991796348357 0.0 True 1000 +BP 2.0 280 auprc supra_input_relative supra_input 0.05163132588355115 0.07368941487309685 -0.022058088989545706 -0.022319021147738243 -0.035115892264668915 -0.010828930523513546 0.0 True 1000 +BP 2.0 280 f_max supra_input_relative supra_relative 0.16090319819984983 0.13151832103319872 0.029384877166651113 0.029662928948434347 0.014618921754134415 0.04568033873410833 0.0 True 1000 +BP 2.0 280 auprc supra_input_relative supra_relative 0.05163132588355115 0.03140843262490831 0.020222893258642835 0.02019872996765573 0.010805318714325426 0.030297476868710052 0.0 True 1000 +BP 2.0 280 f_max supra_input_output supra_input 0.19442959339214827 0.19298127282714725 0.0014483205650010211 -0.0007040590490854448 -0.0090283165271402 0.00639264861749586 0.886 False 1000 +BP 2.0 280 auprc supra_input_output supra_input 0.07889988553924517 0.07368941487309685 0.005210470666148312 0.005159087173056824 0.0013295541449735524 0.009409841327016003 0.008 True 1000 +BP 2.0 280 f_max supra_input_output supra_output 0.19442959339214827 0.17754422727003788 0.01688536612211039 0.016517779420194514 0.005812513656117802 0.030171147078578927 0.002 True 1000 +BP 2.0 280 auprc supra_input_output supra_output 0.07889988553924517 0.06543747274327048 0.013462412795974682 0.013432902369844275 0.0066669488347831825 0.02108861043050281 0.0 True 1000 +BP 2.0 280 f_max supra_relative_output supra_relative 0.1334460693994402 0.13151832103319872 0.0019277483662414885 0.001281315981514228 -0.0032737667732135956 0.0046779271559358165 0.384 False 1000 +BP 2.0 280 auprc supra_relative_output supra_relative 0.031228862219874462 0.03140843262490831 -0.00017957040503385063 -0.0002136851225374418 -0.00523189001773528 0.0044360059746145365 0.922 False 1000 +BP 2.0 280 f_max supra_relative_output supra_output 0.1334460693994402 0.17754422727003788 -0.044098157870597676 -0.04500087989247657 -0.0670335722715342 -0.023845848831727832 0.0 True 1000 +BP 2.0 280 auprc supra_relative_output supra_output 0.031228862219874462 0.06543747274327048 -0.03420861052339602 -0.034457621041143964 -0.049569530845337405 -0.021197199663569324 0.0 True 1000 +BP 2.0 280 f_max full supra_input_relative 0.1586593787825127 0.16090319819984983 -0.0022438194173371173 -0.0013895108680303998 -0.005163691539931328 0.0028709704677585142 0.436 False 1000 +BP 2.0 280 auprc full supra_input_relative 0.050503835419353965 0.05163132588355115 -0.0011274904641971825 -0.001100444434107521 -0.005475686128804402 0.0030154411793788844 0.608 False 1000 +BP 2.0 280 f_max full supra_input_output 0.1586593787825127 0.19442959339214827 -0.03577021460963556 -0.034526557213781374 -0.053737126797426815 -0.017688732701067866 0.0 True 1000 +BP 2.0 280 auprc full supra_input_output 0.050503835419353965 0.07889988553924517 -0.0283960501198912 -0.02857855275490259 -0.041877892776797376 -0.017882979926020406 0.0 True 1000 +BP 2.0 280 f_max full supra_relative_output 0.1586593787825127 0.1334460693994402 0.025213309383072507 0.026992102098889716 0.012863259574906898 0.04242819418027794 0.0 True 1000 +BP 2.0 280 auprc full supra_relative_output 0.050503835419353965 0.031228862219874462 0.019274973199479503 0.01931197065608565 0.01177851972134148 0.02790009694079797 0.0 True 1000 +BP 2.0 280 f_max single naive 0.16961760380902954 0.13750683947562561 0.03211076433340393 0.03337933783541494 0.0027764523467290344 0.0673389018047483 0.028 True 1000 +BP 2.0 280 auprc single naive 0.06849099189384009 0.05141761288677794 0.01707337900706215 0.018027819851339326 -0.003030699277458974 0.04463894098015902 0.094 False 1000 +BP 2.0 280 f_max supra naive 0.17306362498704847 0.13750683947562561 0.035556785511422856 0.03679006103533557 0.005480030997847609 0.07085083983374751 0.02 True 1000 +BP 2.0 280 auprc supra naive 0.07174461568612162 0.05141761288677794 0.02032700279934368 0.021274817448492092 -0.0006076370736711062 0.04882742689348591 0.06 False 1000 +BP 2.0 280 f_max supra_input naive 0.19298127282714725 0.13750683947562561 0.055474433351521635 0.0581378475423607 0.02761894458713147 0.09125516779682778 0.0 True 1000 +BP 2.0 280 auprc supra_input naive 0.07368941487309685 0.05141761288677794 0.022271801986318916 0.023030620203827134 0.00229689348020616 0.047932829235212394 0.026 True 1000 +BP 2.0 280 f_max supra_relative naive 0.13151832103319872 0.13750683947562561 -0.005988518442426899 -0.005366186800910077 -0.03366511564033397 0.024268869467217218 0.678 False 1000 +BP 2.0 280 auprc supra_relative naive 0.03140843262490831 0.05141761288677794 -0.020009180261869625 -0.019487130911566845 -0.03311144567790638 -0.003632861999881536 0.012 True 1000 +BP 2.0 280 f_max supra_output naive 0.17754422727003788 0.13750683947562561 0.040037387794412266 0.04091600907308073 0.012331700099170074 0.07302076176489213 0.008 True 1000 +BP 2.0 280 auprc supra_output naive 0.06543747274327048 0.05141761288677794 0.014019859856492546 0.014756805007039683 -0.004792512617146979 0.0378431320921719 0.154 False 1000 +BP 2.0 280 f_max supra_input_relative naive 0.16090319819984983 0.13750683947562561 0.023396358724224214 0.02429674214752427 -0.0030758115198014965 0.05549761584533118 0.082 False 1000 +BP 2.0 280 auprc supra_input_relative naive 0.05163132588355115 0.05141761288677794 0.00021371299677321043 0.0007115990560888874 -0.016314618875848445 0.02015108078503528 1.0 False 1000 +BP 2.0 280 f_max supra_input_output naive 0.19442959339214827 0.13750683947562561 0.056922753916522656 0.05743378849327525 0.027470692243892477 0.0896271123937604 0.0 True 1000 +BP 2.0 280 auprc supra_input_output naive 0.07889988553924517 0.05141761288677794 0.027482272652467228 0.028189707376883956 0.006577109703718517 0.053711336976517386 0.008 True 1000 +BP 2.0 280 f_max supra_relative_output naive 0.1334460693994402 0.13750683947562561 -0.00406077007618541 -0.004084870819395849 -0.03334658828931204 0.02584378689868615 0.764 False 1000 +BP 2.0 280 auprc supra_relative_output naive 0.031228862219874462 0.05141761288677794 -0.020188750666903475 -0.01970081603410428 -0.03330068906372104 -0.004095098473822269 0.014 True 1000 +BP 2.0 280 f_max full naive 0.1586593787825127 0.13750683947562561 0.021152539306887097 0.022907231279493873 -0.003573911772098757 0.05282434803981673 0.108 False 1000 +BP 2.0 280 auprc full naive 0.050503835419353965 0.05141761288677794 -0.0009137774674239721 -0.0003888453780186335 -0.016906512530189214 0.01882257414594033 0.904 False 1000 +BP 2.0 280 f_max full single 0.1586593787825127 0.16961760380902954 -0.010958225026516832 -0.010472106555921066 -0.026639812607469636 0.004787293376492602 0.168 False 1000 +BP 2.0 280 auprc full single 0.050503835419353965 0.06849099189384009 -0.01798715647448612 -0.018416665229357962 -0.032204117392600144 -0.006461301200293548 0.004 True 1000 +BP 4.0 276 f_max supra single 0.12225491312943826 0.11708071265233112 0.005174200477107135 0.0034657094654235686 -0.001677355980962806 0.009217942677176386 0.174 False 1000 +BP 4.0 276 auprc supra single 0.036024747250942145 0.031958241999146354 0.004066505251795791 0.004023007334352549 0.0011420926072675465 0.008295875533666777 0.0 True 1000 +BP 4.0 276 f_max supra_input supra 0.1415351523410215 0.12225491312943826 0.019280239211583228 0.01766096820010047 -0.002569605317307302 0.038973807484330734 0.112 False 1000 +BP 4.0 276 auprc supra_input supra 0.03704168130007926 0.036024747250942145 0.001016934049137115 0.0009392872621281502 -0.00883256153358061 0.009909274535227483 0.818 False 1000 +BP 4.0 276 f_max supra_relative supra 0.08373433234879325 0.12225491312943826 -0.038520580780645006 -0.03974862312832235 -0.06187154167831402 -0.01930363445709081 0.0 True 1000 +BP 4.0 276 auprc supra_relative supra 0.01289216911036889 0.036024747250942145 -0.023132578140573255 -0.023854228021458396 -0.04083341469850693 -0.011753473035175972 0.0 True 1000 +BP 4.0 276 f_max supra_output supra 0.12170029235321507 0.12225491312943826 -0.0005546207762231886 -0.0018068497421068544 -0.015781637444263072 0.010813211124900038 0.818 False 1000 +BP 4.0 276 auprc supra_output supra 0.028871544568472095 0.036024747250942145 -0.00715320268247005 -0.007518335759070577 -0.01705076751389702 -0.0007495135122330082 0.024 True 1000 +BP 4.0 276 f_max supra_input_relative supra_input 0.0966619842666126 0.1415351523410215 -0.04487316807440889 -0.04450470996640021 -0.07104937649614447 -0.020782372563228197 0.0 True 1000 +BP 4.0 276 auprc supra_input_relative supra_input 0.020290622359582438 0.03704168130007926 -0.016751058940496822 -0.016844175647136972 -0.02867742290693026 -0.006946110408006533 0.0 True 1000 +BP 4.0 276 f_max supra_input_relative supra_relative 0.0966619842666126 0.08373433234879325 0.012927651917819347 0.012904881362022605 0.0001685273800913248 0.02577431115303292 0.05 True 1000 +BP 4.0 276 auprc supra_input_relative supra_relative 0.020290622359582438 0.01289216911036889 0.007398453249213548 0.007949339636449577 0.0015388027178561818 0.015703845168025855 0.01 True 1000 +BP 4.0 276 f_max supra_input_output supra_input 0.13624185783526765 0.1415351523410215 -0.005293294505753832 -0.003515709325001232 -0.010908917299436575 0.0033264989127430138 0.306 False 1000 +BP 4.0 276 auprc supra_input_output supra_input 0.03637106977533471 0.03704168130007926 -0.0006706115247445479 -0.0009071094181243111 -0.004278487889723218 0.0019060283819847298 0.532 False 1000 +BP 4.0 276 f_max supra_input_output supra_output 0.13624185783526765 0.12170029235321507 0.014541565482052585 0.01595210861720609 0.0028841892414396817 0.032764741851625145 0.01 True 1000 +BP 4.0 276 auprc supra_input_output supra_output 0.03637106977533471 0.028871544568472095 0.007499525206862617 0.007550513603074416 0.0028742152270996176 0.013174013006473978 0.002 True 1000 +BP 4.0 276 f_max supra_relative_output supra_relative 0.08724122315327773 0.08373433234879325 0.0035068908044844777 0.002014632355605338 -0.001953344581897343 0.005503501814115476 0.246 False 1000 +BP 4.0 276 auprc supra_relative_output supra_relative 0.012685259590633123 0.01289216911036889 -0.0002069095197357665 4.925907804026423e-05 -0.003728150114821094 0.0042060437334531355 0.992 False 1000 +BP 4.0 276 f_max supra_relative_output supra_output 0.08724122315327773 0.12170029235321507 -0.03445906919993734 -0.03592714103061015 -0.05976040866812411 -0.013652968982739492 0.0 True 1000 +BP 4.0 276 auprc supra_relative_output supra_output 0.012685259590633123 0.028871544568472095 -0.01618628497783897 -0.016286633184347556 -0.02699104781000357 -0.007644523917528479 0.0 True 1000 +BP 4.0 276 f_max full supra_input_relative 0.09498655717961088 0.0966619842666126 -0.0016754270870017185 -0.0019735411334702 -0.007465742525066166 0.0007703574706583962 0.18 False 1000 +BP 4.0 276 auprc full supra_input_relative 0.021022876794778367 0.020290622359582438 0.0007322544351959295 0.0005458342090130938 -0.003593185862223967 0.004583191146754343 0.77 False 1000 +BP 4.0 276 f_max full supra_input_output 0.09498655717961088 0.13624185783526765 -0.041255300655656774 -0.04296254177486918 -0.06798583886792324 -0.01993605858415505 0.0 True 1000 +BP 4.0 276 auprc full supra_input_output 0.021022876794778367 0.03637106977533471 -0.015348192980556345 -0.015391232019999567 -0.02618885470135927 -0.005928317164978757 0.002 True 1000 +BP 4.0 276 f_max full supra_relative_output 0.09498655717961088 0.08724122315327773 0.0077453340263331505 0.008916707872947068 -0.004214683248146299 0.022802635373956295 0.18 False 1000 +BP 4.0 276 auprc full supra_relative_output 0.021022876794778367 0.012685259590633123 0.008337617204145244 0.008445914767422406 0.002905557554482061 0.015055561377751055 0.002 True 1000 +BP 4.0 276 f_max single naive 0.11708071265233112 0.046529709699944764 0.07055100295238637 0.07480312348733449 0.04444734178507228 0.10952849208584207 0.0 True 1000 +BP 4.0 276 auprc single naive 0.031958241999146354 0.013590508225684453 0.018367733773461903 0.019703566319562937 0.005760216742088609 0.0403272611231339 0.002 True 1000 +BP 4.0 276 f_max supra naive 0.12225491312943826 0.046529709699944764 0.07572520342949349 0.07826883295275806 0.048419753356825385 0.1133105324233124 0.0 True 1000 +BP 4.0 276 auprc supra naive 0.036024747250942145 0.013590508225684453 0.022434239025257693 0.02372657365391549 0.007717555748769079 0.0464466025111101 0.0 True 1000 +BP 4.0 276 f_max supra_input naive 0.1415351523410215 0.046529709699944764 0.09500544264107671 0.09592980115285853 0.06633105657625901 0.12801868976884445 0.0 True 1000 +BP 4.0 276 auprc supra_input naive 0.03704168130007926 0.013590508225684453 0.02345117307439481 0.024665860916043638 0.01072327901984696 0.04270064933866439 0.0 True 1000 +BP 4.0 276 f_max supra_relative naive 0.08373433234879325 0.046529709699944764 0.03720462264884849 0.03852020982443571 0.013521390474971279 0.0666714070446147 0.002 True 1000 +BP 4.0 276 auprc supra_relative naive 0.01289216911036889 0.013590508225684453 -0.0006983391153155636 -0.00012765436754291114 -0.0072242787251023954 0.009891694584843609 0.866 False 1000 +BP 4.0 276 f_max supra_output naive 0.12170029235321507 0.046529709699944764 0.07517058265327031 0.0764619832106512 0.04799964345074092 0.10752825684903768 0.0 True 1000 +BP 4.0 276 auprc supra_output naive 0.028871544568472095 0.013590508225684453 0.015281036342787642 0.01620823789484491 0.004085511236460339 0.03262602656215179 0.004 True 1000 +BP 4.0 276 f_max supra_input_relative naive 0.0966619842666126 0.046529709699944764 0.050132274566667835 0.05142509118645831 0.02620065666470451 0.0796558440589294 0.0 True 1000 +BP 4.0 276 auprc supra_input_relative naive 0.020290622359582438 0.013590508225684453 0.006700114133897985 0.007821685268906666 -0.0022540778084713183 0.022043959993201634 0.166 False 1000 +BP 4.0 276 f_max supra_input_output naive 0.13624185783526765 0.046529709699944764 0.08971214813532288 0.09241409182785729 0.06466988727056751 0.12261396495177515 0.0 True 1000 +BP 4.0 276 auprc supra_input_output naive 0.03637106977533471 0.013590508225684453 0.02278056154965026 0.02375875149791933 0.009997670781269157 0.04090563211814514 0.0 True 1000 +BP 4.0 276 f_max supra_relative_output naive 0.08724122315327773 0.046529709699944764 0.040711513453332966 0.04053484218004105 0.014231854543195728 0.06873467690530333 0.002 True 1000 +BP 4.0 276 auprc supra_relative_output naive 0.012685259590633123 0.013590508225684453 -0.0009052486350513301 -7.839528950264695e-05 -0.007774219700456941 0.011058416492602604 0.876 False 1000 +BP 4.0 276 f_max full naive 0.09498655717961088 0.046529709699944764 0.04845684747966612 0.04945155005298811 0.025318167251367086 0.07798460945455203 0.0 True 1000 +BP 4.0 276 auprc full naive 0.021022876794778367 0.013590508225684453 0.007432368569093914 0.008367519477919758 -0.002306854606695461 0.023410503388675452 0.172 False 1000 +BP 4.0 276 f_max full single 0.09498655717961088 0.11708071265233112 -0.022094155472720242 -0.02535157343434637 -0.04764563304082319 -0.006007076740980322 0.012 True 1000 +BP 4.0 276 auprc full single 0.021022876794778367 0.031958241999146354 -0.010935365204367987 -0.011336046841643178 -0.02344663883750386 -0.001456376564127707 0.018 True 1000 +MF 0.0 392 f_max supra single 0.3504597362612787 0.35184878665395036 -0.0013890503926716757 -0.0012164337107802641 -0.011423381331893715 0.00940429566330763 0.8 False 1000 +MF 0.0 392 auprc supra single 0.1776717251689381 0.1773342076968634 0.0003375174720746943 0.0005647073093582895 -0.0031886639790258313 0.004646816718468438 0.8 False 1000 +MF 0.0 392 f_max supra_input supra 0.36737678045727645 0.3504597362612787 0.016917044195997766 0.016845007822230203 -0.0007454060601018641 0.035476190188295634 0.064 False 1000 +MF 0.0 392 auprc supra_input supra 0.17784905555489533 0.1776717251689381 0.0001773303859572417 0.0002037154677632386 -0.008923313092032495 0.010294996319620615 0.938 False 1000 +MF 0.0 392 f_max supra_relative supra 0.34065815985623116 0.3504597362612787 -0.00980157640504753 -0.00926977003242962 -0.03233907957084341 0.015000734175344834 0.452 False 1000 +MF 0.0 392 auprc supra_relative supra 0.16111075915801953 0.1776717251689381 -0.016560966010918554 -0.016539404775849087 -0.035982723960896384 0.004246144600566147 0.108 False 1000 +MF 0.0 392 f_max supra_output supra 0.3576744070429309 0.3504597362612787 0.0072146707816522215 0.007688930448635647 -0.007961399243639604 0.02352153530185721 0.34 False 1000 +MF 0.0 392 auprc supra_output supra 0.18648654900225867 0.1776717251689381 0.008814823833320579 0.008783267072154178 -0.0015674636943321708 0.018011222978485765 0.08 False 1000 +MF 0.0 392 f_max supra_input_relative supra_input 0.36420255420510744 0.36737678045727645 -0.00317422625216901 -0.002577745219751396 -0.024001896567753096 0.019543711728160437 0.822 False 1000 +MF 0.0 392 auprc supra_input_relative supra_input 0.197624731265526 0.17784905555489533 0.019775675710630658 0.019836499754096657 0.004854970576853503 0.03384666051458282 0.008 True 1000 +MF 0.0 392 f_max supra_input_relative supra_relative 0.36420255420510744 0.34065815985623116 0.023544394348876285 0.023537032634908427 -0.0033936329095489394 0.04846514694094781 0.088 False 1000 +MF 0.0 392 auprc supra_input_relative supra_relative 0.197624731265526 0.16111075915801953 0.036513972107506454 0.03657961999770897 0.017934853007527576 0.05476650888586681 0.0 True 1000 +MF 0.0 392 f_max supra_input_output supra_input 0.36763063939325596 0.36737678045727645 0.0002538589359795118 0.0011896052776207303 -0.010307134735980926 0.011671736230424085 0.826 False 1000 +MF 0.0 392 auprc supra_input_output supra_input 0.19671267213108037 0.17784905555489533 0.018863616576185044 0.01881431604516286 0.013142195126681378 0.02462253027444928 0.0 True 1000 +MF 0.0 392 f_max supra_input_output supra_output 0.36763063939325596 0.3576744070429309 0.009956232350325056 0.010345682651215288 0.00018309936108096321 0.02134881599913515 0.048 True 1000 +MF 0.0 392 auprc supra_input_output supra_output 0.19671267213108037 0.18648654900225867 0.010226123128821707 0.010234764440771926 0.002908971532555304 0.017373858395265365 0.008 True 1000 +MF 0.0 392 f_max supra_relative_output supra_relative 0.34592202257534416 0.34065815985623116 0.005263862719112999 0.007897222731226828 -0.010975760915426536 0.02978687239825713 0.478 False 1000 +MF 0.0 392 auprc supra_relative_output supra_relative 0.19128826041227334 0.16111075915801953 0.030177501254253808 0.030368242340195292 0.019026143723306403 0.04124151662873966 0.0 True 1000 +MF 0.0 392 f_max supra_relative_output supra_output 0.34592202257534416 0.3576744070429309 -0.011752384467586752 -0.009061477749838438 -0.03435780544594651 0.016612217747847838 0.472 False 1000 +MF 0.0 392 auprc supra_relative_output supra_output 0.19128826041227334 0.18648654900225867 0.004801711410014675 0.005045570492192033 -0.0168936449882481 0.02784812012074203 0.68 False 1000 +MF 0.0 392 f_max full supra_input_relative 0.3839210840186379 0.36420255420510744 0.01971852981353045 0.019801154103820023 0.007354371999006546 0.03319912492230597 0.002 True 1000 +MF 0.0 392 auprc full supra_input_relative 0.2187751255190095 0.197624731265526 0.0211503942534835 0.021044673673853823 0.013549158630995869 0.028851033797561876 0.0 True 1000 +MF 0.0 392 f_max full supra_input_output 0.3839210840186379 0.36763063939325596 0.016290444625381928 0.016033803606447898 -0.003417589681960031 0.03447132655667559 0.094 False 1000 +MF 0.0 392 auprc full supra_input_output 0.2187751255190095 0.19671267213108037 0.022062453387929115 0.022066857382787613 0.006417422211222205 0.03749572653000221 0.004 True 1000 +MF 0.0 392 f_max full supra_relative_output 0.3839210840186379 0.34592202257534416 0.037999061443293736 0.03544096400750163 0.014702548446940678 0.05628451565749084 0.0 True 1000 +MF 0.0 392 auprc full supra_relative_output 0.2187751255190095 0.19128826041227334 0.027486865106736147 0.027256051331367506 0.010216801619678957 0.043434613806572334 0.0 True 1000 +MF 0.0 392 f_max single naive 0.35184878665395036 0.7203660688206672 -0.36851728216671686 -0.36810742234284777 -0.42487275349867976 -0.3126980313423542 0.0 True 1000 +MF 0.0 392 auprc single naive 0.1773342076968634 0.4203418528418113 -0.2430076451449479 -0.24311572586376343 -0.29630679105123425 -0.19179000242229108 0.0 True 1000 +MF 0.0 392 f_max supra naive 0.3504597362612787 0.7203660688206672 -0.36990633255938854 -0.36932385605362805 -0.42418018729242707 -0.31574151042842735 0.0 True 1000 +MF 0.0 392 auprc supra naive 0.1776717251689381 0.4203418528418113 -0.24267012767287321 -0.24255101855440514 -0.29637674655126567 -0.1910582314173708 0.0 True 1000 +MF 0.0 392 f_max supra_input naive 0.36737678045727645 0.7203660688206672 -0.3529892883633908 -0.3524788482313978 -0.40887439197286385 -0.30156463250795984 0.0 True 1000 +MF 0.0 392 auprc supra_input naive 0.17784905555489533 0.4203418528418113 -0.24249279728691597 -0.24234730308664187 -0.2981726406789954 -0.19094637909044704 0.0 True 1000 +MF 0.0 392 f_max supra_relative naive 0.34065815985623116 0.7203660688206672 -0.37970790896443607 -0.3785936260860576 -0.43097780067369634 -0.32931012761938 0.0 True 1000 +MF 0.0 392 auprc supra_relative naive 0.16111075915801953 0.4203418528418113 -0.25923109368379177 -0.2590904233302542 -0.30832905085330276 -0.2118755496647809 0.0 True 1000 +MF 0.0 392 f_max supra_output naive 0.3576744070429309 0.7203660688206672 -0.3626916617777363 -0.3616349256049924 -0.41674455498033314 -0.31108515563191486 0.0 True 1000 +MF 0.0 392 auprc supra_output naive 0.18648654900225867 0.4203418528418113 -0.23385530383955264 -0.23376775148225096 -0.2875959216237593 -0.1836006426323842 0.0 True 1000 +MF 0.0 392 f_max supra_input_relative naive 0.36420255420510744 0.7203660688206672 -0.3561635146155598 -0.3550565934511492 -0.40851266925814644 -0.30373879881211835 0.0 True 1000 +MF 0.0 392 auprc supra_input_relative naive 0.197624731265526 0.4203418528418113 -0.22271712157628532 -0.22251080333254528 -0.27754537749482794 -0.17238934916000984 0.0 True 1000 +MF 0.0 392 f_max supra_input_output naive 0.36763063939325596 0.7203660688206672 -0.35273542942741126 -0.3512892429537771 -0.40742727046247434 -0.3012762761238246 0.0 True 1000 +MF 0.0 392 auprc supra_input_output naive 0.19671267213108037 0.4203418528418113 -0.22362918071073093 -0.22353298704147903 -0.27906258656459015 -0.17358113353991383 0.0 True 1000 +MF 0.0 392 f_max supra_relative_output naive 0.34592202257534416 0.7203660688206672 -0.37444404624532307 -0.3706964033548308 -0.4238873712902062 -0.3214657583434163 0.0 True 1000 +MF 0.0 392 auprc supra_relative_output naive 0.19128826041227334 0.4203418528418113 -0.22905359242953796 -0.22872218099005895 -0.28048061711845934 -0.18147992614226435 0.0 True 1000 +MF 0.0 392 f_max full naive 0.3839210840186379 0.7203660688206672 -0.33644498480202933 -0.33525543934732915 -0.3891472044051814 -0.2844799390151764 0.0 True 1000 +MF 0.0 392 auprc full naive 0.2187751255190095 0.4203418528418113 -0.20156672732280181 -0.20146612965869146 -0.25535177353222466 -0.15048529431144114 0.0 True 1000 +MF 0.0 392 f_max full single 0.3839210840186379 0.35184878665395036 0.03207229736468753 0.032851982995518565 0.0081441472123679 0.055334560969615164 0.006 True 1000 +MF 0.0 392 auprc full single 0.2187751255190095 0.1773342076968634 0.041440917822146095 0.041649596205072005 0.02482983495919265 0.05775178593043713 0.0 True 1000 +MF 2.0 151 f_max supra single 0.3420345280007133 0.3427980848534931 -0.0007635568527797898 -0.0006334753279696734 -0.007849046136836412 0.006752116983727203 0.856 False 1000 +MF 2.0 151 auprc supra single 0.19920622762740417 0.1890117765436093 0.010194451083794881 0.010465748170430672 0.0032160286937445745 0.018676005636825595 0.002 True 1000 +MF 2.0 151 f_max supra_input supra 0.34634258566256804 0.3420345280007133 0.004308057661854714 0.0037493046982647786 -0.018881599361330333 0.024502718098392197 0.764 False 1000 +MF 2.0 151 auprc supra_input supra 0.21902353881255027 0.19920622762740417 0.019817311185146097 0.019134168127874017 -0.0002786676263036025 0.03994769604972892 0.054 False 1000 +MF 2.0 151 f_max supra_relative supra 0.2668190504161003 0.3420345280007133 -0.075215477584613 -0.07148022821355723 -0.10851339578021373 -0.03706256777945971 0.0 True 1000 +MF 2.0 151 auprc supra_relative supra 0.10136836912622044 0.19920622762740417 -0.09783785850118373 -0.09741150496238746 -0.13373371628147188 -0.06360324130799547 0.0 True 1000 +MF 2.0 151 f_max supra_output supra 0.3283248203031117 0.3420345280007133 -0.013709707697601603 -0.01328519858831538 -0.03458018719296121 0.007496467120463653 0.214 False 1000 +MF 2.0 151 auprc supra_output supra 0.19624132476947065 0.19920622762740417 -0.0029649028579335213 -0.00316490050792971 -0.024709706608148467 0.017625983586380192 0.76 False 1000 +MF 2.0 151 f_max supra_input_relative supra_input 0.29759678021539177 0.34634258566256804 -0.04874580544717627 -0.04594938006583786 -0.06822124084744828 -0.023987618552640064 0.002 True 1000 +MF 2.0 151 auprc supra_input_relative supra_input 0.16166630451106856 0.21902353881255027 -0.05735723430148171 -0.05650950154078808 -0.08003257811537483 -0.03565618706855883 0.0 True 1000 +MF 2.0 151 f_max supra_input_relative supra_relative 0.29759678021539177 0.2668190504161003 0.030777729799291442 0.029280152845984155 0.0036004626814405668 0.05742231912041817 0.03 True 1000 +MF 2.0 151 auprc supra_input_relative supra_relative 0.16166630451106856 0.10136836912622044 0.06029793538484812 0.06003617154947339 0.03471884045094995 0.08722974868270436 0.0 True 1000 +MF 2.0 151 f_max supra_input_output supra_input 0.3387240186494688 0.34634258566256804 -0.007618567013099209 -0.008076970089546921 -0.01771094442768386 0.0007044817438132564 0.078 False 1000 +MF 2.0 151 auprc supra_input_output supra_input 0.21495445913612463 0.21902353881255027 -0.004069079676425641 -0.004001510447409798 -0.011982147829835572 0.002761426147323762 0.246 False 1000 +MF 2.0 151 f_max supra_input_output supra_output 0.3387240186494688 0.3283248203031117 0.010399198346357108 0.008957533197033235 -0.003731736145821803 0.021288209851554902 0.168 False 1000 +MF 2.0 151 auprc supra_input_output supra_output 0.21495445913612463 0.19624132476947065 0.018713134366653977 0.018297558188393924 0.008941481013205778 0.029231461484347878 0.0 True 1000 +MF 2.0 151 f_max supra_relative_output supra_relative 0.2833439796045256 0.2668190504161003 0.016524929188425297 0.012316894066810014 -0.002480004447797534 0.026615220490583125 0.126 False 1000 +MF 2.0 151 auprc supra_relative_output supra_relative 0.11782829271639662 0.10136836912622044 0.016459923590176187 0.01647569190154201 0.002046805768770581 0.032523617871591075 0.02 True 1000 +MF 2.0 151 f_max supra_relative_output supra_output 0.2833439796045256 0.3283248203031117 -0.0449808406985861 -0.04587813555843184 -0.07897748371180338 -0.014165339260743636 0.006 True 1000 +MF 2.0 151 auprc supra_relative_output supra_output 0.11782829271639662 0.19624132476947065 -0.07841303205307402 -0.07777091255291575 -0.111069074008383 -0.049378508664473304 0.0 True 1000 +MF 2.0 151 f_max full supra_input_relative 0.3071719115131589 0.29759678021539177 0.009575131297767159 0.006247909412720043 -0.006343220124909852 0.018432845682571915 0.3 False 1000 +MF 2.0 151 auprc full supra_input_relative 0.15972442628853586 0.16166630451106856 -0.0019418782225326947 -0.0020176769213716794 -0.015560299545944897 0.013015900848466546 0.754 False 1000 +MF 2.0 151 f_max full supra_input_output 0.3071719115131589 0.3387240186494688 -0.031552107136309904 -0.03162450056357089 -0.05365770783520404 -0.01036951073929182 0.008 True 1000 +MF 2.0 151 auprc full supra_input_output 0.15972442628853586 0.21495445913612463 -0.05523003284758876 -0.05452566801474996 -0.07796095643894714 -0.03534703321486821 0.0 True 1000 +MF 2.0 151 f_max full supra_relative_output 0.3071719115131589 0.2833439796045256 0.023827931908633304 0.023211168191894182 -0.0024236237344238596 0.04853985452836605 0.074 False 1000 +MF 2.0 151 auprc full supra_relative_output 0.15972442628853586 0.11782829271639662 0.04189613357213924 0.0415428027265597 0.019957824008984524 0.06600541041968762 0.0 True 1000 +MF 2.0 151 f_max single naive 0.3427980848534931 0.0747337284956894 0.2680643563578037 0.26869075740659915 0.20816841071341075 0.3302295183185279 0.0 True 1000 +MF 2.0 151 auprc single naive 0.1890117765436093 0.026582066823136727 0.16242970972047255 0.1633684093071387 0.1115145538621808 0.21723017442502746 0.0 True 1000 +MF 2.0 151 f_max supra naive 0.3420345280007133 0.0747337284956894 0.26730079950502394 0.26805728207862944 0.2067552890333757 0.3298118690091098 0.0 True 1000 +MF 2.0 151 auprc supra naive 0.19920622762740417 0.026582066823136727 0.17262416080426746 0.17383415747756942 0.11980123272155366 0.23421760031979955 0.0 True 1000 +MF 2.0 151 f_max supra_input naive 0.34634258566256804 0.0747337284956894 0.27160885716687866 0.27180658677689423 0.21492159532223898 0.32837850175044425 0.0 True 1000 +MF 2.0 151 auprc supra_input naive 0.21902353881255027 0.026582066823136727 0.19244147198941353 0.19296832560544339 0.1362026148735223 0.2526925477605205 0.0 True 1000 +MF 2.0 151 f_max supra_relative naive 0.2668190504161003 0.0747337284956894 0.19208532192041092 0.19657705386507218 0.14756549787607917 0.2493834796032456 0.0 True 1000 +MF 2.0 151 auprc supra_relative naive 0.10136836912622044 0.026582066823136727 0.07478630230308371 0.07642265251518193 0.04059509119220879 0.1202155842612466 0.0 True 1000 +MF 2.0 151 f_max supra_output naive 0.3283248203031117 0.0747337284956894 0.25359109180742234 0.25477208349031405 0.19854094189733623 0.3100423291306011 0.0 True 1000 +MF 2.0 151 auprc supra_output naive 0.19624132476947065 0.026582066823136727 0.1696592579463339 0.1706692569696397 0.11586605523968752 0.22879714912553256 0.0 True 1000 +MF 2.0 151 f_max supra_input_relative naive 0.29759678021539177 0.0747337284956894 0.22286305171970236 0.22585720671105633 0.172775517146243 0.27801496006699666 0.0 True 1000 +MF 2.0 151 auprc supra_input_relative naive 0.16166630451106856 0.026582066823136727 0.13508423768793182 0.13645882406465534 0.08957002937195822 0.18926703216630056 0.0 True 1000 +MF 2.0 151 f_max supra_input_output naive 0.3387240186494688 0.0747337284956894 0.2639902901537794 0.2637296166873473 0.20703571910144747 0.32102419434177965 0.0 True 1000 +MF 2.0 151 auprc supra_input_output naive 0.21495445913612463 0.026582066823136727 0.1883723923129879 0.18896681515803362 0.13276130401774144 0.2515725464463723 0.0 True 1000 +MF 2.0 151 f_max supra_relative_output naive 0.2833439796045256 0.0747337284956894 0.20861025110883621 0.2088939479318822 0.16103379048141578 0.2584291966067251 0.0 True 1000 +MF 2.0 151 auprc supra_relative_output naive 0.11782829271639662 0.026582066823136727 0.0912462258932599 0.09289834441672395 0.05334200487349824 0.1411540858646441 0.0 True 1000 +MF 2.0 151 f_max full naive 0.3071719115131589 0.0747337284956894 0.23243818301746952 0.23210511612377638 0.1811544506733097 0.28370226528712533 0.0 True 1000 +MF 2.0 151 auprc full naive 0.15972442628853586 0.026582066823136727 0.13314235946539915 0.13444114714328367 0.08716528223552117 0.18531879994595524 0.0 True 1000 +MF 2.0 151 f_max full single 0.3071719115131589 0.3427980848534931 -0.03562617334033419 -0.03658564128282271 -0.06140143281012552 -0.011671213467656871 0.01 True 1000 +MF 2.0 151 auprc full single 0.15972442628853586 0.1890117765436093 -0.029287350255073424 -0.028927262163855078 -0.05256742471672103 -0.004213290495183715 0.02 True 1000 +MF 4.0 145 f_max supra single 0.31620643824473516 0.3167811928767344 -0.0005747546319992414 0.0004177473752856677 -0.0072014370198171945 0.008452231588557896 0.972 False 1000 +MF 4.0 145 auprc supra single 0.18565479835779616 0.18328900688605118 0.0023657914717449824 0.0024542969740007125 -0.002122408201735552 0.006928872130983516 0.28 False 1000 +MF 4.0 145 f_max supra_input supra 0.33381836824773203 0.31620643824473516 0.01761193000299688 0.01392410612287826 -0.008880210073205642 0.03772718414629681 0.228 False 1000 +MF 4.0 145 auprc supra_input supra 0.20050425945779105 0.18565479835779616 0.014849461099994882 0.014556659387429532 -0.005191084648157036 0.03463717589833894 0.132 False 1000 +MF 4.0 145 f_max supra_relative supra 0.24360821271781707 0.31620643824473516 -0.07259822552691808 -0.07185493525165615 -0.10764743421625188 -0.037285104322552644 0.0 True 1000 +MF 4.0 145 auprc supra_relative supra 0.0866384924224616 0.18565479835779616 -0.09901630593533456 -0.09849955748143771 -0.14153486834806797 -0.06160051976520385 0.0 True 1000 +MF 4.0 145 f_max supra_output supra 0.3138096336503749 0.31620643824473516 -0.002396804594360258 -0.003892290115837782 -0.025060479206152446 0.018174452861521275 0.682 False 1000 +MF 4.0 145 auprc supra_output supra 0.18058233744285324 0.18565479835779616 -0.005072460914942928 -0.004941931947204727 -0.02318195017213066 0.012236228280592707 0.598 False 1000 +MF 4.0 145 f_max supra_input_relative supra_input 0.26976449185413537 0.33381836824773203 -0.06405387639359666 -0.05996587507197815 -0.08655313337209777 -0.03377158385280094 0.0 True 1000 +MF 4.0 145 auprc supra_input_relative supra_input 0.1260701210247072 0.20050425945779105 -0.07443413843308386 -0.07456628484079408 -0.10589021219802079 -0.048596853232950814 0.0 True 1000 +MF 4.0 145 f_max supra_input_relative supra_relative 0.26976449185413537 0.24360821271781707 0.026156279136318294 0.02581316630255626 -0.001367341710171309 0.054287731174962314 0.074 False 1000 +MF 4.0 145 auprc supra_input_relative supra_relative 0.1260701210247072 0.0866384924224616 0.03943162860224558 0.03848993202807314 0.014860292707146965 0.06557618072317248 0.0 True 1000 +MF 4.0 145 f_max supra_input_output supra_input 0.3223521734024969 0.33381836824773203 -0.011466194845235145 -0.010569492506604817 -0.02156070257588503 0.00031297612746291747 0.064 False 1000 +MF 4.0 145 auprc supra_input_output supra_input 0.19524934449529005 0.20050425945779105 -0.005254914962500995 -0.005050740285052841 -0.011759600933973336 0.0016832688640272136 0.13 False 1000 +MF 4.0 145 f_max supra_input_output supra_output 0.3223521734024969 0.3138096336503749 0.008542539752121991 0.007246903732111225 -0.006701466682769336 0.020492087620683164 0.294 False 1000 +MF 4.0 145 auprc supra_input_output supra_output 0.19524934449529005 0.18058233744285324 0.014667007052436815 0.014447851049581418 0.004610222607999981 0.025561030236240175 0.002 True 1000 +MF 4.0 145 f_max supra_relative_output supra_relative 0.25331622046992747 0.24360821271781707 0.009708007752110392 0.0076189810274111215 -0.014256221816485568 0.02277431697810299 0.396 False 1000 +MF 4.0 145 auprc supra_relative_output supra_relative 0.09342925429690763 0.0866384924224616 0.006790761874446022 0.007147816698363236 -0.006999141939293879 0.021301457716008068 0.316 False 1000 +MF 4.0 145 f_max supra_relative_output supra_output 0.25331622046992747 0.3138096336503749 -0.06049341318044743 -0.06034366410840724 -0.0982056555690464 -0.021611072206226465 0.0 True 1000 +MF 4.0 145 auprc supra_relative_output supra_output 0.09342925429690763 0.18058233744285324 -0.08715308314594561 -0.08640980883586973 -0.12303838364815538 -0.052680535698449285 0.0 True 1000 +MF 4.0 145 f_max full supra_input_relative 0.2768034568720887 0.26976449185413537 0.007038965017953314 0.003641870735865019 -0.0076847107306769686 0.014997496876061995 0.488 False 1000 +MF 4.0 145 auprc full supra_input_relative 0.12957851381254135 0.1260701210247072 0.003508392787834158 0.0033838300818872184 -0.006769593488420815 0.014680146229028395 0.502 False 1000 +MF 4.0 145 f_max full supra_input_output 0.2768034568720887 0.3223521734024969 -0.045548716530408206 -0.0457545118295083 -0.07032349923998814 -0.022215580011890543 0.0 True 1000 +MF 4.0 145 auprc full supra_input_output 0.12957851381254135 0.19524934449529005 -0.0656708306827487 -0.06613171447385403 -0.09422081401254256 -0.04217039942523295 0.0 True 1000 +MF 4.0 145 f_max full supra_relative_output 0.2768034568720887 0.25331622046992747 0.023487236402161216 0.021836056011010154 -0.008851747985014629 0.052348117419346855 0.162 False 1000 +MF 4.0 145 auprc full supra_relative_output 0.12957851381254135 0.09342925429690763 0.03614925951563372 0.034725945411597126 0.011670997851153452 0.060185188185883844 0.01 True 1000 +MF 4.0 145 f_max single naive 0.3167811928767344 0.07527512705860413 0.24150606581813028 0.24207595551186056 0.17777865177458785 0.31024357089798504 0.0 True 1000 +MF 4.0 145 auprc single naive 0.18328900688605118 0.023373921180975445 0.15991508570507573 0.16099611352864093 0.10404723784761793 0.22166768182108806 0.0 True 1000 +MF 4.0 145 f_max supra naive 0.31620643824473516 0.07527512705860413 0.24093131118613104 0.24249370288714625 0.17827710439623556 0.30947135325683073 0.0 True 1000 +MF 4.0 145 auprc supra naive 0.18565479835779616 0.023373921180975445 0.16228087717682071 0.16345041050264167 0.1060875771692564 0.2247465214855931 0.0 True 1000 +MF 4.0 145 f_max supra_input naive 0.33381836824773203 0.07527512705860413 0.2585432411891279 0.2564178090100245 0.1905256588892453 0.32147523288356095 0.0 True 1000 +MF 4.0 145 auprc supra_input naive 0.20050425945779105 0.023373921180975445 0.1771303382768156 0.17800706989007117 0.12159268498744982 0.24292394579779644 0.0 True 1000 +MF 4.0 145 f_max supra_relative naive 0.24360821271781707 0.07527512705860413 0.16833308565921296 0.1706387676354901 0.11604816332145075 0.22618759082261472 0.0 True 1000 +MF 4.0 145 auprc supra_relative naive 0.0866384924224616 0.023373921180975445 0.06326457124148616 0.06495085302120396 0.02862224695927931 0.10842327806723642 0.0 True 1000 +MF 4.0 145 f_max supra_output naive 0.3138096336503749 0.07527512705860413 0.23853450659177078 0.23860141277130845 0.1756120935910325 0.3045365870573729 0.0 True 1000 +MF 4.0 145 auprc supra_output naive 0.18058233744285324 0.023373921180975445 0.1572084162618778 0.15850847855543693 0.104273330598404 0.2228898006836614 0.0 True 1000 +MF 4.0 145 f_max supra_input_relative naive 0.26976449185413537 0.07527512705860413 0.19448936479553125 0.19645193393804636 0.14303844353174883 0.2521300756325815 0.0 True 1000 +MF 4.0 145 auprc supra_input_relative naive 0.1260701210247072 0.023373921180975445 0.10269619984373174 0.10344078504927709 0.059814782558696544 0.15286129753087702 0.0 True 1000 +MF 4.0 145 f_max supra_input_output naive 0.3223521734024969 0.07527512705860413 0.24707704634389277 0.24584831650341965 0.18272971066751295 0.3112081354483578 0.0 True 1000 +MF 4.0 145 auprc supra_input_output naive 0.19524934449529005 0.023373921180975445 0.1718754233143146 0.17295632960501833 0.11540052484693619 0.23696926167666685 0.0 True 1000 +MF 4.0 145 f_max supra_relative_output naive 0.25331622046992747 0.07527512705860413 0.17804109341132335 0.1782577486629012 0.1250855818106096 0.2312711528196573 0.0 True 1000 +MF 4.0 145 auprc supra_relative_output naive 0.09342925429690763 0.023373921180975445 0.07005533311593218 0.07209866971956719 0.03571045536537444 0.11686882189271738 0.0 True 1000 +MF 4.0 145 f_max full naive 0.2768034568720887 0.07527512705860413 0.20152832981348456 0.20009380467391136 0.14471946655919174 0.25729365180597896 0.0 True 1000 +MF 4.0 145 auprc full naive 0.12957851381254135 0.023373921180975445 0.1062045926315659 0.10682461513116431 0.06244746839800014 0.15703992623217922 0.0 True 1000 +MF 4.0 145 f_max full single 0.2768034568720887 0.3167811928767344 -0.039977736004645714 -0.041982150837949206 -0.06777002902767933 -0.016130965235034385 0.006 True 1000 +MF 4.0 145 auprc full single 0.12957851381254135 0.18328900688605118 -0.053710493073509835 -0.054171498397476626 -0.08213257301927634 -0.02705587135057773 0.0 True 1000 +CC 0.0 422 f_max supra single 0.38066936485600744 0.37939213745252304 0.0012772274034844044 0.0009561916561670674 -0.006048446261899587 0.008091863392665427 0.762 False 1000 +CC 0.0 422 auprc supra single 0.22136146113555719 0.22098897907182266 0.00037248206373452497 0.00031686346617743386 -0.003110578036826946 0.0042174734747241075 0.888 False 1000 +CC 0.0 422 f_max supra_input supra 0.37210747432733937 0.38066936485600744 -0.00856189052866807 -0.00691018110334618 -0.01908194462737339 0.005045101162849828 0.266 False 1000 +CC 0.0 422 auprc supra_input supra 0.20675314042971912 0.22136146113555719 -0.014608320705838063 -0.014555670383441442 -0.02368496590757875 -0.006159669818640373 0.0 True 1000 +CC 0.0 422 f_max supra_relative supra 0.35128104782911407 0.38066936485600744 -0.029388317026893374 -0.029422105314631773 -0.04536389864179257 -0.013101032489692278 0.0 True 1000 +CC 0.0 422 auprc supra_relative supra 0.17984949000101644 0.22136146113555719 -0.04151197113454075 -0.04203121200239163 -0.05666515558154241 -0.028516040192055433 0.0 True 1000 +CC 0.0 422 f_max supra_output supra 0.3892766795082576 0.38066936485600744 0.008607314652250175 0.00875133916260461 -0.0011344178184834498 0.019056437862673074 0.09 False 1000 +CC 0.0 422 auprc supra_output supra 0.2513872673562999 0.22136146113555719 0.030025806220742718 0.029750739068430596 0.02162500972864383 0.038583658955354844 0.0 True 1000 +CC 0.0 422 f_max supra_input_relative supra_input 0.37429348759444325 0.37210747432733937 0.0021860132671038812 0.00040783984581503553 -0.013690032990548835 0.012430765831001852 0.944 False 1000 +CC 0.0 422 auprc supra_input_relative supra_input 0.2056131265692438 0.20675314042971912 -0.0011400138604753252 -0.0012738206466654499 -0.010771892246514153 0.008306657383542535 0.77 False 1000 +CC 0.0 422 f_max supra_input_relative supra_relative 0.37429348759444325 0.35128104782911407 0.023012439765329185 0.022919764057100625 0.006643515888462521 0.039379885423672066 0.006 True 1000 +CC 0.0 422 auprc supra_input_relative supra_relative 0.2056131265692438 0.17984949000101644 0.02576363656822736 0.026201720972284737 0.0118981849137941 0.04138664853048038 0.0 True 1000 +CC 0.0 422 f_max supra_input_output supra_input 0.38818181751479036 0.37210747432733937 0.01607434318745099 0.013998084211977197 0.005230424153387932 0.022501516777223694 0.006 True 1000 +CC 0.0 422 auprc supra_input_output supra_input 0.2478064810347812 0.20675314042971912 0.04105334060506208 0.04072871587652165 0.0337521572994022 0.04832385091937476 0.0 True 1000 +CC 0.0 422 f_max supra_input_output supra_output 0.38818181751479036 0.3892766795082576 -0.0010948619934672554 -0.001663436053973594 -0.009678089219707108 0.005981675276944 0.642 False 1000 +CC 0.0 422 auprc supra_input_output supra_output 0.2478064810347812 0.2513872673562999 -0.003580786321518703 -0.003577693575350386 -0.008251631877123619 0.0009413154784029958 0.148 False 1000 +CC 0.0 422 f_max supra_relative_output supra_relative 0.3557514236403979 0.35128104782911407 0.004470375811283844 0.005360345908115125 -0.0026557696876714507 0.015021420295628482 0.22 False 1000 +CC 0.0 422 auprc supra_relative_output supra_relative 0.2156900609998714 0.17984949000101644 0.03584057099885496 0.03577917014196715 0.027885297674437582 0.043817829710173305 0.0 True 1000 +CC 0.0 422 f_max supra_relative_output supra_output 0.3557514236403979 0.3892766795082576 -0.033525255867859705 -0.032813098569121256 -0.048711142796917954 -0.016533200433883996 0.0 True 1000 +CC 0.0 422 auprc supra_relative_output supra_output 0.2156900609998714 0.2513872673562999 -0.03569720635642851 -0.036002780928855074 -0.052210483690118346 -0.019619970427479694 0.0 True 1000 +CC 0.0 422 f_max full supra_input_relative 0.3837827299538696 0.37429348759444325 0.009489242359426331 0.009507430470035657 0.0012254125797486629 0.01777572475196882 0.024 True 1000 +CC 0.0 422 auprc full supra_input_relative 0.23613261087318776 0.2056131265692438 0.030519484303943967 0.030226857543838526 0.023721538537114232 0.03738546787061514 0.0 True 1000 +CC 0.0 422 f_max full supra_input_output 0.3837827299538696 0.38818181751479036 -0.004399087560920778 -0.004082813896126504 -0.017539899263883635 0.007260917868831219 0.564 False 1000 +CC 0.0 422 auprc full supra_input_output 0.23613261087318776 0.2478064810347812 -0.011673870161593436 -0.011775678979348572 -0.022183539738155563 -0.0017574213888264453 0.02 True 1000 +CC 0.0 422 f_max full supra_relative_output 0.3837827299538696 0.3557514236403979 0.02803130631347167 0.027066848619021158 0.012334401763519852 0.042105421373339905 0.0 True 1000 +CC 0.0 422 auprc full supra_relative_output 0.23613261087318776 0.2156900609998714 0.020442549873316368 0.020649408374156117 0.008387002824536737 0.03379953706080492 0.002 True 1000 +CC 0.0 422 f_max single naive 0.37939213745252304 0.5918263432357462 -0.2124342057832232 -0.21394788534777762 -0.24622353554683654 -0.1816051936815687 0.0 True 1000 +CC 0.0 422 auprc single naive 0.22098897907182266 0.5411543162750881 -0.32016533720326545 -0.32054908004725735 -0.3563153908739898 -0.2820924142472484 0.0 True 1000 +CC 0.0 422 f_max supra naive 0.38066936485600744 0.5918263432357462 -0.2111569783797388 -0.21299169369161058 -0.2444231630428557 -0.18038987172101154 0.0 True 1000 +CC 0.0 422 auprc supra naive 0.22136146113555719 0.5411543162750881 -0.31979285513953093 -0.32023221658107986 -0.3562203273692557 -0.28201749803679843 0.0 True 1000 +CC 0.0 422 f_max supra_input naive 0.37210747432733937 0.5918263432357462 -0.21971886890840686 -0.21990187479495674 -0.2514608082821474 -0.18717986412665635 0.0 True 1000 +CC 0.0 422 auprc supra_input naive 0.20675314042971912 0.5411543162750881 -0.334401175845369 -0.33478788696452133 -0.3698159852149924 -0.2968107792277366 0.0 True 1000 +CC 0.0 422 f_max supra_relative naive 0.35128104782911407 0.5918263432357462 -0.24054529540663216 -0.2424137990062423 -0.27722837979354487 -0.20782072738739948 0.0 True 1000 +CC 0.0 422 auprc supra_relative naive 0.17984949000101644 0.5411543162750881 -0.3613048262740717 -0.36226342858347155 -0.395675381911539 -0.3244710127345613 0.0 True 1000 +CC 0.0 422 f_max supra_output naive 0.3892766795082576 0.5918263432357462 -0.20254966372748862 -0.20424035452900594 -0.23704325187705075 -0.16982922681334403 0.0 True 1000 +CC 0.0 422 auprc supra_output naive 0.2513872673562999 0.5411543162750881 -0.28976704891878824 -0.29048147751264924 -0.3263777458617887 -0.2512619036114687 0.0 True 1000 +CC 0.0 422 f_max supra_input_relative naive 0.37429348759444325 0.5918263432357462 -0.21753285564130298 -0.21949403494914171 -0.2527153603714463 -0.18606227169696735 0.0 True 1000 +CC 0.0 422 auprc supra_input_relative naive 0.2056131265692438 0.5411543162750881 -0.33554118970584434 -0.33606170761118676 -0.3724260890576838 -0.2968358386440717 0.0 True 1000 +CC 0.0 422 f_max supra_input_output naive 0.38818181751479036 0.5918263432357462 -0.20364452572095587 -0.20590379058297956 -0.23841438766743897 -0.1719499757662758 0.0 True 1000 +CC 0.0 422 auprc supra_input_output naive 0.2478064810347812 0.5411543162750881 -0.29334783524030694 -0.2940591710879996 -0.3287613494353998 -0.2555867662453542 0.0 True 1000 +CC 0.0 422 f_max supra_relative_output naive 0.3557514236403979 0.5918263432357462 -0.23607491959534832 -0.2370534530981272 -0.2707136599710847 -0.20130045209020114 0.0 True 1000 +CC 0.0 422 auprc supra_relative_output naive 0.2156900609998714 0.5411543162750881 -0.3254642552752167 -0.3264842584415043 -0.3617717170988194 -0.2865982330026093 0.0 True 1000 +CC 0.0 422 f_max full naive 0.3837827299538696 0.5918263432357462 -0.20804361328187665 -0.20998660447910605 -0.24295524843332167 -0.17691842925519993 0.0 True 1000 +CC 0.0 422 auprc full naive 0.23613261087318776 0.5411543162750881 -0.3050217054019004 -0.30583485006734823 -0.34266622155713167 -0.26621121153892613 0.0 True 1000 +CC 0.0 422 f_max full single 0.3837827299538696 0.37939213745252304 0.004390592501346546 0.00396128086867158 -0.008170065232661127 0.014079453317019082 0.46 False 1000 +CC 0.0 422 auprc full single 0.23613261087318776 0.22098897907182266 0.015143631801365104 0.014714229979909075 0.0018809104236648141 0.027487042038745736 0.024 True 1000 +CC 2.0 288 f_max supra single 0.20315757312646796 0.20418705801361683 -0.0010294848871488793 -0.0013509578842089062 -0.006385685423044436 0.003392205581195689 0.568 False 1000 +CC 2.0 288 auprc supra single 0.05664095581953725 0.057550814323517045 -0.0009098585039797949 -0.0008037520194745427 -0.004278088008178984 0.0017921948837255328 0.608 False 1000 +CC 2.0 288 f_max supra_input supra 0.19227286600693066 0.20315757312646796 -0.01088470711953729 -0.011530582115614753 -0.027158670770112373 0.002449037804491337 0.138 False 1000 +CC 2.0 288 auprc supra_input supra 0.055718082535733425 0.05664095581953725 -0.0009228732838038253 -0.0009150768214094145 -0.007369197613330233 0.005585926339207184 0.812 False 1000 +CC 2.0 288 f_max supra_relative supra 0.17406137853913475 0.20315757312646796 -0.029096194587333202 -0.030119315388915352 -0.05970593023830935 -0.004198838507561558 0.026 True 1000 +CC 2.0 288 auprc supra_relative supra 0.041937928499600215 0.05664095581953725 -0.014703027319937036 -0.015463505166546479 -0.02969097274248222 -0.0030948017548744393 0.01 True 1000 +CC 2.0 288 f_max supra_output supra 0.20872491324071593 0.20315757312646796 0.005567340114247976 0.0050478865675110994 -0.013256233506242462 0.02115600926297091 0.57 False 1000 +CC 2.0 288 auprc supra_output supra 0.05933234754454002 0.05664095581953725 0.0026913917250027675 0.0022141579340367473 -0.004913716588983489 0.008861529463500084 0.538 False 1000 +CC 2.0 288 f_max supra_input_relative supra_input 0.20014085427621897 0.19227286600693066 0.007867988269288306 0.008308104331184363 -0.012402987081305817 0.026381935410547844 0.39 False 1000 +CC 2.0 288 auprc supra_input_relative supra_input 0.0505428753460521 0.055718082535733425 -0.005175207189681327 -0.005629293648483378 -0.014912328548227882 0.002277238287626438 0.156 False 1000 +CC 2.0 288 f_max supra_input_relative supra_relative 0.20014085427621897 0.17406137853913475 0.026079475737084218 0.026896837604484965 0.009249823218211573 0.04561211998252867 0.0 True 1000 +CC 2.0 288 auprc supra_input_relative supra_relative 0.0505428753460521 0.041937928499600215 0.008604946846451883 0.008919134696653687 -0.0018022064861046134 0.021116389375983455 0.112 False 1000 +CC 2.0 288 f_max supra_input_output supra_input 0.21260013711238593 0.19227286600693066 0.020327271105455264 0.02022683843943401 0.006869692073333969 0.03445378181015158 0.008 True 1000 +CC 2.0 288 auprc supra_input_output supra_input 0.0631971384788975 0.055718082535733425 0.00747905594316408 0.007364774361528893 0.002018318444689218 0.013119525157395609 0.006 True 1000 +CC 2.0 288 f_max supra_input_output supra_output 0.21260013711238593 0.20872491324071593 0.003875223871669997 0.0036483697563081557 -0.0046872476662023 0.013122132114674629 0.442 False 1000 +CC 2.0 288 auprc supra_input_output supra_output 0.0631971384788975 0.05933234754454002 0.003864790934357487 0.004235539606082731 -0.001034605772638793 0.010651996817984706 0.118 False 1000 +CC 2.0 288 f_max supra_relative_output supra_relative 0.1838313800517334 0.17406137853913475 0.009770001512598658 0.011582909782713266 0.004835162175676895 0.021727077750226826 0.0 True 1000 +CC 2.0 288 auprc supra_relative_output supra_relative 0.04285708001859051 0.041937928499600215 0.0009191515189902952 0.0007297252013120376 -0.00791764426981235 0.008373701671714561 0.842 False 1000 +CC 2.0 288 f_max supra_relative_output supra_output 0.1838313800517334 0.20872491324071593 -0.02489353318898252 -0.02358429217371319 -0.048791648589844326 0.00048693905774220683 0.052 False 1000 +CC 2.0 288 auprc supra_relative_output supra_output 0.04285708001859051 0.05933234754454002 -0.016475267525949508 -0.01694793789927119 -0.03379276656444611 -0.0029940466156277395 0.02 True 1000 +CC 2.0 288 f_max full supra_input_relative 0.20841152876548033 0.20014085427621897 0.00827067448926136 0.007889985435814664 -0.0010288490030088422 0.018599474303484843 0.098 False 1000 +CC 2.0 288 auprc full supra_input_relative 0.05106006325346844 0.0505428753460521 0.0005171879074163457 0.0005457330211323909 -0.004839542455239672 0.005649043016876954 0.816 False 1000 +CC 2.0 288 f_max full supra_input_output 0.20841152876548033 0.21260013711238593 -0.004188608346905598 -0.004028748672434982 -0.021671406756898146 0.014122788681748278 0.652 False 1000 +CC 2.0 288 auprc full supra_input_output 0.05106006325346844 0.0631971384788975 -0.012137075225429061 -0.01244833498887988 -0.021569271041998316 -0.003915312787968645 0.004 True 1000 +CC 2.0 288 f_max full supra_relative_output 0.20841152876548033 0.1838313800517334 0.02458014871374692 0.02320391325758636 0.0037025486480581084 0.04272179332240621 0.026 True 1000 +CC 2.0 288 auprc full supra_relative_output 0.05106006325346844 0.04285708001859051 0.008202983234877934 0.008735142516474042 -0.0017025264282565721 0.0197430521880688 0.09 False 1000 +CC 2.0 288 f_max single naive 0.20418705801361683 0.27592110996968483 -0.071734051956068 -0.07239490884582424 -0.11727492221223386 -0.029070503197256747 0.0 True 1000 +CC 2.0 288 auprc single naive 0.057550814323517045 0.14642831318147828 -0.08887749885796123 -0.08740965690305234 -0.11804136552041551 -0.05622840932286792 0.0 True 1000 +CC 2.0 288 f_max supra naive 0.20315757312646796 0.27592110996968483 -0.07276353684321687 -0.07374586673003312 -0.11894232162924956 -0.03134203709980915 0.0 True 1000 +CC 2.0 288 auprc supra naive 0.05664095581953725 0.14642831318147828 -0.08978735736194103 -0.0882134089225269 -0.11849360676254797 -0.057504165161128294 0.0 True 1000 +CC 2.0 288 f_max supra_input naive 0.19227286600693066 0.27592110996968483 -0.08364824396275417 -0.08527644884564789 -0.12727568869633857 -0.0455571666554978 0.0 True 1000 +CC 2.0 288 auprc supra_input naive 0.055718082535733425 0.14642831318147828 -0.09071023064574485 -0.0891284857439363 -0.1168974793843485 -0.06121617012761723 0.0 True 1000 +CC 2.0 288 f_max supra_relative naive 0.17406137853913475 0.27592110996968483 -0.10185973143055008 -0.10386518211894849 -0.14368934752522244 -0.06575239380107709 0.0 True 1000 +CC 2.0 288 auprc supra_relative naive 0.041937928499600215 0.14642831318147828 -0.10449038468187807 -0.10367691408907337 -0.12760280607825533 -0.07871547623002399 0.0 True 1000 +CC 2.0 288 f_max supra_output naive 0.20872491324071593 0.27592110996968483 -0.0671961967289689 -0.06869798016252204 -0.11047952876199424 -0.02715838219326824 0.0 True 1000 +CC 2.0 288 auprc supra_output naive 0.05933234754454002 0.14642831318147828 -0.08709596563693826 -0.08599925098849014 -0.11561940246736555 -0.05730089671687069 0.0 True 1000 +CC 2.0 288 f_max supra_input_relative naive 0.20014085427621897 0.27592110996968483 -0.07578025569346586 -0.07696834451446352 -0.12028944380836926 -0.03627555787236931 0.0 True 1000 +CC 2.0 288 auprc supra_input_relative naive 0.0505428753460521 0.14642831318147828 -0.09588543783542619 -0.09475777939241968 -0.12072210177921129 -0.06837168529872072 0.0 True 1000 +CC 2.0 288 f_max supra_input_output naive 0.21260013711238593 0.27592110996968483 -0.0633209728572989 -0.06504961040621388 -0.10756630282839946 -0.022829549966603187 0.0 True 1000 +CC 2.0 288 auprc supra_input_output naive 0.0631971384788975 0.14642831318147828 -0.08323117470258078 -0.08176371138240741 -0.11087331928781965 -0.052527641864815344 0.0 True 1000 +CC 2.0 288 f_max supra_relative_output naive 0.1838313800517334 0.27592110996968483 -0.09208972991795142 -0.09228227233623522 -0.13342398246564113 -0.05301420173356625 0.0 True 1000 +CC 2.0 288 auprc supra_relative_output naive 0.04285708001859051 0.14642831318147828 -0.10357123316288777 -0.10294718888776133 -0.12507681732056378 -0.07947604358071408 0.0 True 1000 +CC 2.0 288 f_max full naive 0.20841152876548033 0.27592110996968483 -0.0675095812042045 -0.06907835907864886 -0.11164701332317001 -0.028567210828058708 0.0 True 1000 +CC 2.0 288 auprc full naive 0.05106006325346844 0.14642831318147828 -0.09536824992800984 -0.0942120463712873 -0.1192390945845958 -0.0678414039024066 0.0 True 1000 +CC 2.0 288 f_max full single 0.20841152876548033 0.20418705801361683 0.004224470751863496 0.0033165497671753673 -0.023407496908620904 0.026221917335539843 0.712 False 1000 +CC 2.0 288 auprc full single 0.05106006325346844 0.057550814323517045 -0.006490751070048602 -0.006802389468234944 -0.02037116588890634 0.005216687472505585 0.286 False 1000 +CC 4.0 202 f_max supra single 0.13060876370294153 0.1310356728406018 -0.00042690913766027516 0.0008149625239812237 -0.008910979138578707 0.012510149893107357 0.936 False 1000 +CC 4.0 202 auprc supra single 0.03129067564188612 0.03256236248146611 -0.00127168683957999 -0.0014848436143586265 -0.00506471472593681 0.0009241368615146235 0.308 False 1000 +CC 4.0 202 f_max supra_input supra 0.1294870590779316 0.13060876370294153 -0.0011217046250099338 -0.004229696634667407 -0.024049837777719685 0.013501436180460371 0.63 False 1000 +CC 4.0 202 auprc supra_input supra 0.03239089050618442 0.03129067564188612 0.0011002148642983017 0.001038477942626906 -0.0046150626760032435 0.006509088111932402 0.67 False 1000 +CC 4.0 202 f_max supra_relative supra 0.06872836842386244 0.13060876370294153 -0.061880395279079084 -0.06616046608162708 -0.10611490616233439 -0.027997412240932464 0.0 True 1000 +CC 4.0 202 auprc supra_relative supra 0.011710666183001015 0.03129067564188612 -0.019580009458885102 -0.020289497638309905 -0.03967087931790851 -0.005504346698262346 0.002 True 1000 +CC 4.0 202 f_max supra_output supra 0.11938378883792156 0.13060876370294153 -0.011224974865019971 -0.01114043155694225 -0.023687629671616344 8.781245496855012e-05 0.054 False 1000 +CC 4.0 202 auprc supra_output supra 0.036629540798907956 0.03129067564188612 0.005338865157021838 0.004589273279537875 -0.0023013828318467463 0.013134087385252813 0.22 False 1000 +CC 4.0 202 f_max supra_input_relative supra_input 0.11704074740568092 0.1294870590779316 -0.012446311672250673 -0.013789511455454307 -0.03070738160567344 0.0011703686230660913 0.07 False 1000 +CC 4.0 202 auprc supra_input_relative supra_input 0.027946086966148243 0.03239089050618442 -0.004444803540036176 -0.004610399692243133 -0.016464016894772088 0.005315178099345718 0.376 False 1000 +CC 4.0 202 f_max supra_input_relative supra_relative 0.11704074740568092 0.06872836842386244 0.04831237898181848 0.04814125799150537 0.015911843352610935 0.08012306801896418 0.006 True 1000 +CC 4.0 202 auprc supra_input_relative supra_relative 0.027946086966148243 0.011710666183001015 0.016235420783147228 0.016717575888693676 0.005294861713877204 0.031042512678575808 0.004 True 1000 +CC 4.0 202 f_max supra_input_output supra_input 0.12533124073004 0.1294870590779316 -0.004155818347891582 -0.0036550691640503652 -0.011315331404053688 0.0026850620921315613 0.332 False 1000 +CC 4.0 202 auprc supra_input_output supra_input 0.035515295230318725 0.03239089050618442 0.0031244047241343054 0.0026206968695134704 -0.0024278311835463882 0.010177318340450095 0.422 False 1000 +CC 4.0 202 f_max supra_input_output supra_output 0.12533124073004 0.11938378883792156 0.005947451892118455 0.0032556657582244776 -0.021219647374154105 0.024031668672765277 0.726 False 1000 +CC 4.0 202 auprc supra_input_output supra_output 0.035515295230318725 0.036629540798907956 -0.001114245568589231 -0.0009300984673974987 -0.008375724689741928 0.006407862955370808 0.816 False 1000 +CC 4.0 202 f_max supra_relative_output supra_relative 0.06701519940569019 0.06872836842386244 -0.0017131690181722559 -0.0015361934012117452 -0.008072470370429779 0.004513977465198175 0.602 False 1000 +CC 4.0 202 auprc supra_relative_output supra_relative 0.013915926101555409 0.011710666183001015 0.0022052599185543934 0.0023016771179036207 -0.00378258605600244 0.011270554426960648 0.536 False 1000 +CC 4.0 202 f_max supra_relative_output supra_output 0.06701519940569019 0.11938378883792156 -0.05236858943223137 -0.05655622792589658 -0.09503724583035703 -0.01805813688361822 0.0 True 1000 +CC 4.0 202 auprc supra_relative_output supra_output 0.013915926101555409 0.036629540798907956 -0.022713614697352547 -0.02257709379994416 -0.045656222229207385 -0.002324568456154377 0.024 True 1000 +CC 4.0 202 f_max full supra_input_relative 0.12338995602996741 0.11704074740568092 0.006349208624286493 0.0033548993282619486 -0.00820947335622468 0.012816177155754251 0.486 False 1000 +CC 4.0 202 auprc full supra_input_relative 0.027627268056174378 0.027946086966148243 -0.0003188189099738656 -0.0004803426949245516 -0.00740536847494708 0.006492443396303753 0.882 False 1000 +CC 4.0 202 f_max full supra_input_output 0.12338995602996741 0.12533124073004 -0.0019412847000725986 -0.006779542963141992 -0.026028475727255158 0.007977729551641667 0.49 False 1000 +CC 4.0 202 auprc full supra_input_output 0.027627268056174378 0.035515295230318725 -0.007888027174144347 -0.007711439256681155 -0.019181544058408533 0.0016794125009099144 0.122 False 1000 +CC 4.0 202 f_max full supra_relative_output 0.12338995602996741 0.06701519940569019 0.056374756624277225 0.05303235072097906 0.02114684951733102 0.08561049424839355 0.0 True 1000 +CC 4.0 202 auprc full supra_relative_output 0.027627268056174378 0.013915926101555409 0.013711341954618969 0.013935556075865507 0.0007410146617407894 0.0276006560724435 0.04 True 1000 +CC 4.0 202 f_max single naive 0.1310356728406018 0.10149307194327688 0.029542600897324925 0.032920078238110446 -0.01324000194001521 0.08089481883943979 0.158 False 1000 +CC 4.0 202 auprc single naive 0.03256236248146611 0.04397862691300466 -0.01141626443153855 -0.009566512835400191 -0.02983142964285966 0.016517252048295968 0.394 False 1000 +CC 4.0 202 f_max supra naive 0.13060876370294153 0.10149307194327688 0.02911569175966465 0.03373504076209167 -0.011377565523286231 0.08123356241518417 0.144 False 1000 +CC 4.0 202 auprc supra naive 0.03129067564188612 0.04397862691300466 -0.01268795127111854 -0.011051356449758817 -0.030570981934231637 0.014154246782051677 0.322 False 1000 +CC 4.0 202 f_max supra_input naive 0.1294870590779316 0.10149307194327688 0.027993987134654716 0.029505344127424265 -0.013644234037860222 0.07534330378465041 0.212 False 1000 +CC 4.0 202 auprc supra_input naive 0.03239089050618442 0.04397862691300466 -0.011587736406820238 -0.01001287850713191 -0.030046363296252578 0.015101797238111782 0.37 False 1000 +CC 4.0 202 f_max supra_relative naive 0.06872836842386244 0.10149307194327688 -0.032764703519414434 -0.032425425319535406 -0.06984777928369755 0.008839388642785509 0.112 False 1000 +CC 4.0 202 auprc supra_relative naive 0.011710666183001015 0.04397862691300466 -0.03226796073000364 -0.03134085408806873 -0.04383713728669182 -0.01460420845334463 0.0 True 1000 +CC 4.0 202 f_max supra_output naive 0.11938378883792156 0.10149307194327688 0.01789071689464468 0.022594609205149423 -0.018667500851358185 0.0670362189022185 0.286 False 1000 +CC 4.0 202 auprc supra_output naive 0.036629540798907956 0.04397862691300466 -0.007349086114096702 -0.006462083170220942 -0.02858909576496848 0.020884542899209067 0.566 False 1000 +CC 4.0 202 f_max supra_input_relative naive 0.11704074740568092 0.10149307194327688 0.015547675462404043 0.015715832671969958 -0.02544879036550782 0.059758500492605136 0.486 False 1000 +CC 4.0 202 auprc supra_input_relative naive 0.027946086966148243 0.04397862691300466 -0.016032539946856415 -0.014623278199375046 -0.03366002823556357 0.009426937134648808 0.2 False 1000 +CC 4.0 202 f_max supra_input_output naive 0.12533124073004 0.10149307194327688 0.023838168786763134 0.025850274963373897 -0.01767086440632318 0.07084809458940183 0.258 False 1000 +CC 4.0 202 auprc supra_input_output naive 0.035515295230318725 0.04397862691300466 -0.008463331682685933 -0.007392181637618441 -0.029577087141730504 0.01853019938684503 0.514 False 1000 +CC 4.0 202 f_max supra_relative_output naive 0.06701519940569019 0.10149307194327688 -0.03447787253758669 -0.033961618720747155 -0.07006360192925574 0.0048954922258761776 0.104 False 1000 +CC 4.0 202 auprc supra_relative_output naive 0.013915926101555409 0.04397862691300466 -0.03006270081144925 -0.0290391769701651 -0.04364108683030369 -0.008383005383078092 0.004 True 1000 +CC 4.0 202 f_max full naive 0.12338995602996741 0.10149307194327688 0.021896884086690535 0.01907073200023191 -0.022486455921009587 0.06447412874860918 0.406 False 1000 +CC 4.0 202 auprc full naive 0.027627268056174378 0.04397862691300466 -0.01635135885683028 -0.015103620894299596 -0.03406326706799786 0.007120679570462745 0.188 False 1000 +CC 4.0 202 f_max full single 0.12338995602996741 0.1310356728406018 -0.00764571681063439 -0.01384934623787854 -0.0410342983417809 0.008404687538050777 0.244 False 1000 +CC 4.0 202 auprc full single 0.027627268056174378 0.03256236248146611 -0.00493509442529173 -0.005537108058899405 -0.01941036403716825 0.00619300333041458 0.368 False 1000 diff --git a/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_provenance.tsv b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_provenance.tsv new file mode 100644 index 0000000..d125c93 --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_provenance.tsv @@ -0,0 +1,16 @@ +rung label adds predictions_file n_rows n_domains score interpro_file interpro_sha256 cohort_interpro_file cohort_interpro_sha256 t0_gaf_file t0_gaf_sha256 t1_gaf_file t1_gaf_sha256 +single single domains - /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_significant.tsv 37536 5114 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +single__p single domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/single/domain_go_associations_significant.tsv 37536 5114 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra + supra-domains supra-domains /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_significant.tsv 144823 23113 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra__p + supra-domains (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra/domain_go_associations_significant.tsv 144823 23113 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input + input propagation input propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_significant.tsv 436620 23379 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input__p + input propagation (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input/domain_go_associations_significant.tsv 436620 23379 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative + relative inference relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_significant.tsv 28282 9440 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative__p + relative inference (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative/domain_go_associations_significant.tsv 28282 9440 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_output + output propagation output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_output/domain_go_annotations_propagated.tsv 979440 23113 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative input + relative input propagation + relative inference /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_significant.tsv 85285 10392 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_relative__p input + relative (p-ranked) sensitivity: score column only /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_relative/domain_go_associations_significant.tsv 85285 10392 -log10(p) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_input_output input + output input + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_input_output/domain_go_annotations_propagated.tsv 1064167 23379 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +supra_relative_output relative + output relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/supra_relative_output/domain_go_annotations_propagated.tsv 152781 9440 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +full full current method input propagation + relative inference + output propagation /home/danbolser/Build/dcGO-2.0/results/ablation-t0interpro/ipr_manual/full/domain_go_annotations_propagated.tsv 255986 10392 -log10(q) /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d +naive CAFA naive baseline - - 19299 0 propagated t0 term frequency /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c /home/danbolser/Build/dcGO-2.0/data/interim/protein2ipr_human_t0_ipr85.dat.gz b99bbafcbc33568c7dd6fb6dac49412ac7ffafaa69261b807218bf971316f28c data/raw/goa_archive/goa_human.gaf.205.gz 69ae7d90743400c6519a6bfe3a1b1578476585f151de39323bfd890cbae13d86 data/raw/goa_annotations/goa_human.gaf.gz 14a33b8d5fcf7bddcdd1710e154f9642e33412ef9c962901ee76891d6ea82d7d diff --git a/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_selection_counts.tsv b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_selection_counts.tsv new file mode 100644 index 0000000..b117a1d --- /dev/null +++ b/validation/ablation_cells/ipr_manual_t0interpro_owncohort/ablation_selection_counts.tsv @@ -0,0 +1,19 @@ +stage aspect min_ic n_proteins n_dropped_vs_ic0 pct_of_ic0 note +t0_annotated_proteins - - 18735 - - proteins with >=1 GO annotation at t0 under the training evidence filter (the training universe) +t1_experimental_proteins - - 16362 - - proteins with >=1 experimental GO annotation at t1 (the gold standard) +proteins_with_domains - - 19393 - - proteins with >=1 InterPro domain (predictable at all by a domain method) +no_knowledge_candidates BP - 310 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains BP - 280 - - the scored cohort before any IC floor +no_knowledge_candidates MF - 418 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains MF - 392 - - the scored cohort before any IC floor +no_knowledge_candidates CC - 453 - - no t0 knowledge in this aspect, gained t1 experimental terms +no_knowledge_with_domains CC - 422 - - the scored cohort before any IC floor +cohort_at_ic_floor BP 0 280 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 2 280 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor BP 4 276 4 98.6 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 0 392 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 2 151 241 38.5 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor MF 4 145 247 37.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 0 422 0 100.0 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 2 288 134 68.2 proteins whose truth is non-empty at this IC floor +cohort_at_ic_floor CC 4 202 220 47.9 proteins whose truth is non-empty at this IC floor diff --git a/validation/ablation_cells/t0_interpro_comparison.tsv b/validation/ablation_cells/t0_interpro_comparison.tsv new file mode 100644 index 0000000..7261313 --- /dev/null +++ b/validation/ablation_cells/t0_interpro_comparison.tsv @@ -0,0 +1,271 @@ +aspect min_ic n_eval_proteins arm_a arm_b method metric observed_a observed_b observed_diff diff_mean diff_ci_lo diff_ci_hi p_value significant n_replicates +BP 0.0 298 selctrl baseline supra_input f_max 0.24391948612898903 0.24291253308477817 0.001006953044210862 0.0006825709267321811 -0.0005106600229348949 0.0020525581286017372 0.306 False 1000 +BP 0.0 298 selctrl baseline supra_input auprc 0.12031130066802086 0.12017471152083997 0.000136589147180885 0.00013701606600465465 -0.0001246397871701137 0.00039313758552268725 0.294 False 1000 +BP 0.0 298 selctrl baseline supra_input_relative f_max 0.21161694252967744 0.21215346671089524 -0.0005365241812178012 -0.0004829671420987096 -0.0013065011018763325 0.0002829306873832899 0.158 False 1000 +BP 0.0 298 selctrl baseline supra_input_relative auprc 0.09092431931044631 0.09119041187089935 -0.00026609256045304064 -0.0002563402462846975 -0.0007685142120837103 0.00013704578499117972 0.234 False 1000 +BP 0.0 298 selctrl baseline supra_input_output f_max 0.24970556786024375 0.24841117122371198 0.001294396636531775 0.0011245933535032448 -0.00040585702467276977 0.0032888721278286465 0.222 False 1000 +BP 0.0 298 selctrl baseline supra_input_output auprc 0.12990810902139693 0.12967768600592786 0.00023042301546907784 0.00024211402036414253 -0.0004023910427596297 0.001001316004499226 0.516 False 1000 +BP 0.0 298 selctrl baseline full f_max 0.22006836976335684 0.22046565516471517 -0.0003972854013583327 -0.00044220963808840974 -0.001195270507210397 0.00020618452284860682 0.142 False 1000 +BP 0.0 298 selctrl baseline full auprc 0.09730776842727502 0.0976470708721623 -0.00033930244488727945 -0.0003360891118908391 -0.0008334648531446989 5.308143775858342e-05 0.118 False 1000 +BP 0.0 298 selctrl baseline naive f_max 0.24522800676034753 0.24522800676034753 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 0.0 298 selctrl baseline naive auprc 0.1548729085557252 0.1548729085557252 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 0.0 298 t0 selctrl supra_input f_max 0.23795056705598774 0.24391948612898903 -0.005968919073001289 -0.0061054497178163305 -0.018304554478768234 0.004921096565806069 0.284 False 1000 +BP 0.0 298 t0 selctrl supra_input auprc 0.11319817605149689 0.12031130066802086 -0.007113124616523969 -0.006875405050702039 -0.01674538879193624 0.0015306750435839075 0.118 False 1000 +BP 0.0 298 t0 selctrl supra_input_relative f_max 0.20953063532557034 0.21161694252967744 -0.002086307204107102 -0.003010279287533016 -0.017991839540687376 0.008960272280914075 0.648 False 1000 +BP 0.0 298 t0 selctrl supra_input_relative auprc 0.08534064572151016 0.09092431931044631 -0.005583673588936147 -0.00552369403072173 -0.01420143682981762 0.0014145957103483926 0.166 False 1000 +BP 0.0 298 t0 selctrl supra_input_output f_max 0.24542121830129587 0.24970556786024375 -0.004284349558947881 -0.004919128469338427 -0.016524640147256187 0.0058327953698688395 0.386 False 1000 +BP 0.0 298 t0 selctrl supra_input_output auprc 0.12250816012290822 0.12990810902139693 -0.007399948898488717 -0.00711871250096937 -0.01780926468941398 0.002745172774714495 0.152 False 1000 +BP 0.0 298 t0 selctrl full f_max 0.21359025032890977 0.22006836976335684 -0.006478119434447066 -0.006391307435302613 -0.020670640921367514 0.0052064108925916515 0.32 False 1000 +BP 0.0 298 t0 selctrl full auprc 0.09146101838127958 0.09730776842727502 -0.005846750045995441 -0.005790887350607555 -0.01590987071852167 0.0023447485185964195 0.192 False 1000 +BP 0.0 298 t0 selctrl naive f_max 0.24522800676034753 0.24522800676034753 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 0.0 298 t0 selctrl naive auprc 0.1548729085557252 0.1548729085557252 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 0.0 298 t0 baseline supra_input f_max 0.23795056705598774 0.24291253308477817 -0.004961966028790427 -0.0054228787910841495 -0.017390019703048234 0.005550999412067338 0.358 False 1000 +BP 0.0 298 t0 baseline supra_input auprc 0.11319817605149689 0.12017471152083997 -0.006976535469343084 -0.006738388984697385 -0.016680812380636404 0.001713411114400694 0.132 False 1000 +BP 0.0 298 t0 baseline supra_input_relative f_max 0.20953063532557034 0.21215346671089524 -0.002622831385324903 -0.0034932464296317254 -0.018080377074880194 0.008551443422512415 0.572 False 1000 +BP 0.0 298 t0 baseline supra_input_relative auprc 0.08534064572151016 0.09119041187089935 -0.005849766149389188 -0.005780034277006428 -0.014731387777890678 0.0011921153622218936 0.142 False 1000 +BP 0.0 298 t0 baseline supra_input_output f_max 0.24542121830129587 0.24841117122371198 -0.002989952922416106 -0.003794535115835182 -0.01605556070374376 0.007038977661748766 0.548 False 1000 +BP 0.0 298 t0 baseline supra_input_output auprc 0.12250816012290822 0.12967768600592786 -0.0071695258830196396 -0.006876598480605227 -0.017770888652576516 0.0027952998844535944 0.166 False 1000 +BP 0.0 298 t0 baseline full f_max 0.21359025032890977 0.22046565516471517 -0.006875404835805399 -0.006833517073391023 -0.021305679647670283 0.004765413053107963 0.29 False 1000 +BP 0.0 298 t0 baseline full auprc 0.09146101838127958 0.0976470708721623 -0.006186052490882721 -0.006126976462498395 -0.016334515081556457 0.0020767896510296173 0.166 False 1000 +BP 0.0 298 t0 baseline naive f_max 0.24522800676034753 0.24522800676034753 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 0.0 298 t0 baseline naive auprc 0.1548729085557252 0.1548729085557252 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 selctrl baseline supra_input f_max 0.19299661151865918 0.1926785041720935 0.00031810734656567696 0.0006626409838661962 -0.00029317216251641473 0.0029732968371849063 0.366 False 1000 +BP 2.0 298 selctrl baseline supra_input auprc 0.07592523255095349 0.0758467729547912 7.845959616228915e-05 7.725810455953002e-05 -0.0002238814142691045 0.000383152937112946 0.62 False 1000 +BP 2.0 298 selctrl baseline supra_input_relative f_max 0.1590048692983248 0.1593493384500786 -0.0003444691517537901 -0.000489327852548689 -0.0016596162578589259 0.0001085326889401494 0.164 False 1000 +BP 2.0 298 selctrl baseline supra_input_relative auprc 0.053680719886573844 0.05394327998613709 -0.0002625600995632485 -0.00025601784824709825 -0.0005441377374458471 -4.855051775046202e-06 0.046 True 1000 +BP 2.0 298 selctrl baseline supra_input_output f_max 0.19290019525631774 0.19098936123387492 0.0019108340224428255 0.0018472018103989616 0.00014992286499216235 0.00421960495971795 0.034 True 1000 +BP 2.0 298 selctrl baseline supra_input_output auprc 0.08150147032223701 0.08173445253549405 -0.0002329822132570436 -0.00019716748872008373 -0.0010123253658007054 0.00041166789849383234 0.56 False 1000 +BP 2.0 298 selctrl baseline full f_max 0.15848590904470627 0.15880298014062047 -0.00031707109591419536 -0.0004345143809888222 -0.0014008812862633896 0.00014637879892431666 0.136 False 1000 +BP 2.0 298 selctrl baseline full auprc 0.054411779329926364 0.0545741843122555 -0.00016240498232913264 -0.00018357941986890658 -0.0005171444236317285 9.407463962404491e-05 0.188 False 1000 +BP 2.0 298 selctrl baseline naive f_max 0.1393754232163564 0.1393754232163564 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 selctrl baseline naive auprc 0.05246757199300765 0.05246757199300765 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 t0 selctrl supra_input f_max 0.18709980332277934 0.19299661151865918 -0.0058968081958798435 -0.0044783572881325565 -0.014721735875452105 0.005926002232643244 0.372 False 1000 +BP 2.0 298 t0 selctrl supra_input auprc 0.06923837639082926 0.07592523255095349 -0.006686856160124224 -0.006526525785285096 -0.015522685568820475 0.0007642206819952388 0.09 False 1000 +BP 2.0 298 t0 selctrl supra_input_relative f_max 0.15601748308402721 0.1590048692983248 -0.002987386214297588 -0.0027971889325956924 -0.014527539229858798 0.007455663089683592 0.632 False 1000 +BP 2.0 298 t0 selctrl supra_input_relative auprc 0.048512655192598404 0.053680719886573844 -0.00516806469397544 -0.004989983728756228 -0.012280663814749716 0.0010266010890097146 0.108 False 1000 +BP 2.0 298 t0 selctrl supra_input_output f_max 0.18832348586599781 0.19290019525631774 -0.004576709390319927 -0.0044665938486428075 -0.014711102996198218 0.00699990986545833 0.42 False 1000 +BP 2.0 298 t0 selctrl supra_input_output auprc 0.07413412064090152 0.08150147032223701 -0.007367349681335483 -0.007091070018633754 -0.017398945539464958 0.0020606825464263225 0.154 False 1000 +BP 2.0 298 t0 selctrl full f_max 0.15384527790027153 0.15848590904470627 -0.004640631144434748 -0.0043329494781153294 -0.01617617712924306 0.006527386737211411 0.442 False 1000 +BP 2.0 298 t0 selctrl full auprc 0.04745326817925877 0.054411779329926364 -0.006958511150667597 -0.0068264770892749165 -0.015769907972665614 0.00011618303570968312 0.06 False 1000 +BP 2.0 298 t0 selctrl naive f_max 0.1393754232163564 0.1393754232163564 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 t0 selctrl naive auprc 0.05246757199300765 0.05246757199300765 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 t0 baseline supra_input f_max 0.18709980332277934 0.1926785041720935 -0.0055787008493141665 -0.00381571630426636 -0.014437836214353968 0.0067810906300614536 0.442 False 1000 +BP 2.0 298 t0 baseline supra_input auprc 0.06923837639082926 0.0758467729547912 -0.006608396563961935 -0.006449267680725565 -0.01529705393767517 0.000886256315009147 0.094 False 1000 +BP 2.0 298 t0 baseline supra_input_relative f_max 0.15601748308402721 0.1593493384500786 -0.0033318553660513783 -0.003286516785144381 -0.015052809659498664 0.006993046600340406 0.566 False 1000 +BP 2.0 298 t0 baseline supra_input_relative auprc 0.048512655192598404 0.05394327998613709 -0.005430624793538688 -0.005246001577003326 -0.012541755872177946 0.0007867777443048014 0.098 False 1000 +BP 2.0 298 t0 baseline supra_input_output f_max 0.18832348586599781 0.19098936123387492 -0.002665875367877102 -0.002619392038243846 -0.013500701244152152 0.008881509234808503 0.622 False 1000 +BP 2.0 298 t0 baseline supra_input_output auprc 0.07413412064090152 0.08173445253549405 -0.007600331894592527 -0.007288237507353839 -0.017766836036733458 0.0018660369613937103 0.15 False 1000 +BP 2.0 298 t0 baseline full f_max 0.15384527790027153 0.15880298014062047 -0.0049577022403489435 -0.004767463859104152 -0.01667006027927278 0.005890808816060489 0.402 False 1000 +BP 2.0 298 t0 baseline full auprc 0.04745326817925877 0.0545741843122555 -0.00712091613299673 -0.007010056509143823 -0.016168686852608534 5.6518381498498506e-05 0.052 False 1000 +BP 2.0 298 t0 baseline naive f_max 0.1393754232163564 0.1393754232163564 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 2.0 298 t0 baseline naive auprc 0.05246757199300765 0.05246757199300765 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 selctrl baseline supra_input f_max 0.13785042940917663 0.1353725463113433 0.002477883097833339 0.001169128283722004 -0.00031183177155790414 0.004081180766757322 0.448 False 1000 +BP 4.0 292 selctrl baseline supra_input auprc 0.036198295212282634 0.03620825930606598 -9.964093783343397e-06 -1.8664952071713896e-05 -0.0002791171348433351 0.0001833252146716981 0.922 False 1000 +BP 4.0 292 selctrl baseline supra_input_relative f_max 0.093217666191155 0.09334110495918051 -0.0001234387680255078 -0.00015840496246174668 -0.0010875658226013822 0.000347384666698316 0.658 False 1000 +BP 4.0 292 selctrl baseline supra_input_relative auprc 0.019850976304688597 0.019904231320105756 -5.3255015417159185e-05 -5.9758020511090194e-05 -0.00023168171565636744 9.297368195055827e-05 0.42 False 1000 +BP 4.0 292 selctrl baseline supra_input_output f_max 0.131901838297952 0.13193626185180712 -3.442355385513407e-05 0.0006220919769639588 -0.0015083166557014608 0.002468037222545681 0.486 False 1000 +BP 4.0 292 selctrl baseline supra_input_output auprc 0.03637360789031955 0.03636362273250887 9.985157810678269e-06 1.1085020117236554e-05 -0.00040603378892110524 0.0003545137022553364 0.886 False 1000 +BP 4.0 292 selctrl baseline full f_max 0.09090782814931803 0.09097113641583583 -6.330826651780586e-05 -0.00014374967531339473 -0.0006237316109388688 0.00022038455965091118 0.364 False 1000 +BP 4.0 292 selctrl baseline full auprc 0.02187984746106082 0.021781587760909608 9.825970015121116e-05 8.431753200096356e-05 -0.0001407046524909964 0.0003693731380691634 0.51 False 1000 +BP 4.0 292 selctrl baseline naive f_max 0.04686917580427525 0.04686917580427525 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 selctrl baseline naive auprc 0.013722538414023608 0.013722538414023608 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 t0 selctrl supra_input f_max 0.13808913314840468 0.13785042940917663 0.00023870373922804666 0.0004389314939385483 -0.005983283727984657 0.00728196857406836 0.918 False 1000 +BP 4.0 292 t0 selctrl supra_input auprc 0.03501200013295161 0.036198295212282634 -0.0011862950793310248 -0.001061587814343561 -0.004887536760224121 0.001982352495400792 0.528 False 1000 +BP 4.0 292 t0 selctrl supra_input_relative f_max 0.09412487118656662 0.093217666191155 0.0009072049954116229 0.0009057256986354 -0.008205193113254813 0.009292654282745038 0.816 False 1000 +BP 4.0 292 t0 selctrl supra_input_relative auprc 0.01917880743576971 0.019850976304688597 -0.0006721688689188855 -0.000603416591660623 -0.003212839137312901 0.0015797808449215568 0.68 False 1000 +BP 4.0 292 t0 selctrl supra_input_output f_max 0.13286708464632122 0.131901838297952 0.0009652463483692253 -0.00035197868948133625 -0.009819638410151163 0.008538650087813342 0.94 False 1000 +BP 4.0 292 t0 selctrl supra_input_output auprc 0.034378134445179405 0.03637360789031955 -0.0019954734451401446 -0.0020192941533390876 -0.006065489087014536 0.0013550815757707802 0.242 False 1000 +BP 4.0 292 t0 selctrl full f_max 0.09256562477377679 0.09090782814931803 0.0016577966244587655 0.0006783387054324163 -0.007731744302391657 0.008123059446304986 0.824 False 1000 +BP 4.0 292 t0 selctrl full auprc 0.019870938340269942 0.02187984746106082 -0.002008909120790877 -0.0019902453169527703 -0.006183206024254851 0.0012077942979732753 0.268 False 1000 +BP 4.0 292 t0 selctrl naive f_max 0.04686917580427525 0.04686917580427525 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 t0 selctrl naive auprc 0.013722538414023608 0.013722538414023608 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 t0 baseline supra_input f_max 0.13808913314840468 0.1353725463113433 0.0027165868370613855 0.0016080597776605523 -0.005519318626568451 0.008601286278947318 0.602 False 1000 +BP 4.0 292 t0 baseline supra_input auprc 0.03501200013295161 0.03620825930606598 -0.0011962591731143682 -0.001080252766415275 -0.004896752129068515 0.002039015163480907 0.532 False 1000 +BP 4.0 292 t0 baseline supra_input_relative f_max 0.09412487118656662 0.09334110495918051 0.0007837662273861151 0.0007473207361736534 -0.008476923881733569 0.009169086600077649 0.866 False 1000 +BP 4.0 292 t0 baseline supra_input_relative auprc 0.01917880743576971 0.019904231320105756 -0.0007254238843360447 -0.0006631746121717132 -0.0034868879308571294 0.0015690150045006906 0.658 False 1000 +BP 4.0 292 t0 baseline supra_input_output f_max 0.13286708464632122 0.13193626185180712 0.0009308227945140912 0.0002701132874826227 -0.009354671912882149 0.008769189236021954 0.924 False 1000 +BP 4.0 292 t0 baseline supra_input_output auprc 0.034378134445179405 0.03636362273250887 -0.0019854882873294663 -0.002008209133221851 -0.00612064318311117 0.0014103688911559446 0.264 False 1000 +BP 4.0 292 t0 baseline full f_max 0.09256562477377679 0.09097113641583583 0.0015944883579409597 0.0005345890301190215 -0.007960270621438434 0.007926156294186436 0.862 False 1000 +BP 4.0 292 t0 baseline full auprc 0.019870938340269942 0.021781587760909608 -0.0019106494206396658 -0.001905927784951807 -0.006023652968334124 0.0012141912895986783 0.28 False 1000 +BP 4.0 292 t0 baseline naive f_max 0.04686917580427525 0.04686917580427525 0.0 0.0 0.0 0.0 1.0 False 1000 +BP 4.0 292 t0 baseline naive auprc 0.013722538414023608 0.013722538414023608 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 selctrl baseline supra_input f_max 0.3655787927448233 0.3631916024226839 0.0023871903221394097 0.0019511734652354394 -0.0005587783810301194 0.005507803689709649 0.162 False 1000 +MF 0.0 406 selctrl baseline supra_input auprc 0.17602961321469285 0.17454437065059558 0.0014852425640972666 0.001471819715400827 0.00019107938362454782 0.0028089027395019305 0.022 True 1000 +MF 0.0 406 selctrl baseline supra_input_relative f_max 0.36075231398177715 0.3547497172674006 0.006002596714376529 0.005599477433561141 -0.0014517772779820731 0.013678378615660477 0.138 False 1000 +MF 0.0 406 selctrl baseline supra_input_relative auprc 0.19859411226663576 0.19628060872201042 0.0023135035446253394 0.002330592346038492 0.000558989061779193 0.004436653299122109 0.008 True 1000 +MF 0.0 406 selctrl baseline supra_input_output f_max 0.3712904130952929 0.3687724328869243 0.002517980208368631 0.0016161910424483815 -0.0008023533609909234 0.005494941683319161 0.338 False 1000 +MF 0.0 406 selctrl baseline supra_input_output auprc 0.19631964007936464 0.1950567828504962 0.0012628572288684303 0.0012427635105743675 0.00025893183321465987 0.002365958622872238 0.012 True 1000 +MF 0.0 406 selctrl baseline full f_max 0.38274701944288786 0.3844201304669909 -0.0016731110241030578 -0.0012627421855386234 -0.004958174093103538 0.0020254248742065167 0.456 False 1000 +MF 0.0 406 selctrl baseline full auprc 0.21842592506441955 0.2166443535881556 0.0017815714762639545 0.0017708984415780256 0.0004098001557408259 0.00327692282768201 0.01 True 1000 +MF 0.0 406 selctrl baseline naive f_max 0.7189989169653068 0.7189989169653068 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 selctrl baseline naive auprc 0.4190445836593093 0.4190445836593093 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 t0 selctrl supra_input f_max 0.36037094536457454 0.3655787927448233 -0.005207847380248765 -0.004743425684637524 -0.01352896360262681 0.0030792810322429896 0.288 False 1000 +MF 0.0 406 t0 selctrl supra_input auprc 0.17171632950127838 0.17602961321469285 -0.0043132837134144675 -0.004319431735169463 -0.00825679135976775 -0.0008812395852940464 0.01 True 1000 +MF 0.0 406 t0 selctrl supra_input_relative f_max 0.3569474047131703 0.36075231398177715 -0.0038049092686068264 -0.004879559020151501 -0.015929058053747464 0.004265860836275462 0.342 False 1000 +MF 0.0 406 t0 selctrl supra_input_relative auprc 0.1908100853598182 0.19859411226663576 -0.007784026906817565 -0.007741108012857684 -0.01593471026118591 -0.0012295141499030883 0.012 True 1000 +MF 0.0 406 t0 selctrl supra_input_output f_max 0.36076776543325934 0.3712904130952929 -0.01052264766203359 -0.009805576242129195 -0.017354019398474474 -0.0018358393394402412 0.012 True 1000 +MF 0.0 406 t0 selctrl supra_input_output auprc 0.1899294765403534 0.19631964007936464 -0.0063901635390112566 -0.00641624249572299 -0.012906087321227567 -0.0010946304783971312 0.016 True 1000 +MF 0.0 406 t0 selctrl full f_max 0.37612656480454426 0.38274701944288786 -0.006620454638343598 -0.006405609863353408 -0.016900809723151517 0.0034292819569281424 0.224 False 1000 +MF 0.0 406 t0 selctrl full auprc 0.21123115567352646 0.21842592506441955 -0.007194769390893085 -0.007185001686889974 -0.014768501208921493 -0.0011181789455723258 0.018 True 1000 +MF 0.0 406 t0 selctrl naive f_max 0.7189989169653068 0.7189989169653068 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 t0 selctrl naive auprc 0.4190445836593093 0.4190445836593093 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 t0 baseline supra_input f_max 0.36037094536457454 0.3631916024226839 -0.002820657058109355 -0.002792252219402085 -0.011006606258331785 0.00531934929031257 0.5 False 1000 +MF 0.0 406 t0 baseline supra_input auprc 0.17171632950127838 0.17454437065059558 -0.002828041149317201 -0.0028476120197686353 -0.00719368139325409 0.000927094245711314 0.166 False 1000 +MF 0.0 406 t0 baseline supra_input_relative f_max 0.3569474047131703 0.3547497172674006 0.0021976874457697027 0.0007199184134096408 -0.011341869925989687 0.012778944361438038 0.894 False 1000 +MF 0.0 406 t0 baseline supra_input_relative auprc 0.1908100853598182 0.19628060872201042 -0.005470523362192226 -0.005410515666819193 -0.014246627414463234 0.0012730218560617814 0.156 False 1000 +MF 0.0 406 t0 baseline supra_input_output f_max 0.36076776543325934 0.3687724328869243 -0.00800466745366496 -0.008189385199680814 -0.016373471467431965 -3.476721373359196e-06 0.048 True 1000 +MF 0.0 406 t0 baseline supra_input_output auprc 0.1899294765403534 0.1950567828504962 -0.005127306310142826 -0.005173478985148623 -0.011829125593604977 3.721385845588667e-05 0.052 False 1000 +MF 0.0 406 t0 baseline full f_max 0.37612656480454426 0.3844201304669909 -0.008293565662446656 -0.007668352048892033 -0.017900155639381733 0.0019048297291908898 0.13 False 1000 +MF 0.0 406 t0 baseline full auprc 0.21123115567352646 0.2166443535881556 -0.005413197914629131 -0.005414103245311948 -0.013584224433898267 0.0009277909400999084 0.11 False 1000 +MF 0.0 406 t0 baseline naive f_max 0.7189989169653068 0.7189989169653068 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 0.0 406 t0 baseline naive auprc 0.4190445836593093 0.4190445836593093 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 selctrl baseline supra_input f_max 0.35670234664936507 0.3564290769327837 0.0002732697165813702 3.6000610206508025e-05 -0.0023885362178341155 0.0018759623086809477 0.776 False 1000 +MF 2.0 159 selctrl baseline supra_input auprc 0.21789011079771334 0.2182883752598947 -0.000398264462181358 -0.00034512014006605395 -0.0008142830210475362 6.403226414353402e-05 0.128 False 1000 +MF 2.0 159 selctrl baseline supra_input_relative f_max 0.30838965519130074 0.3100201127981691 -0.001630457606868363 -0.0005232044148982275 -0.004235223369172197 0.0009453975949989117 1.0 False 1000 +MF 2.0 159 selctrl baseline supra_input_relative auprc 0.1712713104376412 0.17072673291029536 0.000544577527345852 0.0004965057219456862 -0.0003308847664301263 0.0017669896238265222 0.282 False 1000 +MF 2.0 159 selctrl baseline supra_input_output f_max 0.3509584986613415 0.3509584986613415 0.0 -1.2956402733920846e-05 0.0 0.0 1.0 False 1000 +MF 2.0 159 selctrl baseline supra_input_output auprc 0.2220098622274635 0.22273791854479633 -0.0007280563173328314 -0.0007745575163466978 -0.00164062572483294 -0.00016102867845972725 0.002 True 1000 +MF 2.0 159 selctrl baseline full f_max 0.31749948602842004 0.31776235662993024 -0.00026287060151020114 -0.0006071170478486416 -0.0034363581477923825 0.0 0.272 False 1000 +MF 2.0 159 selctrl baseline full auprc 0.16934947196131758 0.16951885671163683 -0.00016938475031924582 -0.0001479242844804295 -0.0005664503941691902 0.0002499587243189283 0.41 False 1000 +MF 2.0 159 selctrl baseline naive f_max 0.07561162501367721 0.07561162501367721 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 selctrl baseline naive auprc 0.026528609897190324 0.026528609897190324 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 t0 selctrl supra_input f_max 0.3351947698693985 0.35670234664936507 -0.021507576779966564 -0.020261238852865623 -0.03765588625995766 -0.004065598825238514 0.002 True 1000 +MF 2.0 159 t0 selctrl supra_input auprc 0.2080034865452522 0.21789011079771334 -0.009886624252461146 -0.009936519287265523 -0.0192801555411068 -0.002336500223537058 0.014 True 1000 +MF 2.0 159 t0 selctrl supra_input_relative f_max 0.28809539534087936 0.30838965519130074 -0.02029425985042138 -0.02090751739446558 -0.04211672803047556 -0.002997244488464833 0.016 True 1000 +MF 2.0 159 t0 selctrl supra_input_relative auprc 0.153532150824977 0.1712713104376412 -0.0177391596126642 -0.017445627225983366 -0.03691537409821943 -0.0002456293300033268 0.044 True 1000 +MF 2.0 159 t0 selctrl supra_input_output f_max 0.3279453161295411 0.3509584986613415 -0.023013182531800402 -0.022216430165731733 -0.03945152287085463 -0.006884327393333081 0.002 True 1000 +MF 2.0 159 t0 selctrl supra_input_output auprc 0.20413914043745163 0.2220098622274635 -0.017870721790011868 -0.017718992720193883 -0.034440929902004785 -0.004171670095312816 0.012 True 1000 +MF 2.0 159 t0 selctrl full f_max 0.2973310593460379 0.31749948602842004 -0.020168426682382168 -0.02032425227513248 -0.04166400407725493 -0.0016167922003535794 0.03 True 1000 +MF 2.0 159 t0 selctrl full auprc 0.15168797716709995 0.16934947196131758 -0.017661494794217636 -0.017433003506431642 -0.033892278636051064 -0.003573298434897598 0.01 True 1000 +MF 2.0 159 t0 selctrl naive f_max 0.07561162501367721 0.07561162501367721 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 t0 selctrl naive auprc 0.026528609897190324 0.026528609897190324 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 t0 baseline supra_input f_max 0.3351947698693985 0.3564290769327837 -0.021234307063385194 -0.020225238242659118 -0.037590010466104634 -0.004724441824031733 0.002 True 1000 +MF 2.0 159 t0 baseline supra_input auprc 0.2080034865452522 0.2182883752598947 -0.010284888714642504 -0.010281639427331579 -0.019370700719363834 -0.002449648710497713 0.01 True 1000 +MF 2.0 159 t0 baseline supra_input_relative f_max 0.28809539534087936 0.3100201127981691 -0.021924717457289744 -0.0214307218093638 -0.042797088890452195 -0.0037624038043718444 0.012 True 1000 +MF 2.0 159 t0 baseline supra_input_relative auprc 0.153532150824977 0.17072673291029536 -0.01719458208531835 -0.016949121504037677 -0.036850816600008725 0.00010978884190162665 0.058 False 1000 +MF 2.0 159 t0 baseline supra_input_output f_max 0.3279453161295411 0.3509584986613415 -0.023013182531800402 -0.022229386568465657 -0.03945152287085463 -0.0067321800401861295 0.002 True 1000 +MF 2.0 159 t0 baseline supra_input_output auprc 0.20413914043745163 0.22273791854479633 -0.0185987781073447 -0.01849355023654058 -0.03545084735930729 -0.004682904853958196 0.006 True 1000 +MF 2.0 159 t0 baseline full f_max 0.2973310593460379 0.31776235662993024 -0.02043129728389237 -0.02093136932298112 -0.04239995643151849 -0.0022794800652085998 0.026 True 1000 +MF 2.0 159 t0 baseline full auprc 0.15168797716709995 0.16951885671163683 -0.01783087954453688 -0.01758092779091207 -0.03412390157567934 -0.0038984005922727983 0.01 True 1000 +MF 2.0 159 t0 baseline naive f_max 0.07561162501367721 0.07561162501367721 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 2.0 159 t0 baseline naive auprc 0.026528609897190324 0.026528609897190324 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 selctrl baseline supra_input f_max 0.33803499974566886 0.3379901625145267 4.4837231142180745e-05 -0.00021660217177162138 -0.003354097074580441 0.0010131759922114571 1.0 False 1000 +MF 4.0 151 selctrl baseline supra_input auprc 0.19917894898248917 0.19967306357084127 -0.000494114588352107 -0.00045291833242098046 -0.0011404860515446846 0.0001351213762761552 0.128 False 1000 +MF 4.0 151 selctrl baseline supra_input_relative f_max 0.28129841286364055 0.28442667359029095 -0.0031282607266504003 -0.0006116287242796502 -0.004766262232058639 0.002746059404360264 1.0 False 1000 +MF 4.0 151 selctrl baseline supra_input_relative auprc 0.13470318550435026 0.13455082433484078 0.00015236116950947953 0.00011964161243202603 -0.0007485610735164774 0.0009444701836957745 0.738 False 1000 +MF 4.0 151 selctrl baseline supra_input_output f_max 0.33136710383897316 0.33136710383897316 0.0 -0.0001260077794799298 -0.0014477162768513972 0.0005754475021676002 1.0 False 1000 +MF 4.0 151 selctrl baseline supra_input_output auprc 0.20407333923335857 0.20408893951484783 -1.5600281489253076e-05 -3.994072622914027e-06 -0.0005522153060413306 0.0005869045860600529 0.952 False 1000 +MF 4.0 151 selctrl baseline full f_max 0.2910231966668611 0.2910231966668611 0.0 3.0798667855441162e-06 -0.0008814587691862076 0.0010537096904045268 1.0 False 1000 +MF 4.0 151 selctrl baseline full auprc 0.13886787332506306 0.13883319922977003 3.467409529303578e-05 4.649178057960738e-06 -0.0008573422003150764 0.0008200647655345065 0.944 False 1000 +MF 4.0 151 selctrl baseline naive f_max 0.07531254466979251 0.07531254466979251 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 selctrl baseline naive auprc 0.02307128915639914 0.02307128915639914 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 t0 selctrl supra_input f_max 0.32507657135803186 0.33803499974566886 -0.012958428387637 -0.012639271696097857 -0.026962902133801793 -2.4160761021387183e-05 0.048 True 1000 +MF 4.0 151 t0 selctrl supra_input auprc 0.19253720279059416 0.19917894898248917 -0.006641746191895004 -0.006724635422745769 -0.015287023133215335 0.0003620475750916202 0.066 False 1000 +MF 4.0 151 t0 selctrl supra_input_relative f_max 0.26256167023394233 0.28129841286364055 -0.01873674262969821 -0.017843731284356512 -0.040210922742873995 0.0006615280028311641 0.064 False 1000 +MF 4.0 151 t0 selctrl supra_input_relative auprc 0.12106071224226855 0.13470318550435026 -0.013642473262081706 -0.013755282433449357 -0.033722405560372475 0.0024654533619811216 0.112 False 1000 +MF 4.0 151 t0 selctrl supra_input_output f_max 0.3140200277746999 0.33136710383897316 -0.01734707606427327 -0.016041738483092878 -0.03088787898998704 -0.003401365091492519 0.01 True 1000 +MF 4.0 151 t0 selctrl supra_input_output auprc 0.18749109239613948 0.20407333923335857 -0.016582246837219095 -0.016514616511567 -0.034592423278408 -0.0021764344199032724 0.016 True 1000 +MF 4.0 151 t0 selctrl full f_max 0.2697141121356497 0.2910231966668611 -0.021309084531211386 -0.02066300852519923 -0.044294548841611 -0.0016860630799094482 0.02 True 1000 +MF 4.0 151 t0 selctrl full auprc 0.12442969869416227 0.13886787332506306 -0.014438174630900788 -0.014740252080721057 -0.03200419123843235 -0.0007946521357218385 0.034 True 1000 +MF 4.0 151 t0 selctrl naive f_max 0.07531254466979251 0.07531254466979251 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 t0 selctrl naive auprc 0.02307128915639914 0.02307128915639914 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 t0 baseline supra_input f_max 0.32507657135803186 0.3379901625145267 -0.01291359115649482 -0.012855873867869478 -0.02721584362749275 -0.0006313767069794147 0.034 True 1000 +MF 4.0 151 t0 baseline supra_input auprc 0.19253720279059416 0.19967306357084127 -0.007135860780247111 -0.007177553755166748 -0.01585198834567724 -9.198879818542052e-05 0.048 True 1000 +MF 4.0 151 t0 baseline supra_input_relative f_max 0.26256167023394233 0.28442667359029095 -0.02186500335634861 -0.01845536000863616 -0.041854468013195575 0.0002835019760349898 0.056 False 1000 +MF 4.0 151 t0 baseline supra_input_relative auprc 0.12106071224226855 0.13455082433484078 -0.013490112092572226 -0.013635640821017331 -0.03302225280923464 0.0028401217850796256 0.112 False 1000 +MF 4.0 151 t0 baseline supra_input_output f_max 0.3140200277746999 0.33136710383897316 -0.01734707606427327 -0.016167746262572804 -0.030964467737357265 -0.00375393366341221 0.012 True 1000 +MF 4.0 151 t0 baseline supra_input_output auprc 0.18749109239613948 0.20408893951484783 -0.016597847118708348 -0.016518610584189915 -0.03436044599121451 -0.0024026675828292105 0.016 True 1000 +MF 4.0 151 t0 baseline full f_max 0.2697141121356497 0.2910231966668611 -0.021309084531211386 -0.020659928658413682 -0.044294548841611 -0.0018690837551962659 0.018 True 1000 +MF 4.0 151 t0 baseline full auprc 0.12442969869416227 0.13883319922977003 -0.014403500535607752 -0.014735602902663094 -0.03183191006714642 -0.0008370136417853962 0.03 True 1000 +MF 4.0 151 t0 baseline naive f_max 0.07531254466979251 0.07531254466979251 0.0 0.0 0.0 0.0 1.0 False 1000 +MF 4.0 151 t0 baseline naive auprc 0.02307128915639914 0.02307128915639914 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 selctrl baseline supra_input f_max 0.3702482776163785 0.3708337150837128 -0.0005854374673343044 -1.852643797950404e-05 -0.0014397935661884169 0.001814982468363457 0.78 False 1000 +CC 0.0 436 selctrl baseline supra_input auprc 0.20742636150879065 0.2069835241870187 0.0004428373217719439 0.0004434675413283667 0.00012120365743711413 0.0008160012535454091 0.01 True 1000 +CC 0.0 436 selctrl baseline supra_input_relative f_max 0.37268360539863454 0.366746026124739 0.0059375792738955435 0.005948179473732003 0.0003404163234163526 0.013142316618051082 0.036 True 1000 +CC 0.0 436 selctrl baseline supra_input_relative auprc 0.20558766875108328 0.20128331727355236 0.0043043514775309155 0.004289159109188009 0.0018969914890842015 0.006955476741729772 0.0 True 1000 +CC 0.0 436 selctrl baseline supra_input_output f_max 0.3869795489823925 0.38630822739324006 0.0006713215891524515 0.0007810147381693951 -0.0002908520539501052 0.0021679267078795078 0.152 False 1000 +CC 0.0 436 selctrl baseline supra_input_output auprc 0.24806639386107782 0.24747278027197522 0.0005936135891025929 0.0005880025551036061 0.00021296915593072306 0.0009532089356227471 0.0 True 1000 +CC 0.0 436 selctrl baseline full f_max 0.38155247242054213 0.37365641473954475 0.007896057680997381 0.007743363424880165 0.0019982814609960267 0.014769975329900329 0.004 True 1000 +CC 0.0 436 selctrl baseline full auprc 0.23398195570502178 0.22909243267235713 0.0048895230326646555 0.00490219390312374 0.001974537254739155 0.008620100999263294 0.0 True 1000 +CC 0.0 436 selctrl baseline naive f_max 0.593050572267429 0.593050572267429 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 selctrl baseline naive auprc 0.5414744252479067 0.5414744252479067 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 t0 selctrl supra_input f_max 0.3657545116537211 0.3702482776163785 -0.004493765962657437 -0.005104506394352691 -0.012557249774137984 0.0017096351091814373 0.18 False 1000 +CC 0.0 436 t0 selctrl supra_input auprc 0.2001142781223428 0.20742636150879065 -0.007312083386447843 -0.007301025337693272 -0.012603913967180958 -0.00292271074733715 0.0 True 1000 +CC 0.0 436 t0 selctrl supra_input_relative f_max 0.36749458141809443 0.37268360539863454 -0.005189023980540108 -0.004964022603059121 -0.011653627968738098 0.0012254728886283259 0.11 False 1000 +CC 0.0 436 t0 selctrl supra_input_relative auprc 0.1990108702115158 0.20558766875108328 -0.006576798539567469 -0.006626052601973371 -0.01178630288366065 -0.002201030888633031 0.004 True 1000 +CC 0.0 436 t0 selctrl supra_input_output f_max 0.3808275731115995 0.3869795489823925 -0.006151975870793025 -0.006453305391087869 -0.012519772043065966 -0.0008069192751070098 0.03 True 1000 +CC 0.0 436 t0 selctrl supra_input_output auprc 0.23984939219421486 0.24806639386107782 -0.008217001666862955 -0.008254049714250712 -0.0140178937130145 -0.003220963651440208 0.0 True 1000 +CC 0.0 436 t0 selctrl full f_max 0.37665530089167504 0.38155247242054213 -0.004897171528867095 -0.005427023044024247 -0.012269187506895348 0.0008677606492765095 0.098 False 1000 +CC 0.0 436 t0 selctrl full auprc 0.22855037107450735 0.23398195570502178 -0.0054315846305144355 -0.005444549102748253 -0.010366932029633279 -0.0010621770537216676 0.006 True 1000 +CC 0.0 436 t0 selctrl naive f_max 0.593050572267429 0.593050572267429 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 t0 selctrl naive auprc 0.5414744252479067 0.5414744252479067 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 t0 baseline supra_input f_max 0.3657545116537211 0.3708337150837128 -0.005079203429991741 -0.005123032832332195 -0.013115756546268727 0.002637295597078948 0.22 False 1000 +CC 0.0 436 t0 baseline supra_input auprc 0.2001142781223428 0.2069835241870187 -0.006869246064675899 -0.006857557796364905 -0.012200141720691563 -0.002399824385387024 0.0 True 1000 +CC 0.0 436 t0 baseline supra_input_relative f_max 0.36749458141809443 0.366746026124739 0.0007485552933554351 0.0009841568706728814 -0.007648135491620567 0.01030359901761475 0.828 False 1000 +CC 0.0 436 t0 baseline supra_input_relative auprc 0.1990108702115158 0.20128331727355236 -0.0022724470620365533 -0.0023368934927853625 -0.008073431557245317 0.00355680391208916 0.426 False 1000 +CC 0.0 436 t0 baseline supra_input_output f_max 0.3808275731115995 0.38630822739324006 -0.0054806542816405734 -0.0056722906529184745 -0.011848612917633937 -0.0001605721176329141 0.042 True 1000 +CC 0.0 436 t0 baseline supra_input_output auprc 0.23984939219421486 0.24747278027197522 -0.007623388077760362 -0.007666047159147106 -0.013467019042769297 -0.0026535819391386084 0.0 True 1000 +CC 0.0 436 t0 baseline full f_max 0.37665530089167504 0.37365641473954475 0.0029988861521302868 0.002316340380855917 -0.006754944944052457 0.011434210374555082 0.644 False 1000 +CC 0.0 436 t0 baseline full auprc 0.22855037107450735 0.22909243267235713 -0.00054206159784978 -0.0005423551996245148 -0.006583313083915059 0.005400192775722239 0.862 False 1000 +CC 0.0 436 t0 baseline naive f_max 0.593050572267429 0.593050572267429 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 0.0 436 t0 baseline naive auprc 0.5414744252479067 0.5414744252479067 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 selctrl baseline supra_input f_max 0.19343405375604655 0.19428464352832564 -0.0008505897722790867 -0.000993461013175189 -0.0036145559728345982 -2.6661436489906774e-05 0.028 True 1000 +CC 2.0 298 selctrl baseline supra_input auprc 0.05806421570392059 0.058071664366317166 -7.448662396575267e-06 -2.0194545251349252e-05 -0.0005807459718713234 0.00046564511602443636 0.956 False 1000 +CC 2.0 298 selctrl baseline supra_input_relative f_max 0.20495063609979508 0.2049562556463094 -5.6195465143182854e-06 0.0003102400599594781 -0.0003953895903786716 0.0019616140136521345 1.0 False 1000 +CC 2.0 298 selctrl baseline supra_input_relative auprc 0.052436747996529906 0.052354786913297154 8.196108323275247e-05 7.280596626557139e-05 -0.0003086427731276186 0.000556192152231057 0.728 False 1000 +CC 2.0 298 selctrl baseline supra_input_output f_max 0.21768741321397259 0.21832997397133352 -0.0006425607573609382 -0.00025862926246245793 -0.004746736960662229 0.005050712155392248 0.868 False 1000 +CC 2.0 298 selctrl baseline supra_input_output auprc 0.0653755442200359 0.06547117512596667 -9.563090593077417e-05 -0.00013801687275609198 -0.0006207345096124144 0.0003256478597764024 0.58 False 1000 +CC 2.0 298 selctrl baseline full f_max 0.20783250955677182 0.21013662553777537 -0.0023041159810035516 -0.002232006586839007 -0.005516998041035179 0.00034699867120360436 0.118 False 1000 +CC 2.0 298 selctrl baseline full auprc 0.052007881921068415 0.05202060376289804 -1.27218418296271e-05 -7.671928322249614e-05 -0.0004258411035349045 0.00024368459845264517 0.644 False 1000 +CC 2.0 298 selctrl baseline naive f_max 0.2720885241633347 0.2720885241633347 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 selctrl baseline naive auprc 0.1451197420848838 0.1451197420848838 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 t0 selctrl supra_input f_max 0.18984512084059801 0.19343405375604655 -0.0035889329154485394 -0.0035119731727945797 -0.01006579630384347 0.0024096490343011073 0.296 False 1000 +CC 2.0 298 t0 selctrl supra_input auprc 0.05384834822245379 0.05806421570392059 -0.004215867481466802 -0.004402200867016015 -0.008311940436429981 -0.00145635183415226 0.004 True 1000 +CC 2.0 298 t0 selctrl supra_input_relative f_max 0.19658370250400492 0.20495063609979508 -0.00836693359579016 -0.008720227056421443 -0.021569709458935955 0.0009646439098102684 0.108 False 1000 +CC 2.0 298 t0 selctrl supra_input_relative auprc 0.048846805703567135 0.052436747996529906 -0.003589942292962771 -0.0035842503202632276 -0.007422022361882154 -0.0006783030932496884 0.01 True 1000 +CC 2.0 298 t0 selctrl supra_input_output f_max 0.2093688753175775 0.21768741321397259 -0.008318537896395084 -0.008363222536890685 -0.016453996809609052 -0.0016180461594799403 0.004 True 1000 +CC 2.0 298 t0 selctrl supra_input_output auprc 0.06107642913396808 0.0653755442200359 -0.004299115086067819 -0.0041987204042616385 -0.009066476337875249 -0.0008228475541341116 0.01 True 1000 +CC 2.0 298 t0 selctrl full f_max 0.20442953930504945 0.20783250955677182 -0.0034029702517223703 -0.004578657864803635 -0.010328028273693142 0.000921344655756775 0.106 False 1000 +CC 2.0 298 t0 selctrl full auprc 0.04934663831207692 0.052007881921068415 -0.0026612436089914968 -0.002646617411920964 -0.0058306764995934846 0.00022806847797684352 0.074 False 1000 +CC 2.0 298 t0 selctrl naive f_max 0.2720885241633347 0.2720885241633347 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 t0 selctrl naive auprc 0.1451197420848838 0.1451197420848838 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 t0 baseline supra_input f_max 0.18984512084059801 0.19428464352832564 -0.004439522687727626 -0.004505434185969768 -0.011431589170788176 0.0012399713825354275 0.15 False 1000 +CC 2.0 298 t0 baseline supra_input auprc 0.05384834822245379 0.058071664366317166 -0.004223316143863377 -0.004422395412267365 -0.008319449476888053 -0.0015059848600958736 0.004 True 1000 +CC 2.0 298 t0 baseline supra_input_relative f_max 0.19658370250400492 0.2049562556463094 -0.008372553142304479 -0.008409986996461963 -0.02116530127271271 0.0014708252137847315 0.12 False 1000 +CC 2.0 298 t0 baseline supra_input_relative auprc 0.048846805703567135 0.052354786913297154 -0.0035079812097300184 -0.003511444353997656 -0.007315224661508878 -0.0006521048750505994 0.01 True 1000 +CC 2.0 298 t0 baseline supra_input_output f_max 0.2093688753175775 0.21832997397133352 -0.008961098653756022 -0.00862185179935314 -0.016151459309773228 -0.0027610142116807077 0.0 True 1000 +CC 2.0 298 t0 baseline supra_input_output auprc 0.06107642913396808 0.06547117512596667 -0.004394745991998593 -0.004336737277017729 -0.009338813882744844 -0.0008728777742392338 0.008 True 1000 +CC 2.0 298 t0 baseline full f_max 0.20442953930504945 0.21013662553777537 -0.005707086232725922 -0.006810664451642641 -0.013905787310552892 -0.0007066979235509645 0.03 True 1000 +CC 2.0 298 t0 baseline full auprc 0.04934663831207692 0.05202060376289804 -0.002673965450821124 -0.00272333669514346 -0.0059656393851869926 0.00015481978782112973 0.066 False 1000 +CC 2.0 298 t0 baseline naive f_max 0.2720885241633347 0.2720885241633347 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 2.0 298 t0 baseline naive auprc 0.1451197420848838 0.1451197420848838 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 selctrl baseline supra_input f_max 0.12782363409703573 0.12764724364036328 0.00017639045667244835 -0.0011766664702829019 -0.008279635686018414 0.0012682792721002614 0.826 False 1000 +CC 4.0 210 selctrl baseline supra_input auprc 0.03586106852445638 0.03567737321150979 0.00018369531294658986 0.00018526439077177565 -0.00023401883698419507 0.0007001994462438017 0.436 False 1000 +CC 4.0 210 selctrl baseline supra_input_relative f_max 0.12554626887725337 0.126156501550602 -0.000610232673348643 -0.0005377130472291748 -0.0022691913452241533 0.00011693748922430024 0.512 False 1000 +CC 4.0 210 selctrl baseline supra_input_relative auprc 0.030681186035091658 0.030784931046188665 -0.00010374501109700757 -4.246500321702527e-05 -0.0008507452908665864 0.0006878445701841173 0.872 False 1000 +CC 4.0 210 selctrl baseline supra_input_output f_max 0.12547653290885707 0.12695996914772711 -0.0014834362388700395 -0.001071620344396877 -0.006804611065657345 9.056970706459644e-05 0.486 False 1000 +CC 4.0 210 selctrl baseline supra_input_output auprc 0.038374445533779236 0.03846729265179205 -9.284711801281487e-05 -5.7211439305214085e-05 -0.000511458249299254 0.0004615278452382722 0.692 False 1000 +CC 4.0 210 selctrl baseline full f_max 0.12476406186624613 0.12477732311435916 -1.3261248113030644e-05 -0.0004017717575151993 -0.0025584311570142353 0.0004088544692996427 0.358 False 1000 +CC 4.0 210 selctrl baseline full auprc 0.02865341327493268 0.0286555098488457 -2.0965739130213157e-06 -1.8289399449834237e-05 -0.00031401509530293477 0.0003033140342578071 0.828 False 1000 +CC 4.0 210 selctrl baseline naive f_max 0.10440217265761789 0.10440217265761789 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 selctrl baseline naive auprc 0.04494942594240296 0.04494942594240296 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 t0 selctrl supra_input f_max 0.1263849974354251 0.12782363409703573 -0.0014386366616106205 -0.003357298574857071 -0.01218206616352713 0.0049338069376099174 0.436 False 1000 +CC 4.0 210 t0 selctrl supra_input auprc 0.03115695182023453 0.03586106852445638 -0.004704116704221849 -0.0045620362594344386 -0.010445461811705973 -0.0006125181902559813 0.012 True 1000 +CC 4.0 210 t0 selctrl supra_input_relative f_max 0.11463965887981685 0.12554626887725337 -0.01090660999743652 -0.009676486581765315 -0.021468344656947365 -0.0004068819384116489 0.036 True 1000 +CC 4.0 210 t0 selctrl supra_input_relative auprc 0.026881474129342588 0.030681186035091658 -0.0037997119057490703 -0.0035762301566695845 -0.00883983020384434 0.00034550944986350764 0.08 False 1000 +CC 4.0 210 t0 selctrl supra_input_output f_max 0.12239568929514537 0.12547653290885707 -0.0030808436137117062 -0.003137956030437431 -0.013268125880109847 0.00878753422618565 0.534 False 1000 +CC 4.0 210 t0 selctrl supra_input_output auprc 0.03416233160249707 0.038374445533779236 -0.004212113931282166 -0.0039021402225310224 -0.009140029219943014 9.29941124523464e-05 0.056 False 1000 +CC 4.0 210 t0 selctrl full f_max 0.12055865495429378 0.12476406186624613 -0.004205406911952342 -0.004282059711024614 -0.01198418404436229 0.0004053936488540811 0.112 False 1000 +CC 4.0 210 t0 selctrl full auprc 0.026574800701653457 0.02865341327493268 -0.0020786125732792232 -0.0016091509906840992 -0.005582996210914986 0.0012620444142581097 0.326 False 1000 +CC 4.0 210 t0 selctrl naive f_max 0.10440217265761789 0.10440217265761789 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 t0 selctrl naive auprc 0.04494942594240296 0.04494942594240296 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 t0 baseline supra_input f_max 0.1263849974354251 0.12764724364036328 -0.0012622462049381722 -0.004533965045139972 -0.015840720709036134 0.004920153465923095 0.422 False 1000 +CC 4.0 210 t0 baseline supra_input auprc 0.03115695182023453 0.03567737321150979 -0.004520421391275259 -0.004376771868662663 -0.010258690205984916 -0.0005189570404277513 0.022 True 1000 +CC 4.0 210 t0 baseline supra_input_relative f_max 0.11463965887981685 0.126156501550602 -0.011516842670785163 -0.010214199628994487 -0.022099281370666718 -0.0006434079269032577 0.03 True 1000 +CC 4.0 210 t0 baseline supra_input_relative auprc 0.026881474129342588 0.030784931046188665 -0.003903456916846078 -0.0036186951598866095 -0.00882984996715066 0.0004336339673029377 0.082 False 1000 +CC 4.0 210 t0 baseline supra_input_output f_max 0.12239568929514537 0.12695996914772711 -0.004564279852581746 -0.004209576374834308 -0.015855826976963718 0.008687481637211381 0.486 False 1000 +CC 4.0 210 t0 baseline supra_input_output auprc 0.03416233160249707 0.03846729265179205 -0.004304961049294981 -0.003959351661836237 -0.009462584124998789 6.0433371415224926e-05 0.058 False 1000 +CC 4.0 210 t0 baseline full f_max 0.12055865495429378 0.12477732311435916 -0.0042186681600653725 -0.004683831468539813 -0.012421332176358092 0.0002708174782559315 0.088 False 1000 +CC 4.0 210 t0 baseline full auprc 0.026574800701653457 0.0286555098488457 -0.0020807091471922445 -0.0016274403901339332 -0.005562258456610127 0.0012163570377792048 0.324 False 1000 +CC 4.0 210 t0 baseline naive f_max 0.10440217265761789 0.10440217265761789 0.0 0.0 0.0 0.0 1.0 False 1000 +CC 4.0 210 t0 baseline naive auprc 0.04494942594240296 0.04494942594240296 0.0 0.0 0.0 0.0 1.0 False 1000 From 26e497382469ea4cfb52f810ef5ef5f584299ffd Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 7 Oct 2026 14:43:03 +0100 Subject: [PATCH 6/7] Guard the default extract and count the t0 arm's own domain proteins Review fixes for the t0 InterPro PR. - extract_human_interpro.py: --source or --gaf without --output is now a usage error. Before, it replaced data/interim/protein2ipr_.dat.gz, the default --interpro of every run and evaluator; the overwrite guard could not catch it (same "goa" rule, and the human extract has no sidecar). - validation/ablation.py: with --cohort-interpro, the proteins_with_domains row counts the cohort file, so it is now labelled as such, and a new transfer_proteins_with_domains row counts the --interpro file. Rescored both arms with the runbook's eval command: t0 19,393 and control 19,522 transfer proteins against the cohort file's 18,908. The other four tables reproduce byte-for-byte, except one s_min in the t0 metrics that differs in the last digit (float order); the committed metrics are kept. - VALIDATION_PLAN section 4: the control arm flips 9 of 414 paired significance calls against the committed cell (t0: 22), none changing sign, including two supra - single F_max losses that no other arm has. The cohort is still chosen by the 2026 protein2ipr, which biases against t0; the own-cohort run makes the same Base -> Base + relative calls. The manifests' commit ac7485a is mapped to its rebased twin 613ead2, and the claim that run_dcgo_human.py was "unchanged since" is replaced by what did change. - REPRODUCIBILITY.md: the two committed subset sidecars predate the evidence_filter field. Co-Authored-By: Claude Opus 5.5 (1M context) --- REPRODUCIBILITY.md | 5 ++- VALIDATION_PLAN.md | 43 +++++++++++++------ extract_human_interpro.py | 11 +++-- .../test_extract_human_interpro.py | 21 +++++++++ tests/unit/test_ablation.py | 12 ++++++ validation/ablation.py | 30 ++++++++++--- .../ablation_selection_counts.tsv | 3 +- .../ablation_selection_counts.tsv | 3 +- 8 files changed, 104 insertions(+), 24 deletions(-) diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md index dc5d94a..c088a66 100644 --- a/REPRODUCIBILITY.md +++ b/REPRODUCIBILITY.md @@ -58,7 +58,10 @@ benchmark) are hashed and header-read the same way but carry no different bytes. A subset's own `.provenance.json` sidecar names the archive it was cut from, the GAFs and the evidence filter that selected it; `scripts/run_t0_interpro_ablation.py` copies it into the evaluation's -`manifests/` so it is committed with the results. +`manifests/` so it is committed with the results. The two sidecars committed +under `validation/ablation_cells/ipr_manual_{t0interpro,selctrl}/` predate +the `evidence_filter` field; their extraction logs record `Evidence codes: +ALL` (`VALIDATION_PLAN.md` §4). The inputs recorded are exactly those the selected ontology declares in `src/ontology_registry.py` (`needs`, plus `hierarchy_needs` when diff --git a/VALIDATION_PLAN.md b/VALIDATION_PLAN.md index 18d503b..df63d61 100644 --- a/VALIDATION_PLAN.md +++ b/VALIDATION_PLAN.md @@ -1314,11 +1314,18 @@ share one cohort, one IC table and one naive baseline. Both checks passed. | CC 2 | 0.194 / 0.193 / 0.190 | −0.004 (−0.010, +0.002) | 0.058 / 0.058 / 0.054 | −0.004\* (−0.008, −0.001) | | CC 4 | 0.128 / 0.128 / 0.126 | −0.001 (−0.012, +0.005) | 0.036 / 0.036 / 0.031 | −0.005\* (−0.010, −0.001) | -- **Protein selection moves nothing material.** Control − baseline is +- **Protein selection barely moves the estimates.** Control − baseline is within ±0.003 for Base, and within −0.003 to +0.008 (F_max) and −0.001 to +0.005 (AUPRC) across all four configurations. A few of these differences are significant only because the two arms are almost identical, so the - intervals are very narrow. + intervals are very narrow. It does move borderline significance calls: + 9 of the control arm's 414 paired calls differ from the committed cell's + (22 for the t0 arm), none with a change of sign. Two are supra − single + F_max, a significant loss in the control arm at MF IC 0 (−0.012, 95% CI + −0.022 to −0.001) and CC IC 2 (−0.004, −0.010 to −0.001); no supra − single + F_max cell is significant in the baseline or t0 arm. A single cell's + significance call is that fragile; the patterns below are what the arms + support. - **Dating the architectures costs a little, mostly in MF.** Across the four configurations, t0 − control is lower in 32 of 36 F_max comparisons (11 significant, none significantly higher). It is lower in 36 of 36 AUPRC @@ -1350,17 +1357,27 @@ share one cohort, one IC table and one naive baseline. Both checks passed. AUPRC. **What this does and does not remove.** It removes the domain-side look-ahead -from this one cell: training and transfer both read InterPro 85.0, released -13 days before GOA 205. It does **not** remove the vocabulary look-ahead -described above. Seven of the nine rungs still train with `go-basic.obo` -2026-06-15 for propagation and the parental background, and retired t0 ids are -remapped to their 2026 successors. The evaluator also uses that ontology for -truth propagation, IC and aspects, as CAFA does. The other temporal analyses -(§2 headline benchmark, multi-ontology breadth, surprise-score held-out test, -all-species and mouse arms) still read current `protein2ipr`. - -Regeneration. The two new arms' runs (`run_dcgo_human.py` unchanged since) -are the runbook's: +from training and transfer in this one cell: both read InterPro 85.0, released +13 days before GOA 205. Which proteins are scored still depends on the current +release, because the cohort is the committed cell's (a domain in the 2026 +`protein2ipr`). That biases against t0 (its 19 / 16 / 14 misses); the +own-cohort run is free of it and makes the same significance calls for +Base → Base + relative in 9/9 cells on both metrics. It does **not** remove +the vocabulary look-ahead described above. Seven of the nine rungs still train +with `go-basic.obo` 2026-06-15 for propagation and the parental background, +and retired t0 ids are remapped to their 2026 successors. The evaluator also +uses that ontology for truth propagation, IC and aspects, as CAFA does. The +other temporal analyses (§2 headline benchmark, multi-ontology breadth, +surprise-score held-out test, all-species and mouse arms) still read current +`protein2ipr`. + +Regeneration. The two new arms' runs are the runbook's. Their 18 rung +manifests record commit `ac7485a`, the pre-rebase twin of `613ead2`, which is +on no branch. `run_dcgo_human.py`, `src/` and `config/` are byte-identical +between the two; `uv.lock` differs only by #74's dev-group `cafaeval` entry, +so the manifests' `uv.lock` hash (`57e605…`) is `ac7485a`'s. Since then the +only changes there are a comment in `config/settings.py` and the extract +sidecar's `evidence_filter` field, which training does not read. ```bash uv run python scripts/run_t0_interpro_ablation.py # t0 arm diff --git a/extract_human_interpro.py b/extract_human_interpro.py index e8f9c1b..d97f5ac 100644 --- a/extract_human_interpro.py +++ b/extract_human_interpro.py @@ -109,7 +109,7 @@ def main(): parser.add_argument( "--source", type=Path, - default=Path("data/raw/interpro_mappings/protein2ipr.dat.gz"), + default=None, help="protein2ipr file to filter, e.g. an archived release " "(default: data/raw/interpro_mappings/protein2ipr.dat.gz)", ) @@ -118,7 +118,8 @@ def main(): type=Path, default=None, help="Output path (default: data/interim/protein2ipr_.dat.gz). " - "The selecting accession list is written beside it", + "The selecting accession list is written beside it. Required with " + "--source or --gaf", ) parser.add_argument( "--force", @@ -128,6 +129,10 @@ def main(): "extract from scripts/extract_species_interpro.py)", ) args = parser.parse_args() + # The default extract is every run's and evaluator's default --interpro; + # a subset of another release or selection must not replace it unasked. + if (args.source or args.gaf) and not args.output: + parser.error("--source and --gaf require an explicit --output") # Parse GOA to get this species' protein IDs from src.goa_parser import parse_goa @@ -180,7 +185,7 @@ def main(): logger.info( f"Step 2: Extracting {args.species} protein annotations from InterPro..." ) - interpro_file = args.source + interpro_file = args.source or Path("data/raw/interpro_mappings/protein2ipr.dat.gz") if not interpro_file.exists(): logger.error(f"InterPro mappings file not found: {interpro_file}") logger.error( diff --git a/tests/integration/test_extract_human_interpro.py b/tests/integration/test_extract_human_interpro.py index 5f67ae9..ab17252 100644 --- a/tests/integration/test_extract_human_interpro.py +++ b/tests/integration/test_extract_human_interpro.py @@ -128,3 +128,24 @@ def test_a_missing_gaf_fails_before_scanning(tmp_path: Path) -> None: assert result.returncode == 1 assert "absent.gaf.gz" in result.stderr assert not output.exists() + + +@pytest.mark.parametrize("flag", ["--source", "--gaf"]) +def test_a_non_default_selection_needs_an_explicit_output( + tmp_path: Path, flag: str +) -> None: + # Without --output it would replace the species' default extract, which + # every run and evaluator reads by default. + default = _protein2ipr(tmp_path / "data/interim/protein2ipr_toy.dat.gz") + before = default.read_bytes() + value = ( + _protein2ipr(tmp_path / "archive.dat.gz") + if flag == "--source" + else _gaf(tmp_path / "t0.gaf.gz", T0_ROWS) + ) + + result = _extract(tmp_path, "--species", "toy", flag, value) + + assert result.returncode == 2 + assert "--output" in result.stderr + assert default.read_bytes() == before diff --git a/tests/unit/test_ablation.py b/tests/unit/test_ablation.py index 251ecea..63ef91c 100644 --- a/tests/unit/test_ablation.py +++ b/tests/unit/test_ablation.py @@ -169,9 +169,21 @@ def test_cohort_proteins_without_transfer_architectures_are_counted(self): } assert missing == {"BP": 1, "MF": 0, "CC": 0} + def test_the_cohort_and_transfer_files_are_counted_separately(self): + # The domain row is the cohort file's; the transfer file gets its own. + rows = { + r["stage"]: r + for r in _rows(transfer_proteins=["P1", "P2", "P3"]) + if r["aspect"] == "-" + } + assert rows["proteins_with_domains"]["n_proteins"] == 4 + assert "--cohort-interpro" in rows["proteins_with_domains"]["note"] + assert rows["transfer_proteins_with_domains"]["n_proteins"] == 3 + def test_no_transfer_row_when_the_cohort_is_the_transfer_file(self): stages = {r["stage"] for r in _rows()} assert "no_knowledge_without_transfer_architectures" not in stages + assert "transfer_proteins_with_domains" not in stages def test_empty_aspects_are_still_reported_as_zero(self): rows = [ diff --git a/validation/ablation.py b/validation/ablation.py index 998c4b5..df30054 100644 --- a/validation/ablation.py +++ b/validation/ablation.py @@ -223,9 +223,11 @@ def selection_stage_counts( has-a-domain restriction; ``benchmark_with_domains`` is the one actually scored. Both are ``{aspect: {protein: truth}}``. ``transfer_proteins``, given when the cohort comes from another protein2ipr than the transfer - step, adds how many cohort proteins have no architecture to transfer - through (scored as misses). + step, adds the transfer file's own protein count and how many cohort + proteins have no architecture to transfer through (scored as misses); + ``all_architecture_proteins`` is then the cohort file's. """ + transfer = set(transfer_proteins) if transfer_proteins is not None else None rows: list[dict] = [ { "stage": "t0_annotated_proteins", @@ -253,9 +255,28 @@ def selection_stage_counts( "n_proteins": len(set(all_architecture_proteins)), "n_dropped_vs_ic0": "-", "pct_of_ic0": "-", - "note": "proteins with >=1 InterPro domain (predictable at all by a domain method)", + "note": ( + "proteins with >=1 InterPro domain in --cohort-interpro " + "(the file that selects the scored cohort)" + if transfer is not None + else "proteins with >=1 InterPro domain (predictable at all by " + "a domain method)" + ), }, ] + if transfer is not None: + rows.append( + { + "stage": "transfer_proteins_with_domains", + "aspect": "-", + "min_ic": "-", + "n_proteins": len(transfer), + "n_dropped_vs_ic0": "-", + "pct_of_ic0": "-", + "note": "proteins with >=1 InterPro domain in --interpro " + "(predictable at all by this run's transfer step)", + } + ) for aspect in ("BP", "MF", "CC"): rows.append( { @@ -279,8 +300,7 @@ def selection_stage_counts( "note": "the scored cohort before any IC floor", } ) - if transfer_proteins is not None: - transfer = set(transfer_proteins) + if transfer is not None: rows.append( { "stage": "no_knowledge_without_transfer_architectures", diff --git a/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv b/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv index e43cd36..2ab6b0b 100644 --- a/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv +++ b/validation/ablation_cells/ipr_manual_selctrl/ablation_selection_counts.tsv @@ -1,7 +1,8 @@ stage aspect min_ic n_proteins n_dropped_vs_ic0 pct_of_ic0 note t0_annotated_proteins - - 18735 - - proteins with >=1 GO annotation at t0 under the training evidence filter (the training universe) t1_experimental_proteins - - 16362 - - proteins with >=1 experimental GO annotation at t1 (the gold standard) -proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain (predictable at all by a domain method) +proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain in --cohort-interpro (the file that selects the scored cohort) +transfer_proteins_with_domains - - 19522 - - proteins with >=1 InterPro domain in --interpro (predictable at all by this run's transfer step) no_knowledge_candidates BP - 310 - - no t0 knowledge in this aspect, gained t1 experimental terms no_knowledge_with_domains BP - 298 - - the scored cohort before any IC floor no_knowledge_without_transfer_architectures BP - 0 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses diff --git a/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv b/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv index dfc3c60..394d53e 100644 --- a/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv +++ b/validation/ablation_cells/ipr_manual_t0interpro/ablation_selection_counts.tsv @@ -1,7 +1,8 @@ stage aspect min_ic n_proteins n_dropped_vs_ic0 pct_of_ic0 note t0_annotated_proteins - - 18735 - - proteins with >=1 GO annotation at t0 under the training evidence filter (the training universe) t1_experimental_proteins - - 16362 - - proteins with >=1 experimental GO annotation at t1 (the gold standard) -proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain (predictable at all by a domain method) +proteins_with_domains - - 18908 - - proteins with >=1 InterPro domain in --cohort-interpro (the file that selects the scored cohort) +transfer_proteins_with_domains - - 19393 - - proteins with >=1 InterPro domain in --interpro (predictable at all by this run's transfer step) no_knowledge_candidates BP - 310 - - no t0 knowledge in this aspect, gained t1 experimental terms no_knowledge_with_domains BP - 298 - - the scored cohort before any IC floor no_knowledge_without_transfer_architectures BP - 19 - - cohort proteins with no domain in --interpro: no dcGO prediction, scored as misses From 723d6b4347ab9f813d149559ac3fafb607995f9a Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 7 Oct 2026 14:56:06 +0100 Subject: [PATCH 7/7] Reuse a t0 subset only if the selecting GAFs' bytes are unchanged The runbook reused an existing subset when its sidecar named the same source, GAF paths and evidence filter. The t1 GAF lives at a current-release path that a new download overwrites, so a stale subset would be reused without a word, and held-out proteins new to the refreshed GAF would score as misses for want of architectures. Extract sidecars now record selection_sha256, the SHA-256 of each selecting GAF in source order (read back as None for older sidecars), and the runbook reuses a subset only when both hashes still match; otherwise it re-extracts and reruns every rung. Raised by CodeRabbit on #75. Co-Authored-By: Claude Opus 5.5 (1M context) --- REPRODUCIBILITY.md | 7 +++-- extract_human_interpro.py | 2 ++ scripts/run_t0_interpro_ablation.py | 17 +++++++++-- src/universe_provenance.py | 11 +++++++ tests/unit/test_run_t0_interpro_ablation.py | 32 ++++++++++++++++----- tests/unit/test_universe_provenance.py | 17 +++++++++++ 6 files changed, 74 insertions(+), 12 deletions(-) diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md index c088a66..34576f1 100644 --- a/REPRODUCIBILITY.md +++ b/REPRODUCIBILITY.md @@ -56,12 +56,13 @@ Inputs passed with `--gaf`/`--interpro` (archived snapshots for the temporal benchmark) are hashed and header-read the same way but carry no `source_url`/`derived_from`: the species' current-release URL would name different bytes. A subset's own `.provenance.json` sidecar names the archive it -was cut from, the GAFs and the evidence filter that selected it; +was cut from, the GAFs (path and SHA-256) and the evidence filter that +selected it; `scripts/run_t0_interpro_ablation.py` copies it into the evaluation's `manifests/` so it is committed with the results. The two sidecars committed under `validation/ablation_cells/ipr_manual_{t0interpro,selctrl}/` predate -the `evidence_filter` field; their extraction logs record `Evidence codes: -ALL` (`VALIDATION_PLAN.md` §4). +the `evidence_filter` and `selection_sha256` fields; their extraction logs +record `Evidence codes: ALL` (`VALIDATION_PLAN.md` §4). The inputs recorded are exactly those the selected ontology declares in `src/ontology_registry.py` (`needs`, plus `hierarchy_needs` when diff --git a/extract_human_interpro.py b/extract_human_interpro.py index d97f5ac..33a96de 100644 --- a/extract_human_interpro.py +++ b/extract_human_interpro.py @@ -136,6 +136,7 @@ def main(): # Parse GOA to get this species' protein IDs from src.goa_parser import parse_goa + from src.run_manifest import sha256_file from src.universe_provenance import ( ProvenanceConflictError, ensure_overwrite_allowed, @@ -209,6 +210,7 @@ def main(): output_file, selection_rule="goa", selection_sources=goa_files, + selection_sha256=[sha256_file(goa_file) for goa_file in goa_files], interpro_source=interpro_file, evidence_filter=args.evidence_filter, n_accessions=n_accessions, diff --git a/scripts/run_t0_interpro_ablation.py b/scripts/run_t0_interpro_ablation.py index c34d497..58162a7 100644 --- a/scripts/run_t0_interpro_ablation.py +++ b/scripts/run_t0_interpro_ablation.py @@ -12,7 +12,7 @@ architectures. (The current subset was selected by the t1 GAF's non-IEA proteins alone, a strict subset of this.) Skipped when the subset exists and its provenance marker records this exact extraction: the same source, - GAFs and evidence filter. + GAFs (by path and SHA-256) and evidence filter. 2. **Train** the nine rungs: exactly the commands recorded in ``validation/ablation_manifests/.json``, with ``--output-dir`` moved under ``--run-dir`` and ``--gaf``/``--interpro`` added. @@ -53,7 +53,7 @@ REPO = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO)) -from src.run_manifest import manifest_filename # noqa: E402 +from src.run_manifest import manifest_filename, sha256_file # noqa: E402 from src.universe_provenance import marker_path, read_marker # noqa: E402 MANIFESTS = REPO / "validation" / "ablation_manifests" @@ -97,6 +97,17 @@ def completed(output_dir: Path) -> bool: ) +def selection_hashes() -> tuple[str, ...] | None: + """SHA-256 of the two selecting GAFs, or ``None`` if either is missing. + + The t1 GAF sits at a current-release path a new download overwrites, so + a subset is reused only if the GAF *bytes* that selected it are unchanged. + """ + if not (T0_GAF.exists() and T1_GAF.exists()): + return None + return (sha256_file(T0_GAF), sha256_file(T1_GAF)) + + def build_steps( source: Path, subset: Path, run_dir: Path, eval_dir: Path ) -> list[tuple[str, list[str]]]: @@ -115,6 +126,8 @@ def build_steps( and marker is not None and (marker.interpro_source, marker.selection_sources, marker.evidence_filter) == (str(source), (str(T0_GAF), str(T1_GAF)), EVIDENCE_FILTER) + and marker.selection_sha256 is not None + and marker.selection_sha256 == selection_hashes() ) if extracting: steps.append( diff --git a/src/universe_provenance.py b/src/universe_provenance.py index 316e370..b2303ef 100644 --- a/src/universe_provenance.py +++ b/src/universe_provenance.py @@ -51,6 +51,10 @@ class UniverseProvenance: evidence_filter: the GAF evidence filter that selected the accessions (``"goa"`` rule only); ``None`` for other rules and for markers written before it was recorded. + selection_sha256: SHA-256 of each ``selection_sources`` file, in the + same order — paths such as the current-release GAF are mutable, so + the path alone does not say which bytes selected the accessions. + ``None`` for markers written before it was recorded. """ selection_rule: str @@ -61,6 +65,7 @@ class UniverseProvenance: tool: str created: str evidence_filter: Optional[str] = None + selection_sha256: Optional[tuple[str, ...]] = None def marker_path(extract_path: Path) -> Path: @@ -82,6 +87,8 @@ def read_marker(extract_path: Path) -> Optional[UniverseProvenance]: try: payload = json.loads(path.read_text(encoding="utf-8")) payload["selection_sources"] = tuple(payload["selection_sources"]) + if payload.get("selection_sha256") is not None: + payload["selection_sha256"] = tuple(payload["selection_sha256"]) return UniverseProvenance(**payload) except (ValueError, TypeError, KeyError) as exc: logger.warning(f"Unreadable provenance marker {path}: {exc}") @@ -99,6 +106,7 @@ def write_marker( tool: str, created: Optional[str] = None, evidence_filter: Optional[str] = None, + selection_sha256: Optional[Sequence[str]] = None, ) -> Path: """Write the sidecar for a freshly written extract; returns its path.""" provenance = UniverseProvenance( @@ -110,6 +118,9 @@ def write_marker( tool=tool, created=created or datetime.now(timezone.utc).isoformat(timespec="seconds"), evidence_filter=evidence_filter, + selection_sha256=( + tuple(selection_sha256) if selection_sha256 is not None else None + ), ) path = marker_path(extract_path) path.write_text(json.dumps(asdict(provenance), indent=2) + "\n", encoding="utf-8") diff --git a/tests/unit/test_run_t0_interpro_ablation.py b/tests/unit/test_run_t0_interpro_ablation.py index ed1a3f7..0dc9031 100644 --- a/tests/unit/test_run_t0_interpro_ablation.py +++ b/tests/unit/test_run_t0_interpro_ablation.py @@ -74,7 +74,22 @@ def test_the_archive_is_verified_and_the_subset_extracted(tmp_path): assert evaluate[evaluate.index("--cohort-interpro") + 1] == str(runbook.COHORT) -def _cut(subset, source, gafs=(runbook.T0_GAF, runbook.T1_GAF), evidence="all"): +#: Stand-ins for the two selecting GAFs' SHA-256s (the real files are not in CI). +GAF_HASHES = ("t0-sha", "t1-sha") + + +@pytest.fixture(autouse=True) +def _gaf_hashes(monkeypatch): + monkeypatch.setattr(runbook, "selection_hashes", lambda: GAF_HASHES) + + +def _cut( + subset, + source, + gafs=(runbook.T0_GAF, runbook.T1_GAF), + evidence="all", + hashes=GAF_HASHES, +): """A subset on disk whose marker records how it was extracted.""" from src.universe_provenance import write_marker @@ -88,6 +103,7 @@ def _cut(subset, source, gafs=(runbook.T0_GAF, runbook.T1_GAF), evidence="all"): n_matched_lines=0, tool="extract_human_interpro.py", evidence_filter=evidence, + selection_sha256=hashes, ) @@ -107,17 +123,19 @@ def test_a_subset_cut_by_this_exact_extraction_is_reused(tmp_path): @pytest.mark.parametrize( - "gafs,evidence", + "gafs,evidence,hashes", [ - ((runbook.T0_GAF,), "all"), # the t1-only no-knowledge proteins missing - ((runbook.T0_GAF, runbook.T1_GAF), "manual"), # the extract default - ((runbook.T0_GAF, runbook.T1_GAF), None), # marker predates the field + ((runbook.T0_GAF,), "all", GAF_HASHES), # t1-only no-knowledge missing + ((runbook.T0_GAF, runbook.T1_GAF), "manual", GAF_HASHES), # extract default + ((runbook.T0_GAF, runbook.T1_GAF), None, GAF_HASHES), # predates the field + ((runbook.T0_GAF, runbook.T1_GAF), "all", ("t0-sha", "old-t1")), # new t1 + ((runbook.T0_GAF, runbook.T1_GAF), "all", None), # predates the hashes ], ) -def test_a_subset_selected_differently_is_cut_again(tmp_path, gafs, evidence): +def test_a_subset_selected_differently_is_cut_again(tmp_path, gafs, evidence, hashes): source = tmp_path / "protein2ipr.dat.gz" subset = tmp_path / "subset.dat.gz" - _cut(subset, source, gafs, evidence) + _cut(subset, source, gafs, evidence, hashes) assert "extract" in _names(source, subset, tmp_path) diff --git a/tests/unit/test_universe_provenance.py b/tests/unit/test_universe_provenance.py index c638d34..8fc2172 100644 --- a/tests/unit/test_universe_provenance.py +++ b/tests/unit/test_universe_provenance.py @@ -64,6 +64,23 @@ def test_a_marker_from_before_the_evidence_filter_reads_as_unknown(self, extract marker_path(extract).write_text(json.dumps(payload)) assert read_marker(extract).evidence_filter is None + def test_selection_hashes_read_back_in_source_order(self, extract): + write_marker( + extract, + selection_rule="goa", + selection_sources=["t0.gaf.gz", "t1.gaf.gz"], + interpro_source="protein2ipr.dat.gz", + n_accessions=2, + n_matched_lines=2, + tool="extract_human_interpro.py", + selection_sha256=["aa", "bb"], + ) + assert read_marker(extract).selection_sha256 == ("aa", "bb") + + def test_a_marker_from_before_the_hashes_reads_as_unknown(self, extract): + write_goa_marker(extract) + assert read_marker(extract).selection_sha256 is None + def test_missing_marker_reads_as_none(self, extract): assert read_marker(extract) is None