From ca9d864fa2c7b65cde75e67d426d6be41adeaca8 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 30 Sep 2026 12:44:55 +0100 Subject: [PATCH 1/5] Entry-resolution tasks: prompts that name a protein, graded against all its entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every MVS task handed the model the PDB id. Real users don't: the first evaluator prompt to fail live was "I wanna see a structure of PDE5A", answered with 1UJ7 — an entry that does not exist. Probing found the quieter failure too: real ids for the wrong protein (a SARS-CoV-2 RBD for "nanobody bound to GFP"). - tasks/mvs_resolve/: six tasks (PDE5A, PCSK9, CFTR, GFP, myoglobin, lysozyme), each with `accepted_ids` = every PDB entry mapped to the protein's UniProt accession (RCSB search, recorded under provenance) and a minimal polymer-cartoon reference on one canonical entry. scripts/generate_resolve_tasks.py regenerates. - grade_mvs(accepted_refs=…): any accepted id in the prediction is folded onto the reference's; anything else (1UJ7) mismatches as before. Runner and escalation pass the task's set through. - Tests: accepted set sanity, any-accepted-entry == 1.0, the 1UJ7 case scores below it, multi-structure references are left alone. 22 pass. - ROADMAP: B4 resolution, B5 colour-scheme prompt drift, B6 structured outputs. Co-Authored-By: Claude Fable 5.1 --- ROADMAP.md | 16 + molbench/escalate.py | 3 +- molbench/mvs.py | 37 +- molbench/runner.py | 6 +- scripts/generate_resolve_tasks.py | 106 ++ tasks/mvs_resolve/res-cftr.json | 130 +++ tasks/mvs_resolve/res-gfp.json | 707 +++++++++++++ tasks/mvs_resolve/res-lysozyme.json | 1468 ++++++++++++++++++++++++++ tasks/mvs_resolve/res-myoglobin.json | 427 ++++++++ tasks/mvs_resolve/res-pcsk9.json | 136 +++ tasks/mvs_resolve/res-pde5a.json | 120 +++ tests/test_harness.py | 57 + 12 files changed, 3208 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_resolve_tasks.py create mode 100644 tasks/mvs_resolve/res-cftr.json create mode 100644 tasks/mvs_resolve/res-gfp.json create mode 100644 tasks/mvs_resolve/res-lysozyme.json create mode 100644 tasks/mvs_resolve/res-myoglobin.json create mode 100644 tasks/mvs_resolve/res-pcsk9.json create mode 100644 tasks/mvs_resolve/res-pde5a.json diff --git a/ROADMAP.md b/ROADMAP.md index a7aef6f..249b5e5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -78,6 +78,22 @@ strong judge, structured protocol, self-variance reporting); rubric quality search to extract interface/contact residues (ligand–protein, protein–protein), giving grounded "show the binding-site residues" / "show the PPI interface" tasks — a realistic, high-value class we currently lack. +* **B4. Entry resolution (STARTED — `tasks/mvs_resolve/`).** Every task used to hand + the model the PDB id; real users name the protein. The first live failure was + "a structure of PDE5A" → PDB 1UJ7 (does not exist), and probing found real ids + for the wrong protein too. Six tasks now grade against *all* entries for the + protein's UniProt accession (`accepted_ids`, folded by the grader). Next: more + targets, and an `exists` sub-metric (did the id resolve at RCSB at all?) — a + hallucinated id and a wrong-but-real id are different failures. +* **B5. Colour schemes.** The chat-driver's prompt now teaches Mol*'s + `custom.molstar_color_theme_name` on a `color` node for rainbow / by-chain / + by-SS colouring (models kept writing `"spectrum"` as a colour). Porting that here + needs the grader to treat a theme-coloured representation as equivalent to the + per-chain component references it would replace; until then the two prompts differ. +* **B6. Structured outputs.** Constrained decoding (`response_format: json_schema`, + `strict`) with a trimmed MVS schema makes invalid trees impossible by + construction. Measure validation failures + F1 against the prompt-only path on + the cheap models. * **B3. Corpus scaling & de-correlation.** More clinical targets (BRCA1, CFTR, kinases, more p53 structures incl. one with a numbering *offset* to exercise the SIFTS bridge); more Tier-1 template diversity so per-skill items are less diff --git a/molbench/escalate.py b/molbench/escalate.py index e2270cf..dbf73a0 100644 --- a/molbench/escalate.py +++ b/molbench/escalate.py @@ -37,8 +37,9 @@ def escalating_grade( tree_threshold: float = 0.999, visual_threshold: float = 0.97, workdir: str | pathlib.Path | None = None, + accepted_refs: set[str] | list[str] | None = None, ) -> dict[str, Any]: - g = grade_mvs(reference_tree, predicted_tree) + g = grade_mvs(reference_tree, predicted_tree, accepted_refs=accepted_refs) out: dict[str, Any] = {"tree_f1": g["f1"], "tier": "tree"} # T0 — tree match is conclusive (identical / normalised-equivalent). diff --git a/molbench/mvs.py b/molbench/mvs.py index 3dc5ebd..3499e74 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -170,8 +170,39 @@ def _path_similarity(p: tuple, q: tuple) -> float: # --- grading ---------------------------------------------------------------------- -def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]: - """Grade a predicted MVS scene against a reference. Returns precision/recall/f1.""" +def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], + accepted: set[str]) -> list[tuple]: + """Treat any accepted structure id in the prediction as the reference's own. + + A prompt that names a protein ("a structure of PDE5A") has many right answers; + the task lists them in ``accepted_ids``. The reference carries one canonical id, + so a prediction that downloaded another accepted entry is rewritten to that id + before matching. An id outside the set (a made-up one, say) stays as it is and + mismatches the download segment like any other wrong answer. + """ + ref_ids = {seg[1][1] for path in ref_paths for seg in path + if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref"} + if len(ref_ids) != 1: + return pred_paths # several structures: no single canonical id to fold onto + canonical = next(iter(ref_ids)) + accepted = {a.lower() for a in accepted} + + def fold(seg: tuple) -> tuple: + if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref" \ + and seg[1][1] in accepted: + return ("download", ("ref", canonical)) + return seg + + return [tuple(fold(seg) for seg in path) for path in pred_paths] + + +def grade_mvs(reference: Any, predicted: Any, + accepted_refs: set[str] | list[str] | None = None) -> dict[str, Any]: + """Grade a predicted MVS scene against a reference. Returns precision/recall/f1. + + ``accepted_refs``: other structure ids that count as the reference's (see + ``_fold_accepted_refs``); used by tasks that name a protein instead of an id. + """ ref_root, pred_root = extract_root(reference), extract_root(predicted) if ref_root is None: return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "error": "bad reference tree"} @@ -181,6 +212,8 @@ def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]: ref_paths = flatten_paths(ref_root) pred_paths = flatten_paths(pred_root) + if accepted_refs: + pred_paths = _fold_accepted_refs(ref_paths, pred_paths, set(accepted_refs)) n_ref, n_pred = len(ref_paths), len(pred_paths) if n_ref == 0: f1 = 1.0 if n_pred == 0 else 0.0 diff --git a/molbench/runner.py b/molbench/runner.py index 9298044..9a1783f 100644 --- a/molbench/runner.py +++ b/molbench/runner.py @@ -196,7 +196,8 @@ def run_mvs_task(model, system: str, task: dict) -> dict[str, Any]: tree, err = extract_json_object(raw) if err: return {"f1": 0.0, "precision": 0.0, "recall": 0.0, "error": err, "raw": raw} - result = mvs_mod.grade_mvs(task["reference_mvs"], tree) + result = mvs_mod.grade_mvs(task["reference_mvs"], tree, + accepted_refs=task.get("accepted_ids")) result["schema_errors"] = mvs_mod.validate_mvs(tree) result["predicted"] = tree result["raw"] = raw @@ -305,7 +306,8 @@ def run(models: list[str], categories: list[str] | None, wd = gallery / model.name.replace("/", "_") / task["id"] pt["escalation"] = escalating_grade( task["reference_mvs"], pt["predicted"], prompt=task.get("prompt"), - render=render_fn, vlm=vlm_fn, workdir=str(wd)) + render=render_fn, vlm=vlm_fn, workdir=str(wd), + accepted_refs=task.get("accepted_ids")) per_task.append(pt) raw_samples[model.name][task["id"]] = [ {"f1": g["f1"], "predicted": g.get("predicted"), diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py new file mode 100644 index 0000000..11875c2 --- /dev/null +++ b/scripts/generate_resolve_tasks.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Entry-resolution tasks: the prompt names a protein, not a PDB id. + +Every other MVS task hands the model the id. Real users don't: the first evaluator +prompt to fail in the wild was "I wanna see a structure of PDE5A", answered with +PDB 1UJ7 — an entry that does not exist. Probing further found the quieter failure: +real ids for the wrong protein (a SARS-CoV-2 RBD for "nanobody bound to GFP"). + +Many entries are a right answer here, so each task lists ``accepted_ids``: every PDB +entry whose polymer maps to the protein's UniProt accession, pulled from RCSB search +at authoring time and recorded under ``provenance``. The reference carries one +canonical entry; the grader folds any accepted id onto it (``mvs._fold_accepted_refs``). +The scene itself is deliberately minimal (a polymer cartoon) so the task measures +resolution, not styling. + + python scripts/generate_resolve_tasks.py # -> tasks/mvs_resolve/*.json +""" + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +import urllib.request + +from molbench.mvs import categorize + +REPO = pathlib.Path(__file__).resolve().parent.parent +OUT = REPO / "tasks" / "mvs_resolve" +CIF = "https://files.rcsb.org/download/{}.cif" +SEARCH = "https://search.rcsb.org/rcsbsearch/v2/query" + +# (slug, prompt, UniProt accession, canonical PDB id, note). Prompts are verbatim or +# lightly edited from the evaluator corpus; the canonical id is a well-known entry. +SPECS = [ + ("pde5a", "I wanna see a structure of PDE5A", "O76074", "1udt", + "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist."), + ("pcsk9", "show the structure of PCSK9", "Q8NBP7", "2p4e", + "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), + ("cftr", "show me the CFTR channel", "P13569", "5uak", + "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain)."), + ("gfp", "show green fluorescent protein", "P42212", "1ema", + "The starter chip's target; the model tends to be right here — a control."), + ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", + "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic."), + ("lysozyme", "Show me hen egg white lysozyme", "P00698", "1lyz", + "Starter chip; hundreds of valid entries — a control."), +] + + +def entries_for(accession: str) -> list[str]: + """All PDB entries with a polymer entity mapped to this UniProt accession.""" + query = { + "query": {"type": "terminal", "service": "text", "parameters": { + "attribute": "rcsb_polymer_entity_container_identifiers." + "reference_sequence_identifiers.database_accession", + "operator": "exact_match", "value": accession}}, + "return_type": "entry", + "request_options": {"paginate": {"start": 0, "rows": 10000}}, + } + req = urllib.request.Request(SEARCH, data=json.dumps(query).encode(), + headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=60) as resp: + if resp.status == 204: + return [] + return sorted(hit["identifier"].lower() for hit in json.load(resp)["result_set"]) + + +def reference(pdb: str) -> dict: + return {"kind": "root", "children": [{ + "kind": "download", "params": {"url": CIF.format(pdb)}, "children": [{ + "kind": "parse", "params": {"format": "mmcif"}, "children": [{ + "kind": "structure", "params": {"type": "model"}, "children": [{ + "kind": "component", "params": {"selector": "polymer"}, "children": [{ + "kind": "representation", "params": {"type": "cartoon"}}]}]}]}]}]} + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + today = dt.date.today().isoformat() + for slug, prompt, accession, canonical, note in SPECS: + ids = entries_for(accession) + if canonical not in ids: + raise SystemExit(f"{slug}: canonical {canonical} is not mapped to {accession}") + ref = reference(canonical) + task = { + "id": f"res-{slug}", + "category": "mvs", + "source": "resolve", + "title": f"Resolve by name: {slug}", + "prompt": prompt, + "reference_mvs": ref, + "accepted_ids": ids, + "categories": categorize(ref) + ["resolve"], + "skills": ["resolve", "download", "component", "representation"], + "notes": note, + "provenance": {"uniprot": accession, "n_entries": len(ids), + "source": "RCSB search, reference_sequence_identifiers", "fetched": today}, + } + (OUT / f"res-{slug}.json").write_text(json.dumps(task, indent=2) + "\n") + print(f"res-{slug:10s} {accession} {len(ids):4d} accepted entries canonical {canonical}") + + +if __name__ == "__main__": + main() diff --git a/tasks/mvs_resolve/res-cftr.json b/tasks/mvs_resolve/res-cftr.json new file mode 100644 index 0000000..3b75542 --- /dev/null +++ b/tasks/mvs_resolve/res-cftr.json @@ -0,0 +1,130 @@ +{ + "id": "res-cftr", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: cftr", + "prompt": "show me the CFTR channel", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/5uak.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "1xmi", + "1xmj", + "2bbo", + "2bbs", + "2bbt", + "2lob", + "2pze", + "2pzf", + "2pzg", + "3gd7", + "3isw", + "4wz6", + "5d2d", + "5d3e", + "5d3f", + "5tf7", + "5tf8", + "5tfa", + "5tfb", + "5tfc", + "5tfd", + "5tff", + "5tfg", + "5tfi", + "5tfj", + "5tgk", + "5uak", + "6gjq", + "6gjs", + "6gju", + "6gk4", + "6gkd", + "6hep", + "6msm", + "6o1v", + "6o2p", + "6uk1", + "6wbs", + "6ze1", + "7qi1", + "7sv7", + "7svd", + "7svr", + "8eig", + "8eio", + "8eiq", + "8ej1", + "8fzq", + "8gls", + "8ubr", + "8v7z", + "8v81", + "9dw4", + "9dw5", + "9dw7", + "9dw8", + "9dw9", + "9mxl", + "9y1q", + "9y4t" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", + "provenance": { + "uniprot": "P13569", + "n_entries": 60, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-gfp.json b/tasks/mvs_resolve/res-gfp.json new file mode 100644 index 0000000..9eb9385 --- /dev/null +++ b/tasks/mvs_resolve/res-gfp.json @@ -0,0 +1,707 @@ +{ + "id": "res-gfp", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: gfp", + "prompt": "show green fluorescent protein", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1ema.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "13lj", + "13lk", + "13ll", + "13lm", + "13ln", + "13lo", + "13lp", + "13lq", + "13lr", + "13ls", + "13lt", + "13lu", + "1b9c", + "1bfp", + "1c4f", + "1cv7", + "1ema", + "1emb", + "1emc", + "1eme", + "1emf", + "1emg", + "1emk", + "1eml", + "1emm", + "1f09", + "1f0b", + "1gfl", + "1h6r", + "1hcj", + "1huy", + "1jby", + "1jbz", + "1jc0", + "1jc1", + "1kp5", + "1kyp", + "1kyr", + "1kys", + "1myw", + "1q4a", + "1q4b", + "1q4c", + "1q4d", + "1q4e", + "1q73", + "1qxt", + "1qy3", + "1qyf", + "1qyo", + "1qyq", + "1rm9", + "1rmm", + "1rmo", + "1rmp", + "1rrx", + "1s6z", + "1w7s", + "1w7t", + "1w7u", + "1yfp", + "1yhg", + "1yhh", + "1yhi", + "1yj2", + "1yjf", + "1z1p", + "1z1q", + "24vi", + "24vl", + "24vm", + "24vn", + "24vo", + "24vp", + "24vq", + "24vr", + "24vs", + "25ex", + "29te", + "2ah8", + "2aha", + "2awj", + "2awk", + "2awl", + "2awm", + "2b3p", + "2b3q", + "2due", + "2duf", + "2dug", + "2duh", + "2dui", + "2emd", + "2emn", + "2emo", + "2fwq", + "2fzu", + "2g16", + "2g2s", + "2g3d", + "2g5z", + "2g6e", + "2h6v", + "2h9w", + "2hcg", + "2hfc", + "2hgd", + "2hgy", + "2hjo", + "2hqz", + "2hrs", + "2jad", + "2o24", + "2o29", + "2o2b", + "2okw", + "2oky", + "2q57", + "2q6p", + "2qrf", + "2qt2", + "2qu1", + "2qz0", + "2wsn", + "2wso", + "2wur", + "2y0g", + "2ydz", + "2ye0", + "2ye1", + "2yfp", + "31fw", + "38oa", + "38rk", + "38rx", + "3ai4", + "3ai5", + "3cb9", + "3cbe", + "3cd1", + "3cd9", + "3dpw", + "3dpx", + "3dpz", + "3dq1", + "3dq2", + "3dq3", + "3dq4", + "3dq5", + "3dq6", + "3dq7", + "3dq8", + "3dq9", + "3dqa", + "3dqc", + "3dqd", + "3dqe", + "3dqf", + "3dqh", + "3dqi", + "3dqj", + "3dqk", + "3dql", + "3dqm", + "3dqn", + "3dqo", + "3dqu", + "3ed8", + "3evp", + "3evr", + "3g9a", + "3gex", + "3gj1", + "3gj2", + "3i19", + "3k1k", + "3la1", + "3ogo", + "3p28", + "3sry", + "3ss0", + "3ssh", + "3ssk", + "3ssl", + "3ssp", + "3sst", + "3ssv", + "3ssy", + "3st0", + "3sv5", + "3svb", + "3svc", + "3svd", + "3sve", + "3u8p", + "3ufz", + "3ug0", + "3v3d", + "3vht", + "3w1c", + "3w1d", + "3ztf", + "4anj", + "4ar7", + "4as8", + "4b5y", + "4bdu", + "4en1", + "4eul", + "4ges", + "4gf6", + "4h47", + "4h48", + "4ik1", + "4ik3", + "4ik4", + "4ik5", + "4ik8", + "4ik9", + "4j88", + "4j89", + "4j8a", + "4jfg", + "4jrb", + "4ka9", + "4kag", + "4kex", + "4kf5", + "4kw4", + "4kw8", + "4kw9", + "4l12", + "4l13", + "4l1i", + "4lqt", + "4lqu", + "4lw5", + "4n3d", + "4ndj", + "4ndk", + "4ogs", + "4orn", + "4p1q", + "4p7h", + "4pa0", + "4pfe", + "4u2v", + "4xbi", + "4xgy", + "4xl5", + "4xov", + "4xow", + "4xvp", + "4z4k", + "4z4m", + "4zf3", + "4zf4", + "4zf5", + "5aqb", + "5b61", + "5bkf", + "5bkg", + "5dpg", + "5dph", + "5dpi", + "5dpj", + "5dqb", + "5dqm", + "5drf", + "5drg", + "5dtx", + "5dty", + "5dtz", + "5du0", + "5ehu", + "5f9g", + "5fgu", + "5hbd", + "5hge", + "5hw9", + "5hzo", + "5j2o", + "5j3n", + "5jzk", + "5jzl", + "5ktg", + "5lel", + "5lem", + "5ma3", + "5ma4", + "5ma5", + "5ma6", + "5ma8", + "5ma9", + "5mad", + "5mak", + "5mse", + "5n9o", + "5nhn", + "5ni3", + "5o89", + "5o8b", + "5o8c", + "5ox8", + "5ox9", + "5oxa", + "5oxb", + "5oxc", + "5t3i", + "5wj2", + "5wj3", + "5wj4", + "5z6y", + "6aa2", + "6aa6", + "6as9", + "6b7r", + "6b7t", + "6b9c", + "6dq0", + "6dq1", + "6efr", + "6fll", + "6fww", + "6gel", + "6gez", + "6go8", + "6go9", + "6gqg", + "6gqh", + "6grm", + "6hr1", + "6hut", + "6ir6", + "6ir7", + "6itc", + "6jgh", + "6jgi", + "6jgj", + "6kkz", + "6kl0", + "6kl1", + "6krg", + "6l27", + "6lnp", + "6lr7", + "6m63", + "6mb2", + "6mdr", + "6mks", + "6mwq", + "6oa8", + "6ofk", + "6ofl", + "6ofm", + "6ofn", + "6ofo", + "6og8", + "6og9", + "6oga", + "6ogb", + "6ogc", + "6pfr", + "6pfs", + "6pft", + "6pfu", + "6qq8", + "6qq9", + "6qqa", + "6qqb", + "6quh", + "6qui", + "6quj", + "6sm0", + "6t39", + "6t3a", + "6t90", + "6uhj", + "6uhk", + "6uhl", + "6uhm", + "6uhn", + "6uho", + "6uhp", + "6uhq", + "6uhr", + "6un4", + "6un5", + "6un6", + "6un7", + "6uz0", + "6uz3", + "6val", + "6vam", + "6vio", + "6wrf", + "6wsg", + "6wv9", + "6wva", + "6wvd", + "6wvf", + "6wvg", + "6xzf", + "6yov", + "6zsm", + "6zsn", + "6zui", + "7a7k", + "7a7l", + "7aa5", + "7amb", + "7amf", + "7amu", + "7bwn", + "7byl", + "7bym", + "7byn", + "7cd7", + "7cd8", + "7edj", + "7fhk", + "7fhl", + "7fhn", + "7fho", + "7k18", + "7km4", + "7ks0", + "7ks3", + "7kuy", + "7l31", + "7o7c", + "7o7d", + "7o7e", + "7o7h", + "7o7u", + "7o7v", + "7o7w", + "7o7x", + "7pca", + "7pcz", + "7pd0", + "7phr", + "7pnn", + "7sah", + "7sai", + "7sqy", + "7ssv", + "7ssx", + "7ssy", + "7ssz", + "7sun", + "7vcm", + "7x5v", + "7y96", + "7ydq", + "7yeu", + "7yv3", + "7yv5", + "8a6g", + "8a6n", + "8a6o", + "8a6p", + "8a6q", + "8a6r", + "8a6s", + "8a7v", + "8a83", + "8aha", + "8ahb", + "8am4", + "8b6s", + "8b6t", + "8ban", + "8bav", + "8bbj", + "8bvg", + "8bxp", + "8c1x", + "8c7i", + "8dfl", + "8dhr", + "8dn2", + "8dn3", + "8dn4", + "8dn5", + "8dpd", + "8dta", + "8et3", + "8f6p", + "8fck", + "8fed", + "8fee", + "8fef", + "8g0i", + "8g4e", + "8hco", + "8iyy", + "8iyz", + "8iz0", + "8iz1", + "8iz2", + "8iz3", + "8j0j", + "8j1e", + "8j6o", + "8j7h", + "8jf7", + "8jkc", + "8jkg", + "8jki", + "8jl2", + "8jl5", + "8jl6", + "8jl7", + "8jll", + "8jlm", + "8jls", + "8jlt", + "8jlu", + "8k4s", + "8kex", + "8ots", + "8ovn", + "8ovo", + "8ovp", + "8ovy", + "8qwj", + "8rl9", + "8rlb", + "8rlc", + "8rle", + "8sfs", + "8sfv", + "8sfx", + "8sfz", + "8sg3", + "8slc", + "8smu", + "8syg", + "8t15", + "8t17", + "8t18", + "8t6k", + "8t6l", + "8t6q", + "8tlm", + "8tu9", + "8u4n", + "8u4o", + "8u4p", + "8u4q", + "8u4r", + "8u4s", + "8uqq", + "8vdp", + "8vdq", + "8vdr", + "8w2l", + "8wg3", + "8wg4", + "8x9p", + "8xtw", + "8xtx", + "8xty", + "8ya0", + "8ya2", + "8zup", + "8zuq", + "8zur", + "8zus", + "8zut", + "9bcf", + "9boi", + "9c74", + "9c7c", + "9c7e", + "9c7p", + "9c7q", + "9c7r", + "9cla", + "9clv", + "9con", + "9cty", + "9d0y", + "9f22", + "9f23", + "9f24", + "9f33", + "9f34", + "9fen", + "9feq", + "9fgi", + "9haa", + "9hum", + "9irv", + "9k8w", + "9k8x", + "9lk2", + "9lk6", + "9lyl", + "9mus", + "9mut", + "9muu", + "9muv", + "9oo8", + "9osf", + "9osi", + "9pqc", + "9qks", + "9qxl", + "9rgb", + "9rl4", + "9rl5", + "9rmc", + "9rn1", + "9rn2", + "9rpb", + "9rpd", + "9s0t", + "9s2u", + "9sdl", + "9spe", + "9t9p", + "9tpa", + "9u9d", + "9ved", + "9vee", + "9vef", + "9w1y", + "9w24", + "9w25", + "9yop", + "9yp4", + "9yp9", + "9yr7", + "9yrg", + "9yrh" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "The starter chip's target; the model tends to be right here \u2014 a control.", + "provenance": { + "uniprot": "P42212", + "n_entries": 637, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-lysozyme.json b/tasks/mvs_resolve/res-lysozyme.json new file mode 100644 index 0000000..4845439 --- /dev/null +++ b/tasks/mvs_resolve/res-lysozyme.json @@ -0,0 +1,1468 @@ +{ + "id": "res-lysozyme", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: lysozyme", + "prompt": "Show me hen egg white lysozyme", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1lyz.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "132l", + "193l", + "194l", + "1a2y", + "1aki", + "1at5", + "1at6", + "1azf", + "1b0d", + "1b2k", + "1bgi", + "1bhz", + "1bvk", + "1bvx", + "1bwh", + "1bwi", + "1bwj", + "1c08", + "1c10", + "1dpw", + "1dpx", + "1dqj", + "1e8l", + "1f0w", + "1f10", + "1f3j", + "1fdl", + "1flq", + "1flu", + "1flw", + "1fly", + "1fn5", + "1g7h", + "1g7i", + "1g7j", + "1g7l", + "1g7m", + "1gpq", + "1gwd", + "1gxv", + "1gxx", + "1h6m", + "1h87", + "1hc0", + "1hel", + "1hem", + "1hen", + "1heo", + "1hep", + "1heq", + "1her", + "1hew", + "1hf4", + "1hsw", + "1hsx", + "1ic4", + "1ic5", + "1ic7", + "1iee", + "1io5", + "1ioq", + "1ior", + "1ios", + "1iot", + "1ir7", + "1ir8", + "1ir9", + "1j1o", + "1j1p", + "1j1x", + "1ja2", + "1ja4", + "1ja6", + "1ja7", + "1jis", + "1jit", + "1jiy", + "1jj0", + "1jj1", + "1jj3", + "1jpo", + "1jto", + "1jtt", + "1kip", + "1kiq", + "1kir", + "1kxw", + "1kxx", + "1kxy", + "1lcn", + "1lj3", + "1lj4", + "1lje", + "1ljf", + "1ljg", + "1ljh", + "1lji", + "1ljj", + "1ljk", + "1lkr", + "1lks", + "1lma", + "1lpi", + "1lsa", + "1lsb", + "1lsc", + "1lsd", + "1lse", + "1lsf", + "1lsg", + "1lsm", + "1lsn", + "1lsy", + "1lsz", + "1lyo", + "1lys", + "1lyz", + "1lz8", + "1lz9", + "1lza", + "1lzb", + "1lzc", + "1lzd", + "1lze", + "1lzg", + "1lzh", + "1lzn", + "1lzt", + "1mel", + "1mlc", + "1n4f", + "1nby", + "1nbz", + "1ndg", + "1ndm", + "1p2c", + "1ps5", + "1qio", + "1qtk", + "1rcm", + "1rfp", + "1ri8", + "1rjc", + "1sf4", + "1sf6", + "1sf7", + "1sfb", + "1sfg", + "1sq2", + "1t3p", + "1t6v", + "1ua6", + "1uc0", + "1uco", + "1uia", + "1uib", + "1uic", + "1uid", + "1uie", + "1uif", + "1uig", + "1uih", + "1uuz", + "1v7s", + "1v7t", + "1vat", + "1vau", + "1vdp", + "1vdq", + "1vds", + "1vdt", + "1ved", + "1vfb", + "1w6z", + "1wtm", + "1wtn", + "1xei", + "1xej", + "1xek", + "1xfp", + "1xgp", + "1xgq", + "1xgr", + "1xgt", + "1xgu", + "1yik", + "1yil", + "1ykx", + "1yky", + "1ykz", + "1yl0", + "1yl1", + "1yqv", + "1z55", + "1zmy", + "1zv5", + "1zvh", + "1zvy", + "23si", + "29lj", + "29mn", + "29mr", + "29my", + "29nx", + "29oh", + "29ol", + "2a6u", + "2a7d", + "2a7f", + "2aub", + "2b5z", + "2blx", + "2bly", + "2bpu", + "2c8o", + "2c8p", + "2cds", + "2cgi", + "2d4i", + "2d4j", + "2d4k", + "2d6b", + "2d91", + "2dqc", + "2dqd", + "2dqe", + "2dqf", + "2dqg", + "2dqh", + "2dqi", + "2dqj", + "2eiz", + "2eks", + "2epe", + "2f2n", + "2f30", + "2f4a", + "2f4g", + "2fbb", + "2g4p", + "2g4q", + "2h9j", + "2h9k", + "2hs7", + "2hs9", + "2hso", + "2htx", + "2hu1", + "2hu3", + "2hub", + "2i25", + "2i26", + "2i6z", + "2iff", + "2lym", + "2lyo", + "2lyz", + "2lzh", + "2lzt", + "2pc2", + "2q0m", + "2vb1", + "2w1l", + "2w1m", + "2w1x", + "2w1y", + "2war", + "2x0a", + "2xbr", + "2xbs", + "2xjw", + "2xth", + "2ybh", + "2ybi", + "2ybj", + "2ybl", + "2ybm", + "2ybn", + "2ydg", + "2yss", + "2yvb", + "2z12", + "2z18", + "2z19", + "2znw", + "2znx", + "2zq3", + "2zq4", + "2zyp", + "30ig", + "30js", + "30ju", + "30jx", + "30jy", + "3a34", + "3a3q", + "3a3r", + "3a67", + "3a6b", + "3a6c", + "3a8z", + "3a90", + "3a91", + "3a92", + "3a93", + "3a94", + "3a95", + "3a96", + "3agg", + "3agh", + "3agi", + "3ajn", + "3atn", + "3ato", + "3aw6", + "3aw7", + "3az4", + "3az5", + "3az6", + "3az7", + "3b6l", + "3b72", + "3d9a", + "3e3d", + "3ems", + "3exd", + "3f6z", + "3g3a", + "3g3b", + "3hfm", + "3iju", + "3ijv", + "3j4g", + "3j6k", + "3kam", + "3lym", + "3lyo", + "3lyt", + "3lyz", + "3lzt", + "3m18", + "3m3u", + "3mbe", + "3n9a", + "3n9c", + "3n9e", + "3ojp", + "3ok0", + "3otp", + "3p4z", + "3p64", + "3p65", + "3p66", + "3p68", + "3qe8", + "3qng", + "3qy4", + "3rnx", + "3rt5", + "3ru5", + "3rw8", + "3rz4", + "3sp3", + "3t6u", + "3tmu", + "3tmv", + "3tmw", + "3tmx", + "3txb", + "3txd", + "3txe", + "3txf", + "3txg", + "3txh", + "3txi", + "3txj", + "3txk", + "3ulr", + "3w6a", + "3wl2", + "3wmk", + "3wpj", + "3wpk", + "3wpl", + "3wu7", + "3wu8", + "3wu9", + "3wua", + "3wul", + "3wum", + "3wun", + "3wvx", + "3wvy", + "3ww5", + "3ww6", + "3wxt", + "3wxu", + "3zek", + "3zvq", + "45kk", + "4a7d", + "4a8a", + "4a8b", + "4aga", + "4axt", + "4b0d", + "4b1a", + "4b49", + "4b4e", + "4b4i", + "4b4j", + "4bad", + "4baf", + "4bap", + "4bs7", + "4c3w", + "4cj2", + "4d9z", + "4dc4", + "4dd0", + "4dd1", + "4dd2", + "4dd3", + "4dd7", + "4dd9", + "4ddb", + "4ddc", + "4dt3", + "4e3u", + "4eof", + "4et8", + "4et9", + "4eta", + "4etb", + "4etc", + "4etd", + "4ete", + "4fjr", + "4g49", + "4g4b", + "4g4c", + "4g4h", + "4gcb", + "4gcc", + "4gcd", + "4gce", + "4gcf", + "4gla", + "4glv", + "4gn3", + "4gn4", + "4gn5", + "4h1p", + "4h8x", + "4h8y", + "4h8z", + "4h90", + "4h91", + "4h92", + "4h93", + "4h94", + "4h9a", + "4h9b", + "4h9c", + "4h9e", + "4h9f", + "4h9h", + "4h9i", + "4hp0", + "4hpi", + "4hsf", + "4htk", + "4htn", + "4htq", + "4hv1", + "4hv2", + "4i8s", + "4ias", + "4iat", + "4ii8", + "4j1a", + "4j1b", + "4j7v", + "4kxi", + "4lfp", + "4lfx", + "4lgk", + "4lt0", + "4lt1", + "4lt2", + "4lt3", + "4lyb", + "4lyc", + "4lym", + "4lyo", + "4lyt", + "4lyz", + "4lzt", + "4m4o", + "4m6d", + "4mr1", + "4n0j", + "4n1c", + "4n1e", + "4n5r", + "4n8z", + "4n9r", + "4neb", + "4nfv", + "4ng1", + "4ng8", + "4ngi", + "4ngj", + "4ngk", + "4ngl", + "4ngo", + "4ngv", + "4ngw", + "4ngy", + "4ngz", + "4nhi", + "4nhp", + "4nhq", + "4nhs", + "4nht", + "4nij", + "4nsg", + "4nsh", + "4nsi", + "4nsj", + "4nwe", + "4nwh", + "4ny5", + "4o34", + "4ooo", + "4oot", + "4owa", + "4owb", + "4owc", + "4owe", + "4owh", + "4p2e", + "4pgj", + "4phi", + "4ppo", + "4prq", + "4pru", + "4qeq", + "4qgz", + "4qy9", + "4r0f", + "4r6c", + "4rds", + "4rlm", + "4rln", + "4rw1", + "4rw2", + "4tsa", + "4tsb", + "4tsc", + "4ttd", + "4tun", + "4tws", + "4u3x", + "4uwn", + "4uwu", + "4uwv", + "4w94", + "4w96", + "4wg1", + "4wg7", + "4wl6", + "4wl7", + "4wld", + "4wlt", + "4wlx", + "4wly", + "4wm1", + "4wm2", + "4wm3", + "4wm4", + "4wm5", + "4wm6", + "4wmg", + "4wo6", + "4wo9", + "4woa", + "4x3b", + "4xad", + "4xen", + "4xjb", + "4xjd", + "4xjf", + "4xjg", + "4xjh", + "4xji", + "4xn6", + "4xyy", + "4yem", + "4yen", + "4yeo", + "4ym8", + "4yop", + "4z3m", + "4z41", + "4z46", + "4z98", + "4zee", + "4zfp", + "4zix", + "5a3e", + "5a3z", + "5amy", + "5apc", + "5apd", + "5ape", + "5apf", + "5b05", + "5b06", + "5b07", + "5b1f", + "5b1g", + "5b59", + "5b5j", + "5c6i", + "5c6j", + "5c6l", + "5d5c", + "5d5f", + "5dl9", + "5dla", + "5dm9", + "5e4p", + "5e9r", + "5ebh", + "5f14", + "5f16", + "5f81", + "5f9u", + "5f9x", + "5fcp", + "5fdj", + "5fek", + "5fel", + "5fhw", + "5fst", + "5gwb", + "5hll", + "5hmj", + "5hmv", + "5hnc", + "5hnl", + "5hq1", + "5i4w", + "5i4x", + "5i4y", + "5i53", + "5i54", + "5i5q", + "5idd", + "5iel", + "5ihg", + "5ii3", + "5ilc", + "5ilf", + "5j7c", + "5jen", + "5k2k", + "5k2n", + "5k2p", + "5k2q", + "5k2r", + "5k2s", + "5k7o", + "5kj9", + "5kki", + "5kkj", + "5kxk", + "5kxl", + "5kxm", + "5kxn", + "5kxo", + "5kxp", + "5kxr", + "5kxs", + "5kxt", + "5kxw", + "5kxx", + "5kxy", + "5kxz", + "5ky1", + "5l3h", + "5l3i", + "5l9j", + "5la5", + "5la8", + "5laf", + "5lag", + "5lan", + "5lin", + "5lio", + "5lvg", + "5lvh", + "5lvi", + "5lvj", + "5lxw", + "5lym", + "5lyt", + "5lyz", + "5m1y", + "5mjj", + "5myy", + "5nbj", + "5ne0", + "5nj1", + "5njm", + "5njp", + "5njq", + "5njr", + "5njs", + "5o4w", + "5o4x", + "5o6q", + "5ob6", + "5ob7", + "5ob8", + "5ob9", + "5ocv", + "5oer", + "5ole", + "5oov", + "5ooy", + "5op1", + "5t3f", + "5tk0", + "5uvj", + "5v4g", + "5v4h", + "5v4i", + "5wr9", + "5wra", + "5wrb", + "5yin", + "5yky", + "6a10", + "6a4n", + "6a4o", + "6a4p", + "6a4q", + "6abn", + "6abz", + "6ac2", + "6ad5", + "6adf", + "6aea", + "6agn", + "6agr", + "6ahh", + "6ahl", + "6apm", + "6b7u", + "6b7v", + "6b7w", + "6bo1", + "6bo2", + "6bre", + "6ciw", + "6d6e", + "6d6f", + "6d6g", + "6d6h", + "6f1l", + "6f1m", + "6f1o", + "6f1p", + "6f1r", + "6f2i", + "6f2j", + "6f2k", + "6f9x", + "6f9y", + "6f9z", + "6fa0", + "6fro", + "6frq", + "6fsj", + "6ftr", + "6g5c", + "6g5v", + "6g5y", + "6g8a", + "6gf0", + "6gnl", + "6gob", + "6goh", + "6goi", + "6goj", + "6h0k", + "6h0l", + "6h3b", + "6h79", + "6ht2", + "6hu5", + "6hy4", + "6hy6", + "6hy8", + "6hyb", + "6ig6", + "6irj", + "6jb2", + "6jb5", + "6jb8", + "6jxp", + "6jxq", + "6jzi", + "6k5q", + "6k8g", + "6kcb", + "6kcd", + "6kd1", + "6lav", + "6ll3", + "6lt5", + "6lyt", + "6lyz", + "6mqv", + "6muz", + "6mx9", + "6o2h", + "6p4a", + "6p4b", + "6p4d", + "6pbb", + "6pl9", + "6pla", + "6plb", + "6q88", + "6q8r", + "6q8t", + "6qea", + "6qnb", + "6qq3", + "6qqc", + "6qqd", + "6qqe", + "6qqf", + "6qww", + "6qwx", + "6qwy", + "6qwz", + "6qx0", + "6qxw", + "6qxx", + "6qxy", + "6qy0", + "6qy1", + "6qy2", + "6qy4", + "6qy5", + "6rnb", + "6rnc", + "6ro3", + "6ro5", + "6rt1", + "6rt3", + "6rt9", + "6rta", + "6rxi", + "6s2n", + "6s7n", + "6set", + "6seu", + "6sew", + "6sex", + "6sez", + "6sij", + "6sik", + "6sil", + "6sim", + "6sr0", + "6sr1", + "6sr2", + "6sr3", + "6sr4", + "6sr5", + "6syc", + "6syd", + "6sye", + "6tvl", + "6tvy", + "6w7p", + "6w8e", + "6web", + "6wec", + "6wgo", + "6wrl", + "6wrm", + "6wx5", + "6ybf", + "6ybi", + "6yjx", + "6yob", + "6yoc", + "6yod", + "6yoe", + "6zs8", + "7a70", + "7ac2", + "7ave", + "7avf", + "7avg", + "7b9j", + "7baz", + "7bb1", + "7bcu", + "7bcx", + "7bd0", + "7bdm", + "7bdz", + "7be0", + "7be1", + "7be2", + "7beb", + "7bec", + "7bhk", + "7bhl", + "7bhm", + "7bhn", + "7bmo", + "7bmp", + "7bmq", + "7bmr", + "7bms", + "7bmt", + "7br5", + "7bvm", + "7bvo", + "7byo", + "7byp", + "7c09", + "7cdk", + "7cdm", + "7cdn", + "7cdo", + "7cdp", + "7cdq", + "7cdr", + "7cds", + "7cdt", + "7cdu", + "7cjz", + "7cqo", + "7cvj", + "7cvl", + "7d01", + "7d02", + "7d04", + "7d05", + "7d0w", + "7deq", + "7der", + "7dln", + "7dtb", + "7dtf", + "7e02", + "7f26", + "7fcu", + "7fcw", + "7fg8", + "7fgu", + "7fgv", + "7jmu", + "7kh5", + "7knk", + "7l84", + "7llp", + "7ln8", + "7ln9", + "7loq", + "7lor", + "7lp6", + "7lpl", + "7lpm", + "7lyz", + "7mrp", + "7njf", + "7nkf", + "7o2o", + "7ol5", + "7ol6", + "7ol7", + "7ol8", + "7p6m", + "7pnh", + "7q04", + "7q05", + "7q06", + "7q0t", + "7q0u", + "7q0v", + "7qnp", + "7qq1", + "7r1q", + "7rxy", + "7ryd", + "7ryk", + "7rz0", + "7rz1", + "7rz2", + "7s27", + "7s28", + "7s29", + "7s2a", + "7s2b", + "7s2c", + "7s2d", + "7s2e", + "7s2f", + "7s2g", + "7s2q", + "7s2u", + "7s2v", + "7s2w", + "7s30", + "7s31", + "7s32", + "7s33", + "7s34", + "7s35", + "7s4w", + "7skw", + "7tum", + "7uly", + "7vei", + "7vgo", + "7vgp", + "7wbd", + "7wbe", + "7wbf", + "7wkr", + "7wuc", + "7wxs", + "7y6a", + "7ynu", + "7ynv", + "7yrk", + "7z6j", + "7zu6", + "8a9d", + "8a9e", + "8a9r", + "8aaz", + "8aj3", + "8aj4", + "8aj5", + "8b3l", + "8b3t", + "8b3u", + "8b3v", + "8bov", + "8boy", + "8bph", + "8bpj", + "8bpu", + "8bqm", + "8bqp", + "8bqq", + "8bqr", + "8bqt", + "8c39", + "8cl5", + "8cl6", + "8cpc", + "8cvu", + "8cvv", + "8cvw", + "8cw0", + "8cw1", + "8cw3", + "8cw5", + "8cw6", + "8cw7", + "8cw8", + "8cwb", + "8cwc", + "8cwd", + "8cwe", + "8cwf", + "8cwg", + "8cwh", + "8d5s", + "8d5t", + "8d5u", + "8d5w", + "8d5x", + "8d5z", + "8d60", + "8d61", + "8d62", + "8d69", + "8d6b", + "8d6i", + "8d75", + "8d77", + "8d7a", + "8d7b", + "8d7c", + "8d7d", + "8d7j", + "8d7l", + "8d7q", + "8d7s", + "8d8a", + "8d8b", + "8d8c", + "8d8d", + "8d8e", + "8d8f", + "8d8g", + "8d8h", + "8dct", + "8dcu", + "8dcv", + "8dcw", + "8dyz", + "8dz7", + "8e54", + "8ezo", + "8ezp", + "8ezu", + "8ezx", + "8f00", + "8f0b", + "8f28", + "8f2g", + "8fp6", + "8fp7", + "8fp8", + "8fpb", + "8fpd", + "8fpm", + "8fpn", + "8fpp", + "8fpr", + "8fpu", + "8fry", + "8fs0", + "8fs9", + "8fsa", + "8fsc", + "8fsf", + "8fsg", + "8fsh", + "8fsm", + "8fsn", + "8fst", + "8fsu", + "8fsv", + "8fsw", + "8fsx", + "8fsy", + "8ft0", + "8ft1", + "8ft2", + "8ft3", + "8gmv", + "8gmw", + "8gq1", + "8h3w", + "8h8t", + "8h8u", + "8h8v", + "8h8w", + "8jqv", + "8k8m", + "8lyz", + "8om8", + "8oms", + "8omt", + "8owc", + "8p1a", + "8p1b", + "8p1c", + "8p1d", + "8p2q", + "8pft", + "8pfu", + "8pfv", + "8pfw", + "8pfx", + "8pfy", + "8ph5", + "8ph6", + "8ph7", + "8ph8", + "8piw", + "8pob", + "8pte", + "8qcu", + "8qv8", + "8reg", + "8reh", + "8rge", + "8rlf", + "8rlg", + "8rlh", + "8rli", + "8rnv", + "8rnw", + "8rnx", + "8rny", + "8rpm", + "8rsh", + "8rtj", + "8rtk", + "8rus", + "8s2u", + "8s2v", + "8scy", + "8sil", + "8w6k", + "8wdi", + "8wzf", + "8wzg", + "8wzr", + "8wzt", + "8wzv", + "8y1r", + "8y1s", + "8y9w", + "8ya1", + "8ya4", + "8ybg", + "8ybh", + "8z51", + "8z52", + "8zst", + "8zsu", + "9b7c", + "9b7e", + "9b7f", + "9c9h", + "9enz", + "9eo2", + "9evu", + "9ex0", + "9ex1", + "9ex2", + "9fa5", + "9fc3", + "9fc4", + "9fc5", + "9fc6", + "9fc7", + "9fc8", + "9fc9", + "9fca", + "9fcb", + "9fcc", + "9fgx", + "9fgy", + "9fmy", + "9fud", + "9fue", + "9gcv", + "9ghx", + "9gi7", + "9gog", + "9gok", + "9gyh", + "9h3e", + "9h7w", + "9hc2", + "9hlk", + "9hmk", + "9hmq", + "9hru", + "9hti", + "9htj", + "9i6j", + "9i6k", + "9i6l", + "9i6m", + "9i6n", + "9i6o", + "9i6p", + "9i8l", + "9ig6", + "9j0l", + "9j0m", + "9k9z", + "9ka0", + "9kj7", + "9kot", + "9l1l", + "9l71", + "9l9r", + "9lmk", + "9lml", + "9lyz", + "9mp3", + "9mp4", + "9mp5", + "9mp6", + "9mp7", + "9mp8", + "9mp9", + "9mpa", + "9orv", + "9orw", + "9orx", + "9ory", + "9orz", + "9os0", + "9os1", + "9os8", + "9oz5", + "9p0y", + "9p0z", + "9p10", + "9p12", + "9p13", + "9p1w", + "9p56", + "9p57", + "9p58", + "9p59", + "9p5a", + "9p5b", + "9p5d", + "9p5e", + "9p5f", + "9p5g", + "9p5q", + "9p5r", + "9p5s", + "9p5t", + "9p5u", + "9p5v", + "9p5w", + "9p5x", + "9p5y", + "9p5z", + "9p60", + "9p61", + "9p62", + "9p63", + "9p64", + "9p65", + "9p66", + "9p67", + "9p68", + "9p69", + "9p6r", + "9qa9", + "9qsq", + "9qtl", + "9qum", + "9rbg", + "9rbt", + "9rbv", + "9rrk", + "9rrl", + "9ruv", + "9rv6", + "9rvi", + "9rvo", + "9rzl", + "9rzr", + "9rzs", + "9rzt", + "9s2a", + "9sm9", + "9smc", + "9tbl", + "9tej", + "9tek", + "9toq", + "9tor", + "9tos", + "9tot", + "9ugi", + "9uyr", + "9v3d", + "9v3g", + "9v3h", + "9v3i", + "9v3j", + "9v3k", + "9v3l", + "9v3m", + "9v3n", + "9v3o", + "9v3p", + "9v3q", + "9v3r", + "9wks", + "9xua" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Starter chip; hundreds of valid entries \u2014 a control.", + "provenance": { + "uniprot": "P00698", + "n_entries": 1398, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-myoglobin.json b/tasks/mvs_resolve/res-myoglobin.json new file mode 100644 index 0000000..3c7d0be --- /dev/null +++ b/tasks/mvs_resolve/res-myoglobin.json @@ -0,0 +1,427 @@ +{ + "id": "res-myoglobin", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: myoglobin", + "prompt": "Show me sperm whale myoglobin", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1mbn.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "101m", + "102m", + "103m", + "104m", + "105m", + "106m", + "107m", + "108m", + "109m", + "110m", + "111m", + "112m", + "1a6g", + "1a6k", + "1a6m", + "1a6n", + "1abs", + "1ajg", + "1ajh", + "1bvc", + "1bvd", + "1bz6", + "1bzp", + "1bzr", + "1ch1", + "1ch2", + "1ch3", + "1ch5", + "1ch7", + "1ch9", + "1cik", + "1cio", + "1co8", + "1co9", + "1cp0", + "1cp5", + "1cpw", + "1cq2", + "1do1", + "1do3", + "1do4", + "1do7", + "1dti", + "1dtm", + "1duk", + "1duo", + "1dxc", + "1dxd", + "1ebc", + "1f63", + "1f65", + "1f6h", + "1fcs", + "1h1x", + "1hjt", + "1iop", + "1irc", + "1j3f", + "1j52", + "1jdo", + "1jp6", + "1jp8", + "1jp9", + "1jpb", + "1jw8", + "1l2k", + "1ltw", + "1lue", + "1mbc", + "1mbd", + "1mbi", + "1mbn", + "1mbo", + "1mcy", + "1mgn", + "1mlf", + "1mlg", + "1mlh", + "1mlj", + "1mlk", + "1mll", + "1mlm", + "1mln", + "1mlo", + "1mlq", + "1mlr", + "1mls", + "1mlu", + "1moa", + "1mob", + "1moc", + "1mod", + "1mti", + "1mtj", + "1mtk", + "1myf", + "1mym", + "1myz", + "1mz0", + "1n9f", + "1n9h", + "1n9i", + "1n9x", + "1naz", + "1o16", + "1obm", + "1ofj", + "1ofk", + "1spe", + "1swm", + "1tes", + "1u7r", + "1u7s", + "1ufj", + "1ufp", + "1v9q", + "1vxa", + "1vxb", + "1vxc", + "1vxd", + "1vxe", + "1vxf", + "1vxg", + "1vxh", + "1wvp", + "1yog", + "1yoh", + "1yoi", + "2blh", + "2bli", + "2blj", + "2bw9", + "2bwh", + "2cmm", + "2d6c", + "2e2y", + "2eb8", + "2eb9", + "2ef2", + "2ekt", + "2eku", + "2evk", + "2evp", + "2g0r", + "2g0s", + "2g0v", + "2g0x", + "2g0z", + "2g10", + "2g11", + "2g12", + "2g14", + "2jho", + "2mb5", + "2mbw", + "2mga", + "2mgb", + "2mgc", + "2mgd", + "2mge", + "2mgf", + "2mgg", + "2mgh", + "2mgi", + "2mgj", + "2mgk", + "2mgl", + "2mgm", + "2mya", + "2myb", + "2myc", + "2myd", + "2mye", + "2oh8", + "2oh9", + "2oha", + "2ohb", + "2spl", + "2spm", + "2spn", + "2spo", + "2w6w", + "2w6x", + "2w6y", + "2z6s", + "2z6t", + "2zsn", + "2zso", + "2zsp", + "2zsq", + "2zsr", + "2zss", + "2zst", + "2zsx", + "2zsy", + "2zsz", + "2zt0", + "2zt1", + "2zt2", + "2zt3", + "2zt4", + "3a2g", + "3ase", + "3e4n", + "3e55", + "3e5i", + "3e5o", + "3ecl", + "3ecx", + "3ecz", + "3ed9", + "3eda", + "3edb", + "3h57", + "3h58", + "3k9z", + "3m38", + "3m39", + "3m3a", + "3m3b", + "3mn0", + "3nml", + "3o89", + "3ogb", + "3sdn", + "3u3e", + "4fwx", + "4fwy", + "4fwz", + "4h07", + "4h0b", + "4it8", + "4lpi", + "4mbn", + "4mxk", + "4mxl", + "4nxa", + "4nxc", + "4of9", + "4ood", + "4pnj", + "4pq6", + "4pqb", + "4pqc", + "4qau", + "4tyx", + "5b84", + "5b85", + "5c6y", + "5hav", + "5hlq", + "5hlu", + "5hlx", + "5iks", + "5ile", + "5ilm", + "5ilp", + "5ilr", + "5jom", + "5kd1", + "5kkk", + "5m3s", + "5mbn", + "5o41", + "5oj9", + "5oja", + "5ojb", + "5ojc", + "5ut7", + "5ut8", + "5ut9", + "5uta", + "5utb", + "5utc", + "5utd", + "5vnu", + "5vrt", + "5vzn", + "5vzo", + "5vzp", + "5vzq", + "5wjk", + "5xkv", + "5xkw", + "5xl0", + "5yce", + "5ych", + "5yzf", + "5zeo", + "5zzf", + "5zzg", + "6cf0", + "6d45", + "6e02", + "6e03", + "6e04", + "6f17", + "6f18", + "6f19", + "6f1a", + "6g5a", + "6g5b", + "6g5t", + "6jp1", + "6krc", + "6krf", + "6m8f", + "6mv0", + "6n02", + "6n03", + "6z4r", + "6z4t", + "7a44", + "7a45", + "7cen", + "7cez", + "7ehx", + "7kyr", + "7l3u", + "7l3y", + "7slh", + "7sli", + "7spe", + "7spf", + "7spg", + "7sph", + "7vdn", + "7vuc", + "7vw4", + "7xc9", + "7xcf", + "7xcq", + "7ylk", + "8eko", + "8ess", + "8esu", + "8f9h", + "8f9i", + "8f9j", + "8f9n", + "8fb0", + "8fdj", + "8j4k", + "8j4l", + "8kfh", + "8kfi", + "8kfj", + "8qba", + "8qbc", + "8xbj", + "8xc2", + "9dth", + "9dti", + "9ijo", + "9lmb", + "9p1e", + "9p1f" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic.", + "provenance": { + "uniprot": "P02185", + "n_entries": 357, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-pcsk9.json b/tasks/mvs_resolve/res-pcsk9.json new file mode 100644 index 0000000..4712907 --- /dev/null +++ b/tasks/mvs_resolve/res-pcsk9.json @@ -0,0 +1,136 @@ +{ + "id": "res-pcsk9", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: pcsk9", + "prompt": "show the structure of PCSK9", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/2p4e.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "10sb", + "2p4e", + "2pmw", + "2qtw", + "2w2m", + "2w2n", + "2w2o", + "2w2p", + "2w2q", + "2xtj", + "37hf", + "3bps", + "3gcw", + "3gcx", + "3h42", + "3m0c", + "3p5b", + "3p5c", + "3sqo", + "4k8r", + "4ne9", + "4nmx", + "4ov6", + "5oca", + "5vl7", + "5vla", + "5vlh", + "5vlk", + "5vll", + "5vlp", + "6e4y", + "6e4z", + "6mv5", + "6olz", + "6om0", + "6om7", + "6u26", + "6u2f", + "6u2n", + "6u2p", + "6u36", + "6u38", + "6u3i", + "6u3x", + "6xib", + "6xic", + "6xid", + "6xie", + "6xif", + "7anq", + "7kev", + "7kfa", + "7s5g", + "7s5h", + "8fpo", + "8fpq", + "8fvl", + "8fvm", + "8fvn", + "8fvo", + "8fvp", + "8fvq", + "8vdv", + "8wfr", + "9hz3", + "9szy" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL).", + "provenance": { + "uniprot": "Q8NBP7", + "n_entries": 66, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tasks/mvs_resolve/res-pde5a.json b/tasks/mvs_resolve/res-pde5a.json new file mode 100644 index 0000000..8b6c508 --- /dev/null +++ b/tasks/mvs_resolve/res-pde5a.json @@ -0,0 +1,120 @@ +{ + "id": "res-pde5a", + "category": "mvs", + "source": "resolve", + "title": "Resolve by name: pde5a", + "prompt": "I wanna see a structure of PDE5A", + "reference_mvs": { + "kind": "root", + "children": [ + { + "kind": "download", + "params": { + "url": "https://files.rcsb.org/download/1udt.cif" + }, + "children": [ + { + "kind": "parse", + "params": { + "format": "mmcif" + }, + "children": [ + { + "kind": "structure", + "params": { + "type": "model" + }, + "children": [ + { + "kind": "component", + "params": { + "selector": "polymer" + }, + "children": [ + { + "kind": "representation", + "params": { + "type": "cartoon" + } + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "accepted_ids": [ + "1rkp", + "1t9r", + "1t9s", + "1tbf", + "1udt", + "1udu", + "1uho", + "1xoz", + "1xp0", + "2chm", + "2h40", + "2h42", + "2h44", + "2xss", + "3b2r", + "3bjc", + "3hc8", + "3hdz", + "3jwq", + "3jwr", + "3lfv", + "3mf0", + "3shy", + "3shz", + "3sie", + "3tge", + "3tgg", + "4g2w", + "4g2y", + "4i9z", + "4ia0", + "4md6", + "4oew", + "4oex", + "5jo3", + "5zz2", + "6acb", + "6iwi", + "6l6e", + "6vbi", + "7faq", + "7far", + "8w4s", + "8w4t", + "8wkf", + "8wkg", + "8xww", + "9lsl", + "9lsm", + "9lss" + ], + "categories": [ + "load", + "representation", + "resolve" + ], + "skills": [ + "resolve", + "download", + "component", + "representation" + ], + "notes": "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist.", + "provenance": { + "uniprot": "O76074", + "n_entries": 50, + "source": "RCSB search, reference_sequence_identifiers", + "fetched": "2026-09-30" + } +} diff --git a/tests/test_harness.py b/tests/test_harness.py index c64dd94..5a89d16 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -196,3 +196,60 @@ def test_run_archives_raw_samples_and_meta(): test_render_state_wrapping() test_run_archives_raw_samples_and_meta() print("all sanity checks passed") + + +# --- entry resolution (tasks/mvs_resolve) ------------------------------------------ + +def _resolve_tasks(): + return [t for t in load_tasks(["mvs"]) if t.get("source") == "resolve"] + + +def _with_pdb(task: dict, pdb: str) -> dict: + """The task's own reference scene, downloading a different entry.""" + tree = json.loads(json.dumps(task["reference_mvs"])) + tree["children"][0]["params"]["url"] = f"https://files.rcsb.org/download/{pdb}.cif" + return tree + + +def test_resolve_tasks_have_a_valid_accepted_set(): + tasks = _resolve_tasks() + assert tasks, "no resolve tasks found" + for task in tasks: + canonical = task["reference_mvs"]["children"][0]["params"]["url"].rsplit("/", 1)[-1][:4] + assert canonical in task["accepted_ids"], f"{task['id']}: canonical id not in accepted_ids" + assert len(task["accepted_ids"]) > 1, f"{task['id']}: a one-entry set defeats the point" + + +def test_any_accepted_entry_scores_as_the_reference(): + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + other = next(i for i in task["accepted_ids"] if i != "1udt") # e.g. 1tbf, PDE5 + tadalafil + result = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, other), + accepted_refs=task["accepted_ids"]) + assert result["f1"] == 1.0, result + + +def test_made_up_entry_is_penalised(): + """The evaluator case: 'a structure of PDE5A' answered with PDB 1UJ7, which does not exist.""" + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + assert "1uj7" not in task["accepted_ids"] + wrong = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1uj7"), + accepted_refs=task["accepted_ids"]) + right = mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1udt"), + accepted_refs=task["accepted_ids"]) + assert right["f1"] == 1.0 + assert wrong["f1"] < 1.0, wrong + # Without an accepted set the same wrong id scores the same: the fold only ever helps. + assert mvs.grade_mvs(task["reference_mvs"], _with_pdb(task, "1uj7"))["f1"] == wrong["f1"] + + +def test_accepted_fold_ignores_multi_structure_references(): + ref = {"kind": "root", "children": [ + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/1hho.cif"}}, + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/2hhb.cif"}}, + ]} + pred = {"kind": "root", "children": [ + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/1a3n.cif"}}, + {"kind": "download", "params": {"url": "https://files.rcsb.org/download/2hhb.cif"}}, + ]} + # Two canonical ids → nothing to fold onto; grading is unchanged. + assert mvs.grade_mvs(ref, pred, accepted_refs=["1a3n"]) == mvs.grade_mvs(ref, pred) From 431268369a64c4211ad012c66d8e8bfb4276c489 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Wed, 30 Sep 2026 13:14:13 +0100 Subject: [PATCH 2/5] Address review: run resolve tests from the CI entry point; survive a download node without params Co-Authored-By: Claude Fable 5.1 --- molbench/mvs.py | 15 ++++++++--- tests/test_harness.py | 60 +++++++++++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/molbench/mvs.py b/molbench/mvs.py index 3499e74..dfe4044 100644 --- a/molbench/mvs.py +++ b/molbench/mvs.py @@ -170,6 +170,15 @@ def _path_similarity(p: tuple, q: tuple) -> float: # --- grading ---------------------------------------------------------------------- +def _download_id(seg: tuple) -> str | None: + """The structure id in a ``download`` path segment, or None (a download node with + no/malformed params has an empty signature — a model can emit that).""" + kind, sig = seg + if kind == "download" and isinstance(sig, tuple) and len(sig) == 2 and sig[0] == "ref": + return sig[1] + return None + + def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], accepted: set[str]) -> list[tuple]: """Treat any accepted structure id in the prediction as the reference's own. @@ -180,16 +189,14 @@ def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple], before matching. An id outside the set (a made-up one, say) stays as it is and mismatches the download segment like any other wrong answer. """ - ref_ids = {seg[1][1] for path in ref_paths for seg in path - if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref"} + ref_ids = {_download_id(seg) for path in ref_paths for seg in path} - {None} if len(ref_ids) != 1: return pred_paths # several structures: no single canonical id to fold onto canonical = next(iter(ref_ids)) accepted = {a.lower() for a in accepted} def fold(seg: tuple) -> tuple: - if seg[0] == "download" and isinstance(seg[1], tuple) and seg[1][0] == "ref" \ - and seg[1][1] in accepted: + if _download_id(seg) in accepted: return ("download", ("ref", canonical)) return seg diff --git a/tests/test_harness.py b/tests/test_harness.py index 5a89d16..8c32069 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -176,28 +176,6 @@ def test_run_archives_raw_samples_and_meta(): assert "raw" in tasks[tid][0], "raw model output must be archived" -if __name__ == "__main__": - test_all_reference_answers_are_schema_valid() - test_self_grading_is_perfect() - test_wrong_prediction_scores_low() - test_partial_credit_for_right_residues_wrong_colour() - test_load_modifiers_are_scored() - test_tooltip_text_is_scored() - test_sidechain_flag_is_scored() - test_mvs_self_grading_is_perfect() - test_mvs_is_metadata_insensitive() - test_mvs_partial_credit_for_wrong_colour() - test_mvs_grey_gray_fold() - test_mvs_non_tree_scores_zero() - test_mvs_grader_survives_non_dict_params() - test_mvs_download_variant_is_normalised() - test_mvs_downstream_credit_survives_upstream_mismatch() - test_escalate_tree_tier() - test_render_state_wrapping() - test_run_archives_raw_samples_and_meta() - print("all sanity checks passed") - - # --- entry resolution (tasks/mvs_resolve) ------------------------------------------ def _resolve_tasks(): @@ -253,3 +231,41 @@ def test_accepted_fold_ignores_multi_structure_references(): ]} # Two canonical ids → nothing to fold onto; grading is unchanged. assert mvs.grade_mvs(ref, pred, accepted_refs=["1a3n"]) == mvs.grade_mvs(ref, pred) + + +def test_accepted_fold_survives_download_without_params(): + """A model may emit a download node with no params; the fold must not crash on it.""" + task = next(t for t in _resolve_tasks() if t["id"] == "res-pde5a") + pred = {"kind": "root", "children": [{"kind": "download", "children": [ + {"kind": "parse", "params": {"format": "mmcif"}}]}]} + result = mvs.grade_mvs(task["reference_mvs"], pred, accepted_refs=task["accepted_ids"]) + assert 0.0 <= result["f1"] < 1.0 + bare_ref = {"kind": "root", "children": [{"kind": "download"}]} + assert mvs.grade_mvs(bare_ref, pred, accepted_refs=["1udt"])["f1"] >= 0.0 + + +if __name__ == "__main__": + test_all_reference_answers_are_schema_valid() + test_self_grading_is_perfect() + test_wrong_prediction_scores_low() + test_partial_credit_for_right_residues_wrong_colour() + test_load_modifiers_are_scored() + test_tooltip_text_is_scored() + test_sidechain_flag_is_scored() + test_mvs_self_grading_is_perfect() + test_mvs_is_metadata_insensitive() + test_mvs_partial_credit_for_wrong_colour() + test_mvs_grey_gray_fold() + test_mvs_non_tree_scores_zero() + test_mvs_grader_survives_non_dict_params() + test_mvs_download_variant_is_normalised() + test_mvs_downstream_credit_survives_upstream_mismatch() + test_escalate_tree_tier() + test_render_state_wrapping() + test_run_archives_raw_samples_and_meta() + test_resolve_tasks_have_a_valid_accepted_set() + test_any_accepted_entry_scores_as_the_reference() + test_made_up_entry_is_penalised() + test_accepted_fold_ignores_multi_structure_references() + test_accepted_fold_survives_download_without_params() + print("all sanity checks passed") From abf42251706006dc7ac44f1297b96f9d736e93c2 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:42:02 +0100 Subject: [PATCH 3/5] Address Codex review: CFTR accepts only near-full-length entries; escalation renders the accepted entry; CI runs every test - generate_resolve_tasks: optional per-task min coverage (entity-level RCSB query); res-cftr drops isolated-domain entries such as 1XMI/2BBO. - escalate: the visual tiers re-point the reference at the accepted entry the prediction loaded, as the tree tier already does. - test_harness: the __main__ block runs every test_* function, so CI (which runs this file directly) can no longer skip a newly added test. Co-Authored-By: Claude Opus 5.5 (1M context) --- molbench/escalate.py | 40 +++++++++++++++++++++++-- scripts/generate_resolve_tasks.py | 49 ++++++++++++++++++++++--------- tasks/mvs_resolve/res-cftr.json | 42 +++----------------------- tests/test_harness.py | 44 +++++++++++++-------------- 4 files changed, 97 insertions(+), 78 deletions(-) diff --git a/molbench/escalate.py b/molbench/escalate.py index dbf73a0..6d383eb 100644 --- a/molbench/escalate.py +++ b/molbench/escalate.py @@ -19,14 +19,50 @@ from __future__ import annotations +import copy import tempfile import pathlib from typing import Any, Callable -from .mvs import grade_mvs +from .mvs import _download_id, extract_root, flatten_paths, grade_mvs from .visual import image_similarity +def _download_ids(tree: Any) -> set[str]: + root = extract_root(tree) + if root is None: + return set() + return {_download_id(seg) for path in flatten_paths(root) for seg in path} - {None} + + +def _reference_on_accepted(reference_tree: dict, predicted_tree: Any, + accepted: set[str] | list[str] | None) -> dict: + """The reference re-pointed at the accepted entry the prediction loaded. + + The tree tier folds an accepted id onto the canonical one; the visual tiers must + do the same, or they compare two different structures and an otherwise + equivalent scene can never be rescued. + """ + ref_ids, pred_ids = _download_ids(reference_tree), _download_ids(predicted_tree) + if not accepted or len(ref_ids) != 1 or len(pred_ids) != 1: + return reference_tree + (ref_id,), (pred_id,) = ref_ids, pred_ids + if pred_id == ref_id or pred_id not in {a.lower() for a in accepted}: + return reference_tree + moved = copy.deepcopy(reference_tree) + + def walk(node: dict) -> None: + params = node.get("params") + if node.get("kind") == "download" and isinstance(params, dict) and "url" in params: + params["url"] = params["url"].replace(f"/{ref_id}.", f"/{pred_id}.") + for c in node.get("children") or []: + if isinstance(c, dict): + walk(c) + + walk(extract_root(moved)) + return moved + + def escalating_grade( reference_tree: dict, predicted_tree: Any, @@ -54,7 +90,7 @@ def escalating_grade( wd.mkdir(parents=True, exist_ok=True) ref_png, pred_png = wd / "ref.png", wd / "pred.png" try: - render(reference_tree, str(ref_png)) + render(_reference_on_accepted(reference_tree, predicted_tree, accepted_refs), str(ref_png)) render(predicted_tree, str(pred_png)) except Exception as e: # noqa: BLE001 - a render failure is itself informative return {**out, "decision": "render-failed", "note": str(e)} diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py index 11875c2..2a936ac 100644 --- a/scripts/generate_resolve_tasks.py +++ b/scripts/generate_resolve_tasks.py @@ -31,15 +31,18 @@ CIF = "https://files.rcsb.org/download/{}.cif" SEARCH = "https://search.rcsb.org/rcsbsearch/v2/query" -# (slug, prompt, UniProt accession, canonical PDB id, note). Prompts are verbatim or -# lightly edited from the evaluator corpus; the canonical id is a well-known entry. +# (slug, prompt, UniProt accession, canonical PDB id, note[, min coverage]). Prompts are +# verbatim or lightly edited from the evaluator corpus; the canonical id is a well-known +# entry. A min coverage (fraction of the UniProt sequence one polymer entity must span) +# is set where the prompt asks for the whole protein: "the CFTR channel" is not answered +# by an isolated NBD1 domain. SPECS = [ ("pde5a", "I wanna see a structure of PDE5A", "O76074", "1udt", "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist."), ("pcsk9", "show the structure of PCSK9", "Q8NBP7", "2p4e", "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), ("cftr", "show me the CFTR channel", "P13569", "5uak", - "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain)."), + "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", 0.7), ("gfp", "show green fluorescent protein", "P42212", "1ema", "The starter chip's target; the model tends to be right here — a control."), ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", @@ -49,14 +52,27 @@ ] -def entries_for(accession: str) -> list[str]: - """All PDB entries with a polymer entity mapped to this UniProt accession.""" +def uniprot_length(accession: str) -> int: + url = f"https://rest.uniprot.org/uniprotkb/{accession}.json?fields=length" + with urllib.request.urlopen(url, timeout=60) as resp: + return json.load(resp)["sequence"]["length"] + + +def entries_for(accession: str, min_length: int | None = None) -> list[str]: + """All PDB entries with a polymer entity mapped to this UniProt accession (and, if + ``min_length`` is set, at least that many residues in the same entity).""" + nodes = [{"type": "terminal", "service": "text", "parameters": { + "attribute": "rcsb_polymer_entity_container_identifiers." + "reference_sequence_identifiers.database_accession", + "operator": "exact_match", "value": accession}}] + if min_length: + nodes.append({"type": "terminal", "service": "text", "parameters": { + "attribute": "entity_poly.rcsb_sample_sequence_length", + "operator": "greater_or_equal", "value": min_length}}) query = { - "query": {"type": "terminal", "service": "text", "parameters": { - "attribute": "rcsb_polymer_entity_container_identifiers." - "reference_sequence_identifiers.database_accession", - "operator": "exact_match", "value": accession}}, - "return_type": "entry", + "query": {"type": "group", "logical_operator": "and", "nodes": nodes}, + # Entity-level, so both conditions hold for the same polymer entity. + "return_type": "polymer_entity", "request_options": {"paginate": {"start": 0, "rows": 10000}}, } req = urllib.request.Request(SEARCH, data=json.dumps(query).encode(), @@ -64,7 +80,8 @@ def entries_for(accession: str) -> list[str]: with urllib.request.urlopen(req, timeout=60) as resp: if resp.status == 204: return [] - return sorted(hit["identifier"].lower() for hit in json.load(resp)["result_set"]) + hits = json.load(resp)["result_set"] + return sorted({hit["identifier"].split("_")[0].lower() for hit in hits}) def reference(pdb: str) -> dict: @@ -79,8 +96,10 @@ def reference(pdb: str) -> dict: def main() -> None: OUT.mkdir(parents=True, exist_ok=True) today = dt.date.today().isoformat() - for slug, prompt, accession, canonical, note in SPECS: - ids = entries_for(accession) + for slug, prompt, accession, canonical, note, *rest in SPECS: + min_coverage = rest[0] if rest else None + min_length = round(min_coverage * uniprot_length(accession)) if min_coverage else None + ids = entries_for(accession, min_length) if canonical not in ids: raise SystemExit(f"{slug}: canonical {canonical} is not mapped to {accession}") ref = reference(canonical) @@ -96,7 +115,9 @@ def main() -> None: "skills": ["resolve", "download", "component", "representation"], "notes": note, "provenance": {"uniprot": accession, "n_entries": len(ids), - "source": "RCSB search, reference_sequence_identifiers", "fetched": today}, + "source": "RCSB search, reference_sequence_identifiers", "fetched": today, + **({"min_coverage": min_coverage, "min_length": min_length} + if min_coverage else {})}, } (OUT / f"res-{slug}.json").write_text(json.dumps(task, indent=2) + "\n") print(f"res-{slug:10s} {accession} {len(ids):4d} accepted entries canonical {canonical}") diff --git a/tasks/mvs_resolve/res-cftr.json b/tasks/mvs_resolve/res-cftr.json index 3b75542..71a6962 100644 --- a/tasks/mvs_resolve/res-cftr.json +++ b/tasks/mvs_resolve/res-cftr.json @@ -48,46 +48,10 @@ ] }, "accepted_ids": [ - "1xmi", - "1xmj", - "2bbo", - "2bbs", - "2bbt", - "2lob", - "2pze", - "2pzf", - "2pzg", - "3gd7", - "3isw", - "4wz6", - "5d2d", - "5d3e", - "5d3f", - "5tf7", - "5tf8", - "5tfa", - "5tfb", - "5tfc", - "5tfd", - "5tff", - "5tfg", - "5tfi", - "5tfj", - "5tgk", "5uak", - "6gjq", - "6gjs", - "6gju", - "6gk4", - "6gkd", - "6hep", "6msm", "6o1v", "6o2p", - "6uk1", - "6wbs", - "6ze1", - "7qi1", "7sv7", "7svd", "7svr", @@ -123,8 +87,10 @@ "notes": "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", "provenance": { "uniprot": "P13569", - "n_entries": 60, + "n_entries": 24, "source": "RCSB search, reference_sequence_identifiers", - "fetched": "2026-09-30" + "fetched": "2026-10-06", + "min_coverage": 0.7, + "min_length": 1036 } } diff --git a/tests/test_harness.py b/tests/test_harness.py index 8c32069..97b6c3c 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -244,28 +244,24 @@ def test_accepted_fold_survives_download_without_params(): assert mvs.grade_mvs(bare_ref, pred, accepted_refs=["1udt"])["f1"] >= 0.0 + + +def test_escalation_renders_the_reference_on_the_accepted_entry(): + from molbench import escalate + task = _resolve_tasks()[0] + other = next(i for i in task["accepted_ids"] if i not in json.dumps(task["reference_mvs"])) + pred = _with_pdb(task, other) + moved = escalate._reference_on_accepted(task["reference_mvs"], pred, task["accepted_ids"]) + assert escalate._download_ids(moved) == {other} + assert escalate._download_ids(task["reference_mvs"]) != {other} # the task is untouched + made_up = _with_pdb(task, "9zzz") + assert escalate._reference_on_accepted(task["reference_mvs"], made_up, task["accepted_ids"]) \ + is task["reference_mvs"] + + if __name__ == "__main__": - test_all_reference_answers_are_schema_valid() - test_self_grading_is_perfect() - test_wrong_prediction_scores_low() - test_partial_credit_for_right_residues_wrong_colour() - test_load_modifiers_are_scored() - test_tooltip_text_is_scored() - test_sidechain_flag_is_scored() - test_mvs_self_grading_is_perfect() - test_mvs_is_metadata_insensitive() - test_mvs_partial_credit_for_wrong_colour() - test_mvs_grey_gray_fold() - test_mvs_non_tree_scores_zero() - test_mvs_grader_survives_non_dict_params() - test_mvs_download_variant_is_normalised() - test_mvs_downstream_credit_survives_upstream_mismatch() - test_escalate_tree_tier() - test_render_state_wrapping() - test_run_archives_raw_samples_and_meta() - test_resolve_tasks_have_a_valid_accepted_set() - test_any_accepted_entry_scores_as_the_reference() - test_made_up_entry_is_penalised() - test_accepted_fold_ignores_multi_structure_references() - test_accepted_fold_survives_download_without_params() - print("all sanity checks passed") + # Run every test_* function, so a new test can't be missed by CI (which runs this file). + tests = [f for name, f in sorted(globals().items()) if name.startswith("test_") and callable(f)] + for t in tests: + t() + print(f"all {len(tests)} harness checks passed") From 782c7a12e27d2369126ef13d559cca523bdf1307 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:46:27 +0100 Subject: [PATCH 4/5] Address Codex review: resolve tasks get their own leaderboard regime; unknown sources fail loudly Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/build_leaderboard.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index 3641ea0..c393487 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -34,18 +34,25 @@ NL = "\n" # How a task's `source` maps to a difficulty regime, ordered easy -> hard. -REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] +REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)"] +# The first three form the difficulty gradient (Finding 03); later regimes test other skills. +_SOURCE_REGIME = { + "generated": "Translation", "curated": "Translation", + "grounded": "Grounded (ligand/SS)", + "clinvar-variant": "Clinical (SIFTS/ClinVar)", "clinvar-hotspots": "Clinical (SIFTS/ClinVar)", + "sifts-named": "Clinical (SIFTS/ClinVar)", + "resolve": "Resolution (by name)", +} # --------------------------------------------------------------------------- # # Data helpers # --------------------------------------------------------------------------- # def _regime_of(source: str) -> str: - if source in ("generated", "curated"): - return "Translation" - if source == "grounded": - return "Grounded (ligand/SS)" - return "Clinical (SIFTS/ClinVar)" + # Explicit, so a new task source can't silently land in another regime's numbers. + if source not in _SOURCE_REGIME: + raise ValueError(f"task source {source!r} has no regime; add it to _SOURCE_REGIME") + return _SOURCE_REGIME[source] @functools.lru_cache(maxsize=1) @@ -340,7 +347,7 @@ def findings(ranked: list) -> str: # Compute it live — with a wider panel this can flip, because format-unreliable # models inject parse-driven spread that swamps the difficulty signal. regimes = _task_regimes() - reg_order = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] + reg_order = REGIMES[:3] reg_spread: dict[str, float] = {} for rg in reg_order: per = [] From 4df4c990ce5bd76c964f4e65ce38d25e0038c9b5 Mon Sep 17 00:00:00 2001 From: Dan Bolser Date: Tue, 6 Oct 2026 19:51:57 +0100 Subject: [PATCH 5/5] Address Codex review: non-gradient regimes shown apart from the difficulty gradient; GFP prompt names the species Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/build_leaderboard.py | 16 ++++++++++++---- scripts/generate_resolve_tasks.py | 2 +- tasks/mvs_resolve/res-gfp.json | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/build_leaderboard.py b/scripts/build_leaderboard.py index c393487..3ed0a11 100644 --- a/scripts/build_leaderboard.py +++ b/scripts/build_leaderboard.py @@ -34,8 +34,10 @@ NL = "\n" # How a task's `source` maps to a difficulty regime, ordered easy -> hard. -REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)", "Resolution (by name)"] -# The first three form the difficulty gradient (Finding 03); later regimes test other skills. +GRADIENT = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"] +# Regimes that test another skill, not a harder step on the gradient: shown below it, unordered. +OTHER_REGIMES = ["Resolution (by name)"] +REGIMES = GRADIENT + OTHER_REGIMES _SOURCE_REGIME = { "generated": "Translation", "curated": "Translation", "grounded": "Grounded (ligand/SS)", @@ -347,7 +349,7 @@ def findings(ranked: list) -> str: # Compute it live — with a wider panel this can flip, because format-unreliable # models inject parse-driven spread that swamps the difficulty signal. regimes = _task_regimes() - reg_order = REGIMES[:3] + reg_order = GRADIENT reg_spread: dict[str, float] = {} for rg in reg_order: per = [] @@ -456,10 +458,15 @@ def regime_table(ranked: list) -> str: f"{html.escape(n.split('/')[-1])}" for n, _ in cols ) body = [] + other_header = False for rg in REGIMES: n_tasks = sum(1 for v in reg.values() if v == rg) if not n_tasks: continue + if rg in OTHER_REGIMES and not other_header: + other_header = True + body.append(f"Other skills " + "(not part of the gradient)") vals = {} for n, m in cols: fs = [t["f1"] for t in m["tasks"] if reg.get(t["id"]) == rg] @@ -483,7 +490,8 @@ def regime_table(ranked: list) -> str:

Difficulty gradient by task regime

Mean MVS F1 by how the answer key is grounded, easy → hard. Harder regimes pull the models apart — the - spread you want from a benchmark.

+ spread you want from a benchmark. Rows under Other skills test a + different ability and are not ordered by difficulty.

{head_cells} diff --git a/scripts/generate_resolve_tasks.py b/scripts/generate_resolve_tasks.py index 2a936ac..c63ef3d 100644 --- a/scripts/generate_resolve_tasks.py +++ b/scripts/generate_resolve_tasks.py @@ -43,7 +43,7 @@ "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."), ("cftr", "show me the CFTR channel", "P13569", "5uak", "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", 0.7), - ("gfp", "show green fluorescent protein", "P42212", "1ema", + ("gfp", "show green fluorescent protein from Aequorea victoria", "P42212", "1ema", "The starter chip's target; the model tends to be right here — a control."), ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn", "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic."), diff --git a/tasks/mvs_resolve/res-gfp.json b/tasks/mvs_resolve/res-gfp.json index 9eb9385..d805057 100644 --- a/tasks/mvs_resolve/res-gfp.json +++ b/tasks/mvs_resolve/res-gfp.json @@ -3,7 +3,7 @@ "category": "mvs", "source": "resolve", "title": "Resolve by name: gfp", - "prompt": "show green fluorescent protein", + "prompt": "show green fluorescent protein from Aequorea victoria", "reference_mvs": { "kind": "root", "children": [ @@ -702,6 +702,6 @@ "uniprot": "P42212", "n_entries": 637, "source": "RCSB search, reference_sequence_identifiers", - "fetched": "2026-09-30" + "fetched": "2026-10-06" } }
Regime