Entry-resolution tasks: name the protein, not the id

ROADMAP.md

@@ -78,6 +78,22 @@ strong judge, structured protocol, self-variance reporting); rubric quality
   search to extract interface/contact residues (ligand–protein, protein–protein),
   giving grounded "show the binding-site residues" / "show the PPI interface"
   tasks — a realistic, high-value class we currently lack.
+* **B4. Entry resolution (STARTED — `tasks/mvs_resolve/`).** Every task used to hand
+  the model the PDB id; real users name the protein. The first live failure was
+  "a structure of PDE5A" → PDB 1UJ7 (does not exist), and probing found real ids
+  for the wrong protein too. Six tasks now grade against *all* entries for the
+  protein's UniProt accession (`accepted_ids`, folded by the grader). Next: more
+  targets, and an `exists` sub-metric (did the id resolve at RCSB at all?) — a
+  hallucinated id and a wrong-but-real id are different failures.
+* **B5. Colour schemes.** The chat-driver's prompt now teaches Mol*'s
+  `custom.molstar_color_theme_name` on a `color` node for rainbow / by-chain /
+  by-SS colouring (models kept writing `"spectrum"` as a colour). Porting that here
+  needs the grader to treat a theme-coloured representation as equivalent to the
+  per-chain component references it would replace; until then the two prompts differ.
+* **B6. Structured outputs.** Constrained decoding (`response_format: json_schema`,
+  `strict`) with a trimmed MVS schema makes invalid trees impossible by
+  construction. Measure validation failures + F1 against the prompt-only path on
+  the cheap models.
 * **B3. Corpus scaling & de-correlation.** More clinical targets (BRCA1, CFTR,
   kinases, more p53 structures incl. one with a numbering *offset* to exercise the
   SIFTS bridge); more Tier-1 template diversity so per-skill items are less

molbench/escalate.py

@@ -19,14 +19,50 @@
 
 from __future__ import annotations
 
+import copy
 import tempfile
 import pathlib
 from typing import Any, Callable
 
-from .mvs import grade_mvs
+from .mvs import _download_id, extract_root, flatten_paths, grade_mvs
 from .visual import image_similarity
 
 
+def _download_ids(tree: Any) -> set[str]:
+    root = extract_root(tree)
+    if root is None:
+        return set()
+    return {_download_id(seg) for path in flatten_paths(root) for seg in path} - {None}
+
+
+def _reference_on_accepted(reference_tree: dict, predicted_tree: Any,
+                           accepted: set[str] | list[str] | None) -> dict:
+    """The reference re-pointed at the accepted entry the prediction loaded.
+
+    The tree tier folds an accepted id onto the canonical one; the visual tiers must
+    do the same, or they compare two different structures and an otherwise
+    equivalent scene can never be rescued.
+    """
+    ref_ids, pred_ids = _download_ids(reference_tree), _download_ids(predicted_tree)
+    if not accepted or len(ref_ids) != 1 or len(pred_ids) != 1:
+        return reference_tree
+    (ref_id,), (pred_id,) = ref_ids, pred_ids
+    if pred_id == ref_id or pred_id not in {a.lower() for a in accepted}:
+        return reference_tree
+    moved = copy.deepcopy(reference_tree)
+
+    def walk(node: dict) -> None:
+        params = node.get("params")
+        if node.get("kind") == "download" and isinstance(params, dict) and "url" in params:
+            params["url"] = params["url"].replace(f"/{ref_id}.", f"/{pred_id}.")
+        for c in node.get("children") or []:
+            if isinstance(c, dict):
+                walk(c)
+
+    walk(extract_root(moved))
+    return moved
+
+
 def escalating_grade(
     reference_tree: dict,
     predicted_tree: Any,
@@ -37,8 +73,9 @@ def escalating_grade(
     tree_threshold: float = 0.999,
     visual_threshold: float = 0.97,
     workdir: str | pathlib.Path | None = None,
+    accepted_refs: set[str] | list[str] | None = None,
 ) -> dict[str, Any]:
-    g = grade_mvs(reference_tree, predicted_tree)
+    g = grade_mvs(reference_tree, predicted_tree, accepted_refs=accepted_refs)
     out: dict[str, Any] = {"tree_f1": g["f1"], "tier": "tree"}
 
     # T0 — tree match is conclusive (identical / normalised-equivalent).
@@ -53,7 +90,7 @@ def escalating_grade(
     wd.mkdir(parents=True, exist_ok=True)
     ref_png, pred_png = wd / "ref.png", wd / "pred.png"
     try:
-        render(reference_tree, str(ref_png))
+        render(_reference_on_accepted(reference_tree, predicted_tree, accepted_refs), str(ref_png))
         render(predicted_tree, str(pred_png))
     except Exception as e:  # noqa: BLE001 - a render failure is itself informative
         return {**out, "decision": "render-failed", "note": str(e)}

molbench/mvs.py

@@ -170,8 +170,46 @@ def _path_similarity(p: tuple, q: tuple) -> float:
 
 # --- grading ----------------------------------------------------------------------
 
-def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]:
-    """Grade a predicted MVS scene against a reference. Returns precision/recall/f1."""
+def _download_id(seg: tuple) -> str | None:
+    """The structure id in a ``download`` path segment, or None (a download node with
+    no/malformed params has an empty signature — a model can emit that)."""
+    kind, sig = seg
+    if kind == "download" and isinstance(sig, tuple) and len(sig) == 2 and sig[0] == "ref":
+        return sig[1]
+    return None
+
+
+def _fold_accepted_refs(ref_paths: list[tuple], pred_paths: list[tuple],
+                        accepted: set[str]) -> list[tuple]:
+    """Treat any accepted structure id in the prediction as the reference's own.
+
+    A prompt that names a protein ("a structure of PDE5A") has many right answers;
+    the task lists them in ``accepted_ids``. The reference carries one canonical id,
+    so a prediction that downloaded another accepted entry is rewritten to that id
+    before matching. An id outside the set (a made-up one, say) stays as it is and
+    mismatches the download segment like any other wrong answer.
+    """
+    ref_ids = {_download_id(seg) for path in ref_paths for seg in path} - {None}
+    if len(ref_ids) != 1:
+        return pred_paths  # several structures: no single canonical id to fold onto
+    canonical = next(iter(ref_ids))
+    accepted = {a.lower() for a in accepted}
+
+    def fold(seg: tuple) -> tuple:
+        if _download_id(seg) in accepted:
+            return ("download", ("ref", canonical))
+        return seg
+
+    return [tuple(fold(seg) for seg in path) for path in pred_paths]
+
+
+def grade_mvs(reference: Any, predicted: Any,
+              accepted_refs: set[str] | list[str] | None = None) -> dict[str, Any]:
+    """Grade a predicted MVS scene against a reference. Returns precision/recall/f1.
+
+    ``accepted_refs``: other structure ids that count as the reference's (see
+    ``_fold_accepted_refs``); used by tasks that name a protein instead of an id.
+    """
     ref_root, pred_root = extract_root(reference), extract_root(predicted)
     if ref_root is None:
         return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "error": "bad reference tree"}
@@ -181,6 +219,8 @@ def grade_mvs(reference: Any, predicted: Any) -> dict[str, Any]:
 
     ref_paths = flatten_paths(ref_root)
     pred_paths = flatten_paths(pred_root)
+    if accepted_refs:
+        pred_paths = _fold_accepted_refs(ref_paths, pred_paths, set(accepted_refs))
     n_ref, n_pred = len(ref_paths), len(pred_paths)
     if n_ref == 0:
         f1 = 1.0 if n_pred == 0 else 0.0

molbench/runner.py

@@ -196,7 +196,8 @@ def run_mvs_task(model, system: str, task: dict) -> dict[str, Any]:
     tree, err = extract_json_object(raw)
     if err:
         return {"f1": 0.0, "precision": 0.0, "recall": 0.0, "error": err, "raw": raw}
-    result = mvs_mod.grade_mvs(task["reference_mvs"], tree)
+    result = mvs_mod.grade_mvs(task["reference_mvs"], tree,
+                               accepted_refs=task.get("accepted_ids"))
     result["schema_errors"] = mvs_mod.validate_mvs(tree)
     result["predicted"] = tree
     result["raw"] = raw
@@ -305,7 +306,8 @@ def run(models: list[str], categories: list[str] | None,
                     wd = gallery / model.name.replace("/", "_") / task["id"]
                     pt["escalation"] = escalating_grade(
                         task["reference_mvs"], pt["predicted"], prompt=task.get("prompt"),
-                        render=render_fn, vlm=vlm_fn, workdir=str(wd))
+                        render=render_fn, vlm=vlm_fn, workdir=str(wd),
+                        accepted_refs=task.get("accepted_ids"))
                 per_task.append(pt)
                 raw_samples[model.name][task["id"]] = [
                     {"f1": g["f1"], "predicted": g.get("predicted"),

scripts/build_leaderboard.py

@@ -34,18 +34,27 @@
 NL = "\n"
 
 # How a task's `source` maps to a difficulty regime, ordered easy -> hard.
-REGIMES = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"]
+GRADIENT = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"]
+# Regimes that test another skill, not a harder step on the gradient: shown below it, unordered.
+OTHER_REGIMES = ["Resolution (by name)"]
+REGIMES = GRADIENT + OTHER_REGIMES
+_SOURCE_REGIME = {
+    "generated": "Translation", "curated": "Translation",
+    "grounded": "Grounded (ligand/SS)",
+    "clinvar-variant": "Clinical (SIFTS/ClinVar)", "clinvar-hotspots": "Clinical (SIFTS/ClinVar)",
+    "sifts-named": "Clinical (SIFTS/ClinVar)",
+    "resolve": "Resolution (by name)",
+}
 
 
 # --------------------------------------------------------------------------- #
 # Data helpers
 # --------------------------------------------------------------------------- #
 def _regime_of(source: str) -> str:
-    if source in ("generated", "curated"):
-        return "Translation"
-    if source == "grounded":
-        return "Grounded (ligand/SS)"
-    return "Clinical (SIFTS/ClinVar)"
+    # Explicit, so a new task source can't silently land in another regime's numbers.
+    if source not in _SOURCE_REGIME:
+        raise ValueError(f"task source {source!r} has no regime; add it to _SOURCE_REGIME")
+    return _SOURCE_REGIME[source]
 
 
 @functools.lru_cache(maxsize=1)
@@ -340,7 +349,7 @@ def findings(ranked: list) -> str:
     # Compute it live — with a wider panel this can flip, because format-unreliable
     # models inject parse-driven spread that swamps the difficulty signal.
     regimes = _task_regimes()
-    reg_order = ["Translation", "Grounded (ligand/SS)", "Clinical (SIFTS/ClinVar)"]
+    reg_order = GRADIENT
     reg_spread: dict[str, float] = {}
     for rg in reg_order:
         per = []
@@ -449,10 +458,15 @@ def regime_table(ranked: list) -> str:
         f"<th>{html.escape(n.split('/')[-1])}</th>" for n, _ in cols
     )
     body = []
+    other_header = False
     for rg in REGIMES:
         n_tasks = sum(1 for v in reg.values() if v == rg)
         if not n_tasks:
             continue
+        if rg in OTHER_REGIMES and not other_header:
+            other_header = True
+            body.append(f"<tr><td class='rowhead' colspan='{len(cols) + 1}'><i>Other skills "
+                        "(not part of the gradient)</i></td></tr>")
         vals = {}
         for n, m in cols:
             fs = [t["f1"] for t in m["tasks"] if reg.get(t["id"]) == rg]
@@ -476,7 +490,8 @@ def regime_table(ranked: list) -> str:
   <h2>Difficulty gradient by task regime</h2>
   <p class="section-sub">Mean MVS&nbsp;F1 by how the answer key is grounded,
      easy&nbsp;&rarr;&nbsp;hard. Harder regimes pull the models apart &mdash; the
-     spread you want from a benchmark.</p>
+     spread you want from a benchmark. Rows under <i>Other skills</i> test a
+     different ability and are not ordered by difficulty.</p>
   <div class="table-scroll">
     <table>
       <thead><tr><th class="left">Regime</th>{head_cells}</tr></thead>

scripts/generate_resolve_tasks.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""
+Entry-resolution tasks: the prompt names a protein, not a PDB id.
+
+Every other MVS task hands the model the id. Real users don't: the first evaluator
+prompt to fail in the wild was "I wanna see a structure of PDE5A", answered with
+PDB 1UJ7 — an entry that does not exist. Probing further found the quieter failure:
+real ids for the wrong protein (a SARS-CoV-2 RBD for "nanobody bound to GFP").
+
+Many entries are a right answer here, so each task lists ``accepted_ids``: every PDB
+entry whose polymer maps to the protein's UniProt accession, pulled from RCSB search
+at authoring time and recorded under ``provenance``. The reference carries one
+canonical entry; the grader folds any accepted id onto it (``mvs._fold_accepted_refs``).
+The scene itself is deliberately minimal (a polymer cartoon) so the task measures
+resolution, not styling.
+
+    python scripts/generate_resolve_tasks.py     # -> tasks/mvs_resolve/*.json
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import pathlib
+import urllib.request
+
+from molbench.mvs import categorize
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+OUT = REPO / "tasks" / "mvs_resolve"
+CIF = "https://files.rcsb.org/download/{}.cif"
+SEARCH = "https://search.rcsb.org/rcsbsearch/v2/query"
+
+# (slug, prompt, UniProt accession, canonical PDB id, note[, min coverage]). Prompts are
+# verbatim or lightly edited from the evaluator corpus; the canonical id is a well-known
+# entry. A min coverage (fraction of the UniProt sequence one polymer entity must span)
+# is set where the prompt asks for the whole protein: "the CFTR channel" is not answered
+# by an isolated NBD1 domain.
+SPECS = [
+    ("pde5a", "I wanna see a structure of PDE5A", "O76074", "1udt",
+     "Evaluator prompt (2026-09-30). Haiku answered 1UJ7, which does not exist."),
+    ("pcsk9", "show the structure of PCSK9", "Q8NBP7", "2p4e",
+     "Probe: Haiku answered 1D0G (death receptor 5 / TRAIL)."),
+    ("cftr", "show me the CFTR channel", "P13569", "5uak",
+     "Probe: Haiku answered 5UAY (a plant Toc75 POTRA domain).", 0.7),
+    ("gfp", "show green fluorescent protein from Aequorea victoria", "P42212", "1ema",
+     "The starter chip's target; the model tends to be right here — a control."),
+    ("myoglobin", "Show me sperm whale myoglobin", "P02185", "1mbn",
+     "Corpus prompt asked for 'crab myoglobin' (no such entry); sperm whale is the classic."),
+    ("lysozyme", "Show me hen egg white lysozyme", "P00698", "1lyz",
+     "Starter chip; hundreds of valid entries — a control."),
+]
+
+
+def uniprot_length(accession: str) -> int:
+    url = f"https://rest.uniprot.org/uniprotkb/{accession}.json?fields=length"
+    with urllib.request.urlopen(url, timeout=60) as resp:
+        return json.load(resp)["sequence"]["length"]
+
+
+def entries_for(accession: str, min_length: int | None = None) -> list[str]:
+    """All PDB entries with a polymer entity mapped to this UniProt accession (and, if
+    ``min_length`` is set, at least that many residues in the same entity)."""
+    nodes = [{"type": "terminal", "service": "text", "parameters": {
+        "attribute": "rcsb_polymer_entity_container_identifiers."
+                     "reference_sequence_identifiers.database_accession",
+        "operator": "exact_match", "value": accession}}]
+    if min_length:
+        nodes.append({"type": "terminal", "service": "text", "parameters": {
+            "attribute": "entity_poly.rcsb_sample_sequence_length",
+            "operator": "greater_or_equal", "value": min_length}})
+    query = {
+        "query": {"type": "group", "logical_operator": "and", "nodes": nodes},
+        # Entity-level, so both conditions hold for the same polymer entity.
+        "return_type": "polymer_entity",
+        "request_options": {"paginate": {"start": 0, "rows": 10000}},
+    }
+    req = urllib.request.Request(SEARCH, data=json.dumps(query).encode(),
+                                 headers={"content-type": "application/json"})
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        if resp.status == 204:
+            return []
+        hits = json.load(resp)["result_set"]
+    return sorted({hit["identifier"].split("_")[0].lower() for hit in hits})
+
+
+def reference(pdb: str) -> dict:
+    return {"kind": "root", "children": [{
+        "kind": "download", "params": {"url": CIF.format(pdb)}, "children": [{
+            "kind": "parse", "params": {"format": "mmcif"}, "children": [{
+                "kind": "structure", "params": {"type": "model"}, "children": [{
+                    "kind": "component", "params": {"selector": "polymer"}, "children": [{
+                        "kind": "representation", "params": {"type": "cartoon"}}]}]}]}]}]}
+
+
+def main() -> None:
+    OUT.mkdir(parents=True, exist_ok=True)
+    today = dt.date.today().isoformat()
+    for slug, prompt, accession, canonical, note, *rest in SPECS:
+        min_coverage = rest[0] if rest else None
+        min_length = round(min_coverage * uniprot_length(accession)) if min_coverage else None
+        ids = entries_for(accession, min_length)
+        if canonical not in ids:
+            raise SystemExit(f"{slug}: canonical {canonical} is not mapped to {accession}")
+        ref = reference(canonical)
+        task = {
+            "id": f"res-{slug}",
+            "category": "mvs",
+            "source": "resolve",
+            "title": f"Resolve by name: {slug}",
+            "prompt": prompt,
+            "reference_mvs": ref,
+            "accepted_ids": ids,
+            "categories": categorize(ref) + ["resolve"],
+            "skills": ["resolve", "download", "component", "representation"],
+            "notes": note,
+            "provenance": {"uniprot": accession, "n_entries": len(ids),
+                           "source": "RCSB search, reference_sequence_identifiers", "fetched": today,
+                           **({"min_coverage": min_coverage, "min_length": min_length}
+                              if min_coverage else {})},
+        }
+        (OUT / f"res-{slug}.json").write_text(json.dumps(task, indent=2) + "\n")
+        print(f"res-{slug:10s} {accession}  {len(ids):4d} accepted entries  canonical {canonical}")
+
+
+if __name__ == "__main__":
+    main()