From 29d98a1d87b9b6601678c45087f62e625223c45e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Miguel=20P=C3=A9rez=20Canales?= Date: Sun, 11 Oct 2026 01:14:23 +0200 Subject: [PATCH 1/2] Las secuencias retiradas vienen de UniParc, y reviewed se queda sin inventar Una pasada completa dejo 28.868 accesiones sin secuencia. La operacion ya avisaba de que podia pasar: una accesion que UniProtKB deja de servir sale ausente del endpoint por lotes mientras X-Total-Results la cuenta, y su propia URL responde 200 con el cuerpo vacio. Lo que no estaba dicho es que esas secuencias siguen existiendo en UniParc, que guarda todas las que UniProtKB ha tenido alguna vez. Sobre una muestra aleatoria de 600 de esas 28.868, la ruta nueva resolvio 600. POR QUE MERECE UN PASE Y NO ES UN FLECO. Las que faltan no faltan al azar. Por nivel de evidencia sobre las 74 releases, no tiene secuencia el 9,2% de las proteinas que ALGUNA VEZ llevaron evidencia experimental y el 7,8% de las de inferencia curada, contra el 0,07% del resto. En un corte antiguo eso es el 7,5% del banco de donantes y en el ultimo es el 0%: un gradiente de visibilidad que se disfraza exactamente de crecimiento del banco, que es una de las dos cosas que un dataset longitudinal existe para medir. LAS DOS DECISIONES QUE UNIPARC OBLIGA A TOMAR, y por que se toman asi. Una accesion responde una fila por version de secuencia que haya tenido: 670 filas para 600 accesiones. Se toma la de `last_seen` mas reciente, que es la que tenia al desaparecer, porque todas las demas proteinas del almacen llevan la secuencia que UniProt sirve HOY y asi el almacen entero queda bajo una sola convencion en vez de hacer de estas 28.868 las unicas con una segunda regla. Las fechas quedan guardadas para que una pasada posterior pueda revisarlo sin volver a preguntar. Y una fila responde por todas las accesiones que comparten esa secuencia exacta, asi que solo se escriben las del lote: las demas pueden estar perfectamente vivas. REVIEWED SE QUEDA EN NULL. UniParc guarda secuencias, no estado de entrada, y la accesion es una que UniProtKB ya no sirve, asi que no hay estado que leer. Escribir False seria indistinguible de una entrada que UniProt llamo no revisada, y NULL ya significa lo que es cierto: nadie pregunto, o nadie pudo contestar. De ahi el `asserts_reviewed` del store, opt-in, con los dos llamantes que SI leen el estado de una cabecera FASTA intactos. Y de paso, dates_pending deja de mentir en seco. El informe declara en su propio docstring que un seco devuelve None y no 0 "porque cero recuperable y no intentado no son lo mismo", y ese campo rompia la regla: devolvia 0 con 644.659 filas esperando, porque el pase que las cuenta es justo el que un seco salta. Un seco es donde alguien decide si la corrida real merece la pena, asi que el unico campo que decia "nada que hacer" era el que no podia saberlo. --- protea/core/operations/_protein_store.py | 38 +++- protea/core/operations/_universe_sources.py | 18 +- .../operations/resolve_protein_sequences.py | 142 ++++++++++++- pyproject.toml | 2 +- tests/test_protein_store.py | 52 ++++- tests/test_resolve_protein_sequences.py | 186 ++++++++++++++++-- 6 files changed, 405 insertions(+), 33 deletions(-) diff --git a/protea/core/operations/_protein_store.py b/protea/core/operations/_protein_store.py index 75f03e4d..f5929d74 100644 --- a/protea/core/operations/_protein_store.py +++ b/protea/core/operations/_protein_store.py @@ -69,10 +69,22 @@ def add(self, other: StoreCounts) -> None: def store_records( - session: Session, records: list[UniProtProteinRecord], emit: EmitFn + session: Session, + records: list[UniProtProteinRecord], + emit: EmitFn, + *, + asserts_reviewed: bool = True, ) -> StoreCounts: """Upsert ``records`` as ``sequence`` + ``protein`` rows. + ``asserts_reviewed=False`` says the source cannot answer the Swiss-Prot + question, so ``reviewed`` is left NULL instead of being filled from the + record. UniParc is such a source: it keeps sequences, not entry status, and + an accession it answers for is one UniProtKB no longer serves, so there is + no status to read. Writing ``False`` there would be indistinguishable from a + TrEMBL entry UniProt actually called unreviewed, and NULL already means the + thing that is true: nobody asked, or nobody could answer. + Does NOT commit: the caller owns the transaction boundary, because the two callers want different ones -- ``insert_proteins`` commits per page so an interrupted ingest leaves a partial-but-consistent dataset, and @@ -129,6 +141,8 @@ def _upsert_proteins( existing_prot: dict[str, Protein], existing_seq_ids: dict[str, int], emit: EmitFn, + *, + asserts_reviewed: bool = True, ) -> tuple[int, int]: """Insert new ``Protein`` rows and conservatively patch the existing ones.""" proteins_inserted = 0 @@ -137,10 +151,12 @@ def _upsert_proteins( for r in records: seq_id = existing_seq_ids[r.sequence_hash] if r.accession in existing_prot: - if apply_record_updates(existing_prot[r.accession], r, seq_id): + if apply_record_updates( + existing_prot[r.accession], r, seq_id, asserts_reviewed=asserts_reviewed + ): proteins_updated += 1 else: - to_add.append(build_protein(r, seq_id)) + to_add.append(build_protein(r, seq_id, asserts_reviewed=asserts_reviewed)) proteins_inserted += 1 if to_add: emit("db.insert_proteins_start", None, {"rows": len(to_add)}, "info") @@ -150,7 +166,13 @@ def _upsert_proteins( return proteins_inserted, proteins_updated -def apply_record_updates(p: Protein, r: UniProtProteinRecord, seq_id: int | None) -> bool: +def apply_record_updates( + p: Protein, + r: UniProtProteinRecord, + seq_id: int | None, + *, + asserts_reviewed: bool = True, +) -> bool: """Patch ``p`` from ``r`` without clobbering; return ``True`` if anything changed. The null-filling is what lets the universe be built in two steps: @@ -174,7 +196,7 @@ def apply_record_updates(p: Protein, r: UniProtProteinRecord, seq_id: int | None if getattr(p, "isoform_index", None) != r.isoform_index: p.isoform_index = r.isoform_index changed = True - if getattr(p, "reviewed", None) is None: + if asserts_reviewed and getattr(p, "reviewed", None) is None: p.reviewed = r.reviewed changed = True if getattr(p, "taxonomy_id", None) in (None, "") and r.taxonomy_id: @@ -192,14 +214,16 @@ def apply_record_updates(p: Protein, r: UniProtProteinRecord, seq_id: int | None return changed -def build_protein(r: UniProtProteinRecord, seq_id: int | None) -> Protein: +def build_protein( + r: UniProtProteinRecord, seq_id: int | None, *, asserts_reviewed: bool = True +) -> Protein: """A new ``Protein`` row from a UniProt record.""" return Protein( accession=r.accession, canonical_accession=r.canonical_accession, is_canonical=r.is_canonical, isoform_index=r.isoform_index, - reviewed=r.reviewed, + reviewed=r.reviewed if asserts_reviewed else None, entry_name=r.entry_name, organism=r.organism, taxonomy_id=r.taxonomy_id, diff --git a/protea/core/operations/_universe_sources.py b/protea/core/operations/_universe_sources.py index af4b5933..2a8f63a9 100644 --- a/protea/core/operations/_universe_sources.py +++ b/protea/core/operations/_universe_sources.py @@ -85,9 +85,7 @@ #: whether an embedding neighbourhood captures what curated alignment captures, #: without touching the scored truth. Measured on GOA 156: 62,363 proteins enter #: by these and by nothing else. -CURATED_INFERENCE_CODES = frozenset( - {"ISS", "ISO", "ISA", "ISM", "IGC", "RCA", "NAS", "IKR", "IRD"} -) +CURATED_INFERENCE_CODES = frozenset({"ISS", "ISO", "ISA", "ISM", "IGC", "RCA", "NAS", "IKR", "IRD"}) #: Excluded, each for its own reason, and the reasons are not interchangeable. #: @@ -344,6 +342,11 @@ class _FetchOutcome: dates_written: int = 0 alias: dict[str, str] = field(default_factory=dict) unresolved: list[str] = field(default_factory=list) + #: Accessions UniProtKB no longer serves and UniParc did. Kept apart from + #: ``fetched`` because they are a different claim: ``fetched`` means UniProt + #: answered for a live entry, this means the entry is gone and the sequence + #: was recovered from the archive. + recovered_from_uniparc: list[str] = field(default_factory=list) demerges: dict[str, list[str]] = field(default_factory=dict) artefactos: dict[str, Any] = field(default_factory=dict) @@ -442,6 +445,12 @@ def resolution_report( ``None`` in a dry run rather than ``0``, because "zero recoverable" and "not attempted" are not the same thing: an earlier report read 0% recoverable merges because ten batches had answered 400 and the failures were tallied as zeroes. + + ``dates_pending`` broke that rule and read ``0`` in a dry run while 644.659 + rows were waiting, because the pass that counts them is the one a dry run + skips. A dry run is exactly where somebody decides whether the real one is + worth starting, so the one field that said "nothing to do" was the one that + could not know. """ return { "candidates": outcome.candidates, @@ -450,9 +459,10 @@ def resolution_report( "proteins_inserted": outcome.inserted, "sequences_inserted": outcome.sequences, "resolved_as_merge": None if dry_run else len(outcome.alias), + "recovered_from_uniparc": None if dry_run else len(outcome.recovered_from_uniparc), "not_retrievable": None if dry_run else len(outcome.unresolved), "demerged": None if dry_run else len(outcome.demerges), - "dates_pending": outcome.dates_pending, + "dates_pending": None if dry_run else outcome.dates_pending, "dates_backfilled": None if dry_run else outcome.dates_written, "artefactos": outcome.artefactos, "dry_run": dry_run, diff --git a/protea/core/operations/resolve_protein_sequences.py b/protea/core/operations/resolve_protein_sequences.py index 023389cf..8600dc73 100644 --- a/protea/core/operations/resolve_protein_sequences.py +++ b/protea/core/operations/resolve_protein_sequences.py @@ -98,7 +98,7 @@ import time from typing import Annotated, Any -from protea_contracts import UniProtProteinRecord +from protea_contracts import UniProtProteinRecord, compute_sequence_hash from pydantic import Field from sqlalchemy import select from sqlalchemy.orm import Session @@ -194,6 +194,7 @@ def execute( if not p.dry_run and candidates: self._fetch_sequences(session, candidates, p, emit, outcome) self._resolve_merges(session, candidates, p, emit, outcome) + self._resolve_retired(session, candidates, p, emit, outcome) if not p.dry_run: self._backfill_dates(session, p, emit, outcome) outcome.artefactos = self._store_artifacts(payload.get("_job_id"), outcome) @@ -328,9 +329,7 @@ def _merge_batch( accession that appears in SEVERAL entries is a demerge, not a merge, and that cannot be decided until the whole response has been read. """ - body = self._uniprot.search_secondary_accessions( - batch, emit=emit, knobs=self._knobs(p) - ) + body = self._uniprot.search_secondary_accessions(batch, emit=emit, knobs=self._knobs(p)) candidates, entry_by_acc = candidates_from(body, set(batch)) records = classify(candidates, outcome.alias, outcome.demerges) counts = StoreCounts() @@ -358,6 +357,139 @@ def _merge_batch( ) return counts + # ---- pass three: what UniProtKB no longer serves, from UniParc ---- + + def _resolve_retired( + self, + session: Session, + candidates: list[str], + p: ResolveProteinSequencesPayload, + emit: EmitFn, + outcome: _FetchOutcome, + ) -> None: + """Rescue from UniParc the accessions UniProtKB has stopped serving. + + The two passes before this one ask UniProtKB, and UniProtKB answers + about entries it still has. An accession it deleted is absent from the + batch endpoint while ``X-Total-Results`` counts it, and its own URL + answers 200 with an empty body -- so after the merge pass what is left + is not "unknown", it is "gone from UniProtKB". UniParc keeps the + sequence of every entry UniProtKB has ever held, deleted ones included. + Measured 2026-10-11 over a random sample of 600 of the 28.868 a full + pass could not place: 600 resolved. + + WHY IT IS WORTH A PASS. The ones missing are not missing at random. By + tier over the 74 releases, 9,2% of the proteins that EVER carried + experimental evidence have no sequence, and 7,8% of the curated + inference tier, against 0,07% of the rest. In an early cut that is 7,5% + of the donor bank and in the last one it is 0%, so the hole is a + visibility gradient that looks exactly like the bank growing -- which is + one of the two things a longitudinal dataset exists to measure. + """ + from protea_sources.uniprot import MAX_UNIPARC_OR_CONDITIONS + + pending = self._still_without_sequence(session, candidates) + if not pending: + return + counts = StoreCounts() + recuperadas: set[str] = set() + for batch in chunks(pending, MAX_UNIPARC_OR_CONDITIONS): + counts.add(self._uniparc_batch(session, batch, p, emit, recuperadas)) + outcome.recovered_from_uniparc = sorted(recuperadas) + outcome.unresolved = sorted(set(pending) - recuperadas) + outcome.updated += counts.proteins_updated + outcome.inserted += counts.proteins_inserted + outcome.sequences += counts.sequences_inserted + emit( + "resolve_protein_sequences.uniparc", + None, + { + "pending": len(pending), + "recovered": len(recuperadas), + "unresolved": len(outcome.unresolved), + }, + "info", + ) + + @staticmethod + def _newest_per_accession(tsv: str, pedidas: set[str]) -> dict[str, Any]: + """The newest UniParc row for each accession that was asked for. + + Both filters are here because both are decisions. An accession answers + one row per sequence version it ever had, and the one with the latest + ``last_seen`` is the sequence it held when it disappeared. A row answers + for every accession sharing that exact sequence, and only the ones in + this batch are ours to write: the rest may be perfectly alive entries + belonging to somebody else's question. + + The version suffix is stripped for MATCHING only (``A0A014NDJ0.1`` is + the accession ``A0A014NDJ0``), and the row keeps it, so nothing is lost. + """ + from protea_sources.uniprot import parse_uniparc_tsv + + mejor: dict[str, Any] = {} + for fila in parse_uniparc_tsv(tsv): + for cruda in fila.accessions: + acc = cruda.split(".")[0] + if acc not in pedidas: + continue + previa = mejor.get(acc) + if previa is None or fila.last_seen > previa.last_seen: + mejor[acc] = fila + return mejor + + def _uniparc_batch( + self, + session: Session, + batch: list[str], + p: ResolveProteinSequencesPayload, + emit: EmitFn, + recuperadas: set[str], + ) -> StoreCounts: + """One UniParc query, and the two choices it forces. + + WHICH ROW. An accession can answer several, one per sequence version + UniParc has ever seen: 670 rows came back for a sample of 600. This + takes the one with the latest ``last_seen``, which is the sequence the + accession held when it disappeared. The alternative -- the version + current at each cut -- would be a better answer and a different corpus: + every other protein here carries the sequence UniProt serves TODAY, so + picking the newest keeps the whole store on one convention instead of + making these 28.868 the only ones with a second rule. The dates are + recorded so a later pass can revisit that without asking UniParc again. + + WHICH ACCESSION. Identical sequences share a UPI, so one row answers for + entries nobody asked about. Only the ones in this batch are written: the + rest are somebody else's accessions and may be perfectly alive. + + ``reviewed`` is left NULL on purpose, see ``store_records``. + """ + tsv = self._uniprot.search_uniparc_tsv( + batch, + fields="upi,accession,first_seen,last_seen,length,sequence", + emit=emit, + knobs=self._knobs(p), + ) + mejor = self._newest_per_accession(tsv, set(batch)) + if not mejor: + return StoreCounts() + records = [ + UniProtProteinRecord( + accession=acc, + canonical_accession=acc, + is_canonical=True, + reviewed=False, + sequence=fila.sequence, + length=len(fila.sequence), + sequence_hash=compute_sequence_hash(fila.sequence), + ) + for acc, fila in sorted(mejor.items()) + ] + counts = store_records(session, records, emit, asserts_reviewed=False) + recuperadas.update(mejor) + session.commit() + return counts + def _still_without_sequence(self, session: Session, candidates: list[str]) -> list[str]: pending: list[str] = [] for chunk in chunks(candidates, _DB_CHUNK): @@ -371,7 +503,7 @@ def _still_without_sequence(self, session: Session, candidates: list[str]) -> li ) return sorted(pending) - # ---- pass three: the dates of everything that already had a sequence ---- + # ---- pass four: the dates of everything that already had a sequence ---- def _backfill_dates( self, diff --git a/pyproject.toml b/pyproject.toml index be5bf778..dfee175b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ priority = "explicit" # protea.core.contracts.operation); the remaining four are optional # plugin extras installed via --with plugins. protea-method = { git = "https://github.com/frapercan/protea-method.git", rev = "a120f613b9d2f1646ab5a566f97a8eadc1310a23" } -protea-sources = { git = "https://github.com/frapercan/protea-sources.git", rev = "946227ca3b8840eb0ea7223c8fc52b746027f9fe" } +protea-sources = { git = "https://github.com/frapercan/protea-sources.git", rev = "fc43ab942ebec811afae25eda3be18b3f37d0e46" } protea-runners = { git = "https://github.com/frapercan/protea-runners.git", rev = "ccd06fd80c12b5195339478dae7255d1ea58f4a6" } protea-backends = { git = "https://github.com/frapercan/protea-backends.git", rev = "9b76719a14d729390be7d0faf957fae06a31ced5" } diff --git a/tests/test_protein_store.py b/tests/test_protein_store.py index f6029c12..ef5af3f8 100644 --- a/tests/test_protein_store.py +++ b/tests/test_protein_store.py @@ -159,8 +159,6 @@ def query_side_effect(*args): assert session.add_all.call_count == 2 - - class TestTheSequencelessRowIsCompletedLater: """The invariant that makes a two-step universe possible. @@ -214,3 +212,53 @@ def test_a_second_patch_changes_nothing(self): apply_record_updates(row, _make_record(), seq_id=42) assert apply_record_updates(row, _make_record(), seq_id=99) is False assert row.sequence_id == 42, "the sequence was already there and is not overwritten" + + +class TestASourceThatCannotAnswerTheSwissprotQuestion: + """``reviewed`` NULL means nobody asked, or nobody could answer. + + UniParc keeps sequences, not entry status, and the accessions it answers for + are ones UniProtKB no longer serves, so there is no status to read. Writing + ``False`` there would be indistinguishable from an entry UniProt actually + called unreviewed, and the corpus already has 407.697 proteins admitted for + having been Swiss-Prot in their day whose evidence is automatic: a wrong + ``reviewed`` on top of that is not recoverable afterwards. + """ + + def _extracted_row(self, accession="P12345"): + from protea.infrastructure.orm.models.protein.protein import Protein + + canonical, is_canonical, isoform = Protein.parse_isoform(accession) + return Protein( + accession=accession, + canonical_accession=canonical, + is_canonical=is_canonical, + isoform_index=isoform, + first_admitted_release=156, + ) + + def test_the_patch_leaves_reviewed_null(self): + from protea.core.operations._protein_store import apply_record_updates + + row = self._extracted_row() + apply_record_updates(row, _make_record(), seq_id=42, asserts_reviewed=False) + assert row.reviewed is None, "a source that cannot answer must not answer" + assert row.sequence_id == 42, "everything else it DOES know still lands" + + def test_a_new_row_is_born_with_reviewed_null(self): + from protea.core.operations._protein_store import build_protein + + fila = build_protein(_make_record(), seq_id=42, asserts_reviewed=False) + assert fila.reviewed is None + assert fila.sequence_id == 42 + + def test_the_default_still_asserts_it(self): + from protea.core.operations._protein_store import apply_record_updates, build_protein + + row = self._extracted_row() + apply_record_updates(row, _make_record(), seq_id=42) + assert row.reviewed is True, ( + "the flag is opt-in: the two callers that DO read the status from a " + "FASTA header must keep writing it" + ) + assert build_protein(_make_record(), seq_id=42).reviewed is True diff --git a/tests/test_resolve_protein_sequences.py b/tests/test_resolve_protein_sequences.py index 3bece363..e714cf83 100644 --- a/tests/test_resolve_protein_sequences.py +++ b/tests/test_resolve_protein_sequences.py @@ -100,8 +100,13 @@ def test_it_imports_no_networking_library(self): assert not importados & {"requests", "urllib", "http", "httpx", "socket"}, importados def test_everything_remote_goes_through_the_plugin(self): - """The plugin's two methods and no others. A third way out would be another - place where the retries can be missing.""" + """The plugin's three methods and no others. A fourth way out would be + another place where the retries can be missing. + + It was two until UniParc was added as the route for what UniProtKB stops + serving. The set is spelled out rather than counted so that adding a + remote call is a deliberate edit here, which is the whole point: this + test is how a private HTTP client gets noticed on the way in.""" import ast import protea.core.operations.resolve_protein_sequences as mod @@ -115,7 +120,11 @@ def test_everything_remote_goes_through_the_plugin(self): and isinstance(n.func.value, ast.Attribute) and n.func.value.attr == "_uniprot" } - assert llamadas == {"fetch_accessions_tsv", "search_secondary_accessions"}, llamadas + assert llamadas == { + "fetch_accessions_tsv", + "search_secondary_accessions", + "search_uniparc_tsv", + }, llamadas def test_the_duplicated_module_no_longer_exists(self): with pytest.raises(ModuleNotFoundError): @@ -176,8 +185,11 @@ def fake_store(_session, records, _emit): ), ): op._resolve_merges( - MagicMock(), list(requested), ResolveProteinSequencesPayload(), - MagicMock(), outcome, + MagicMock(), + list(requested), + ResolveProteinSequencesPayload(), + MagicMock(), + outcome, ) return outcome, stored @@ -439,9 +451,13 @@ def test_the_two_paths_agree_on_the_field_names(self): _, from_tsv = _parse_tsv(self._TSV)[0] from_json = _audit_dates_of( - {"entryAudit": {"firstPublicDate": "1987-08-13", - "lastSequenceUpdateDate": "2003-08-22", - "sequenceVersion": 2}} + { + "entryAudit": { + "firstPublicDate": "1987-08-13", + "lastSequenceUpdateDate": "2003-08-22", + "sequenceVersion": 2, + } + } ) assert from_tsv._fields == from_json._fields assert from_tsv == from_json @@ -526,9 +542,7 @@ def test_the_cap_travels_into_the_report(self): def test_with_no_cap_the_report_says_so_too(self): op = _op() - with patch.object( - ResolveProteinSequencesOperation, "_without_sequence", return_value=[] - ): + with patch.object(ResolveProteinSequencesOperation, "_without_sequence", return_value=[]): out = op.execute(MagicMock(), {"dry_run": True}, emit=MagicMock()) assert out.result["limit"] is None @@ -546,12 +560,156 @@ def test_execute_accepts_the_shape_base_worker_delivers(self): import uuid op = _op() - with patch.object( - ResolveProteinSequencesOperation, "_without_sequence", return_value=[] - ): + with patch.object(ResolveProteinSequencesOperation, "_without_sequence", return_value=[]): out = op.execute( MagicMock(), {"dry_run": True, "_job_id": str(uuid.uuid4())}, emit=MagicMock(), ) assert out.result["dry_run"] is True + + +#: A real UniParc answer, trimmed to the two shapes that force a choice: one +#: accession with two sequence versions, and one row shared by four entries of +#: which only one was asked for. +_UP_HEADER = "Entry\tUniProtKB\tFirst seen\tLast seen\tLength\tSequence" +_UP_VIEJA = "UPI000000AAA1\tQ11111.1\t2011-03-01\t2014-07-17\t4\tAAAA" +_UP_NUEVA = "UPI000000AAA2\tQ11111.2\t2014-07-18\t2024-07-17\t4\tCCCC" +_UP_COMPARTIDA = "UPI000000BBB1\tQ22222.1; Z99999.1; Z88888\t2011-03-01\t2026-09-02\t4\tGGGG" + + +class TestUniparcAnswersForWhatUniprotkbDeleted: + """The third pass exists because "absent from UniProtKB" is not "gone". + + After the merge pass what is left are accessions UniProt deleted. UniParc + keeps their sequences, and they are not missing at random: 9,2% of the + proteins that ever carried experimental evidence are in that set, which in + an early cut is 7,5% of the donor bank and in the last one is 0%. + """ + + def _run(self, tsv, batch, *, store=None): + op = _op() + op._uniprot.search_uniparc_tsv.return_value = tsv + contados = [] + + def _store(session, records, emit, **kw): + contados.append((list(records), kw)) + return store or StoreCounts() + + recuperadas: set[str] = set() + with patch("protea.core.operations.resolve_protein_sequences.store_records", _store): + op._uniparc_batch( + MagicMock(), + batch, + ResolveProteinSequencesPayload(), + lambda *a, **k: None, + recuperadas, + ) + return contados[0] if contados else ([], {}), recuperadas + + def test_the_latest_version_wins_when_an_accession_has_several(self): + (records, _), _ = self._run( + "\n".join([_UP_HEADER, _UP_VIEJA, _UP_NUEVA]) + "\n", ["Q11111"] + ) + assert [r.accession for r in records] == ["Q11111"] + assert records[0].sequence == "CCCC", ( + "an accession answers one row per sequence version it ever had; this " + "takes the one it held when it disappeared, so the whole store stays " + "on one convention instead of these being the only rows with a second rule" + ) + + def test_the_order_of_the_rows_does_not_decide_it(self): + (a, _), _ = self._run("\n".join([_UP_HEADER, _UP_VIEJA, _UP_NUEVA]) + "\n", ["Q11111"]) + (b, _), _ = self._run("\n".join([_UP_HEADER, _UP_NUEVA, _UP_VIEJA]) + "\n", ["Q11111"]) + assert a[0].sequence == b[0].sequence == "CCCC" + + def test_only_the_accessions_asked_for_are_written(self): + (records, _), recuperadas = self._run( + "\n".join([_UP_HEADER, _UP_COMPARTIDA]) + "\n", ["Q22222"] + ) + assert [r.accession for r in records] == ["Q22222"] + assert recuperadas == {"Q22222"}, ( + "identical sequences share a UPI, so a row answers for entries nobody " + "asked about; Z99999 and Z88888 may be perfectly alive and are not ours " + "to overwrite" + ) + + def test_it_does_not_assert_the_swissprot_status(self): + (_, kw), _ = self._run("\n".join([_UP_HEADER, _UP_NUEVA]) + "\n", ["Q11111"]) + assert kw.get("asserts_reviewed") is False, ( + "UniParc keeps sequences, not entry status, and the accession is one " + "UniProtKB no longer serves; writing False would be indistinguishable " + "from an entry UniProt actually called unreviewed" + ) + + def test_an_accession_uniparc_does_not_know_is_not_invented(self): + (records, _), recuperadas = self._run(_UP_HEADER + "\n", ["Q11111"]) + assert records == [] and recuperadas == set() + + def test_the_length_comes_from_the_sequence_and_not_from_the_column(self): + mentira = "UPI000000CCC1\tQ33333.1\t2011-03-01\t2026-09-02\t999\tAAAA" + (records, _), _ = self._run("\n".join([_UP_HEADER, mentira]) + "\n", ["Q33333"]) + assert records[0].length == 4 + + +class TestTheThirdPassReportsWhatItRecovered: + def test_recovered_and_unresolved_split_the_pending_without_overlap(self): + op = _op() + op._uniprot.search_uniparc_tsv.return_value = "\n".join([_UP_HEADER, _UP_NUEVA]) + "\n" + outcome = _FetchOutcome() + with ( + patch.object(op, "_still_without_sequence", return_value=["Q11111", "Q44444"]), + patch( + "protea.core.operations.resolve_protein_sequences.store_records", + return_value=StoreCounts(), + ), + ): + op._resolve_retired( + MagicMock(), + ["Q11111", "Q44444"], + ResolveProteinSequencesPayload(), + lambda *a, **k: None, + outcome, + ) + assert outcome.recovered_from_uniparc == ["Q11111"] + assert outcome.unresolved == ["Q44444"] + assert not set(outcome.recovered_from_uniparc) & set(outcome.unresolved) + + def test_nothing_pending_asks_uniparc_nothing(self): + op = _op() + outcome = _FetchOutcome() + with patch.object(op, "_still_without_sequence", return_value=[]): + op._resolve_retired( + MagicMock(), + ["Q11111"], + ResolveProteinSequencesPayload(), + lambda *a, **k: None, + outcome, + ) + assert not op._uniprot.search_uniparc_tsv.called + + def test_the_pass_runs_after_the_merges_and_not_before(self): + fuente = inspect.getsource(ResolveProteinSequencesOperation.execute) + assert fuente.index("_resolve_merges") < fuente.index("_resolve_retired"), ( + "UniParc is the archive: asking it first would recover a sequence for " + "an accession that UniProtKB would have resolved as a merge, and the " + "merge carries an identity that the archive does not" + ) + + +class TestADryRunDoesNotReportZeroForWhatItDidNotCount: + def test_dates_pending_is_none_in_a_dry_run(self): + from protea.core.operations._universe_sources import resolution_report + + r = resolution_report(dry_run=True, outcome=_FetchOutcome(), elapsed=0.1) + assert r["dates_pending"] is None, ( + "the pass that counts them is the one a dry run skips, so 0 read as " + "'no dates pending' while 644.659 rows were waiting; a dry run is " + "exactly where somebody decides whether the real run is worth starting" + ) + + def test_the_real_run_still_reports_the_number(self): + from protea.core.operations._universe_sources import resolution_report + + r = resolution_report(dry_run=False, outcome=_FetchOutcome(dates_pending=7), elapsed=0.1) + assert r["dates_pending"] == 7 From 6247a52424df54e5e832020e258db38542c727ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Miguel=20P=C3=A9rez=20Canales?= Date: Sun, 11 Oct 2026 01:23:39 +0200 Subject: [PATCH 2/2] El lock acompana al pin `poetry install` fallaba con "pyproject.toml changed significantly since poetry.lock was last generated", y con el se caian ocho comprobaciones que no tenian nada que ver: test, lint, docs, mutation y pip-audit instalan dependencias antes de hacer su trabajo. Regenerado y comprobado linea a linea: cambian la referencia de protea-sources, su resolved_reference y el content-hash. Nada mas. El torch +cpu sigue donde estaba, que es lo unico que daba miedo de tocar este fichero. --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9004438c..792e0dfe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4095,8 +4095,8 @@ requests = ">=2.31" [package.source] type = "git" url = "https://github.com/frapercan/protea-sources.git" -reference = "946227ca3b8840eb0ea7223c8fc52b746027f9fe" -resolved_reference = "946227ca3b8840eb0ea7223c8fc52b746027f9fe" +reference = "fc43ab942ebec811afae25eda3be18b3f37d0e46" +resolved_reference = "fc43ab942ebec811afae25eda3be18b3f37d0e46" [[package]] name = "protobuf" @@ -7010,4 +7010,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "e294322c9ac332d92ab6f73db2536c35ef2268f854d0c71d4ba850339da46ec4" +content-hash = "f8d7f2ae7566d96f7427c2011695222b04894a96c1bfa897db216a652dc98cd7"