test: gate measured coverage and Observatory contracts (#998)

.github/workflows/ci.yml

@@ -333,6 +333,35 @@ jobs:
           if-no-files-found: error
           include-hidden-files: true
 
+  python-coverage:
+    name: python / measured coverage floors
+    needs: ci-config
+    if: needs.ci-config.outputs.python == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: ./.github/actions/docker-mirror
+      - uses: ./.github/actions/python-workspace
+      - name: Measure workspace and required PostgreSQL controls and enforce floors
+        run: make test-coverage
+      - name: Publish per-source line/branch totals and skip reasons
+        if: always()
+        run: |
+          python3 scripts/test_summary.py test-results/coverage >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/coverage_report.py test-results/coverage/coverage.json \
+            --output test-results/coverage/totals.json >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        if: always()
+        with:
+          name: workspace-line-branch-coverage
+          path: test-results/coverage/
+          if-no-files-found: error
+          include-hidden-files: true
+
   windows-portability:
     name: windows / compose portability
     needs: ci-config
@@ -369,6 +398,10 @@ jobs:
         id: observatory_install
         run: npm ci
 
+      - name: Install authoritative API for the method/schema drift gate
+        if: needs.ci-config.outputs.frontend == 'true'
+        uses: ./.github/actions/python-workspace
+
       - name: Check formatting with Prettier
         if: ${{ !cancelled() && steps.observatory_install.outcome == 'success' }}
         run: npx prettier --check .
@@ -387,12 +420,32 @@ jobs:
 
       - name: Build production bundle
         if: ${{ !cancelled() && steps.observatory_install.outcome == 'success' }}
+        id: observatory_build
         run: npm run build
 
       - name: Run tests
         if: ${{ !cancelled() && steps.observatory_install.outcome == 'success' }}
         run: npm test
 
+      - name: Install pinned browser driver and Chromium
+        if: ${{ !cancelled() && steps.observatory_build.outcome == 'success' }}
+        # Disposable CI-only driver, kept out of the product dependency graph.
+        run: | # zizmor: ignore[adhoc-packages]
+          npm install --global agent-browser@0.38.1
+          agent-browser install --with-deps
+      - name: Smoke the built Node app against the controlled authenticated API
+        if: ${{ !cancelled() && steps.observatory_build.outcome == 'success' }}
+        env:
+          SMOKE_ARTIFACT_DIR: ${{ github.workspace }}/test-results/browser
+        run: node scripts/browser-smoke.mjs
+      - name: Upload built browser smoke evidence
+        if: always() && needs.ci-config.outputs.frontend == 'true'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: built-observatory-browser-smoke
+          path: test-results/browser/
+          if-no-files-found: error
+
       - name: Install writer dependencies
         if: ${{ !cancelled() && needs.ci-config.outputs.writer == 'true' }}
         id: writer_install
@@ -448,6 +501,7 @@ jobs:
       - python-quality
       - python-core-tests
       - python-package-tests
+      - python-coverage
       - windows-portability
       - frontend
     steps:
@@ -459,6 +513,7 @@ jobs:
           PYTHON_QUALITY: ${{ needs.python-quality.result }}
           PYTHON_CORE_TESTS: ${{ needs.python-core-tests.result }}
           PYTHON_PACKAGE_TESTS: ${{ needs.python-package-tests.result }}
+          PYTHON_COVERAGE: ${{ needs.python-coverage.result }}
           WINDOWS_PORTABILITY: ${{ needs.windows-portability.result }}
           FRONTEND: ${{ needs.frontend.result }}
           SELECT_PYTHON: ${{ needs.ci-config.outputs.python }}
@@ -480,6 +535,7 @@ jobs:
             echo "| python / core tests | ${PYTHON_CORE_TESTS} |"
             echo "| integration / Windows contracts | ${WINDOWS_PORTABILITY} |"
             echo "| python / package tests | ${PYTHON_PACKAGE_TESTS} |"
+            echo "| python / line and branch floors | ${PYTHON_COVERAGE} |"
             echo "| behaviour / JavaScript | ${FRONTEND} |"
           } >> "$GITHUB_STEP_SUMMARY"
 
@@ -491,7 +547,7 @@ jobs:
           if [ "$SELECT_PYTHON" = true ]; then expected_python=success; fi
           if [ "$SELECT_GROUPS" != '[]' ]; then expected_groups=success; fi
           if [ "$SELECT_FRONTEND" = true ] || [ "$SELECT_WRITER" = true ] || [ "$SELECT_PLUGIN" = true ]; then expected_node=success; fi
-          for result in "$INSTALLED_PROVIDER_ARTIFACTS" "$PYTHON_CORE_TESTS" "$WINDOWS_PORTABILITY"; do
+          for result in "$INSTALLED_PROVIDER_ARTIFACTS" "$PYTHON_CORE_TESTS" "$PYTHON_COVERAGE" "$WINDOWS_PORTABILITY"; do
             test "$result" = "$expected_python"
           done
           test "$PYTHON_PACKAGE_TESTS" = "$expected_groups"

.github/workflows/integration.yml

@@ -57,6 +57,9 @@ jobs:
       - name: Run named behavioral contracts
         if: ${{ !cancelled() && steps.workspace.outcome == 'success' }}
         run: uv run --locked python scripts/run_integration.py
+      - name: Publish designated suite execution and skip reasons
+        if: always()
+        run: python3 scripts/test_summary.py test-results/integration >> "$GITHUB_STEP_SUMMARY"
       - name: Upload required suite results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f

Makefile

@@ -29,8 +29,9 @@ LANE ?= all
 BASE ?= origin/main
 PYMDX_DOCS_DIR ?= docs-site
 PYMDX_DOCS_PORT ?= 3000
+COVERAGE_DIR ?= test-results/coverage
 
-.PHONY: setup setup-js install test \
+.PHONY: setup setup-js install test test-coverage \
 	dagster superset hub minio pgweb trino nessie grafana prometheus api hasura openmetadata catalog docs-open \
 	check lint lint-sql lint-python format-python typecheck-python \
 	dependency-refresh dependency-refresh-check \
@@ -56,6 +57,21 @@ install:
 test:
 	uv run --locked pytest
 
+test-coverage:
+	@mkdir -p $(COVERAGE_DIR)
+	@args=(--cov=src/phlo --cov=scripts); \
+	for source in packages/*/src; do args+=("--cov=$$source"); done; \
+	COVERAGE_FILE=$(COVERAGE_DIR)/.coverage uv run --locked pytest -m 'not integration' \
+		"$${args[@]}" --cov-branch --cov-report=json:$(COVERAGE_DIR)/coverage.json \
+		--cov-report= --junitxml=$(COVERAGE_DIR)/results.xml --tb=short -ra || exit $$?; \
+	COVERAGE_FILE=$(COVERAGE_DIR)/.coverage uv run --locked python -m pytest \
+		-p scripts.ci_required -o addopts= -m integration \
+		packages/phlo-api/tests/test_operation_controls_postgres.py \
+		"$${args[@]}" --cov-append --cov-branch \
+		--cov-report=json:$(COVERAGE_DIR)/coverage.json --cov-report= \
+		--junitxml=$(COVERAGE_DIR)/postgres.xml --tb=short -ra
+	python3 scripts/coverage_report.py $(COVERAGE_DIR)/coverage.json --output $(COVERAGE_DIR)/totals.json
+
 test-core-regression:
 	uv run --locked pytest $(CORE_REGRESSION_TEST_PATHS) -m core_regression $(CORE_REGRESSION_PYTEST_ARGS)
 

conftest.py

@@ -85,6 +85,19 @@ def _register_workspace_plugins() -> None:
 # Import fixtures from phlo_testing - these are auto-discovered by pytest
 
 
+def pytest_collection_modifyitems(items):
+    """Service integration directories must not leak into the default unit lane."""
+    missing = [
+        item.nodeid
+        for item in items
+        if "integration" in item.path.parts and not item.get_closest_marker("integration")
+    ]
+    if missing:
+        raise pytest.UsageError(
+            "Integration paths require the integration marker:\n" + "\n".join(missing)
+        )
+
+
 @pytest.fixture(autouse=True)
 def reset_discovery_caches():
     """Keep mocked entry points and manifest inputs isolated between tests."""

docs/contributing/testing-gates.md

@@ -0,0 +1,135 @@
+# Test and contract gates
+
+## Coverage and required lanes
+
+`make test-coverage` runs the complete non-integration workspace with branch
+coverage, then appends the required PostgreSQL operation-control suite before
+checking floors. Docker is required: the suite provisions disposable PostgreSQL
+and rejects skips through `scripts.ci_required`. Its public lifecycle tests cover
+shared claims, replay, evidence resolution, namespace isolation, advisory locks,
+and bounded rate storage. It measures `src/phlo`, `scripts`, and every
+`packages/*/src` tree.
+`scripts/coverage_report.py` publishes separate line and branch percentages and
+checks each against `tests/coverage-floors.json`. A new source tree without a
+measurement or floor fails. The existing package-specific coverage collection
+remains in CI.
+
+The floors use a fresh workspace measurement, not the historical September
+aggregate. They are rounded down to one decimal place to avoid display-rounding
+failures. The JSON artifact contains the unrounded measurements. A floor is a
+minimum, not a claim that uncovered code is safe.
+
+Quality benchmark tests run every worker with real Parquet and DuckDB data and
+check emitted counts, late failure positions, merge-key checksums, and module
+provenance. They include a partial final file and a singleton final delete batch.
+The comparison test launches real before/after worker processes. These are
+correctness checks, not speed or memory thresholds.
+
+`scripts/test_summary.py` publishes executed, skipped, failed, and errored test
+counts from JUnit reports, together with skip and xfail reasons. Ordinary
+non-integration runs can contain optional skips. Named required lanes cannot:
+the existing `scripts.ci_required` plugin rejects skips, xfails, and empty
+execution. `test_missing_minio_cannot_pass_required_lane` runs a real MinIO
+integration selection against a reserved, non-listening port and verifies both
+the failing exit status and the published skip reason.
+
+The required contracts in `scripts/run_integration.py` are Iceberg, Dagster,
+API integration, MinIO migration, dbt quality, schema generation, schema
+conversion, and ingestion. They do not claim complete external-service coverage.
+
+The [fix regression-test-or-justification item proposed in PR1109](https://github.com/phlohouse/phlo/blob/ac75026a89985d009bcb3430830e1b892af67fa4/.github/PULL_REQUEST_TEMPLATE.md#L18-L20)
+belongs to the single PR template. This change does not add a competing template
+or alter repository settings.
+
+## Surviving clock, oracle, and taxonomy findings
+
+The three cited idempotency tests now synchronise with entered and release
+events. Asset metadata deadline tests block a worker on an event instead of
+guessing when a sleep finishes. Query and service timeout tests await a pending
+future that the production deadline cancels. The Loki heartbeat test waits for
+the fetch to start instead of a scheduling delay.
+
+Remaining positive sleeps in API tests poll actual query-worker completion,
+Dagster subprocess execution, or a disposable service. Root lifecycle tests
+wait for real child-process exits and signal handling. Zero-duration async
+sleeps yield control or construct an awaitable fake and do not delay a test.
+Those external boundaries remain clock-driven. The inventory command is
+`rg 'sleep\(' packages/phlo-api/tests tests`.
+
+The MinIO suite no longer tests calls made only to its own `MagicMock`, or
+literal policy dictionaries that never reach production. Production contributor
+and migration tests still exercise configuration, credentials, and storage
+operations. Real-service tests remain integration-marked.
+
+Collection rejects tests under an `integration` directory without an integration
+marker. Restore reconciliation and supported-version upgrade tests use injected
+in-process providers, so they live under `tests/operations`, not a service lane.
+`test_integration_directory_requires_marker` proves both rejected and accepted
+collection.
+
+## Built browser and schema checks
+
+From `packages/phlo-observatory/src/phlo_observatory`, `npm run build` followed
+by `node scripts/browser-smoke.mjs` starts the bundled Node server and a
+controlled HTTP API. The smoke navigates the real application, executes SQL
+through CodeMirror, checks results, and verifies permission denial removes stale
+results. The HTTP fixture verifies the caller bearer and staging environment.
+CI uploads screenshots and sanitised request evidence. This is not live
+FastAPI, Dagster, PostgreSQL, or Trino acceptance.
+
+The existing rendered accessibility and keyboard lineage tests remain the
+accessibility checks. `route-behaviour.test.tsx` adds pending mutation, completed
+result, permission failure, empty result, and registered loading/error boundaries.
+The browser covers actual navigation and the real editor.
+
+`contracts.test.mjs` generates OpenAPI from the installed FastAPI application.
+`scripts/client-contracts.mjs` discovers actual production `phloApi` calls and
+evaluates their actual Zod response parsers. It compares HTTP methods and response
+types, required fields, unions, enums, and nested arrays/objects. JSON-text calls
+use their production inner schemas after verifying the real `jsonText` helper
+decodes JSON and rejects parse failures. Tests also execute that production
+transform. CSV and NDJSON require string response schemas; no-content responses
+must have no body. An OpenAPI default annotation does not make a field required.
+Negative tests remove GET, rename required fields in ordinary and JSON-text
+responses, and break text or no-content declarations. URL-only checks and
+duplicated hand-written fixture schemas are not the authority. Untyped API
+responses remain failures until API and product owners align their contracts.
+
+## Retained Node runtime and handler ownership
+
+The owner-approved #998 scope retains the authenticated TanStack Start Node
+runtime. ADR 0032's historical pure-SPA target does not require deleting working
+identity forwarding. Phlo API owns authorisation, provider access, queries,
+governance, and backend state. Node handlers validate inputs, forward the user's
+identity, parse responses, and assemble presentation data only.
+
+The current 66 production handlers all belong to that API-adapter category.
+There are no UI-only server handlers or approved duplicate backend handlers.
+Each named handler below stays in its API adapter module. Legacy consumer
+migration and URL retirement belong to #1035, not this guard.
+
+| Adapter file | Count | Handlers retained as authenticated API adapters |
+|---|---:|---|
+| admin.ts | 7 | `getAdminIdentity`, `changeMemberRoles`, `createInvitation`, `createServiceAccount`, `revokeServiceAccount`, `getAuditLog`, `exportAuditLog` |
+| assets.ts | 12 | `getAssetPreview`, `rollbackAssetSnapshot`, `getAssetList`, `getAssetDetail`, `startAssetExactRowCount`, `getMaterializationEstimate`, `materializeAsset`, `backfillAsset`, `createAuditProposal`, `getAuditProposal`, `testAuditProposal`, `publishAuditProposal` |
+| branches.ts | 9 | `getBranchesPage`, `getWapRuns`, `getBranchDetail`, `createBranch`, `deleteBranch`, `runBranchChecks`, `rebaseBranch`, `trialMerge`, `signAndMerge` |
+| core.ts | 2 | `getShell`, `getOverview` |
+| datasets.ts | 1 | `getDatasetInventory` |
+| incidents.ts | 10 | `getSchemaDecisions`, `recordSchemaDecision`, `getIncidentList`, `getIncidentDetail`, `createIncident`, `updateIncident`, `setIncidentSubscription`, `createFollowUp`, `updateFollowUp`, `resolveIncident` |
+| pipelines.ts | 7 | `getPipelineList`, `getPipelineJob`, `getRunLogPage`, `launchJob`, `changeSchedule`, `cancelRun`, `retryRun` |
+| query.ts | 11 | `getQueryHistory`, `getQueryWorkspace`, `submitQuery`, `explainQuery`, `getQuerySession`, `getQueryIncidentTargets`, `pinQueryToIncident`, `cancelQuery`, `downloadQueryCsv`, `saveQuery`, `deleteSavedQuery` |
+| reports.ts | 1 | `getRunReport` |
+| settings.ts | 2 | `getSettings`, `saveSettings` |
+| staging.ts | 4 | `getStagingOverview`, `runStagingChecks`, `resyncStaging`, `promoteStaging` |
+
+`scripts/adapter-boundary.mjs` inspects production TypeScript with the TypeScript
+AST. It rejects provider/database dependencies, I/O outside the authenticated
+transport, shared backend credentials/configuration, SQL in adapters,
+noncanonical endpoints, and server functions outside the API adapter directory.
+Negative tests insert prohibited calls. The guard constrains code boundaries;
+it does not prove arbitrary new algorithms have no backend semantics.
+
+`forwarding.test.ts` uses a real HTTP listener to prove caller bearer precedence,
+trusted forwarded bearer fallback, absence of a configured shared-token bypass,
+method/body/environment preservation, HTTP failures, unavailable API failures,
+and rejection of cross-environment responses.

packages/phlo-api/tests/test_loki_api.py

@@ -166,8 +166,10 @@ async def fetch_entries(**_: object) -> LogQueryResult:
 @pytest.mark.anyio
 async def test_run_log_regex_timeout_does_not_stall_heartbeat_request(monkeypatch) -> None:
     secret_message = "a" * 30 + "!"
+    fetched = asyncio.Event()
 
     async def fetch_entries(**_: object) -> LogQueryResult:
+        fetched.set()
         return LogQueryResult(
             entries=[LogEntry(timestamp="1", level="info", message=secret_message, metadata={})],
             has_more=False,
@@ -187,7 +189,7 @@ async def heartbeat() -> dict[str, bool]:
         filter_request = asyncio.create_task(
             client.get("/api/loki/runs/run-1", params={"regex": r"(a+)+$"})
         )
-        await asyncio.sleep(0.02)
+        await asyncio.wait_for(fetched.wait(), timeout=5)
         heartbeat_response = await client.get("/heartbeat")
         filter_response = await filter_request