diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fcbd107..f680459 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: target: x86_64-unknown-linux-gnu - os: macos-latest target: aarch64-apple-darwin - - os: macos-13 + - os: macos-15-intel target: x86_64-apple-darwin steps: - uses: actions/checkout@v4 @@ -58,7 +58,7 @@ jobs: name=phlo-transform-${{ matrix.target }} mkdir -p "dist/$name" cp "target/${{ matrix.target }}/release/phlo-transform" "dist/$name/" - cp README.md LICENSE "dist/$name/" 2>/dev/null || true + cp README.md LICENSE "dist/$name/" cd dist && tar -czf "$name.tar.gz" "$name" echo "artifact=$name.tar.gz" >> "$GITHUB_OUTPUT" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index f598dd2..2ca362c 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,20 @@ the `CLEAN`/`REVIEW`/`UNSUPPORTED` classification, and the report format. ## Status -Audited against code and tests. Phases 0–1 are complete; Phases 2–8 are +Audited against code and tests. Phase 0 is complete; Phases 1–8 are implemented but **partial**, with gaps documented in each roadmap phase's implementation notes and in [`docs/roadmap/README.md`](docs/roadmap/README.md). - **Phase 0 — compiler spike: done.** Multi-root discovery, stable IDs, ordinary-SQL dependency resolution without `ref()`, deterministic DAG, `check`/`list`/`inspect` with JSON. See [`docs/architecture.md`](docs/architecture.md). -- **Phase 1 — MVP build engine: done.** Trino adapter, view/table +- **Phase 1 — MVP build engine: partial.** Trino adapter, view/table materialisations, `plan`/`apply`/`run`/`test`, bounded-concurrency scheduler, custom SQL tests, SQLite run history, cancellation and versioned artifacts. `--retries N` retries retryable adapter failures with bounded backoff; `--resume`/`--retry-failed` continue interrupted runs. See [`docs/engine.md`](docs/engine.md). + Gap: no partition-level scheduler tests. - **Phase 2 — typed compiler and lineage: partial.** Typed semantic IR, schema provider boundary and catalogue enrichment, column resolution and type inference, inferred output schemas, a canonical lineage graph behind @@ -132,8 +133,8 @@ implementation notes and in [`docs/roadmap/README.md`](docs/roadmap/README.md). with no infrastructure; `--adapter trino` targets a Trino cluster. See the adapter boundary in [`docs/engine.md`](docs/engine.md). -All eight numbered phases have implementations; only Phases 0–1 are complete -against their acceptance criteria. +All eight numbered phases have implementations; only Phase 0 is complete +against the audit's criteria. ## Toolchain @@ -196,7 +197,7 @@ runs a disposable container and is executed explicitly in CI. - [Full specification](SPEC.md) - [Blog series: what Phlo Transform is, from first principles](docs/blog/README.md) - [Implementation roadmap](docs/roadmap/README.md) -- [Convergence review](docs/convergence-review.md) — the system model, hardening pass and remaining debt +- [v0.1 release notes](docs/v0.1-release-notes.md) and the [final release review](docs/v0.1-final-review.md) - [Compiler architecture](docs/architecture.md) - [Engine architecture](docs/engine.md) - [Semantic compiler](docs/semantic.md) diff --git a/crates/phlo-transform-daemon/src/lib.rs b/crates/phlo-transform-daemon/src/lib.rs index e36ac8f..67e8faf 100644 --- a/crates/phlo-transform-daemon/src/lib.rs +++ b/crates/phlo-transform-daemon/src/lib.rs @@ -373,6 +373,7 @@ async fn status(State(service): State>) -> ApiResult { let errors = snapshot.errors().count(); Ok(Json(json!({ "workspace_root": snapshot.workspace_root.as_ref().map(|path| path.to_string_lossy().replace('\\', "/")), + "version": env!("CARGO_PKG_VERSION"), "compiler_semantics_version": phlo_transform_core::COMPILER_SEMANTICS_VERSION, "models": snapshot.models.len(), "sources": snapshot.sources().len(), diff --git a/crates/phlo-transform-daemon/tests/api.rs b/crates/phlo-transform-daemon/tests/api.rs index 8ca4eb9..0a29f5f 100644 --- a/crates/phlo-transform-daemon/tests/api.rs +++ b/crates/phlo-transform-daemon/tests/api.rs @@ -44,6 +44,11 @@ async fn exposes_semantic_api() { let status = get_json(&client, &format!("{base}/status")).await; assert_eq!(status["models"], 3); assert_eq!(status["sources"], 1); + assert_eq!( + status["version"], + env!("CARGO_PKG_VERSION"), + "status reports the daemon's release version" + ); let models = get_json(&client, &format!("{base}/v1/models")).await; assert_eq!(models["models"].as_array().unwrap().len(), 3); diff --git a/docs/blog/07-leaving-dbt.md b/docs/blog/07-leaving-dbt.md index a57b7f5..8bfd3b7 100644 --- a/docs/blog/07-leaving-dbt.md +++ b/docs/blog/07-leaving-dbt.md @@ -113,7 +113,5 @@ answer in one line per post: 6. Production changes are written, audited, and promoted on evidence. 7. Getting here from dbt is a command, not a rewrite. -Sources and further reading: [`SPEC.md`](../../SPEC.md) for the full design, -[`docs/roadmap/`](../roadmap/README.md) for honest implementation status, -and [`docs/validation.md`](../validation.md) for what an end-to-end audit -found and fixed. +Sources and further reading: [`SPEC.md`](../../SPEC.md) for the full design +and [`docs/roadmap/`](../roadmap/README.md) for honest implementation status. diff --git a/docs/convergence-review.md b/docs/convergence-review.md deleted file mode 100644 index 3077a8e..0000000 --- a/docs/convergence-review.md +++ /dev/null @@ -1,241 +0,0 @@ -# Convergence review - -A repository-wide hardening pass over `phlo-transform` after the roadmap -implementation: one audit of whether the compiler, state, execution, -lakehouse environments, promotion workflow, CLI and machine API all agree -about the same underlying model — and the fixes where they did not. - -## The model, in one paragraph - -A *logical environment* (`--environment`/`--ref`, aliases that must agree) -labels execution and scopes state. On a Nessie deployment it maps to a -candidate branch plus a branch-scoped physical catalog; without Nessie it is -an honest state-scoping label and nothing more. `--from` names the base ref -a candidate is cut from. `--catalog` pins the physical target; the generated -candidate catalog name is `phlo__<8-hex-sha256>`. -`--trino-catalog` is unrelated — it is the session catalog the connection -opens. `main` and the unlabeled default share the deployment catalog and -fold each other's state records. Promotion is authorised only by evidence -bound to the exact candidate and target commits being merged. - -## What diverged, and what changed - -### Environments: fabricated evidence closed - -The CLI used to record `--environment ci/foo` on a run while executing -against the *default* catalog — a labelled run bound to a branch head it -never wrote, which later promotion would have treated as candidate evidence. -Now CLI and daemon share `EnvironmentContext::resolve`: - -- `run`/`apply` resolve in `Ensure` mode — provision the branch and catalog, - then compile retargeted. -- `plan`/`test` resolve in `ReadOnly` mode — the same catalog precedence - (explicit override → recorded binding → generated name) and the same - retargeted compile, but nothing is created and no evidence is written. - A `plan(environment=X)` names exactly the targets `run(environment=X)` - executes. -- `ReadOnly` also applies `Ensure`'s provisioning requirement: when - resolution lands on the generated catalog and the adapter cannot - provision catalogs (`supports_catalog_provisioning`), the preview fails - closed too — it never names a target a run would refuse. An explicit or - recorded user-managed pin is the escape hatch. -- Nessie present but isolation unprovisionable (no adapter or no - catalog-facing URI) fails closed with `NotConfigured`/`API007` — the run - never proceeds on the default target while claiming a candidate. -- Nessie absent entirely: the environment is a state label and the CLI - prints a note saying so, rather than implying isolation that does not - exist. -- `--environment` and `--ref` given with different values is an error. -- `resume`/`retry_failed` infer the environment from the stored run — no - more needing to repeat `--ref` — and still refuse a flag that disagrees. - -### Orchestration: one implementation, two surfaces - -Promotion, candidate cleanup, run-reference binding, run lookup and test -execution were implemented twice — CLI and daemon had parallel copies that -had already drifted. They now live once in `phlo-transform-engine`: - -- `evaluate_promotion` — gathers candidate/target refs, the env-scoped run - and its model/seed/test records, the provisioning artifact, the hash-bound - branch diff, live contract breaks, merge check and lineage standing, then - evaluates the gates. `promotion.request()` pins the target hash at merge - so a racing commit is rejected by Nessie. -- `persist_promotion`, `cleanup_candidate`, `bind_run_reference`, - `find_unique_run` (distinguishing no-match from ambiguous), and - `execute_tests` are shared by both surfaces. - -Net effect: ~700 lines of duplicated orchestration removed; the CLI's -`promote` and the daemon's promote operation authorise a merge under -literally identical rules. - -### Catalog ownership - -Provisioning records how each catalog was established -(`created`/`unverified`/`unmanaged`) plus whether Phlo provably owns it. -A pre-existing catalog is accepted only on a recorded binding — the -generated name is a public convention anyone can mint, so it proves -nothing on its own, and a `ref create` intent record cannot vouch for a -catalog that appeared later. Nor can a legacy `unverified` record: the -`catalog_owned_by_phlo` flag was introduced with the vetting rule, so a -record lacking it may have been adopted on the name alone and is not -binding evidence. A catalog another candidate claims is refused; an -`unmanaged` generated name fails closed; a refused `unverified` catalog -rolls back a just-created branch so a failed provisioning attempt leaves -nothing behind. `cleanup_candidate` drops only -catalogs Phlo owns — including legacy `created` records predating the -ownership flag — and reports every failure rather than leaving silent -leftovers. Environment artifacts are removed only after the branch and -catalog are both gone. - -### Two Nessie addresses - -`--nessie-endpoint`/`PHLO_NESSIE_ENDPOINT` is the REST endpoint this process -calls. `--nessie-catalog-uri`/`PHLO_NESSIE_CATALOG_URI` (new) is the address -written into provisioned catalogs — what Trino's Iceberg connector dials. -They default to the same value; the split exists because Trino in a -container cannot reach `http://127.0.0.1:`. The daemon's -`ServiceConfig.nessie_uri` already modelled this; the CLI now passes the -catalog URI to provisioning instead of conflating the two. - -### Planner performance: measured, then fixed - -A committed scaling measurement (`plan_scaling_adapter_metadata`, ignored by -default) established the baseline with 5ms simulated metadata latency: - -| models | scenario | before | after | -|---:|---|---:|---:| -| 100 | cold | 0.7s | 16ms | -| 100 | warm | 1.4s | 63ms | -| 1,000 | cold | 7.6s | 56ms | -| 1,000 | warm | 14.8s | 515ms | -| 5,000 | cold | 45s | 305ms | -| 5,000 | warm | 82s | 2.6s | - -Three changes produced this, in order of leverage: - -1. **Batched metadata.** `Adapter::relations_exist` and - `relation_columns_many` batch relation probes through - `information_schema` (Trino groups by catalog/schema; the trait defaults - keep other adapters serial). One call replaces N `SELECT 1 … LIMIT 0` - round trips. A missing relation in `relation_columns_many` reports `Err` - rather than an empty schema — an empty schema would misclassify every - column as added. -2. **Bounded parallelism.** `output_identity` (a per-table `$snapshots` - read that cannot batch) is prefetched with `buffered(16)` before the - decision loop. `decide` is now synchronous over `ModelEvidence` — all - I/O happens in one prefetch pass. -3. **Killed the O(N²) CPU work.** `Selection::get` scanned `members` - linearly with a `logical_name()` allocation per comparison, called per - model — ~7.7s of pure CPU at 5k models. The planner builds a - `BTreeMap<&str, &SelectedModel>` once. The per-model "required by" BFS - became one multi-source BFS (`requiring_models`). - -State reads were bulked at the same time: `materialized_in`, `seeds_in` and -the new `StateStore::materialized_by_hashes` (real implementations for -SQLite and Postgres) replace per-model queries, and the default↔`main` -record fold lives in shared `materialized_scope`/`seeds_scope` helpers so -the planner, branch diff and CLI/daemon all scope identically. - -### Golden path, proven end-to-end - -`crates/phlo-transform-cli/tests/golden_path.rs` drives the real binary -through the full lifecycle against testcontainers Trino + Nessie: workspace -init → candidate provisioning → isolated run → branch diff → gate -evaluation → promotion → branch and catalog cleanup. It runs in CI -alongside the existing container tests. Stale-evidence rejection — a moved -candidate or target head, an unbound run — is covered by the engine's audit -tests rather than this path. The golden path's first real run is what -surfaced the Nessie endpoint/catalog-URI conflation. - -### Product-surface reconciliation - -- `doctor` gained a Nessie check: endpoint reachability plus the - catalog-URI the warehouse would be provisioned with, when it differs. -- Flag help now describes one model consistently across every subcommand. -- Docs audited and reconciled: `engine.md` no longer defers implemented - phases (its Deferred section listed contracts, incremental, WAP, diff and - the daemon — all shipped); `README.md`/`roadmap/README.md` Phase 8 no - longer claims a missing plan endpoint; `wap.md`/`SPEC.md` describe - read-only `plan`/`test` resolution, the `--environment`/`--ref` alias - rule and `--nessie-catalog-uri`; `architecture.md` acknowledges the - post-Phase-0 crates. - -## Evidence and provenance model - -Promotion evidence chains four sources: the live Nessie refs, the state -store's env-scoped run/model/seed/test records, the store's immutable -audit-evidence records (environment bindings, branch diffs, lineage diffs — -exported to `environment*.json`, `branch_diff.json`, `lineage_diff.json`), -and live-computed contract breaks. Everything is hash-bound: the diff -evidence records both refs' resolved heads, the run binds to the -candidate's post-run head, and `promote` re-checks both hashes at merge -time. Evidence in a shared state backend is portable across CI stages; a -store read failure fails closed rather than substituting local files that -could disagree with what another stage recorded. Unknown provenance fails -closed — a branch Phlo did not create has no recorded base, and "no -evidence" never reads as "no changes". - -## Crate boundaries - -The layout holds: `cli → {trino, duckdb, daemon} → engine → {core, -openlineage, nessie} → core → sql`. `cli` and `daemon` are composition -roots; the adapters implement the engine's trait and nothing reaches past -it; `dbt` lowers into `SemanticProject` without touching execution; `nessie` -is a leaf client. No splits were needed — the fix was moving orchestration -*into* the engine, not drawing new crate lines. - -## Remaining debt - -- **`output_identity` is inherently per-relation.** Warm plans at 5k models - still spend ~2.6s on 5000 metadata reads at 16-way concurrency — correct, - but a ceiling. -- **`cached` is classification only** — cross-environment reuse is planned - and evidenced, not executed as a data copy. -- **Daemon**: reload is a full recompile (no targeted invalidation); - progress is polled, not pushed. -- **`warm` cost asymmetry**: `relation_exists` batches but - `source_state`/`output_identity` do not — a non-Iceberg catalog offering - bulk metadata could narrow this further. - -## Maturity - -Levels: **Mature** — hardened *and* operationally complete; nothing here -is there yet. **Strong** — failure modes handled deliberately and covered -by tests; fails closed where safety demands. **Production-credible** — -works end to end with documented gaps. **Needs hardening** — the happy -path works; failure modes unexplored. **Experimental** — a stub or -sketch, present but not exercised. - -| area | level | evidence / remaining gap | -|---|---|---| -| compiler + diagnostics | Strong | typed IR, deterministic DAG, stable IDs, contracts; unsupported SQL degrades to `Unknown` honestly rather than guessing | -| dbt translator | Production-credible | 100% of jaffle_shop/canvas-exemplar convert CLEAN; dynamic Jinja/macros, exposures, metrics are REVIEW/UNSUPPORTED by design | -| lineage + impact | Production-credible | canonical graph, OpenLineage export, `--diff` across refs; offline column lineage needs catalog schemas; unsupported SQL → `Unknown` | -| planner + state/cache | Strong | batched warehouse+state evidence, synchronous `decide`, cache reuse requires same-relation + same-adapter + strong output identity; `cached` still classifies — reuse is not executed | -| execution runner | Strong | bounded concurrency, `--retries` backoff on retryable adapter failures, timeouts, cancellation, `--resume`/`--retry-failed`; DuckDB has no remote cancel | -| incremental models | Production-credible | append/merge/partition/window strategies, watermarks, schema-change rebuilds; Trino `MERGE` verified live; partition replace is a column-list delete, and watermark coverage is not per-adapter | -| state store | Strong | env-scoped records, `main`↔default fold, SQLite/Postgres parity tested; watermarks deliberately do not fold | -| environments + provisioning | Strong | one resolve path, fail-closed, recorded bindings, ownership-aware cleanup, branch rollback on rejected catalogs | -| WAP + promotion | Strong | hash-bound evidence, provenance-required base, merge-time hash recheck, gates fail closed, live golden-path E2E; evidence is state-store backed and portable across machines via Postgres | -| data diff | Production-credible | keyed/tolerance/partition/sampled diff live-tested; no distribution summaries, no example-value redaction | -| daemon API | Production-credible | versioned ops API, idempotency, cancellation, coherent snapshots; reload is a full recompile and progress is polled | -| CLI | Strong | every surface shares engine orchestration; JSON output everywhere; `--environment`/`--ref` agreement enforced | -| workflow integration | Needs hardening | ownership + `quality_gate` exist; host workflow tasks, run correlation and gating APIs are Phase 7 debt | - -The model now holds together: one environment resolution path, one -promotion audit, one ownership story, one state scope — and every surface -(CLI, daemon, plan preview, continuation runs, diff, promote, cleanup) -reads from it identically. The fail-closed posture is tested rather than -asserted: unprovisionable environments error, unverifiable catalogs are -refused, stale evidence blocks, and the golden path proves the whole loop -on real infrastructure. - -## Top priorities from here - -1. Cache-reuse *execution* — `cached` currently classifies; actually - reusing the materialised output closes the loop. -2. Targeted daemon invalidation — reload by changed paths rather than full - recompile. -3. Bulk source-state/output-identity reads where the catalog allows. -4. `plan` output — surface the resolved physical catalog once at the top - (it is currently only per-model in each target row). diff --git a/docs/daemon.md b/docs/daemon.md index 9d60142..53b3378 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -32,7 +32,7 @@ launched with (`adapter`, `state`, `nessie`). | Endpoint | CLI equivalent | Notes | |---|---|---| -| `GET /status`, `GET /v1/status` | — | workspace root, semantics version, counts, capabilities, last update | +| `GET /status`, `GET /v1/status` | — | workspace root, release version, compiler semantics version, counts, capabilities, last update | | `GET /v1/check` | `check --json` | diagnostics / check report | | `GET /v1/models` | `list --json` | models, sources and tests | | `GET /v1/models/{id}` | `inspect --json` | model detail | diff --git a/docs/release-validation.md b/docs/release-validation.md deleted file mode 100644 index d66d812..0000000 --- a/docs/release-validation.md +++ /dev/null @@ -1,200 +0,0 @@ -# Release validation — v0.1.0 candidate - -Validated `phlohouse/phlo-transform` in a completely fresh container -(`rust:1.93-bookworm`, aarch64, OrbStack). Nothing from the host toolchain was -used: the image runs `cargo install --path crates/phlo-transform-cli --locked` -against a clean copy of the repo and clones the external test projects from -GitHub. Build from scratch: ~5m40s (dominated by the bundled DuckDB C++ -compile). - -External projects: - -- `dbt-labs/canvas-exemplar` — realistic dbt translation stress test -- `dbt-labs/jaffle_shop_duckdb` — DuckDB-compatible end-to-end execution test - -## Getting-started path - -The documented path works verbatim with no undocumented setup: - -```text -phlo-transform init / check / plan --adapter duckdb / run --adapter duckdb -phlo-transform explain example.daily_events / phlo-transform doctor -``` - -All succeed; `doctor` reports workspace/compile/adapter/state/nessie health and -warns (correctly, exit 0) when no adapter is configured for a command that -does not need one. - -## Translation coverage - -### canvas-exemplar (14 models, 6 sources, 6 macros, 3 packages, 25 exposures) - -Upstream checkout ships no `profiles.yml`. Two runs are reported: the -untouched project, and the project plus a minimal `type: duckdb` profile -(which dbt itself also requires to run). - -| Classification | Initial | After fixes | Upstream (no profile) | With DuckDB profile | -|---|---|---|---|---| -| CLEAN models | 5 (35.7%) | 9 (64.3%) | **13 (92.9%)** | **14 (100%)** | -| REVIEW models | 9 | 5 | 1 | 0 | -| CLEAN macros | — | — | 6 | 6 | -| CLEAN packages | — | — | 1 (`dbt_utils`) | 2 (+ `dbt_date`) | -| UNSUPPORTED | 5 macros, 25 exposures | unchanged | 29 property-only resources (exposures/metrics/semantic models + `unit_tests`/`groups`) | same | - -Compatibility pass (current): the five REVIEW models converted cleanly — -`cents_to_dollars` inlines via `adapter.dispatch` → `default__` variant -(statically selected), `dbt_utils.generate_surrogate_key` lowers to -`md5(concat_ws(…))`, and `dbt_utils.star` lowers to `* exclude (…)` for the -provable argument subset. `metricflow_time_spine` uses -`dbt_date.get_base_dates`, which lowers to a DuckDB-specific -`generate_series` spine only when the source profile declares -`type: duckdb` — without a profile it stays REVIEW rather than emit -backend-specific SQL for an unknown destination. -The `generate_schema_name` override is evaluated statically for every -exercised `(resource_type, schema)` case — canvas's cases never reach its -`target.name` arm, so it proves target-independent and CLEAN. When a macro -does branch on `target.name` and the profile declares multiple outputs, -every output is evaluated and the macro stays REVIEW (naming the diverging -target) unless all agree — a Phlo workspace is not dbt-target-specific. -Declared packages are CLEAN when every observed call site lowers; the -unpinned `dbt-audit-helper` git dependency stays REVIEW. -Semantic-layer resources remain UNSUPPORTED by design. - -Note: the analysis report counts all 6 declared sources; the generated -workspace reports 3 — only sources actually referenced by models are -inferred into the Phlo graph. Intended, but worth knowing when comparing -counts. - -### jaffle_shop_duckdb (5 models, 3 seeds) - -| | Initial | After fixes | After compat pass | -|---|---|---|---| -| CLEAN models | 1 | 1 | **5 (100%)** | -| REVIEW models | 4 | 4 | 0 | -| CLEAN seeds | 0 | 0 | **3** | - -The `orders.sql` pivot (`{% set payment_methods = [...] %}` + `{% for %}` + -`{% if not loop.last %}`) now expands statically at translation time, and -seeds became runnable inputs rather than manual prerequisites. - -## End-to-end execution (jaffle_shop_duckdb → DuckDB) - -The translated workspace runs verbatim with no manual seed loading: the -three CSVs are copied to `seeds/` and the runner loads them -(`CREATE OR REPLACE TABLE main.raw_* AS SELECT * FROM read_csv_auto(…)`) -before building models. - -- `doctor`, `check`, `plan`, `run`, `test`, `inspect`, `explain`, `lineage`, - `manifest` — all exit 0. -- 3 seeds load; 5 models build; 20 tests pass (unique, relationships, - accepted_values from `arguments:` syntax, not_null directives). -- **State:** seed CSV content hashes are recorded in `seed_loads`; a changed - CSV re-plans the seed (and its downstream models, via the `csv:` source - state) and an unchanged CSV is `SKIP`. Source-table appends still trigger - rebuilds as before. - -## Scaling benchmark (synthetic workspaces, DuckDB `:memory:`) - -| Workspace | `check` before | `check` after | `plan` before | `plan` after | -|---|---|---|---|---| -| 100 models, 5 layers | 0.00s | 0.00s | 0.13s | 0.20s | -| 1,000 models, 5 layers | 0.37s | 0.07s | 0.93s | 0.25s | -| 5,000 models, 5 layers | 8.51s | 0.32s | 18.10s | 2.05s | -| 5,000 models, all independent | 15.02s | 0.19s | — | 0.95s | -| 5,000 models, single chain | 3.83s | 0.19s | — | 1.10s | - -Peak RSS at 5,000 models: ~135 MB (`check`), ~168 MB (`plan`). `check` is now -effectively linear; `plan` is dominated by per-model `relation_exists` -catalogue probes, which is inherent to a state-aware planner. - -## Bugs found and fixed - -1. **`ref()` to seeds failed to resolve (DBT001).** Seed names are known - resources; a ref now resolves to the relation dbt would materialise the - CSV as (file stem in the target schema). Seeds are native runnable - inputs — the CSV is copied to `seeds/` and loaded automatically — so - both the seed and the referencing model classify CLEAN. - `relationships` test targets pointing at seeds resolve the same way. -2. **dbt `arguments:` test syntax unsupported.** The modern form - `accepted_values: {arguments: {values: [...]}}` (and `relationships: - {arguments: {to, field}}`) produced false "lacks a column or values" / - "target could not be resolved" findings. `arguments` is now merged into - the test arg map for both model and source tests. -3. **`dbt.date_trunc` unnecessarily unsupported.** The dbt cross-database - shim `dbt.date_trunc('day', 'col')` now lowers to native - `date_trunc('day', col)` (the string-literal column argument becomes a - bare identifier). Other package/project macros still classify REVIEW — - Jinja is not executed, by design. -4. **DuckDB source state silently broken for unqualified sources.** A bare - `from raw_events` resolves through DuckDB's search path to - `main.raw_events`, but enrichment qualified it as `default.raw_events`; - the resulting `DESCRIBE` error then discarded *all* source-state and - schema enrichment. Appended source rows were invisible and every plan - reported `SKIP` forever. Fixes: unqualified sources default to `main` - under the DuckDB adapter, `collect_source_states` failure no longer - discards schema enrichment, and `relation_columns` returns empty on a - missing relation instead of erroring. -5. **Quadratic compile time.** `Resolver` linear-scanned every model up to - four times per relation reference; duplicate detection and the - type-analysis loop each re-scanned the model list per model. All are now - hash/B-tree indexed — 5,000-model `check` went 8.5s → 0.3s. - -## Regression coverage added - -- `fixtures/dbt-seeds` — seed `ref()`, `arguments:` test syntax, - `dbt.date_trunc`, and a `get_base_dates` call with no `profiles.yml` - staying REVIEW. -- `fixtures/dbt-jaffle` — extended with `{% set %}`/`{% for %}` expansion - (`payments_pivot`), `dbt_utils.generate_surrogate_key` + - dispatching `cents_to_dollars` (`orders_enriched`), - `dbt_date.get_base_dates` (`time_spine`), `dbt_utils.star` - (`package_users`, `package_users_aliased`/`package_users_renamed` → - REVIEW), grouped `not_constant`, and a `dbt_utils.expression_is_true` - test. -- `fixtures/dbt-dynamic` — a dynamic `{% for %}` over `run_query` stays - REVIEW and keeps `--verify` failing. -- `fixtures/native-seeds` — native `seeds/**/*.csv` discovery, - `[seeds] schema`, content-hash versioning. -- `phlo-transform-dbt::seed_refs_arguments_syntax_and_dbt_builtins`. -- `phlo-transform-engine::seed_loads_before_models_and_skips_when_unchanged`, - `seed_content_change_replans_the_load`, - `failed_seed_load_blocks_dependent_models`, and - `seed_tests_pull_the_seed_into_the_plan` — plan/apply/state over a - fake adapter. -- `phlo-transform-core::csv_seeds_are_discovered_and_compiled`. -- `phlo-transform-cli::unqualified_source_appends_trigger_rebuilds_on_duckdb` - — full lifecycle e2e asserting a source append reclassifies the model - `BUILD`, not `SKIP`. - -## Known limitations (documented) - -- dbt semantic-layer resources (metrics, semantic models, exposures), - snapshots, analyses, `unit_tests`/`groups`, and disabled models are - UNSUPPORTED by design. -- Dynamic macros and Jinja constructs (`{% set %}` capture blocks, `{% for %}` - over non-literals, `run_query`, `{% call %}`, macro bodies containing - `{% %}` statements, e.g. `generate_schema_name`) are never executed; - call sites are REVIEW and must be rewritten by hand. -- Seeds translate to native CSV inputs (DuckDB `load_csv`); adapters - without a `load_csv` implementation report them UNSUPPORTED. -- DuckDB `source_state` fingerprints schema + row count: in-place updates - that preserve the row count remain invisible (Iceberg snapshot IDs are the - precise mechanism). -- `inspect`/`explain` without `--adapter` compile offline and can report a - spurious `changed` status; pass `--adapter` for catalogue-enriched state. -- The Trino/Nessie path (WAP, promotion, diffs) was not exercised live in - this validation — it has dedicated `--ignored` E2E tests and CI coverage. - -## Verdict - -**Ready for v0.1.0.** The documented install and getting-started path works -in a clean container with no undocumented steps; the dbt translator produces -honest, actionable classifications on realistic projects (all -jaffle_shop_duckdb models and all canvas-exemplar models convert CLEAN -(13/14 untouched; 14/14 with a declared DuckDB profile); -every remaining REVIEW/UNSUPPORTED is a real dynamic, backend-ambiguous, or -semantic-layer construct a human must decide on); the DuckDB lifecycle — seeds, plan, run, test, -state-aware re-runs — is verified end-to-end; and compile performance is -comfortable to at least 5,000 models. The version number already says -what it is: an early release with documented gaps, and the docs match the -behaviour observed here. diff --git a/docs/roadmap/03-state-aware-execution.md b/docs/roadmap/03-state-aware-execution.md index 7434a8b..8ed74d6 100644 --- a/docs/roadmap/03-state-aware-execution.md +++ b/docs/roadmap/03-state-aware-execution.md @@ -361,17 +361,23 @@ Phase 3 is **partially implemented** (audited against code and tests). See - SQLite `model_versions` table with `record_materialized`, `materialized_version` and `materialized_by_hash`. - `Planner` is state-aware: `build` / `skip` / `cached` with structured - `ChangeReason`s. + `ChangeReason`s. `cached` executes verified adoption: the plan carries the + source materialisation's provenance and the runner re-verifies the live + output identity (Iceberg snapshot) before adopting, falling back to `build` + on stale evidence. - Runner records materialisations and rejects stale plans. - `inspect` exposes desired/current versions and status. Tests cover formatting/comment stability, SQL/config/materialisation/upstream and source-state invalidation, owner/tag non-invalidation, second-run skip, -cross-environment `cached` classification, and stale-plan rejection. - -Remaining gaps: `cached` is a classification only (no cross-environment reuse -execution); no direct test for `compiler_semantics_change`. Non-Iceberg sources -fall back to a schema fingerprint rather than data state. +cross-environment `cached` adoption, stale-identity build fallback, and +stale-plan rejection — plus the live Nessie e2e adopting inherited +materialisations with zero model SQL. + +Remaining gaps: no direct test for `compiler_semantics_change`. Non-Iceberg +sources fall back to a schema fingerprint rather than data state. `cached` +adoption is same-adapter only and requires the relation to be visible through +the target environment's binding (true for Nessie branch inheritance). Deferred, as listed above: optimiser semantics. diff --git a/docs/roadmap/05-nessie-wap.md b/docs/roadmap/05-nessie-wap.md index 066abb9..d287dca 100644 --- a/docs/roadmap/05-nessie-wap.md +++ b/docs/roadmap/05-nessie-wap.md @@ -361,15 +361,16 @@ Implemented: against Nessie 0.108) and in-memory client; references, branch creation, non-destructive merge checks, merge against an expected target hash, assign (rollback) and delete. -- `apply --ref --from ` creates the candidate branch from the - base and a branch-scoped Trino catalog (dynamic catalog management; Trino - cannot switch Nessie refs at query time) and compiles/applies against it, so - candidate data is isolated from `main`. +- `run`/`apply --ref --from ` creates the candidate branch + from the base and a branch-scoped Trino catalog (dynamic catalog management; + Trino cannot switch Nessie refs at query time) and compiles/applies against + it, so candidate data is isolated from `main`. - `promote` enforces a successful candidate run, target staleness via the recorded base hash (`environment.json`), merge conflicts, an optional data-diff gate (`--require-diff`) and a breaking-schema gate (`--allow-breaking-schema`), and can remove the candidate with `--cleanup`; - it writes `promotion.json`. + it writes `promotion.json` and persists the record in the state store's + `promotions` table. - `rollback --ref --to ` moves a reference. Missing / deviated: @@ -380,6 +381,4 @@ Missing / deviated: opt-in (`--cleanup`) rather than automatic. - Views are unsupported by Trino's Nessie Iceberg catalog, so view models cannot run in a Nessie candidate environment. -- Promotion state is written to an artifact but not persisted in the state - store. diff --git a/docs/roadmap/08-daemon-agent-apis.md b/docs/roadmap/08-daemon-agent-apis.md index dde849b..206dcd8 100644 --- a/docs/roadmap/08-daemon-agent-apis.md +++ b/docs/roadmap/08-daemon-agent-apis.md @@ -443,7 +443,10 @@ Implemented: - `phlo-transform-daemon`: `WorkspaceService` holding an immutable compiled snapshot, versioned `/v1` routes for status/check/models/inspect/lineage/ - impact/graph, and a polling file watcher that reloads on change. + impact/graph/plan plus diff and state reads, tracked `POST /v1/operations` + mutations (`run`/`test`/`promote`/`resume`/`retry_failed`/`reload`) with + idempotency keys and cooperative cancellation, and a polling file watcher + that reloads on change. - CLI `daemon` command binds `127.0.0.1`. - API and CLI share the same report DTOs; errors carry stable codes. - Tests cover the HTTP surface, explicit reload and watcher-driven reload @@ -451,7 +454,6 @@ Implemented: Missing / deviated: -- No plan endpoint on the API (the roadmap lists plan information). - Reload is a **coherent full recompile**, not dependency-aware targeted invalidation (acceptance criterion 2 is deviated). - No concurrency/snapshot-consistency test under simultaneous reads and writes. diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index 23cdfa3..e206ef0 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -37,14 +37,14 @@ Audited against code and tests (not commit messages). Legend: **done**, | 0 compiler spike | done | discovery, IDs, `sqlparser-rs`, CTE-aware relations, resolver, DAG, diagnostics, CLI JSON; parse errors carry file but not spans | | 1 MVP engine | partial | Trino adapter + plan/apply/run/test, scheduler, state, artifacts, cancellation, `--retries` backoff + `--resume`/`--retry-failed` continuations. Gaps: partition-level scheduler tests absent | | 2 typed compiler/lineage | partial | semantic IR, resolver, type inference, contracts, generated tests, lineage/impact; nested `array`/`map`/`row` types parsed; `lineage --upstream/--downstream` filter direction; `impact` accepts model or column and renders consumers. Gaps: offline column lineage needs catalogue schemas (model-level impact works offline); unsupported SQL is `Unknown` | -| 3 state-aware execution | partial | versions, skip/build/cached, stale plans. Source states are wired: `Adapter::source_state` reads Iceberg snapshots (schema-fingerprint fallback) and CLI plan/apply/run enrich compilation with them; cached reuse covered by test. Gaps: non-Iceberg sources use schema rather than data state; no cache *reuse* execution (classification only) | +| 3 state-aware execution | partial | versions, skip/build/cached, stale plans. Source states are wired: `Adapter::source_state` reads Iceberg snapshots (schema-fingerprint fallback) and CLI plan/apply/run enrich compilation with them. `cached` is executable: verified cross-environment adoption (Iceberg snapshot identity checked at plan and run time) with build fallback on stale evidence — covered live on Nessie. Gaps: non-Iceberg sources use schema rather than data state; adoption requires the relation to be visible through the target environment's binding (true for Nessie branch inheritance); no cross-adapter reuse | | 4 incremental models | partial | `@incremental` strategies; append/merge; partition replaces touched partitions; time-window uses a committed watermark with typed predicate and applied overlap; schema classification forces full rebuilds; Trino `MERGE` verified live on Iceberg. Gaps: incremental partition replacement uses a column-list delete (no metadata pruning); no schema/watermark test against every adapter | | 5 Nessie + WAP | partial | Nessie client boundary (REST v2, live-verified), `promote`/`rollback`/promotion artifact. `run`/`apply --ref --from ` auto-creates the branch and a branch-scoped Trino catalog and writes candidate data; `plan`/`test --ref` resolve the same environment read-only. Catalog ownership is recorded so cleanup drops only catalogs Phlo created; `--environment` and `--ref` are aliases that must agree; `--nessie-catalog-uri` splits the warehouse-facing URI from the client endpoint. Promotion gates (run/tests/blocked/schema/data_diff/base/conflicts) run through one shared engine evaluation for CLI and daemon; live Nessie + Trino/Iceberg E2E covers isolation, diff, promotion, stale rejection and cleanup — including a CLI golden path (`golden_path.rs`). Gaps: no Iceberg snapshot in the promotion record, no automatic rebase of an advanced target; Nessie Iceberg catalog has no views | | 6 data diff | partial | keyed diff with per-column counts and configured policies/tolerances; real `TABLESAMPLE` sampling; partition comparison via Iceberg `$partitions` metadata (row-count fallback); populated schema diffs; stale-diff invalidation at promotion; live Trino coverage for keyed/tolerance/partition/sampled. Gaps: no statistical distribution summaries; no example-value redaction | | 7 workflow integration | partial | ownership, typed graph with `quality_gate`, cross-workflow policy, registered consumers. Missing: host workflow tasks/graph, transform-group invocation API, run correlation, gating API, e2e | | 8 daemon + agent APIs | partial | versioned local HTTP API, coherent snapshot, watcher/reload, `/v1/plan` with read-only environment resolution, operations API, API tests. Missing: targeted invalidation (full reload), benchmark, push diagnostics | -| dbt migration | done | `translate --from dbt` (phlo-transform-dbt): project/YAML/model/test/macro discovery, Jinja scan-classify (no execution), `ref`/`source`/`var`/`config` lowering, materialisation + incremental strategy mapping (merge/key, insert_overwrite/partition, append, watermark `is_incremental()` → `window`), generic-test conversion (`unique`/`not_null`/`accepted_values`/`relationships`/`unique_combination_of_columns` plus `dbt_utils.expression_is_true`/`accepted_range`/`not_constant`/`not_empty_string`), `CLEAN`/`REVIEW`/`UNSUPPORTED` classification, human + `--json` report, `.phlo/migration/dbt-translation.json` manifest, `--out`/`--overwrite`/`--verify` (non-zero exit when the generated workspace fails `check`). Compat pass: native CSV seeds (`seeds/**/*.csv` copy + `[seeds]`/`[seed.*]` schema + content-hash state + DuckDB `load_csv`), static `{% set %}`/`{% for %}`/`{% if %}` expansion, static lowering of `dbt_utils.generate_surrogate_key`/`star`/`safe_cast`, `dbt_date.get_base_dates`, `dbt.current_timestamp`/`date_trunc`, and restricted project-macro inlining (incl. `adapter.dispatch` → `default__` variants) — 100% of jaffle_shop_duckdb and canvas-exemplar models convert CLEAN. See `docs/dbt-migration.md` and `docs/release-validation.md`. Gaps: dynamic Jinja/macros and `run_query` are REVIEW (never executed); exposures, metrics, semantic models, snapshots, `unit_tests`/`groups` unsupported; dbt `source()`/model `meta` beyond owner lands only in the manifest | -| second adapter | done | `phlo-transform-duckdb` implements the `Adapter` trait over embedded DuckDB (bundled); `--adapter duckdb`/`--duckdb-path` selects it in the CLI; merge is emulated as delete+insert; tested in-memory, file-backed and end-to-end via a translated dbt project including incremental second runs (see `docs/validation.md`). Gaps: no Iceberg/Nessie awareness (source_state is a schema + row-count fingerprint — in-place updates are invisible), no remote cancellation | +| dbt migration | done | `translate --from dbt` (phlo-transform-dbt): project/YAML/model/test/macro discovery, Jinja scan-classify (no execution), `ref`/`source`/`var`/`config` lowering, materialisation + incremental strategy mapping (merge/key, insert_overwrite/partition, append, watermark `is_incremental()` → `window`), generic-test conversion (`unique`/`not_null`/`accepted_values`/`relationships`/`unique_combination_of_columns` plus `dbt_utils.expression_is_true`/`accepted_range`/`not_constant`/`not_empty_string`), `CLEAN`/`REVIEW`/`UNSUPPORTED` classification, human + `--json` report, `.phlo/migration/dbt-translation.json` manifest, `--out`/`--overwrite`/`--verify` (non-zero exit when the generated workspace fails `check`). Compat pass: native CSV seeds (`seeds/**/*.csv` copy + `[seeds]`/`[seed.*]` schema + content-hash state + DuckDB `load_csv`), static `{% set %}`/`{% for %}`/`{% if %}` expansion, static lowering of `dbt_utils.generate_surrogate_key`/`star`/`safe_cast`, `dbt_date.get_base_dates`, `dbt.current_timestamp`/`date_trunc`, and restricted project-macro inlining (incl. `adapter.dispatch` → `default__` variants) — 100% of jaffle_shop_duckdb and canvas-exemplar models convert CLEAN. See `docs/dbt-migration.md`. Gaps: dynamic Jinja/macros and `run_query` are REVIEW (never executed); exposures, metrics, semantic models, snapshots, `unit_tests`/`groups` unsupported; dbt `source()`/model `meta` beyond owner lands only in the manifest | +| second adapter | done | `phlo-transform-duckdb` implements the `Adapter` trait over embedded DuckDB (bundled); `--adapter duckdb`/`--duckdb-path` selects it in the CLI; merge is emulated as delete+insert; tested in-memory, file-backed and end-to-end via a translated dbt project including incremental second runs. Gaps: no Iceberg/Nessie awareness (source_state is a schema + row-count fingerprint — in-place updates are invisible), no remote cancellation | | developer UX | partial | `explain`, `doctor`, `init`, `manifest` commands added; `--adapter` selection and clearer adapter errors. Gaps: no `fmt` command (no native formatter exists), plan output unchanged | M0 and M1 are met. M2–M4 are met only at the documented partial level above. diff --git a/docs/v0.1-final-review.md b/docs/v0.1-final-review.md new file mode 100644 index 0000000..3145ba0 --- /dev/null +++ b/docs/v0.1-final-review.md @@ -0,0 +1,324 @@ +# v0.1 final release review + +Final release-gate review of `phlo-transform` as v0.1.0. Scope: verify the +baseline, run the full validation suite live, check packaging/install/ +upgrade paths, and record a verdict. Not an architecture review. + +This document now also carries the validation evidence that previously lived +in the intermediate review reports (convergence review, dbt-migration audit, +fresh-container validation, readiness report) — those files were removed +from the tree and their load-bearing findings folded in here. + +## Baseline + +| Item | Value | +|---|---| +| Commit reviewed | `5fcb695698a7b344dccde8a2a6ddb12623299502` (`main`, post-PR #18) | +| Rust toolchain | `rustc 1.93.0 (254b59607)`, pinned via `.mise.toml` (`rustfmt`, `clippy`) | +| Crate versions | all 10 workspace crates at `0.1.0` (workspace-inherited) | +| `rust-version` | `1.93` | +| Compiler semantics | `COMPILER_SEMANTICS_VERSION = 3` (`phlo-transform-core/src/version.rs`) | +| Artifact schema | `SCHEMA_VERSION = 3` (`phlo-transform-engine/src/artifacts.rs`) | +| Daemon API | `/v1/...` — reads, `POST /v1/operations`, cancel, reload | +| Adapters | `duckdb` (embedded, bundled) and `trino` (HTTP) | +| State stores | SQLite (default `.phlo/transform/state.db`), Postgres (`--state postgres://…`, `NoTls`) | + +Feature claims verified against code: executable `cached` adoption +(`PlanAction::Cached` → verified adopt-or-build path in `run.rs`), +catalog-free target version input (`schema.table|materialization`), +environment-qualified source compilation, composite-key assertions, +`ref delete` catalog cleanup, Trino `load_csv`, retryable Nessie/identity-read +classification, evidence-id-bound promotion records, and the +`skip`/`cached`/`build` semantics all exist as documented. + +## Validation + +Run on this checkout (macOS arm64, OrbStack Docker): + +| Suite | Command | Result | +|---|---|---| +| Format | `cargo fmt --all -- --check` | clean | +| Lint | `cargo clippy --workspace --all-targets -- -D warnings` | clean | +| Unit/integration | `cargo test --workspace` | **582 passed, 0 failed**, 10 ignored | +| Postgres shared-state | `PHLO_TEST_POSTGRES_URL=… cargo test -p phlo-transform-engine` | 5/5 vs real Postgres 16 | +| Trino e2e | `--test trino_e2e -- --ignored` | 3/3 (run+test, keyed diff, partition/tolerance/schema/sampling diff) | +| Nessie WAP | `--test nessie_wap_e2e -- --ignored` | 2/2 (WAP promotion; candidate adopts inherited materialisations, zero model SQL) | +| Daemon env isolation | `--test nessie_e2e -- --ignored` (daemon crate) | 1/1 (candidate run cannot touch base tables) | +| Incremental MERGE | `--test trino_incremental_e2e -- --ignored` | 1/1 (Trino MERGE on Iceberg) | +| CLI golden path | `--test golden_path -- --ignored` | 1/1 (init→promote on live Trino+Nessie) | +| Dogfood (DuckDB) | `scripts/dogfood.sh` | 16/16 | +| Dogfood (live) | `PHLO_TRINO_ENDPOINT`+`PHLO_NESSIE_ENDPOINT`+`PHLO_NESSIE_CATALOG_URI` | **24/24** — incl. `run --ref` asserted `cached` for every model via JSON, `diff`, `promote --check`, `ref delete` cleanup | + +The 10 ignored tests are the eight Docker e2e above plus two scaling +benchmarks run explicitly — `plan_scaling_e2e` on real Trino and +`plan_scaling_adapter_metadata` against a counting adapter. All Docker e2e +were run here and pass. +Dogfood output was clean: no warnings, no retries, no suspicious skips; the +warm run reported all-skip as designed. + +### Performance (real Trino + Nessie + Iceberg, testcontainers on OrbStack, Apple M2 Pro) + +Per-model `output_identity` snapshot reads dominate warm plans — the drift +check is what makes a recorded version *evidence*. + +| Models | cold plan | materialise | warm plan | changed-subset | `--since` | candidate plan | candidate run | candidate miss (plan / run) | +|---|---|---|---|---|---|---|---|---| +| 100 | 1.2s / 1 call | 10.0s | 1.1s / 101 | 1.1s / 102 | 0.13s / 7 | 1.2s / 101 — 100% cached | 0.9s — 100 cached, 0 built | 1.3s / 1.6s — 95 cached + 5 built | +| 1,000 | 0.6s / 1 call | 132s | 13.8s / 1,001 | 13.5s / 1,002 | 0.4s / 12 | 13.1s / 1,001 — 100% cached | 12.6s — 1,000 cached, 0 built | 13.0s / 13.3s — 990 cached + 10 built | +| 5,000 | 1.8s / 2 calls | 291s | 27.7s / 5,001 | 56.6s / 5,002 | 1.2s / 52 | 80.7s / 5,001 — 100% cached | 57.5s — 5,000 cached, 0 built | 57.9s / 79.2s — 4,950 cached + 50 built | + +(Adapter calls counted on a wrapping `CountingAdapter`; "materialise" is the +run that creates the tables, a prerequisite not a planning measure. Candidate +run = zero model SQL; its adapter calls are the identity verification reads +plus schema provisioning.) + +Compiler scaling is comfortable well past release scale: on synthetic +workspaces, `check` is 0.32s and `plan` 2.05s at 5,000 models / 5 layers +(DuckDB `:memory:`), peak RSS ~170 MB. + +### Correctness evidence + +Bugs found by actual use across the earlier audit passes, each locked in by +a regression test (full narratives removed with the intermediate reports; +the regressions remain): + +- **`cached` was classification-only** — adoption now persists an + environment-local record claiming the source run verbatim + (`a_cached_run_adopts_the_verified_output_into_the_environment`, + `a_cached_run_falls_back_to_build_when_identity_drifts`, + `candidate_reuses_inherited_materialisations_without_rebuilding`). +- **Model versions were not portable across environments** — the catalog is + an environment binding, not model content; target input is + `schema.table|materialization` and semantics v3 + (`cache_reuse_matches_across_catalogs_on_the_same_content_slot`, + `a_catalog_retarget_adopts_the_visible_output`, + `a_catalog_retarget_without_verifiable_identity_rebuilds`). +- **Composite `@key a,b` generated per-column unique tests** — now one + composite unique plus `not_null` per column + (`a_composite_key_asserts_one_composite_unique`). +- **Source references bypassed the environment catalog** + (`a_candidate_compile_qualifies_source_references`). +- **`ref delete` leaked the provisioned Trino catalog** — routed through + `cleanup_candidate`; verified live in dogfood. +- **Trino had no `load_csv`** — implemented with RFC 4180 parsing, type + inference, batched inserts. +- **Transient identity reads masqueraded as "no verifiable output"** — + non-missing-relation errors now retry (150ms, 400ms) before reporting + `None`; anything still unreadable fails closed to a build. +- **Nessie commit hiccups were never retryable** — `EXTERNAL`, + `INSUFFICIENT_RESOURCES` and the Nessie signature inside + `GENERIC_INTERNAL_ERROR` are retryable. +- **A catalog retarget left plans undetectably stale** — `Plan::staleness` + also requires `compiled.target.display() == model.target` + (`a_plan_is_stale_when_the_compilation_retargets_the_model`). +- **`retry_failed` dismissed now-cacheable failures** — `Build | Cached` + counts as executable (`retry_failed_executes_a_now_cacheable_model`). +- **Promotion evidence ids were re-looked-up after the gates ran** — audits + return the consulted record's id; no second read + (`audit_evidence_ids_name_the_record_the_audit_consulted`, + `a_promotion_records_the_evidence_ids_the_gates_evaluated`). +- **`translate --verify` exited 0 on failing output** — + `translate_verify_failure_exits_nonzero`. +- **DuckDB `source_state` fingerprinted schema only** — appends invisible to + the planner; now includes `count(*)` + (`unqualified_source_appends_trigger_rebuilds_on_duckdb`). +- **Quadratic compile time** — resolver/type-analysis scans are indexed; + 5,000-model `check` 8.5s → 0.3s. +- **dbt `arguments:` test syntax, seed `ref()`, `dbt.date_trunc`** — all + lowered correctly with fixtures (`dbt-seeds`, `dbt-jaffle`, + `native-seeds`). + +### dbt migration validation + +Audited against three fixtures (`dbt-clean`, `dbt-shop`, `dbt-jaffle`) and +two real projects in a fresh container: + +- `dbt-shop` (sources, staging views, mart, keyed + watermark incrementals, + folder config, schema tests, `var()`): translates 100% CLEAN, runs + end-to-end on DuckDB with no manual edits — keyed merge updates rows, + window incremental appends only past-watermark rows, third run full SKIP. +- `jaffle_shop_duckdb`: **5/5 models CLEAN**, 3 seeds load, 20 generated + tests pass. +- `dbt-labs/canvas-exemplar`: **13/14 models CLEAN untouched, 14/14 with a + declared DuckDB profile**; macros classify honestly; exposures/metrics/ + semantic-layer resources UNSUPPORTED by design. +- Dynamic Jinja stays REVIEW/UNSUPPORTED and fails loudly, never + miscompiled — the deliberate limit. + +## Clean install + +```bash +CARGO_INSTALL_ROOT=/tmp/… cargo install --path crates/phlo-transform-cli --locked +``` + +Installs `phlo-transform` 0.1.0. On a fresh directory: +`init` → `check` → `run --adapter duckdb` → `plan` (all-skip second time) → +`doctor` → `explain` all pass. No repository-relative assumptions; the bundled +DuckDB adapter needs no services. The documented Trino/Nessie path is covered +by the live dogfood leg and `golden_path` e2e. + +Independently verified in a completely fresh `rust:1.93-bookworm` container: +`cargo install --path crates/phlo-transform-cli --locked` from scratch +(~5m40s, dominated by the bundled DuckDB C++ compile), then the documented +`init`/`check`/`plan`/`run` path verbatim with no undocumented setup. + +`phlo-transform --version` → `phlo-transform 0.1.0`. The daemon `/v1/status` +payload now also reports `version` alongside `compiler_semantics_version` +(added this pass, asserted in `tests/api.rs`). + +## Packaging + +All manifests carry `name`, `version` (0.1.0), `edition 2021`, `license = +Apache-2.0`, `repository`, `rust-version`, `description`. Internal +dependencies are `path + version = "0.1.0"` — the publishable form. + +`cargo package --allow-dirty` results: + +| Crate | Result | +|---|---| +| `phlo-transform-sql`, `phlo-transform-nessie` | package **and** verify-build cleanly | +| `phlo-transform-core`, `-dbt`, `-engine`, `-openlineage`, `-trino`, `-daemon`, `-duckdb`, `-cli` | **package file lists are clean**, but full `cargo package` cannot resolve until their internal deps exist on crates.io — expected pre-publish | + +Package contents inspected: `src/`, `tests/` (+ insta snapshots), manifests +only — no fixtures, no generated state, no Docker/testcontainers assets, no +oversized files. + +**Publish order** if crates.io release is ever wanted: +`sql` → `core`, `nessie`, `openlineage`, `dbt` → `engine` → +`duckdb`, `trino`, `daemon` → `cli`. No crate is marked `publish = false`; +the v0.1 distribution channel is the GitHub-release binaries built by +`release.yml`, so crates.io staging does not gate this release. + +## Upgrade compatibility + +Verified by hand-building realistic old state and opening it with the +installed binary: + +- **SQLite** — a pre-evidence, pre-`effective_key`, pre-`output_identity`, + pre-v3 database (no `evidence`/`promotions` tables, short column sets, + `compiler_version='2'` record): opens cleanly; all tables created and + columns added **additively**; the legacy row reads via `state model` + (`Adapter: `); `plan` then reports `compiler semantics changed` + and rebuilds once — the documented one-time rebuild wave. +- **Postgres** — a faithful old schema (BIGINT counters, no `evidence` table, + no `adapter`/`output_identity`/`effective_key_json`/`seq`/`plan_json` + columns): `ensure_schema` adds everything non-destructively; + `state runs`/`state model` read pre-existing rows correctly. +- **Artifacts** — `lineage_diff.json`, `environment*.json` and promotion + `record_json` carry `#[serde(default)]` on every field added since they + were first written (`base_kind`, `candidate`, `environment`, `created_at`, + `catalog_status`, `catalog_owned_by_phlo`, `evidence`); old payloads + deserialize. `manifest`/`plan`/`run`/`diff`/`branch_diff` artifacts are + write-only — the tool never reads them back. +- **Intentional incompatibility** — none found beyond the designed ones: + pre-v3 records hash-miss once and rebuild; records without + `adapter`/`output_identity` can never authorise cache reuse (fail closed). + +Edge case noted, not a blocker: pointing `--state postgres://` at a +*foreign* table named `runs` with incompatible column types kills the state +worker thread; the CLI exits `error: state store error: postgres worker +stopped`. Loud and contained — phlo-written schemas always used BIGINT +counters, so this only triggers on databases phlo never created. + +## Security / hygiene + +- No credentials, tokens, `.env` files or private endpoints tracked; CI uses + a throwaway `phlo:phlo` service password only. +- Postgres URLs have credentials stripped before they can reach error text + (`state_location_display`), covered by `unreachable_postgres_state_url_errors`. +- VHS doc-generation tapes hardcoded a developer's home path — **fixed** to a + generic `PATH` + comment. +- `.gitignore` covers `.phlo/`, `target/`, corpus data (285M local corpus is + untracked; only `corpus/packages.lock` is committed, intentionally). +- All 39 fixture dirs are referenced by tests; no stale scripts; README links + all resolve; CI test-target names all exist. +- **`LICENSE` was missing** despite `license = "Apache-2.0"` on every crate + and `release.yml` copying `LICENSE` into release tarballs — **added**, and + the first added copy was itself corrupted (Section 8 spliced warranty text + where the limitation-of-liability terms belong); **replaced verbatim** with + the official ASF `LICENSE-2.0.txt`. +- Stale `cached` "classification only" claims corrected: `docs/roadmap/README.md` + Phase-3 row and the Phase-3 implementation notes updated. Two further stale + audit notes fixed: Phase-5 claimed promotion records were artifact-only + (they persist to the state store's `promotions` table since PR #17) and + Phase-8 claimed the API had no plan endpoint (`/v1/plan` exists). The + README status section also disagreed with the audit table — it called + Phase 1 "done" where the audit records "partial" (no partition-level + scheduler tests); aligned to the audited verdict. +- Intermediate point-in-time review records (convergence review, dbt + migration audit, fresh-container validation, readiness report) were + removed from the tree; their load-bearing evidence is folded into this + document above. + +## Documentation audit + +README covers: what it is, install, DuckDB quickstart, Trino env vars, +selectors, dbt translation, and honest per-phase status with links. +`docs/engine.md` documents `skip`/`cached`/`build` and cache adoption; +`docs/wap.md` environments/refs/promotion; `docs/state.md` versions and +migrations; `docs/daemon.md` the machine API; `docs/dbt-migration*.md` the +translator and its limits; `docs/dbt-compatibility-corpus.md` reproducible +measured coverage. A new user can reach every required concept from the +README's doc links. + +Release notes drafted at [`docs/v0.1-release-notes.md`](v0.1-release-notes.md). + +## CI / release workflow + +`ci.yml` is the release gate: fmt + clippy + `cargo test --workspace` + the +Postgres service tests + all five Docker e2e targets. `release.yml` fires only +on `v*` tags: builds the CLI for linux-x86_64, macOS arm64 and macOS x86_64, +packages tarballs, creates a GitHub release. Nothing auto-publishes crates or +tags anything. + +Two fixes this pass: + +- The x86_64 leg pinned `macos-13`, a runner image GitHub retired in + December 2025 — moved to `macos-15-intel` so the tag build can still + schedule. +- `cp README.md LICENSE "dist/$name/" 2>/dev/null || true` would have let a + release silently ship without its license — now a hard failure if either + file is absent. + +## Known limitations + +- Warm-plan cost is O(models × snapshot read): ~28s at 5,000 models on a + small Nessie/Trino pair (`--since` stays sub-second). +- Postgres state has no TLS — trusted networks only. +- Nessie Iceberg catalogs cannot store views — views must materialise as + tables. +- dbt dynamic Jinja is deliberately limited: macros/snapshots/`run_query` are + `REVIEW`/`UNSUPPORTED`, never executed. +- No cross-adapter/cross-engine cache reuse by design. +- `plan --ref` on an unprovisioned candidate previews an empty target (a + printed note explains `run --ref` provisions and may adopt). +- Daemon reload is a full recompile; responses terser than the CLI. +- DuckDB `source_state` fingerprints schema + row count: in-place updates + that preserve row count are invisible (Iceberg snapshot IDs are the + precise mechanism). +- `inspect`/`explain` without `--adapter` compile offline and can report a + spurious `changed` status; pass `--adapter` for catalogue-enriched state. + +## Release blockers + +None known. (Real findings fixed in this pass: missing `LICENSE`, the +corrupted license text that briefly replaced it, the retired `macos-13` +release runner, the license-copy `|| true`, and the stale `cached`/phase +claims.) + +## Final verdict + +**Ready for v0.1.0** + +## Post-v0.1 priorities + +1. Batched/alternative output-identity reads — the ~28s warm plan at 5k is + the largest recurring cost; every candidate run pays it. +2. Postgres TLS support — required before shared state over untrusted + networks. +3. Provisioned-candidate `plan` parity — show expected adoption without a + preceding `run --ref`. +4. `state`/`state evidence` ergonomics — make the promotion → evidence id → + payload join legible in human output. +5. Transient-resilience budget review at 10k+ models — confirm the + 150/400ms retry delays and `buffered(16)` window still suffice. diff --git a/docs/v0.1-readiness.md b/docs/v0.1-readiness.md deleted file mode 100644 index 3220c5c..0000000 --- a/docs/v0.1-readiness.md +++ /dev/null @@ -1,261 +0,0 @@ -# v0.1 release-candidate readiness - -The question this report answers: can `phlo-transform` be called a v0.1 -release candidate based on what it actually does on real infrastructure — -not on architecture or unit tests alone. - -## Executive verdict - -**Ready for v0.1 RC.** - -Every acceptance gate below is met against live Trino/Iceberg/Nessie -infrastructure, a real Postgres state store, and a 120-model dogfood -workspace exercised end to end — including the last piece that was -classification-only: `cached` now has real execution semantics (verified -adoption, not just a label). The known limitations are documented, bounded, -and fail closed rather than silently wrong. - -## What was validated - -- **Workspace suite.** `cargo test --workspace` — all unit and integration - tests pass (engine 173, CLI 79 incl. golden-path snapshots, core, - daemon API, nessie client, dbt importer). -- **Real Trino + Iceberg + Nessie e2e** (testcontainers, Docker): the - existing WAP promotion flow plus a new candidate cache-reuse test that - proves a candidate branch adopts inherited materialisations with zero - model SQL, preserves source run/timestamp provenance, and falls back to - a real build when a table is externally rewritten. -- **Dogfood script** (`scripts/dogfood.sh`, 120-model generated - workspace): 24/24 checks — `check`, `list`, cold/warm runs, tests, - lineage/impact, `plan --since`, deliberate failure, `--retry-failed`, - `--resume` refusal on a finished run, state inspection — plus the live - leg: `run` on `main`, `ref create`, `run --ref` that asserts every model - result is `cached` (JSON-checked, not just exit code), `diff`, - `promote --check`, `ref delete` with catalog cleanup. -- **Real planning benchmark** at 100 / 1,000 / 5,000 models including - candidate cache-reuse and forced-miss tiers (numbers below). -- **Postgres state store** against real Postgres 16 — 5/5 tests. -- **Daemon** environment isolation (candidate run cannot touch base - tables) and bearer-auth gating (401 without token on every endpoint - except `/status`). -- **Install/upgrade**: `cargo install --path` produces a working binary; - `init`/`check`/`run --adapter duckdb` pass on a clean workspace; a - pre-evidence SQLite `state.db` opens, migrates (missing tables created, - missing columns added), and serves reads without manual surgery; a - Postgres URL's credentials do not appear in connect errors. -- **Other ignored e2e**: `golden_path`, `trino_e2e` ×3 (run+test, keyed - diff, partitioned diff), `trino_incremental_e2e` (MERGE on Iceberg), - `nessie_wap_e2e`, daemon `nessie_e2e` — all pass. - -## Correctness findings - -Bugs found by actual use, each locked in by a regression test: - -- **`Cached` was classification-only.** The planner could say a model was - reusable but the runner never wrote a target-environment record, so the - adoption was neither durable nor honest. Now a verified adoption - persists an environment-local `MaterializedRecord` claiming the source - run id and timestamp verbatim. Tests: `a_cached_run_adopts_the_verified_output_into_the_environment`, - `a_cached_run_falls_back_to_build_when_identity_drifts`, - `a_cached_run_adopts_the_source_environments_watermark` (engine tests); - `candidate_reuses_inherited_materialisations_without_rebuilding` - (Trino/Nessie e2e). -- **Model versions were not portable across environments.** The version - target input included the catalog, so a Nessie candidate compiling - against its own catalog produced different hashes and could never match - a base record. The catalog is an environment binding, not model content: - the target input is now `schema.table|materialization` and - `COMPILER_SEMANTICS_VERSION` is 3. Retarget detection survives through - full target-display comparison (`changed_models` treats a moved target - as changed; the planner emits `target_change`). Tests: - `cache_reuse_matches_across_catalogs_on_the_same_content_slot`, - `cache_reuse_rejects_a_different_content_slot`, - `a_catalog_retarget_adopts_the_visible_output`, - `a_catalog_retarget_without_verifiable_identity_rebuilds`. -- **Composite `@key a,b` generated per-column unique tests** instead of - one `unique(a, b)`. Now `not_null` per column plus a composite unique. - Test: `a_composite_key_asserts_one_composite_unique`. -- **Source references bypassed the environment catalog** — a candidate - read the base branch (or failed) while writing the candidate. The - rewriter now emits `relation_for_source` for unqualified sources under a - default catalog. Tests: `a_candidate_compile_qualifies_source_references`, - `compiled_sql_rewrites_models_and_preserves_sources`. -- **`ref delete` leaked the provisioned Trino catalog.** It now routes - through `cleanup_candidate`; a Phlo-provisioned catalog is dropped, an - unproven one is left and reported. Verified live in the dogfood script. -- **Trino had no `load_csv`** — any project with seeds could not run on - Trino. Implemented with RFC 4180 parsing, type inference, batched - inserts. -- **Transient identity reads masqueraded as "no verifiable output".** - `iceberg_snapshot` swallowed every error as `None`, so a Nessie/Trino - blip demoted an unchanged model to a rebuild — observed at scale in the - benchmark (a 1,000-model candidate run that adopted 999 and rebuilt 1, - and earlier 2-of-5,000 warm-plan misses). Non-missing-relation errors - now retry (150ms, 400ms) before reporting `None`; a real missing - relation still answers immediately, and anything still unreadable fails - closed to a build. -- **Nessie commit hiccups were never retryable.** `EXTERNAL`, - `INSUFFICIENT_RESOURCES` and the Nessie-client signature inside - `GENERIC_INTERNAL_ERROR` are now classified retryable so the runner's - `RetryPolicy` applies. Covered by `phlo-transform-trino` unit tests. -- **A catalog retarget left plans undetectably stale.** With the catalog - out of the version hash, `Plan::staleness` compared only hashes — an - old plan could execute against a compilation bound to another physical - target. It now also requires `compiled.target.display() == model.target`. - Test: `a_plan_is_stale_when_the_compilation_retargets_the_model`. -- **`retry_failed` dismissed now-cacheable failures as "nothing to - retry".** The work check counted only `Build` actions, so a failed model - newly satisfiable by adoption was excluded and its tests dropped. - `Build | Cached` now counts as executable. Test: - `retry_failed_executes_a_now_cacheable_model`. -- **Promotion evidence ids were re-looked-up after the gates ran.** A - concurrent writer landing a newer evidence row between evaluation and - the post-hoc id query could be recorded as the authorising row. The - audit-resolution functions now return the consulted record's id with the - evidence (`AuditEvidence.diff_evidence_id`, `LineageEvidence.evidence_id`, - `read_environment_evidence`), and `PromotionEvaluation` carries those - ids forward — no second read. Tests: - `audit_evidence_ids_name_the_record_the_audit_consulted`, - `a_promotion_records_the_evidence_ids_the_gates_evaluated`. - -## Cache reuse - -`skip` and `cached` now have distinct, user-legible meanings: - -- **`skip`** — this environment's own record vouches for the desired - version *at this target*, and the live identity check confirms the - relation still holds the recorded output. -- **`cached`** — no record in *this* environment fully vouches, but a - matching record elsewhere does, and this environment's target provably - already holds the same physical output. Execution adopts it: no model - SQL runs. -- **`build`** — nothing vouches; the model executes. - -Cross-environment reuse (the Nessie case) requires all of: identical -desired version hash; same content slot `schema.table` (catalogs may -differ — that is exactly branch inheritance); same adapter; a non-empty -strong `output_identity` (Iceberg snapshot id) recorded on the source; -and a live `output_identity` read through *this* environment's binding -matching it at plan time and again at run time. Records from adapters that -cannot prove physical identity (DuckDB, non-Iceberg Trino) never authorise -reuse; legacy records without adapter or identity are inert. - -Run output explains itself — a cached model prints `adopted the output -recorded in — snapshot: verified live`; a stale-evidence -fallback prints `cache evidence stale at run time: now reports -…; building instead`. - -Same-environment reuse covers the retarget case: a same-version record -whose target display moved adopts the verified output into an updated -record rather than rebuilding. - -**Known limitations.** Adoption requires the physical relation to already -be visible through the target environment's binding — true for Nessie -branch inheritance, not for a candidate pointed at an isolated warehouse -(it builds, correctly). There is no cross-*adapter* or cross-*engine* -reuse by design. A `plan --ref` against a never-provisioned candidate is a -read-only preview of an empty target; the CLI now prints a note saying a -`run --ref` provisions first and may adopt. - -## Performance - -Real Trino + Nessie + Iceberg (testcontainers on OrbStack, Apple M2 Pro). -Per-model `output_identity` snapshot reads dominate warm plans and are the -cost of honesty — the drift check is what makes a recorded version -*evidence*. - -| Models | cold plan | materialise | warm plan | changed-subset | `--since` | candidate plan | candidate run | candidate miss (plan / run) | -|---|---|---|---|---|---|---|---|---| -| 100 | 1.2s / 1 call | 10.0s | 1.1s / 101 | 1.1s / 102 | 0.13s / 7 | 1.2s / 101 — 100% cached | 0.9s — 100 cached, 0 built | 1.3s / 1.6s — 95 cached + 5 built | -| 1,000 | 0.6s / 1 call | 132s | 13.8s / 1,001 | 13.5s / 1,002 | 0.4s / 12 | 13.1s / 1,001 — 100% cached | 12.6s — 1,000 cached, 0 built | 13.0s / 13.3s — 990 cached + 10 built | -| 5,000 | 1.8s / 2 calls | 291s | 27.7s / 5,001 | 56.6s / 5,002 | 1.2s / 52 | 80.7s / 5,001 — 100% cached | 57.5s — 5,000 cached, 0 built | 57.9s / 79.2s — 4,950 cached + 50 built | - -(Adapter calls counted on the wrapping `CountingAdapter`; "materialise" is -the run that creates the tables, a prerequisite not a planning measure. -Candidate run = zero model SQL; its adapter calls are the identity -verification reads plus schema provisioning.) - -The honest finding stands: a warm 5k plan is ~28s because 5,000 snapshot -reads run `buffered(16)` against a single small Nessie/Trino pair. The -`--since` path stays sub-second because it only probes the change set. -Candidate adoption is the same shape as a warm plan — the adoption itself -is free; the verification reads are the cost. - -## UX - -Improved this pass: - -- `run` prints cached-adoption reasons per model; the `skip`/`cached`/ - `build` distinction is documented in SPEC §39 and `docs/engine.md`. -- `plan --ref`/`test --ref` on a never-provisioned candidate prints a - note explaining the preview sees an empty target and that `run --ref` - provisions + may adopt — previously it silently reported every model - `missing_relation`. -- `compiler semantics changed` is an explicit reason when a pre-v3 record - meets a v3 workspace (the one-time rebuild wave on upgrade explains - itself). -- The dogfood's candidate run is asserted `cached` in JSON, not just - exit-code green. - -Still awkward (accepted for v0.1): the unprovisioned-candidate preview -cannot show adoption without provisioning (the note is the mitigation); -`--since` depends on a git checkout; the daemon and CLI surface the same -concepts but daemon responses remain more terse. - -## Upgrade/migration - -- **SQLite state**: additive-only migrations (`CREATE TABLE IF NOT - EXISTS`, idempotent `ALTER TABLE ADD COLUMN`, legacy `effective_key` - backfill). Verified by opening a hand-built pre-evidence database — all - missing tables/columns appear and legacy rows read correctly. No manual - surgery. -- **Postgres state**: same additive schema approach; no TLS on the client - (`NoTls`) — documented for trusted networks. -- **Compiler semantics v2 → v3**: old records hash-miss once and rebuild - with an explicit `compiler_semantics_change` reason. Legacy records - lacking `adapter`/`output_identity` can never authorise cache reuse — - fail closed by construction. -- **Promotion records**: `evidence` (branch-diff / lineage-diff / - environment ids) is optional via serde defaults — records written before - evidence tracking deserialize with `None`; `promotions.record_json` - schema is unchanged. -- **Artifacts**: `plan.json`/`run.json`/`manifest.json` schemas are - additive; `Cached` results carry `output_identity` and reasons. - -## Remaining work - -**Release blocker** — none known. - -**Post-v0.1** (documented limitations, real but not blocking): - -- Warm-plan cost is O(models × snapshot read): ~28s at 5,000. First thing - to attack if large workspaces need fast no-op plans — needs a batched - identity read Trino/Nessie cannot serve today, or a documented opt-out - for callers accepting drift risk. -- Postgres state client has no TLS (`NoTls`) — trusted networks only. -- Nessie Iceberg catalogs cannot store views — platform limitation; - Nessie-targeted projects must materialise views as tables. - -**Future capability** (deliberately out of scope): a generic distributed -cache abstraction (this pass solves the Phlo/Iceberg/Nessie case only); -cross-adapter reuse; batched snapshot reads. - -## Top five priorities after v0.1 - -From observed use, not roadmap speculation: - -1. **Batched/alternative output-identity reads** — the 28s warm plan at - 5k is the single largest recurring cost; every candidate run pays it - too. -2. **Postgres TLS support** — the only thing standing between "local/trusted - network" and "shared CI over untrusted networks" deployments. -3. **Provisioned-candidate `plan` parity** — a `plan --ref` that can show - adoption without a preceding `run` would remove the most confusing - output a new user sees. -4. **`state`/`state evidence` ergonomics** — evidence ids on promotion - records are now queryable; making the join obvious in human output - (`promotion → evidence id → payload`) closes the audit loop for - non-JSON readers. -5. **Transient-resilience budget review** — identity-read retries fixed - the observed flake; at 10k+ models, watch whether 150/400ms retry - delays suffice or the concurrency window needs tuning. diff --git a/docs/v0.1-release-notes.md b/docs/v0.1-release-notes.md new file mode 100644 index 0000000..b06c77e --- /dev/null +++ b/docs/v0.1-release-notes.md @@ -0,0 +1,140 @@ +# Phlo Transform v0.1.0 + +## What it is + +Phlo Transform is a workspace-native SQL transformation compiler and +execution engine written in Rust. It parses ordinary SQL, resolves the +workspace into one typed dependency graph without `ref()`, and treats +planning, state, lineage, testing, diffing and audited promotion as +first-class, compiler-driven concepts. It is intentionally not a dbt +compatibility layer — it is a smaller, more predictable SQL build system. + +## Highlights + +- **Plan before apply.** Every `plan` explains *what* will run and *why* — + per-model actions (`build`/`skip`/`cached`) with named reasons, before + anything mutates. +- **Content-addressed state.** Model versions hash semantics, not text: + formatting and comments never trigger rebuilds; upstream changes cascade + correctly downstream. +- **Executable cache reuse.** `cached` is real: a Nessie candidate adopts a + `main` materialisation with zero model SQL, verified live against the + physical table identity (Iceberg snapshot), and falls back to a real + build when evidence is stale. +- **Nessie-native environments and WAP.** `run --ref` provisions an + isolated candidate branch + catalog; `promote` merges only after audit + gates (staleness, branch diff, breaking-schema, lineage diff) pass. +- **Column lineage and contracts.** Canonical lineage graph, column-level + impact analysis, OpenLineage export, schema contracts with + breaking-change gates, declared renames. +- **dbt migration.** `translate --from dbt` lowers dbt projects into native + Phlo workspaces with an honest `CLEAN`/`REVIEW`/`UNSUPPORTED` + classification and a written report — it does not pretend Jinja programs + are SQL. +- **Machine API.** `phlo-transform daemon` exposes the same compiler/engine + truth over versioned local JSON (`/v1/...`) with tracked operations, + idempotency keys and cooperative cancellation. + +## Core workflow + +```bash +phlo-transform init +phlo-transform check +phlo-transform plan --adapter duckdb +phlo-transform run --adapter duckdb # plan + apply + tests +phlo-transform test --adapter duckdb +phlo-transform explain +``` + +`--since ` plans and runs only what changed relative to a ref. +`--resume`/`--retry-failed` continue interrupted runs; `--retries N` retries +retryable adapter failures with bounded backoff. + +## Lakehouse / WAP + +With Trino + Iceberg + Nessie configured (`--trino-endpoint`, +`--nessie-endpoint`, optionally `--nessie-catalog-uri` when the warehouse +reaches Nessie on a different address): + +```bash +phlo-transform run --adapter trino --catalog # materialise on main +phlo-transform ref create ci/pr-1 --from main +phlo-transform run --ref ci/pr-1 # candidate adopts main's + # verified outputs +phlo-transform diff --from ci/pr-1 --to main +phlo-transform promote ci/pr-1 --to main --check +phlo-transform ref delete ci/pr-1 # drops phlo-owned catalog +``` + +## State and caching + +- State store: SQLite (default, `.phlo/transform/state.db`) or Postgres + (`--state postgres://...`). Both migrate additively on open — old + databases need no manual surgery. +- `skip` — this environment's record vouches *and* the live identity check + confirms the relation still holds it. +- `cached` — a matching record elsewhere vouches *and* this environment's + target provably holds the same physical output; adoption runs no model + SQL. +- `build` — nothing vouches; the model executes. +- Records from adapters that cannot prove physical identity (DuckDB, + non-Iceberg Trino) never authorise reuse — fail closed by design. + +## Lineage, contracts, diffs + +- `lineage `, `impact `; `lineage --diff` compares + graphs across git refs. +- Contracts declared in config; breaking changes gate promotion; declared + renames (`renamed_from`) are not breaking. +- `diff` compares candidate vs base data with keyed policies, tolerances, + sampling and Iceberg `$partitions` metadata. + +## dbt migration + +```bash +phlo-transform -r translate --from dbt --check +phlo-transform -r translate --from dbt --out generated/ --verify +``` + +`ref()`/`source()`/`var()`, materialisations, keys, schema tests, tags and +config inheritance are lowered to directives + `phlo.toml`/`transform.toml`. + +## Machine API + +`phlo-transform daemon --port 7070` serves `/v1/...` reads and +`POST /v1/operations` for mutations with idempotency keys and +`/v1/operations/{id}/cancel`. `/v1/status` reports version, semantics +version, counts and capabilities. `--token` gates everything except +liveness behind `Authorization: Bearer`. + +## Known limitations + +These are real and deliberately documented — none are silent: + +- **Warm plans read one Iceberg snapshot per model.** At 5,000 models a + warm plan is ~28s against a single Nessie/Trino pair (`--since` stays + sub-second). This is the cost of an honest drift check. +- **Postgres state has no TLS** (`NoTls`) — use it on trusted networks. +- **Nessie Iceberg catalogs cannot store views** — Nessie-targeted + projects must materialise views as tables. +- **dbt dynamic Jinja is deliberately limited** — static config/`var`/ + `ref`/`source` translate; macros, snapshots and dynamic control flow are + classified `REVIEW`/`UNSUPPORTED` rather than miscompiled. +- **No cross-adapter or cross-engine cache reuse** — reuse is + Phlo/Iceberg/Nessie only, by design. +- `plan --ref` on a never-provisioned candidate previews an empty target; + `run --ref` provisions first and may adopt. +- The daemon reloads by full recompile (no targeted invalidation), and its + responses are more terse than the CLI's. + +## Installation + +```bash +cargo install --path crates/phlo-transform-cli --locked +phlo-transform --version # phlo-transform 0.1.0 +``` + +No external services are required for the DuckDB path — the adapter is +embedded. Prebuilt binaries are attached to the GitHub release for +`x86_64-unknown-linux-gnu`, `aarch64-apple-darwin` and +`x86_64-apple-darwin`. diff --git a/docs/validation.md b/docs/validation.md deleted file mode 100644 index 026dbe6..0000000 --- a/docs/validation.md +++ /dev/null @@ -1,87 +0,0 @@ -# Validation — dbt migration end-to-end - -This documents a usability audit of the migration workflow, run against three -fixture projects: - -| Fixture | Shape | -|---|---| -| `fixtures/dbt-clean` | Minimal project: one source, staging view, table mart, `not_null` test. Translates 100% CLEAN. | -| `fixtures/dbt-shop` | Realistic DuckDB-compatible project: sources, staging views, a table mart, a keyed (`merge`) incremental, a watermark (`append` + `is_incremental()`) incremental, folder-level `+materialized`/`+tags`, schema tests, `var()`. Translates 100% CLEAN. | -| `fixtures/dbt-jaffle` | Jaffle-style project exercising the REVIEW/UNSUPPORTED paths: package macros (`dbt_utils.star`), a project macro, an `ephemeral` model, an `is_incremental()` else-branch, a seed, a snapshot, a singular test, `packages.yml`. | - -## Workflow exercised - -```text -phlo-transform -r translate --from dbt --check -phlo-transform -r translate --from dbt --out --verify -# seed source tables in a DuckDB file -phlo-transform -r doctor / check / list / plan / run / test -phlo-transform -r explain / inspect / lineage / manifest -# mutate sources, re-run, confirm incremental behaviour and no-op third run -``` - -## What worked - -- `translate --check` classification is accurate and honest: macro call sites - and the ephemeral model are REVIEW, the snapshot is UNSUPPORTED, everything - else is CLEAN. Generated SQL is minimal — `{{ config }}` blocks become - directives, `ref`/`source`/`var` resolve to names/literals, and no - `phlo.toml` is emitted when the project needs no defaults. -- `--verify` compiles the generated workspace and reports real diagnostics. -- Residual Jinja in REVIEW models fails loudly at `check` time with a parse - error pointing at the generated file, as designed. -- `dbt-shop` translated, compiled, planned, ran and tested on DuckDB with no - manual edits. The keyed incremental merged an updated row; the - time-window incremental appended only rows past its watermark; a third run - with no upstream change was a full SKIP. -- `doctor` reports workspace/compile/adapter/state/nessie health clearly and exits - non-zero on problems. -- Generated tests (`unique`, `not_null`, `accepted_values`, key-folded - assertions) all executed and passed. - -## Problems found and fixed - -1. **`translate --verify` exited 0 on a failing generated workspace.** The - command printed `check failed` with diagnostics but always returned - success, so CI could not detect REVIEW models that do not compile. It now - exits non-zero when the verify report is not ok. Files are still written — - REVIEW output is intentionally inspectable. -2. **`inspect` always reported `status: changed`.** The desired version in the - report is the *short* hash while the recorded materialised version was - compared by *full* hash, so even a just-built model showed `changed`. The - comparison now uses the short hash on both sides. -3. **DuckDB `source_state` only fingerprinted the schema.** Appended source - rows were invisible to the planner, so incremental models — the main reason - to run incrementally — were always SKIP on DuckDB. The fingerprint now - includes `count(*)`, which is metadata-cheap on DuckDB and catches the - dominant append case. -4. **DuckDB timestamp/time values were rendered as Rust debug strings.** - `max()` returned e.g. `Microsecond 1704272400000000`, which broke - the time-window watermark predicate (`CAST('Microsecond ...' AS timestamp)` - is invalid SQL) and corrupted any test/diff output containing timestamps. - `ValueRef::Timestamp`/`Time64` now render as `YYYY-MM-DD HH:MM:SS.ffffff` - / `HH:MM:SS.ffffff` literals. - -## Coverage added - -- `fixtures/dbt-shop` — the third fixture above. -- `translate_verify_failure_exits_nonzero` — regression test for fix 1. -- `translated_dbt_project_runs_on_duckdb` — full lifecycle e2e: translate, - `--verify`, seed DuckDB, `run`, mutate sources, `run` again (asserting - merge/window row counts), no-change `plan` is SKIP, `inspect` reports - `unchanged`. - -## Remaining limitations - -- REVIEW models with residual Jinja fail `check` with a generic parse error; - the diagnostic does not mention that the `{{ ... }}` is untranslated Jinja. -- DuckDB `source_state` (schema + row count) still cannot see in-place updates - that preserve row count; Iceberg snapshot ids remain the only precise source - state. -- `inspect`/`explain` computed offline (no `--adapter`) cannot observe source - state, so a model reading external sources derives a different desired - version than the enriched one `run` uses; pass `--adapter` for accurate - state. (The catalog no longer contributes — it is an environment binding, - not a version input.) -- `dbt_utils.star`-style macro call sites are preserved verbatim and flagged - REVIEW — they require manual rewrite, as documented. diff --git a/docs/vhs/incremental.tape b/docs/vhs/incremental.tape index 292f865..2bcd57c 100644 --- a/docs/vhs/incremental.tape +++ b/docs/vhs/incremental.tape @@ -8,7 +8,7 @@ Set Padding 16 Set TypingSpeed 40ms Set Theme "Catppuccin Mocha" -Env PATH "/Users/garethprice/Developer/phlo-transform/target/debug:/usr/bin:/bin:/opt/homebrew/bin" +# Requires `phlo-transform` on PATH (e.g. cargo install --path crates/phlo-transform-cli) Env SHELL "/bin/zsh" Type "cd /tmp/phlo-demo/generated" diff --git a/docs/vhs/run.tape b/docs/vhs/run.tape index a4b4f80..5651c1f 100644 --- a/docs/vhs/run.tape +++ b/docs/vhs/run.tape @@ -8,7 +8,7 @@ Set Padding 16 Set TypingSpeed 40ms Set Theme "Catppuccin Mocha" -Env PATH "/Users/garethprice/Developer/phlo-transform/target/debug:/usr/bin:/bin:/opt/homebrew/bin" +# Requires `phlo-transform` on PATH (e.g. cargo install --path crates/phlo-transform-cli) Env SHELL "/bin/zsh" Type "cd /tmp/phlo-demo/generated" diff --git a/docs/vhs/translate.tape b/docs/vhs/translate.tape index 0b347f7..0bc25ba 100644 --- a/docs/vhs/translate.tape +++ b/docs/vhs/translate.tape @@ -8,7 +8,7 @@ Set Padding 16 Set TypingSpeed 40ms Set Theme "Catppuccin Mocha" -Env PATH "/Users/garethprice/Developer/phlo-transform/target/debug:/usr/bin:/bin:/opt/homebrew/bin" +# Requires `phlo-transform` on PATH (e.g. cargo install --path crates/phlo-transform-cli) Env SHELL "/bin/zsh" Type "cd /tmp/phlo-demo/dbt-shop"