diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md new file mode 100644 index 00000000..8306da31 --- /dev/null +++ b/FINAL_REPORT.md @@ -0,0 +1,163 @@ +# Final report: chronicle#179 passive pass-through anchors + +## Result + +Implemented and validated the TY2023 passive pass-through administrative +anchors requested in chronicle#179. The work adds 20 Form 8960 facts and +extends the existing Publication 1304 Table 1.4 record set with 160 partnership +and S corporation facts, for 180 net-new TY2023 facts. Every affected record set +and fact has `provenance_class: administrative`. + +No network access, fetch, push, or pull request was used. + +## Record sets and fact counts + +Form 8960 package ID: `soi-form-8960-2023`. + +Each Form 8960 record set has one all-filers row and two facts: a return count +and a USD amount. The ten record sets therefore emit 20 facts: + +- `irs_soi.ty2023.form_8960.line_1` +- `irs_soi.ty2023.form_8960.line_2` +- `irs_soi.ty2023.form_8960.line_3` +- `irs_soi.ty2023.form_8960.line_4a` +- `irs_soi.ty2023.form_8960.line_4b` +- `irs_soi.ty2023.form_8960.line_4c` +- `irs_soi.ty2023.form_8960.line_5d` +- `irs_soi.ty2023.form_8960.line_8` +- `irs_soi.ty2023.form_8960.line_12` +- `irs_soi.ty2023.form_8960.line_17` + +Publication 1304 package ID: `soi-table-1-4`. + +- Existing record set extended: `irs_soi.ty2023.table_1_4`. +- TY2023 total: 20 AGI rows × 37 measures = 740 facts. +- Net-new partnership and S corporation slice: 20 rows × 8 measures = 160 + facts. +- Complete Schedule E entity slice, including the four existing estate/trust + measures: 20 rows × 12 measures = 240 facts. +- TY2020 remains at its original 29 measures and 580 facts because that + publication combines partnership and S corporation values. The new separate + measures are source-accurately scoped to TY2021–TY2023. + +Amounts are normalized from published thousands of dollars to `unit: usd` +with `value_scale: 1000`; return counts use `unit: count`. Table 1.4 net-loss +amounts remain the publisher's positive magnitudes. + +## Source identity and scope notes + +Form 8960 is pinned to: + +- URL: `https://www.irs.gov/pub/irs-pdf/p4801.pdf` +- Vintage: `tax_year_2023_rev_6_2026` +- Publication identity: Publication 4801 (Rev. 6-2026), pages 214–215 +- Curated source artifact SHA-256: + `a831947133806455b971560102bf6d7710865ece481616844bead7c9b81958fd` +- Supervisor-verified extract SHA-256: + `1cafe8a381e83c5365b6a2ef54695ac6b4beca369eb61338169e24f2581ef080` +- Full Publication 4801 transcript SHA-256: + `0c157394542a0b3e2cbfe361d6ac4c744568c6476bada9d7a7647470b54a9364` + +Table 1.4 is pinned to: + +- URL: `https://www.irs.gov/pub/irs-soi/23in14ar.xls` +- SHA-256: + `b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af` +- Size: 115,712 bytes + +The source-package description and manifest document the identified data +boundary: IRS SOI entity-side partnership tables 23pa01, 23pa04, 23pa06, +23pa10, and 23pa23 provide no passive/nonpassive split by industry, while Table +1.4 provides entity income and loss without a passive split. Form 8960 line 4 +is therefore the sole administrative anchor for the passive pass-through NIIT +base. + +## Form 8960 line 4 reconciliation + +The publisher's displayed amounts in thousands do not satisfy literal equality: + +- Line 4a: `$1,185,607,258,000` +- Line 4b: `-$1,076,350,273,000` +- Their sum: `$109,256,985,000` +- Published line 4c: `$109,256,984,000` +- Residual: `$1,000`, exactly one published reporting unit + +The ingestion preserves all publisher values. The unit-level consistency test +asserts the exact `$1,000` residual and reconciliation within one reporting +unit; forcing exact equality would falsify one of the supplied source values. + +## Microcosm concept IDs + +The primary passive-plus-rental NIIT base is: + +- `irs_soi.form_8960.line_4c` + +Its associated line 4 inputs and counts are: + +- `irs_soi.form_8960.line_4a` +- `irs_soi.form_8960.line_4a.return_count` +- `irs_soi.form_8960.line_4b` +- `irs_soi.form_8960.line_4b.return_count` +- `irs_soi.form_8960.line_4c.return_count` + +The other Form 8960 concepts are: + +- `irs_soi.form_8960.line_1` +- `irs_soi.form_8960.line_1.return_count` +- `irs_soi.form_8960.line_2` +- `irs_soi.form_8960.line_2.return_count` +- `irs_soi.form_8960.line_3` +- `irs_soi.form_8960.line_3.return_count` +- `irs_soi.form_8960.line_5d` +- `irs_soi.form_8960.line_5d.return_count` +- `irs_soi.form_8960.line_8` +- `irs_soi.form_8960.line_8.return_count` +- `irs_soi.form_8960.line_12` +- `irs_soi.form_8960.line_12.return_count` +- `irs_soi.form_8960.line_17` +- `irs_soi.form_8960.line_17.return_count` + +The Table 1.4 entity concepts are: + +- `irs_soi.partnership_net_income` +- `irs_soi.returns_with_partnership_net_income` +- `irs_soi.partnership_net_loss` +- `irs_soi.returns_with_partnership_net_loss` +- `irs_soi.s_corporation_net_income` +- `irs_soi.returns_with_s_corporation_net_income` +- `irs_soi.s_corporation_net_loss` +- `irs_soi.returns_with_s_corporation_net_loss` +- `irs_soi.estate_and_trust_net_income` +- `irs_soi.returns_with_estate_and_trust_net_income` +- `irs_soi.estate_and_trust_net_loss` +- `irs_soi.returns_with_estate_and_trust_net_loss` + +## Validation results + +- `ruff check chronicle policyengine_chronicle db scripts tests`: passed. +- Governance, boundary, facts-only, consumer-contract, alias-drift, and Form + 8960 focused tests: 67 passed. +- Full `pytest -q`: 650 passed, 1 skipped, 0 failed in 27m22s. The skip and 14 + dependency warnings are pre-existing/non-blocking. +- `validate-package soi-form-8960-2023 --year 2023`: valid; 10 record sets, 20 + measures, 20 source records, 10 regions. +- `build-suite soi-form-8960-2023 --year 2023`: valid; 20 facts and consumer + facts, 66 source cells, 10 source rows, 10 regions, lineage coverage 1.0, no + acceptance errors. +- `validate-package soi-table-1-4 --year 2023`: valid; 1 record set, 37 + measures, 20 rows, 740 source records, 1 region. +- `build-suite soi-table-1-4 --year 2023`: valid; 740 facts and consumer facts, + 8,109 source cells, 1 region, lineage coverage 1.0, no acceptance errors. +- Direct Table 1.4 fact builds for TY2020–TY2023: valid; 580, 740, 740, and 740 + facts respectively. +- Canonical merged-bundle test: passed; 145,402 facts across 121 included source + packages, including 33,957 IRS SOI facts. + +The repository has no Makefile. All checks used its Python harness and pytest +suite through the existing offline virtual environment. + +## Changelog + +The functional change fragment is `changelog.d/179.added.md`, following the +parent PolicyEngine repository convention. `CHANGELOG.md` was not edited +manually. diff --git a/PROGRESS.md b/PROGRESS.md new file mode 100644 index 00000000..478dc907 --- /dev/null +++ b/PROGRESS.md @@ -0,0 +1,51 @@ +# Progress: chronicle#179 passive pass-through anchors + +## State + +Complete. Both source ingestions, repository-wide snapshots, changelog, and +final validation are finished on branch `passive-pass-through-anchors` in +`.claude/worktrees/passive-179`. + +## Done + +- Read the applicable repository instructions in `AGENTS.md` and the parent + `CLAUDE.md`. +- Confirmed that the existing `irs_soi.table_1_4` package already pins the + supplied TY2023 workbook, so this task will extend that package rather than + duplicate it. +- Mapped record-set, fact, provenance, source-identity, test, and changelog + conventions; the repository has no Makefile or established changelog file. +- Added `soi-form-8960-2023`: 10 tax-year record sets and 20 administrative + facts (return count plus USD amount for each requested line), pinned to + Publication 4801 (Rev. 6-2026) and its supervisor-verified transcripts. +- Added source-level tests for every Form 8960 value, unit, provenance pin, and + the published one-$1,000 line 4 rounding residual. +- Passed the focused tests (10 tests), source-package validation, and the full + Form 8960 source-suite build (20 consumer facts; all reports valid). +- Extended the existing `irs_soi.ty2023.table_1_4` record set with partnership + and S-corporation net-income/net-loss return counts and amounts across all 20 + AGI classes (160 new facts); the existing estate/trust measures remain intact. +- Added strict per-measure year scoping so TY2020 retains its combined + partnership/S-corporation layout while TY2021 through TY2023 ingest the + separately published lines. +- Passed Table 1.4 source-package compilation for TY2020 and TY2023, focused + package/suite tests, Ruff checks, and full fact builds for TY2020 through + TY2023 (580 facts in 2020; 740 facts in each later year). +- Ran the full 651-test suite once: 649 passed, one skipped, and the sole + failure was the expected stale merged-bundle count snapshot. Updated that + snapshot from the generated bundle's exact coverage report (+180 facts and + +1 source package). +- Rebuilt the canonical merged bundle after the snapshot update; its complete + aggregate-contract test passed (1 test in 12m31s). +- Added the required functional-change fragment at + `changelog.d/179.added.md`, following the parent repository convention. +- Passed the final repository-wide checks: Ruff; 67 targeted governance, + boundary, facts-only, consumer-contract, alias, and Form tests; both package + validators and source-suite builds; and the full suite with 650 passed, one + skipped, and no failures. +- Wrote the requested handoff, record-set/fact inventory, exact Microcosm + concept IDs, and check results to `FINAL_REPORT.md`. + +## Next + +- No implementation work remains. The branch is ready for supervisor review. diff --git a/changelog.d/179.added.md b/changelog.d/179.added.md new file mode 100644 index 00000000..a200dd5d --- /dev/null +++ b/changelog.d/179.added.md @@ -0,0 +1 @@ +Added TY2023 IRS SOI Form 8960 passive pass-through NIIT anchors and Publication 1304 Table 1.4 partnership and S corporation Schedule E facts by AGI class. diff --git a/chronicle/source_package.py b/chronicle/source_package.py index f875d77b..0914a00f 100644 --- a/chronicle/source_package.py +++ b/chronicle/source_package.py @@ -219,6 +219,7 @@ "soi-filing-season-week47-2024-eitc-total": Path( "irs_soi/filing_season_week47_2024" ), + "soi-form-8960-2023": Path("irs_soi/form_8960_2023"), "soi-table-4-3": Path("irs_soi/table_4_3"), "soi-state-2022": Path("irs_soi/state_2022"), "soi-congressional-district-2022": Path("irs_soi/congressional_district_2022"), @@ -908,6 +909,7 @@ def to_record_set_spec(self, year: int) -> SourceRecordSetSpec: measures = tuple( _measure_from_mapping(measure, year=year) for measure in _required(self.payload, "measures", "record_set") + if _mapping_applies_to_year(measure, year=year, context="measure") ) return SourceRecordSetSpec( record_set_id=_render_required_string( @@ -2114,6 +2116,35 @@ def _required(payload: dict[str, Any], key: str, context: str) -> Any: return payload[key] +def _mapping_applies_to_year( + payload: dict[str, Any], + *, + year: int, + context: str, +) -> bool: + """Return whether a declarative mapping applies to the requested year.""" + years = payload.get("years") + if years is None: + return True + identifier = payload.get(f"{context}_id") + context_label = f"{context} {identifier!r}" if identifier else context + if not isinstance(years, list) or not years: + raise ValueError(f"{context_label} years must be a non-empty list") + if any(isinstance(item, bool) or not isinstance(item, int) for item in years): + raise ValueError(f"{context_label} years entries must be integers") + if len(set(years)) != len(years): + raise ValueError(f"{context_label} years entries must be unique") + if "column_by_year" in payload: + columns = payload["column_by_year"] + for declared_year in years: + if declared_year not in columns and str(declared_year) not in columns: + raise ValueError( + f"{context_label} column_by_year has no entry for " + f"declared year {declared_year}" + ) + return year in years + + def _year_mapping(files_by_year: dict[Any, Any], year: int) -> dict[str, str]: if year in files_by_year: return files_by_year[year] diff --git a/db/data/irs_soi/form_8960_2023/irs_soi_form_8960_2023.csv b/db/data/irs_soi/form_8960_2023/irs_soi_form_8960_2023.csv new file mode 100644 index 00000000..e7d07eb1 --- /dev/null +++ b/db/data/irs_soi/form_8960_2023/irs_soi_form_8960_2023.csv @@ -0,0 +1,11 @@ +form_line,label,return_count,amount_thousands,count_page,amount_page +line_1,Taxable interest,8200131,209033554,214,215 +line_2,Ordinary dividends,6877553,349823646,214,215 +line_3,Annuities,176061,6970315,214,215 +line_4a,"Rental real estate, royalties, partnerships, S corporations, trusts, trades or businesses, etc.",4988033,1185607258,214,215 +line_4b,Adjustment for net income or loss derived in the ordinary course of a non-section 1411 trade or business,4038235,-1076350273,214,215 +line_4c,Combine lines 4a and 4b,2260296,109256984,214,215 +line_5d,Combine lines 5a through 5c,6432002,556322852,214,215 +line_8,Total investment income,8976709,1222447069,214,215 +line_12,Net investment income,8211516,1197238417,214,215 +line_17,Net investment income tax for individuals,8104840,39330017,214,215 diff --git a/db/data/irs_soi/form_8960_2023/manifest.yaml b/db/data/irs_soi/form_8960_2023/manifest.yaml new file mode 100644 index 00000000..07007337 --- /dev/null +++ b/db/data/irs_soi/form_8960_2023/manifest.yaml @@ -0,0 +1,42 @@ +source_id: irs_soi +package_id: soi-form-8960-2023 +dataset: irs_soi_publication_4801_form_8960_2023 +source_page: https://www.irs.gov/pub/irs-pdf/p4801.pdf +publication_revision: Publication 4801 (Rev. 6-2026) +table: Form 8960, Net Investment Income Tax—Individuals, Estates, and Trusts, + pages 214–215 +files: + 2023: + filename: irs_soi_form_8960_2023.csv + source_url: https://www.irs.gov/pub/irs-pdf/p4801.pdf + sha256: a831947133806455b971560102bf6d7710865ece481616844bead7c9b81958fd + size_bytes: 795 + source_table: >- + Publication 4801 (Rev. 6-2026), TY2023 Form 8960 facsimile, number of + returns and amounts of selected lines + year: 2023 + notes: >- + Curated CSV transcription of the ten requested Form 8960 lines from the + supervisor-verified p4801-form8960-extract.txt (SHA-256 + 1cafe8a381e83c5365b6a2ef54695ac6b4beca369eb61338169e24f2581ef080), + checked against the full Publication 4801 text extraction p4801.txt + (SHA-256 + 0c157394542a0b3e2cbfe361d6ac4c744568c6476bada9d7a7647470b54a9364). + The canonical publisher artifact is Publication 4801 (Rev. 6-2026) at + source_url. Page 214 reports return counts; page 215 reports amounts in + thousands of dollars. The CSV keeps the published integers and signs + verbatim. SOI's entity-side partnership tables 23pa01, 23pa04, 23pa06, + 23pa10, and 23pa23 publish no passive/nonpassive split by industry, while + Publication 1304 Table 1.4 reports partnership, S-corporation, and + estate/trust income and loss without a passive split. Form 8960 line 4 is + therefore the sole administrative anchor for the passive pass-through + NIIT base. The three independently reported line 4 values have a + one-thousand-dollar rounding residual: line 4a plus line 4b is + 109,256,985 thousand dollars, while published line 4c is 109,256,984 + thousand dollars. License: US government work, public domain. + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/irs_soi/soi-form-8960-2023/2023/a831947133806455b971560102bf6d7710865ece481616844bead7c9b81958fd/irs_soi_form_8960_2023.csv + uri: r2://ledger-raw/raw/irs_soi/soi-form-8960-2023/2023/a831947133806455b971560102bf6d7710865ece481616844bead7c9b81958fd/irs_soi_form_8960_2023.csv diff --git a/db/data/irs_soi/table_1_4/manifest.yaml b/db/data/irs_soi/table_1_4/manifest.yaml index 64e243d7..12a2c022 100644 --- a/db/data/irs_soi/table_1_4/manifest.yaml +++ b/db/data/irs_soi/table_1_4/manifest.yaml @@ -3,6 +3,14 @@ package_id: soi-table-1-4 dataset: irs_soi_publication_1304_table_1_4 source_page: https://www.irs.gov/statistics/soi-tax-stats-individual-income-tax-returns-complete-report-publication-1304-basic-tables-part-1 table: Table 1.4 +notes: >- + Table 1.4 publishes partnership, S corporation, and estate/trust net-income + and net-loss return counts and amounts by AGI class, but it does not identify + the passive share. IRS SOI entity-side partnership tables 23pa01, 23pa04, + 23pa06, 23pa10, and 23pa23 likewise publish no passive/nonpassive split by + industry. Publication 4801 Form 8960 line 4 is therefore the sole + administrative anchor for the passive pass-through NIIT base. Table 1.4 loss + amounts are reported and ingested as positive magnitudes. files: 2020: filename: 20in14ar.xls diff --git a/packages/irs_soi/form_8960_2023/source_package.yaml b/packages/irs_soi/form_8960_2023/source_package.yaml new file mode 100644 index 00000000..a2426166 --- /dev/null +++ b/packages/irs_soi/form_8960_2023/source_package.yaml @@ -0,0 +1,596 @@ +# SOI's entity-side partnership tables (23pa01/04/06/10/23) have no +# passive/nonpassive industry split, and Publication 1304 Table 1.4 reports +# entity income/loss without a passive split. Form 8960 line 4 is therefore the +# sole administrative anchor for the passive pass-through NIIT base. +schema_version: ledger.source_package.v1 +package_id: soi-form-8960-2023 +label: IRS SOI Publication 4801 TY2023 Form 8960 line-item estimates +artifact: + source_name: irs_soi + source_table: Publication 4801 Form 8960, pages 214–215 + resource_package: db + resource_directory: data/irs_soi/form_8960_2023 + manifest: manifest.yaml + vintage: tax_year_2023_rev_6_2026 + extracted_at: "2026-08-19" + extraction_method: >- + manual transcription from the supervisor-verified Publication 4801 + (Rev. 6-2026) Form 8960 text extraction into a curated CSV; page 214 return + counts and page 215 amounts in thousands of dollars kept verbatim + parser: delimited_text_full_rows + sheet_name: form_8960_selected_lines + artifact_year: 2023 +record_sets: + - record_set_id: irs_soi.ty2023.form_8960.line_1 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_1.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_1 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 1 + ordinal: 0 + row_number: 2 + expected_row_header_column: A + expected_row_header: line_1 + guard_cells: + - column: B + expected_value: Taxable interest + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 1 taxable interest + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_1.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 1 taxable interest + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_1 + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_2 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_2.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_2 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 2 + ordinal: 0 + row_number: 3 + expected_row_header_column: A + expected_row_header: line_2 + guard_cells: + - column: B + expected_value: Ordinary dividends + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 2 ordinary dividends + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_2.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 2 ordinary dividends + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_2 + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_3 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_3.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_3 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 3 + ordinal: 0 + row_number: 4 + expected_row_header_column: A + expected_row_header: line_3 + guard_cells: + - column: B + expected_value: Annuities + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 3 annuities + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_3.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 3 annuities + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_3 + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_4a + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_4a.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_4a + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 4a + ordinal: 0 + row_number: 5 + expected_row_header_column: A + expected_row_header: line_4a + guard_cells: + - column: B + expected_value: >- + Rental real estate, royalties, partnerships, S corporations, + trusts, trades or businesses, etc. + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 4a + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_4a.return_count + unit: count + aggregation: sum + - measure_id: amount + label: >- + Form 8960 line 4a rental, royalty, pass-through, trust, and business + income or loss before the non-section 1411 adjustment + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_4a + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_4b + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_4b.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_4b + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 4b + ordinal: 0 + row_number: 6 + expected_row_header_column: A + expected_row_header: line_4b + guard_cells: + - column: B + expected_value: >- + Adjustment for net income or loss derived in the ordinary course + of a non-section 1411 trade or business + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 4b + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_4b.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 4b non-section 1411 trade or business adjustment + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_4b + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_4c + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_4c.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_4c + description: >- + Form 8960 line 4c is the administrative NIIT passive-plus-rental base + after the line 4b non-section 1411 adjustment. SOI's entity-side + partnership tables 23pa01, 23pa04, 23pa06, 23pa10, and 23pa23 publish no + passive/nonpassive split by industry, and Publication 1304 Table 1.4 + reports entity income and loss without a passive split. Form 8960 line 4 + is therefore the sole administrative anchor for the passive pass-through + NIIT base. + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 4c + ordinal: 0 + row_number: 7 + expected_row_header_column: A + expected_row_header: line_4c + guard_cells: + - column: B + expected_value: Combine lines 4a and 4b + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting the Form 8960 line 4c NIIT passive-plus-rental base + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_4c.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 4c NIIT passive-plus-rental base + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_4c + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_5d + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_5d.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_5d + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 5d + ordinal: 0 + row_number: 8 + expected_row_header_column: A + expected_row_header: line_5d + guard_cells: + - column: B + expected_value: Combine lines 5a through 5c + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 5d + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_5d.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 5d net property disposition gain or loss + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_5d + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_8 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_8.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_8 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 8 + ordinal: 0 + row_number: 9 + expected_row_header_column: A + expected_row_header: line_8 + guard_cells: + - column: B + expected_value: Total investment income + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 8 total investment income + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_8.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 8 total investment income + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_8 + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_12 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_12.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_12 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 12 + ordinal: 0 + row_number: 10 + expected_row_header_column: A + expected_row_header: line_12 + guard_cells: + - column: B + expected_value: Net investment income + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 12 net investment income + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_12.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 12 net investment income + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_12 + unit: usd + aggregation: sum + value_scale: 1000 + + - record_set_id: irs_soi.ty2023.form_8960.line_17 + provenance_class: administrative + record_set_spec_id: irs_soi.form_8960.line_17.v1 + source_record_id_prefix: irs_soi.ty2023.form_8960.line_17 + sheet_name: form_8960_selected_lines + period_type: tax_year + period: 2023 + geography_id: 0100000US + geography_level: country + geography_name: United States + geography_vintage: 2020_census + entity: tax_unit + entity_role: filing_unit + domain: form_8960_filers + groupby_dimension: irs_soi.form_8960.line + rows: + - value_id: all + label: All Form 8960 filings reporting line 17 + ordinal: 0 + row_number: 11 + expected_row_header_column: A + expected_row_header: line_17 + guard_cells: + - column: B + expected_value: Net investment income tax for individuals + label: published line label + - column: E + expected_value: 214 + label: return-count page + - column: F + expected_value: 215 + label: amount page + table_record_kind: total + measures: + - measure_id: return_count + label: Returns reporting Form 8960 line 17 individual NIIT + ordinal: 0 + column: C + source_column_id: return_count + expected_column_header_row: 1 + expected_column_header: return_count + concept: irs_soi.form_8960.line_17.return_count + unit: count + aggregation: sum + - measure_id: amount + label: Form 8960 line 17 net investment income tax for individuals + ordinal: 1 + column: D + source_column_id: amount_thousands + expected_column_header_row: 1 + expected_column_header: amount_thousands + concept: irs_soi.form_8960.line_17 + unit: usd + aggregation: sum + value_scale: 1000 diff --git a/packages/irs_soi/table_1_4/source_package.yaml b/packages/irs_soi/table_1_4/source_package.yaml index 5b1ff8cd..9d3fceaa 100644 --- a/packages/irs_soi/table_1_4/source_package.yaml +++ b/packages/irs_soi/table_1_4/source_package.yaml @@ -1,6 +1,6 @@ schema_version: ledger.source_package.v1 package_id: soi-table-1-4 -label: IRS SOI Publication 1304 Table 1.4 total wages +label: IRS SOI Publication 1304 Table 1.4 income items by AGI class artifact: source_name: irs_soi source_table: Publication 1304 Table 1.4 @@ -853,3 +853,146 @@ record_sets: (PolicyEngine/populace#451 item 2). legal_vintage: tax_year_{year} value_scale: 1000 + # Table 1.4 publishes separate partnership and S corporation legs from + # TY2021 onward. TY2020 combines them, so these measures are deliberately + # scoped away from 2020 rather than duplicating the combined values. + - measure_id: partnership_net_income_returns + label: Returns with partnership net income + ordinal: 29 + years: [2021, 2022, 2023] + column: BP + column_by_year: + 2021: BD + 2022: BP + 2023: BP + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 55 + 2022: 67 + 2023: 67 + concept: irs_soi.returns_with_partnership_net_income + unit: count + aggregation: sum + - measure_id: partnership_net_income_amount + label: Partnership net income + ordinal: 30 + years: [2021, 2022, 2023] + column: BQ + column_by_year: + 2021: BE + 2022: BQ + 2023: BQ + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 56 + 2022: 68 + 2023: 68 + concept: irs_soi.partnership_net_income + unit: usd + aggregation: sum + value_scale: 1000 + - measure_id: partnership_net_loss_returns + label: Returns with partnership net loss + ordinal: 31 + years: [2021, 2022, 2023] + column: BR + column_by_year: + 2021: BF + 2022: BR + 2023: BR + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 57 + 2022: 69 + 2023: 69 + concept: irs_soi.returns_with_partnership_net_loss + unit: count + aggregation: sum + - measure_id: partnership_net_loss_amount + label: Partnership net loss (positive magnitude) + ordinal: 32 + years: [2021, 2022, 2023] + column: BS + column_by_year: + 2021: BG + 2022: BS + 2023: BS + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 58 + 2022: 70 + 2023: 70 + concept: irs_soi.partnership_net_loss + unit: usd + aggregation: sum + value_scale: 1000 + - measure_id: s_corporation_net_income_returns + label: Returns with S corporation net income + ordinal: 33 + years: [2021, 2022, 2023] + column: BT + column_by_year: + 2021: BH + 2022: BT + 2023: BT + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 59 + 2022: 71 + 2023: 71 + concept: irs_soi.returns_with_s_corporation_net_income + unit: count + aggregation: sum + - measure_id: s_corporation_net_income_amount + label: S corporation net income + ordinal: 34 + years: [2021, 2022, 2023] + column: BU + column_by_year: + 2021: BI + 2022: BU + 2023: BU + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 60 + 2022: 72 + 2023: 72 + concept: irs_soi.s_corporation_net_income + unit: usd + aggregation: sum + value_scale: 1000 + - measure_id: s_corporation_net_loss_returns + label: Returns with S corporation net loss + ordinal: 35 + years: [2021, 2022, 2023] + column: BV + column_by_year: + 2021: BJ + 2022: BV + 2023: BV + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 61 + 2022: 73 + 2023: 73 + concept: irs_soi.returns_with_s_corporation_net_loss + unit: count + aggregation: sum + - measure_id: s_corporation_net_loss_amount + label: S corporation net loss (positive magnitude) + ordinal: 36 + years: [2021, 2022, 2023] + column: BW + column_by_year: + 2021: BK + 2022: BW + 2023: BW + expected_column_header_row: 8 + expected_column_header_by_year: + 2021: 62 + 2022: 74 + 2023: 74 + concept: irs_soi.s_corporation_net_loss + unit: usd + aggregation: sum + value_scale: 1000 diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 945bfea5..a1a22dde 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -27,16 +27,16 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "aggregate_duplicate_key_count": 0, "entity_count": 9, "error_count": 0, - "fact_count": 145222, + "fact_count": 145402, "geography_count": 9390, "period_count": 144, "semantic_duplicate_key_count": 12, "skipped_source_count": 10, "source_count": 40, - "source_package_count": 120, + "source_package_count": 121, "warning_count": 1, } - assert len(rows) == 145222 + assert len(rows) == 145402 assert {row["provenance_class"] for row in rows} <= { "administrative", "census", @@ -54,7 +54,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert rows[0]["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert rows[0]["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert source_packages["source_package_count"] == 120 + assert source_packages["source_package_count"] == 121 assert source_packages["skipped_source_count"] == 10 assert sorted(item["source"] for item in source_packages["skipped_sources"]) == [ "census-acs-s0101-congressional-district-age-2024", @@ -68,7 +68,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 145222 + assert coverage["fact_count"] == 145402 assert coverage["counts"]["by_source"] == { "bea": 445, "bfp_economic_outlook": 5, @@ -87,7 +87,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "hhs_acf_tanf": 110, "hmrc": 20533, "ici": 12, - "irs_soi": 33777, + "irs_soi": 33957, "isc": 2, "jrc_euromod_be": 18, "kff": 52, @@ -112,7 +112,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "welshgov": 198, } table_counts = coverage["counts"]["by_source_table"] - assert len(table_counts) == 115 + assert len(table_counts) == 116 assert table_counts["irs_soi:Congressional District Data 2022"] == 26880 assert ( table_counts[ @@ -123,6 +123,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert table_counts["ssa:SSI Monthly Statistics, December 2024, Table 1"] == 4 assert table_counts["irs_soi:Publication 1304 Table 1.1"] == 80 + assert table_counts["irs_soi:Publication 1304 Table 1.4"] == 740 + assert ( + table_counts["irs_soi:Publication 4801 Form 8960, pages 214–215"] == 20 + ) assert ( table_counts[ "irs_soi:Publication 1304 Table 2.5 EITC by AGI and qualifying children" @@ -374,7 +378,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "tax_year:2020": 9, "tax_year:2021": 9, "tax_year:2022": 34951, - "tax_year:2023": 48436, + "tax_year:2023": 48616, "tax_year:2024": 40, } assert coverage["counts"]["by_geography"]["country:BE"] == 31 @@ -382,7 +386,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert coverage["counts"]["by_geography"]["nuts1:BE2"] == 17 assert coverage["counts"]["by_geography"]["nuts1:BE3"] == 6 assert coverage["counts"]["by_geography"]["commune:11002"] == 1 - assert coverage["counts"]["by_geography"]["country:0100000US"] == 2109 + assert coverage["counts"]["by_geography"]["country:0100000US"] == 2289 assert coverage["counts"]["by_geography"]["state:0400000US06"] == 217 assert ( coverage["counts"]["by_geography"]["congressional_district:5001700US0601"] == 56 @@ -399,7 +403,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "institutional_sector": 103, "pension_plan": 2, "person": 56753, - "tax_unit": 33783, + "tax_unit": 33963, } assert not coverage["duplicates"]["aggregate_fact_keys"] assert len(coverage["duplicates"]["semantic_fact_keys"]) == 12 diff --git a/tests/test_chronicle_irs_soi_form_8960.py b/tests/test_chronicle_irs_soi_form_8960.py new file mode 100644 index 00000000..95ad1c2f --- /dev/null +++ b/tests/test_chronicle_irs_soi_form_8960.py @@ -0,0 +1,114 @@ +"""Tests for TY2023 IRS SOI Publication 4801 Form 8960 facts.""" + +from __future__ import annotations + +from chronicle.core import validate_facts +from chronicle.source_package import load_source_package, validate_source_package +from chronicle.sources.cells import validate_source_cells +from chronicle.sources.rows import validate_source_rows + + +EXPECTED_COUNTS = { + "line_1": 8_200_131, + "line_2": 6_877_553, + "line_3": 176_061, + "line_4a": 4_988_033, + "line_4b": 4_038_235, + "line_4c": 2_260_296, + "line_5d": 6_432_002, + "line_8": 8_976_709, + "line_12": 8_211_516, + "line_17": 8_104_840, +} + +EXPECTED_AMOUNTS_USD = { + "line_1": 209_033_554_000, + "line_2": 349_823_646_000, + "line_3": 6_970_315_000, + "line_4a": 1_185_607_258_000, + "line_4b": -1_076_350_273_000, + "line_4c": 109_256_984_000, + "line_5d": 556_322_852_000, + "line_8": 1_222_447_069_000, + "line_12": 1_197_238_417_000, + "line_17": 39_330_017_000, +} + + +def _facts_by_line_and_measure(): + package = load_source_package("soi-form-8960-2023") + source_rows = package.build_source_rows(2023) + cells = package.build_source_cells(2023, source_rows=source_rows) + facts = package.build_facts(2023, cells=cells, source_rows=source_rows) + return ( + package, + source_rows, + cells, + facts, + { + (fact.layout.record_set_id.rsplit(".", 1)[-1], fact.layout.measure_id): fact + for fact in facts + }, + ) + + +def test_form_8960_package_emits_requested_counts_and_amounts(): + package, source_rows, cells, facts, facts_by_line_and_measure = ( + _facts_by_line_and_measure() + ) + + assert package.package_id == "soi-form-8960-2023" + assert len(package.record_sets) == 10 + assert len(source_rows) == 10 + assert len(facts) == 20 + assert validate_source_rows(source_rows).valid + assert validate_source_cells(cells).valid + assert validate_facts(facts).valid + + for line, expected_count in EXPECTED_COUNTS.items(): + count = facts_by_line_and_measure[(line, "return_count")] + amount = facts_by_line_and_measure[(line, "amount")] + assert count.value == expected_count + assert count.measure.unit == "count" + assert count.measure.concept == f"irs_soi.form_8960.{line}.return_count" + assert amount.value == EXPECTED_AMOUNTS_USD[line] + assert amount.measure.unit == "usd" + assert amount.measure.concept == f"irs_soi.form_8960.{line}" + + assert {fact.provenance_class for fact in facts} == {"administrative"} + assert {fact.source.url for fact in facts} == { + "https://www.irs.gov/pub/irs-pdf/p4801.pdf" + } + assert {fact.source.vintage for fact in facts} == {"tax_year_2023_rev_6_2026"} + assert {fact.source.source_sha256 for fact in facts} == { + "a831947133806455b971560102bf6d7710865ece481616844bead7c9b81958fd" + } + assert all(fact.source.raw_r2_uri for fact in facts) + + +def test_form_8960_line_4_arithmetic_is_consistent_to_published_unit(): + _, _, _, _, facts_by_line_and_measure = _facts_by_line_and_measure() + line_4a = facts_by_line_and_measure[("line_4a", "amount")].value + line_4b = facts_by_line_and_measure[("line_4b", "amount")].value + line_4c = facts_by_line_and_measure[("line_4c", "amount")].value + + # Publication 4801 displays independently estimated amounts in $000. The + # reported components therefore leave exactly one published unit of + # rounding residual; changing any fact to force equality would break + # publisher fidelity. + residual = line_4a + line_4b - line_4c + assert residual == 1_000 + assert abs(residual) <= 1_000 + + +def test_form_8960_source_package_validates(): + report = validate_source_package("soi-form-8960-2023", year=2023) + + assert report.valid + assert report.counts == { + "measure_count": 20, + "record_set_count": 10, + "row_count": 10, + "source_record_count": 20, + "source_region_count": 10, + } diff --git a/tests/test_chronicle_soi_targets.py b/tests/test_chronicle_soi_targets.py index b57a808f..b2ecf864 100644 --- a/tests/test_chronicle_soi_targets.py +++ b/tests/test_chronicle_soi_targets.py @@ -127,7 +127,7 @@ def test_build_soi_table_1_4_wage_facts_from_packaged_source(): } assert validate_facts(facts).valid - assert len(facts) == 580 + assert len(facts) == 740 assert ( facts_by_concept_and_range[("irs_soi.returns_with_total_wages", "all")].value == 128_591_050 diff --git a/tests/test_chronicle_source_package.py b/tests/test_chronicle_source_package.py index 3c506362..9df47b0a 100644 --- a/tests/test_chronicle_source_package.py +++ b/tests/test_chronicle_source_package.py @@ -168,6 +168,55 @@ def test_record_set_provenance_fields_propagate_to_specs_and_facts(): assert survey_spec.survey_instrument == "ACS 1-year" +def test_record_set_measure_year_scope_preserves_legacy_layout(): + package_path = REPO_ROOT / "packages" / "irs_soi" / "table_1_4" + package = load_source_package(package_path) + legacy_set = package.build_source_record_set_specs(2020)[0] + current_set = package.build_source_record_set_specs(2023)[0] + legacy_record = package.build_source_record_specs(2020)[0] + + assert len(legacy_set.measures) == 29 + assert len(current_set.measures) == 37 + assert legacy_record.layout is not None + assert legacy_record.layout.record_set_spec_hash == "22ed3e367531f2331f744623" + assert "partnership_net_income_amount" not in { + measure.measure_id for measure in legacy_set.measures + } + assert "partnership_net_income_amount" in { + measure.measure_id for measure in current_set.measures + } + + +@pytest.mark.parametrize( + ("years", "message"), + [ + ([], "non-empty list"), + ("2023", "non-empty list"), + ([2023, "2024"], "must be integers"), + ([True, 2023], "must be integers"), + ([2023, 2023], "must be unique"), + ], +) +def test_record_set_measure_year_scope_rejects_malformed_values(years, message): + source_path = REPO_ROOT / "packages" / "irs_soi" / "table_1_4" + payload = yaml.safe_load((source_path / "source_package.yaml").read_text()) + record_set = payload["record_sets"][0] + record_set["measures"][-1]["years"] = years + + with pytest.raises(ValueError, match=message): + DeclarativeRecordSet(record_set).to_record_set_spec(2020) + + +def test_record_set_measure_year_scope_requires_every_column_mapping(): + source_path = REPO_ROOT / "packages" / "irs_soi" / "table_1_4" + payload = yaml.safe_load((source_path / "source_package.yaml").read_text()) + record_set = payload["record_sets"][0] + record_set["measures"][-1]["years"].append(2024) + + with pytest.raises(ValueError, match="no entry for declared year 2024"): + DeclarativeRecordSet(record_set).to_record_set_spec(2020) + + def test_every_source_package_record_set_declares_provenance_class(): missing: list[str] = [] malformed: list[str] = [] @@ -394,9 +443,14 @@ def test_source_package_path_builds_valid_soi_table_1_4_facts(): assert package.package_id == "soi-table-1-4" assert len(cells) == 8109 - assert len(facts) == 580 + assert len(facts) == 740 assert validate_facts(facts).valid assert facts[0].source.source_table == "Publication 1304 Table 1.4" + assert facts[0].source.url == "https://www.irs.gov/pub/irs-soi/23in14ar.xls" + assert ( + facts[0].source.source_sha256 + == "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af" + ) assert ( values_by_record["irs_soi.ty2023.table_1_4.all.alimony_received_amount"].value == 6_686_429_000 @@ -405,6 +459,54 @@ def test_source_package_path_builds_valid_soi_table_1_4_facts(): values_by_record["irs_soi.ty2023.table_1_4.all.alimony_paid_amount"].value == 7_497_135_000 ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.partnership_net_income_returns" + ].value + == 3_266_345 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.partnership_net_income_amount" + ].value + == 452_077_038_000 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.partnership_net_loss_returns" + ].value + == 2_103_634 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.partnership_net_loss_amount" + ].value + == 168_884_100_000 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.s_corporation_net_income_returns" + ].value + == 4_138_040 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.s_corporation_net_income_amount" + ].value + == 797_450_861_000 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.s_corporation_net_loss_returns" + ].value + == 1_415_974 + ) + assert ( + values_by_record[ + "irs_soi.ty2023.table_1_4.all.s_corporation_net_loss_amount" + ].value + == 89_543_715_000 + ) def test_bea_nipa_total_wages_package_preserves_bea_concept(): diff --git a/tests/test_chronicle_suite.py b/tests/test_chronicle_suite.py index b4b2af3e..a41b1d56 100644 --- a/tests/test_chronicle_suite.py +++ b/tests/test_chronicle_suite.py @@ -138,12 +138,12 @@ def test_build_source_suite_supports_soi_table_1_4(tmp_path): "artifact_count": 1, "agent_acceptance_error_count": 0, "concept_alignment_count": 8, - "constraint_count": 1044, - "consumer_fact_count": 580, - "fact_count": 580, + "constraint_count": 1332, + "consumer_fact_count": 740, + "fact_count": 740, "lineage_coverage": 1.0, "source_cell_count": 8109, - "source_record_count": 580, + "source_record_count": 740, "source_region_count": 1, "source_row_count": 0, }