From 569df8273388595dcc9b92ceb56fcd4d99349ce6 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 15:17:00 -0500 Subject: [PATCH 1/2] feat(nowcast): anchor the computers family's exports on published 2017 (#1270) Census domestic exports put the computers family (3341) at 1.28 times BEA's 2017 exports, and terminals (334118) export more than they produce, so its imports exceed its use. Each 3341 commodity's export cell is now its published 2017 F04000 cell moved by its own Census growth, the #763 construction for imports. Terminals' exports fall to 0.74-0.92 of output in 2017-2023 (were 1.01-1.13 in 2022-2024); 2024 sits at 1.00 because output falls while Census exports grow. Needs a rebuild to reach the tables. Co-Authored-By: Claude Opus 5.5 --- .../test_nowcast_export_anchoring.py | 42 +++++++++ bedrock/transform/iot/nowcast.py | 14 ++- .../transform/iot/nowcast_export_anchoring.py | 86 +++++++++++++++++++ 3 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py create mode 100644 bedrock/transform/iot/nowcast_export_anchoring.py diff --git a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py new file mode 100644 index 000000000..c64073978 --- /dev/null +++ b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py @@ -0,0 +1,42 @@ +"""The computers family's exports sit on the published 2017 cells (#1270).""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from bedrock.transform.iot.nowcast_export_anchoring import anchor_family_exports + +M = 1e6 +IDX = ['334111', '334118', '334119', '336111'] + + +def _anchor(now: list[float], base: list[float], pub: list[float]) -> pd.Series: + return anchor_family_exports( + pd.Series(now, index=IDX) * M, + pd.Series(base, index=IDX) * M, + pd.Series(pub, index=IDX) * M, + ) + + +def test_anchored_rows_are_published_2017_moved_by_own_growth() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * 10.0 / 8.0 * M) + assert out['334118'] == pytest.approx(6.0 * 11.0 / 9.0 * M) + + +def test_2017_reproduces_the_published_cells() -> None: + out = _anchor([8.0, 9.0, 0.0, 25.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * M) + assert out['334118'] == pytest.approx(6.0 * M) + + +def test_rows_outside_the_family_keep_census() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['336111'] == pytest.approx(30.0 * M) + + +def test_census_only_and_published_only_rows() -> None: + out = _anchor([10.0, 4.0, 0.0, 30.0], [8.0, 3.0, 0.0, 25.0], [7.0, 0.0, 2.0, 23.0]) + assert out['334118'] == pytest.approx(4.0 * M) # published 2017 empty: Census + assert out['334119'] == pytest.approx(2.0 * M) # Census 2017 empty: held flat diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index 14889be00..abc5882e7 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -461,7 +461,9 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat F03000 is Inventories_ for every year in ``INVENTORIES_YEARS`` (#529, #746); 2024 is all-zero. F04000 is mapped Trade_Exports_ Detail mass for every year in - ``TRADE_OVERLAY_YEARS``. S00900/F04000 uses the rest-of-world identity + ``TRADE_OVERLAY_YEARS``, with the computers family anchored on the + published 2017 cells (:mod:`~bedrock.transform.iot.nowcast_export_anchoring`, + #1270). S00900/F04000 uses the rest-of-world identity against frozen 2017 Supply T016. The seventeen NIPA-built columns are **conditioned on the published annual @@ -479,12 +481,18 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat y = _nipa_final_demand_block(pd.DataFrame(fbs), int(year)) if year in TRADE_OVERLAY_YEARS: + from bedrock.transform.iot.nowcast_export_anchoring import ( # noqa: PLC0415 + anchor_family_exports, + published_2017_exports, + ) from bedrock.transform.iot.nowcast_s00300_use import ( # noqa: PLC0415 s00300_f02n00_usd, ) - exports = _trade_fbs_commodity_vector( - f'Trade_Exports_{year}', download_sources_ok + exports = anchor_family_exports( + _trade_fbs_commodity_vector(f'Trade_Exports_{year}', download_sources_ok), + _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), + published_2017_exports(), ) y['F04000'] = exports.reindex(y.index).fillna(0.0) pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise')) diff --git a/bedrock/transform/iot/nowcast_export_anchoring.py b/bedrock/transform/iot/nowcast_export_anchoring.py new file mode 100644 index 000000000..b794a7632 --- /dev/null +++ b/bedrock/transform/iot/nowcast_export_anchoring.py @@ -0,0 +1,86 @@ +"""Anchor the export column of chosen families on the published 2017 cells (#1270). + +The export column (``F04000``) is Census domestic exports (#762) mapped to BEA +commodities, and nothing conditions it. For the computers family (``3341``) +that puts exports above domestic output. Terminals and peripherals (``334118``) +export $9.2bn in 2017 against BEA's $6.1bn and its own $8.1bn of output, and +the whole family runs 1.28 times published. Census re-exports of terminals +($16.4bn in 2017) are already excluded, so the excess is in the domestic +exports themselves. + +A row that exports more than it produces leaves imports with nowhere to go: +total use excluding exports falls below imports, so the import matrix assigns +industries more of the commodity than they buy (the rows #1282 warns about). + +The construction is the #763 one for imports +(:func:`~bedrock.transform.iot.nowcast_import_conditioning.anchored_mcif`): +each commodity's exports are its **published 2017 detail ``F04000`` cell** +moved by **its own Census line's growth**. 2017 reproduces BEA, and every year +moves by its own observed series. + +⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES`. Every other commodity +keeps its Census value. +""" + +from __future__ import annotations + +import logging + +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + +#: NAICS families whose exports are anchored on the published 2017 cells. +#: ``3341`` computers: Census domestic exports run 1.28 times BEA's 2017 family +#: total, and terminals export more than they produce (#1270). +ANCHORED_EXPORT_FAMILIES: frozenset[str] = frozenset({'3341'}) + +#: Below this ($) a published or Census value counts as empty. +EMPTY_USD = 1e6 + + +def anchor_family_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = ANCHORED_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each anchored family's cells on published 2017 x own growth. + + For a commodity inside *families*: + + - published 2017 and Census 2017 both carry it: + ``published_2017 x (census_t / census_2017)``; + - published 2017 is empty but Census has it: keep the Census value; + - Census 2017 is empty but published has it: hold the published value. + + Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + in_family = pd.Series( + [str(c)[:4] in families for c in exports_now.index], index=exports_now.index + ) + has_pub = pub >= EMPTY_USD + has_base = base >= EMPTY_USD + with np.errstate(divide='ignore', invalid='ignore'): + growth = (exports_now / base).replace([np.inf, -np.inf], np.nan).fillna(0.0) + + out = exports_now.copy() + both = in_family & has_pub & has_base + out[both] = pub[both] * growth[both] + pub_only = in_family & has_pub & ~has_base + out[pub_only] = pub[pub_only] + return out + + +def published_2017_exports() -> pd.Series: + """BEA's published 2017 detail export column (``F04000``), USD.""" + from bedrock.extract.iot.io_2017 import ( # noqa: PLC0415 + _load_2017_detail_supply_use_usa, + ) + + use = _load_2017_detail_supply_use_usa('Use_SUT_detail') + use.columns = use.columns.str.strip() + return pd.to_numeric(use['F04000'], errors='coerce').fillna(0.0) * 1e6 From 1331e58b1f03196089836871a759f5629553bb7d Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 19:52:47 -0500 Subject: [PATCH 2/2] feat(nowcast): anchor the pharmaceutical family's exports on family growth (#1194, #1270) Review on #1295 (Ben): Mo's 325411 finding is the same Census-vs-BEA gap. Census puts preparations on 325411 ($7.3bn of 2017 exports against $0.08bn published) and 325413, so a row's own Census line is no mover: own-growth anchoring would put biologics (325414) at $52.7bn of 2024 exports against $48.0bn of output. FAMILY_GROWTH_EXPORT_FAMILIES = {'3254'} holds BEA's 2017 split of the family and moves it by the family's Census growth (1.81 by 2024). All four rows stay below output in 2017-2024 (325414 peaks at 0.87). anchored_exports applies both modes. Also from the review: rebuild_nowcast.yml pinned to ubuntu-24.04, as #1297 did for CI, and the rebuild script's anchor comment covers exports. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rebuild_nowcast.yml | 4 +- .../test_nowcast_export_anchoring.py | 30 +++++++++- bedrock/transform/iot/nowcast.py | 6 +- .../transform/iot/nowcast_export_anchoring.py | 56 ++++++++++++++++++- scripts/rebuild_nowcast_year.py | 2 +- 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rebuild_nowcast.yml b/.github/workflows/rebuild_nowcast.yml index 425825b08..cd47ed1fc 100644 --- a/.github/workflows/rebuild_nowcast.yml +++ b/.github/workflows/rebuild_nowcast.yml @@ -21,7 +21,7 @@ env: jobs: tables: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write @@ -65,7 +65,7 @@ jobs: ghg: needs: tables - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write diff --git a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py index c64073978..0f821d4fa 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py @@ -5,7 +5,10 @@ import pandas as pd import pytest -from bedrock.transform.iot.nowcast_export_anchoring import anchor_family_exports +from bedrock.transform.iot.nowcast_export_anchoring import ( + anchor_family_exports, + anchor_family_growth_exports, +) M = 1e6 IDX = ['334111', '334118', '334119', '336111'] @@ -40,3 +43,28 @@ def test_census_only_and_published_only_rows() -> None: out = _anchor([10.0, 4.0, 0.0, 30.0], [8.0, 3.0, 0.0, 25.0], [7.0, 0.0, 2.0, 23.0]) assert out['334118'] == pytest.approx(4.0 * M) # published 2017 empty: Census assert out['334119'] == pytest.approx(2.0 * M) # Census 2017 empty: held flat + + +PHARMA = ['325411', '325412', '325414', '336111'] + + +def test_family_growth_keeps_bea_split_and_moves_by_the_family() -> None: + now = pd.Series([14.0, 20.0, 46.0, 30.0], index=PHARMA) * M + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(now, base, pub) + + growth = (14.0 + 20.0 + 46.0) / (7.0 + 17.0 + 20.0) + for code, published in (('325411', 0.1), ('325412', 30.0), ('325414', 22.5)): + assert out[code] == pytest.approx(published * growth * M) + assert out['336111'] == pytest.approx(30.0 * M) # outside the family + + +def test_family_growth_in_2017_reproduces_bea() -> None: + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(base, base, pub) + + for code in ('325411', '325412', '325414'): + assert out[code] == pytest.approx(pub[code]) + assert out['336111'] == pytest.approx(base['336111']) diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index abc5882e7..a9d0b44fb 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -461,7 +461,7 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat F03000 is Inventories_ for every year in ``INVENTORIES_YEARS`` (#529, #746); 2024 is all-zero. F04000 is mapped Trade_Exports_ Detail mass for every year in - ``TRADE_OVERLAY_YEARS``, with the computers family anchored on the + ``TRADE_OVERLAY_YEARS``, with the computers and pharmaceutical families anchored on the published 2017 cells (:mod:`~bedrock.transform.iot.nowcast_export_anchoring`, #1270). S00900/F04000 uses the rest-of-world identity against frozen 2017 Supply T016. @@ -482,14 +482,14 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat if year in TRADE_OVERLAY_YEARS: from bedrock.transform.iot.nowcast_export_anchoring import ( # noqa: PLC0415 - anchor_family_exports, + anchored_exports, published_2017_exports, ) from bedrock.transform.iot.nowcast_s00300_use import ( # noqa: PLC0415 s00300_f02n00_usd, ) - exports = anchor_family_exports( + exports = anchored_exports( _trade_fbs_commodity_vector(f'Trade_Exports_{year}', download_sources_ok), _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), published_2017_exports(), diff --git a/bedrock/transform/iot/nowcast_export_anchoring.py b/bedrock/transform/iot/nowcast_export_anchoring.py index b794a7632..e0feaf451 100644 --- a/bedrock/transform/iot/nowcast_export_anchoring.py +++ b/bedrock/transform/iot/nowcast_export_anchoring.py @@ -18,7 +18,9 @@ moved by **its own Census line's growth**. 2017 reproduces BEA, and every year moves by its own observed series. -⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES`. Every other commodity +⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES` (each row moved by its +own Census growth) and :data:`FAMILY_GROWTH_EXPORT_FAMILIES` (each row moved by +its family's growth, where Census misclassifies rows). Every other commodity keeps its Census value. """ @@ -31,11 +33,22 @@ logger = logging.getLogger(__name__) -#: NAICS families whose exports are anchored on the published 2017 cells. +#: NAICS families whose exports are anchored on the published 2017 cells, each +#: row moved by **its own** Census line's growth. #: ``3341`` computers: Census domestic exports run 1.28 times BEA's 2017 family #: total, and terminals export more than they produce (#1270). ANCHORED_EXPORT_FAMILIES: frozenset[str] = frozenset({'3341'}) +#: NAICS families whose exports are anchored on the published 2017 cells, every +#: row moved by the **family's** Census growth: BEA's 2017 split of the family, +#: moved by the family level. For families whose Census rows are misclassified, +#: so a row's own line is no mover. +#: ``3254`` pharmaceuticals: Census puts preparations on ``325411`` medicinal +#: ingredients ($7.3bn of exports in 2017 against $0.08bn published) and +#: ``325413`` diagnostics. With each row's own growth, biologics (``325414``) +#: would export $52.7bn in 2024 against $48.0bn of output (#1194, #1270). +FAMILY_GROWTH_EXPORT_FAMILIES: frozenset[str] = frozenset({'3254'}) + #: Below this ($) a published or Census value counts as empty. EMPTY_USD = 1e6 @@ -75,6 +88,45 @@ def anchor_family_exports( return out +def anchor_family_growth_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = FAMILY_GROWTH_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each family on BEA's 2017 split, moved by the family's growth. + + For each family in *families*, every commodity's exports are + ``published_2017 x (family census_t / family census_2017)``. A commodity BEA + publishes no 2017 exports for gets none: its Census mass is the + misclassification being corrected. A family with no Census 2017 mass keeps + its Census values. Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + family = pd.Series([str(c)[:4] for c in exports_now.index], index=exports_now.index) + out = exports_now.copy() + for code in sorted(families): + rows = family == code + base_total = float(base[rows].sum()) + if base_total < EMPTY_USD: + continue + growth = float(exports_now[rows].sum()) / base_total + out[rows] = pub[rows] * growth + logger.info( + 'exports %s: BEA 2017 split moved by the family growth %.3f', code, growth + ) + return out + + +def anchored_exports( + exports_now: pd.Series, exports_2017: pd.Series, published_2017: pd.Series +) -> pd.Series: + """Both anchorings: own-growth rows, then family-growth families.""" + rows = anchor_family_exports(exports_now, exports_2017, published_2017) + return anchor_family_growth_exports(rows, exports_2017, published_2017) + + def published_2017_exports() -> pd.Series: """BEA's published 2017 detail export column (``F04000``), USD.""" from bedrock.extract.iot.io_2017 import ( # noqa: PLC0415 diff --git a/scripts/rebuild_nowcast_year.py b/scripts/rebuild_nowcast_year.py index 257918809..afbb0e902 100644 --- a/scripts/rebuild_nowcast_year.py +++ b/scripts/rebuild_nowcast_year.py @@ -240,7 +240,7 @@ def main() -> None: common.download_fba_on_api_error = False check_flags() - # 2017 is the import anchor every later year is conditioned against. + # 2017 is the import and export anchor every later year is conditioned against. build_trade(sorted({2017, year}), tag) run_step('bedrock.transform.iot.nowcast_sut_assembly', '--years', str(year))