diff --git a/.github/workflows/rebuild_nowcast.yml b/.github/workflows/rebuild_nowcast.yml index 425825b08..cd47ed1fc 100644 --- a/.github/workflows/rebuild_nowcast.yml +++ b/.github/workflows/rebuild_nowcast.yml @@ -21,7 +21,7 @@ env: jobs: tables: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write @@ -65,7 +65,7 @@ jobs: ghg: needs: tables - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write diff --git a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py new file mode 100644 index 000000000..0f821d4fa --- /dev/null +++ b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py @@ -0,0 +1,70 @@ +"""The computers family's exports sit on the published 2017 cells (#1270).""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from bedrock.transform.iot.nowcast_export_anchoring import ( + anchor_family_exports, + anchor_family_growth_exports, +) + +M = 1e6 +IDX = ['334111', '334118', '334119', '336111'] + + +def _anchor(now: list[float], base: list[float], pub: list[float]) -> pd.Series: + return anchor_family_exports( + pd.Series(now, index=IDX) * M, + pd.Series(base, index=IDX) * M, + pd.Series(pub, index=IDX) * M, + ) + + +def test_anchored_rows_are_published_2017_moved_by_own_growth() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * 10.0 / 8.0 * M) + assert out['334118'] == pytest.approx(6.0 * 11.0 / 9.0 * M) + + +def test_2017_reproduces_the_published_cells() -> None: + out = _anchor([8.0, 9.0, 0.0, 25.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * M) + assert out['334118'] == pytest.approx(6.0 * M) + + +def test_rows_outside_the_family_keep_census() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['336111'] == pytest.approx(30.0 * M) + + +def test_census_only_and_published_only_rows() -> None: + out = _anchor([10.0, 4.0, 0.0, 30.0], [8.0, 3.0, 0.0, 25.0], [7.0, 0.0, 2.0, 23.0]) + assert out['334118'] == pytest.approx(4.0 * M) # published 2017 empty: Census + assert out['334119'] == pytest.approx(2.0 * M) # Census 2017 empty: held flat + + +PHARMA = ['325411', '325412', '325414', '336111'] + + +def test_family_growth_keeps_bea_split_and_moves_by_the_family() -> None: + now = pd.Series([14.0, 20.0, 46.0, 30.0], index=PHARMA) * M + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(now, base, pub) + + growth = (14.0 + 20.0 + 46.0) / (7.0 + 17.0 + 20.0) + for code, published in (('325411', 0.1), ('325412', 30.0), ('325414', 22.5)): + assert out[code] == pytest.approx(published * growth * M) + assert out['336111'] == pytest.approx(30.0 * M) # outside the family + + +def test_family_growth_in_2017_reproduces_bea() -> None: + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(base, base, pub) + + for code in ('325411', '325412', '325414'): + assert out[code] == pytest.approx(pub[code]) + assert out['336111'] == pytest.approx(base['336111']) diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index 14889be00..a9d0b44fb 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -461,7 +461,9 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat F03000 is Inventories_ for every year in ``INVENTORIES_YEARS`` (#529, #746); 2024 is all-zero. F04000 is mapped Trade_Exports_ Detail mass for every year in - ``TRADE_OVERLAY_YEARS``. S00900/F04000 uses the rest-of-world identity + ``TRADE_OVERLAY_YEARS``, with the computers and pharmaceutical families anchored on the + published 2017 cells (:mod:`~bedrock.transform.iot.nowcast_export_anchoring`, + #1270). S00900/F04000 uses the rest-of-world identity against frozen 2017 Supply T016. The seventeen NIPA-built columns are **conditioned on the published annual @@ -479,12 +481,18 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat y = _nipa_final_demand_block(pd.DataFrame(fbs), int(year)) if year in TRADE_OVERLAY_YEARS: + from bedrock.transform.iot.nowcast_export_anchoring import ( # noqa: PLC0415 + anchored_exports, + published_2017_exports, + ) from bedrock.transform.iot.nowcast_s00300_use import ( # noqa: PLC0415 s00300_f02n00_usd, ) - exports = _trade_fbs_commodity_vector( - f'Trade_Exports_{year}', download_sources_ok + exports = anchored_exports( + _trade_fbs_commodity_vector(f'Trade_Exports_{year}', download_sources_ok), + _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), + published_2017_exports(), ) y['F04000'] = exports.reindex(y.index).fillna(0.0) pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise')) diff --git a/bedrock/transform/iot/nowcast_export_anchoring.py b/bedrock/transform/iot/nowcast_export_anchoring.py new file mode 100644 index 000000000..e0feaf451 --- /dev/null +++ b/bedrock/transform/iot/nowcast_export_anchoring.py @@ -0,0 +1,138 @@ +"""Anchor the export column of chosen families on the published 2017 cells (#1270). + +The export column (``F04000``) is Census domestic exports (#762) mapped to BEA +commodities, and nothing conditions it. For the computers family (``3341``) +that puts exports above domestic output. Terminals and peripherals (``334118``) +export $9.2bn in 2017 against BEA's $6.1bn and its own $8.1bn of output, and +the whole family runs 1.28 times published. Census re-exports of terminals +($16.4bn in 2017) are already excluded, so the excess is in the domestic +exports themselves. + +A row that exports more than it produces leaves imports with nowhere to go: +total use excluding exports falls below imports, so the import matrix assigns +industries more of the commodity than they buy (the rows #1282 warns about). + +The construction is the #763 one for imports +(:func:`~bedrock.transform.iot.nowcast_import_conditioning.anchored_mcif`): +each commodity's exports are its **published 2017 detail ``F04000`` cell** +moved by **its own Census line's growth**. 2017 reproduces BEA, and every year +moves by its own observed series. + +⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES` (each row moved by its +own Census growth) and :data:`FAMILY_GROWTH_EXPORT_FAMILIES` (each row moved by +its family's growth, where Census misclassifies rows). Every other commodity +keeps its Census value. +""" + +from __future__ import annotations + +import logging + +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + +#: NAICS families whose exports are anchored on the published 2017 cells, each +#: row moved by **its own** Census line's growth. +#: ``3341`` computers: Census domestic exports run 1.28 times BEA's 2017 family +#: total, and terminals export more than they produce (#1270). +ANCHORED_EXPORT_FAMILIES: frozenset[str] = frozenset({'3341'}) + +#: NAICS families whose exports are anchored on the published 2017 cells, every +#: row moved by the **family's** Census growth: BEA's 2017 split of the family, +#: moved by the family level. For families whose Census rows are misclassified, +#: so a row's own line is no mover. +#: ``3254`` pharmaceuticals: Census puts preparations on ``325411`` medicinal +#: ingredients ($7.3bn of exports in 2017 against $0.08bn published) and +#: ``325413`` diagnostics. With each row's own growth, biologics (``325414``) +#: would export $52.7bn in 2024 against $48.0bn of output (#1194, #1270). +FAMILY_GROWTH_EXPORT_FAMILIES: frozenset[str] = frozenset({'3254'}) + +#: Below this ($) a published or Census value counts as empty. +EMPTY_USD = 1e6 + + +def anchor_family_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = ANCHORED_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each anchored family's cells on published 2017 x own growth. + + For a commodity inside *families*: + + - published 2017 and Census 2017 both carry it: + ``published_2017 x (census_t / census_2017)``; + - published 2017 is empty but Census has it: keep the Census value; + - Census 2017 is empty but published has it: hold the published value. + + Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + in_family = pd.Series( + [str(c)[:4] in families for c in exports_now.index], index=exports_now.index + ) + has_pub = pub >= EMPTY_USD + has_base = base >= EMPTY_USD + with np.errstate(divide='ignore', invalid='ignore'): + growth = (exports_now / base).replace([np.inf, -np.inf], np.nan).fillna(0.0) + + out = exports_now.copy() + both = in_family & has_pub & has_base + out[both] = pub[both] * growth[both] + pub_only = in_family & has_pub & ~has_base + out[pub_only] = pub[pub_only] + return out + + +def anchor_family_growth_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = FAMILY_GROWTH_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each family on BEA's 2017 split, moved by the family's growth. + + For each family in *families*, every commodity's exports are + ``published_2017 x (family census_t / family census_2017)``. A commodity BEA + publishes no 2017 exports for gets none: its Census mass is the + misclassification being corrected. A family with no Census 2017 mass keeps + its Census values. Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + family = pd.Series([str(c)[:4] for c in exports_now.index], index=exports_now.index) + out = exports_now.copy() + for code in sorted(families): + rows = family == code + base_total = float(base[rows].sum()) + if base_total < EMPTY_USD: + continue + growth = float(exports_now[rows].sum()) / base_total + out[rows] = pub[rows] * growth + logger.info( + 'exports %s: BEA 2017 split moved by the family growth %.3f', code, growth + ) + return out + + +def anchored_exports( + exports_now: pd.Series, exports_2017: pd.Series, published_2017: pd.Series +) -> pd.Series: + """Both anchorings: own-growth rows, then family-growth families.""" + rows = anchor_family_exports(exports_now, exports_2017, published_2017) + return anchor_family_growth_exports(rows, exports_2017, published_2017) + + +def published_2017_exports() -> pd.Series: + """BEA's published 2017 detail export column (``F04000``), USD.""" + from bedrock.extract.iot.io_2017 import ( # noqa: PLC0415 + _load_2017_detail_supply_use_usa, + ) + + use = _load_2017_detail_supply_use_usa('Use_SUT_detail') + use.columns = use.columns.str.strip() + return pd.to_numeric(use['F04000'], errors='coerce').fillna(0.0) * 1e6 diff --git a/scripts/rebuild_nowcast_year.py b/scripts/rebuild_nowcast_year.py index 257918809..afbb0e902 100644 --- a/scripts/rebuild_nowcast_year.py +++ b/scripts/rebuild_nowcast_year.py @@ -240,7 +240,7 @@ def main() -> None: common.download_fba_on_api_error = False check_flags() - # 2017 is the import anchor every later year is conditioned against. + # 2017 is the import and export anchor every later year is conditioned against. build_trade(sorted({2017, year}), tag) run_step('bedrock.transform.iot.nowcast_sut_assembly', '--years', str(year))