diff --git a/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py b/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py index 76dbcebc1..8805e5ee9 100644 --- a/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py +++ b/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py @@ -175,13 +175,18 @@ def _to_summary() -> pd.Series: def service_rows() -> list[str]: - """Summary commodity rows from ``42`` onward, without government.""" - goods = ('111', '113', '21', '22', '23', '3') - return sorted( - {s for s in _to_summary() if not s.startswith(goods) and not s.startswith('G')} - - set(SEPARATE_ROWS) + """Summary commodity rows from ``42`` onward, without government. + + The same cut the build conditions service exports on + (:func:`~bedrock.transform.iot.nowcast_fd_conditioning.is_service_group`), + so the grader and the conditioning can't disagree on what a service is. + """ + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + is_service_group, ) + return sorted({s for s in _to_summary() if is_service_group(s)}) + def default_build() -> str: """The balanced build the canonical config ships, e.g. ``v1.0.0_5514c3b``.""" diff --git a/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py b/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py index f4f97560a..f701f707e 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py @@ -19,6 +19,8 @@ EMPTY_GROUP_USD, _commodity_to_summary, condition_fd_on_summary, + condition_service_exports_on_summary, + is_service_group, summary_condition_factors, ) @@ -81,3 +83,28 @@ def test_empty_and_sign_flipped_groups_hold_at_one(y: pd.DataFrame) -> None: def test_years_outside_the_workbook_are_refused(y: pd.DataFrame) -> None: with pytest.raises(ValueError, match='summary Use SUT'): summary_condition_factors(y, 2025) + + +# --- service exports (#1250) ------------------------------------------------ + + +def test_service_exports_land_on_the_published_rows_and_goods_stay() -> None: + groups = _commodity_to_summary() + exports = pd.Series(2.0e9, index=groups.index) + out = condition_service_exports_on_summary(exports, YEAR) + published = _load_usa_summary_sut('Use_SUT_summary', YEAR) # type: ignore[arg-type] + published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6 + ours = out.groupby(groups).sum() + for group in ('514', '561', '55', '5412OP'): + assert is_service_group(group) + assert ours[group] == pytest.approx(published.at[group, 'F040'], rel=1e-9) + goods = groups.map(lambda g: not is_service_group(g)) + pd.testing.assert_series_equal(out[goods], exports[goods]) + within = groups.eq('5412OP') + assert out[within].nunique() == 1 # the detail split inside a group holds + + +def test_service_groups_exclude_goods_government_and_accounting_rows() -> None: + assert not any(is_service_group(g) for g in ('111CA', '22', '23', '325', 'GFGD')) + assert not any(is_service_group(g) for g in ('Other', 'Used')) + assert all(is_service_group(g) for g in ('42', '481', '523', '81')) diff --git a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py index d03edc2ef..a1c675dba 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py @@ -310,3 +310,64 @@ def test_a_group_whose_scale_would_flip_its_sign_keeps_its_seed( ) assert report.loc[('TOP', 'g1'), 'held'] == pytest.approx(2.0) assert any('flip their sign' in w for w in warned) + + +# --- the import hold (#1288) ------------------------------------------------ + + +def test_the_import_hold_targets_the_published_groups_less_frozen( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(asm, 'commodity_groups', lambda: GROUPS) + monkeypatch.setattr( + asm, 'published_import_groups', lambda year: pd.Series({'g1': 5.0, 'g2': 3.0}) + ) + frozen = pd.DataFrame({'i1': 0.0, 'MCIF': [0.0, 1.5, 0.0]}, index=GROUPS.index) + (hold,) = asm.import_group_holds(2017, frozen) + assert hold.column == 'MCIF' + assert hold.targets.to_dict() == pytest.approx({'g1': 3.5, 'g2': 3.0}) + + +def test_held_imports_keep_each_group_where_the_rows_would_move_them() -> None: + """The rows want c1 and c2 to lose 2 of supply and c3 to gain 2; without + the hold, imports move from g1 to g2; held, i1 absorbs it instead.""" + supply = pd.DataFrame( + {'i1': [4.0, 4.0, 2.0], 'MCIF': [2.0, 2.0, 2.0]}, index=['c1', 'c2', 'c3'] + ) + rows = pd.Series({'c1': 5.0, 'c2': 5.0, 'c3': 6.0}) + cols = pd.Series({'i1': 10.0, 'MCIF': 6.0}) + mask = SutMask.from_pattern(supply) + hold = GroupHold('MCIF', GROUPS, pd.Series({'g1': 4.0, 'g2': 2.0})) + free, _ = _balance_block( + supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True + ) + held, _ = _balance_block( + supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True, holds=(hold,) + ) + assert free.loc[['c1', 'c2'], 'MCIF'].sum() < 4.0 + assert held.loc[['c1', 'c2'], 'MCIF'].sum() == pytest.approx(4.0, abs=1e-6) + assert held.at['c3', 'MCIF'] == pytest.approx(2.0, abs=1e-6) + pd.testing.assert_series_equal(held.sum(axis=1), rows, check_names=False) + + +# --- fixed service exports (#1250) ------------------------------------------ + + +def test_service_export_cells_are_fixed_and_goods_stay_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + asm, + 'commodity_groups', + lambda: pd.Series({'c1': '514', 'c2': '325', 'c3': '561'}), + ) + use = pd.DataFrame( + {'i1': [1.0, 1.0, 1.0], 'F04000': [2.0, 3.0, 0.0]}, index=['c1', 'c2', 'c3'] + ) + zero = pd.DataFrame(False, index=use.index, columns=use.columns) + mask = SutMask( + structural_zero=zero, fixed_value=zero.copy(), sign_lock=zero.astype(int) + ) + fixed = asm.fix_service_exports(mask, use).fixed_value + assert fixed['F04000'].to_dict() == {'c1': True, 'c2': False, 'c3': True} + assert not fixed['i1'].any() diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index a9d0b44fb..9d077305c 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -494,6 +494,14 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), published_2017_exports(), ) + from bedrock.utils.config.usa_config import get_usa_config # noqa: PLC0415 + + if get_usa_config().condition_service_exports_on_summary: + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + condition_service_exports_on_summary, + ) + + exports = condition_service_exports_on_summary(exports, int(year)) y['F04000'] = exports.reindex(y.index).fillna(0.0) pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise')) y.loc['S00900', 'F04000'] = -pce + _s00900_export_identity_usd() diff --git a/bedrock/transform/iot/nowcast_fd_conditioning.py b/bedrock/transform/iot/nowcast_fd_conditioning.py index 4c42d54e9..d816f087f 100644 --- a/bedrock/transform/iot/nowcast_fd_conditioning.py +++ b/bedrock/transform/iot/nowcast_fd_conditioning.py @@ -28,10 +28,13 @@ targets otherwise) and large fixes on FD-owned rows — custom programming's 2023 residual fell 76 -> 30bn $M in the counterfactual. -⚠️ ``F03000`` and ``F04000`` are NOT conditioned. Inventories and exports are -their own sourced constructions (#746, #729/#771) with their own graders; +⚠️ ``F03000`` and ``F04000`` are NOT conditioned here. Inventories and exports +are their own sourced constructions (#746, #729/#771) with their own graders; their summary-level standing is recorded in the About doc, and exports are -bl-young's lane. +bl-young's lane. With ``condition_service_exports_on_summary`` set, the +**service** rows of ``F04000`` are conditioned the same way +(:func:`condition_service_exports_on_summary`, #1250); goods exports stay on +Census trade. Guards, in the order applied per (group, column): @@ -95,6 +98,68 @@ def _commodity_to_summary() -> pd.Series: return pd.Series({code: parents[0] for code, parents in mapping.items()}) +def _group_factors(ours: pd.Series, pub: pd.Series) -> tuple[pd.Series, float, float]: + """published / ours per group, with the guards; plus the USD each guard left.""" + empty = ours.abs() < EMPTY_GROUP_USD + flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0) + factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0) + factor[empty | flipped] = 1.0 + unreachable = float(pub[empty].abs().sum()) + sign_held = float((pub - ours)[flipped].abs().sum()) + return factor, unreachable, sign_held + + +#: Summary groups whose exports stay on Census goods trade: agriculture to +#: manufacturing, and government. ``Other`` and ``Used`` are accounting rows. +GOODS_SUMMARY_PREFIXES: tuple[str, ...] = ('111', '113', '21', '22', '23', '3', 'G') +ACCOUNTING_SUMMARY_ROWS: tuple[str, ...] = ('Other', 'Used') + + +def is_service_group(group: str) -> bool: + """Whether summary commodity *group* is a service row (``42`` onward).""" + return not str(group).startswith(GOODS_SUMMARY_PREFIXES) and ( + str(group) not in ACCOUNTING_SUMMARY_ROWS + ) + + +def condition_service_exports_on_summary(exports: pd.Series, year: int) -> pd.Series: + """*exports* with each service group scaled to the summary Use ``F040``, USD. + + Service exports are each row's published 2017 value moved by a blend of + ITA categories (#771). The blends drift by about 40% on data processing + (``514``), administrative and support (``561``) and management of + companies (``55``) by 2024, and the balance barely moves exports, so the + drift reaches the tables (#1250). BEA publishes the summary rows every + year, so each service group is held to them; the blends only split a + group between its detail rows. Goods rows and rows with no summary group + are returned unchanged, and the guards of :func:`summary_condition_factors` + apply. + """ + if year not in SUMMARY_SUT_YEARS: + raise ValueError( + f'the summary Use SUT covers {SUMMARY_SUT_YEARS[0]}-' + f'{SUMMARY_SUT_YEARS[-1]}; got {year}' + ) + groups = _commodity_to_summary().reindex(exports.index) + service = groups.map(lambda g: isinstance(g, str) and is_service_group(g)) + service = service.fillna(False).astype(bool) + published = _load_usa_summary_sut('Use_SUT_summary', year) # type: ignore[arg-type] + published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6 + ours = exports[service].groupby(groups[service]).sum() + pub = published['F040'].reindex(ours.index).fillna(0.0) + factor, unreachable, sign_held = _group_factors(ours, pub) + if unreachable or sign_held: + logger.info( + 'service export conditioning %s: %.0fM published unreachable (our ' + 'group empty), %.0fM held on sign disagreement', + year, + unreachable / 1e6, + sign_held / 1e6, + ) + per_row = groups.map(factor).where(service, 1.0).astype(float).fillna(1.0) + return exports * per_row + + def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame: """Summary group x column scale factors for ``y`` at ``year``, guards applied. @@ -119,13 +184,10 @@ def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame: # The summary workbooks truncate the codes to four characters # (F01000 -> F010). pub = published[column[:4]].reindex(ours.index).fillna(0.0) - empty = ours.abs() < EMPTY_GROUP_USD - flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0) - factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0) - factor[empty | flipped] = 1.0 + factor, missed, held = _group_factors(ours, pub) factors[column] = factor - unreachable += float(pub[empty].abs().sum()) - sign_held += float((pub - ours)[flipped].abs().sum()) + unreachable += missed + sign_held += held if unreachable or sign_held: logger.info( 'FD conditioning %s: %.0fM published unreachable (our group ' diff --git a/bedrock/transform/iot/nowcast_sut_assembly.py b/bedrock/transform/iot/nowcast_sut_assembly.py index ef5b2f060..fe8971390 100644 --- a/bedrock/transform/iot/nowcast_sut_assembly.py +++ b/bedrock/transform/iot/nowcast_sut_assembly.py @@ -914,14 +914,47 @@ def published_tax_groups(year: int) -> pd.DataFrame: ``SUB`` is negative, as BEA stores it and as the balance carries it. """ + return _published_supply_groups(int(year), HELD_TAX_COLUMNS) + + +def _published_supply_groups(year: int, columns: Sequence[str]) -> pd.DataFrame: + """Summary Supply *columns* for *year*, $M, by summary commodity.""" supply = _load_usa_summary_sut('Supply_summary', year) # type: ignore[arg-type] supply = supply.rename(columns=lambda c: str(c).strip()) groups = set(commodity_groups()) - frame = supply[list(HELD_TAX_COLUMNS)].apply(pd.to_numeric, errors='coerce') + frame = supply[list(columns)].apply(pd.to_numeric, errors='coerce') frame.index = [str(i).strip() for i in frame.index] return frame.loc[[g for g in frame.index if g in groups]].fillna(0.0) +#: The Supply import column the balance can hold to BEA's summary groups. +HELD_IMPORT_COLUMN = 'MCIF' + + +def published_import_groups(year: int) -> pd.Series: + """The summary Supply ``MCIF`` column for *year*, $M, by summary commodity.""" + return _published_supply_groups(int(year), (HELD_IMPORT_COLUMN,))[ + HELD_IMPORT_COLUMN + ] + + +def import_group_holds(year: int, frozen_supply: pd.DataFrame) -> tuple[GroupHold, ...]: + """The engine's ``MCIF`` group hold: published group sums less frozen, $M. + + #785 already scales each summary group of the seed's imports to BEA's + summary ``MCIF``, but the balance otherwise holds imports only as one soft + column total (T7), so the row identities move them between groups. In + 2017, service imports enter at the published $267.2bn and leave at + $247.0bn (#1288). Held by group, they keep the published sums and the + detail split inside each group stays free. + """ + groups = commodity_groups().reindex(frozen_supply.index) + frozen = frozen_supply[HELD_IMPORT_COLUMN].astype(float).groupby(groups).sum() + published = published_import_groups(int(year)) + target = published - frozen.reindex(published.index).fillna(0.0) + return (GroupHold(column=HELD_IMPORT_COLUMN, groups=groups, targets=target),) + + def commodity_groups() -> pd.Series: """Each detail commodity's summary commodity group.""" return pd.Series( @@ -1035,6 +1068,38 @@ def rows(z_supply: pd.DataFrame) -> pd.DataFrame: return rows +#: The Use column whose service cells are fixed when exports are conditioned. +EXPORT_COLUMN = 'F04000' + + +def fix_service_exports(mask: SutMask, use: pd.DataFrame) -> SutMask: + """New Use mask with the service ``F04000`` cells Tier-1 fixed against *use*. + + With ``condition_service_exports_on_summary``, the seed's service exports + already sit on BEA's summary ``F040`` groups (#1250). The balance + otherwise moves them freely: on ``v1.0.0_fd271dc`` it took pipelines' + (``486``) error from 47 to 527 $M a year, and banking's (``521CI``) from + 51 to 673. Fixed, they leave the balance as conditioned, and the rows + close on their other cells. Structural zeros stay structural. + """ + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + is_service_group, + ) + + groups = commodity_groups().reindex(mask.index) + service = groups.map(lambda g: isinstance(g, str) and is_service_group(g)) + rows = service.fillna(False).astype(bool) & ~mask.structural_zero[EXPORT_COLUMN] + fixed = mask.fixed_value.copy() + fixed.loc[rows, EXPORT_COLUMN] = True + out = SutMask( + structural_zero=mask.structural_zero, + fixed_value=fixed, + sign_lock=mask.sign_lock, + ) + out.validate_against(use) + return out + + def assemble( year: int, *, @@ -1057,6 +1122,9 @@ def assemble( When ``pin_refinery_secondaries_at_go_mix`` is on, the Supply seed is then rewritten so held co-product cells enter GRAS at seed share times the T17-at-seed column pin, and Tier-1 is overlaid from that held list. + + When ``condition_service_exports_on_summary`` is on, the service export + cells are Tier-1 fixed too (:func:`fix_service_exports`). """ # Gross output and the summary supply-use controls must be one BEA annual # update; the targets balance one against the other (#1157). @@ -1077,6 +1145,8 @@ def assemble( seeds, masks, pin = apply_refinery_secondaries_go_mix_pin( int(year), seeds, masks, targets ) + if get_usa_config().condition_service_exports_on_summary: + masks = {**masks, 'use': fix_service_exports(masks['use'], seeds['use'])} return YearBalance( year=int(year), seeds=seeds, @@ -1100,6 +1170,7 @@ def balance_year( constrain_electricity_pce_cell: bool | None = None, convert_tax_rows: bool = True, hold_tax_groups: bool = True, + hold_import_groups: bool | None = None, ) -> YearBalance: """Assemble and balance one year: split, offset, engine, restore. @@ -1110,7 +1181,11 @@ def balance_year( Use ``T00TOP``/``T00SUB`` rows stay a conversion of the Supply columns. *hold_tax_groups* holds ``TOP``/``SUB`` to the published summary groups, with the detail free inside each group (:func:`tax_group_holds`). + *hold_import_groups* does the same for ``MCIF`` (:func:`import_group_holds`); + it defaults to the ``hold_imports_on_summary_groups`` config flag. """ + if hold_import_groups is None: + hold_import_groups = get_usa_config().hold_imports_on_summary_groups mode = resolve_pce_constraint(pce_constraint, constrain_electricity_pce_cell) assembled = assemble( int(year), @@ -1135,7 +1210,12 @@ def run( pce_eia_band_m=band, tax_rows=tax_row_operator(year, frozen) if convert_tax_rows else None, group_holds=( - tax_group_holds(year, frozen['supply']) if hold_tax_groups else () + (tax_group_holds(year, frozen['supply']) if hold_tax_groups else ()) + + ( + import_group_holds(year, frozen['supply']) + if hold_import_groups + else () + ) ), ) return out, restore_fixed_blocks(out.blocks, frozen) diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index e985be69e..53672f2f1 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -304,6 +304,19 @@ class USAConfig(BaseModel): # and a purchaser-to-producer ratio of 0.011 (#1194). See # nowcast_trade_margins.supply_following_weight. trade_margin_receivers_follow_supply: bool = False # DRI: WesIngwersen + # Hold the Step 5 balance's imports (MCIF) to BEA's summary Supply MCIF by + # summary group, as TOP/SUB are held, instead of one soft column total. + # The row identities otherwise pull service imports off the published + # groups: 2017 enters at $267.2bn and leaves at $247.0bn (#1288). See + # nowcast_sut_assembly.import_group_holds. + hold_imports_on_summary_groups: bool = False # DRI: WesIngwersen + # Scale each service summary group of exports (F04000) to BEA's summary + # Use F040, keeping the ITA growth blends only for the detail split. The + # blends drift about 40% on 514, 561 and 55 by 2024 (#1250). The Step 5 + # balance then keeps those cells fixed. See + # nowcast_fd_conditioning.condition_service_exports_on_summary and + # nowcast_sut_assembly.fix_service_exports. + condition_service_exports_on_summary: bool = False # DRI: WesIngwersen # Smooth each manufacturing industry's share of the expense seed's survey # total across the survey changes (census 2017 -> ASM 2018-21 -> census # 2022 -> AIES): ASM shares benchmarked to both censuses, a 3-year centred