From 2b79c473e9ffd86cdc297554e7526d841980c378 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Fri, 13 Nov 2026 09:13:13 -0500 Subject: [PATCH 1/4] feat(nowcast): hold imports to BEA's summary groups and condition service exports on summary F040 (#1288, #1250) Two flags, both off: - hold_imports_on_summary_groups: the Step 5 balance holds MCIF by summary group (import_group_holds, the same GroupHold as TOP/SUB) instead of one soft column total. In 2017 service imports enter at the published $267.2bn and leave at $247.0bn. - condition_service_exports_on_summary: each service summary group of F04000 is scaled to BEA's summary Use F040; the ITA growth blends keep only the detail split. Goods exports stay on Census trade. Co-Authored-By: Claude Opus 5.5 --- .../iot/__tests__/test_derive_gross_output.py | 6 +- .../iot/__tests__/test_ec_go_adjustment.py | 6 +- .../test_mut_use_to_import_matrix.py | 6 +- .../__tests__/test_nowcast_fd_conditioning.py | 27 +++++++ .../test_nowcast_supply_go_control.py | 12 +-- .../__tests__/test_nowcast_tax_row_closer.py | 38 +++++++++ .../__tests__/test_nowcast_trade_margins.py | 6 +- bedrock/transform/iot/derived_price_index.py | 34 ++++---- .../iot/eia_utility_go_adjustment.py | 3 +- bedrock/transform/iot/helpers.py | 8 +- bedrock/transform/iot/nowcast.py | 8 ++ .../transform/iot/nowcast_fd_conditioning.py | 80 ++++++++++++++++--- bedrock/transform/iot/nowcast_mut.py | 6 +- .../transform/iot/nowcast_redefinitions.py | 5 +- bedrock/transform/iot/nowcast_subsidies.py | 6 +- .../iot/nowcast_supply_go_control.py | 2 +- bedrock/transform/iot/nowcast_sut_assembly.py | 49 +++++++++++- .../iot/nowcast_transport_margins.py | 2 +- bedrock/transform/iot/sut_use_to_mut_use.py | 2 +- bedrock/transform/iot/tax_subsidy_layer.py | 2 +- bedrock/transform/iot/use_price_bridge.py | 2 +- bedrock/utils/config/usa_config.py | 14 +++- 22 files changed, 255 insertions(+), 69 deletions(-) diff --git a/bedrock/transform/iot/__tests__/test_derive_gross_output.py b/bedrock/transform/iot/__tests__/test_derive_gross_output.py index 4cc8ab879..6ffe1e4e5 100644 --- a/bedrock/transform/iot/__tests__/test_derive_gross_output.py +++ b/bedrock/transform/iot/__tests__/test_derive_gross_output.py @@ -193,6 +193,6 @@ def test_2017_redefinition_roundtrip() -> None: pd.testing.assert_series_equal(gap, column_sum_gap, check_names=False, rtol=1e-10) # 3. Absolute per-industry error < $11M (≤ 10 in BEA's million-dollar units) - assert ( - gap.abs().max() <= 11_000_000 - ), f'Max abs gap {gap.abs().max():.0f} exceeds BEA rounding tolerance' + assert gap.abs().max() <= 11_000_000, ( + f'Max abs gap {gap.abs().max():.0f} exceeds BEA rounding tolerance' + ) diff --git a/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py b/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py index 773af247a..a1b109fd2 100644 --- a/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py +++ b/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py @@ -68,9 +68,9 @@ def test_the_chained_years_hold_the_manufacturing_total_not_the_groups( ), year got = adjusted.loc[members, year].groupby(parents[members]).sum() want = raw.loc[members, year].groupby(parents[members]).sum() - assert ( - float((got - want).abs().sum()) > 10_000.0 - ), f'{year}: no group moved; the chain is a no-op' + assert float((got - want).abs().sum()) > 10_000.0, ( + f'{year}: no group moved; the chain is a no-op' + ) def test_nothing_before_2022_and_nothing_outside_manufacturing_moves( diff --git a/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py b/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py index d52a49ac6..c098644d5 100644 --- a/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py +++ b/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py @@ -177,9 +177,9 @@ def test_f05000_is_written_negative_and_the_seven_columns_zero() -> None: ) assert table.loc[FIRST, USA_2017_FINAL_DEMAND_IMPORT_CODE] == pytest.approx(-400.0) - assert ( - table[list(ZERO_IMPORT_FINAL_DEMAND_CODES)].to_numpy() == 0.0 - ).all(), 'BEA measured these at zero; they belong in the frame as zeros' + assert (table[list(ZERO_IMPORT_FINAL_DEMAND_CODES)].to_numpy() == 0.0).all(), ( + 'BEA measured these at zero; they belong in the frame as zeros' + ) def test_the_matrix_has_the_axes_the_published_table_has() -> None: diff --git a/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py b/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py index f4f97560a..f701f707e 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py @@ -19,6 +19,8 @@ EMPTY_GROUP_USD, _commodity_to_summary, condition_fd_on_summary, + condition_service_exports_on_summary, + is_service_group, summary_condition_factors, ) @@ -81,3 +83,28 @@ def test_empty_and_sign_flipped_groups_hold_at_one(y: pd.DataFrame) -> None: def test_years_outside_the_workbook_are_refused(y: pd.DataFrame) -> None: with pytest.raises(ValueError, match='summary Use SUT'): summary_condition_factors(y, 2025) + + +# --- service exports (#1250) ------------------------------------------------ + + +def test_service_exports_land_on_the_published_rows_and_goods_stay() -> None: + groups = _commodity_to_summary() + exports = pd.Series(2.0e9, index=groups.index) + out = condition_service_exports_on_summary(exports, YEAR) + published = _load_usa_summary_sut('Use_SUT_summary', YEAR) # type: ignore[arg-type] + published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6 + ours = out.groupby(groups).sum() + for group in ('514', '561', '55', '5412OP'): + assert is_service_group(group) + assert ours[group] == pytest.approx(published.at[group, 'F040'], rel=1e-9) + goods = groups.map(lambda g: not is_service_group(g)) + pd.testing.assert_series_equal(out[goods], exports[goods]) + within = groups.eq('5412OP') + assert out[within].nunique() == 1 # the detail split inside a group holds + + +def test_service_groups_exclude_goods_government_and_accounting_rows() -> None: + assert not any(is_service_group(g) for g in ('111CA', '22', '23', '325', 'GFGD')) + assert not any(is_service_group(g) for g in ('Other', 'Used')) + assert all(is_service_group(g) for g in ('42', '481', '523', '81')) diff --git a/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py b/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py index 2b63f19c7..4f415b74b 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py @@ -148,9 +148,9 @@ def test_converged_columns_land_on_their_go_share(controlled: pd.DataFrame) -> N # note, so selecting on the note excluded all 19 manufacturing groups and # left this test passing while checking none of them. fitted_groups = set(diagnostics.index[~diagnostics['kept_seed']]) - assert ( - fitted_groups & gc.released_groups() - ), 'no released group reached the GO-share check; the selector is wrong again' + assert fitted_groups & gc.released_groups(), ( + 'no released group reached the GO-share check; the selector is wrong again' + ) wedge = gc._wedge(YEAR, controlled) target = gc.gross_output_at_basic(YEAR, wedge) @@ -256,9 +256,9 @@ def test_released_groups_is_the_union_over_every_conditioner() -> None: assert config.chain_manufacturing_on_aies assert config.rebase_utility_gross_output_on_eia - assert released == aies_released() | { - '22' - }, 'the union picked up a group no conditioner claims' + assert released == aies_released() | {'22'}, ( + 'the union picked up a group no conditioner claims' + ) # manufacturing's groups plus utilities, and nothing else assert all(group.startswith('3') or group == '22' for group in released), sorted( released diff --git a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py index d03edc2ef..f16a3b8ad 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py @@ -310,3 +310,41 @@ def test_a_group_whose_scale_would_flip_its_sign_keeps_its_seed( ) assert report.loc[('TOP', 'g1'), 'held'] == pytest.approx(2.0) assert any('flip their sign' in w for w in warned) + + +# --- the import hold (#1288) ------------------------------------------------ + + +def test_the_import_hold_targets_the_published_groups_less_frozen( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(asm, 'commodity_groups', lambda: GROUPS) + monkeypatch.setattr( + asm, 'published_import_groups', lambda year: pd.Series({'g1': 5.0, 'g2': 3.0}) + ) + frozen = pd.DataFrame({'i1': 0.0, 'MCIF': [0.0, 1.5, 0.0]}, index=GROUPS.index) + (hold,) = asm.import_group_holds(2017, frozen) + assert hold.column == 'MCIF' + assert hold.targets.to_dict() == pytest.approx({'g1': 3.5, 'g2': 3.0}) + + +def test_held_imports_keep_each_group_where_the_rows_would_move_them() -> None: + """The rows want c1 and c2 to lose 2 of supply and c3 to gain 2; without + the hold, imports move from g1 to g2; held, i1 absorbs it instead.""" + supply = pd.DataFrame( + {'i1': [4.0, 4.0, 2.0], 'MCIF': [2.0, 2.0, 2.0]}, index=['c1', 'c2', 'c3'] + ) + rows = pd.Series({'c1': 5.0, 'c2': 5.0, 'c3': 6.0}) + cols = pd.Series({'i1': 10.0, 'MCIF': 6.0}) + mask = SutMask.from_pattern(supply) + hold = GroupHold('MCIF', GROUPS, pd.Series({'g1': 4.0, 'g2': 2.0})) + free, _ = _balance_block( + supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True + ) + held, _ = _balance_block( + supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True, holds=(hold,) + ) + assert free.loc[['c1', 'c2'], 'MCIF'].sum() < 4.0 + assert held.loc[['c1', 'c2'], 'MCIF'].sum() == pytest.approx(4.0, abs=1e-6) + assert held.at['c3', 'MCIF'] == pytest.approx(2.0, abs=1e-6) + pd.testing.assert_series_equal(held.sum(axis=1), rows, check_names=False) diff --git a/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py b/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py index 94eed7c31..d371b74f6 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py @@ -414,9 +414,9 @@ def test_receiving_shares_are_frozen_at_2017() -> None: assert list(received.index) == list(anchor.index) assert (received / received.sum() - anchor_shares).abs().max() < 1e-12 - assert min(levels.values()) < max( - levels.values() - ), 'the levels are identical too, so the shares matching proves nothing' + assert min(levels.values()) < max(levels.values()), ( + 'the levels are identical too, so the shares matching proves nothing' + ) # --- #1194: receivers that follow their own supply ------------------------- diff --git a/bedrock/transform/iot/derived_price_index.py b/bedrock/transform/iot/derived_price_index.py index 332aa2466..032960b9e 100644 --- a/bedrock/transform/iot/derived_price_index.py +++ b/bedrock/transform/iot/derived_price_index.py @@ -47,18 +47,18 @@ #: release, so the final year is the mean of the quarters the summary table #: has published for it -- Q1-Q2 in a Q2 release, not a full year. END_YEAR_BY_VINTAGE: dict[BeaDataVersion, int] = { - "2025Q2": 2025, - "2026Q2": 2026, + '2025Q2': 2025, + '2026Q2': 2026, } -SECTOR_CODE_COL = "sector_code" +SECTOR_CODE_COL = 'sector_code' DISAGGREGATED_CODES = [ - "562111", # Solid waste collection - "562HAZ", # Hazardous waste collection treatment and disposal - "562212", # Solid waste landfilling - "562213", # Solid waste combustors and incinerators - "562910", # Remediation services - "562920", # Material separation/recovery facilities - "562OTH", # Other waste collection and treatment services + '562111', # Solid waste collection + '562HAZ', # Hazardous waste collection treatment and disposal + '562212', # Solid waste landfilling + '562213', # Solid waste combustors and incinerators + '562910', # Remediation services + '562920', # Material separation/recovery facilities + '562OTH', # Other waste collection and treatment services ] @@ -89,14 +89,14 @@ def _combine_pi_detail_summary(end_year: int, vintage: BeaDataVersion) -> pd.Dat pi_summary=_aggregate_latest_pi_summary_quarterly__annual(end_year, vintage) ), on=SECTOR_CODE_COL, - how="left", + how='left', ) YEAR_COLS = pi.columns.difference( pd.Index([SECTOR_CODE_COL, SECTOR_NAME_COL, SECTOR_SUMMARY_CODE_COL]) ) - waste_pi = pi[pi[SECTOR_CODE_COL] == "562000"] - assert waste_pi.shape[0] == 1, "found multiple aggregated waste rows" + waste_pi = pi[pi[SECTOR_CODE_COL] == '562000'] + assert waste_pi.shape[0] == 1, 'found multiple aggregated waste rows' waste_disaggregated = pd.concat( [waste_pi.assign(sector_code=code) for code in DISAGGREGATED_CODES], @@ -125,7 +125,7 @@ def _aggregate_detail_pi(vintage: BeaDataVersion) -> pd.DataFrame: .pipe(lambda ser: ser[ser > 1]) .index.tolist() ) - print(f"handling duplicated codes {duplicated_codes}") + print(f'handling duplicated codes {duplicated_codes}') pi_detail_reagg = pi_detail.loc[ ~pi_detail[SECTOR_CODE_COL].isin(duplicated_codes), @@ -148,11 +148,11 @@ def _aggregate_detail_pi(vintage: BeaDataVersion) -> pd.DataFrame: pi_agg = wgt_avg_pi.to_frame().transpose().assign(sector_code=code) pi_detail_reagg = pd.concat([pi_detail_reagg, pi_agg], ignore_index=True) - assert pi_detail_reagg[SECTOR_CODE_COL].is_unique, "found duplicate secotr codes" + assert pi_detail_reagg[SECTOR_CODE_COL].is_unique, 'found duplicate secotr codes' assert_sets_equal( - expected=set(BEA_2017_INDUSTRY_CODES) | {"335220"}, + expected=set(BEA_2017_INDUSTRY_CODES) | {'335220'}, actual=set(pi_detail_reagg[SECTOR_CODE_COL]), - if_fail="exception", + if_fail='exception', ) return pi_detail_reagg diff --git a/bedrock/transform/iot/eia_utility_go_adjustment.py b/bedrock/transform/iot/eia_utility_go_adjustment.py index bbe07ccfe..02dc82dd1 100644 --- a/bedrock/transform/iot/eia_utility_go_adjustment.py +++ b/bedrock/transform/iot/eia_utility_go_adjustment.py @@ -576,8 +576,7 @@ def check() -> int: for code in HELD_OUT: if code in CONTROLLED: # fixed list: the retail basis may add S00101, S00202 failures.append( - f'{code} is in HELD_OUT and must not be controlled: ' - f'{HELD_OUT[code]}' + f'{code} is in HELD_OUT and must not be controlled: {HELD_OUT[code]}' ) table = report() diff --git a/bedrock/transform/iot/helpers.py b/bedrock/transform/iot/helpers.py index 0c4b49898..d8206d1ae 100644 --- a/bedrock/transform/iot/helpers.py +++ b/bedrock/transform/iot/helpers.py @@ -8,14 +8,14 @@ load_bea_v2017_industry_to_bea_v2017_summary, ) -SECTOR_CODE_COL = "sector_code" +SECTOR_CODE_COL = 'sector_code' def map_pi_summary__detail(pi_summary: pd.DataFrame) -> pd.DataFrame: """Map BEA summary-level PI rows onto their corresponding detail sectors.""" summary__detail_mapping = pd.DataFrame.from_dict( load_bea_v2017_industry_to_bea_v2017_summary(), - orient="index", + orient='index', columns=[SECTOR_SUMMARY_CODE_COL], ) summary__detail_mapping[SECTOR_CODE_COL] = summary__detail_mapping.index @@ -23,7 +23,7 @@ def map_pi_summary__detail(pi_summary: pd.DataFrame) -> pd.DataFrame: return pi_summary.merge( summary__detail_mapping, on=SECTOR_SUMMARY_CODE_COL, - how="left", + how='left', ).set_index(SECTOR_NAME_COL) @@ -33,4 +33,4 @@ def map_detail_table(df: pd.DataFrame) -> pd.DataFrame: list(PRICE_INDEX_DETAIL_NAME_TO_BEA_2017_INDUSTRY_MAPPING.items()), columns=[SECTOR_NAME_COL, SECTOR_CODE_COL], ).explode(SECTOR_CODE_COL) - return df.merge(mapping, on=SECTOR_NAME_COL, how="left") + return df.merge(mapping, on=SECTOR_NAME_COL, how='left') diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index a9d0b44fb..9d077305c 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -494,6 +494,14 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), published_2017_exports(), ) + from bedrock.utils.config.usa_config import get_usa_config # noqa: PLC0415 + + if get_usa_config().condition_service_exports_on_summary: + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + condition_service_exports_on_summary, + ) + + exports = condition_service_exports_on_summary(exports, int(year)) y['F04000'] = exports.reindex(y.index).fillna(0.0) pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise')) y.loc['S00900', 'F04000'] = -pce + _s00900_export_identity_usd() diff --git a/bedrock/transform/iot/nowcast_fd_conditioning.py b/bedrock/transform/iot/nowcast_fd_conditioning.py index 4c42d54e9..d816f087f 100644 --- a/bedrock/transform/iot/nowcast_fd_conditioning.py +++ b/bedrock/transform/iot/nowcast_fd_conditioning.py @@ -28,10 +28,13 @@ targets otherwise) and large fixes on FD-owned rows — custom programming's 2023 residual fell 76 -> 30bn $M in the counterfactual. -⚠️ ``F03000`` and ``F04000`` are NOT conditioned. Inventories and exports are -their own sourced constructions (#746, #729/#771) with their own graders; +⚠️ ``F03000`` and ``F04000`` are NOT conditioned here. Inventories and exports +are their own sourced constructions (#746, #729/#771) with their own graders; their summary-level standing is recorded in the About doc, and exports are -bl-young's lane. +bl-young's lane. With ``condition_service_exports_on_summary`` set, the +**service** rows of ``F04000`` are conditioned the same way +(:func:`condition_service_exports_on_summary`, #1250); goods exports stay on +Census trade. Guards, in the order applied per (group, column): @@ -95,6 +98,68 @@ def _commodity_to_summary() -> pd.Series: return pd.Series({code: parents[0] for code, parents in mapping.items()}) +def _group_factors(ours: pd.Series, pub: pd.Series) -> tuple[pd.Series, float, float]: + """published / ours per group, with the guards; plus the USD each guard left.""" + empty = ours.abs() < EMPTY_GROUP_USD + flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0) + factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0) + factor[empty | flipped] = 1.0 + unreachable = float(pub[empty].abs().sum()) + sign_held = float((pub - ours)[flipped].abs().sum()) + return factor, unreachable, sign_held + + +#: Summary groups whose exports stay on Census goods trade: agriculture to +#: manufacturing, and government. ``Other`` and ``Used`` are accounting rows. +GOODS_SUMMARY_PREFIXES: tuple[str, ...] = ('111', '113', '21', '22', '23', '3', 'G') +ACCOUNTING_SUMMARY_ROWS: tuple[str, ...] = ('Other', 'Used') + + +def is_service_group(group: str) -> bool: + """Whether summary commodity *group* is a service row (``42`` onward).""" + return not str(group).startswith(GOODS_SUMMARY_PREFIXES) and ( + str(group) not in ACCOUNTING_SUMMARY_ROWS + ) + + +def condition_service_exports_on_summary(exports: pd.Series, year: int) -> pd.Series: + """*exports* with each service group scaled to the summary Use ``F040``, USD. + + Service exports are each row's published 2017 value moved by a blend of + ITA categories (#771). The blends drift by about 40% on data processing + (``514``), administrative and support (``561``) and management of + companies (``55``) by 2024, and the balance barely moves exports, so the + drift reaches the tables (#1250). BEA publishes the summary rows every + year, so each service group is held to them; the blends only split a + group between its detail rows. Goods rows and rows with no summary group + are returned unchanged, and the guards of :func:`summary_condition_factors` + apply. + """ + if year not in SUMMARY_SUT_YEARS: + raise ValueError( + f'the summary Use SUT covers {SUMMARY_SUT_YEARS[0]}-' + f'{SUMMARY_SUT_YEARS[-1]}; got {year}' + ) + groups = _commodity_to_summary().reindex(exports.index) + service = groups.map(lambda g: isinstance(g, str) and is_service_group(g)) + service = service.fillna(False).astype(bool) + published = _load_usa_summary_sut('Use_SUT_summary', year) # type: ignore[arg-type] + published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6 + ours = exports[service].groupby(groups[service]).sum() + pub = published['F040'].reindex(ours.index).fillna(0.0) + factor, unreachable, sign_held = _group_factors(ours, pub) + if unreachable or sign_held: + logger.info( + 'service export conditioning %s: %.0fM published unreachable (our ' + 'group empty), %.0fM held on sign disagreement', + year, + unreachable / 1e6, + sign_held / 1e6, + ) + per_row = groups.map(factor).where(service, 1.0).astype(float).fillna(1.0) + return exports * per_row + + def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame: """Summary group x column scale factors for ``y`` at ``year``, guards applied. @@ -119,13 +184,10 @@ def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame: # The summary workbooks truncate the codes to four characters # (F01000 -> F010). pub = published[column[:4]].reindex(ours.index).fillna(0.0) - empty = ours.abs() < EMPTY_GROUP_USD - flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0) - factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0) - factor[empty | flipped] = 1.0 + factor, missed, held = _group_factors(ours, pub) factors[column] = factor - unreachable += float(pub[empty].abs().sum()) - sign_held += float((pub - ours)[flipped].abs().sum()) + unreachable += missed + sign_held += held if unreachable or sign_held: logger.info( 'FD conditioning %s: %.0fM published unreachable (our group ' diff --git a/bedrock/transform/iot/nowcast_mut.py b/bedrock/transform/iot/nowcast_mut.py index 7353ada7e..a4d9f9b40 100644 --- a/bedrock/transform/iot/nowcast_mut.py +++ b/bedrock/transform/iot/nowcast_mut.py @@ -219,7 +219,7 @@ def _conform_use_sut(use: pd.DataFrame, rest_of_world_pce: float) -> pd.DataFram if rest_of_world_pce > 0: raise ValueError( f'rest-of-world PCE adjustment is {rest_of_world_pce:,.0f} USD; ' - 'it removes nonresidents\' spending, so it cannot be positive' + "it removes nonresidents' spending, so it cannot be positive" ) commodities = list(USA_2017_COMMODITY_CODES) unexpected = set(commodities) - set(use.index) - {'4200ID', 'S00900'} @@ -726,8 +726,8 @@ def _negative_producer_value( if enforce and float(-short.min()) > GATE_ATOL: raise AssertionError( f"{year} producers' value is ${-report['worst_usd_m']:,.3f}M below " - f"zero at {report['worst_cell']} with the margin cap on; the " - f"{good} margin given up exceeds what its buyers purchase, which " + f'zero at {report["worst_cell"]} with the margin cap on; the ' + f'{good} margin given up exceeds what its buyers purchase, which ' 'is a Step 5 balance to fix, not a placement' ) return report diff --git a/bedrock/transform/iot/nowcast_redefinitions.py b/bedrock/transform/iot/nowcast_redefinitions.py index 1382d09fb..5b923dc44 100644 --- a/bedrock/transform/iot/nowcast_redefinitions.py +++ b/bedrock/transform/iot/nowcast_redefinitions.py @@ -480,8 +480,7 @@ def _stored_before( matches = sorted(directory.glob(pattern), key=lambda p: p.stat().st_mtime) if not matches: raise FileNotFoundError( - f'no {pattern} in {directory}; ' - 'run bedrock.transform.iot.nowcast_mut first' + f'no {pattern} in {directory}; run bedrock.transform.iot.nowcast_mut first' ) return pd.read_parquet(matches[-1]), matches[-1].name @@ -556,7 +555,7 @@ def save_after_redef( 'before_inputs': before_inputs or [], 'primary_source_meta': source_lineage(before_inputs or [], directory), 'method': ( - 'Make: 2017 detail movement pattern on the year\'s own ' + "Make: 2017 detail movement pattern on the year's own " 'cells; Use: 2017 cell-ratio carry with commodity-row ' 'closure and VA column closure; FD invariant; imports ' 're-allocated along the after-redef rows (#816 rule)' diff --git a/bedrock/transform/iot/nowcast_subsidies.py b/bedrock/transform/iot/nowcast_subsidies.py index 408091e03..eb2ada769 100644 --- a/bedrock/transform/iot/nowcast_subsidies.py +++ b/bedrock/transform/iot/nowcast_subsidies.py @@ -266,13 +266,13 @@ def published_sub_by_group(year: int) -> pd.Series: raise ValueError( f'The {year} summary Supply SUB column is positive on ' f'{sorted(groups.index[groups > 0])}. BEA stores this column negative; ' - f'a positive cell means the workbook was read on the Use table\'s sign ' + f"a positive cell means the workbook was read on the Use table's sign " f'convention.' ) if abs(float(groups.sum()) - total) > _ROUNDING_TOLERANCE: raise ValueError( f'The {year} summary Supply SUB groups sum to {groups.sum():,.0f} USD ' - f'against the workbook\'s own T017 of {total:,.0f}. A group row is ' + f"against the workbook's own T017 of {total:,.0f}. A group row is " f'being dropped or double-read.' ) return groups.rename('SUB') @@ -320,7 +320,7 @@ def published_sub_by_commodity() -> pd.Series: raise ValueError( f'The published 2017 Supply SUB column is positive on ' f'{sorted(column.index[column > 0])}. BEA stores this column negative; ' - f'a positive cell means the workbook was read on the Use table\'s sign ' + f"a positive cell means the workbook was read on the Use table's sign " f'convention, which doubles the subsidy wedge in T015.' ) return column diff --git a/bedrock/transform/iot/nowcast_supply_go_control.py b/bedrock/transform/iot/nowcast_supply_go_control.py index 5a2b0fb5d..2a011d7d3 100644 --- a/bedrock/transform/iot/nowcast_supply_go_control.py +++ b/bedrock/transform/iot/nowcast_supply_go_control.py @@ -737,7 +737,7 @@ def go_controlled_supply_block( raise ValueError( f'the {year} tax wedge did not settle in {WEDGE_PASSES} passes; the last ' f'moved {moved:,.0f} USD on some industry against a tolerance of ' - f'{WEDGE_TOLERANCE:,.0f}. T00TOP is allocated on this block\'s market ' + f"{WEDGE_TOLERANCE:,.0f}. T00TOP is allocated on this block's market " f'shares, so target and answer chase each other - a wedge that will not ' f'settle means the fit is moving market shares far more than expected.' ) diff --git a/bedrock/transform/iot/nowcast_sut_assembly.py b/bedrock/transform/iot/nowcast_sut_assembly.py index ef5b2f060..e1e73407e 100644 --- a/bedrock/transform/iot/nowcast_sut_assembly.py +++ b/bedrock/transform/iot/nowcast_sut_assembly.py @@ -180,7 +180,7 @@ def resolve_pce_constraint( mode = get_usa_config().electricity_pce_constraint_mode if mode == 'none': raise ValueError( - "constrain_electricity_pce_cell=True requires " + 'constrain_electricity_pce_cell=True requires ' "electricity_pce_constraint_mode != 'none'" ) return mode @@ -914,14 +914,47 @@ def published_tax_groups(year: int) -> pd.DataFrame: ``SUB`` is negative, as BEA stores it and as the balance carries it. """ + return _published_supply_groups(int(year), HELD_TAX_COLUMNS) + + +def _published_supply_groups(year: int, columns: Sequence[str]) -> pd.DataFrame: + """Summary Supply *columns* for *year*, $M, by summary commodity.""" supply = _load_usa_summary_sut('Supply_summary', year) # type: ignore[arg-type] supply = supply.rename(columns=lambda c: str(c).strip()) groups = set(commodity_groups()) - frame = supply[list(HELD_TAX_COLUMNS)].apply(pd.to_numeric, errors='coerce') + frame = supply[list(columns)].apply(pd.to_numeric, errors='coerce') frame.index = [str(i).strip() for i in frame.index] return frame.loc[[g for g in frame.index if g in groups]].fillna(0.0) +#: The Supply import column the balance can hold to BEA's summary groups. +HELD_IMPORT_COLUMN = 'MCIF' + + +def published_import_groups(year: int) -> pd.Series: + """The summary Supply ``MCIF`` column for *year*, $M, by summary commodity.""" + return _published_supply_groups(int(year), (HELD_IMPORT_COLUMN,))[ + HELD_IMPORT_COLUMN + ] + + +def import_group_holds(year: int, frozen_supply: pd.DataFrame) -> tuple[GroupHold, ...]: + """The engine's ``MCIF`` group hold: published group sums less frozen, $M. + + #785 already scales each summary group of the seed's imports to BEA's + summary ``MCIF``, but the balance otherwise holds imports only as one soft + column total (T7), so the row identities move them between groups. In + 2017, service imports enter at the published $267.2bn and leave at + $247.0bn (#1288). Held by group, they keep the published sums and the + detail split inside each group stays free. + """ + groups = commodity_groups().reindex(frozen_supply.index) + frozen = frozen_supply[HELD_IMPORT_COLUMN].astype(float).groupby(groups).sum() + published = published_import_groups(int(year)) + target = published - frozen.reindex(published.index).fillna(0.0) + return (GroupHold(column=HELD_IMPORT_COLUMN, groups=groups, targets=target),) + + def commodity_groups() -> pd.Series: """Each detail commodity's summary commodity group.""" return pd.Series( @@ -1100,6 +1133,7 @@ def balance_year( constrain_electricity_pce_cell: bool | None = None, convert_tax_rows: bool = True, hold_tax_groups: bool = True, + hold_import_groups: bool | None = None, ) -> YearBalance: """Assemble and balance one year: split, offset, engine, restore. @@ -1110,7 +1144,11 @@ def balance_year( Use ``T00TOP``/``T00SUB`` rows stay a conversion of the Supply columns. *hold_tax_groups* holds ``TOP``/``SUB`` to the published summary groups, with the detail free inside each group (:func:`tax_group_holds`). + *hold_import_groups* does the same for ``MCIF`` (:func:`import_group_holds`); + it defaults to the ``hold_imports_on_summary_groups`` config flag. """ + if hold_import_groups is None: + hold_import_groups = get_usa_config().hold_imports_on_summary_groups mode = resolve_pce_constraint(pce_constraint, constrain_electricity_pce_cell) assembled = assemble( int(year), @@ -1135,7 +1173,12 @@ def run( pce_eia_band_m=band, tax_rows=tax_row_operator(year, frozen) if convert_tax_rows else None, group_holds=( - tax_group_holds(year, frozen['supply']) if hold_tax_groups else () + (tax_group_holds(year, frozen['supply']) if hold_tax_groups else ()) + + ( + import_group_holds(year, frozen['supply']) + if hold_import_groups + else () + ) ), ) return out, restore_fixed_blocks(out.blocks, frozen) diff --git a/bedrock/transform/iot/nowcast_transport_margins.py b/bedrock/transform/iot/nowcast_transport_margins.py index e335b54f0..9258b345a 100644 --- a/bedrock/transform/iot/nowcast_transport_margins.py +++ b/bedrock/transform/iot/nowcast_transport_margins.py @@ -1375,7 +1375,7 @@ def joint_mode_fit() -> pd.DataFrame: if float(unreached.sum()) > 0.01 * float(column_margin.sum()): raise ValueError( f'the joint fit cannot reach {unreached.sum():,.0f} USD of the ' - f'published column on the modes\' combined support - the mode ' + f"published column on the modes' combined support - the mode " f'crosswalks exclude commodities BEA says receive transport margin.' ) return fitted diff --git a/bedrock/transform/iot/sut_use_to_mut_use.py b/bedrock/transform/iot/sut_use_to_mut_use.py index 3584bb860..583dedff9 100644 --- a/bedrock/transform/iot/sut_use_to_mut_use.py +++ b/bedrock/transform/iot/sut_use_to_mut_use.py @@ -230,7 +230,7 @@ def v00200_row(va_block: pd.DataFrame) -> pd.Series: positive = sorted(values.index[flagged]) raise ValueError( f'{row} is positive on {len(positive)} industries ' - f'({positive[:5]}). This block is on BEA\'s published sign ' + f"({positive[:5]}). This block is on BEA's published sign " f'convention, where the subsidy rows are subtracted; the build ' f'stores them negative and adds them. Summing a BEA-signed ' f'block overstates V00200 by twice the subsidy total.' diff --git a/bedrock/transform/iot/tax_subsidy_layer.py b/bedrock/transform/iot/tax_subsidy_layer.py index 5774a7507..7d1f5dd95 100644 --- a/bedrock/transform/iot/tax_subsidy_layer.py +++ b/bedrock/transform/iot/tax_subsidy_layer.py @@ -488,7 +488,7 @@ def restrict_to_cells( if not shifted.empty: logger.warning( 'Sales tax of %s USD on %d commodities (largest %s, %s USD) has no ' - 'room in the target table\'s trade margins and moved to the ' + "room in the target table's trade margins and moved to the " 'producer-level tax (top_rest).', f'{shifted.sum():,.0f}', len(shifted), diff --git a/bedrock/transform/iot/use_price_bridge.py b/bedrock/transform/iot/use_price_bridge.py index 31120e178..716e01817 100644 --- a/bedrock/transform/iot/use_price_bridge.py +++ b/bedrock/transform/iot/use_price_bridge.py @@ -646,7 +646,7 @@ def _strip_sales_tax( f'{int(cells.to_numpy().sum())} cells, by ' f'{over.where(cells, 0.0).to_numpy().sum():,.0f} USD (first ' f'{cells.any(axis=1).idxmax()}), which would leave a negative ' - "tax-free margin; the fiscal layer was restricted to another " + 'tax-free margin; the fiscal layer was restricted to another ' "table's margins." ) out[kind] = amounts[kind] - tax diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index e985be69e..b88fbe615 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -304,6 +304,17 @@ class USAConfig(BaseModel): # and a purchaser-to-producer ratio of 0.011 (#1194). See # nowcast_trade_margins.supply_following_weight. trade_margin_receivers_follow_supply: bool = False # DRI: WesIngwersen + # Hold the Step 5 balance's imports (MCIF) to BEA's summary Supply MCIF by + # summary group, as TOP/SUB are held, instead of one soft column total. + # The row identities otherwise pull service imports off the published + # groups: 2017 enters at $267.2bn and leaves at $247.0bn (#1288). See + # nowcast_sut_assembly.import_group_holds. + hold_imports_on_summary_groups: bool = False # DRI: WesIngwersen + # Scale each service summary group of exports (F04000) to BEA's summary + # Use F040, keeping the ITA growth blends only for the detail split. The + # blends drift about 40% on 514, 561 and 55 by 2024 (#1250). See + # nowcast_fd_conditioning.condition_service_exports_on_summary. + condition_service_exports_on_summary: bool = False # DRI: WesIngwersen # Smooth each manufacturing industry's share of the expense seed's survey # total across the survey changes (census 2017 -> ASM 2018-21 -> census # 2022 -> AIES): ASM shares benchmarked to both censuses, a 3-year centred @@ -565,8 +576,7 @@ def _validate_electric_output_basis(self) -> USAConfig: and not self.electricity_accounting_residual ): raise ValueError( - 'electric_output_on_eia_level requires ' - 'electricity_accounting_residual' + 'electric_output_on_eia_level requires electricity_accounting_residual' ) if self.electricity_delivery_margin and ( self.implement_electricity_disaggregation From 98e0a7df6927b691b3b179139a44e27cb2451905 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Fri, 13 Nov 2026 09:15:21 -0500 Subject: [PATCH 2/4] Restore files that ruff format reformatted; keep black's formatting Co-Authored-By: Claude Opus 5.5 --- .../iot/__tests__/test_derive_gross_output.py | 6 ++-- .../iot/__tests__/test_ec_go_adjustment.py | 6 ++-- .../test_mut_use_to_import_matrix.py | 6 ++-- .../test_nowcast_supply_go_control.py | 12 +++---- .../__tests__/test_nowcast_trade_margins.py | 6 ++-- bedrock/transform/iot/derived_price_index.py | 34 +++++++++---------- .../iot/eia_utility_go_adjustment.py | 3 +- bedrock/transform/iot/helpers.py | 8 ++--- bedrock/transform/iot/nowcast_mut.py | 6 ++-- .../transform/iot/nowcast_redefinitions.py | 5 +-- bedrock/transform/iot/nowcast_subsidies.py | 6 ++-- .../iot/nowcast_supply_go_control.py | 2 +- bedrock/transform/iot/nowcast_sut_assembly.py | 2 +- .../iot/nowcast_transport_margins.py | 2 +- bedrock/transform/iot/sut_use_to_mut_use.py | 2 +- bedrock/transform/iot/tax_subsidy_layer.py | 2 +- bedrock/transform/iot/use_price_bridge.py | 2 +- bedrock/utils/config/usa_config.py | 3 +- 18 files changed, 58 insertions(+), 55 deletions(-) diff --git a/bedrock/transform/iot/__tests__/test_derive_gross_output.py b/bedrock/transform/iot/__tests__/test_derive_gross_output.py index 6ffe1e4e5..4cc8ab879 100644 --- a/bedrock/transform/iot/__tests__/test_derive_gross_output.py +++ b/bedrock/transform/iot/__tests__/test_derive_gross_output.py @@ -193,6 +193,6 @@ def test_2017_redefinition_roundtrip() -> None: pd.testing.assert_series_equal(gap, column_sum_gap, check_names=False, rtol=1e-10) # 3. Absolute per-industry error < $11M (≤ 10 in BEA's million-dollar units) - assert gap.abs().max() <= 11_000_000, ( - f'Max abs gap {gap.abs().max():.0f} exceeds BEA rounding tolerance' - ) + assert ( + gap.abs().max() <= 11_000_000 + ), f'Max abs gap {gap.abs().max():.0f} exceeds BEA rounding tolerance' diff --git a/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py b/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py index a1b109fd2..773af247a 100644 --- a/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py +++ b/bedrock/transform/iot/__tests__/test_ec_go_adjustment.py @@ -68,9 +68,9 @@ def test_the_chained_years_hold_the_manufacturing_total_not_the_groups( ), year got = adjusted.loc[members, year].groupby(parents[members]).sum() want = raw.loc[members, year].groupby(parents[members]).sum() - assert float((got - want).abs().sum()) > 10_000.0, ( - f'{year}: no group moved; the chain is a no-op' - ) + assert ( + float((got - want).abs().sum()) > 10_000.0 + ), f'{year}: no group moved; the chain is a no-op' def test_nothing_before_2022_and_nothing_outside_manufacturing_moves( diff --git a/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py b/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py index c098644d5..d52a49ac6 100644 --- a/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py +++ b/bedrock/transform/iot/__tests__/test_mut_use_to_import_matrix.py @@ -177,9 +177,9 @@ def test_f05000_is_written_negative_and_the_seven_columns_zero() -> None: ) assert table.loc[FIRST, USA_2017_FINAL_DEMAND_IMPORT_CODE] == pytest.approx(-400.0) - assert (table[list(ZERO_IMPORT_FINAL_DEMAND_CODES)].to_numpy() == 0.0).all(), ( - 'BEA measured these at zero; they belong in the frame as zeros' - ) + assert ( + table[list(ZERO_IMPORT_FINAL_DEMAND_CODES)].to_numpy() == 0.0 + ).all(), 'BEA measured these at zero; they belong in the frame as zeros' def test_the_matrix_has_the_axes_the_published_table_has() -> None: diff --git a/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py b/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py index 4f415b74b..2b63f19c7 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_supply_go_control.py @@ -148,9 +148,9 @@ def test_converged_columns_land_on_their_go_share(controlled: pd.DataFrame) -> N # note, so selecting on the note excluded all 19 manufacturing groups and # left this test passing while checking none of them. fitted_groups = set(diagnostics.index[~diagnostics['kept_seed']]) - assert fitted_groups & gc.released_groups(), ( - 'no released group reached the GO-share check; the selector is wrong again' - ) + assert ( + fitted_groups & gc.released_groups() + ), 'no released group reached the GO-share check; the selector is wrong again' wedge = gc._wedge(YEAR, controlled) target = gc.gross_output_at_basic(YEAR, wedge) @@ -256,9 +256,9 @@ def test_released_groups_is_the_union_over_every_conditioner() -> None: assert config.chain_manufacturing_on_aies assert config.rebase_utility_gross_output_on_eia - assert released == aies_released() | {'22'}, ( - 'the union picked up a group no conditioner claims' - ) + assert released == aies_released() | { + '22' + }, 'the union picked up a group no conditioner claims' # manufacturing's groups plus utilities, and nothing else assert all(group.startswith('3') or group == '22' for group in released), sorted( released diff --git a/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py b/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py index d371b74f6..94eed7c31 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_trade_margins.py @@ -414,9 +414,9 @@ def test_receiving_shares_are_frozen_at_2017() -> None: assert list(received.index) == list(anchor.index) assert (received / received.sum() - anchor_shares).abs().max() < 1e-12 - assert min(levels.values()) < max(levels.values()), ( - 'the levels are identical too, so the shares matching proves nothing' - ) + assert min(levels.values()) < max( + levels.values() + ), 'the levels are identical too, so the shares matching proves nothing' # --- #1194: receivers that follow their own supply ------------------------- diff --git a/bedrock/transform/iot/derived_price_index.py b/bedrock/transform/iot/derived_price_index.py index 032960b9e..332aa2466 100644 --- a/bedrock/transform/iot/derived_price_index.py +++ b/bedrock/transform/iot/derived_price_index.py @@ -47,18 +47,18 @@ #: release, so the final year is the mean of the quarters the summary table #: has published for it -- Q1-Q2 in a Q2 release, not a full year. END_YEAR_BY_VINTAGE: dict[BeaDataVersion, int] = { - '2025Q2': 2025, - '2026Q2': 2026, + "2025Q2": 2025, + "2026Q2": 2026, } -SECTOR_CODE_COL = 'sector_code' +SECTOR_CODE_COL = "sector_code" DISAGGREGATED_CODES = [ - '562111', # Solid waste collection - '562HAZ', # Hazardous waste collection treatment and disposal - '562212', # Solid waste landfilling - '562213', # Solid waste combustors and incinerators - '562910', # Remediation services - '562920', # Material separation/recovery facilities - '562OTH', # Other waste collection and treatment services + "562111", # Solid waste collection + "562HAZ", # Hazardous waste collection treatment and disposal + "562212", # Solid waste landfilling + "562213", # Solid waste combustors and incinerators + "562910", # Remediation services + "562920", # Material separation/recovery facilities + "562OTH", # Other waste collection and treatment services ] @@ -89,14 +89,14 @@ def _combine_pi_detail_summary(end_year: int, vintage: BeaDataVersion) -> pd.Dat pi_summary=_aggregate_latest_pi_summary_quarterly__annual(end_year, vintage) ), on=SECTOR_CODE_COL, - how='left', + how="left", ) YEAR_COLS = pi.columns.difference( pd.Index([SECTOR_CODE_COL, SECTOR_NAME_COL, SECTOR_SUMMARY_CODE_COL]) ) - waste_pi = pi[pi[SECTOR_CODE_COL] == '562000'] - assert waste_pi.shape[0] == 1, 'found multiple aggregated waste rows' + waste_pi = pi[pi[SECTOR_CODE_COL] == "562000"] + assert waste_pi.shape[0] == 1, "found multiple aggregated waste rows" waste_disaggregated = pd.concat( [waste_pi.assign(sector_code=code) for code in DISAGGREGATED_CODES], @@ -125,7 +125,7 @@ def _aggregate_detail_pi(vintage: BeaDataVersion) -> pd.DataFrame: .pipe(lambda ser: ser[ser > 1]) .index.tolist() ) - print(f'handling duplicated codes {duplicated_codes}') + print(f"handling duplicated codes {duplicated_codes}") pi_detail_reagg = pi_detail.loc[ ~pi_detail[SECTOR_CODE_COL].isin(duplicated_codes), @@ -148,11 +148,11 @@ def _aggregate_detail_pi(vintage: BeaDataVersion) -> pd.DataFrame: pi_agg = wgt_avg_pi.to_frame().transpose().assign(sector_code=code) pi_detail_reagg = pd.concat([pi_detail_reagg, pi_agg], ignore_index=True) - assert pi_detail_reagg[SECTOR_CODE_COL].is_unique, 'found duplicate secotr codes' + assert pi_detail_reagg[SECTOR_CODE_COL].is_unique, "found duplicate secotr codes" assert_sets_equal( - expected=set(BEA_2017_INDUSTRY_CODES) | {'335220'}, + expected=set(BEA_2017_INDUSTRY_CODES) | {"335220"}, actual=set(pi_detail_reagg[SECTOR_CODE_COL]), - if_fail='exception', + if_fail="exception", ) return pi_detail_reagg diff --git a/bedrock/transform/iot/eia_utility_go_adjustment.py b/bedrock/transform/iot/eia_utility_go_adjustment.py index 02dc82dd1..bbe07ccfe 100644 --- a/bedrock/transform/iot/eia_utility_go_adjustment.py +++ b/bedrock/transform/iot/eia_utility_go_adjustment.py @@ -576,7 +576,8 @@ def check() -> int: for code in HELD_OUT: if code in CONTROLLED: # fixed list: the retail basis may add S00101, S00202 failures.append( - f'{code} is in HELD_OUT and must not be controlled: {HELD_OUT[code]}' + f'{code} is in HELD_OUT and must not be controlled: ' + f'{HELD_OUT[code]}' ) table = report() diff --git a/bedrock/transform/iot/helpers.py b/bedrock/transform/iot/helpers.py index d8206d1ae..0c4b49898 100644 --- a/bedrock/transform/iot/helpers.py +++ b/bedrock/transform/iot/helpers.py @@ -8,14 +8,14 @@ load_bea_v2017_industry_to_bea_v2017_summary, ) -SECTOR_CODE_COL = 'sector_code' +SECTOR_CODE_COL = "sector_code" def map_pi_summary__detail(pi_summary: pd.DataFrame) -> pd.DataFrame: """Map BEA summary-level PI rows onto their corresponding detail sectors.""" summary__detail_mapping = pd.DataFrame.from_dict( load_bea_v2017_industry_to_bea_v2017_summary(), - orient='index', + orient="index", columns=[SECTOR_SUMMARY_CODE_COL], ) summary__detail_mapping[SECTOR_CODE_COL] = summary__detail_mapping.index @@ -23,7 +23,7 @@ def map_pi_summary__detail(pi_summary: pd.DataFrame) -> pd.DataFrame: return pi_summary.merge( summary__detail_mapping, on=SECTOR_SUMMARY_CODE_COL, - how='left', + how="left", ).set_index(SECTOR_NAME_COL) @@ -33,4 +33,4 @@ def map_detail_table(df: pd.DataFrame) -> pd.DataFrame: list(PRICE_INDEX_DETAIL_NAME_TO_BEA_2017_INDUSTRY_MAPPING.items()), columns=[SECTOR_NAME_COL, SECTOR_CODE_COL], ).explode(SECTOR_CODE_COL) - return df.merge(mapping, on=SECTOR_NAME_COL, how='left') + return df.merge(mapping, on=SECTOR_NAME_COL, how="left") diff --git a/bedrock/transform/iot/nowcast_mut.py b/bedrock/transform/iot/nowcast_mut.py index a4d9f9b40..7353ada7e 100644 --- a/bedrock/transform/iot/nowcast_mut.py +++ b/bedrock/transform/iot/nowcast_mut.py @@ -219,7 +219,7 @@ def _conform_use_sut(use: pd.DataFrame, rest_of_world_pce: float) -> pd.DataFram if rest_of_world_pce > 0: raise ValueError( f'rest-of-world PCE adjustment is {rest_of_world_pce:,.0f} USD; ' - "it removes nonresidents' spending, so it cannot be positive" + 'it removes nonresidents\' spending, so it cannot be positive' ) commodities = list(USA_2017_COMMODITY_CODES) unexpected = set(commodities) - set(use.index) - {'4200ID', 'S00900'} @@ -726,8 +726,8 @@ def _negative_producer_value( if enforce and float(-short.min()) > GATE_ATOL: raise AssertionError( f"{year} producers' value is ${-report['worst_usd_m']:,.3f}M below " - f'zero at {report["worst_cell"]} with the margin cap on; the ' - f'{good} margin given up exceeds what its buyers purchase, which ' + f"zero at {report['worst_cell']} with the margin cap on; the " + f"{good} margin given up exceeds what its buyers purchase, which " 'is a Step 5 balance to fix, not a placement' ) return report diff --git a/bedrock/transform/iot/nowcast_redefinitions.py b/bedrock/transform/iot/nowcast_redefinitions.py index 5b923dc44..1382d09fb 100644 --- a/bedrock/transform/iot/nowcast_redefinitions.py +++ b/bedrock/transform/iot/nowcast_redefinitions.py @@ -480,7 +480,8 @@ def _stored_before( matches = sorted(directory.glob(pattern), key=lambda p: p.stat().st_mtime) if not matches: raise FileNotFoundError( - f'no {pattern} in {directory}; run bedrock.transform.iot.nowcast_mut first' + f'no {pattern} in {directory}; ' + 'run bedrock.transform.iot.nowcast_mut first' ) return pd.read_parquet(matches[-1]), matches[-1].name @@ -555,7 +556,7 @@ def save_after_redef( 'before_inputs': before_inputs or [], 'primary_source_meta': source_lineage(before_inputs or [], directory), 'method': ( - "Make: 2017 detail movement pattern on the year's own " + 'Make: 2017 detail movement pattern on the year\'s own ' 'cells; Use: 2017 cell-ratio carry with commodity-row ' 'closure and VA column closure; FD invariant; imports ' 're-allocated along the after-redef rows (#816 rule)' diff --git a/bedrock/transform/iot/nowcast_subsidies.py b/bedrock/transform/iot/nowcast_subsidies.py index eb2ada769..408091e03 100644 --- a/bedrock/transform/iot/nowcast_subsidies.py +++ b/bedrock/transform/iot/nowcast_subsidies.py @@ -266,13 +266,13 @@ def published_sub_by_group(year: int) -> pd.Series: raise ValueError( f'The {year} summary Supply SUB column is positive on ' f'{sorted(groups.index[groups > 0])}. BEA stores this column negative; ' - f"a positive cell means the workbook was read on the Use table's sign " + f'a positive cell means the workbook was read on the Use table\'s sign ' f'convention.' ) if abs(float(groups.sum()) - total) > _ROUNDING_TOLERANCE: raise ValueError( f'The {year} summary Supply SUB groups sum to {groups.sum():,.0f} USD ' - f"against the workbook's own T017 of {total:,.0f}. A group row is " + f'against the workbook\'s own T017 of {total:,.0f}. A group row is ' f'being dropped or double-read.' ) return groups.rename('SUB') @@ -320,7 +320,7 @@ def published_sub_by_commodity() -> pd.Series: raise ValueError( f'The published 2017 Supply SUB column is positive on ' f'{sorted(column.index[column > 0])}. BEA stores this column negative; ' - f"a positive cell means the workbook was read on the Use table's sign " + f'a positive cell means the workbook was read on the Use table\'s sign ' f'convention, which doubles the subsidy wedge in T015.' ) return column diff --git a/bedrock/transform/iot/nowcast_supply_go_control.py b/bedrock/transform/iot/nowcast_supply_go_control.py index 2a011d7d3..5a2b0fb5d 100644 --- a/bedrock/transform/iot/nowcast_supply_go_control.py +++ b/bedrock/transform/iot/nowcast_supply_go_control.py @@ -737,7 +737,7 @@ def go_controlled_supply_block( raise ValueError( f'the {year} tax wedge did not settle in {WEDGE_PASSES} passes; the last ' f'moved {moved:,.0f} USD on some industry against a tolerance of ' - f"{WEDGE_TOLERANCE:,.0f}. T00TOP is allocated on this block's market " + f'{WEDGE_TOLERANCE:,.0f}. T00TOP is allocated on this block\'s market ' f'shares, so target and answer chase each other - a wedge that will not ' f'settle means the fit is moving market shares far more than expected.' ) diff --git a/bedrock/transform/iot/nowcast_sut_assembly.py b/bedrock/transform/iot/nowcast_sut_assembly.py index e1e73407e..fd06a228b 100644 --- a/bedrock/transform/iot/nowcast_sut_assembly.py +++ b/bedrock/transform/iot/nowcast_sut_assembly.py @@ -180,7 +180,7 @@ def resolve_pce_constraint( mode = get_usa_config().electricity_pce_constraint_mode if mode == 'none': raise ValueError( - 'constrain_electricity_pce_cell=True requires ' + "constrain_electricity_pce_cell=True requires " "electricity_pce_constraint_mode != 'none'" ) return mode diff --git a/bedrock/transform/iot/nowcast_transport_margins.py b/bedrock/transform/iot/nowcast_transport_margins.py index 9258b345a..e335b54f0 100644 --- a/bedrock/transform/iot/nowcast_transport_margins.py +++ b/bedrock/transform/iot/nowcast_transport_margins.py @@ -1375,7 +1375,7 @@ def joint_mode_fit() -> pd.DataFrame: if float(unreached.sum()) > 0.01 * float(column_margin.sum()): raise ValueError( f'the joint fit cannot reach {unreached.sum():,.0f} USD of the ' - f"published column on the modes' combined support - the mode " + f'published column on the modes\' combined support - the mode ' f'crosswalks exclude commodities BEA says receive transport margin.' ) return fitted diff --git a/bedrock/transform/iot/sut_use_to_mut_use.py b/bedrock/transform/iot/sut_use_to_mut_use.py index 583dedff9..3584bb860 100644 --- a/bedrock/transform/iot/sut_use_to_mut_use.py +++ b/bedrock/transform/iot/sut_use_to_mut_use.py @@ -230,7 +230,7 @@ def v00200_row(va_block: pd.DataFrame) -> pd.Series: positive = sorted(values.index[flagged]) raise ValueError( f'{row} is positive on {len(positive)} industries ' - f"({positive[:5]}). This block is on BEA's published sign " + f'({positive[:5]}). This block is on BEA\'s published sign ' f'convention, where the subsidy rows are subtracted; the build ' f'stores them negative and adds them. Summing a BEA-signed ' f'block overstates V00200 by twice the subsidy total.' diff --git a/bedrock/transform/iot/tax_subsidy_layer.py b/bedrock/transform/iot/tax_subsidy_layer.py index 7d1f5dd95..5774a7507 100644 --- a/bedrock/transform/iot/tax_subsidy_layer.py +++ b/bedrock/transform/iot/tax_subsidy_layer.py @@ -488,7 +488,7 @@ def restrict_to_cells( if not shifted.empty: logger.warning( 'Sales tax of %s USD on %d commodities (largest %s, %s USD) has no ' - "room in the target table's trade margins and moved to the " + 'room in the target table\'s trade margins and moved to the ' 'producer-level tax (top_rest).', f'{shifted.sum():,.0f}', len(shifted), diff --git a/bedrock/transform/iot/use_price_bridge.py b/bedrock/transform/iot/use_price_bridge.py index 716e01817..31120e178 100644 --- a/bedrock/transform/iot/use_price_bridge.py +++ b/bedrock/transform/iot/use_price_bridge.py @@ -646,7 +646,7 @@ def _strip_sales_tax( f'{int(cells.to_numpy().sum())} cells, by ' f'{over.where(cells, 0.0).to_numpy().sum():,.0f} USD (first ' f'{cells.any(axis=1).idxmax()}), which would leave a negative ' - 'tax-free margin; the fiscal layer was restricted to another ' + "tax-free margin; the fiscal layer was restricted to another " "table's margins." ) out[kind] = amounts[kind] - tax diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index b88fbe615..6ca34a458 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -576,7 +576,8 @@ def _validate_electric_output_basis(self) -> USAConfig: and not self.electricity_accounting_residual ): raise ValueError( - 'electric_output_on_eia_level requires electricity_accounting_residual' + 'electric_output_on_eia_level requires ' + 'electricity_accounting_residual' ) if self.electricity_delivery_margin and ( self.implement_electricity_disaggregation From f95e14c3d03ed1af84afc76e52b53243740048f0 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Fri, 13 Nov 2026 09:22:01 -0500 Subject: [PATCH 3/4] Fix the service export cells in the Step 5 balance when they are conditioned (#1250) Co-Authored-By: Claude Opus 5.5 --- .../__tests__/test_nowcast_tax_row_closer.py | 23 ++++++++++++ bedrock/transform/iot/nowcast_sut_assembly.py | 37 +++++++++++++++++++ bedrock/utils/config/usa_config.py | 6 ++- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py index f16a3b8ad..a1c675dba 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py @@ -348,3 +348,26 @@ def test_held_imports_keep_each_group_where_the_rows_would_move_them() -> None: assert held.loc[['c1', 'c2'], 'MCIF'].sum() == pytest.approx(4.0, abs=1e-6) assert held.at['c3', 'MCIF'] == pytest.approx(2.0, abs=1e-6) pd.testing.assert_series_equal(held.sum(axis=1), rows, check_names=False) + + +# --- fixed service exports (#1250) ------------------------------------------ + + +def test_service_export_cells_are_fixed_and_goods_stay_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + asm, + 'commodity_groups', + lambda: pd.Series({'c1': '514', 'c2': '325', 'c3': '561'}), + ) + use = pd.DataFrame( + {'i1': [1.0, 1.0, 1.0], 'F04000': [2.0, 3.0, 0.0]}, index=['c1', 'c2', 'c3'] + ) + zero = pd.DataFrame(False, index=use.index, columns=use.columns) + mask = SutMask( + structural_zero=zero, fixed_value=zero.copy(), sign_lock=zero.astype(int) + ) + fixed = asm.fix_service_exports(mask, use).fixed_value + assert fixed['F04000'].to_dict() == {'c1': True, 'c2': False, 'c3': True} + assert not fixed['i1'].any() diff --git a/bedrock/transform/iot/nowcast_sut_assembly.py b/bedrock/transform/iot/nowcast_sut_assembly.py index fd06a228b..fe8971390 100644 --- a/bedrock/transform/iot/nowcast_sut_assembly.py +++ b/bedrock/transform/iot/nowcast_sut_assembly.py @@ -1068,6 +1068,38 @@ def rows(z_supply: pd.DataFrame) -> pd.DataFrame: return rows +#: The Use column whose service cells are fixed when exports are conditioned. +EXPORT_COLUMN = 'F04000' + + +def fix_service_exports(mask: SutMask, use: pd.DataFrame) -> SutMask: + """New Use mask with the service ``F04000`` cells Tier-1 fixed against *use*. + + With ``condition_service_exports_on_summary``, the seed's service exports + already sit on BEA's summary ``F040`` groups (#1250). The balance + otherwise moves them freely: on ``v1.0.0_fd271dc`` it took pipelines' + (``486``) error from 47 to 527 $M a year, and banking's (``521CI``) from + 51 to 673. Fixed, they leave the balance as conditioned, and the rows + close on their other cells. Structural zeros stay structural. + """ + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + is_service_group, + ) + + groups = commodity_groups().reindex(mask.index) + service = groups.map(lambda g: isinstance(g, str) and is_service_group(g)) + rows = service.fillna(False).astype(bool) & ~mask.structural_zero[EXPORT_COLUMN] + fixed = mask.fixed_value.copy() + fixed.loc[rows, EXPORT_COLUMN] = True + out = SutMask( + structural_zero=mask.structural_zero, + fixed_value=fixed, + sign_lock=mask.sign_lock, + ) + out.validate_against(use) + return out + + def assemble( year: int, *, @@ -1090,6 +1122,9 @@ def assemble( When ``pin_refinery_secondaries_at_go_mix`` is on, the Supply seed is then rewritten so held co-product cells enter GRAS at seed share times the T17-at-seed column pin, and Tier-1 is overlaid from that held list. + + When ``condition_service_exports_on_summary`` is on, the service export + cells are Tier-1 fixed too (:func:`fix_service_exports`). """ # Gross output and the summary supply-use controls must be one BEA annual # update; the targets balance one against the other (#1157). @@ -1110,6 +1145,8 @@ def assemble( seeds, masks, pin = apply_refinery_secondaries_go_mix_pin( int(year), seeds, masks, targets ) + if get_usa_config().condition_service_exports_on_summary: + masks = {**masks, 'use': fix_service_exports(masks['use'], seeds['use'])} return YearBalance( year=int(year), seeds=seeds, diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index 6ca34a458..53672f2f1 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -312,8 +312,10 @@ class USAConfig(BaseModel): hold_imports_on_summary_groups: bool = False # DRI: WesIngwersen # Scale each service summary group of exports (F04000) to BEA's summary # Use F040, keeping the ITA growth blends only for the detail split. The - # blends drift about 40% on 514, 561 and 55 by 2024 (#1250). See - # nowcast_fd_conditioning.condition_service_exports_on_summary. + # blends drift about 40% on 514, 561 and 55 by 2024 (#1250). The Step 5 + # balance then keeps those cells fixed. See + # nowcast_fd_conditioning.condition_service_exports_on_summary and + # nowcast_sut_assembly.fix_service_exports. condition_service_exports_on_summary: bool = False # DRI: WesIngwersen # Smooth each manufacturing industry's share of the expense seed's survey # total across the survey changes (census 2017 -> ASM 2018-21 -> census From 71be7b472e8b9f2bc9fc5491b5008d3081651df3 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Fri, 13 Nov 2026 13:08:40 -0500 Subject: [PATCH 4/4] Grade service trade on the same service cut the build conditions on (#1303 review) service_rows() now calls nowcast_fd_conditioning.is_service_group, so the grader and the export conditioning can't disagree on what a service is. Same 40 summary rows as before. Co-Authored-By: Claude Opus 5.5 --- .../trade_data/service_trade_vs_published.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py b/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py index 76dbcebc1..8805e5ee9 100644 --- a/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py +++ b/bedrock/analysis/nowcasting/trade_data/service_trade_vs_published.py @@ -175,13 +175,18 @@ def _to_summary() -> pd.Series: def service_rows() -> list[str]: - """Summary commodity rows from ``42`` onward, without government.""" - goods = ('111', '113', '21', '22', '23', '3') - return sorted( - {s for s in _to_summary() if not s.startswith(goods) and not s.startswith('G')} - - set(SEPARATE_ROWS) + """Summary commodity rows from ``42`` onward, without government. + + The same cut the build conditions service exports on + (:func:`~bedrock.transform.iot.nowcast_fd_conditioning.is_service_group`), + so the grader and the conditioning can't disagree on what a service is. + """ + from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415 + is_service_group, ) + return sorted({s for s in _to_summary() if is_service_group(s)}) + def default_build() -> str: """The balanced build the canonical config ships, e.g. ``v1.0.0_5514c3b``."""