From 11440e0d1e48b509d497a7cb6e284b8bddb3906e Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:26:03 -0500 Subject: [PATCH 1/6] feat(nab): scrap correction for the national balance (#1270) derive_Vnorm_scrap_corrected divides each industry's market shares by 1 - s, so A q counts its purchases at U / (1 - s) and the Use row no longer adds up to q. derive_scrap_correction returns the difference by commodity, -sum_j U[i, j] s_j / (1 - s_j), and derive_y_dom_from_use_final_demand includes it. The scrap share moves to _scrap_share, shared with derive_Vnorm_scrap_corrected (no change to A). On v1.0.0_8365fa4 it clears 562HAZ (1.0% and 1.6% in 2022-23) and the iron and steel gap. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 67 +++++++++++++++++++------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 61ed7a9c0..d04dc8442 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -308,6 +308,23 @@ def derive_q() -> pd.Series[float]: return q +def _scrap_share(industries: pd.Index, x: pd.Series[float]) -> pd.Series[float]: + """Each industry's scrap output (BEA's ``S00401``) as a share of its output.""" + scrap_detail = load_detail_V_usa().loc[:, 'S00401'] + scrap_fraction = industry_corresp() @ scrap_detail + if get_usa_config().implement_electricity_disaggregation: + parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap_fraction = scrap_fraction.drop( + ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' + ) + for code in ELECTRICITY_DISAGG_SECTORS: + scrap_fraction.loc[code] = parent_scrap + # The delivery sector has no scrap: the reindex below gives it zero. + scrap_fraction = scrap_fraction.reindex(industries, fill_value=0.0) + x_aligned = x.reindex(industries, fill_value=0.0) + return (scrap_fraction / x_aligned).fillna(0.0) + + @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -331,22 +348,7 @@ def derive_Vnorm_scrap_corrected( Vnorm = compute_Vnorm_matrix(V=V, q=q) - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) - x_aligned = x.reindex(V.index, fill_value=0.0) - - V_scrap_corrected = Vnorm.divide( - (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 - ) + V_scrap_corrected = Vnorm.divide(1.0 - _scrap_share(V.index, x), axis=0) V_scrap_corrected = V_scrap_corrected.reindex( index=V.index, columns=V.columns, fill_value=0.0 ) @@ -1035,6 +1037,31 @@ def derive_negative_purchase_sales() -> pd.Series[float]: return out +def derive_scrap_correction() -> pd.Series[float]: + """By commodity, the scrap correction for the national balance. + + ``derive_Vnorm_scrap_corrected`` takes scrap (``S00401``) out of each + industry's output and charges all of its inputs to its other products, + dividing its market shares by ``1 - s``, with ``s`` its scrap share. So + ``A q`` counts industry ``j``'s purchases of commodity ``i`` at + ``U[i, j] / (1 - s_j)``, not ``U[i, j]``, and the Use row no longer adds up + to ``q``. This returns the difference, ``-sum_j U[i, j] s_j / (1 - s_j)``, + for the final demand the national balance reads. Values are zero or + negative. + + It is a model correction, not a transaction, so it is not a published + final-demand column. It is largest where scrap-making industries buy + heavily: hazardous waste (``562HAZ``), which buys mainly from itself, and + iron and steel (``331110``) (#1270). + """ + V = derive_V() + s = _scrap_share(V.index, compute_x(V=V)) + Udom = derive_U_set().Udom + out = -Udom.mul((s / (1.0 - s)).reindex(Udom.columns, fill_value=0.0)).sum(axis=1) + out.index.name = 'sector' + return out + + def derive_y_dom_from_use_final_demand() -> pd.Series[float]: """Domestic final demand plus exports, read from the Use table. @@ -1045,13 +1072,17 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. - It also includes ``derive_negative_purchase_sales``: the sales BEA records - as negative intermediate purchases, which ``derive_U_set`` takes out of U. + It also includes two corrections for what the model does to U: + ``derive_negative_purchase_sales``, the sales BEA records as negative + intermediate purchases, which ``derive_U_set`` takes out of U; and + ``derive_scrap_correction``, for the inputs the scrap correction in A moves + onto scrap-making industries' other products. """ Y = derive_Ytot_full_cs_matrix() Uimp = derive_U_with_negatives().Uimp y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) y = y.add(derive_negative_purchase_sales(), fill_value=0.0) + y = y.add(derive_scrap_correction(), fill_value=0.0) y.index.name = 'sector' return y From 15407647c89f3332152f9129c1fcaca551ccfe24 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:26:05 -0500 Subject: [PATCH 2/6] test(nab): leave BEA's accounting sectors out of q = L y, as USEEIO 2 does (#1270) ACCOUNTING_SECTORS (scrap, used goods, noncomparable imports, the rest-of-world adjustment) have no output of their own, so the check can't hold for them. compare_output_vs_leontief_x_demand still computes L @ y on every sector and compares the rest. The xfail reason now quotes the one 2024 failure, 334118; 2017-2023 pass. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/test_eeio_diagnostics.py | 21 +++++++++++++++++-- bedrock/utils/validation/eeio_diagnostics.py | 16 ++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 0a45aebfd..59bd9eabc 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -417,8 +417,8 @@ def test_compare_output_and_L_y( @pytest.mark.xfail( reason=( 'Make and Use disagree on commodity output. On MUT v1.0.0_8365fa4 ' - '(2024), with the F09000 sales in y: 334118 (+4.4%, imports above ' - 'use) and S00402 ($4M on q = 0).' + '(2024) one sector fails: 334118 (+4.4%), whose imports exceed its ' + 'use. 2017-2023 pass.' ), ) def test_domestic_L_y_from_use_final_demand_equals_q( @@ -600,3 +600,20 @@ def test_total_lci_equals_e(canonical_usa_config: None) -> None: f'total LCI≈E failed ({len(result.failing_sectors)} cells): ' f'{result.failing_sectors[:20]}' ) + + +def test_output_vs_leontief_skips_the_accounting_sectors() -> None: + """S00402 has no output, so q = L @ y isn't checked for it (#1270).""" + sectors = ['1111A0', 'S00402'] + L = pd.DataFrame(np.eye(2), index=sectors, columns=sectors) + q = pd.Series([100.0, 0.0], index=sectors) + + off_on_used_goods = compare_output_vs_leontief_x_demand( + output=q, L=L, y=pd.Series([100.0, -6.0e6], index=sectors) + ) + assert off_on_used_goods.passed + + off_on_oilseeds = compare_output_vs_leontief_x_demand( + output=q, L=L, y=pd.Series([103.0, 0.0], index=sectors) + ) + assert not off_on_oilseeds.passed diff --git a/bedrock/utils/validation/eeio_diagnostics.py b/bedrock/utils/validation/eeio_diagnostics.py index de24772d9..3d0c56639 100644 --- a/bedrock/utils/validation/eeio_diagnostics.py +++ b/bedrock/utils/validation/eeio_diagnostics.py @@ -28,6 +28,12 @@ # (e.g. S00402 used goods) compare |diff| against atol instead of (diff / q). _VALIDATE_RESULT_ATOL = 1e-4 +# BEA's accounting rows: scrap, used and secondhand goods, noncomparable +# imports and the rest-of-world adjustment. They have no output of their own +# (S00402's q is 0), so q = L @ y can't hold for them and isn't checked, as in +# USEEIO 2. Of the four, only S00402 is a Cornerstone commodity today (#1270). +ACCOUNTING_SECTORS: frozenset[str] = frozenset({'S00300', 'S00401', 'S00402', 'S00900'}) + @dc.dataclass class DiagnosticResult: @@ -347,7 +353,7 @@ def compare_output_vs_leontief_x_demand( Compares the total sector output (commodity or industry) against the model result calculation of L @ y. Pass/fail: ``|diff| <= tolerance * |value| + atol`` for all sectors (see - ``validate_result``). + ``validate_result``) except the ``ACCOUNTING_SECTORS``, which aren't checked. Parameters ---------- @@ -387,8 +393,14 @@ def compare_output_vs_leontief_x_demand( output_check = backcompute_q_from_L_and_y(L=L, y=y) name = "compare output and L * y" + # L @ y uses every sector; only the comparison leaves the accounting rows out. + checked = output.index.difference(list(ACCOUNTING_SECTORS), sort=False) d_result = validate_result( - name, output, output_check, tolerance=tolerance, include_details=include_details + name, + output.loc[checked], + output_check.loc[checked], + tolerance=tolerance, + include_details=include_details, ) return d_result From c6acb8e8073aa0cfe50e90e06a2016305b9983b6 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:51:03 -0500 Subject: [PATCH 3/6] Revert the national-balance scrap correction (#1293) It offset a double count in A and B instead of fixing it. #1293 fixes the scrap correction itself. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 67 +++++++------------------- 1 file changed, 18 insertions(+), 49 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index d04dc8442..61ed7a9c0 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -308,23 +308,6 @@ def derive_q() -> pd.Series[float]: return q -def _scrap_share(industries: pd.Index, x: pd.Series[float]) -> pd.Series[float]: - """Each industry's scrap output (BEA's ``S00401``) as a share of its output.""" - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(industries, fill_value=0.0) - x_aligned = x.reindex(industries, fill_value=0.0) - return (scrap_fraction / x_aligned).fillna(0.0) - - @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -348,7 +331,22 @@ def derive_Vnorm_scrap_corrected( Vnorm = compute_Vnorm_matrix(V=V, q=q) - V_scrap_corrected = Vnorm.divide(1.0 - _scrap_share(V.index, x), axis=0) + scrap_detail = load_detail_V_usa().loc[:, 'S00401'] + scrap_fraction = industry_corresp() @ scrap_detail + if get_usa_config().implement_electricity_disaggregation: + parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap_fraction = scrap_fraction.drop( + ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' + ) + for code in ELECTRICITY_DISAGG_SECTORS: + scrap_fraction.loc[code] = parent_scrap + # The delivery sector has no scrap: the reindex below gives it zero. + scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) + x_aligned = x.reindex(V.index, fill_value=0.0) + + V_scrap_corrected = Vnorm.divide( + (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 + ) V_scrap_corrected = V_scrap_corrected.reindex( index=V.index, columns=V.columns, fill_value=0.0 ) @@ -1037,31 +1035,6 @@ def derive_negative_purchase_sales() -> pd.Series[float]: return out -def derive_scrap_correction() -> pd.Series[float]: - """By commodity, the scrap correction for the national balance. - - ``derive_Vnorm_scrap_corrected`` takes scrap (``S00401``) out of each - industry's output and charges all of its inputs to its other products, - dividing its market shares by ``1 - s``, with ``s`` its scrap share. So - ``A q`` counts industry ``j``'s purchases of commodity ``i`` at - ``U[i, j] / (1 - s_j)``, not ``U[i, j]``, and the Use row no longer adds up - to ``q``. This returns the difference, ``-sum_j U[i, j] s_j / (1 - s_j)``, - for the final demand the national balance reads. Values are zero or - negative. - - It is a model correction, not a transaction, so it is not a published - final-demand column. It is largest where scrap-making industries buy - heavily: hazardous waste (``562HAZ``), which buys mainly from itself, and - iron and steel (``331110``) (#1270). - """ - V = derive_V() - s = _scrap_share(V.index, compute_x(V=V)) - Udom = derive_U_set().Udom - out = -Udom.mul((s / (1.0 - s)).reindex(Udom.columns, fill_value=0.0)).sum(axis=1) - out.index.name = 'sector' - return out - - def derive_y_dom_from_use_final_demand() -> pd.Series[float]: """Domestic final demand plus exports, read from the Use table. @@ -1072,17 +1045,13 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. - It also includes two corrections for what the model does to U: - ``derive_negative_purchase_sales``, the sales BEA records as negative - intermediate purchases, which ``derive_U_set`` takes out of U; and - ``derive_scrap_correction``, for the inputs the scrap correction in A moves - onto scrap-making industries' other products. + It also includes ``derive_negative_purchase_sales``: the sales BEA records + as negative intermediate purchases, which ``derive_U_set`` takes out of U. """ Y = derive_Ytot_full_cs_matrix() Uimp = derive_U_with_negatives().Uimp y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) y = y.add(derive_negative_purchase_sales(), fill_value=0.0) - y = y.add(derive_scrap_correction(), fill_value=0.0) y.index.name = 'sector' return y From eed4e65142cab37318a61ebf98dee4886f1ad55c Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:59:25 -0500 Subject: [PATCH 4/6] fix(eeio): don't divide market shares by 1 - s; x already excludes scrap (#1293) USEEIO divides market shares by 1 - s, with s scrap's share of output including scrap. The Cornerstone Make table has no scrap (S00401 maps with weight 0), so x is already net of it, and U / x and E / x already land on the other products. Dividing again counted scrap twice in A and B. derive_Vnorm_scrap_corrected now returns the plain market shares, which gives exactly USEEIO's A and B. B @ q now equals E (4,808.491 in 2024; it was 4,812.5). N falls for scrap-making industries: remediation -11.1%, metal stampings -5.9%, hazardous waste -2.4%, landfills -1.5%; median -0.17%. New integration test: B @ q conserves E. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 34 ++++++------------- .../__tests__/test_eeio_diagnostics.py | 15 ++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 61ed7a9c0..008879bdf 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -112,8 +112,6 @@ split_vector_using_agg_ratio, ) from bedrock.utils.schemas.cornerstone_schemas import ( - ELECTRICITY_AGGREGATE_SECTOR, - ELECTRICITY_DISAGG_SECTORS, active_cornerstone_commodities, validate_cornerstone, ) @@ -313,7 +311,16 @@ def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, target_year: int = 0, ) -> pd.DataFrame: - """Scrap-corrected V norm. Inflation is applied via ``derive_V``. + """Market shares ``V q̂⁻¹``, net of scrap. Inflation is applied via ``derive_V``. + + USEEIO divides market shares by ``1 - s``, with ``s`` scrap's share of an + industry's output **including** scrap, so that the industry's inputs and + emissions land on its other products. The Cornerstone Make table has no + scrap: ``S00401`` maps with weight 0, so ``x`` is already output without + scrap, and ``U / x`` and ``E / x`` already land on the other products. + Dividing by ``1 - s`` again would count scrap twice (#1293). The plain + market shares give exactly USEEIO's A and B: + ``U x_incl⁻¹ V_n / (1 - scrap / x_incl) = U x_excl⁻¹ V_n``. When ``target_year`` is not positive and inflation is on, uses ``USAConfig.model_base_year``. @@ -327,27 +334,8 @@ def derive_Vnorm_scrap_corrected( V = derive_V(use_inflation, effective_year) q = compute_q(V=V) - x = compute_x(V=V) - - Vnorm = compute_Vnorm_matrix(V=V, q=q) - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) - x_aligned = x.reindex(V.index, fill_value=0.0) - - V_scrap_corrected = Vnorm.divide( - (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 - ) - V_scrap_corrected = V_scrap_corrected.reindex( + V_scrap_corrected = compute_Vnorm_matrix(V=V, q=q).reindex( index=V.index, columns=V.columns, fill_value=0.0 ) validate_cornerstone(V_scrap_corrected, 'V') diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 59bd9eabc..7b52210a2 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -569,6 +569,21 @@ def test_domestic_lci_equals_e(canonical_usa_config: None) -> None: ) +@pytest.mark.eeio_integration +def test_b_times_q_conserves_emissions(canonical_usa_config: None) -> None: + """``B @ q`` sums to E: commodity factors neither add nor lose emissions. + + It failed by 4.0 Mt when the scrap correction divided market shares by + ``1 - s`` on an ``x`` that already excluded scrap (#1293). The LCI≈E test + above can't see that, because it puts the same market shares on both sides. + """ + B = derive_B() + q = derive_Aq_scaled().scaled_q + assert float((B.reindex(columns=q.index, fill_value=0.0) @ q).sum()) == ( + pytest.approx(float(derive_E_usa().values.sum()), rel=1e-9) + ) + + @pytest.mark.eeio_integration @pytest.mark.xfail( reason=( From 9ecd027abdd8189c2f82b70c017b76dc56fc4b29 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 18:01:03 -0500 Subject: [PATCH 5/6] fix(eeio): keep the scrap correction where x is BEA gross output (#1293) Review on #1292 (Ben): when use_ghg_year_x_in_B or the deflation branch puts BEA gross output in B's x, that output includes scrap, so the 1 / (1 - s) raise was doing real work there. derive_B_via_vnorm now divides the market shares by 1 - s on that path only, with s = scrap / output including scrap (scrap_share_of_output; electricity children share a parent's scrap by output). Both paths are now USEEIO-exact: v0.3 B matches E / (x_GO (1 - s)) V_n to 9e-16, and v1.0 B q = E (4,808.491). Also fixes the stale Vnorm column-sum comment in inflation_helpers_usa. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 32 +++++++++++++++++++ .../utils/economic/inflation_helpers_usa.py | 10 +++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 008879bdf..ba1413caf 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -112,6 +112,8 @@ split_vector_using_agg_ratio, ) from bedrock.utils.schemas.cornerstone_schemas import ( + ELECTRICITY_AGGREGATE_SECTOR, + ELECTRICITY_DISAGG_SECTORS, active_cornerstone_commodities, validate_cornerstone, ) @@ -306,6 +308,30 @@ def derive_q() -> pd.Series[float]: return q +def scrap_share_of_output() -> pd.Series[float]: + """Each industry's scrap (``S00401``) over its output **including** scrap. + + That is USEEIO's ``s``. The Cornerstone Make has no scrap, so + ``derive_Vnorm_scrap_corrected`` needs no adjustment where ``x`` is its row + sums. Where ``x`` is BEA gross output, which includes scrap, B divides the + market shares by ``1 - s`` (``derive_B_via_vnorm``, #1293). + + A parent's scrap is shared across the electricity children by output. + """ + V = derive_V() + x = compute_x(V=V) + scrap = industry_corresp() @ load_detail_V_usa().loc[:, 'S00401'] + if get_usa_config().implement_electricity_disaggregation: + parent = float(scrap.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap = scrap.drop(ELECTRICITY_AGGREGATE_SECTOR, errors='ignore') + children = [c for c in ELECTRICITY_DISAGG_SECTORS if c in x.index] + child_x = float(x.reindex(children).sum()) + for code in children: + scrap.loc[code] = parent * float(x[code]) / child_x if child_x else 0.0 + scrap = scrap.reindex(V.index, fill_value=0.0) + return (scrap / (x.reindex(V.index, fill_value=0.0) + scrap)).fillna(0.0) + + @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -822,6 +848,12 @@ def derive_B_via_vnorm() -> pd.DataFrame: else: x = derive_x_after_redefinition() if cfg.use_ghg_year_x_in_B else derive_x() Vnorm = derive_Vnorm_scrap_corrected() + if cfg.usa_detail_io_source != 'nowcast' and ( + cfg.deflate_x_to_detail_io_year_for_B or cfg.use_ghg_year_x_in_B + ): + # BEA gross output includes scrap, so the scrap correction applies here, + # as in USEEIO: divide by 1 - s with s over output including scrap (#1293). + Vnorm = Vnorm.divide(1.0 - scrap_share_of_output(), axis=0) Bi = E.divide(x, axis=1).fillna(0.0) B = Bi @ Vnorm validate_cornerstone(B, 'B') diff --git a/bedrock/utils/economic/inflation_helpers_usa.py b/bedrock/utils/economic/inflation_helpers_usa.py index 81d8d096b..8b234ca69 100644 --- a/bedrock/utils/economic/inflation_helpers_usa.py +++ b/bedrock/utils/economic/inflation_helpers_usa.py @@ -423,12 +423,10 @@ def get_vnorm_adjusted_commodity_price_ratio( aligned = industry_ratio.reindex(Vnorm.index, fill_value=1.0) # Normalize V_norm columns to sum to 1 so the dot-product is a true weighted - # *average*, not a weighted *sum*. Scrap correction in - # `derive_Vnorm_scrap_corrected` is a row-axis scaling that - # leaves column sums slightly > 1 — using the un-renormalized form would - # produce a non-1 ratio at year=year (off by the column-sum drift, ~5–7%). - # Done locally here rather than upstream because A/B matrix consumers want - # the un-renormalized scrap-corrected V_norm. + # *average*, not a weighted *sum*. `derive_Vnorm_scrap_corrected` returns the + # plain market shares (#1293), whose columns already sum to 1, so this is a + # guard: it keeps the ratio at 1 for year = year if the shares ever change. + # B's gross-output path scales them by 1 / (1 - s) separately. column_sums = Vnorm.sum(axis=0) weights = Vnorm.divide(column_sums.where(column_sums > 1e-9, 1.0), axis=1) commodity_ratio = aligned @ weights From 6a9083e838cef1e107edef641d18b42ec9d378cb Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 18:56:49 -0500 Subject: [PATCH 6/6] fix(eeio): share a split industry's scrap by output; mock it in the B steps test (#1293) Review on #1292 (Ben): industry_corresp gave each of 562000's seven waste children 1/7 of its scrap, so remediation's s was 7.28% against the parent's 0.72%, and v0.3 B ran 7.0% high on 562213. scrap_share_of_output now shares a split BEA industry's scrap across its Cornerstone children by output, so every child gets the parent's share (0.722% for all seven; nothing outside 562* moves). test_useeio_b_adjust_divide_transform_steps now mocks scrap_share_of_output (I1 0.1, I2 0) and expects 0.08 / 0.9, which pins the gross-output branch. Co-Authored-By: Claude Opus 5.5 --- .../transform/eeio/__tests__/test_useeio_b_steps.py | 10 ++++++++-- bedrock/transform/eeio/usa_matrices.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py b/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py index ba7d391da..13aa1d797 100644 --- a/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py +++ b/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py @@ -60,6 +60,12 @@ def test_useeio_b_adjust_divide_transform_steps( lambda original_year, target_year: ratio, ) monkeypatch.setattr(dc, "derive_Vnorm_scrap_corrected", lambda: vnorm) + # Gross output includes scrap, so this branch divides by 1 - s (#1293). + monkeypatch.setattr( + dc, + "scrap_share_of_output", + lambda: pd.Series([0.1, 0.0], index=["I1", "I2"]), + ) try: out = dc.derive_B_via_vnorm() @@ -69,10 +75,10 @@ def test_useeio_b_adjust_divide_transform_steps( # Step 1: adjusted x = x_nominal / ratio = [125, 160] # Step 2: Bi = E / adjusted x = [0.08, 0.125] - # Step 3: B = Bi @ I = Bi + # Step 3: B = Bi @ (I / (1 - s)), with s = [0.1, 0] expected = pd.DataFrame( [ - [0.08, 0.125], # CO2 + [0.08 / 0.9, 0.125], # CO2 [0.0, 0.0], # CH4 [0.0, 0.0], # N2O [0.0, 0.0], # HFCs diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index ba1413caf..487c9a66b 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -316,11 +316,19 @@ def scrap_share_of_output() -> pd.Series[float]: sums. Where ``x`` is BEA gross output, which includes scrap, B divides the market shares by ``1 - s`` (``derive_B_via_vnorm``, #1293). - A parent's scrap is shared across the electricity children by output. + A BEA industry split into several Cornerstone industries (the waste + children of ``562000``, the electricity children) shares its scrap across + them **by output**, not by the correspondence's equal weights. So every + child gets its parent's scrap share, as in USEEIO's waste disaggregation. """ V = derive_V() x = compute_x(V=V) - scrap = industry_corresp() @ load_detail_V_usa().loc[:, 'S00401'] + corresp = industry_corresp() + by_output = corresp.gt(0).mul(x.reindex(corresp.index).fillna(0.0), axis=0) + totals = by_output.sum(axis=0) + # A BEA industry whose children have no output keeps the correspondence. + weights = by_output.div(totals.where(totals > 0), axis=1).fillna(corresp) + scrap = weights @ load_detail_V_usa().loc[:, 'S00401'] if get_usa_config().implement_electricity_disaggregation: parent = float(scrap.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) scrap = scrap.drop(ELECTRICITY_AGGREGATE_SECTOR, errors='ignore')