From b8453af75b1c5aff61d8a19be3f43e456543bc16 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 12:42:30 -0500 Subject: [PATCH 01/15] fix: clip negatives on a copy, not the cached input (#1270) handle_negative_matrix_values and handle_negative_vector_values set negatives to zero in place. derive_U_set passes them the cached output of derive_U_with_negatives, so once derive_U_set ran, every later reader of derive_U_with_negatives saw the clipped frame. Both now return a copy. Co-Authored-By: Claude Opus 5.5 --- bedrock/utils/math/__tests__/__init__.py | 0 .../math/__tests__/test_handle_negatives.py | 30 +++++++++++++++++++ bedrock/utils/math/handle_negatives.py | 15 +++++++--- 3 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 bedrock/utils/math/__tests__/__init__.py create mode 100644 bedrock/utils/math/__tests__/test_handle_negatives.py diff --git a/bedrock/utils/math/__tests__/__init__.py b/bedrock/utils/math/__tests__/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bedrock/utils/math/__tests__/test_handle_negatives.py b/bedrock/utils/math/__tests__/test_handle_negatives.py new file mode 100644 index 000000000..150832e49 --- /dev/null +++ b/bedrock/utils/math/__tests__/test_handle_negatives.py @@ -0,0 +1,30 @@ +"""The negative-value handlers clip to zero and leave their input unchanged (#1270).""" + +from __future__ import annotations + +import pandas as pd + +from bedrock.utils.math.handle_negatives import ( + handle_negative_matrix_values, + handle_negative_vector_values, +) + + +def test_matrix_negatives_are_zero_and_the_input_keeps_them() -> None: + df = pd.DataFrame([[1.0, -2.0], [-3.0, 4.0]], index=['a', 'b'], columns=['a', 'b']) + out = handle_negative_matrix_values(df) + pd.testing.assert_frame_equal( + out, + pd.DataFrame([[1.0, 0.0], [0.0, 4.0]], index=['a', 'b'], columns=['a', 'b']), + ) + assert df.loc['a', 'b'] == -2.0 + assert df.loc['b', 'a'] == -3.0 + + +def test_vector_negatives_are_zero_and_the_input_keeps_them() -> None: + ser = pd.Series([1.0, -2.0, float('nan')], index=['a', 'b', 'c']) + out = handle_negative_vector_values(ser) + pd.testing.assert_series_equal( + out, pd.Series([1.0, 0.0, float('nan')], index=['a', 'b', 'c']) + ) + assert ser['b'] == -2.0 diff --git a/bedrock/utils/math/handle_negatives.py b/bedrock/utils/math/handle_negatives.py index 3aacc16ba..682899e78 100644 --- a/bedrock/utils/math/handle_negatives.py +++ b/bedrock/utils/math/handle_negatives.py @@ -15,14 +15,21 @@ def handle_negative_matrix_values(df: pd.DataFrame) -> pd.DataFrame: """ There are negative values in U or A matrix, set them to 0 TODO(mo.li) some reason why this might be + + Returns a copy. Inputs are often cached frames (``derive_U_with_negatives``), + so clipping in place would change what later callers read (#1270). """ - df[df < 0] = 0 - return df + out = df.copy() + out[out < 0] = 0 + return out def handle_negative_vector_values(ser: pd.Series[float]) -> pd.Series[float]: """ There are negative values in vector, set them to 0 + + Returns a copy, as ``handle_negative_matrix_values`` does. """ - ser[ser < 0] = 0 - return ser + out = ser.copy() + out[out < 0] = 0 + return out From 871eb2e33e7aedc2e9fac401d445ccd5b6885438 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 13:43:32 -0500 Subject: [PATCH 02/15] fix(nab): move BEA's negative intermediate entries to final demand (#1270) BEA records some sales as negative intermediate purchases: industries selling used goods (S00402) and federal nondefense government selling farm commodities from its stocks (1111A0 bought by S00600). derive_U_set sets them to zero, so the Use row no longer added up to q. The Use-side final demand now carries them. Only cells whose Use value is negative move; a cell negative only because imports exceed use still shows as an error. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index a90d10783..8dc528818 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -1023,10 +1023,20 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: takes. Unlike ``derive_y_nab`` it never sees ``A`` or ``q``, so ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. + + BEA records some sales as negative intermediate purchases: industries + selling used goods (``S00402``, e.g. airlines selling aircraft) and federal + nondefense government selling farm commodities from its stocks (``1111A0`` + bought by ``S00600``). ``derive_U_set`` sets these cells to zero, so they are + added to final demand here, and the Use row still adds up to ``q`` (#1270). + Only cells whose Use value is itself negative move. A cell that is negative + only because imports exceed use stays out, so that error still shows. """ Y = derive_Ytot_full_cs_matrix() - Uimp = derive_U_with_negatives().Uimp - y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) + uset = derive_U_with_negatives() + y = Y.sum(axis=1).add(uset.Uimp.sum(axis=1), fill_value=0.0) + sold = uset.Udom.where((uset.Udom + uset.Uimp) < 0, 0.0).clip(upper=0.0) + y = y.add(sold.sum(axis=1), fill_value=0.0) y.index.name = 'sector' return y From 014f03f4f8db263e2d69ad7decdaafce1b0f1517 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 13:50:11 -0500 Subject: [PATCH 03/15] feat(publish): F09000 column for sales BEA records as negative purchases (#1270) derive_negative_purchase_sales returns, by commodity, the domestic Use cells that are negative in the Use table itself (used-goods sales, government sales from stocks). The Use-side final demand and a new published U column, F09000 "Sales recorded as negative intermediate purchases", carry them, so the NIPA-sourced final-demand columns are unchanged and each published U row still sums to output. F09000 is kept out of FINAL_DEMANDS. Co-Authored-By: Claude Opus 5.5 --- bedrock/publish/excel/writer.py | 3 ++ bedrock/publish/model_objects.py | 35 ++++++++++++++++-- bedrock/transform/eeio/usa_matrices.py | 37 +++++++++++++------ .../taxonomy/cornerstone/final_demand.py | 8 ++++ 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/bedrock/publish/excel/writer.py b/bedrock/publish/excel/writer.py index ab709df6a..682c893e7 100644 --- a/bedrock/publish/excel/writer.py +++ b/bedrock/publish/excel/writer.py @@ -236,10 +236,13 @@ def _build_final_demand_meta_df() -> pd.DataFrame: from bedrock.utils.taxonomy.cornerstone.final_demand import ( FINAL_DEMAND_DESC, FINAL_DEMANDS, + NEGATIVE_PURCHASE_SALES_CODE, + NEGATIVE_PURCHASE_SALES_DESC, ) return pd.DataFrame( [{'Code': code, 'Name': FINAL_DEMAND_DESC[code]} for code in FINAL_DEMANDS] + + [{'Code': NEGATIVE_PURCHASE_SALES_CODE, 'Name': NEGATIVE_PURCHASE_SALES_DESC}] ) diff --git a/bedrock/publish/model_objects.py b/bedrock/publish/model_objects.py index 9ce926c53..aba99e9fa 100644 --- a/bedrock/publish/model_objects.py +++ b/bedrock/publish/model_objects.py @@ -95,9 +95,30 @@ def get_V() -> pd.DataFrame: return derive_V() +def _with_negative_purchase_sales( + fd: pd.DataFrame, sales: pd.Series, code: str +) -> pd.DataFrame: + """``fd`` plus the column of sales BEA records as negative purchases (#1270). + + ``derive_U_set`` sets those cells to zero, so the column puts them back on + the final-demand side and each published U row still sums to output. + """ + column = sales.reindex(fd.index, fill_value=0.0) + if abs(column.sum() - sales.sum()) > 1.0: + raise ValueError( + f'{code}: {sales.sum() - column.sum():,.0f} of negative-purchase ' + 'sales sit on rows the final-demand block does not have' + ) + return fd.assign(**{code: column}) + + @functools.cache def get_U() -> pd.DataFrame: - from bedrock.utils.taxonomy.cornerstone.final_demand import FINAL_DEMANDS + from bedrock.transform.eeio.usa_matrices import derive_negative_purchase_sales + from bedrock.utils.taxonomy.cornerstone.final_demand import ( + FINAL_DEMANDS, + NEGATIVE_PURCHASE_SALES_CODE, + ) if _reaggregation_efs_active(): from bedrock.transform.eeio.usa_matrices import ( @@ -108,7 +129,11 @@ def get_U() -> pd.DataFrame: uset = derive_U_set_reaggregated() intermediate = uset.Udom + uset.Uimp - fd_block = derive_disagg_Ytot_reaggregated()[list(FINAL_DEMANDS)] + fd_block = _with_negative_purchase_sales( + derive_disagg_Ytot_reaggregated()[list(FINAL_DEMANDS)], + derive_negative_purchase_sales(), + NEGATIVE_PURCHASE_SALES_CODE, + ) return assemble_extended_U( intermediate=intermediate, fd=fd_block, @@ -124,7 +149,11 @@ def get_U() -> pd.DataFrame: uset = derive_U_set() intermediate = uset.Udom + uset.Uimp - fd_block = derive_disagg_Ytot_with_trade()[list(FINAL_DEMANDS)] + fd_block = _with_negative_purchase_sales( + derive_disagg_Ytot_with_trade()[list(FINAL_DEMANDS)], + derive_negative_purchase_sales(), + NEGATIVE_PURCHASE_SALES_CODE, + ) return assemble_extended_U( intermediate=intermediate, fd=fd_block, diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 8dc528818..61ed7a9c0 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -1014,6 +1014,27 @@ def derive_y_nab() -> pd.Series[float]: return backcompute_y_from_A_and_q(A=aq.Adom, q=aq.scaled_q) +def derive_negative_purchase_sales() -> pd.Series[float]: + """By commodity, the sales BEA records as negative intermediate purchases. + + Examples are industries selling used goods (``S00402``, e.g. airlines + selling aircraft) and federal nondefense government selling farm + commodities from its stocks (``1111A0`` bought by ``S00600``). + ``derive_U_set`` sets these cells to zero. The Use-side final demand and + the published U's ``F09000`` column carry them instead, so each Use row + still adds up to ``q`` (#1270). Values are negative. + + Only domestic cells whose Use value is itself negative count. A cell that + is negative only because imports exceed use is an error, not a sale, so it + stays out and still shows. + """ + uset = derive_U_with_negatives() + sold = uset.Udom.where((uset.Udom + uset.Uimp) < 0, 0.0).clip(upper=0.0) + out = sold.sum(axis=1) + out.index.name = 'sector' + return out + + def derive_y_dom_from_use_final_demand() -> pd.Series[float]: """Domestic final demand plus exports, read from the Use table. @@ -1024,19 +1045,13 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. - BEA records some sales as negative intermediate purchases: industries - selling used goods (``S00402``, e.g. airlines selling aircraft) and federal - nondefense government selling farm commodities from its stocks (``1111A0`` - bought by ``S00600``). ``derive_U_set`` sets these cells to zero, so they are - added to final demand here, and the Use row still adds up to ``q`` (#1270). - Only cells whose Use value is itself negative move. A cell that is negative - only because imports exceed use stays out, so that error still shows. + It also includes ``derive_negative_purchase_sales``: the sales BEA records + as negative intermediate purchases, which ``derive_U_set`` takes out of U. """ Y = derive_Ytot_full_cs_matrix() - uset = derive_U_with_negatives() - y = Y.sum(axis=1).add(uset.Uimp.sum(axis=1), fill_value=0.0) - sold = uset.Udom.where((uset.Udom + uset.Uimp) < 0, 0.0).clip(upper=0.0) - y = y.add(sold.sum(axis=1), fill_value=0.0) + Uimp = derive_U_with_negatives().Uimp + y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) + y = y.add(derive_negative_purchase_sales(), fill_value=0.0) y.index.name = 'sector' return y diff --git a/bedrock/utils/taxonomy/cornerstone/final_demand.py b/bedrock/utils/taxonomy/cornerstone/final_demand.py index fce2e7945..a334120bf 100644 --- a/bedrock/utils/taxonomy/cornerstone/final_demand.py +++ b/bedrock/utils/taxonomy/cornerstone/final_demand.py @@ -48,3 +48,11 @@ 'F10N00': 'State and local: Gross investment in intellectual property products', 'F10S00': 'State and local: Gross investment in structures', } + +# A Cornerstone column, not in BEA's tables: sales that BEA records as negative +# intermediate purchases (used goods, government sales from stocks). The model +# sets negative intermediate cells to zero, and this column carries them, so the +# published U rows still add up to commodity output (#1270). It is kept out of +# FINAL_DEMANDS because the BEA-sourced Y has no such column. +NEGATIVE_PURCHASE_SALES_CODE = 'F09000' +NEGATIVE_PURCHASE_SALES_DESC = 'Sales recorded as negative intermediate purchases' From e950634822f3d91c13495242d24e49fe8505d34d Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 13:57:57 -0500 Subject: [PATCH 04/15] test: quote the balance with the F09000 sales in the xfail reason (#1270) Co-Authored-By: Claude Opus 5.5 --- bedrock/utils/validation/__tests__/test_eeio_diagnostics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 2051418e6..0a45aebfd 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -417,8 +417,8 @@ def test_compare_output_and_L_y( @pytest.mark.xfail( reason=( 'Make and Use disagree on commodity output. On MUT v1.0.0_8365fa4 ' - '(2024) 3 sectors fail, $2.9bn: S00402 (q = 0, +$2.0bn), 1111A0 ' - '(+1.4%) and 334118 (+4.4%).' + '(2024), with the F09000 sales in y: 334118 (+4.4%, imports above ' + 'use) and S00402 ($4M on q = 0).' ), ) def test_domestic_L_y_from_use_final_demand_equals_q( From 11440e0d1e48b509d497a7cb6e284b8bddb3906e Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:26:03 -0500 Subject: [PATCH 05/15] feat(nab): scrap correction for the national balance (#1270) derive_Vnorm_scrap_corrected divides each industry's market shares by 1 - s, so A q counts its purchases at U / (1 - s) and the Use row no longer adds up to q. derive_scrap_correction returns the difference by commodity, -sum_j U[i, j] s_j / (1 - s_j), and derive_y_dom_from_use_final_demand includes it. The scrap share moves to _scrap_share, shared with derive_Vnorm_scrap_corrected (no change to A). On v1.0.0_8365fa4 it clears 562HAZ (1.0% and 1.6% in 2022-23) and the iron and steel gap. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 67 +++++++++++++++++++------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 61ed7a9c0..d04dc8442 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -308,6 +308,23 @@ def derive_q() -> pd.Series[float]: return q +def _scrap_share(industries: pd.Index, x: pd.Series[float]) -> pd.Series[float]: + """Each industry's scrap output (BEA's ``S00401``) as a share of its output.""" + scrap_detail = load_detail_V_usa().loc[:, 'S00401'] + scrap_fraction = industry_corresp() @ scrap_detail + if get_usa_config().implement_electricity_disaggregation: + parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap_fraction = scrap_fraction.drop( + ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' + ) + for code in ELECTRICITY_DISAGG_SECTORS: + scrap_fraction.loc[code] = parent_scrap + # The delivery sector has no scrap: the reindex below gives it zero. + scrap_fraction = scrap_fraction.reindex(industries, fill_value=0.0) + x_aligned = x.reindex(industries, fill_value=0.0) + return (scrap_fraction / x_aligned).fillna(0.0) + + @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -331,22 +348,7 @@ def derive_Vnorm_scrap_corrected( Vnorm = compute_Vnorm_matrix(V=V, q=q) - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) - x_aligned = x.reindex(V.index, fill_value=0.0) - - V_scrap_corrected = Vnorm.divide( - (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 - ) + V_scrap_corrected = Vnorm.divide(1.0 - _scrap_share(V.index, x), axis=0) V_scrap_corrected = V_scrap_corrected.reindex( index=V.index, columns=V.columns, fill_value=0.0 ) @@ -1035,6 +1037,31 @@ def derive_negative_purchase_sales() -> pd.Series[float]: return out +def derive_scrap_correction() -> pd.Series[float]: + """By commodity, the scrap correction for the national balance. + + ``derive_Vnorm_scrap_corrected`` takes scrap (``S00401``) out of each + industry's output and charges all of its inputs to its other products, + dividing its market shares by ``1 - s``, with ``s`` its scrap share. So + ``A q`` counts industry ``j``'s purchases of commodity ``i`` at + ``U[i, j] / (1 - s_j)``, not ``U[i, j]``, and the Use row no longer adds up + to ``q``. This returns the difference, ``-sum_j U[i, j] s_j / (1 - s_j)``, + for the final demand the national balance reads. Values are zero or + negative. + + It is a model correction, not a transaction, so it is not a published + final-demand column. It is largest where scrap-making industries buy + heavily: hazardous waste (``562HAZ``), which buys mainly from itself, and + iron and steel (``331110``) (#1270). + """ + V = derive_V() + s = _scrap_share(V.index, compute_x(V=V)) + Udom = derive_U_set().Udom + out = -Udom.mul((s / (1.0 - s)).reindex(Udom.columns, fill_value=0.0)).sum(axis=1) + out.index.name = 'sector' + return out + + def derive_y_dom_from_use_final_demand() -> pd.Series[float]: """Domestic final demand plus exports, read from the Use table. @@ -1045,13 +1072,17 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. - It also includes ``derive_negative_purchase_sales``: the sales BEA records - as negative intermediate purchases, which ``derive_U_set`` takes out of U. + It also includes two corrections for what the model does to U: + ``derive_negative_purchase_sales``, the sales BEA records as negative + intermediate purchases, which ``derive_U_set`` takes out of U; and + ``derive_scrap_correction``, for the inputs the scrap correction in A moves + onto scrap-making industries' other products. """ Y = derive_Ytot_full_cs_matrix() Uimp = derive_U_with_negatives().Uimp y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) y = y.add(derive_negative_purchase_sales(), fill_value=0.0) + y = y.add(derive_scrap_correction(), fill_value=0.0) y.index.name = 'sector' return y From 15407647c89f3332152f9129c1fcaca551ccfe24 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:26:05 -0500 Subject: [PATCH 06/15] test(nab): leave BEA's accounting sectors out of q = L y, as USEEIO 2 does (#1270) ACCOUNTING_SECTORS (scrap, used goods, noncomparable imports, the rest-of-world adjustment) have no output of their own, so the check can't hold for them. compare_output_vs_leontief_x_demand still computes L @ y on every sector and compares the rest. The xfail reason now quotes the one 2024 failure, 334118; 2017-2023 pass. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/test_eeio_diagnostics.py | 21 +++++++++++++++++-- bedrock/utils/validation/eeio_diagnostics.py | 16 ++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 0a45aebfd..59bd9eabc 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -417,8 +417,8 @@ def test_compare_output_and_L_y( @pytest.mark.xfail( reason=( 'Make and Use disagree on commodity output. On MUT v1.0.0_8365fa4 ' - '(2024), with the F09000 sales in y: 334118 (+4.4%, imports above ' - 'use) and S00402 ($4M on q = 0).' + '(2024) one sector fails: 334118 (+4.4%), whose imports exceed its ' + 'use. 2017-2023 pass.' ), ) def test_domestic_L_y_from_use_final_demand_equals_q( @@ -600,3 +600,20 @@ def test_total_lci_equals_e(canonical_usa_config: None) -> None: f'total LCI≈E failed ({len(result.failing_sectors)} cells): ' f'{result.failing_sectors[:20]}' ) + + +def test_output_vs_leontief_skips_the_accounting_sectors() -> None: + """S00402 has no output, so q = L @ y isn't checked for it (#1270).""" + sectors = ['1111A0', 'S00402'] + L = pd.DataFrame(np.eye(2), index=sectors, columns=sectors) + q = pd.Series([100.0, 0.0], index=sectors) + + off_on_used_goods = compare_output_vs_leontief_x_demand( + output=q, L=L, y=pd.Series([100.0, -6.0e6], index=sectors) + ) + assert off_on_used_goods.passed + + off_on_oilseeds = compare_output_vs_leontief_x_demand( + output=q, L=L, y=pd.Series([103.0, 0.0], index=sectors) + ) + assert not off_on_oilseeds.passed diff --git a/bedrock/utils/validation/eeio_diagnostics.py b/bedrock/utils/validation/eeio_diagnostics.py index de24772d9..3d0c56639 100644 --- a/bedrock/utils/validation/eeio_diagnostics.py +++ b/bedrock/utils/validation/eeio_diagnostics.py @@ -28,6 +28,12 @@ # (e.g. S00402 used goods) compare |diff| against atol instead of (diff / q). _VALIDATE_RESULT_ATOL = 1e-4 +# BEA's accounting rows: scrap, used and secondhand goods, noncomparable +# imports and the rest-of-world adjustment. They have no output of their own +# (S00402's q is 0), so q = L @ y can't hold for them and isn't checked, as in +# USEEIO 2. Of the four, only S00402 is a Cornerstone commodity today (#1270). +ACCOUNTING_SECTORS: frozenset[str] = frozenset({'S00300', 'S00401', 'S00402', 'S00900'}) + @dc.dataclass class DiagnosticResult: @@ -347,7 +353,7 @@ def compare_output_vs_leontief_x_demand( Compares the total sector output (commodity or industry) against the model result calculation of L @ y. Pass/fail: ``|diff| <= tolerance * |value| + atol`` for all sectors (see - ``validate_result``). + ``validate_result``) except the ``ACCOUNTING_SECTORS``, which aren't checked. Parameters ---------- @@ -387,8 +393,14 @@ def compare_output_vs_leontief_x_demand( output_check = backcompute_q_from_L_and_y(L=L, y=y) name = "compare output and L * y" + # L @ y uses every sector; only the comparison leaves the accounting rows out. + checked = output.index.difference(list(ACCOUNTING_SECTORS), sort=False) d_result = validate_result( - name, output, output_check, tolerance=tolerance, include_details=include_details + name, + output.loc[checked], + output_check.loc[checked], + tolerance=tolerance, + include_details=include_details, ) return d_result From c6acb8e8073aa0cfe50e90e06a2016305b9983b6 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:51:03 -0500 Subject: [PATCH 07/15] Revert the national-balance scrap correction (#1293) It offset a double count in A and B instead of fixing it. #1293 fixes the scrap correction itself. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 67 +++++++------------------- 1 file changed, 18 insertions(+), 49 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index d04dc8442..61ed7a9c0 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -308,23 +308,6 @@ def derive_q() -> pd.Series[float]: return q -def _scrap_share(industries: pd.Index, x: pd.Series[float]) -> pd.Series[float]: - """Each industry's scrap output (BEA's ``S00401``) as a share of its output.""" - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(industries, fill_value=0.0) - x_aligned = x.reindex(industries, fill_value=0.0) - return (scrap_fraction / x_aligned).fillna(0.0) - - @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -348,7 +331,22 @@ def derive_Vnorm_scrap_corrected( Vnorm = compute_Vnorm_matrix(V=V, q=q) - V_scrap_corrected = Vnorm.divide(1.0 - _scrap_share(V.index, x), axis=0) + scrap_detail = load_detail_V_usa().loc[:, 'S00401'] + scrap_fraction = industry_corresp() @ scrap_detail + if get_usa_config().implement_electricity_disaggregation: + parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap_fraction = scrap_fraction.drop( + ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' + ) + for code in ELECTRICITY_DISAGG_SECTORS: + scrap_fraction.loc[code] = parent_scrap + # The delivery sector has no scrap: the reindex below gives it zero. + scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) + x_aligned = x.reindex(V.index, fill_value=0.0) + + V_scrap_corrected = Vnorm.divide( + (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 + ) V_scrap_corrected = V_scrap_corrected.reindex( index=V.index, columns=V.columns, fill_value=0.0 ) @@ -1037,31 +1035,6 @@ def derive_negative_purchase_sales() -> pd.Series[float]: return out -def derive_scrap_correction() -> pd.Series[float]: - """By commodity, the scrap correction for the national balance. - - ``derive_Vnorm_scrap_corrected`` takes scrap (``S00401``) out of each - industry's output and charges all of its inputs to its other products, - dividing its market shares by ``1 - s``, with ``s`` its scrap share. So - ``A q`` counts industry ``j``'s purchases of commodity ``i`` at - ``U[i, j] / (1 - s_j)``, not ``U[i, j]``, and the Use row no longer adds up - to ``q``. This returns the difference, ``-sum_j U[i, j] s_j / (1 - s_j)``, - for the final demand the national balance reads. Values are zero or - negative. - - It is a model correction, not a transaction, so it is not a published - final-demand column. It is largest where scrap-making industries buy - heavily: hazardous waste (``562HAZ``), which buys mainly from itself, and - iron and steel (``331110``) (#1270). - """ - V = derive_V() - s = _scrap_share(V.index, compute_x(V=V)) - Udom = derive_U_set().Udom - out = -Udom.mul((s / (1.0 - s)).reindex(Udom.columns, fill_value=0.0)).sum(axis=1) - out.index.name = 'sector' - return out - - def derive_y_dom_from_use_final_demand() -> pd.Series[float]: """Domestic final demand plus exports, read from the Use table. @@ -1072,17 +1045,13 @@ def derive_y_dom_from_use_final_demand() -> pd.Series[float]: ``L_dom @ y`` reproduces ``q`` only when Make and Use agree on each commodity's output. Negatives are kept, as in the published tables. - It also includes two corrections for what the model does to U: - ``derive_negative_purchase_sales``, the sales BEA records as negative - intermediate purchases, which ``derive_U_set`` takes out of U; and - ``derive_scrap_correction``, for the inputs the scrap correction in A moves - onto scrap-making industries' other products. + It also includes ``derive_negative_purchase_sales``: the sales BEA records + as negative intermediate purchases, which ``derive_U_set`` takes out of U. """ Y = derive_Ytot_full_cs_matrix() Uimp = derive_U_with_negatives().Uimp y = Y.sum(axis=1).add(Uimp.sum(axis=1), fill_value=0.0) y = y.add(derive_negative_purchase_sales(), fill_value=0.0) - y = y.add(derive_scrap_correction(), fill_value=0.0) y.index.name = 'sector' return y From eed4e65142cab37318a61ebf98dee4886f1ad55c Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 14:59:25 -0500 Subject: [PATCH 08/15] fix(eeio): don't divide market shares by 1 - s; x already excludes scrap (#1293) USEEIO divides market shares by 1 - s, with s scrap's share of output including scrap. The Cornerstone Make table has no scrap (S00401 maps with weight 0), so x is already net of it, and U / x and E / x already land on the other products. Dividing again counted scrap twice in A and B. derive_Vnorm_scrap_corrected now returns the plain market shares, which gives exactly USEEIO's A and B. B @ q now equals E (4,808.491 in 2024; it was 4,812.5). N falls for scrap-making industries: remediation -11.1%, metal stampings -5.9%, hazardous waste -2.4%, landfills -1.5%; median -0.17%. New integration test: B @ q conserves E. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 34 ++++++------------- .../__tests__/test_eeio_diagnostics.py | 15 ++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 61ed7a9c0..008879bdf 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -112,8 +112,6 @@ split_vector_using_agg_ratio, ) from bedrock.utils.schemas.cornerstone_schemas import ( - ELECTRICITY_AGGREGATE_SECTOR, - ELECTRICITY_DISAGG_SECTORS, active_cornerstone_commodities, validate_cornerstone, ) @@ -313,7 +311,16 @@ def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, target_year: int = 0, ) -> pd.DataFrame: - """Scrap-corrected V norm. Inflation is applied via ``derive_V``. + """Market shares ``V q̂⁻¹``, net of scrap. Inflation is applied via ``derive_V``. + + USEEIO divides market shares by ``1 - s``, with ``s`` scrap's share of an + industry's output **including** scrap, so that the industry's inputs and + emissions land on its other products. The Cornerstone Make table has no + scrap: ``S00401`` maps with weight 0, so ``x`` is already output without + scrap, and ``U / x`` and ``E / x`` already land on the other products. + Dividing by ``1 - s`` again would count scrap twice (#1293). The plain + market shares give exactly USEEIO's A and B: + ``U x_incl⁻¹ V_n / (1 - scrap / x_incl) = U x_excl⁻¹ V_n``. When ``target_year`` is not positive and inflation is on, uses ``USAConfig.model_base_year``. @@ -327,27 +334,8 @@ def derive_Vnorm_scrap_corrected( V = derive_V(use_inflation, effective_year) q = compute_q(V=V) - x = compute_x(V=V) - - Vnorm = compute_Vnorm_matrix(V=V, q=q) - scrap_detail = load_detail_V_usa().loc[:, 'S00401'] - scrap_fraction = industry_corresp() @ scrap_detail - if get_usa_config().implement_electricity_disaggregation: - parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) - scrap_fraction = scrap_fraction.drop( - ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' - ) - for code in ELECTRICITY_DISAGG_SECTORS: - scrap_fraction.loc[code] = parent_scrap - # The delivery sector has no scrap: the reindex below gives it zero. - scrap_fraction = scrap_fraction.reindex(V.index, fill_value=0.0) - x_aligned = x.reindex(V.index, fill_value=0.0) - - V_scrap_corrected = Vnorm.divide( - (1.0 - (scrap_fraction / x_aligned).fillna(0.0)), axis=0 - ) - V_scrap_corrected = V_scrap_corrected.reindex( + V_scrap_corrected = compute_Vnorm_matrix(V=V, q=q).reindex( index=V.index, columns=V.columns, fill_value=0.0 ) validate_cornerstone(V_scrap_corrected, 'V') diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 59bd9eabc..7b52210a2 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -569,6 +569,21 @@ def test_domestic_lci_equals_e(canonical_usa_config: None) -> None: ) +@pytest.mark.eeio_integration +def test_b_times_q_conserves_emissions(canonical_usa_config: None) -> None: + """``B @ q`` sums to E: commodity factors neither add nor lose emissions. + + It failed by 4.0 Mt when the scrap correction divided market shares by + ``1 - s`` on an ``x`` that already excluded scrap (#1293). The LCI≈E test + above can't see that, because it puts the same market shares on both sides. + """ + B = derive_B() + q = derive_Aq_scaled().scaled_q + assert float((B.reindex(columns=q.index, fill_value=0.0) @ q).sum()) == ( + pytest.approx(float(derive_E_usa().values.sum()), rel=1e-9) + ) + + @pytest.mark.eeio_integration @pytest.mark.xfail( reason=( From 569df8273388595dcc9b92ceb56fcd4d99349ce6 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 15:17:00 -0500 Subject: [PATCH 09/15] feat(nowcast): anchor the computers family's exports on published 2017 (#1270) Census domestic exports put the computers family (3341) at 1.28 times BEA's 2017 exports, and terminals (334118) export more than they produce, so its imports exceed its use. Each 3341 commodity's export cell is now its published 2017 F04000 cell moved by its own Census growth, the #763 construction for imports. Terminals' exports fall to 0.74-0.92 of output in 2017-2023 (were 1.01-1.13 in 2022-2024); 2024 sits at 1.00 because output falls while Census exports grow. Needs a rebuild to reach the tables. Co-Authored-By: Claude Opus 5.5 --- .../test_nowcast_export_anchoring.py | 42 +++++++++ bedrock/transform/iot/nowcast.py | 14 ++- .../transform/iot/nowcast_export_anchoring.py | 86 +++++++++++++++++++ 3 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py create mode 100644 bedrock/transform/iot/nowcast_export_anchoring.py diff --git a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py new file mode 100644 index 000000000..c64073978 --- /dev/null +++ b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py @@ -0,0 +1,42 @@ +"""The computers family's exports sit on the published 2017 cells (#1270).""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from bedrock.transform.iot.nowcast_export_anchoring import anchor_family_exports + +M = 1e6 +IDX = ['334111', '334118', '334119', '336111'] + + +def _anchor(now: list[float], base: list[float], pub: list[float]) -> pd.Series: + return anchor_family_exports( + pd.Series(now, index=IDX) * M, + pd.Series(base, index=IDX) * M, + pd.Series(pub, index=IDX) * M, + ) + + +def test_anchored_rows_are_published_2017_moved_by_own_growth() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * 10.0 / 8.0 * M) + assert out['334118'] == pytest.approx(6.0 * 11.0 / 9.0 * M) + + +def test_2017_reproduces_the_published_cells() -> None: + out = _anchor([8.0, 9.0, 0.0, 25.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['334111'] == pytest.approx(7.0 * M) + assert out['334118'] == pytest.approx(6.0 * M) + + +def test_rows_outside_the_family_keep_census() -> None: + out = _anchor([10.0, 11.0, 0.0, 30.0], [8.0, 9.0, 0.0, 25.0], [7.0, 6.0, 0.0, 23.0]) + assert out['336111'] == pytest.approx(30.0 * M) + + +def test_census_only_and_published_only_rows() -> None: + out = _anchor([10.0, 4.0, 0.0, 30.0], [8.0, 3.0, 0.0, 25.0], [7.0, 0.0, 2.0, 23.0]) + assert out['334118'] == pytest.approx(4.0 * M) # published 2017 empty: Census + assert out['334119'] == pytest.approx(2.0 * M) # Census 2017 empty: held flat diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index 14889be00..abc5882e7 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -461,7 +461,9 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat F03000 is Inventories_ for every year in ``INVENTORIES_YEARS`` (#529, #746); 2024 is all-zero. F04000 is mapped Trade_Exports_ Detail mass for every year in - ``TRADE_OVERLAY_YEARS``. S00900/F04000 uses the rest-of-world identity + ``TRADE_OVERLAY_YEARS``, with the computers family anchored on the + published 2017 cells (:mod:`~bedrock.transform.iot.nowcast_export_anchoring`, + #1270). S00900/F04000 uses the rest-of-world identity against frozen 2017 Supply T016. The seventeen NIPA-built columns are **conditioned on the published annual @@ -479,12 +481,18 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat y = _nipa_final_demand_block(pd.DataFrame(fbs), int(year)) if year in TRADE_OVERLAY_YEARS: + from bedrock.transform.iot.nowcast_export_anchoring import ( # noqa: PLC0415 + anchor_family_exports, + published_2017_exports, + ) from bedrock.transform.iot.nowcast_s00300_use import ( # noqa: PLC0415 s00300_f02n00_usd, ) - exports = _trade_fbs_commodity_vector( - f'Trade_Exports_{year}', download_sources_ok + exports = anchor_family_exports( + _trade_fbs_commodity_vector(f'Trade_Exports_{year}', download_sources_ok), + _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), + published_2017_exports(), ) y['F04000'] = exports.reindex(y.index).fillna(0.0) pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise')) diff --git a/bedrock/transform/iot/nowcast_export_anchoring.py b/bedrock/transform/iot/nowcast_export_anchoring.py new file mode 100644 index 000000000..b794a7632 --- /dev/null +++ b/bedrock/transform/iot/nowcast_export_anchoring.py @@ -0,0 +1,86 @@ +"""Anchor the export column of chosen families on the published 2017 cells (#1270). + +The export column (``F04000``) is Census domestic exports (#762) mapped to BEA +commodities, and nothing conditions it. For the computers family (``3341``) +that puts exports above domestic output. Terminals and peripherals (``334118``) +export $9.2bn in 2017 against BEA's $6.1bn and its own $8.1bn of output, and +the whole family runs 1.28 times published. Census re-exports of terminals +($16.4bn in 2017) are already excluded, so the excess is in the domestic +exports themselves. + +A row that exports more than it produces leaves imports with nowhere to go: +total use excluding exports falls below imports, so the import matrix assigns +industries more of the commodity than they buy (the rows #1282 warns about). + +The construction is the #763 one for imports +(:func:`~bedrock.transform.iot.nowcast_import_conditioning.anchored_mcif`): +each commodity's exports are its **published 2017 detail ``F04000`` cell** +moved by **its own Census line's growth**. 2017 reproduces BEA, and every year +moves by its own observed series. + +⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES`. Every other commodity +keeps its Census value. +""" + +from __future__ import annotations + +import logging + +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + +#: NAICS families whose exports are anchored on the published 2017 cells. +#: ``3341`` computers: Census domestic exports run 1.28 times BEA's 2017 family +#: total, and terminals export more than they produce (#1270). +ANCHORED_EXPORT_FAMILIES: frozenset[str] = frozenset({'3341'}) + +#: Below this ($) a published or Census value counts as empty. +EMPTY_USD = 1e6 + + +def anchor_family_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = ANCHORED_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each anchored family's cells on published 2017 x own growth. + + For a commodity inside *families*: + + - published 2017 and Census 2017 both carry it: + ``published_2017 x (census_t / census_2017)``; + - published 2017 is empty but Census has it: keep the Census value; + - Census 2017 is empty but published has it: hold the published value. + + Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + in_family = pd.Series( + [str(c)[:4] in families for c in exports_now.index], index=exports_now.index + ) + has_pub = pub >= EMPTY_USD + has_base = base >= EMPTY_USD + with np.errstate(divide='ignore', invalid='ignore'): + growth = (exports_now / base).replace([np.inf, -np.inf], np.nan).fillna(0.0) + + out = exports_now.copy() + both = in_family & has_pub & has_base + out[both] = pub[both] * growth[both] + pub_only = in_family & has_pub & ~has_base + out[pub_only] = pub[pub_only] + return out + + +def published_2017_exports() -> pd.Series: + """BEA's published 2017 detail export column (``F04000``), USD.""" + from bedrock.extract.iot.io_2017 import ( # noqa: PLC0415 + _load_2017_detail_supply_use_usa, + ) + + use = _load_2017_detail_supply_use_usa('Use_SUT_detail') + use.columns = use.columns.str.strip() + return pd.to_numeric(use['F04000'], errors='coerce').fillna(0.0) * 1e6 From 9ecd027abdd8189c2f82b70c017b76dc56fc4b29 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 18:01:03 -0500 Subject: [PATCH 10/15] fix(eeio): keep the scrap correction where x is BEA gross output (#1293) Review on #1292 (Ben): when use_ghg_year_x_in_B or the deflation branch puts BEA gross output in B's x, that output includes scrap, so the 1 / (1 - s) raise was doing real work there. derive_B_via_vnorm now divides the market shares by 1 - s on that path only, with s = scrap / output including scrap (scrap_share_of_output; electricity children share a parent's scrap by output). Both paths are now USEEIO-exact: v0.3 B matches E / (x_GO (1 - s)) V_n to 9e-16, and v1.0 B q = E (4,808.491). Also fixes the stale Vnorm column-sum comment in inflation_helpers_usa. Co-Authored-By: Claude Opus 5.5 --- bedrock/transform/eeio/usa_matrices.py | 32 +++++++++++++++++++ .../utils/economic/inflation_helpers_usa.py | 10 +++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index 008879bdf..ba1413caf 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -112,6 +112,8 @@ split_vector_using_agg_ratio, ) from bedrock.utils.schemas.cornerstone_schemas import ( + ELECTRICITY_AGGREGATE_SECTOR, + ELECTRICITY_DISAGG_SECTORS, active_cornerstone_commodities, validate_cornerstone, ) @@ -306,6 +308,30 @@ def derive_q() -> pd.Series[float]: return q +def scrap_share_of_output() -> pd.Series[float]: + """Each industry's scrap (``S00401``) over its output **including** scrap. + + That is USEEIO's ``s``. The Cornerstone Make has no scrap, so + ``derive_Vnorm_scrap_corrected`` needs no adjustment where ``x`` is its row + sums. Where ``x`` is BEA gross output, which includes scrap, B divides the + market shares by ``1 - s`` (``derive_B_via_vnorm``, #1293). + + A parent's scrap is shared across the electricity children by output. + """ + V = derive_V() + x = compute_x(V=V) + scrap = industry_corresp() @ load_detail_V_usa().loc[:, 'S00401'] + if get_usa_config().implement_electricity_disaggregation: + parent = float(scrap.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) + scrap = scrap.drop(ELECTRICITY_AGGREGATE_SECTOR, errors='ignore') + children = [c for c in ELECTRICITY_DISAGG_SECTORS if c in x.index] + child_x = float(x.reindex(children).sum()) + for code in children: + scrap.loc[code] = parent * float(x[code]) / child_x if child_x else 0.0 + scrap = scrap.reindex(V.index, fill_value=0.0) + return (scrap / (x.reindex(V.index, fill_value=0.0) + scrap)).fillna(0.0) + + @functools.cache def derive_Vnorm_scrap_corrected( apply_inflation: bool | None = None, @@ -822,6 +848,12 @@ def derive_B_via_vnorm() -> pd.DataFrame: else: x = derive_x_after_redefinition() if cfg.use_ghg_year_x_in_B else derive_x() Vnorm = derive_Vnorm_scrap_corrected() + if cfg.usa_detail_io_source != 'nowcast' and ( + cfg.deflate_x_to_detail_io_year_for_B or cfg.use_ghg_year_x_in_B + ): + # BEA gross output includes scrap, so the scrap correction applies here, + # as in USEEIO: divide by 1 - s with s over output including scrap (#1293). + Vnorm = Vnorm.divide(1.0 - scrap_share_of_output(), axis=0) Bi = E.divide(x, axis=1).fillna(0.0) B = Bi @ Vnorm validate_cornerstone(B, 'B') diff --git a/bedrock/utils/economic/inflation_helpers_usa.py b/bedrock/utils/economic/inflation_helpers_usa.py index 81d8d096b..8b234ca69 100644 --- a/bedrock/utils/economic/inflation_helpers_usa.py +++ b/bedrock/utils/economic/inflation_helpers_usa.py @@ -423,12 +423,10 @@ def get_vnorm_adjusted_commodity_price_ratio( aligned = industry_ratio.reindex(Vnorm.index, fill_value=1.0) # Normalize V_norm columns to sum to 1 so the dot-product is a true weighted - # *average*, not a weighted *sum*. Scrap correction in - # `derive_Vnorm_scrap_corrected` is a row-axis scaling that - # leaves column sums slightly > 1 — using the un-renormalized form would - # produce a non-1 ratio at year=year (off by the column-sum drift, ~5–7%). - # Done locally here rather than upstream because A/B matrix consumers want - # the un-renormalized scrap-corrected V_norm. + # *average*, not a weighted *sum*. `derive_Vnorm_scrap_corrected` returns the + # plain market shares (#1293), whose columns already sum to 1, so this is a + # guard: it keeps the ratio at 1 for year = year if the shares ever change. + # B's gross-output path scales them by 1 / (1 - s) separately. column_sums = Vnorm.sum(axis=0) weights = Vnorm.divide(column_sums.where(column_sums > 1e-9, 1.0), axis=1) commodity_ratio = aligned @ weights From 6a9083e838cef1e107edef641d18b42ec9d378cb Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 18:56:49 -0500 Subject: [PATCH 11/15] fix(eeio): share a split industry's scrap by output; mock it in the B steps test (#1293) Review on #1292 (Ben): industry_corresp gave each of 562000's seven waste children 1/7 of its scrap, so remediation's s was 7.28% against the parent's 0.72%, and v0.3 B ran 7.0% high on 562213. scrap_share_of_output now shares a split BEA industry's scrap across its Cornerstone children by output, so every child gets the parent's share (0.722% for all seven; nothing outside 562* moves). test_useeio_b_adjust_divide_transform_steps now mocks scrap_share_of_output (I1 0.1, I2 0) and expects 0.08 / 0.9, which pins the gross-output branch. Co-Authored-By: Claude Opus 5.5 --- .../transform/eeio/__tests__/test_useeio_b_steps.py | 10 ++++++++-- bedrock/transform/eeio/usa_matrices.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py b/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py index ba7d391da..13aa1d797 100644 --- a/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py +++ b/bedrock/transform/eeio/__tests__/test_useeio_b_steps.py @@ -60,6 +60,12 @@ def test_useeio_b_adjust_divide_transform_steps( lambda original_year, target_year: ratio, ) monkeypatch.setattr(dc, "derive_Vnorm_scrap_corrected", lambda: vnorm) + # Gross output includes scrap, so this branch divides by 1 - s (#1293). + monkeypatch.setattr( + dc, + "scrap_share_of_output", + lambda: pd.Series([0.1, 0.0], index=["I1", "I2"]), + ) try: out = dc.derive_B_via_vnorm() @@ -69,10 +75,10 @@ def test_useeio_b_adjust_divide_transform_steps( # Step 1: adjusted x = x_nominal / ratio = [125, 160] # Step 2: Bi = E / adjusted x = [0.08, 0.125] - # Step 3: B = Bi @ I = Bi + # Step 3: B = Bi @ (I / (1 - s)), with s = [0.1, 0] expected = pd.DataFrame( [ - [0.08, 0.125], # CO2 + [0.08 / 0.9, 0.125], # CO2 [0.0, 0.0], # CH4 [0.0, 0.0], # N2O [0.0, 0.0], # HFCs diff --git a/bedrock/transform/eeio/usa_matrices.py b/bedrock/transform/eeio/usa_matrices.py index ba1413caf..487c9a66b 100644 --- a/bedrock/transform/eeio/usa_matrices.py +++ b/bedrock/transform/eeio/usa_matrices.py @@ -316,11 +316,19 @@ def scrap_share_of_output() -> pd.Series[float]: sums. Where ``x`` is BEA gross output, which includes scrap, B divides the market shares by ``1 - s`` (``derive_B_via_vnorm``, #1293). - A parent's scrap is shared across the electricity children by output. + A BEA industry split into several Cornerstone industries (the waste + children of ``562000``, the electricity children) shares its scrap across + them **by output**, not by the correspondence's equal weights. So every + child gets its parent's scrap share, as in USEEIO's waste disaggregation. """ V = derive_V() x = compute_x(V=V) - scrap = industry_corresp() @ load_detail_V_usa().loc[:, 'S00401'] + corresp = industry_corresp() + by_output = corresp.gt(0).mul(x.reindex(corresp.index).fillna(0.0), axis=0) + totals = by_output.sum(axis=0) + # A BEA industry whose children have no output keeps the correspondence. + weights = by_output.div(totals.where(totals > 0), axis=1).fillna(corresp) + scrap = weights @ load_detail_V_usa().loc[:, 'S00401'] if get_usa_config().implement_electricity_disaggregation: parent = float(scrap.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) scrap = scrap.drop(ELECTRICITY_AGGREGATE_SECTOR, errors='ignore') From 1331e58b1f03196089836871a759f5629553bb7d Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 19:52:47 -0500 Subject: [PATCH 12/15] feat(nowcast): anchor the pharmaceutical family's exports on family growth (#1194, #1270) Review on #1295 (Ben): Mo's 325411 finding is the same Census-vs-BEA gap. Census puts preparations on 325411 ($7.3bn of 2017 exports against $0.08bn published) and 325413, so a row's own Census line is no mover: own-growth anchoring would put biologics (325414) at $52.7bn of 2024 exports against $48.0bn of output. FAMILY_GROWTH_EXPORT_FAMILIES = {'3254'} holds BEA's 2017 split of the family and moves it by the family's Census growth (1.81 by 2024). All four rows stay below output in 2017-2024 (325414 peaks at 0.87). anchored_exports applies both modes. Also from the review: rebuild_nowcast.yml pinned to ubuntu-24.04, as #1297 did for CI, and the rebuild script's anchor comment covers exports. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rebuild_nowcast.yml | 4 +- .../test_nowcast_export_anchoring.py | 30 +++++++++- bedrock/transform/iot/nowcast.py | 6 +- .../transform/iot/nowcast_export_anchoring.py | 56 ++++++++++++++++++- scripts/rebuild_nowcast_year.py | 2 +- 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rebuild_nowcast.yml b/.github/workflows/rebuild_nowcast.yml index 425825b08..cd47ed1fc 100644 --- a/.github/workflows/rebuild_nowcast.yml +++ b/.github/workflows/rebuild_nowcast.yml @@ -21,7 +21,7 @@ env: jobs: tables: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write @@ -65,7 +65,7 @@ jobs: ghg: needs: tables - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 360 permissions: id-token: write diff --git a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py index c64073978..0f821d4fa 100644 --- a/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py +++ b/bedrock/transform/iot/__tests__/test_nowcast_export_anchoring.py @@ -5,7 +5,10 @@ import pandas as pd import pytest -from bedrock.transform.iot.nowcast_export_anchoring import anchor_family_exports +from bedrock.transform.iot.nowcast_export_anchoring import ( + anchor_family_exports, + anchor_family_growth_exports, +) M = 1e6 IDX = ['334111', '334118', '334119', '336111'] @@ -40,3 +43,28 @@ def test_census_only_and_published_only_rows() -> None: out = _anchor([10.0, 4.0, 0.0, 30.0], [8.0, 3.0, 0.0, 25.0], [7.0, 0.0, 2.0, 23.0]) assert out['334118'] == pytest.approx(4.0 * M) # published 2017 empty: Census assert out['334119'] == pytest.approx(2.0 * M) # Census 2017 empty: held flat + + +PHARMA = ['325411', '325412', '325414', '336111'] + + +def test_family_growth_keeps_bea_split_and_moves_by_the_family() -> None: + now = pd.Series([14.0, 20.0, 46.0, 30.0], index=PHARMA) * M + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(now, base, pub) + + growth = (14.0 + 20.0 + 46.0) / (7.0 + 17.0 + 20.0) + for code, published in (('325411', 0.1), ('325412', 30.0), ('325414', 22.5)): + assert out[code] == pytest.approx(published * growth * M) + assert out['336111'] == pytest.approx(30.0 * M) # outside the family + + +def test_family_growth_in_2017_reproduces_bea() -> None: + base = pd.Series([7.0, 17.0, 20.0, 25.0], index=PHARMA) * M + pub = pd.Series([0.1, 30.0, 22.5, 23.0], index=PHARMA) * M + out = anchor_family_growth_exports(base, base, pub) + + for code in ('325411', '325412', '325414'): + assert out[code] == pytest.approx(pub[code]) + assert out['336111'] == pytest.approx(base['336111']) diff --git a/bedrock/transform/iot/nowcast.py b/bedrock/transform/iot/nowcast.py index abc5882e7..a9d0b44fb 100644 --- a/bedrock/transform/iot/nowcast.py +++ b/bedrock/transform/iot/nowcast.py @@ -461,7 +461,7 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat F03000 is Inventories_ for every year in ``INVENTORIES_YEARS`` (#529, #746); 2024 is all-zero. F04000 is mapped Trade_Exports_ Detail mass for every year in - ``TRADE_OVERLAY_YEARS``, with the computers family anchored on the + ``TRADE_OVERLAY_YEARS``, with the computers and pharmaceutical families anchored on the published 2017 cells (:mod:`~bedrock.transform.iot.nowcast_export_anchoring`, #1270). S00900/F04000 uses the rest-of-world identity against frozen 2017 Supply T016. @@ -482,14 +482,14 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat if year in TRADE_OVERLAY_YEARS: from bedrock.transform.iot.nowcast_export_anchoring import ( # noqa: PLC0415 - anchor_family_exports, + anchored_exports, published_2017_exports, ) from bedrock.transform.iot.nowcast_s00300_use import ( # noqa: PLC0415 s00300_f02n00_usd, ) - exports = anchor_family_exports( + exports = anchored_exports( _trade_fbs_commodity_vector(f'Trade_Exports_{year}', download_sources_ok), _trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok), published_2017_exports(), diff --git a/bedrock/transform/iot/nowcast_export_anchoring.py b/bedrock/transform/iot/nowcast_export_anchoring.py index b794a7632..e0feaf451 100644 --- a/bedrock/transform/iot/nowcast_export_anchoring.py +++ b/bedrock/transform/iot/nowcast_export_anchoring.py @@ -18,7 +18,9 @@ moved by **its own Census line's growth**. 2017 reproduces BEA, and every year moves by its own observed series. -⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES`. Every other commodity +⚠️ Applied only inside :data:`ANCHORED_EXPORT_FAMILIES` (each row moved by its +own Census growth) and :data:`FAMILY_GROWTH_EXPORT_FAMILIES` (each row moved by +its family's growth, where Census misclassifies rows). Every other commodity keeps its Census value. """ @@ -31,11 +33,22 @@ logger = logging.getLogger(__name__) -#: NAICS families whose exports are anchored on the published 2017 cells. +#: NAICS families whose exports are anchored on the published 2017 cells, each +#: row moved by **its own** Census line's growth. #: ``3341`` computers: Census domestic exports run 1.28 times BEA's 2017 family #: total, and terminals export more than they produce (#1270). ANCHORED_EXPORT_FAMILIES: frozenset[str] = frozenset({'3341'}) +#: NAICS families whose exports are anchored on the published 2017 cells, every +#: row moved by the **family's** Census growth: BEA's 2017 split of the family, +#: moved by the family level. For families whose Census rows are misclassified, +#: so a row's own line is no mover. +#: ``3254`` pharmaceuticals: Census puts preparations on ``325411`` medicinal +#: ingredients ($7.3bn of exports in 2017 against $0.08bn published) and +#: ``325413`` diagnostics. With each row's own growth, biologics (``325414``) +#: would export $52.7bn in 2024 against $48.0bn of output (#1194, #1270). +FAMILY_GROWTH_EXPORT_FAMILIES: frozenset[str] = frozenset({'3254'}) + #: Below this ($) a published or Census value counts as empty. EMPTY_USD = 1e6 @@ -75,6 +88,45 @@ def anchor_family_exports( return out +def anchor_family_growth_exports( + exports_now: pd.Series, + exports_2017: pd.Series, + published_2017: pd.Series, + families: frozenset[str] = FAMILY_GROWTH_EXPORT_FAMILIES, +) -> pd.Series: + """Exports with each family on BEA's 2017 split, moved by the family's growth. + + For each family in *families*, every commodity's exports are + ``published_2017 x (family census_t / family census_2017)``. A commodity BEA + publishes no 2017 exports for gets none: its Census mass is the + misclassification being corrected. A family with no Census 2017 mass keeps + its Census values. Every commodity outside *families* keeps ``exports_now``. + """ + pub = published_2017.reindex(exports_now.index).fillna(0.0).clip(lower=0.0) + base = exports_2017.reindex(exports_now.index).fillna(0.0) + family = pd.Series([str(c)[:4] for c in exports_now.index], index=exports_now.index) + out = exports_now.copy() + for code in sorted(families): + rows = family == code + base_total = float(base[rows].sum()) + if base_total < EMPTY_USD: + continue + growth = float(exports_now[rows].sum()) / base_total + out[rows] = pub[rows] * growth + logger.info( + 'exports %s: BEA 2017 split moved by the family growth %.3f', code, growth + ) + return out + + +def anchored_exports( + exports_now: pd.Series, exports_2017: pd.Series, published_2017: pd.Series +) -> pd.Series: + """Both anchorings: own-growth rows, then family-growth families.""" + rows = anchor_family_exports(exports_now, exports_2017, published_2017) + return anchor_family_growth_exports(rows, exports_2017, published_2017) + + def published_2017_exports() -> pd.Series: """BEA's published 2017 detail export column (``F04000``), USD.""" from bedrock.extract.iot.io_2017 import ( # noqa: PLC0415 diff --git a/scripts/rebuild_nowcast_year.py b/scripts/rebuild_nowcast_year.py index 257918809..afbb0e902 100644 --- a/scripts/rebuild_nowcast_year.py +++ b/scripts/rebuild_nowcast_year.py @@ -240,7 +240,7 @@ def main() -> None: common.download_fba_on_api_error = False check_flags() - # 2017 is the import anchor every later year is conditioned against. + # 2017 is the import and export anchor every later year is conditioned against. build_trade(sorted({2017, year}), tag) run_step('bedrock.transform.iot.nowcast_sut_assembly', '--years', str(year)) From 146bdb06b12cc894785716af76f81b238ad752d4 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 20:17:13 -0500 Subject: [PATCH 13/15] fix(publish): F09000 on the reaggregated U, its sign in the name, and a test mock (#1270) Review on #1290 (Ben): - get_U's reaggregation branch sums the negative-purchase sales on the electricity children into 221100, as the final-demand block does, instead of reindexing them onto the 405 axis. - The F09000 description says its values are negative. - test_get_U_commodity_axis_is_parent_not_children mocks derive_negative_purchase_sales (on the 407 axis), so it no longer runs on real data. Co-Authored-By: Claude Opus 5.5 --- bedrock/publish/model_objects.py | 8 +++++++- .../eeio/__tests__/test_electricity_reaggregation.py | 9 +++++++++ bedrock/utils/taxonomy/cornerstone/final_demand.py | 7 +++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/bedrock/publish/model_objects.py b/bedrock/publish/model_objects.py index aba99e9fa..ce46dcafd 100644 --- a/bedrock/publish/model_objects.py +++ b/bedrock/publish/model_objects.py @@ -121,6 +121,9 @@ def get_U() -> pd.DataFrame: ) if _reaggregation_efs_active(): + from bedrock.transform.eeio.sector_disagg_pipeline import ( + reaggregate_electricity_children_y, + ) from bedrock.transform.eeio.usa_matrices import ( derive_disagg_Ytot_reaggregated, derive_U_set_reaggregated, @@ -131,7 +134,10 @@ def get_U() -> pd.DataFrame: intermediate = uset.Udom + uset.Uimp fd_block = _with_negative_purchase_sales( derive_disagg_Ytot_reaggregated()[list(FINAL_DEMANDS)], - derive_negative_purchase_sales(), + # Sum any sales on the G/T/D children into 221100, as the FD block does. + reaggregate_electricity_children_y( + derive_negative_purchase_sales().to_frame() + ).iloc[:, 0], NEGATIVE_PURCHASE_SALES_CODE, ) return assemble_extended_U( diff --git a/bedrock/transform/eeio/__tests__/test_electricity_reaggregation.py b/bedrock/transform/eeio/__tests__/test_electricity_reaggregation.py index e1f6c8dd0..49dd007ee 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_reaggregation.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_reaggregation.py @@ -333,6 +333,15 @@ def test_get_U_commodity_axis_is_parent_not_children( 'bedrock.transform.eeio.usa_matrices.derive_VA_reaggregated', lambda: va, ) + monkeypatch.setattr( + 'bedrock.transform.eeio.usa_matrices.derive_negative_purchase_sales', + # On the 407 axis, as derive_U_with_negatives is when disaggregation is on. + lambda: pd.Series( + 0.0, + index=[c for c in commodities if c != ELECTRICITY_AGGREGATE] + + list(ELECTRICITY_DISAGG_SECTORS), + ), + ) model_objects.get_U.cache_clear() try: u = model_objects.get_U() diff --git a/bedrock/utils/taxonomy/cornerstone/final_demand.py b/bedrock/utils/taxonomy/cornerstone/final_demand.py index a334120bf..4ef7c6077 100644 --- a/bedrock/utils/taxonomy/cornerstone/final_demand.py +++ b/bedrock/utils/taxonomy/cornerstone/final_demand.py @@ -53,6 +53,9 @@ # intermediate purchases (used goods, government sales from stocks). The model # sets negative intermediate cells to zero, and this column carries them, so the # published U rows still add up to commodity output (#1270). It is kept out of -# FINAL_DEMANDS because the BEA-sourced Y has no such column. +# FINAL_DEMANDS because the BEA-sourced Y has no such column. Its values are +# negative: a sale, recorded the way BEA records it. NEGATIVE_PURCHASE_SALES_CODE = 'F09000' -NEGATIVE_PURCHASE_SALES_DESC = 'Sales recorded as negative intermediate purchases' +NEGATIVE_PURCHASE_SALES_DESC = ( + 'Sales recorded as negative intermediate purchases (values are negative)' +) From 25db6d0f7a73cae9b4395e48282f209c36aab52a Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 20:26:13 -0500 Subject: [PATCH 14/15] feat(nowcast): turn on trade_margin_receivers_follow_supply in the v1.0 configs (#1194) Step 4 spread each year's trade margin over the receiving goods by frozen 2017 shares, so goods whose supply grew slower than total trade got margins above their supply (motor homes 336213 at 1.17 of supply in 2024; Phi 0.19). #1260 added the flag; this turns it on in the nine v1.0 configs and adds it to the rebuild's flag check. Needs a rebuild. Co-Authored-By: Claude Opus 5.5 --- bedrock/utils/config/__tests__/test_usa_config.py | 1 + bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml | 3 +++ .../utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml | 3 +++ scripts/rebuild_nowcast_year.py | 1 + 11 files changed, 29 insertions(+) diff --git a/bedrock/utils/config/__tests__/test_usa_config.py b/bedrock/utils/config/__tests__/test_usa_config.py index 61ec10182..e4a0a19f2 100644 --- a/bedrock/utils/config/__tests__/test_usa_config.py +++ b/bedrock/utils/config/__tests__/test_usa_config.py @@ -466,6 +466,7 @@ def test_v1_0_yearly_yamls_load(year: int) -> None: 'cap_margins_at_purchase', 'pin_refinery_secondaries_at_go_mix', 'air_margin_on_aies_level', + 'trade_margin_receivers_follow_supply', } assert all(getattr(cfg, flag) for flag in rebuild_flags) diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml index 972158b5f..d0d008830 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml @@ -60,3 +60,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml index 5fc034cb8..dc849b5ad 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml index 2f0f9df68..d4ac3502e 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml index d93475ba9..76b1a2b6f 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml index 12b51b1da..8284ae75b 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml index f6004b24b..8e173efee 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml index 8d94adec3..60f5c16a6 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml index 6ed12d81e..392df0350 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml index 0f8850589..f081059be 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml @@ -59,3 +59,6 @@ cap_margins_at_purchase: True pin_refinery_secondaries_at_go_mix: True # Air freight revenue at the AIES level from 2023 (#1251, #1274). air_margin_on_aies_level: True +# Spread each year's trade margin over the goods by their own supply, not by +# frozen 2017 shares (#1194, #1260). +trade_margin_receivers_follow_supply: True diff --git a/scripts/rebuild_nowcast_year.py b/scripts/rebuild_nowcast_year.py index afbb0e902..e8affd69d 100644 --- a/scripts/rebuild_nowcast_year.py +++ b/scripts/rebuild_nowcast_year.py @@ -54,6 +54,7 @@ 'cap_margins_at_purchase', 'pin_refinery_secondaries_at_go_mix', 'air_margin_on_aies_level', + 'trade_margin_receivers_follow_supply', ) AFTER_MUT = 'Nowcast_Detail_Use_AfterRedef' GHG_METHODS = ( From 5dd8fb50fe0679727560a8ca3af23b32e7fc6695 Mon Sep 17 00:00:00 2001 From: WesIngwersen Date: Thu, 12 Nov 2026 22:12:44 -0500 Subject: [PATCH 15/15] Pin v1.0 to the CI rebuild v1.0.0_fd271dc (#1270, #1194) Run 43349311057 rebuilt 2017-2024 on main 83dfd6ba (the #1289-#1295 stack) plus trade_margin_receivers_follow_supply. Uploaded to GCS, nothing overwritten. The nine v1.0 configs, the GHG FBS upstream pin, both 2024 pins (new sha256), the B time-series assessment and the config tests move from 8365fa4 to fd271dc. q = L_dom y passes in every year 2017-2024, so test_domestic_L_y_from_use_final_demand_equals_q drops its xfail. Co-Authored-By: Claude Opus 5.5 --- .../ghg_fbs_pin/upstream_inputs_pin.json | 64 +++++++++---------- .../nowcasting/utilities_expense_seed.py | 2 +- .../time_series_matrix_assessment.py | 4 +- .../utils/config/__tests__/test_usa_config.py | 10 +-- .../configs/2026_usa_cornerstone_v1_0.yaml | 8 +-- .../2026_usa_cornerstone_v1_0_2017.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2018.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2019.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2020.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2021.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2022.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2023.yaml | 11 ++-- .../2026_usa_cornerstone_v1_0_2024.yaml | 11 ++-- .../cornerstone_ghg_fbs_2024_pin.json | 4 +- .../nowcast_balanced_sut_2024_pin.json | 8 +-- .../__tests__/test_eeio_diagnostics.py | 7 -- 16 files changed, 98 insertions(+), 97 deletions(-) diff --git a/bedrock/analysis/ghg_fbs_pin/upstream_inputs_pin.json b/bedrock/analysis/ghg_fbs_pin/upstream_inputs_pin.json index bd48768bb..5e75a6dbe 100644 --- a/bedrock/analysis/ghg_fbs_pin/upstream_inputs_pin.json +++ b/bedrock/analysis/ghg_fbs_pin/upstream_inputs_pin.json @@ -5,105 +5,105 @@ "methods": { "GHG_national_Cornerstone_2017": { "fbs": { - "Energy_manufacturing_national_nowcast_2017": "Energy_manufacturing_national_nowcast_2017_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2017": "Energy_highway_fuel_shares_national_2017_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2017": "Energy_industrial_distillate_national_2017_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2017": "Energy_manufacturing_national_nowcast_2017_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2017": "Energy_highway_fuel_shares_national_2017_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2017": "Energy_industrial_distillate_national_2017_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2017, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2018": { "fbs": { - "Energy_manufacturing_national_nowcast_2018": "Energy_manufacturing_national_nowcast_2018_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2018": "Energy_highway_fuel_shares_national_2018_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2018": "Energy_industrial_distillate_national_2018_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2018": "Energy_manufacturing_national_nowcast_2018_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2018": "Energy_highway_fuel_shares_national_2018_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2018": "Energy_industrial_distillate_national_2018_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2018, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2019": { "fbs": { - "Energy_manufacturing_national_nowcast_2019": "Energy_manufacturing_national_nowcast_2019_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2019": "Energy_highway_fuel_shares_national_2019_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2019": "Energy_industrial_distillate_national_2019_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2019": "Energy_manufacturing_national_nowcast_2019_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2019": "Energy_highway_fuel_shares_national_2019_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2019": "Energy_industrial_distillate_national_2019_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2019, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2020": { "fbs": { - "Energy_manufacturing_national_nowcast_2020": "Energy_manufacturing_national_nowcast_2020_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2020": "Energy_highway_fuel_shares_national_2020_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2020": "Energy_industrial_distillate_national_2020_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2020": "Energy_manufacturing_national_nowcast_2020_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2020": "Energy_highway_fuel_shares_national_2020_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2020": "Energy_industrial_distillate_national_2020_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2020, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2021": { "fbs": { - "Energy_manufacturing_national_nowcast_2021": "Energy_manufacturing_national_nowcast_2021_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2021": "Energy_highway_fuel_shares_national_2021_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2021": "Energy_industrial_distillate_national_2021_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2021": "Energy_manufacturing_national_nowcast_2021_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2021": "Energy_highway_fuel_shares_national_2021_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2021": "Energy_industrial_distillate_national_2021_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2021, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2022": { "fbs": { - "Energy_manufacturing_national_nowcast_2022": "Energy_manufacturing_national_nowcast_2022_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2022": "Energy_highway_fuel_shares_national_2022_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2022": "Energy_industrial_distillate_national_2022_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2022": "Energy_manufacturing_national_nowcast_2022_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2022": "Energy_highway_fuel_shares_national_2022_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2022": "Energy_industrial_distillate_national_2022_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2022, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2023": { "fbs": { - "Energy_manufacturing_national_nowcast_2023": "Energy_manufacturing_national_nowcast_2023_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2023": "Energy_highway_fuel_shares_national_2023_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2023": "Energy_industrial_distillate_national_2023_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2023": "Energy_manufacturing_national_nowcast_2023_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2023": "Energy_highway_fuel_shares_national_2023_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2023": "Energy_industrial_distillate_national_2023_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2023, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } }, "GHG_national_Cornerstone_2024": { "fbs": { - "Energy_manufacturing_national_nowcast_2024": "Energy_manufacturing_national_nowcast_2024_v1.0.0_8365fa4.parquet", - "Energy_highway_fuel_shares_national_2024": "Energy_highway_fuel_shares_national_2024_v1.0.0_8365fa4.parquet", - "Energy_industrial_distillate_national_2024": "Energy_industrial_distillate_national_2024_v1.0.0_8365fa4.parquet" + "Energy_manufacturing_national_nowcast_2024": "Energy_manufacturing_national_nowcast_2024_v1.0.0_fd271dc.parquet", + "Energy_highway_fuel_shares_national_2024": "Energy_highway_fuel_shares_national_2024_v1.0.0_fd271dc.parquet", + "Energy_industrial_distillate_national_2024": "Energy_industrial_distillate_national_2024_v1.0.0_fd271dc.parquet" }, "nowcast_mut": { "year": 2024, "stage": "after", - "vintage": "v1.0.0_8365fa4", + "vintage": "v1.0.0_fd271dc", "tables": ["Make", "Use", "Import", "Margins"] } } diff --git a/bedrock/analysis/nowcasting/utilities_expense_seed.py b/bedrock/analysis/nowcasting/utilities_expense_seed.py index 3e5318277..0464bb434 100644 --- a/bedrock/analysis/nowcasting/utilities_expense_seed.py +++ b/bedrock/analysis/nowcasting/utilities_expense_seed.py @@ -174,7 +174,7 @@ #: PUDL release and revises past years (``v2026.8.0`` to the November release #: moved 2024 fuel cost +0.09% and 2016-2024 resale revenue +0.02-0.05%), so #: the balanced SUT pin could not be rebuilt in CI. ``v2026.8.0`` is what -#: ``v1.0.0_8365fa4`` was built on; move it only with a rebuild and re-pin. +#: ``v1.0.0_fd271dc`` was built on; move it only with a rebuild and re-pin. PUDL = 'https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop/v2026.8.0' #: EIA form 923, every fuel receipt at every reporting power plant, 2008-2024, diff --git a/bedrock/analysis/time_series_B_matrix/time_series_matrix_assessment.py b/bedrock/analysis/time_series_B_matrix/time_series_matrix_assessment.py index 69b04e458..4134f00e6 100644 --- a/bedrock/analysis/time_series_B_matrix/time_series_matrix_assessment.py +++ b/bedrock/analysis/time_series_B_matrix/time_series_matrix_assessment.py @@ -64,8 +64,8 @@ #: Constant-dollar year for real B/N (and default normalization year). DOLLAR_YEAR = 2017 -FBS_VINTAGE = 'v1.0.0_8365fa4' -MUT_VINTAGE = 'v1.0.0_8365fa4' +FBS_VINTAGE = 'v1.0.0_fd271dc' +MUT_VINTAGE = 'v1.0.0_fd271dc' #: Year-stem GHG FBS basename template (matches ``2026_usa_cornerstone_v1_0_{year}``). FBS_STEM = 'GHG_national_Cornerstone_{year}' diff --git a/bedrock/utils/config/__tests__/test_usa_config.py b/bedrock/utils/config/__tests__/test_usa_config.py index e4a0a19f2..0938ee3db 100644 --- a/bedrock/utils/config/__tests__/test_usa_config.py +++ b/bedrock/utils/config/__tests__/test_usa_config.py @@ -426,7 +426,7 @@ def test_v1_0_yearly_yamls_load(year: int) -> None: electricity flags its MUT was built with and the delivery sector. Named ``2026_usa_cornerstone_v1_0_{year}``. Each member pins - ``GHG_national_Cornerstone_{year}_v1.0.0_8365fa4``. The unsuffixed + ``GHG_national_Cornerstone_{year}_v1.0.0_fd271dc``. The unsuffixed ``2026_usa_cornerstone_v1_0`` (live ``CANONICAL_USA_CONFIG``) matches the 2024 series member. ``2025_usa_cornerstone_v0_6`` keeps the facilities-transportation 2024 pin for the v0.6 release / waterfall FINAL. @@ -444,9 +444,9 @@ def test_v1_0_yearly_yamls_load(year: int) -> None: assert not cfg.deflate_x_to_detail_io_year_for_B assert cfg.bea_price_index_vintage == '2026Q2' assert cfg.cornerstone_ghg_fbs_filename == ( - f'GHG_national_Cornerstone_{year}_v1.0.0_8365fa4.parquet' + f'GHG_national_Cornerstone_{year}_v1.0.0_fd271dc.parquet' ) - assert cfg.nowcast_mut_vintage == 'v1.0.0_8365fa4' + assert cfg.nowcast_mut_vintage == 'v1.0.0_fd271dc' electricity_flags = { 'electric_output_on_retail_revenue', 'electricity_accounting_residual', @@ -457,7 +457,7 @@ def test_v1_0_yearly_yamls_load(year: int) -> None: 'electricity_delivery_margin', } assert all(getattr(cfg, flag) for flag in electricity_flags) - # Fixes merged 2026-11-10 and 11-11, rebuilt in v1.0.0_8365fa4 (#1249). + # Fixes merged 2026-11-10 and 11-11, rebuilt in v1.0.0_fd271dc (#1249). rebuild_flags = { 'fill_withheld_aies_codes', 'hold_aies_outliers_on_bea', @@ -503,7 +503,7 @@ def test_v1_0_release_yaml_matches_2024_series_member() -> None: y2024 = _load_usa_config_from_file_name('2026_usa_cornerstone_v1_0_2024.yaml') assert base.model_dump() == y2024.model_dump() assert base.cornerstone_ghg_fbs_filename == ( - 'GHG_national_Cornerstone_2024_v1.0.0_8365fa4.parquet' + 'GHG_national_Cornerstone_2024_v1.0.0_fd271dc.parquet' ) set_global_usa_config('2026_usa_cornerstone_v1_0.yaml') assert _select_cornerstone_ghg_fbs_base_name() == 'GHG_national_Cornerstone_2024' diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml index d0d008830..4e5c6110d 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0.yaml @@ -2,7 +2,7 @@ # # Unsuffixed family release for calendar year 2024. Same resolved fields as # 2026_usa_cornerstone_v1_0_2024 (the series member). Detail MUT is the -# after-redefinition build named by nowcast_mut_vintage: v1.0.0_8365fa4, the +# after-redefinition build named by nowcast_mut_vintage: v1.0.0_fd271dc, the # 2017-2024 rebuild on the 2026 BEA annual update with electric output on EIA's # 2017 level and the 2026-11-10 fixes below (#1249). bea_price_index_vintage # 2026Q2 is the index those tables were built with. The GHG loader returns @@ -22,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2024 usa_ghg_data_year: 2024 bea_price_index_vintage: '2026Q2' @@ -30,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2024_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2024_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -47,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml index dc849b5ad..e16ee575c 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2017.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2017. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2017. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2017. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2017 usa_ghg_data_year: 2017 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2017_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2017_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml index d4ac3502e..da3586b28 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2018.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2018. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2018. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2018. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2018 usa_ghg_data_year: 2018 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2018_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2018_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml index 76b1a2b6f..f1d029b0c 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2019.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2019. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2019. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2019. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2019 usa_ghg_data_year: 2019 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2019_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2019_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml index 8284ae75b..be628e59d 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2020.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2020. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2020. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2020. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2020 usa_ghg_data_year: 2020 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2020_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2020_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml index 8e173efee..dd30a2378 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2021.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2021. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2021. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2021. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2021 usa_ghg_data_year: 2021 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2021_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2021_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml index 60f5c16a6..7a9b55899 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2022.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2022. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2022. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2022. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2022 usa_ghg_data_year: 2022 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2022_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2022_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml index 392df0350..9cf338449 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2023.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2023. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2023. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2023. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2023 usa_ghg_data_year: 2023 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2023_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2023_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml index f081059be..7c08996ba 100644 --- a/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml +++ b/bedrock/utils/config/configs/2026_usa_cornerstone_v1_0_2024.yaml @@ -1,9 +1,10 @@ # Cornerstone 2026 model - v1.0, calendar year 2024. # # Same methodology as 2026_usa_cornerstone_v1_0 for year 2024. -# nowcast_mut_vintage v1.0.0_8365fa4 is the 2017-2024 rebuild on the 2026 BEA +# nowcast_mut_vintage v1.0.0_fd271dc is the 2017-2024 rebuild on the 2026 BEA # annual update, with electric output on EIA's 2017 level and the 2026-11-10 -# fixes below (#1249). +# fixes below (#1249), the balance and export fixes of 11-12 (#1270, #1293) +# and the trade margin receivers (#1194). # bea_price_index_vintage 2026Q2 is the index those tables were built with. The # GHG loader returns GHG_national_Cornerstone_2024. # cornerstone_ghg_fbs_filename names the GCS object loaded for E. When @@ -21,7 +22,7 @@ iot_before_or_after_redefinition: after # Data selection ##### usa_detail_io_source: nowcast -nowcast_mut_vintage: 'v1.0.0_8365fa4' +nowcast_mut_vintage: 'v1.0.0_fd271dc' usa_base_io_data_year: 2024 usa_ghg_data_year: 2024 bea_price_index_vintage: '2026Q2' @@ -29,7 +30,7 @@ bea_price_index_vintage: '2026Q2' ##### # Methodology selection ##### -cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2024_v1.0.0_8365fa4.parquet +cornerstone_ghg_fbs_filename: GHG_national_Cornerstone_2024_v1.0.0_fd271dc.parquet apply_io_year_adjustments: False implement_waste_disaggregation: True waste_weights_year: match_io @@ -46,7 +47,7 @@ hold_electric_gas_on_eia: True # applied after Step 7. electric_output_on_eia_level: True electricity_delivery_margin: True -# Fixes merged 2026-11-10, rebuilt together in v1.0.0_8365fa4 (#1249). +# Fixes merged 2026-11-10, rebuilt together in v1.0.0_fd271dc (#1249). # AIES output chain (#1231): withheld codes, 325190 on BEA, gas distribution. fill_withheld_aies_codes: True hold_aies_outliers_on_bea: True diff --git a/bedrock/utils/snapshots/cornerstone_ghg_fbs_2024_pin.json b/bedrock/utils/snapshots/cornerstone_ghg_fbs_2024_pin.json index 3f8499fc0..e09f809ee 100644 --- a/bedrock/utils/snapshots/cornerstone_ghg_fbs_2024_pin.json +++ b/bedrock/utils/snapshots/cornerstone_ghg_fbs_2024_pin.json @@ -2,6 +2,6 @@ "method": "GHG_national_Cornerstone_2024", "usa_config": "2026_usa_cornerstone_v1_0", "gcs_sub_bucket": "transform/output_data", - "filename": "GHG_national_Cornerstone_2024_v1.0.0_8365fa4.parquet", - "sha256": "067cd99beb43b586e26135387d4bd273ad98812e8684a1e3e43e60f1aa88d4d1" + "filename": "GHG_national_Cornerstone_2024_v1.0.0_fd271dc.parquet", + "sha256": "f846719311e07765a06f7b0ce1cd6f0e09bc48caf51bd9ab3238d7a6de148b5d" } diff --git a/bedrock/utils/snapshots/nowcast_balanced_sut_2024_pin.json b/bedrock/utils/snapshots/nowcast_balanced_sut_2024_pin.json index 1bb9afdc1..7a2419fbf 100644 --- a/bedrock/utils/snapshots/nowcast_balanced_sut_2024_pin.json +++ b/bedrock/utils/snapshots/nowcast_balanced_sut_2024_pin.json @@ -4,12 +4,12 @@ "usa_config": "2026_usa_cornerstone_v1_0", "tables": { "use": { - "filename": "Balanced_Detail_Use_SUT_2024_v1.0.0_8365fa4.parquet", - "sha256": "db0dfdb2e23efa4511e1d54c57570ddfc0aa842a6f148cb753f487ce6ee5f889" + "filename": "Balanced_Detail_Use_SUT_2024_v1.0.0_fd271dc.parquet", + "sha256": "2a6130da528c889723a82261c029608927245afdac8d541e8bd71ee16c0d29e9" }, "supply": { - "filename": "Balanced_Detail_Supply_2024_v1.0.0_8365fa4.parquet", - "sha256": "68e283b2f1f76936cb825d467ac5c8e6fb919c10f26d6a95bc7f99ea9543460a" + "filename": "Balanced_Detail_Supply_2024_v1.0.0_fd271dc.parquet", + "sha256": "cc9d68deb93f96aae21d2da8776c743ce942c449d4d005d123cadce1aaa14abc" } } } diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 7b52210a2..85cee4028 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -414,13 +414,6 @@ def test_compare_output_and_L_y( @pytest.mark.eeio_integration -@pytest.mark.xfail( - reason=( - 'Make and Use disagree on commodity output. On MUT v1.0.0_8365fa4 ' - '(2024) one sector fails: 334118 (+4.4%), whose imports exceed its ' - 'use. 2017-2023 pass.' - ), -) def test_domestic_L_y_from_use_final_demand_equals_q( canonical_usa_config: None, ) -> None: