Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -175,13 +175,18 @@ def _to_summary() -> pd.Series:


def service_rows() -> list[str]:
"""Summary commodity rows from ``42`` onward, without government."""
goods = ('111', '113', '21', '22', '23', '3')
return sorted(
{s for s in _to_summary() if not s.startswith(goods) and not s.startswith('G')}
- set(SEPARATE_ROWS)
"""Summary commodity rows from ``42`` onward, without government.

The same cut the build conditions service exports on
(:func:`~bedrock.transform.iot.nowcast_fd_conditioning.is_service_group`),
so the grader and the conditioning can't disagree on what a service is.
"""
from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415
is_service_group,
)

return sorted({s for s in _to_summary() if is_service_group(s)})


def default_build() -> str:
"""The balanced build the canonical config ships, e.g. ``v1.0.0_5514c3b``."""
Expand Down
27 changes: 27 additions & 0 deletions bedrock/transform/iot/__tests__/test_nowcast_fd_conditioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
EMPTY_GROUP_USD,
_commodity_to_summary,
condition_fd_on_summary,
condition_service_exports_on_summary,
is_service_group,
summary_condition_factors,
)

Expand Down Expand Up @@ -81,3 +83,28 @@ def test_empty_and_sign_flipped_groups_hold_at_one(y: pd.DataFrame) -> None:
def test_years_outside_the_workbook_are_refused(y: pd.DataFrame) -> None:
with pytest.raises(ValueError, match='summary Use SUT'):
summary_condition_factors(y, 2025)


# --- service exports (#1250) ------------------------------------------------


def test_service_exports_land_on_the_published_rows_and_goods_stay() -> None:
groups = _commodity_to_summary()
exports = pd.Series(2.0e9, index=groups.index)
out = condition_service_exports_on_summary(exports, YEAR)
published = _load_usa_summary_sut('Use_SUT_summary', YEAR) # type: ignore[arg-type]
published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6
ours = out.groupby(groups).sum()
for group in ('514', '561', '55', '5412OP'):
assert is_service_group(group)
assert ours[group] == pytest.approx(published.at[group, 'F040'], rel=1e-9)
goods = groups.map(lambda g: not is_service_group(g))
pd.testing.assert_series_equal(out[goods], exports[goods])
within = groups.eq('5412OP')
assert out[within].nunique() == 1 # the detail split inside a group holds


def test_service_groups_exclude_goods_government_and_accounting_rows() -> None:
assert not any(is_service_group(g) for g in ('111CA', '22', '23', '325', 'GFGD'))
assert not any(is_service_group(g) for g in ('Other', 'Used'))
assert all(is_service_group(g) for g in ('42', '481', '523', '81'))
61 changes: 61 additions & 0 deletions bedrock/transform/iot/__tests__/test_nowcast_tax_row_closer.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,64 @@ def test_a_group_whose_scale_would_flip_its_sign_keeps_its_seed(
)
assert report.loc[('TOP', 'g1'), 'held'] == pytest.approx(2.0)
assert any('flip their sign' in w for w in warned)


# --- the import hold (#1288) ------------------------------------------------


def test_the_import_hold_targets_the_published_groups_less_frozen(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(asm, 'commodity_groups', lambda: GROUPS)
monkeypatch.setattr(
asm, 'published_import_groups', lambda year: pd.Series({'g1': 5.0, 'g2': 3.0})
)
frozen = pd.DataFrame({'i1': 0.0, 'MCIF': [0.0, 1.5, 0.0]}, index=GROUPS.index)
(hold,) = asm.import_group_holds(2017, frozen)
assert hold.column == 'MCIF'
assert hold.targets.to_dict() == pytest.approx({'g1': 3.5, 'g2': 3.0})


def test_held_imports_keep_each_group_where_the_rows_would_move_them() -> None:
"""The rows want c1 and c2 to lose 2 of supply and c3 to gain 2; without
the hold, imports move from g1 to g2; held, i1 absorbs it instead."""
supply = pd.DataFrame(
{'i1': [4.0, 4.0, 2.0], 'MCIF': [2.0, 2.0, 2.0]}, index=['c1', 'c2', 'c3']
)
rows = pd.Series({'c1': 5.0, 'c2': 5.0, 'c3': 6.0})
cols = pd.Series({'i1': 10.0, 'MCIF': 6.0})
mask = SutMask.from_pattern(supply)
hold = GroupHold('MCIF', GROUPS, pd.Series({'g1': 4.0, 'g2': 2.0}))
free, _ = _balance_block(
supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True
)
held, _ = _balance_block(
supply, mask, rows, cols, rtol=1e-12, close_rows_exactly=True, holds=(hold,)
)
assert free.loc[['c1', 'c2'], 'MCIF'].sum() < 4.0
assert held.loc[['c1', 'c2'], 'MCIF'].sum() == pytest.approx(4.0, abs=1e-6)
assert held.at['c3', 'MCIF'] == pytest.approx(2.0, abs=1e-6)
pd.testing.assert_series_equal(held.sum(axis=1), rows, check_names=False)


# --- fixed service exports (#1250) ------------------------------------------


def test_service_export_cells_are_fixed_and_goods_stay_free(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
asm,
'commodity_groups',
lambda: pd.Series({'c1': '514', 'c2': '325', 'c3': '561'}),
)
use = pd.DataFrame(
{'i1': [1.0, 1.0, 1.0], 'F04000': [2.0, 3.0, 0.0]}, index=['c1', 'c2', 'c3']
)
zero = pd.DataFrame(False, index=use.index, columns=use.columns)
mask = SutMask(
structural_zero=zero, fixed_value=zero.copy(), sign_lock=zero.astype(int)
)
fixed = asm.fix_service_exports(mask, use).fixed_value
assert fixed['F04000'].to_dict() == {'c1': True, 'c2': False, 'c3': True}
assert not fixed['i1'].any()
8 changes: 8 additions & 0 deletions bedrock/transform/iot/nowcast.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,14 @@ def derive_initial_Y_pur(year: int, download_sources_ok: bool = False) -> pd.Dat
_trade_fbs_commodity_vector('Trade_Exports_2017', download_sources_ok),
published_2017_exports(),
)
from bedrock.utils.config.usa_config import get_usa_config # noqa: PLC0415

if get_usa_config().condition_service_exports_on_summary:
from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415
condition_service_exports_on_summary,
)

exports = condition_service_exports_on_summary(exports, int(year))
y['F04000'] = exports.reindex(y.index).fillna(0.0)
pce = float(pd.to_numeric(y.loc['S00900', 'F01000'], errors='raise'))
y.loc['S00900', 'F04000'] = -pce + _s00900_export_identity_usd()
Expand Down
80 changes: 71 additions & 9 deletions bedrock/transform/iot/nowcast_fd_conditioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,13 @@
targets otherwise) and large fixes on FD-owned rows — custom programming's
2023 residual fell 76 -> 30bn $M in the counterfactual.

⚠️ ``F03000`` and ``F04000`` are NOT conditioned. Inventories and exports are
their own sourced constructions (#746, #729/#771) with their own graders;
⚠️ ``F03000`` and ``F04000`` are NOT conditioned here. Inventories and exports
are their own sourced constructions (#746, #729/#771) with their own graders;
their summary-level standing is recorded in the About doc, and exports are
bl-young's lane.
bl-young's lane. With ``condition_service_exports_on_summary`` set, the
**service** rows of ``F04000`` are conditioned the same way
(:func:`condition_service_exports_on_summary`, #1250); goods exports stay on
Census trade.

Guards, in the order applied per (group, column):

Expand Down Expand Up @@ -95,6 +98,68 @@ def _commodity_to_summary() -> pd.Series:
return pd.Series({code: parents[0] for code, parents in mapping.items()})


def _group_factors(ours: pd.Series, pub: pd.Series) -> tuple[pd.Series, float, float]:
"""published / ours per group, with the guards; plus the USD each guard left."""
empty = ours.abs() < EMPTY_GROUP_USD
flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0)
factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0)
factor[empty | flipped] = 1.0
unreachable = float(pub[empty].abs().sum())
sign_held = float((pub - ours)[flipped].abs().sum())
return factor, unreachable, sign_held


#: Summary groups whose exports stay on Census goods trade: agriculture to
#: manufacturing, and government. ``Other`` and ``Used`` are accounting rows.
GOODS_SUMMARY_PREFIXES: tuple[str, ...] = ('111', '113', '21', '22', '23', '3', 'G')
ACCOUNTING_SUMMARY_ROWS: tuple[str, ...] = ('Other', 'Used')


def is_service_group(group: str) -> bool:
"""Whether summary commodity *group* is a service row (``42`` onward)."""
return not str(group).startswith(GOODS_SUMMARY_PREFIXES) and (
str(group) not in ACCOUNTING_SUMMARY_ROWS
)


def condition_service_exports_on_summary(exports: pd.Series, year: int) -> pd.Series:
"""*exports* with each service group scaled to the summary Use ``F040``, USD.

Service exports are each row's published 2017 value moved by a blend of
ITA categories (#771). The blends drift by about 40% on data processing
(``514``), administrative and support (``561``) and management of
companies (``55``) by 2024, and the balance barely moves exports, so the
drift reaches the tables (#1250). BEA publishes the summary rows every
year, so each service group is held to them; the blends only split a
group between its detail rows. Goods rows and rows with no summary group
are returned unchanged, and the guards of :func:`summary_condition_factors`
apply.
"""
if year not in SUMMARY_SUT_YEARS:
raise ValueError(
f'the summary Use SUT covers {SUMMARY_SUT_YEARS[0]}-'
f'{SUMMARY_SUT_YEARS[-1]}; got {year}'
)
groups = _commodity_to_summary().reindex(exports.index)
service = groups.map(lambda g: isinstance(g, str) and is_service_group(g))
service = service.fillna(False).astype(bool)
published = _load_usa_summary_sut('Use_SUT_summary', year) # type: ignore[arg-type]
published = published.apply(pd.to_numeric, errors='coerce').fillna(0.0) * 1e6
ours = exports[service].groupby(groups[service]).sum()
pub = published['F040'].reindex(ours.index).fillna(0.0)
factor, unreachable, sign_held = _group_factors(ours, pub)
if unreachable or sign_held:
logger.info(
'service export conditioning %s: %.0fM published unreachable (our '
'group empty), %.0fM held on sign disagreement',
year,
unreachable / 1e6,
sign_held / 1e6,
)
per_row = groups.map(factor).where(service, 1.0).astype(float).fillna(1.0)
return exports * per_row


def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame:
"""Summary group x column scale factors for ``y`` at ``year``, guards applied.

Expand All @@ -119,13 +184,10 @@ def summary_condition_factors(y: pd.DataFrame, year: int) -> pd.DataFrame:
# The summary workbooks truncate the codes to four characters
# (F01000 -> F010).
pub = published[column[:4]].reindex(ours.index).fillna(0.0)
empty = ours.abs() < EMPTY_GROUP_USD
flipped = ~empty & (np.sign(ours) != np.sign(pub)) & (pub.abs() >= 1.0)
factor = (pub / ours.replace(0.0, np.nan)).fillna(1.0)
factor[empty | flipped] = 1.0
factor, missed, held = _group_factors(ours, pub)
factors[column] = factor
unreachable += float(pub[empty].abs().sum())
sign_held += float((pub - ours)[flipped].abs().sum())
unreachable += missed
sign_held += held
if unreachable or sign_held:
logger.info(
'FD conditioning %s: %.0fM published unreachable (our group '
Expand Down
84 changes: 82 additions & 2 deletions bedrock/transform/iot/nowcast_sut_assembly.py
Original file line number Diff line number Diff line change
Expand Up @@ -914,14 +914,47 @@ def published_tax_groups(year: int) -> pd.DataFrame:

``SUB`` is negative, as BEA stores it and as the balance carries it.
"""
return _published_supply_groups(int(year), HELD_TAX_COLUMNS)


def _published_supply_groups(year: int, columns: Sequence[str]) -> pd.DataFrame:
"""Summary Supply *columns* for *year*, $M, by summary commodity."""
supply = _load_usa_summary_sut('Supply_summary', year) # type: ignore[arg-type]
supply = supply.rename(columns=lambda c: str(c).strip())
groups = set(commodity_groups())
frame = supply[list(HELD_TAX_COLUMNS)].apply(pd.to_numeric, errors='coerce')
frame = supply[list(columns)].apply(pd.to_numeric, errors='coerce')
frame.index = [str(i).strip() for i in frame.index]
return frame.loc[[g for g in frame.index if g in groups]].fillna(0.0)


#: The Supply import column the balance can hold to BEA's summary groups.
HELD_IMPORT_COLUMN = 'MCIF'


def published_import_groups(year: int) -> pd.Series:
"""The summary Supply ``MCIF`` column for *year*, $M, by summary commodity."""
return _published_supply_groups(int(year), (HELD_IMPORT_COLUMN,))[
HELD_IMPORT_COLUMN
]


def import_group_holds(year: int, frozen_supply: pd.DataFrame) -> tuple[GroupHold, ...]:
"""The engine's ``MCIF`` group hold: published group sums less frozen, $M.

#785 already scales each summary group of the seed's imports to BEA's
summary ``MCIF``, but the balance otherwise holds imports only as one soft
column total (T7), so the row identities move them between groups. In
2017, service imports enter at the published $267.2bn and leave at
$247.0bn (#1288). Held by group, they keep the published sums and the
detail split inside each group stays free.
"""
groups = commodity_groups().reindex(frozen_supply.index)
frozen = frozen_supply[HELD_IMPORT_COLUMN].astype(float).groupby(groups).sum()
published = published_import_groups(int(year))
target = published - frozen.reindex(published.index).fillna(0.0)
return (GroupHold(column=HELD_IMPORT_COLUMN, groups=groups, targets=target),)


def commodity_groups() -> pd.Series:
"""Each detail commodity's summary commodity group."""
return pd.Series(
Expand Down Expand Up @@ -1035,6 +1068,38 @@ def rows(z_supply: pd.DataFrame) -> pd.DataFrame:
return rows


#: The Use column whose service cells are fixed when exports are conditioned.
EXPORT_COLUMN = 'F04000'


def fix_service_exports(mask: SutMask, use: pd.DataFrame) -> SutMask:
"""New Use mask with the service ``F04000`` cells Tier-1 fixed against *use*.

With ``condition_service_exports_on_summary``, the seed's service exports
already sit on BEA's summary ``F040`` groups (#1250). The balance
otherwise moves them freely: on ``v1.0.0_fd271dc`` it took pipelines'
(``486``) error from 47 to 527 $M a year, and banking's (``521CI``) from
51 to 673. Fixed, they leave the balance as conditioned, and the rows
close on their other cells. Structural zeros stay structural.
"""
from bedrock.transform.iot.nowcast_fd_conditioning import ( # noqa: PLC0415
is_service_group,
)

groups = commodity_groups().reindex(mask.index)
service = groups.map(lambda g: isinstance(g, str) and is_service_group(g))
rows = service.fillna(False).astype(bool) & ~mask.structural_zero[EXPORT_COLUMN]
fixed = mask.fixed_value.copy()
fixed.loc[rows, EXPORT_COLUMN] = True
out = SutMask(
structural_zero=mask.structural_zero,
fixed_value=fixed,
sign_lock=mask.sign_lock,
)
out.validate_against(use)
return out


def assemble(
year: int,
*,
Expand All @@ -1057,6 +1122,9 @@ def assemble(
When ``pin_refinery_secondaries_at_go_mix`` is on, the Supply seed is then
rewritten so held co-product cells enter GRAS at seed share times the
T17-at-seed column pin, and Tier-1 is overlaid from that held list.

When ``condition_service_exports_on_summary`` is on, the service export
cells are Tier-1 fixed too (:func:`fix_service_exports`).
"""
# Gross output and the summary supply-use controls must be one BEA annual
# update; the targets balance one against the other (#1157).
Expand All @@ -1077,6 +1145,8 @@ def assemble(
seeds, masks, pin = apply_refinery_secondaries_go_mix_pin(
int(year), seeds, masks, targets
)
if get_usa_config().condition_service_exports_on_summary:
masks = {**masks, 'use': fix_service_exports(masks['use'], seeds['use'])}
return YearBalance(
year=int(year),
seeds=seeds,
Expand All @@ -1100,6 +1170,7 @@ def balance_year(
constrain_electricity_pce_cell: bool | None = None,
convert_tax_rows: bool = True,
hold_tax_groups: bool = True,
hold_import_groups: bool | None = None,
) -> YearBalance:
"""Assemble and balance one year: split, offset, engine, restore.

Expand All @@ -1110,7 +1181,11 @@ def balance_year(
Use ``T00TOP``/``T00SUB`` rows stay a conversion of the Supply columns.
*hold_tax_groups* holds ``TOP``/``SUB`` to the published summary groups,
with the detail free inside each group (:func:`tax_group_holds`).
*hold_import_groups* does the same for ``MCIF`` (:func:`import_group_holds`);
it defaults to the ``hold_imports_on_summary_groups`` config flag.
"""
if hold_import_groups is None:
hold_import_groups = get_usa_config().hold_imports_on_summary_groups
mode = resolve_pce_constraint(pce_constraint, constrain_electricity_pce_cell)
assembled = assemble(
int(year),
Expand All @@ -1135,7 +1210,12 @@ def run(
pce_eia_band_m=band,
tax_rows=tax_row_operator(year, frozen) if convert_tax_rows else None,
group_holds=(
tax_group_holds(year, frozen['supply']) if hold_tax_groups else ()
(tax_group_holds(year, frozen['supply']) if hold_tax_groups else ())
+ (
import_group_holds(year, frozen['supply'])
if hold_import_groups
else ()
)
),
)
return out, restore_fixed_blocks(out.blocks, frozen)
Expand Down
Loading
Loading