Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions docs/internal/steps/expansion_design.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ class StudyLoader(Step):
- **Chain resolution**: When this step appears inside a larger Chain, `with_input()` resolves
it so the built chain integrates into the parent chain.

- **UID consistency**: `_exca_uid_dict_override` on Step delegates to the resolved Chain
representation, so `StudyLoader(transforms=[T1])` and `Chain([StudyLoader(), T1])`
- **UID consistency**: `_exca_uid_dict_override` on Step exports the resolution,
so `StudyLoader(transforms=[T1])` and `Chain([StudyLoader(), T1])`
produce the same UID.

### How caching works
Expand Down Expand Up @@ -103,6 +103,8 @@ return `self` from `_resolve_step()`, so re-resolution is a no-op.
### 5. UID consistency via `_exca_uid_dict_override`

`utils.py` is updated to support `None` return (opt-out). Step's override:
- Fast path `None` if `"has_resolve"` not in `_step_flags`
- `None` if `_resolve_step()` returns `self`
- Otherwise delegates to the returned Step's `_exca_uid_dict_override()`
- Otherwise the resolution's uid export

`ConfigExporter` applies overrides recursively: nested resolutions affect cache
keys and `uid.yaml`; `full-uid.yaml`/`config.yaml` retain declared configs.
4 changes: 1 addition & 3 deletions exca/steps/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,12 +299,10 @@ def _make_paths(self, aligned: tp.Sequence[Step]) -> backends.StepPaths:
return paths

def _exca_uid_dict_override(self) -> dict[str, tp.Any] | None:
if "has_resolve" not in self._step_flags:
return None
built = utils.resolved_step(self)
if built is self:
return None
return built._exca_uid_dict_override()
return exca.utils.ConfigExporter(uid=True, exclude_defaults=True).apply(built)

def lookup(
self,
Expand Down
7 changes: 4 additions & 3 deletions exca/steps/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,11 @@ def _run_items(self, batch: items.StepItems) -> items.StepItems:
f"a step), got {self.infra!r}"
)
cbatches = []
for child in self.steps:
uids = [identity.materialize_uid(child, v) for v in batch]
for variant in self.steps:
resolved = utils.resolved_step(variant)
uids = [identity.materialize_uid(resolved, v) for v in batch]
child_batch = items.StepItems(source=dict(zip(uids, batch)), uids=uids)
cbatches.append(self.infra._prepare(child, child_batch))
cbatches.append(self.infra._prepare(resolved, child_batch))
with self.infra._claim(cbatches) as claimed:
if claimed.ready:
self.infra._execute(claimed.ready)
Expand Down
29 changes: 28 additions & 1 deletion exca/steps/test_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import exca

from . import backends, conftest, identity, items, utils
from . import backends, conftest, helpers, identity, items, utils
from .base import Chain, Step

# =============================================================================
Expand Down Expand Up @@ -450,6 +450,33 @@ def test_resolve_step_uid_consistency() -> None:
assert step_uid == chain_uid


class _ResolvesToMult(Step):
def _run(self, value: float) -> float:
return value

def _resolve_step(self) -> Step:
return conftest.Mult(infra=self.infra)


class _StepWithBody(Step):
body: Step

def _run(self, value: float) -> float:
return value


def test_nested_resolution_drives_uid() -> None:
resolving_uid = identity.step_uid([_StepWithBody(body=_ResolvesToMult())])
assert resolving_uid == identity.step_uid([_StepWithBody(body=conftest.Mult())])


def test_parallel_caches_the_resolved_step_result(tmp_path: Path) -> None:
infra: tp.Any = {"backend": "Cached", "folder": tmp_path}
sweep = helpers.Parallel(steps=[_ResolvesToMult()], infra=infra)
sweep.run(5.0)
assert sweep.steps[0].lookup(5.0).result() == 10.0


def test_resolve_step_runtime_checks(tmp_path: Path) -> None:
force_infra: tp.Any = {"backend": "Cached", "folder": tmp_path, "mode": "force"}
chain_infra: tp.Any = {"backend": "Cached", "folder": tmp_path}
Expand Down
Loading