Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions docs/guides/parallel.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,53 @@ molecule test --all --workers cpus-1
`molecule.yml` so that the default scenario handles infrastructure
create/destroy while workers run the test sequences.

### Scenario slicing with `--slice`

When scenario names contain directory separators (e.g.
`appliance_vlans/gathered`, `appliance_vlans/merged`), the `--slice`
flag controls how scenarios are grouped into work units dispatched to
workers.

| `--slice` | Behavior |
|-----------|----------|
| `1` (default) | Group by the first path segment — all CRUD states for a resource run sequentially on one worker. |
| `2` | Each leaf scenario is an independent work unit (no grouping). |

```bash
# Group by resource (default when --workers is used)
molecule test --all --workers 4

# Treat each scenario independently
molecule test --all --workers 4 --slice 2
```

`slice` can also be set in the config file (`config.yml` or
`molecule.yml`) so it travels with the project:

```yaml
# extensions/molecule/config.yml
shared_state: true
slice: 1
```

The CLI flag overrides the config value when both are present.

For example, with 186 scenarios across 37 resources, `slice: 1`
produces 37 work units instead of 186, each containing ~5 scenarios
that run sequentially within the worker.

The CLI `--slice` flag requires `--workers` > 1. When defined only
in the config file, `slice` is silently ignored in sequential mode.

### How it works

1. The **default scenario's `create`** runs first (serial, main process).
2. Prerun tasks run for all scenarios (serial, main process).
3. Scenarios are submitted to a `ProcessPoolExecutor` with the specified
number of workers. Each worker reconstructs a `Config` from the
scenario's `molecule.yml` and runs the scenario's sequence, skipping
`create` and `destroy` (handled by the default scenario).
3. Scenarios are grouped into slices according to `--slice` depth and
submitted to a `ProcessPoolExecutor` with the specified number of
workers. Each worker reconstructs a `Config` from each scenario's
`molecule.yml` and runs the scenario's sequence, skipping `create`
and `destroy` (handled by the default scenario).
4. Results are collected as workers complete.
5. The **default scenario's `destroy`** runs last (serial, main process).

Expand All @@ -58,6 +97,7 @@ molecule test --all --workers 4 --continue-on-failure
### Incompatible options

- `--workers` > 1 cannot be combined with `--destroy=never`.
- `--slice` cannot be used when `--workers` is 1 (no parallelism).

---

Expand Down
11 changes: 11 additions & 0 deletions src/molecule/click_cfg.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,17 @@ def subcommand(self) -> CliOption:
nargs=1,
)

@property
def slice(self) -> CliOption:
"""Scenario grouping depth for worker dispatch."""
return CliOption(
name="slice",
help="Directory depth at which scenarios are grouped into worker units. "
"1 groups by top-level resource, 2 treats each leaf scenario independently.",
default="1",
experimental=True,
)

@property
def workers(self) -> CliOption:
"""Worker count for concurrent scenario execution."""
Expand Down
3 changes: 2 additions & 1 deletion src/molecule/command/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def execute(self, action_args: list[str] | None = None) -> None: # noqa: ARG002


@click_command_ex()
@common_options("continue_on_failure", "parallel", "workers")
@common_options("continue_on_failure", "parallel", "slice", "workers")
def check(ctx: click.Context) -> None: # pragma: no cover
"""Use the provisioner to perform a Dry-Run (destroy, dependency, create, prepare, converge).

Expand All @@ -74,6 +74,7 @@ def check(ctx: click.Context) -> None: # pragma: no cover
"parallel": parallel,
"report": ctx.params["report"],
"shared_state": ctx.params["shared_state"],
"slice": int(ctx.params["slice"]),
"subcommand": subcommand,
"workers": resolve_workers(ctx.params["workers"]),
}
Expand Down
3 changes: 2 additions & 1 deletion src/molecule/command/destroy.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def execute(self, action_args: list[str] | None = None) -> None: # noqa: ARG002


@click_command_ex()
@common_options("continue_on_failure", "driver_name_with_choices", "parallel", "workers")
@common_options("continue_on_failure", "driver_name_with_choices", "parallel", "slice", "workers")
def destroy(ctx: click.Context) -> None: # pragma: no cover
"""Use the provisioner to destroy the instances.

Expand All @@ -78,6 +78,7 @@ def destroy(ctx: click.Context) -> None: # pragma: no cover
"driver_name": ctx.params["driver_name"],
"report": ctx.params["report"],
"shared_state": ctx.params["shared_state"],
"slice": int(ctx.params["slice"]),
"subcommand": subcommand,
"workers": resolve_workers(ctx.params["workers"]),
}
Expand Down
2 changes: 2 additions & 0 deletions src/molecule/command/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def execute(self, action_args: list[str] | None = None) -> None:
"driver_name_with_choices",
"platform_name_with_default",
"parallel",
"slice",
"workers",
"ansible_args",
)
Expand Down Expand Up @@ -80,6 +81,7 @@ def test(ctx: click.Context) -> None: # pragma: no cover
"platform_name": ctx.params["platform_name"],
"report": ctx.params["report"],
"shared_state": ctx.params["shared_state"],
"slice": int(ctx.params["slice"]),
"subcommand": subcommand,
"workers": resolve_workers(ctx.params["workers"]),
}
Expand Down
25 changes: 20 additions & 5 deletions src/molecule/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,19 @@ def _apply_cli_overrides(self) -> None:
This method modifies the config dictionary to include CLI overrides,
creating a single source of truth for configuration values.
"""
# Apply shared_state CLI override ONLY if it was explicitly provided via CLI
ctx = click.get_current_context(silent=True)
if "shared_state" not in self.command_args or ctx is None:
if ctx is None:
return
source = ctx.get_parameter_source("shared_state")
if source == click.core.ParameterSource.COMMANDLINE:
self.config["shared_state"] = self.command_args["shared_state"]

if "shared_state" in self.command_args:
source = ctx.get_parameter_source("shared_state")
if source == click.core.ParameterSource.COMMANDLINE:
self.config["shared_state"] = self.command_args["shared_state"]

if "slice" in self.command_args:
source = ctx.get_parameter_source("slice")
if source == click.core.ParameterSource.COMMANDLINE:
self.config["slice"] = self.command_args["slice"]

def _apply_env_overrides(self) -> None:
"""Apply environment variable overrides to command_args.
Expand Down Expand Up @@ -222,6 +228,15 @@ def shared_state(self) -> bool:
"""
return self.config.get("shared_state", False)

@property
def slice(self) -> int:
"""Directory depth for grouping scenarios into worker units.

Returns:
The slice depth (1 = group by top-level resource, 2 = each leaf independently).
"""
return int(self.config.get("slice", 1))

@property
def command_borders(self) -> bool:
"""Return if command borders are enabled."""
Expand Down
7 changes: 7 additions & 0 deletions src/molecule/data/molecule.json
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,13 @@
"title": "Shared State",
"type": "boolean"
},
"slice": {
"default": 1,
"description": "Directory depth for grouping scenarios into worker units. 1 groups by top-level resource, 2 treats each leaf scenario independently.",
"minimum": 1,
"title": "Slice",
"type": "integer"
},
"verifier": {
"$ref": "#/$defs/VerifierModel"
}
Expand Down
11 changes: 11 additions & 0 deletions src/molecule/scenarios.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,17 @@ def shared_state(self) -> bool:
"""
return any(scenario.config.shared_state for scenario in self.all)

@property
def slice(self) -> int:
"""Directory depth for grouping scenarios into worker units.

Returns:
The configured slice depth from the first scenario's config (default 1).
"""
if self.all:
return self.all[0].config.slice
return 1

def print_matrix(self) -> None:
"""Show the matrix for all scenarios."""
tree = {}
Expand Down
4 changes: 4 additions & 0 deletions src/molecule/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ class ConfigData(TypedDict, total=False):
provisioner: Provisioner config.
scenario: Scenario config.
shared_state: Should state be shared between scenarios.
slice: Directory depth for grouping scenarios into worker units.
verifier: Verifier config.
"""

Expand All @@ -306,6 +307,7 @@ class ConfigData(TypedDict, total=False):
provisioner: ProvisionerData
scenario: ScenarioData
shared_state: bool
slice: int
verifier: VerifierData


Expand Down Expand Up @@ -342,6 +344,7 @@ class CommandArgs(TypedDict, total=False):
report: Whether to show an after-run summary report.
scenario_name: Name of the scenario to target.
shared_state: Whether (some) state should be shared between scenarios.
slice: Directory depth for grouping scenarios into worker units.
subcommand: Name of subcommand being run.
workers: Number of concurrent worker processes for parallel scenario execution.
command_borders: Whether to enable borders around command output.
Expand All @@ -358,6 +361,7 @@ class CommandArgs(TypedDict, total=False):
report: bool
scenario_name: str
shared_state: bool
slice: int
subcommand: str
workers: int
command_borders: bool
Loading
Loading