diff --git a/.claude/hooks/posttooluse_format.py b/.claude/hooks/posttooluse_format.py new file mode 100644 index 0000000..bb637f4 --- /dev/null +++ b/.claude/hooks/posttooluse_format.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""PostToolUse hook: auto-format and lint a Python file after Edit/Write. + +Reads a JSON hook payload on stdin with `tool_name` and `tool_input`. +Runs `black` then `ruff check --fix` on the touched file. If ruff still +reports issues afterwards, prints them to stderr and exits 2 so the model +sees them as advisory feedback (not a hard block). +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + + +def run(args: list[str]) -> subprocess.CompletedProcess | None: + try: + # nosec B603 - args are built here from sys.executable plus a fixed + # tool name; no shell is involved and nothing is user-supplied. + return subprocess.run( # noqa: S603 # nosec B603 + args, check=False, capture_output=True, text=True, timeout=60 + ) + except FileNotFoundError: + return None + except Exception: + return None + + +def main() -> None: + try: + payload = json.load(sys.stdin) + except Exception: + sys.exit(0) + + tool_input = payload.get("tool_input", {}) or {} + file_path = tool_input.get("file_path", "") or "" + + if not file_path.endswith(".py"): + sys.exit(0) + + if not os.path.exists(file_path): + sys.exit(0) + + black_result = run([sys.executable, "-m", "black", file_path]) + if black_result is None: + # black not installed / not runnable - tolerate silently. + sys.exit(0) + + ruff_fix_result = run([sys.executable, "-m", "ruff", "check", "--fix", file_path]) + if ruff_fix_result is None: + # ruff not installed / not runnable - tolerate silently. + sys.exit(0) + + ruff_check_result = run([sys.executable, "-m", "ruff", "check", file_path]) + if ruff_check_result is None: + sys.exit(0) + + if ruff_check_result.returncode != 0: + output = (ruff_check_result.stdout or "") + (ruff_check_result.stderr or "") + print(output.strip(), file=sys.stderr) + sys.exit(2) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.claude/hooks/pretooluse_guard.py b/.claude/hooks/pretooluse_guard.py new file mode 100644 index 0000000..00ddfeb --- /dev/null +++ b/.claude/hooks/pretooluse_guard.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""PreToolUse hook: block dangerous git operations and edits to protected data dirs. + +Reads a JSON hook payload on stdin with `tool_name` and `tool_input`. +Exit 0 to allow. Exit 2 (with a stderr message) to block the tool call. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys + +# Directories that must never be written to or staged. +PROTECTED_DIR_PATTERNS = [ + r"(^|[/\\])data([/\\]|$)", + r"(^|[/\\])logs([/\\]|$)", + r"(^|[/\\])d:[/\\]l0_raw([/\\]|$)", + r"(^|[/\\])d:[/\\]l1_processed([/\\]|$)", +] + +PROTECTED_STAGE_PATTERNS = [ + r"(^|[/\\])data([/\\]|$)", + r"(^|[/\\])logs([/\\]|$)", + r"(^|[/\\])\.venv([/\\]|$)", + r"(^|[/\\])venv([/\\]|$)", + r"(^|[/\\])htmlcov([/\\]|$)", +] + +EXACT_PROTECTED_STAGE_FILES = { + "config/config.yaml", + "config\\config.yaml", + ".coverage", + "coverage.xml", +} + + +def block(message: str) -> None: + print(message, file=sys.stderr) + sys.exit(2) + + +def split_commands(command: str) -> list[str]: + """Split a shell command on &&, ;, and | into individual segments.""" + # Not a full shell parser, but good enough to catch chained dangerous + # commands. Avoid splitting inside quotes where reasonably possible. + parts = re.split(r"&&|\|\||;|\|", command) + return [p.strip() for p in parts if p.strip()] + + +def current_branch() -> str | None: + try: + # nosec B603 B607 - fixed argv, no shell. Resolving git by PATH is + # intentional so the hook works across platforms and installs. + result = subprocess.run( # noqa: S603, S607 # nosec B603 B607 + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + return None + return None + + +def tokens_of(segment: str) -> list[str]: + return segment.split() + + +def is_git_subcommand(tokens: list[str], sub: str, index: int = 1) -> bool: + return len(tokens) > index and tokens[0] == "git" and tokens[index] == sub + + +def check_git_push(tokens: list[str], segment: str) -> None: + if not is_git_subcommand(tokens, "push"): + return + + has_force = False + has_force_with_lease = False + for tok in tokens[2:]: + if tok == "--force-with-lease" or tok.startswith("--force-with-lease="): + has_force_with_lease = True + elif tok in ("--force", "-f"): + has_force = True + + # Determine push target (best-effort: look for main/master in args) + targets_main = bool(re.search(r"\b(main|master)\b", segment)) + + if has_force and not has_force_with_lease: + block( + "Blocked: 'git push --force'/'-f' is not allowed. " + "Use '--force-with-lease' instead, and never on main/master." + ) + + if has_force_with_lease and targets_main: + block( + "Blocked: 'git push --force-with-lease' targeting main/master " + "is not allowed." + ) + + if targets_main and not (has_force or has_force_with_lease): + block("Blocked: 'git push' targeting main/master is not allowed.") + + +def check_git_commit(tokens: list[str]) -> None: + if not is_git_subcommand(tokens, "commit"): + return + + if any(t in ("--no-verify", "-n") for t in tokens[2:]): + block("Blocked: 'git commit --no-verify'/'-n' is not allowed.") + + branch = current_branch() + if branch in ("main", "master"): + block( + f"Blocked: 'git commit' while on branch '{branch}'. " + "Create a feature branch first (see the git-branch-pr skill)." + ) + + +def check_git_dangerous(tokens: list[str], segment: str) -> None: + if not tokens or tokens[0] != "git": + return + + sub = tokens[1] if len(tokens) > 1 else "" + + if sub == "reset" and "--hard" in tokens[2:]: + block("Blocked: 'git reset --hard' is not allowed.") + + if sub == "clean": + flag_blob = "".join(t.lstrip("-") for t in tokens[2:] if t.startswith("-")) + if "f" in flag_blob and ("d" in flag_blob or "x" in flag_blob): + block("Blocked: 'git clean' with force+destructive flags is not allowed.") + + if sub == "rebase": + branch = current_branch() + if branch in ("main", "master"): + block("Blocked: 'git rebase' while on main/master is not allowed.") + + if sub == "filter-branch": + block("Blocked: 'git filter-branch' is not allowed.") + + if sub == "push" and "--mirror" in tokens[2:]: + block("Blocked: 'git push --mirror' is not allowed.") + + if sub == "update-ref" and "-d" in tokens[2:]: + block("Blocked: 'git update-ref -d' is not allowed.") + + if sub == "reflog" and len(tokens) > 2 and tokens[2] == "delete": + block("Blocked: 'git reflog delete' is not allowed.") + + +def check_git_add(tokens: list[str]) -> None: + if not tokens or tokens[0] != "git": + return + if len(tokens) < 2 or tokens[1] not in ("add", "stage"): + return + + for raw in tokens[2:]: + if raw.startswith("-"): + continue + normalized = raw.strip("'\"") + norm_slash = normalized.replace("\\", "/") + if ( + norm_slash in EXACT_PROTECTED_STAGE_FILES + or normalized in EXACT_PROTECTED_STAGE_FILES + or norm_slash.endswith((".coverage", "coverage.xml")) + ): + block(f"Blocked: staging protected file '{normalized}' is not allowed.") + for pattern in PROTECTED_STAGE_PATTERNS: + if re.search(pattern, normalized, re.IGNORECASE): + block( + f"Blocked: staging path '{normalized}' under a protected " + "directory (data/, logs/, .venv/, venv/, htmlcov/) is not " + "allowed." + ) + + +def check_bash(command: str) -> None: + for segment in split_commands(command): + tokens = tokens_of(segment) + check_git_push(tokens, segment) + check_git_commit(tokens) + check_git_dangerous(tokens, segment) + check_git_add(tokens) + + +def check_edit_write(file_path: str) -> None: + if not file_path: + return + normalized = file_path.replace("\\", "/") + for pattern in PROTECTED_DIR_PATTERNS: + if re.search(pattern, normalized, re.IGNORECASE): + block( + f"Blocked: writing to '{file_path}' under a protected data " + "directory (data/, logs/, D:/L0_raw, D:/L1_processed) is not " + "allowed." + ) + if re.search(pattern, file_path, re.IGNORECASE): + block( + f"Blocked: writing to '{file_path}' under a protected data " + "directory (data/, logs/, D:/L0_raw, D:/L1_processed) is not " + "allowed." + ) + + +def main() -> None: + try: + payload = json.load(sys.stdin) + except Exception: + # If we can't parse the payload, fail open (allow) rather than + # blocking unrelated tool calls due to a hook plumbing issue. + sys.exit(0) + + tool_name = payload.get("tool_name", "") + tool_input = payload.get("tool_input", {}) or {} + + if tool_name == "Bash": + command = tool_input.get("command", "") or "" + check_bash(command) + elif tool_name in ("Edit", "Write"): + file_path = tool_input.get("file_path", "") or "" + check_edit_write(file_path) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..19e8298 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,35 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "python .claude/hooks/pretooluse_guard.py" + } + ] + }, + { + "matcher": "Edit|Write", + "hooks": [ + { + "type": "command", + "command": "python .claude/hooks/pretooluse_guard.py" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Edit|Write", + "hooks": [ + { + "type": "command", + "command": "python .claude/hooks/posttooluse_format.py" + } + ] + } + ] + } +} diff --git a/.claude/skills/git-branch-pr/SKILL.md b/.claude/skills/git-branch-pr/SKILL.md new file mode 100644 index 0000000..601292a --- /dev/null +++ b/.claude/skills/git-branch-pr/SKILL.md @@ -0,0 +1,77 @@ +--- +name: git-branch-pr +description: Use when creating a feature/fix branch, keeping a branch current with main, or opening/merging a pull request in this repo. Covers branch naming, never committing to main, rebase vs merge, gh pr create using the repo's PR template, CI-green requirement, squash-merge default, and safe force-push rules. +--- + +# git-branch-pr + +## Branch naming and creation + +- Prefixes: `feat/`, `fix/`, `docs/`, `refactor/`. +- Never commit directly to `main`. +- Always branch from an up-to-date `main`: + ```powershell + git checkout main + git pull origin main + git checkout -b feat/my-slug + ``` + +## Keeping a branch current with main + +- **Unpushed / solo work on the branch**: prefer `git rebase main` to keep + history linear. +- **Once the branch is pushed and/or shared with others**: prefer + `git merge main` instead of rebasing, to avoid rewriting shared history. + +## Opening a PR + +Use `gh pr create`, filling in the sections from +`.github/pull_request_template.md` (Summary, Problem, Approach, Changes +Made, Tests, Risks, Rollback Plan, Checklist): + +```powershell +gh pr create --title "feat: short description" --body "$(cat <<'EOF' +## Summary +... + +## Problem +... + +## Approach +... + +### Changes Made +- ... + +## Tests +- ... + +## Risks +- ... + +## Rollback Plan +- ... + +## Checklist +- [ ] Lint/format/typecheck clean (ruff, black, mypy) +- [ ] Tests updated; coverage >=70% +- [ ] Documentation updated (README, CHANGELOG, docs/) +- [ ] Pre-commit hooks pass +- [ ] CI pipeline green +EOF +)" +``` + +## Merging + +- Require CI green before merge — do not merge on red or pending checks. +- Squash-merge is the default merge strategy for this repo. +- Delete the branch after merge (`gh pr merge --squash --delete-branch`, or + delete manually if merged via the web UI). + +## Force-pushing + +- Never force-push shared/pushed branches with plain `--force`. +- If a force-push is genuinely needed after a rebase, use + `--force-with-lease` only, and only on your own feature branch — never on + `main`/`master`. diff --git a/.claude/skills/git-commit/SKILL.md b/.claude/skills/git-commit/SKILL.md new file mode 100644 index 0000000..c102080 --- /dev/null +++ b/.claude/skills/git-commit/SKILL.md @@ -0,0 +1,52 @@ +--- +name: git-commit +description: Use when the user asks to commit changes in this repo. Covers deliberate file staging (never bulk `git add -A`/`.`), Conventional Commits message style matching this repo's history, running pre-commit on staged files, and files that must never be staged (config/config.yaml, data/, logs/, venv dirs, coverage artifacts). +--- + +# git-commit + +## Staging: deliberate, never bulk + +- Never run `git add -A` or `git add .`. +- Enumerate each path explicitly, e.g. `git add src/eddypro_batch_processor/core.py tests/test_core.py`. +- Before staging, run `git status` to see the full set of changes and decide, + file by file, what belongs in this commit. +- Never stage: + - `config/config.yaml` (the user's live machine-specific working config) + - anything under `data/`, `logs/`, `.venv/`, `venv/`, `htmlcov/` + - `.coverage`, `coverage.xml` + If any of these show up in `git status` as modified/untracked, leave them + unstaged and mention it to the user rather than silently including them. + +## Commit message style + +Follow Conventional Commits, matching this repo's actual history: + +``` +feat: add metadata population planning and analysis docs +fix(cli): correct exit code on validation failure +docs: align logging config docs +refactor: simplify project filenames and clean up docs +test: add tests for ECMD validation +chore: bump dependency pins +``` + +- Type prefixes: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`. +- Optional scope in parens, e.g. `feat(logging):`, `fix(core):`. +- Subject in imperative mood, no trailing period. +- Add a body with bullet points for anything non-trivial. + +## Before committing + +1. `git status` — confirm exactly the intended files are staged and nothing + from the never-stage list slipped in. +2. `git diff --staged` — review the actual diff, not just filenames. +3. `pre-commit run --files ` — run hooks against just the + staged files before committing (list them explicitly, don't use + `--all-files` for a routine commit). +4. Fix anything pre-commit flags, re-stage, and re-run before committing. + +## Never + +- Never use `git commit --no-verify` or `-n` to skip hooks. +- Never commit on `main`/`master` directly — see the `git-branch-pr` skill. diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md new file mode 100644 index 0000000..696af28 --- /dev/null +++ b/.claude/skills/release/SKILL.md @@ -0,0 +1,60 @@ +--- +name: release +description: Use when cutting a new release of eddypro-batch-processor — bumping the version, moving CHANGELOG [Unreleased] entries into a dated section, verifying the installed version, and tagging. Do not use for routine feature commits. +--- + +# release + +## 1. Bump the version + +- Edit `version = "..."` in `pyproject.toml` to the new version. +- **Do not** hand-edit `__version__` in + `src/eddypro_batch_processor/__init__.py` — it is now derived + automatically via `importlib.metadata` from the installed package + metadata, so it must not be set manually. + +## 2. Update CHANGELOG.md + +- Move the contents of the `## [Unreleased]` section into a new dated + section following [Keep a Changelog](https://keepachangelog.com/) + format: + + ```markdown + ## [Unreleased] + + ## [0.4.0] - 2026-08-18 + + ### Added + - ... + + ### Fixed + - ... + ``` + +- Leave `## [Unreleased]` at the top, empty, ready for the next cycle. +- Keep category headers (`Added`, `Changed`, `Deprecated`, `Removed`, + `Fixed`, `Security`) consistent with existing entries. + +## 3. Verify + +Reinstall/refresh the editable install if needed, then confirm the +version resolves correctly: + +```powershell +.venv\Scripts\Activate.ps1 +pip install -e . --no-deps +python -c "import eddypro_batch_processor as m; print(m.__version__)" +``` + +The printed version must match the new `pyproject.toml` version. + +## 4. Tag and push + +```powershell +git tag v0.4.0 +git push origin v0.4.0 +``` + +Only push the tag once the version bump and CHANGELOG commit are merged to +`main` via the normal PR flow (see the `git-branch-pr` skill) — do not tag +an unmerged branch. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf21f59..ed73cd4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,9 +38,6 @@ jobs: - name: Run ruff linting run: ruff check . --output-format=github - - name: Run ruff formatting check - run: ruff format --check . - - name: Run black formatting check run: black --check --diff . diff --git a/.gitignore b/.gitignore index 2cf4ae1..1ea06fb 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ config/* # Test coverage reports .coverage .coverage.* +**/.coverage +**/.coverage.* coverage.xml htmlcov/ .pytest_cache/ @@ -60,6 +62,7 @@ Desktop.ini *.sublime-workspace *.code-workspace .mypy_cache/ +**/.mypy_cache/ .ruff_cache/ .dmypy.json dmypy.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a22b6fe..a638f78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,14 +13,14 @@ repos: - id: debug-statements - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.8 + rev: v0.16.3 hooks: - - id: ruff + - id: ruff-check args: [--fix, --exit-non-zero-on-fix] types_or: [python, pyi, jupyter] - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 26.5.1 hooks: - id: black types_or: [python, pyi] @@ -31,7 +31,7 @@ repos: - id: mypy additional_dependencies: [types-PyYAML, types-requests] args: [--ignore-missing-imports, --no-strict-optional] - exclude: ^(tests/|docs/|src/eddypro_batch_processor\.py) + exclude: ^(tests/|docs/) - repo: https://github.com/PyCQA/bandit rev: 1.7.5 @@ -39,4 +39,4 @@ repos: - id: bandit args: [-c, pyproject.toml] additional_dependencies: ["bandit[toml]", "pbr"] - exclude: ^(tests/|src/eddypro_batch_processor\.py) + exclude: ^tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9788dd8..cdd7181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,53 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`--version` global CLI flag** to print the installed package version and exit. + +- **`monitoring_enabled` config key and `--monitor`/`--no-monitor` CLI flags** + (available on both `run` and `scenarios`). When disabled, no metrics files + are written and `metrics_interval_seconds` is ignored. + +- **`--reports-dir` on `scenarios`**, matching the flag already available on + `run` and `status`. + +- **Multiprocessing wired for `run`**: `--mp --max-proc N` (or + `multiprocessing: true` / `max_processes: N` in config) now parallelizes + `run` across years, one worker per year, up to `max_processes` concurrent + workers. See [docs/MULTI_YEAR_RUNS.md](docs/MULTI_YEAR_RUNS.md). + +- **Scenario HTML reports**: `scenarios` now generates a per-scenario HTML + report at `{output_dir}/{scenario_suffix}/reports/run_report.html`, plus + one aggregate comparison report across all scenarios. + +- **Run manifest provenance**: manifests now include a `provenance` block + (git SHA + dirty flag, package version, EddyPro executable path and + SHA256 checksum, `sys.argv`), a `manifest_schema_version: 2` field, and a + per-year `years[]` array (`{year, status, duration_seconds, error, + output_dir}`) so failed years are visible instead of silently vanishing. + - Investigation doc on EddyPro execution path differences between `run` and `scenarios`. +### Fixed + +- **Performance monitoring was measuring the wrong process.** The monitor + previously sampled the `cmd.exe`/shell wrapper used to launch EddyPro, so + CPU and disk I/O were always reported as `0.0`. It now samples the whole + EddyPro process tree, producing real CPU, memory, and disk numbers, + including derived disk rates (`read_mb_per_s`, `write_mb_per_s`) and a + CPU/MEMORY/DISK_THROUGHPUT/DISK_IOPS bottleneck classification shown in + the HTML report and run manifest. + +- **`config_checksum` is now a real SHA256** of the canonicalised config + JSON (plus a separate SHA256 of the raw config file). It previously used + Python's per-process-salted `hash()`, so the checksum changed on every + run even with an identical config, making it useless for detecting real + changes. + +- **Manifest timestamps are now UTC with explicit offset**, and the + manifest is written atomically (temp file + `os.replace`) at the start of + a run (`status: "running"`) and rewritten at the end — including when + every year fails, so a manifest always exists after a run is attempted. + - **Static .metadata population from ECMD** - Selects the ECMD row closest to but not later than the processing year - Populates site, timing, and instrument fields plus station identifiers diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e9d6707 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,67 @@ +# CLAUDE.md + +Project brief for Claude Code sessions in this repo. + +## What this is + +`eddypro-batch-processor` batch-runs LI-COR EddyPro over multi-year +eddy-covariance flux data on Windows. It generates EddyPro `.eddypro` +project files from templates, runs `eddypro_rp`/`eddypro_fcc` as +subprocesses across years/sites/parameter scenarios, monitors process +performance, and produces manifests plus HTML/Plotly reports. + +## Environment + +- Windows, PowerShell primary shell. +- Virtualenv lives at **`.venv`**. A stale second `venv/` directory also + exists in the repo — always use `.venv`, never `venv`. +- Activate: `.venv\Scripts\Activate.ps1` +- Install (editable, with dev deps): `pip install -e ".[dev]"` + +## Commands + +```powershell +.venv\Scripts\Activate.ps1 +pytest +ruff check . +black . +mypy src/ +pre-commit run --all-files +``` + +## Module map (`src/eddypro_batch_processor/`) + +- `cli.py` — argparse CLI, entry point `eddypro-batch`; subcommands + `run`, `scenarios`, `validate`, `status`. +- `core.py` — orchestration: config loading, subprocess execution of + `eddypro_rp` / `eddypro_fcc`. +- `monitor.py` — psutil-based process-tree performance sampling during runs. +- `analysis.py` — bottleneck classification from monitoring samples. +- `report.py` — run manifests and HTML/Plotly report generation. +- `ini_tools.py` — EddyPro `.eddypro` INI templating and patching. +- `ecmd.py` — ECMD metadata CSV handling. +- `scenarios.py` — parameter Cartesian-product scenario generation. +- `validation.py` — config and ECMD validation. + +## Domain rules — read before acting + +- **EddyPro runs take hours.** Never launch a real run speculatively just + to "check" something — use `--dry-run` instead. +- **Never write to or delete anything under** `data/`, `logs/`, + `D:/L0_raw/`, or `D:/L1_processed/`. These hold irreplaceable raw and + processed field data. +- `config/config.yaml` is the user's live working config with + machine-specific paths. Don't commit changes to it. + +## Code quality + +Detailed code-quality, testing, and doc-maintenance rules live in +`.github/copilot-instructions.md` — treat that as the source of truth +rather than duplicating it here. + +## Definition of Done + +- [ ] Tests pass (`pytest`) +- [ ] Lint/format/type-check clean (`ruff check .`, `black .`, `mypy src/`) +- [ ] Relevant docs updated (README, `docs/*.md` as applicable) +- [ ] `CHANGELOG.md` entry added under `[Unreleased]` diff --git a/README.md b/README.md index 693c521..a62cf96 100644 --- a/README.md +++ b/README.md @@ -170,38 +170,29 @@ source venv/bin/activate local `bin/` copy strategy as `scenarios`. If `eddypro_rp` fails, `eddypro_fcc` is skipped. -3. **Run a single scenario with specific parameters:** +3. **Process several years with the same settings (multi-year run):** ```bash - eddypro-batch --config config/config.yaml run --site GL-ZaF --years 2021 --rot-meth 1 --tlag-meth 2 --detrend-meth 0 --despike-meth 1 --hf-meth 4 + eddypro-batch --config config/config.yaml run --site GL-Dsk --years 2020 2021 2022 2023 2024 2025 --mp --max-proc 6 ``` -4. **Test all combinations of parameter scenarios (Cartesian product):** + See [MULTI_YEAR_RUNS.md](docs/MULTI_YEAR_RUNS.md) for a complete worked + example (config file and pure-CLI forms, choosing `max_processes`, + reading the performance bottleneck report). - This example tests all 16 combinations (2×2×2×2): +4. **Test combinations of processing parameters (scenario matrix):** ```bash eddypro-batch --config config/config.yaml scenarios --site GL-ZaF --years 2021 --rot-meth 1 3 --tlag-meth 2 4 --detrend-meth 0 1 --despike-meth 0 1 ``` - With high-frequency correction methods added, you can test up to 32 combinations (2×2×2×2×2): - - ```bash - eddypro-batch --config config/config.yaml scenarios --site GL-ZaF --years 2021 --rot-meth 1 3 --tlag-meth 2 4 --detrend-meth 0 1 --despike-meth 0 1 --hf-meth 1 4 - ``` - - **Parameter meanings:** - - - `--rot-meth 1 3` → Rotation methods: 1=Double Rotation (DR), 3=Planar Fit (PF) - - `--tlag-meth 2 4` → Time lag methods: 2=Constant (CMD), 4=Automatic Optimization (AO) - - `--detrend-meth 0 1` → Detrending: 0=Block Average (BA), 1=Linear Detrending (LD) - - `--despike-meth 0 1` → Spike removal: 0=Vickers & Mahrt (1997), 1=Mauder et al. (2013) - - `--hf-meth 1 4` → High-frequency spectral correction: 1=Moncrieff et al. (1997) analytic, 4=Fratini et al. (2012) in situ/analytic - - Each scenario runs independently and produces separate output files in - `scenario{suffix}` directories (e.g., `scenario_rot1_tlag2_det0_spk1`). - - See [SCENARIOS.md](docs/SCENARIOS.md) for detailed documentation on scenario runs. + This tests all 16 combinations (2×2×2×2) of rotation, time lag, detrend, + and spike-removal methods; add `--hf-meth 1 4` for up to 32. Each + scenario runs independently and produces its own output directory and + HTML report. See [SCENARIOS.md](docs/SCENARIOS.md) for the full parameter + table, naming conventions, and more examples — note that a *scenario* + run (many parameter combinations, one year) is different from a + *multi-year* run (one set of parameters, many years) shown above. 5. **Dry-run mode (generate files without executing EddyPro):** @@ -219,80 +210,26 @@ source venv/bin/activate For detailed information, see the `docs/` directory: +- **[MULTI_YEAR_RUNS.md](docs/MULTI_YEAR_RUNS.md)** – Worked multi-year run example (same settings, many years) — start here if that's your use case - **[USAGE.md](docs/USAGE.md)** – Complete CLI usage guide with all command examples and options - **[CONFIG.md](docs/CONFIG.md)** – Configuration file reference, YAML structure, and ECMD format specifications -- **[SCENARIOS.md](docs/SCENARIOS.md)** – Scenario matrix runs, parameter testing, and naming conventions +- **[SCENARIOS.md](docs/SCENARIOS.md)** – Scenario matrix runs (parameter combinations), naming conventions - **[REPORTING.md](docs/REPORTING.md)** – Understanding reports, performance metrics, and manifest structure - **[OUTPUT_FILE_TRACKING.md](docs/OUTPUT_FILE_TRACKING.md)** – Machine-readable output file tracking in manifests - **[KNOWN_ISSUES_AND_TODO.md](docs/KNOWN_ISSUES_AND_TODO.md)** – Known issues, gaps, and roadmap items - **[DEVELOPMENT.md](docs/DEVELOPMENT.md)** – Contributing guidelines, development setup, and testing - **[ARCHITECTURE.md](docs/ARCHITECTURE.md)** – System design and module organization -- **[plan/IMPROVEMENT_PLAN.md](docs/plan/IMPROVEMENT_PLAN.md)** – Project roadmap and completed milestones +- **[plan/implemented/IMPROVEMENT_PLAN.md](docs/plan/implemented/IMPROVEMENT_PLAN.md)** – Project roadmap and completed milestones ## Key Capabilities -### Configuration Validation - -Catch configuration errors before processing: +- **Configuration validation** (`eddypro-batch validate`) — required keys/types, path existence, ECMD schema and sanity checks +- **Multi-year runs** — process several years with identical settings, optionally in parallel (`--mp --max-proc N`); see [MULTI_YEAR_RUNS.md](docs/MULTI_YEAR_RUNS.md) +- **Scenario matrix testing** (`eddypro-batch scenarios`) — Cartesian product of up to 32 parameter combinations by default (`--max-scenarios`); see [SCENARIOS.md](docs/SCENARIOS.md) +- **Performance monitoring** — CPU, memory, and disk I/O sampled from the whole EddyPro process tree, with a CPU/MEMORY/DISK_THROUGHPUT/DISK_IOPS bottleneck classification; toggle with `monitoring_enabled` / `--monitor` / `--no-monitor` +- **Reporting** — HTML reports (for both `run` and `scenarios`, plus an aggregate comparison report for `scenarios`) and JSON `run_manifest.json` with provenance (git SHA, config checksum, EddyPro executable checksum, `sys.argv`) -```bash -eddypro-batch --config config/config.yaml validate -``` - -Validates: - -- Required config keys and types -- Path existence (EddyPro executable, input directories, ECMD file) -- ECMD schema (required columns, data types) -- Sanity checks (positive values, non-empty fields) - -### Scenario Matrix Testing - -Test multiple parameter combinations systematically. The `scenarios` command creates a Cartesian product of all parameter values: - -```bash -eddypro-batch --config config/config.yaml scenarios --site GL-ZaF --years 2021 --rot-meth 1 3 --tlag-meth 2 4 --detrend-meth 0 1 --despike-meth 0 1 -``` - -This creates 16 scenarios (2×2×2×2) with all combinations: - -- Rotation: Double Rotation (1) and Planar Fit (3) -- Time lag: Constant (2) and Automatic Optimization (4) -- Detrending: Block Average (0) and Linear Detrending (1) -- Spike removal: Vickers & Mahrt (0) and Mauder et al. (1) - -Each scenario is named uniquely (e.g., `scenario_rot1_tlag2_det0_spk1`) and -processed independently. Results are tracked in the run manifest for comparison. - -**Note:** Maximum 32 scenarios allowed (configurable via `--max-scenarios`). See [SCENARIOS.md](docs/SCENARIOS.md) for details. - -### Performance Monitoring - -Track resource usage during processing: - -- CPU utilization (process and system) -- Memory usage (RSS, peak) -- Disk I/O (read/write MB, IOPS) -- Processing duration - -Metrics are saved per scenario and aggregated in reports. - -### Comprehensive Reporting - -Generates detailed reports after each run: - -- **HTML reports** with interactive Plotly charts (CPU, memory, I/O over time) -- **JSON manifests** (`run_manifest.json`) for programmatic analysis - - Complete scenario results with success/failure status - - Machine-readable output file tracking (all EddyPro CSV outputs) - - Duration, metrics summary, and configuration snapshot -- **Per-scenario metrics** (CSV time series of resource usage) -- **Provenance capture** (config checksum, git SHA, Python environment, package versions) - -Reports are saved to `{output_dir}/reports/` by default. HTML reports are -generated for `run` executions; `scenarios` currently writes a run manifest only. -See [REPORTING.md](docs/REPORTING.md) and [OUTPUT_FILE_TRACKING.md](docs/OUTPUT_FILE_TRACKING.md) -for details. +See [CONFIG.md](docs/CONFIG.md) and [REPORTING.md](docs/REPORTING.md) for full details. ## Configuration Example @@ -303,17 +240,17 @@ years_to_process: [2021, 2022, 2023] input_dir_pattern: "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" output_dir_pattern: "D:/L1_processed/{site_id}/{year}/ec_rflux" ecmd_file: "D:/L1_processed/{site_id}/ecmd/{site_id}_ecmd.csv" - multiprocessing: False -max_processes: 16 -stream_output: True -log_level: INFO - -metrics_interval_seconds: 0.5 -report_charts: plotly # Options: plotly, svg, none +monitoring_enabled: true +report_charts: plotly ``` -See [CONFIG.md](docs/CONFIG.md) for all options and details. +This is a short excerpt. The full, authoritative example with every key and +comments is [`config/config.yaml.example`](config/config.yaml.example); see +[CONFIG.md](docs/CONFIG.md) for the key-by-key reference and +[MULTI_YEAR_RUNS.md](docs/MULTI_YEAR_RUNS.md) / +[`examples/multi_year_config.yaml`](examples/multi_year_config.yaml) for a +complete real-world example. ## Contributing diff --git a/config/config.yaml.example b/config/config.yaml.example index f4bea90..20e8bdc 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -1,65 +1,37 @@ # Configuration for EddyPro processing job +# +# This file is the single source of truth for the full set of configuration +# keys accepted by eddypro-batch. Copy it to config/config.yaml and edit the +# values for your site. See docs/CONFIG.md for the full reference and +# docs/MULTI_YEAR_RUNS.md for a worked multi-year example. -# Specify the path to the EddyPro executable +# Specify the path to the EddyPro executable (eddypro_rp.exe on Windows) eddypro_executable: "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe" # Specify the site ID you want to process -#site_id: GL-ZaF -#site_id: GL-ZaH -#site_id: GL-Dsk site_id: GL-NuF -#site_id: GL-NuH # List of years you want to process for this site years_to_process: - - #- 2000 - #- 2001 - #- 2002 - #- 2003 - #- 2004 - #- 2005 - #- 2016 - #- 2007 - #- 2008 - #- 2009 - #- 2010 - #- 2011 - #- 2012 - #- 2013 - #- 2014 - #- 2015 - #- 2016 - #- 2017 - #- 2018 - #- 2019 - #- 2020 - #- 2021 - #- 2022 - #- 2023 - 2024 # Input directory pattern for each year and site # Use `{year}` and `{site_id}` as placeholders -#input_dir_pattern: "C:/Users/au710242/Code/Python/eddypro_batch_processor/data/raw/{site_id}/{year}" - input_dir_pattern: "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" # Output directory pattern for each year and site # Use `{year}` and `{site_id}` as placeholders -#output_dir_pattern: "C:/Users/au710242/Code/Python/eddypro_batch_processor/data/processed/{site_id}/{year}" - output_dir_pattern: "D:/L1_processed/{site_id}/{year}/ec_rflux" -# Path to the ECMD CSV file -#ecmd_file: "C:/Users/au710242/Code/Python/eddypro_batch_processor/data/GL-ZaF_ecmd.csv" +# Path to the ECMD CSV file (site instrument/metadata history) ecmd_file: "D:/L1_processed/{site_id}/ecmd/{site_id}_ecmd.csv" -# Enable or disable multiprocessing -multiprocessing: False # Set to False to disable multiprocessing +# Enable or disable multiprocessing across years (see docs/MULTI_YEAR_RUNS.md) +multiprocessing: False # Set to True to process years in parallel -# Maximum number of CPU cores to use for multiprocessing -max_processes: 16 # Adjust this number based on your requirements +# Maximum number of worker processes when multiprocessing is enabled +# Rule of thumb: one worker per year, bounded by physical cores and disk throughput +max_processes: 16 # Control output streaming (EddyPro subprocess outputs) stream_output: True # Set to False to keep EddyPro output quiet @@ -77,9 +49,26 @@ log_backup_count: 5 # Capture EddyPro stdout/stderr in logs log_eddypro_output: true -# Performance monitoring configuration -metrics_interval_seconds: 0.5 # Sampling interval for performance monitoring +# Enable or disable performance monitoring (CPU/memory/disk sampling). +# When False, no metrics files are written and metrics_interval_seconds is +# ignored. See docs/CONFIG.md and docs/MULTI_YEAR_RUNS.md. +monitoring_enabled: true + +# Performance monitoring sampling interval in seconds (only used when +# monitoring_enabled is true) +metrics_interval_seconds: 0.5 + +# Optional: tune how the bottleneck analyser classifies a run. Defaults assume a +# mechanical disk; raise the disk limits substantially for NVMe. See +# docs/CONFIG.md for the full table of keys and defaults. +#performance_thresholds: +# cpu_high_percent: 90 +# memory_high_percent: 85 +# disk_high_mb_per_s: 100 # Reporting configuration reports_dir: null # Optional: Custom reports directory (default: {output_dir}/reports) report_charts: plotly # Chart engine: plotly, svg, or none (default: plotly) + +# Optional: override the EddyPro project template +project_template: null # null = use config/EddyProProject_template.ini diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 842716a..4dd798d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -114,15 +114,20 @@ graph TD - Populate static metadata from ECMD row 5. **Processing Execution** (`core.py`) - - `run`: executes configured EddyPro executable directly - - `scenarios`: copies EddyPro binaries and runs both `eddypro_rp` and `eddypro_fcc` - - Capture performance metrics + - Both `run` and `scenarios` copy EddyPro binaries to a local `bin/` + folder and run `eddypro_rp` followed by `eddypro_fcc` (fcc is skipped + if rp fails) + - `run` parallelizes across years when `multiprocessing: true` + (one worker per year, up to `max_processes`) + - Capture performance metrics (unless `monitoring_enabled: false`) - Handle errors and timeouts 6. **Report Generation** (`report.py`) - - Generate HTML reports with charts (currently for `run` executions) - - Create JSON manifests - - Package performance data + - Generate HTML reports with charts for both `run` and `scenarios` + (`scenarios` additionally produces an aggregate comparison report) + - Create JSON manifests with provenance (git SHA, package version, + EddyPro executable checksum, `sys.argv`) and per-year status + - Package performance data, including bottleneck classification --- @@ -366,7 +371,7 @@ The architecture provides several extension points for future enhancements: ### Optional Dependencies - **psutil**: System monitoring (graceful fallback) -- **plotly**: Interactive charts (SVG fallback) +- **plotly**: Interactive charts (if absent, reports render without charts; no automatic SVG fallback — select `report_charts: svg` explicitly) - **jinja2**: Advanced templating (future use) ### Development Dependencies @@ -390,29 +395,10 @@ The architecture provides several extension points for future enhancements: ### Configuration Structure -```yaml -# Core settings -eddypro_executable: "/path/to/eddypro" -site_id: "GL-ZaF" -years_to_process: [2021, 2022] - -# Path patterns -input_dir_pattern: "/data/raw/{site_id}/{year}" -output_dir_pattern: "/data/processed/{site_id}/{year}" -ecmd_file: "/data/{site_id}_ecmd.csv" - -# Processing options -multiprocessing: true -max_processes: 4 -stream_output: true - -# Monitoring -metrics_interval_seconds: 0.5 - -# Reporting -reports_dir: null # Auto-generate -report_charts: "plotly" # plotly, svg, none -``` +See [`config/config.yaml.example`](../config/config.yaml.example) for the +full, authoritative set of keys (core settings, path patterns, +multiprocessing, monitoring, and reporting options), and +[CONFIG.md](CONFIG.md) for the key-by-key reference. ### Template System diff --git a/docs/CONFIG.md b/docs/CONFIG.md index e4f87d8..8ba4539 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -12,59 +12,30 @@ Override with: `--config /path/to/config.yaml` ### Complete Example +The full, authoritative example config lives at +[`config/config.yaml.example`](../config/config.yaml.example) — copy it to +`config/config.yaml` and edit it. A short excerpt: + ```yaml -# EddyPro executable path eddypro_executable: "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe" - -# Site identification -site_id: GL-ZaF - -# Years to process +site_id: GL-NuF years_to_process: - - 2021 - - 2022 - - 2023 - -# Input directory pattern (use {year} and {site_id} placeholders) + - 2024 input_dir_pattern: "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" - -# Output directory pattern (use {year} and {site_id} placeholders) output_dir_pattern: "D:/L1_processed/{site_id}/{year}/ec_rflux" - -# ECMD (Extended Configuration Metadata) file path ecmd_file: "D:/L1_processed/{site_id}/ecmd/{site_id}_ecmd.csv" - -# Multiprocessing settings multiprocessing: False max_processes: 16 - -# Output streaming (EddyPro subprocess outputs) -stream_output: True - -# Logging level -log_level: INFO - -# Optional log file path (null disables file logging) -log_file: "logs/eddypro_processing.log" - -# Log rotation (bytes) and backup count (0 disables rotation) -log_max_bytes: 10485760 -log_backup_count: 5 - -# Capture EddyPro stdout/stderr in logs -log_eddypro_output: true - -# Performance monitoring +monitoring_enabled: true metrics_interval_seconds: 0.5 - -# Reporting -reports_dir: null # null = use default ({output_dir}/reports) -report_charts: plotly # Options: plotly, svg, none - -# Optional: project template override -project_template: null # null = use config/EddyProProject_template.ini +reports_dir: null +report_charts: plotly ``` +For a fully worked multi-year example (same settings applied across several +years), see [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) and +[`examples/multi_year_config.yaml`](../examples/multi_year_config.yaml). + ## Required Configuration Keys The following keys **must** be present in your configuration file: @@ -79,8 +50,8 @@ The following keys **must** be present in your configuration file: | `ecmd_file` | str | Path to ECMD CSV file | | `stream_output` | bool | Enable/disable real-time output | | `log_level` | str | Logging level | -| `multiprocessing` | bool | Enable/disable multiprocessing | -| `max_processes` | int | Maximum number of processes | +| `multiprocessing` | bool | Enable/disable multiprocessing across years | +| `max_processes` | int | Maximum number of parallel worker processes | | `metrics_interval_seconds` | float | Performance monitoring interval | | `reports_dir` | str or null | Custom reports directory | | `report_charts` | str | Chart engine for reports | @@ -94,6 +65,7 @@ The following keys **must** be present in your configuration file: | `log_max_bytes` | int or null | Max log file size in bytes before rotation (0 disables rotation) | | `log_backup_count` | int or null | Number of rotated log files to keep | | `log_eddypro_output` | bool | Write EddyPro stdout/stderr to logs | +| `monitoring_enabled` | bool | Enable/disable performance monitoring (default: `true`) | ## Configuration Details @@ -156,7 +128,9 @@ years_to_process: - Each item must be a valid integer (typically 4-digit year) **Notes:** -- Years are processed sequentially (or in parallel if multiprocessing is enabled) +- Years are processed in parallel (one worker per year, up to `max_processes`) + when `multiprocessing: true`; otherwise sequentially. See + [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) for a worked example. - Used in path placeholders (`{year}`) --- @@ -264,8 +238,12 @@ multiprocessing: True - When enabled, `max_processes` must be positive **Notes:** -- Multiprocessing settings are validated but not yet wired into execution. - The `run` and `scenarios` commands currently execute sequentially. +- When `True`, the `run` command processes years in parallel across up to + `max_processes` worker processes (one worker per year). `scenarios` + processes years sequentially, running the full scenario batch for each + year before moving to the next. +- Enable via CLI with `--mp` (and `--max-proc N`); see + [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md). --- @@ -398,6 +376,70 @@ log_eddypro_output: true --- +### monitoring_enabled + +**Type:** Boolean + +**Default:** `true` + +**Description:** Enable or disable performance monitoring (CPU, memory, and +disk sampling of the EddyPro process tree) during processing. + +**Example:** +```yaml +monitoring_enabled: false +``` + +**Interaction with `metrics_interval_seconds`:** +- When `true` (default), the monitor samples at `metrics_interval_seconds` + and writes `metrics_*.csv` files per year/scenario. +- When `false`, no metrics files are written and `metrics_interval_seconds` + is ignored entirely — validation does not require it to be positive. + +**CLI Override:** `--monitor` / `--no-monitor` (available on both `run` and +`scenarios`). + +--- + +### performance_thresholds + +**Type:** Mapping (optional) + +**Default:** see table below + +**Description:** Tunes how the bottleneck analyser classifies a run. The +defaults assume a mechanical disk; on NVMe storage the disk limits should be +raised substantially or every run will be reported as disk-bound. + +| Key | Default | Meaning | +|-----|---------|---------| +| `cpu_high_percent` | 90 | At or above this sustained (p95) CPU, status is RED | +| `cpu_moderate_percent` | 70 | At or above this, status is YELLOW | +| `cpu_idle_percent` | 40 | Below this, a busy disk is read as the limiting factor | +| `memory_high_percent` | 85 | System memory use that counts as RED | +| `memory_moderate_percent` | 70 | System memory use that counts as YELLOW | +| `disk_high_mb_per_s` | 100 | Combined read+write throughput counting as RED | +| `disk_moderate_mb_per_s` | 50 | Throughput counting as YELLOW | +| `disk_high_iops` | 1000 | Combined IOPS above which latency is the suspect | + +Unknown keys are ignored, so a config written for a newer version still loads. + +**Example (NVMe):** +```yaml +performance_thresholds: + disk_high_mb_per_s: 2000 + disk_moderate_mb_per_s: 1000 +``` + +**CLI Override:** none — config only. + +**Notes:** +- Disable for maximum throughput on large batches where the small sampling + overhead matters, or when you no longer need per-run performance data. See + [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md). + +--- + ### metrics_interval_seconds **Type:** Float @@ -462,9 +504,15 @@ report_charts: plotly ``` **Fallback Behavior:** -- If `plotly` is selected but not installed, automatically falls back to `svg` with a warning +- If `plotly` is selected but the `plotly` package is not installed, the + report is still generated: chart sections show a + "Plotly not installed" note instead of a chart, and a debug-level log + message records the import failure. There is no automatic switch to `svg`. + Set `report_charts: svg` (or pass `--report-charts svg`) explicitly if you + don't have `plotly` installed. -**Scope:** Currently used for HTML reports generated by the `run` command. +**Scope:** Used for HTML reports generated by both `run` and `scenarios` +(per-scenario reports plus one aggregate comparison report). **CLI Override:** ```bash @@ -638,5 +686,6 @@ eddypro-batch validate && eddypro-batch run ## See Also - [USAGE.md](USAGE.md) – CLI usage and examples +- [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) – Worked multi-year run example - [SCENARIOS.md](SCENARIOS.md) – Scenario matrix runs - [REPORTING.md](REPORTING.md) – Understanding reports diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 312a6aa..3c8e19b 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -679,4 +679,4 @@ pytest --log-cli-level=DEBUG - [USAGE.md](USAGE.md) – CLI usage - [CONFIG.md](CONFIG.md) – Configuration options -- [plan/IMPROVEMENT_PLAN.md](plan/IMPROVEMENT_PLAN.md) – Project roadmap +- [plan/implemented/IMPROVEMENT_PLAN.md](plan/implemented/IMPROVEMENT_PLAN.md) – Project roadmap diff --git a/docs/KNOWN_ISSUES_AND_TODO.md b/docs/KNOWN_ISSUES_AND_TODO.md index 1b99c28..820052b 100644 --- a/docs/KNOWN_ISSUES_AND_TODO.md +++ b/docs/KNOWN_ISSUES_AND_TODO.md @@ -5,48 +5,57 @@ It is intended to keep docs aligned with the actual behavior in the codebase. ## Current Known Issues -### 1) Scenario runs depend on `eddypro_fcc` +### 1) Scenario (and run) execution depends on `eddypro_fcc` -- **What happens:** Scenario execution uses `run_eddypro_with_monitoring()` which - copies the EddyPro binaries and runs both `eddypro_rp` and `eddypro_fcc`. +- **What happens:** Both `run` and `scenarios` use + `run_eddypro_with_monitoring()`, which copies the EddyPro binaries and runs + both `eddypro_rp` and `eddypro_fcc`. - **Failure mode:** If `eddypro_fcc` is missing from the same directory as - `eddypro_executable`, or if `eddypro_fcc` cannot run on the host, scenarios fail. -- **Impact:** Scenario runs can error even when a regular `run` succeeds. -- **Notes:** This is likely the source of the “pipeline still cannot run everything - without errors” observation. It is consistent with failures seen when - `eddypro_fcc` is absent or not executable. + `eddypro_executable`, or if `eddypro_fcc` cannot run on the host, processing + fails for that year/scenario. +- **Impact:** Runs can error even when only the raw-processing step + (`eddypro_rp`) would have succeeded. -### 2) Execution path mismatch (`run` vs `scenarios`) +### 2) Execution path parity (`run` vs `scenarios`) -- **`run`:** Executes the configured `eddypro_executable` directly. -- **`scenarios`:** Runs `eddypro_rp` then `eddypro_fcc` from a copied `bin/` folder. -- **Impact:** Outputs and error modes can diverge between `run` and `scenarios`. - This is a deliberate design today but should be documented as a limitation. +- **`run`:** Copies EddyPro binaries to a local `bin/` folder and runs + `eddypro_rp` then `eddypro_fcc` (same strategy as `scenarios`). +- **`scenarios`:** Runs `eddypro_rp` then `eddypro_fcc` from a copied `bin/` + folder, once per scenario. +- **Status:** The two commands now use the same execution strategy, so this + is no longer a source of divergent outputs/error modes. It remains listed + here because the two commands still differ in other ways worth knowing: + `run` parallelizes across years when `multiprocessing: true`; `scenarios` + processes years sequentially and parallelizes scenarios within a year not + at all (scenarios run one after another). -### 3) Metrics schema mismatch with report chart loader +### 3) ~~Metrics schema mismatch with report chart loader~~ — FIXED -- **Observed:** The monitor writes raw metrics with fields like - `system_cpu_percent` and `process_memory_rss`. -- **Report loader expects:** `cpu_percent`, `memory_mb`, `read_mb`, `write_mb`. -- **Impact:** HTML charts can be empty or missing lines even when metrics exist. +Performance monitoring now samples the whole EddyPro process tree (not the +`cmd.exe`/shell wrapper), and the report chart loader's expected fields +(`cpu_percent`, `memory_mb`, `read_mb`, `write_mb`) match what the monitor +writes. CPU, memory, and disk figures in `run_report.html` reflect actual +EddyPro resource usage, including derived disk rates and a +CPU/MEMORY/DISK_THROUGHPUT/DISK_IOPS bottleneck classification. -### 4) Scenario reports are not generated +### 4) ~~Scenario reports are not generated~~ — FIXED -- **Current behavior:** `scenarios` writes only `run_manifest.json`. -- **Expected by docs:** Per-scenario HTML reports and an aggregate report. -- **Impact:** Users do not get HTML reports for scenario runs today. +`scenarios` now generates a per-scenario HTML report at +`{output_dir}/{scenario_suffix}/reports/run_report.html`, plus one aggregate +comparison report and `run_manifest.json` under `reports_dir`. -### 5) `status` output and scenario manifest schema mismatch +### 5) ~~`status` output and scenario manifest schema mismatch~~ — FIXED -- **`status` prints:** `scenario_name` from the run manifest. -- **`scenarios` manifest entries:** `scenario_index` + `scenario_suffix` without - `scenario_name`. -- **Impact:** Status output can show “unknown” for scenario name entries. +The run manifest's scenario entries and the `status` command's reader are now +aligned, so scenario names/suffixes display correctly instead of showing +"unknown". -### 6) Multiprocessing flags are not wired +### 6) ~~Multiprocessing flags are not wired~~ — FIXED -- `multiprocessing` and `max_processes` are validated but not applied in execution. -- Runs are currently sequential regardless of these settings. +`multiprocessing` and `max_processes` are wired into `run`: years are +processed in parallel, one worker per year, up to `max_processes` concurrent +workers. Enable via config (`multiprocessing: true`) or CLI (`--mp +--max-proc N`). See [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md). ### 7) CLI flag ambiguity @@ -55,41 +64,21 @@ It is intended to keep docs aligned with the actual behavior in the codebase. ## Investigations / Suspected Root Causes -- **`eddypro_fcc` availability:** Scenario runner explicitly requires it. If - installations provide only `eddypro_rp`, scenario runs will fail. -- **Execution environment:** Scenario runs copy binaries to a local `bin/` folder; - missing dependencies or licensing checks can fail after copy. +- **`eddypro_fcc` availability:** Both `run` and `scenarios` explicitly + require it. If installations provide only `eddypro_rp`, processing will + fail (see Issue 1). +- **Execution environment:** Both commands copy binaries to a local `bin/` + folder; missing dependencies or licensing checks can fail after copy. -## Roadmap Items (Planned, Not Implemented Yet) - -### Monitoring Toggle - -See [MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md](plan/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md). - -Planned: -- Config `monitoring_enabled: true|false` (default `true`). -- CLI flags `--monitor` / `--no-monitor` for `run` and `scenarios`. -- Skip metrics files and monitor creation when disabled. -- Validation: only enforce positive `metrics_interval_seconds` if enabled. - -### Performance Analysis & Reporting Enhancements - -See [PERFORMANCE_ANALYSIS_DESIGN.md](plan/PERFORMANCE_ANALYSIS_DESIGN.md). +## TODO Checklist (High Priority) -Planned: -- New `analysis.py` with `BottleneckAnalyzer` and `ScenarioAnalysis` models. -- Traffic-light executive summary (CPU/RAM/Disk bottleneck classification). -- Plotly subplots for CPU, memory, and disk I/O with thresholds. -- Per-scenario HTML reports and aggregated comparison matrix. -- Unified baseline scenario model for `run` so analysis is consistent. +- [ ] Decide how to handle `eddypro_fcc` missing on `run`/`scenarios` (fail + fast vs. fallback to `eddypro_rp` only). +- [ ] Consider parallelizing `scenarios` across scenarios/years the same way + `run` now parallelizes across years. -## TODO Checklist (High Priority) +## See Also -- [ ] Decide how to handle `eddypro_fcc` missing on scenario runs (fail fast vs. - fallback to `eddypro_rp` only). -- [ ] Align metrics schema between monitor outputs and report chart loader. -- [ ] Add HTML report generation for `scenarios` or document as intentionally - unsupported. -- [ ] Wire `multiprocessing` and `max_processes` or mark as deprecated. -- [ ] Add `monitoring_enabled` config + CLI flags as per plan. -- [ ] Implement performance analysis module and integrate with reporting. +- [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) – multi-year run behavior and caveats +- [SCENARIOS.md](SCENARIOS.md) – scenario run behavior and caveats +- [REPORTING.md](REPORTING.md) – manifest and report schema diff --git a/docs/MULTI_YEAR_RUNS.md b/docs/MULTI_YEAR_RUNS.md new file mode 100644 index 0000000..36e5f54 --- /dev/null +++ b/docs/MULTI_YEAR_RUNS.md @@ -0,0 +1,200 @@ +# Multi-Year Runs + +This page walks through processing several years of data for one site with +the **same** processing settings. It is deliberately separate from +[SCENARIOS.md](SCENARIOS.md). + +> A **multi-year run** applies the *same* processing settings across several +> years. A **scenario run** applies *several combinations of EddyPro +> processing parameters* (`rot_meth`, `tlag_meth`, `detrend_meth`, +> `despike_meth`, `hf_meth`) to the same year(s). They are independent +> features; this page covers the former. + +If you want to compare parameter combinations, see [SCENARIOS.md](SCENARIOS.md) +instead. The two can be combined (a scenario matrix run also accepts multiple +`--years`), but this page focuses on the plain `run` command. + +## Worked example: GL-Dsk, 2020-2025 + +The example below processes six years (2020-2025) of site `GL-Dsk` with +identical settings, parallelized across years with performance monitoring +enabled. + +### Option 1: As a config file + +[`examples/multi_year_config.yaml`](../examples/multi_year_config.yaml) is a +complete, ready-to-edit config for this scenario: + +```yaml +eddypro_executable: "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe" +site_id: GL-Dsk +years_to_process: [2020, 2021, 2022, 2023, 2024, 2025] +input_dir_pattern: "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" +output_dir_pattern: "D:/L1_processed/{site_id}/{year}/ec_rflux_sc26" +ecmd_file: "D:/L1_processed/{site_id}/ecmd/{site_id}_ecmd.csv" +multiprocessing: true +max_processes: 6 +monitoring_enabled: true +metrics_interval_seconds: 1.0 +reports_dir: "D:/L1_processed/GL-Dsk/reports/2020-2025" +report_charts: plotly +``` + +(See the file itself for the remaining required keys — `stream_output`, +`log_level`, and the optional `log_*` keys — needed for `validate` to pass.) + +Run it: + +```powershell +eddypro-batch --config examples/multi_year_config.yaml validate +eddypro-batch --config examples/multi_year_config.yaml run --dry-run +eddypro-batch --config examples/multi_year_config.yaml run +``` + +### Option 2: As pure CLI (no config file edits) + +Every setting can be supplied on the command line instead, overriding +whatever is in `config/config.yaml`: + +```powershell +eddypro-batch run ` + --site GL-Dsk ` + --years 2020 2021 2022 2023 2024 2025 ` + --input-dir-pattern "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" ` + --output-dir-pattern "D:/L1_processed/{site_id}/{year}/ec_rflux_sc26" ` + --rot-meth 1 --tlag-meth 2 --detrend-meth 0 --despike-meth 0 --hf-meth 1 ` + --mp --max-proc 6 ` + --metrics-interval 1.0 ` + --reports-dir "D:/L1_processed/GL-Dsk/reports/2020-2025" ` + --report-charts plotly +``` + +Note that `--rot-meth`, `--tlag-meth`, `--detrend-meth`, `--despike-meth`, and +`--hf-meth` on `run` each take a single value — they patch the same INI +parameter for every year, which is exactly the "same settings, many years" +behavior a multi-year run is for. (On `scenarios`, the same flag names accept +*multiple* values and form a Cartesian product instead — see +[SCENARIOS.md](SCENARIOS.md).) + +## Always dry-run first + +Rehearse before committing to a multi-hour run: + +```powershell +eddypro-batch --config examples/multi_year_config.yaml run --dry-run +``` + +`--dry-run`: +- Creates each year's output directory and `.eddypro` project file +- Materializes `.metadata` and `_dynamic_metadata.txt` from the ECMD file +- Runs the same preflight validation as a real run (input path exists, files + present, metadata sane) +- Writes the run manifest and HTML report as if all years "succeeded" + +`--dry-run` does **not**: +- Invoke `eddypro_rp` or `eddypro_fcc` +- Produce any EddyPro output CSVs +- Sample performance metrics (there is no process to monitor) + +## Monitoring overhead: `--no-monitor` + +Performance monitoring samples the whole EddyPro process tree (CPU, memory, +disk reads/writes) at `metrics_interval_seconds`. It has real but usually +small overhead. For maximum throughput on a large batch — many years, tight +disk budget, or a machine you also need for other work — disable it: + +```powershell +eddypro-batch --config examples/multi_year_config.yaml run --no-monitor +``` + +or set `monitoring_enabled: false` in the config. With monitoring disabled, +no `metrics_*.csv` files are written and `metrics_interval_seconds` is +ignored. Keep monitoring on while you are still tuning `max_processes` — the +bottleneck report is what tells you whether to raise or lower it. + +## What lands where + +For each processed year: + +``` +{output_dir_pattern}/{site_id}/{year}/ # e.g. .../GL-Dsk/2021/ec_rflux_sc26/ +├── GL-Dsk.eddypro # generated project file +├── GL-Dsk.metadata +├── GL-Dsk_dynamic_metadata.txt +├── eddypro_GL-Dsk_fluxnet_*.csv +├── eddypro_GL-Dsk_full_output_*.csv +├── eddypro_GL-Dsk_metadata_*.csv +├── eddypro_GL-Dsk_qc_details_*.csv +├── metrics_rp.csv # performance samples, eddypro_rp phase +└── metrics_fcc.csv # performance samples, eddypro_fcc phase +``` + +And once, for the whole run, under `reports_dir`: + +``` +D:/L1_processed/GL-Dsk/reports/2020-2025/ +├── run_manifest.json +└── run_report.html +``` + +`run_manifest.json` includes a `years[]` array with one entry per year — +`{year, status, duration_seconds, error, output_dir}` — so a failed year is +visible in the manifest even though the run as a whole continues. It also +carries a `provenance` block (git SHA + dirty flag, package version, EddyPro +executable path/checksum, `sys.argv`), a real SHA256 `config_checksum`, and +`manifest_schema_version: 2`. See [REPORTING.md](REPORTING.md) for the full +schema. + +## Reading the bottleneck traffic light + +`run_report.html` classifies each year's run as CPU, MEMORY, +DISK_THROUGHPUT, or DISK_IOPS bound, based on the sampled metrics. Use it to +decide whether `max_processes` is set well: + +| Classification | What it means | What to do | +|---|---|---| +| CPU | Workers are compute-bound; cores are the limit | Lower `max_processes` toward the physical core count if you see thrashing; otherwise this is healthy utilization | +| MEMORY | Workers are approaching available RAM | Lower `max_processes`, or process fewer years concurrently | +| DISK_THROUGHPUT | Aggregate read/write MB/s is saturating the disk | Lower `max_processes`, or move input/output to faster storage (SSD/NVMe) | +| DISK_IOPS | Many small reads/writes are the limit (common with many small raw files) | Move data to faster storage; concatenating raw files can help more than adding workers | + +With N years running in parallel, remember that each worker's own CPU% is a +share of the *whole machine* — a healthy multi-year run with `max_processes: +6` on an 8-core machine will show each worker around 60-80% CPU, not 100%. +Read the aggregate/system columns in the metrics CSVs, not just one worker's +process columns, when deciding whether you're actually CPU-bound. + +## Choosing `max_processes` + +Start from **one worker per year** and cap it by whichever is smaller: + +- Physical CPU cores available (leave 1-2 free for the OS and monitoring) +- What your input/output disks can sustain without becoming the bottleneck + (see the traffic-light table above) + +For the GL-Dsk example (6 years, `max_processes: 6`), if the bottleneck +report comes back DISK_THROUGHPUT or DISK_IOPS, drop `max_processes` to 3-4 +before adding more years, or move `D:/L0_raw` / `D:/L1_processed` to faster +storage. + +## Caveats + +- **Years are processed independently.** A failure in one year (bad ECMD + row, missing input files, EddyPro crash) is logged and does not stop the + other years — the run continues and the failure is recorded per-year in + the manifest's `years[]` array. +- **`reports_dir` defaults to the first year's output directory** if you + don't set it (see `cli.py` around line 533). For a multi-year run this + default is confusing (why would the 2025 report live under `.../2020/`?), + which is why the example above sets `reports_dir` explicitly. +- Both `run` and `scenarios` execute `eddypro_rp` followed by `eddypro_fcc` + for every year; if `eddypro_rp` fails, `eddypro_fcc` is skipped for that + year and the year is marked failed. + +## See also + +- [SCENARIOS.md](SCENARIOS.md) – parameter-combination testing (the other + kind of "multi" run) +- [CONFIG.md](CONFIG.md) – full configuration key reference +- [REPORTING.md](REPORTING.md) – manifest and report schema, provenance +- [USAGE.md](USAGE.md) – complete CLI reference diff --git a/docs/REPORTING.md b/docs/REPORTING.md index fae0b99..892d406 100644 --- a/docs/REPORTING.md +++ b/docs/REPORTING.md @@ -7,9 +7,13 @@ This document describes the structure, location, and interpretation of reports g The batch processor generates comprehensive reports for each run, including: - Run manifests (machine-readable JSON) -- HTML reports with interactive visualizations (for `run` executions) +- HTML reports with interactive visualizations (for both `run` and + `scenarios`; `scenarios` additionally produces one aggregate comparison + report across all scenarios) - Per-scenario metrics and metadata - Performance time series data +- A bottleneck classification (CPU / MEMORY / DISK_THROUGHPUT / DISK_IOPS) + for each run/scenario, based on the sampled metrics ## Report Location @@ -55,21 +59,40 @@ eddypro-batch run --reports-dir /custom/reports ```json { "run_id": "GL-ZaF_20251002_100530", - "timestamp": "2025-10-02T10:05:30", - "start_time": "2025-10-02T10:05:30", - "end_time": "2025-10-02T11:20:15", + "manifest_schema_version": 2, + "status": "completed", + "timestamp": "2025-10-02T10:05:30+00:00", + "start_time": "2025-10-02T10:05:30+00:00", + "end_time": "2025-10-02T11:20:15+00:00", "duration_seconds": 4485.2, "site_id": "GL-ZaF", "years_processed": [2021, 2022], - "config_checksum": "a1b2c3d4", + "years": [ + { + "year": 2021, + "status": "success", + "duration_seconds": 2240.1, + "error": null, + "output_dir": "/path/to/output/2021" + }, + { + "year": 2022, + "status": "failed", + "duration_seconds": 12.4, + "error": "ECMD file not found for site GL-ZaF: ...", + "output_dir": "/path/to/output/2022" + } + ], + "config_checksum": "6c3e...sha256-of-canonical-config-json", + "config_file_checksum": "a91f...sha256-of-config-yaml-file", "config_snapshot": {"...": "..."}, - "overall_success": true, + "overall_success": false, "scenarios": [ { "scenario_name": "baseline", "scenario_params": {}, - "start_time": "2025-10-02T10:05:30", - "end_time": "2025-10-02T11:20:15", + "start_time": "2025-10-02T10:05:30+00:00", + "end_time": "2025-10-02T11:20:15+00:00", "duration_seconds": 4485.2, "success": true } @@ -93,10 +116,23 @@ eddypro-batch run --reports-dir /custom/reports "plotly": "5.22.0" } }, + "provenance": { + "git_sha": "d34dbeef...", + "git_dirty": false, + "package_version": "0.3.0", + "eddypro_executable": "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe", + "eddypro_executable_checksum": "9f1c...sha256-of-eddypro_rp.exe", + "argv": ["eddypro-batch", "run", "--site", "GL-ZaF", "--years", "2021", "2022"] + }, "dry_run": false } ``` +The manifest is written **atomically** (to a temp file, then renamed into +place) at the start of the run with `status: "running"`, and rewritten with +the final `status` and all other fields when the run ends — including when +every year fails, so a manifest always exists after a run is attempted. + --- ### run_report.html @@ -193,7 +229,18 @@ timestamp,relative_time,system_cpu_percent,system_memory_total,system_memory_ava - `process_io_read_bytes`, `process_io_write_bytes`, `process_io_read_count`, `process_io_write_count` Column presence can vary by platform and psutil capabilities. The CSV is a raw -time series; summary statistics are written to `metrics_summary{suffix}.json`. +time series; summary statistics are written to `metrics_summary{suffix}.json`, +including derived disk rates (`read_mb_per_s`, `write_mb_per_s`) and the +CPU/MEMORY/DISK_THROUGHPUT/DISK_IOPS bottleneck classification shown in the +HTML report. + +**Note:** Metrics sample the whole EddyPro process tree (not just the +`cmd.exe`/shell wrapper that launches it), so CPU and disk figures reflect +actual EddyPro resource usage. + +If `monitoring_enabled: false` (or `--no-monitor`), none of these files are +written and `metrics_interval_seconds` is ignored. See +[CONFIG.md](CONFIG.md#monitoring_enabled). --- @@ -263,32 +310,43 @@ Throughput: 3.2 scenarios/hour ## Provenance & Reproducibility -### Config Hash +### Config Checksums -**Purpose:** Unique identifier for configuration state +**Purpose:** Detect configuration changes between runs -**Generation:** SHA256 hash of config file content (excluding comments/whitespace) +**Generation:** Two independent SHA256 checksums are recorded: +- `config_checksum` – SHA256 of the canonicalised config as JSON + (`json.dumps(config, sort_keys=True)`), so key ordering in the YAML file + doesn't matter +- `config_file_checksum` – SHA256 of the raw `config.yaml` file content **Use:** - Compare runs to detect config changes -- Link outputs to exact configuration used +- Link outputs to the exact configuration used + +### Provenance Block -### Git SHA +**Purpose:** Link reports to the exact code, executable, and invocation used -**Purpose:** Link reports to source code version +**Captured (under `provenance` in the manifest):** +- `git_sha` and `git_dirty` – commit hash and whether the working tree had + uncommitted changes at runtime (when run from a git checkout) +- `package_version` – installed `eddypro-batch-processor` package version +- `eddypro_executable` and `eddypro_executable_checksum` – path and SHA256 + of the EddyPro executable used +- `argv` – the exact command-line invocation (`sys.argv`) -**Captured:** Git commit hash at runtime (if repository available) +**Not captured:** input raw-data file checksums are not recorded. **Use:** -- Reproduce results with specific code version -- Track code changes between runs +- Reproduce results with a specific code version and EddyPro build +- Track code and executable changes between runs ### Environment Snapshot -**Captured:** +**Captured (under `environment`):** - Python version - Package versions (psutil, plotly, yaml, etc.) -- EddyPro version (if detectable) **Use:** - Reproduce results with identical environment @@ -388,13 +446,15 @@ eddypro-batch run --metrics-interval 2.0 # Sample every 2 seconds **Symptom:** HTML report displays chart placeholders or errors **Possible Causes:** -- Plotly not installed +- Plotly not installed (the report still generates, with a "Plotly not + installed" note in place of each chart — there is no automatic fallback + to SVG) - Incompatible Plotly version - Browser JavaScript disabled **Solutions:** - Install Plotly: `pip install plotly` -- Use SVG fallback: `--report-charts svg` +- Or explicitly select the SVG engine: `--report-charts svg` - Update browser or enable JavaScript ### Large Metrics Files @@ -413,5 +473,6 @@ eddypro-batch run --metrics-interval 2.0 # Sample every 2 seconds ## See Also - [USAGE.md](USAGE.md) – CLI usage and examples +- [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) – Worked multi-year run example, including how to read the bottleneck table - [SCENARIOS.md](SCENARIOS.md) – Scenario matrix runs - [CONFIG.md](CONFIG.md) – Configuration options diff --git a/docs/SCENARIOS.md b/docs/SCENARIOS.md index ca69e37..5da53d1 100644 --- a/docs/SCENARIOS.md +++ b/docs/SCENARIOS.md @@ -4,6 +4,11 @@ This document explains how to use the `scenarios` command to test multiple param ## Overview +> A **scenario run** applies *several combinations* of EddyPro processing +> parameters to the same year(s). This is different from a **multi-year +> run**, which applies the *same* settings across several years — see +> [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) for that. + The `scenarios` command allows you to run a Cartesian product of EddyPro processing parameters, enabling systematic testing of different configurations. This is useful for: - Sensitivity analysis of processing methods @@ -196,13 +201,16 @@ Located in `{output_dir}/scenario{suffix}/`: - `scenario_manifest{suffix}.json` – scenario metadata and metrics - `metrics.csv` – performance time series (CPU, memory, I/O) -### Aggregated Reports +### Per-Scenario and Aggregated Reports -Located in `{output_dir}/reports/`: +Each scenario gets its own HTML report at +`{output_dir}/{scenario_suffix}/reports/run_report.html`. In addition, an +aggregate comparison report and `run_manifest.json` are written to the run's +`reports_dir` (default `{output_dir}/reports/`): - `run_manifest.json` – Summary of all scenarios -- `run_report.html` – Interactive report with charts (currently generated for `run` executions) -- Scenario comparison tables and visualizations +- `run_report.html` – Aggregate comparison report across all scenarios, with + charts and the bottleneck classification per scenario See [REPORTING.md](REPORTING.md) for details on report structure and interpretation. @@ -367,5 +375,6 @@ This processes each year with all scenario combinations. ## See Also - [USAGE.md](USAGE.md) – General CLI usage +- [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md) – Multi-year runs (same settings, many years) - [CONFIG.md](CONFIG.md) – Configuration options - [REPORTING.md](REPORTING.md) – Understanding reports and manifests diff --git a/docs/USAGE.md b/docs/USAGE.md index 53c7054..1aec6e6 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -27,6 +27,7 @@ These options can be used with any command: - `--config PATH`: Path to the configuration YAML file (default: `config/config.yaml`) - `--log-level LEVEL`: Set the logging level (choices: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`; default: `INFO`) +- `--version`: Print the installed `eddypro-batch` version and exit - `--help`, `-h`: Show help message and exit **Logging to file:** Configure `log_file` in your config to write logs to a file @@ -63,6 +64,8 @@ eddypro-batch run [OPTIONS] | `--mp` | flag | Enable multiprocessing | | `--max-proc N` | int | Maximum number of processes for multiprocessing | | `--dry-run` | flag | Generate files without executing EddyPro | +| `--monitor` | flag | Enable performance monitoring (default) | +| `--no-monitor` | flag | Disable performance monitoring (no metrics files; ignores `--metrics-interval`) | | `--metrics-interval SECONDS` | float | Performance monitoring sampling interval (default: 0.5) | | `--reports-dir PATH` | str | Custom reports directory (default: `{output_dir}/reports`) | | `--report-charts ENGINE` | str | Chart engine for reports (choices: `plotly`, `svg`, `none`; default: `plotly`) | @@ -125,11 +128,16 @@ eddypro-batch scenarios [OPTIONS] | `--max-scenarios N` | int | Maximum number of scenarios allowed (default: 32) | | `--site SITE_ID` | str | Site ID to process | | `--years YEAR [YEAR ...]` | int | Years to process | -| `--metrics-interval SECONDS` | float | Performance monitoring sampling interval (default: 0.5) | | `--dry-run` | flag | Generate files without executing EddyPro | +| `--monitor` | flag | Enable performance monitoring (default) | +| `--no-monitor` | flag | Disable performance monitoring (no metrics files; ignores `--metrics-interval`) | +| `--metrics-interval SECONDS` | float | Performance monitoring sampling interval (default: 0.5) | +| `--reports-dir PATH` | str | Custom reports directory (default: `{output_dir}/reports`) | -**Reporting:** The `scenarios` command currently writes `run_manifest.json` only. -HTML reports are generated by the `run` command. +**Reporting:** `scenarios` generates a per-scenario HTML report at +`{output_dir}/{scenario_suffix}/reports/run_report.html` for each scenario, +plus one aggregate comparison report and `run_manifest.json` under +`reports_dir`. See [REPORTING.md](REPORTING.md). **Examples:** @@ -225,16 +233,16 @@ eddypro-batch --log-level WARNING validate Validation Report ============================================================ -✓ Config Structure: OK -✓ Config Sanity: OK -❌ Paths: 1 error(s) - • EddyPro executable not found: /path/to/eddypro_rp.exe +[PASS] Config Structure: OK +[PASS] Config Sanity: OK +[FAIL] Paths: 1 error(s) + - EddyPro executable not found: /path/to/eddypro_rp.exe → Check 'eddypro_executable' path in config -✓ Ecmd Schema: OK -✓ Ecmd Sanity: OK +[PASS] Ecmd Schema: OK +[PASS] Ecmd Sanity: OK ============================================================ -❌ Total errors: 1 +[FAIL] Total errors: 1 ``` --- @@ -300,6 +308,10 @@ summary table for scenarios, duration, and success status. eddypro-batch run --site GL-ZaF --years 2020 2021 2022 2023 --mp --max-proc 4 ``` +For a complete, real-world multi-year walkthrough (config file and pure-CLI +forms, choosing `max_processes`, reading the bottleneck report), see +[MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md). + **Test scenario matrix for specific site and year:** ```bash eddypro-batch scenarios \ @@ -326,6 +338,11 @@ eddypro-batch --log-level DEBUG validate All commands read from a YAML configuration file (default: `config/config.yaml`). See [CONFIG.md](CONFIG.md) for details on all available options. +## Multi-Year Runs + +For a complete worked example of processing several years with the same +settings (config file and pure-CLI forms), see [MULTI_YEAR_RUNS.md](MULTI_YEAR_RUNS.md). + ## Scenario Processing For detailed information about how scenarios work, naming conventions, and output structure, see [SCENARIOS.md](SCENARIOS.md). diff --git a/docs/plan/implemented/METADATA_POPULATION_PLAN.md b/docs/plan/implemented/METADATA_POPULATION_PLAN.md index 31f3de5..25b8cc5 100644 --- a/docs/plan/implemented/METADATA_POPULATION_PLAN.md +++ b/docs/plan/implemented/METADATA_POPULATION_PLAN.md @@ -2,9 +2,9 @@ ## Results (current behavior) -- The .metadata file is currently created by copying a site-specific or generic template during scenario setup in [src/eddypro_batch_processor/core.py](src/eddypro_batch_processor/core.py). -- The .eddypro file is written via the INI utilities in [src/eddypro_batch_processor/ini_tools.py](src/eddypro_batch_processor/ini_tools.py). -- Dynamic metadata (.txt) is generated from the ECMD CSV in [src/eddypro_batch_processor/ecmd.py](src/eddypro_batch_processor/ecmd.py). +- The .metadata file is currently created by copying a site-specific or generic template during scenario setup in [src/eddypro_batch_processor/core.py](../../../src/eddypro_batch_processor/core.py). +- The .eddypro file is written via the INI utilities in [src/eddypro_batch_processor/ini_tools.py](../../../src/eddypro_batch_processor/ini_tools.py). +- Dynamic metadata (.txt) is generated from the ECMD CSV in [src/eddypro_batch_processor/ecmd.py](../../../src/eddypro_batch_processor/ecmd.py). - There is no current logic that populates .metadata values from ECMD. ## Required behavior (per request) @@ -49,7 +49,7 @@ Populate the following .metadata values based on ECMD input (column names on the ### 1. Add ECMD row selector utility -Create a new function in [src/eddypro_batch_processor/ecmd.py](src/eddypro_batch_processor/ecmd.py) to load and select the ECMD row for a given site and year. +Create a new function in [src/eddypro_batch_processor/ecmd.py](../../../src/eddypro_batch_processor/ecmd.py) to load and select the ECMD row for a given site and year. **Proposed behavior:** @@ -66,7 +66,7 @@ Create a new function in [src/eddypro_batch_processor/ecmd.py](src/eddypro_batch ### 2. Add metadata patcher in ini_tools.py -Add a function in [src/eddypro_batch_processor/ini_tools.py](src/eddypro_batch_processor/ini_tools.py) that: +Add a function in [src/eddypro_batch_processor/ini_tools.py](../../../src/eddypro_batch_processor/ini_tools.py) that: - Reads the .metadata template (already copied into the scenario output dir). - Applies the static updates (`file_name`, `site_id`). @@ -86,7 +86,7 @@ Update the flow so that .metadata population is invoked **after** the .eddypro f 1. Writes the .eddypro file (existing `write_ini_file()` logic). 2. Populates the .metadata file using the ECMD selector and patcher. -This keeps the sequencing requirement (“inside ini_tools.py just after writing the .eddypro file”) while minimizing changes in the core orchestration in [src/eddypro_batch_processor/core.py](src/eddypro_batch_processor/core.py). +This keeps the sequencing requirement (“inside ini_tools.py just after writing the .eddypro file”) while minimizing changes in the core orchestration in [src/eddypro_batch_processor/core.py](../../../src/eddypro_batch_processor/core.py). ### 4. Tests @@ -97,14 +97,14 @@ Add unit tests for: - End-to-end scenario run in dry-run mode to ensure .metadata is populated. Likely locations: -- [tests/test_ecmd.py](tests/test_ecmd.py) (new or existing) -- [tests/test_ini_tools.py](tests/test_ini_tools.py) -- [tests/test_e2e_integration.py](tests/test_e2e_integration.py) +- [tests/test_ecmd.py](../../../tests/test_ecmd.py) (new or existing) +- [tests/test_ini_tools.py](../../../tests/test_ini_tools.py) +- [tests/test_e2e_integration.py](../../../tests/test_e2e_integration.py) ### 5. Documentation and changelog -- Update [docs/CONFIG.md](docs/CONFIG.md) or [docs/SCENARIOS.md](docs/SCENARIOS.md) only if user-facing behavior needs to be documented. -- Add an entry under **[Unreleased]** in [CHANGELOG.md](CHANGELOG.md). +- Update [docs/CONFIG.md](../../../docs/CONFIG.md) or [docs/SCENARIOS.md](../../../docs/SCENARIOS.md) only if user-facing behavior needs to be documented. +- Add an entry under **[Unreleased]** in [CHANGELOG.md](../../../CHANGELOG.md). ## Open questions (for confirmation) diff --git a/docs/plan/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md b/docs/plan/implemented/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md similarity index 73% rename from docs/plan/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md rename to docs/plan/implemented/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md index 206bfaf..d68848d 100644 --- a/docs/plan/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md +++ b/docs/plan/implemented/MONITORING_TOGGLE_IMPLEMENTATION_PLAN.md @@ -13,7 +13,7 @@ Add a way to disable performance monitoring via CLI and config while keeping cur ### 1) CLI wiring Update argument parsing and overrides in: -- [src/eddypro_batch_processor/cli.py](src/eddypro_batch_processor/cli.py) +- [src/eddypro_batch_processor/cli.py](../../../src/eddypro_batch_processor/cli.py) Tasks: - Add `--monitor` / `--no-monitor` flags to `run` command. @@ -23,7 +23,7 @@ Tasks: ### 2) Config schema and validation Update validation rules in: -- [src/eddypro_batch_processor/validation.py](src/eddypro_batch_processor/validation.py) +- [src/eddypro_batch_processor/validation.py](../../../src/eddypro_batch_processor/validation.py) Tasks: - Include `monitoring_enabled` in required keys (or treat as optional with default). @@ -32,8 +32,8 @@ Tasks: ### 3) Core execution path Update monitoring entry points in: -- [src/eddypro_batch_processor/core.py](src/eddypro_batch_processor/core.py) -- [src/eddypro_batch_processor/monitor.py](src/eddypro_batch_processor/monitor.py) +- [src/eddypro_batch_processor/core.py](../../../src/eddypro_batch_processor/core.py) +- [src/eddypro_batch_processor/monitor.py](../../../src/eddypro_batch_processor/monitor.py) Tasks: - Add `monitoring_enabled: bool` parameter to `run_subprocess_with_monitoring()`. @@ -43,11 +43,11 @@ Tasks: ### 4) Tests Update or add tests in: -- [tests/test_cli.py](tests/test_cli.py) -- [tests/test_cli_functions.py](tests/test_cli_functions.py) -- [tests/test_e2e_integration.py](tests/test_e2e_integration.py) -- [tests/test_validation.py](tests/test_validation.py) -- [tests/test_monitor.py](tests/test_monitor.py) +- [tests/test_cli.py](../../../tests/test_cli.py) +- [tests/test_cli_functions.py](../../../tests/test_cli_functions.py) +- [tests/test_e2e_integration.py](../../../tests/test_e2e_integration.py) +- [tests/test_validation.py](../../../tests/test_validation.py) +- [tests/test_monitor.py](../../../tests/test_monitor.py) Test cases: - CLI flags set/clear `monitoring_enabled`. @@ -57,11 +57,11 @@ Test cases: ### 5) Documentation updates Update: -- [README.md](README.md) -- [docs/USAGE.md](docs/USAGE.md) -- [docs/CONFIG.md](docs/CONFIG.md) -- [docs/REPORTING.md](docs/REPORTING.md) -- [CHANGELOG.md](CHANGELOG.md) +- [README.md](../../../README.md) +- [docs/USAGE.md](../../../docs/USAGE.md) +- [docs/CONFIG.md](../../../docs/CONFIG.md) +- [docs/REPORTING.md](../../../docs/REPORTING.md) +- [CHANGELOG.md](../../../CHANGELOG.md) Docs tasks: - Add config option `monitoring_enabled` with default and description. @@ -71,24 +71,24 @@ Docs tasks: ## Detailed Implementation Steps 1) **Add config key** -- In [docs/CONFIG.md](docs/CONFIG.md), document: +- In [docs/CONFIG.md](../../../docs/CONFIG.md), document: - `monitoring_enabled: true` (default). - If `false`, no metrics collected; `metrics_interval_seconds` ignored. 2) **CLI additions** -- In [src/eddypro_batch_processor/cli.py](src/eddypro_batch_processor/cli.py): +- In [src/eddypro_batch_processor/cli.py](../../../src/eddypro_batch_processor/cli.py): - Add flags: - `--monitor` (enable monitoring) - `--no-monitor` (disable monitoring) - In config overrides, set `config["monitoring_enabled"]` accordingly. 3) **Validation rules** -- In [src/eddypro_batch_processor/validation.py](src/eddypro_batch_processor/validation.py): +- In [src/eddypro_batch_processor/validation.py](../../../src/eddypro_batch_processor/validation.py): - Validate `monitoring_enabled` is bool. - If `monitoring_enabled` is `False`, skip positive check for `metrics_interval_seconds`. 4) **Core runtime logic** -- In [src/eddypro_batch_processor/core.py](src/eddypro_batch_processor/core.py): +- In [src/eddypro_batch_processor/core.py](../../../src/eddypro_batch_processor/core.py): - Add parameter `monitoring_enabled: bool` to `run_subprocess_with_monitoring`. - If disabled, run subprocess without `MonitoredOperation`. - Ensure no metrics files emitted. @@ -100,8 +100,8 @@ Docs tasks: 6) **Docs + changelog** - Update CLI usage examples with `--no-monitor`. -- Mention effect on reports/metrics in [docs/REPORTING.md](docs/REPORTING.md). -- Add entry to [CHANGELOG.md](CHANGELOG.md) under `[Unreleased]`. +- Mention effect on reports/metrics in [docs/REPORTING.md](../../../docs/REPORTING.md). +- Add entry to [CHANGELOG.md](../../../CHANGELOG.md) under `[Unreleased]`. ## Acceptance Criteria - `eddypro-batch run --no-monitor` runs without creating metrics files. diff --git a/docs/plan/PERFORMANCE_ANALYSIS_DESIGN.md b/docs/plan/implemented/PERFORMANCE_ANALYSIS_DESIGN.md similarity index 100% rename from docs/plan/PERFORMANCE_ANALYSIS_DESIGN.md rename to docs/plan/implemented/PERFORMANCE_ANALYSIS_DESIGN.md diff --git a/examples/README.md b/examples/README.md index df81df5..584f5dd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,10 +5,15 @@ This directory contains example configurations and sample data to help you get s ## Contents - **`config.yaml`** - Basic configuration example +- **`multi_year_config.yaml`** - Full multi-year run example (site GL-Dsk, 2020-2025); see [docs/MULTI_YEAR_RUNS.md](../docs/MULTI_YEAR_RUNS.md) - **`sample_ecmd.csv`** - Example ECMD metadata file format - **`basic_project.ini`** - Minimal EddyPro project configuration - **`README.md`** - This file +For the full, authoritative config reference with every key, see +[`config/config.yaml.example`](../config/config.yaml.example) and +[docs/CONFIG.md](../docs/CONFIG.md). + ## Quick Start 1. Copy `config.yaml` to your project's `config/` directory diff --git a/examples/config.yaml b/examples/config.yaml index e9517b1..9157af3 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -1,5 +1,8 @@ # Example Configuration for EddyPro Batch Processor -# Copy this file to config/config.yaml and customize for your environment +# Copy this file to config/config.yaml and customize for your environment. +# For the full annotated reference (every key), see config/config.yaml.example +# and docs/CONFIG.md. For a real multi-year worked example, see +# examples/multi_year_config.yaml and docs/MULTI_YEAR_RUNS.md. # EddyPro executable path (update for your installation) eddypro_executable: "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe" @@ -17,12 +20,19 @@ output_dir_pattern: "data/processed/{site_id}/{year}" ecmd_file: "data/{site_id}_ecmd.csv" # Processing options -multiprocessing: false # Set to true for parallel processing -max_processes: 4 # Adjust based on your CPU cores -stream_output: true # Show EddyPro output during processing +multiprocessing: false # Set to true for parallel processing across years +max_processes: 4 # Adjust based on your CPU cores +stream_output: true # Show EddyPro output during processing -# Logging and monitoring +# Logging log_level: INFO # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL +log_file: null # Optional: path to a log file (null disables file logging) +log_max_bytes: 10485760 # Log rotation size in bytes (0 disables rotation) +log_backup_count: 5 # Number of rotated log files to keep +log_eddypro_output: true # Capture EddyPro stdout/stderr in logs + +# Performance monitoring +monitoring_enabled: true # Set to false to skip CPU/memory/disk sampling entirely metrics_interval_seconds: 0.5 # Reporting diff --git a/examples/multi_year_config.yaml b/examples/multi_year_config.yaml new file mode 100644 index 0000000..b972b70 --- /dev/null +++ b/examples/multi_year_config.yaml @@ -0,0 +1,55 @@ +# Multi-year run example: site GL-Dsk, 2020-2025. +# +# A multi-year run applies the SAME processing settings across several years. +# This is distinct from a scenario run (see docs/SCENARIOS.md), which applies +# several combinations of processing parameters to the same year(s). +# See docs/MULTI_YEAR_RUNS.md for the full worked walkthrough, including the +# pure-CLI equivalent of this file. +# +# Usage: +# eddypro-batch --config examples/multi_year_config.yaml validate +# eddypro-batch --config examples/multi_year_config.yaml run --dry-run +# eddypro-batch --config examples/multi_year_config.yaml run + +# Path to the EddyPro raw-processing executable +eddypro_executable: "C:/Program Files/LI-COR/EddyPro-7.0.9/bin/eddypro_rp.exe" + +# Site identifier +site_id: GL-Dsk + +# Six years processed with identical settings +years_to_process: [2020, 2021, 2022, 2023, 2024, 2025] + +# Input/output directory patterns ({site_id} and {year} are substituted per year) +input_dir_pattern: "D:/L0_raw/{site_id}/{year}/ec/rflux_csv" +output_dir_pattern: "D:/L1_processed/{site_id}/{year}/ec_rflux_sc26" + +# Site instrument/metadata history (ECMD) +ecmd_file: "D:/L1_processed/{site_id}/ecmd/{site_id}_ecmd.csv" + +# Parallelize across years: one EddyPro run per year, up to 6 concurrent workers. +# Choose max_processes based on physical cores and disk throughput (see +# docs/MULTI_YEAR_RUNS.md for guidance on reading the bottleneck report). +multiprocessing: true +max_processes: 6 + +# Output streaming and logging +stream_output: true +log_level: INFO +log_file: "logs/GL-Dsk_eddypro_processing.log" +log_max_bytes: 10485760 +log_backup_count: 5 +log_eddypro_output: true + +# Performance monitoring: sample CPU/memory/disk every second per year-worker. +# Set monitoring_enabled: false for maximum throughput on large batches. +monitoring_enabled: true +metrics_interval_seconds: 1.0 + +# Reports directory set explicitly: it otherwise defaults to the FIRST +# processed year's output directory, which is confusing for a multi-year run. +reports_dir: "D:/L1_processed/GL-Dsk/reports/2020-2025" +report_charts: plotly + +# Use the default project template +project_template: null diff --git a/pyproject.toml b/pyproject.toml index b6ea128..6fc1ada 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,9 +34,13 @@ dependencies = [ dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", - "ruff>=0.1.0", - "black>=23.0.0", - "mypy>=1.0.0", + # Upper bounds are deliberate: these are formatters/linters whose output + # changes between releases. Without them CI installs the newest version and + # disagrees with the pinned pre-commit hooks, which is green locally and red + # in CI. Bump these and .pre-commit-config.yaml together. + "ruff>=0.16.3,<0.17", + "black>=26.5.1,<27", + "mypy>=1.8.0", "pre-commit>=3.0.0", "bandit[toml]>=1.7.0", "types-PyYAML>=6.0.0", @@ -73,6 +77,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "SIM", "TRY", "PL"] ignore = [ "TRY003", # Allow custom exception messages "PLR0913", # Allow many function arguments (common in scientific code) + "PLR0917", # Same, for positional count: these are keyword-called in practice "PLR2004", # Allow magic values in non-test code (scientific constants) "PLR0912", # Allow many branches (legacy code) "PLR0915", # Allow many statements (legacy code) diff --git a/requirements.txt b/requirements.txt index a84d4fa..5fa7102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -certifi==2024.8.30 -charset-normalizer==3.4.0 -idna==3.10 -PyYAML==6.0.2 -requests==2.32.3 -urllib3==2.2.3 -psutil>=5.8.0 +# Runtime dependencies. Authoritative source is pyproject.toml [project.dependencies]; +# this file exists for `pip install -r requirements.txt` convenience and must be kept +# in sync with pyproject.toml. +pyyaml>=6.0 +psutil>=5.9.0 plotly>=5.0.0 +tqdm>=4.64.0 +pandas>=1.5.0 diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index 22bda2a..0000000 --- a/src/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# src/__init__.py - -""" -EddyPro Batch Processor Package - -This package provides functionality to automate and manage EddyPro processing tasks. -Currently under development - refactoring from monolithic script to modular package. -""" - -# Re-export key functions will be implemented during Milestone 2-3 -# For now, keeping this minimal to avoid import errors during transition - -__all__: list[str] = [ - # Will be populated as modules are refactored -] diff --git a/src/eddypro_batch_processor/__init__.py b/src/eddypro_batch_processor/__init__.py index b912f3f..29fcfdd 100644 --- a/src/eddypro_batch_processor/__init__.py +++ b/src/eddypro_batch_processor/__init__.py @@ -1,9 +1,16 @@ """EddyPro Batch Processor - Automated EddyPro processing with scenario support.""" -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version + +try: + __version__ = _version("eddypro-batch-processor") +except PackageNotFoundError: # pragma: no cover - source checkout without install + __version__ = "0.0.0+unknown" + __author__ = "Rasmus Jensen" __email__ = "raje@ecos.au.dk" from .core import EddyProBatchProcessor, load_config, validate_config -__all__ = ["EddyProBatchProcessor", "load_config", "validate_config"] +__all__ = ["EddyProBatchProcessor", "__version__", "load_config", "validate_config"] diff --git a/src/eddypro_batch_processor/analysis.py b/src/eddypro_batch_processor/analysis.py new file mode 100644 index 0000000..d3fb400 --- /dev/null +++ b/src/eddypro_batch_processor/analysis.py @@ -0,0 +1,340 @@ +""" +Bottleneck analysis for EddyPro performance metrics. + +Digests the ``metrics.csv`` time series produced by +:mod:`eddypro_batch_processor.monitor` into summary statistics and a +traffic-light classification that answers the practical question: *was this run +limited by the CPU, by memory, or by the disk?* + +Deliberately import-light -- stdlib ``csv`` and arithmetic only, no pandas -- so +that it can be used from the reporting path without pulling in heavy imports. +""" + +from __future__ import annotations + +import csv +import logging +from collections.abc import Mapping +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Literal, TypedDict + +logger = logging.getLogger(__name__) + +Status = Literal["GREEN", "YELLOW", "RED", "UNKNOWN"] +Bottleneck = Literal["CPU", "MEMORY", "DISK_THROUGHPUT", "DISK_IOPS", "NONE", "UNKNOWN"] + + +class PerformanceThresholds(TypedDict, total=False): + """Tunable limits used to classify a run. Override via ``config.yaml``.""" + + cpu_high_percent: float + cpu_moderate_percent: float + cpu_idle_percent: float + memory_high_percent: float + memory_moderate_percent: float + disk_high_mb_per_s: float + disk_moderate_mb_per_s: float + disk_high_iops: float + + +DEFAULT_THRESHOLDS: dict[str, float] = { + "cpu_high_percent": 90.0, + "cpu_moderate_percent": 70.0, + # Below this, the CPU is considered idle enough that a high I/O rate is + # evidence the run was waiting on the disk rather than computing. + "cpu_idle_percent": 40.0, + "memory_high_percent": 85.0, + "memory_moderate_percent": 70.0, + # Roughly a mechanical-disk ceiling; raise substantially for NVMe. + "disk_high_mb_per_s": 100.0, + "disk_moderate_mb_per_s": 50.0, + "disk_high_iops": 1000.0, +} + + +@dataclass +class MetricStats: + """Summary statistics for one metric series.""" + + mean: float = 0.0 + max: float = 0.0 + p95: float = 0.0 + + def as_dict(self) -> dict[str, float]: + return asdict(self) + + +@dataclass +class ScenarioAnalysis: + """Result of analysing one metrics time series.""" + + scenario_name: str = "baseline" + sample_count: int = 0 + duration_seconds: float = 0.0 + + cpu: MetricStats = field(default_factory=MetricStats) + memory_mb: MetricStats = field(default_factory=MetricStats) + system_memory_percent: MetricStats = field(default_factory=MetricStats) + read_mb_per_s: MetricStats = field(default_factory=MetricStats) + write_mb_per_s: MetricStats = field(default_factory=MetricStats) + read_iops: MetricStats = field(default_factory=MetricStats) + write_iops: MetricStats = field(default_factory=MetricStats) + + total_read_mb: float = 0.0 + total_write_mb: float = 0.0 + peak_memory_mb: float = 0.0 + + cpu_status: Status = "UNKNOWN" + memory_status: Status = "UNKNOWN" + disk_status: Status = "UNKNOWN" + primary_bottleneck: Bottleneck = "UNKNOWN" + explanation: str = "No metrics available." + + def to_dict(self) -> dict[str, Any]: + """Serialise for inclusion in a manifest or report.""" + data = asdict(self) + return data + + +def _to_float(value: Any) -> float | None: + """Parse a CSV cell into a float, tolerating blanks and junk.""" + if value is None or value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _stats(values: list[float]) -> MetricStats: + """Compute mean/max/p95 for a series.""" + if not values: + return MetricStats() + ordered = sorted(values) + n = len(ordered) + if n == 1: + p95 = ordered[0] + else: + index = 0.95 * (n - 1) + lower = int(index) + upper = min(lower + 1, n - 1) + weight = index - lower + p95 = ordered[lower] * (1 - weight) + ordered[upper] * weight + return MetricStats( + mean=round(sum(ordered) / n, 3), + max=round(ordered[-1], 3), + p95=round(p95, 3), + ) + + +class BottleneckAnalyzer: + """Classify a metrics time series into a primary bottleneck.""" + + def __init__(self, thresholds: Mapping[str, Any] | None = None): + merged: dict[str, float] = dict(DEFAULT_THRESHOLDS) + if thresholds: + # Unknown keys are ignored rather than rejected, so a config written + # for a newer version does not break an older install. + merged.update( + {k: float(v) for k, v in thresholds.items() if k in DEFAULT_THRESHOLDS} + ) + self.thresholds: dict[str, float] = merged + + # ------------------------------------------------------------------ + + def analyze( + self, metrics_csv_path: str | Path, scenario_name: str = "baseline" + ) -> ScenarioAnalysis: + """ + Analyse a metrics CSV file. + + Args: + metrics_csv_path: Path to a ``metrics*.csv`` written by the monitor. + scenario_name: Label carried through to the report. + + Returns: + A :class:`ScenarioAnalysis`. Always returns an object -- a missing or + unreadable file yields an ``UNKNOWN`` classification rather than raising. + """ + path = Path(metrics_csv_path) + rows = self._read_rows(path) + if not rows: + return ScenarioAnalysis( + scenario_name=scenario_name, + explanation=f"No usable samples found in {path.name}.", + ) + return self.analyze_rows(rows, scenario_name=scenario_name) + + def analyze_rows( + self, rows: list[dict[str, Any]], scenario_name: str = "baseline" + ) -> ScenarioAnalysis: + """Analyse already-parsed metric rows.""" + + def series(key: str) -> list[float]: + return [v for v in (_to_float(r.get(key)) for r in rows) if v is not None] + + cpu = series("cpu_percent") + memory = series("memory_mb") + sys_mem = series("system_memory_percent") + read_rate = series("read_mb_per_s") + write_rate = series("write_mb_per_s") + read_iops = series("read_iops") + write_iops = series("write_iops") + read_total = series("read_mb") + write_total = series("write_mb") + rel_time = series("relative_time") + + analysis = ScenarioAnalysis( + scenario_name=scenario_name, + sample_count=len(rows), + duration_seconds=round(rel_time[-1], 2) if rel_time else 0.0, + cpu=_stats(cpu), + memory_mb=_stats(memory), + system_memory_percent=_stats(sys_mem), + read_mb_per_s=_stats(read_rate), + write_mb_per_s=_stats(write_rate), + read_iops=_stats(read_iops), + write_iops=_stats(write_iops), + total_read_mb=round(read_total[-1], 3) if read_total else 0.0, + total_write_mb=round(write_total[-1], 3) if write_total else 0.0, + peak_memory_mb=round(max(memory), 3) if memory else 0.0, + ) + + self._classify(analysis) + return analysis + + # ------------------------------------------------------------------ + + def _read_rows(self, path: Path) -> list[dict[str, Any]]: + """Read a metrics CSV, returning [] on any failure.""" + if not path.exists(): + logger.debug(f"Metrics file not found: {path}") + return [] + try: + with path.open("r", encoding="utf-8", newline="") as f: + return list(csv.DictReader(f)) + except Exception: + logger.exception(f"Failed to read metrics from {path}") + return [] + + def _classify(self, a: ScenarioAnalysis) -> None: + """Assign per-resource statuses and the primary bottleneck.""" + t = self.thresholds + + # CPU: judged on sustained load, so p95 rather than the peak. + a.cpu_status = self._level( + a.cpu.p95, t["cpu_moderate_percent"], t["cpu_high_percent"] + ) + + # Memory: system-wide pressure matters more than the process footprint, + # but fall back to the process if system memory was not recorded. + mem_metric = ( + a.system_memory_percent.p95 if a.system_memory_percent.max > 0 else 0.0 + ) + a.memory_status = self._level( + mem_metric, t["memory_moderate_percent"], t["memory_high_percent"] + ) + + # Disk: combined read+write throughput at p95. + disk_rate = a.read_mb_per_s.p95 + a.write_mb_per_s.p95 + total_iops = a.read_iops.p95 + a.write_iops.p95 + disk_by_rate = self._level( + disk_rate, t["disk_moderate_mb_per_s"], t["disk_high_mb_per_s"] + ) + disk_by_iops = "RED" if total_iops > t["disk_high_iops"] else "GREEN" + a.disk_status = "RED" if "RED" in (disk_by_rate, disk_by_iops) else disk_by_rate + + cpu_idle = a.cpu.p95 < t["cpu_idle_percent"] + + # Priority: saturated CPU is the clearest signal. Otherwise a busy disk + # paired with an idle CPU means the run was waiting on I/O. + if a.cpu_status == "RED": + a.primary_bottleneck = "CPU" + a.explanation = ( + f"CPU saturated: sustained (p95) utilisation {a.cpu.p95:.1f}%. " + f"Processing is compute-bound; more parallel years will not help " + f"unless spare cores are available." + ) + elif a.memory_status == "RED": + a.primary_bottleneck = "MEMORY" + a.explanation = ( + f"Memory pressure: system memory reached {mem_metric:.1f}% (p95), " + f"peak process-tree RSS {a.peak_memory_mb:.0f} MB. Reduce " + f"max_processes to avoid swapping." + ) + elif cpu_idle and disk_by_rate == "RED": + a.primary_bottleneck = "DISK_THROUGHPUT" + a.explanation = ( + f"Disk-bound: {disk_rate:.1f} MB/s sustained while the CPU sat at " + f"{a.cpu.p95:.1f}%. The run is waiting on storage -- move the data " + f"to a faster disk before adding parallelism." + ) + elif cpu_idle and disk_by_iops == "RED": + a.primary_bottleneck = "DISK_IOPS" + a.explanation = ( + f"Disk-bound on latency: {total_iops:.0f} IOPS (p95) with the CPU at " + f"{a.cpu.p95:.1f}%. Many small reads -- typical of raw files split " + f"into short intervals." + ) + elif a.cpu_status == "YELLOW": + a.primary_bottleneck = "CPU" + a.explanation = ( + f"Moderately CPU-bound: sustained utilisation {a.cpu.p95:.1f}%. " + f"Some headroom remains." + ) + else: + a.primary_bottleneck = "NONE" + a.explanation = ( + f"No clear bottleneck: CPU {a.cpu.p95:.1f}% (p95), disk " + f"{disk_rate:.1f} MB/s, peak memory {a.peak_memory_mb:.0f} MB. " + f"There is headroom to increase max_processes." + ) + + @staticmethod + def _level(value: float, moderate: float, high: float) -> Status: + """Map a value onto a traffic light.""" + if value >= high: + return "RED" + if value >= moderate: + return "YELLOW" + return "GREEN" + + +def analyze_metrics_files( + metrics_paths: dict[str, Path], + thresholds: Mapping[str, Any] | None = None, +) -> list[ScenarioAnalysis]: + """ + Analyse several metrics files at once. + + Args: + metrics_paths: Mapping of scenario name -> metrics CSV path. + thresholds: Optional threshold overrides. + + Returns: + One :class:`ScenarioAnalysis` per input, in insertion order. + """ + analyzer = BottleneckAnalyzer(thresholds) + return [ + analyzer.analyze(path, scenario_name=name) + for name, path in metrics_paths.items() + ] + + +def dominant_bottleneck(analyses: list[ScenarioAnalysis]) -> str: + """ + Pick the most frequently observed limiting resource across analyses. + + A real bottleneck outranks ``NONE`` even when ``NONE`` is more common: one + saturated year is the actionable finding, not the quiet ones. + """ + if not analyses: + return "UNKNOWN" + counts: dict[str, int] = {} + for a in analyses: + counts[a.primary_bottleneck] = counts.get(a.primary_bottleneck, 0) + 1 + ranked = sorted( + counts.items(), key=lambda kv: (kv[0] in ("NONE", "UNKNOWN"), -kv[1]) + ) + return ranked[0][0] diff --git a/src/eddypro_batch_processor/cli.py b/src/eddypro_batch_processor/cli.py index 9ed5c57..4b001e0 100644 --- a/src/eddypro_batch_processor/cli.py +++ b/src/eddypro_batch_processor/cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ EddyPro Batch Processor CLI. @@ -9,14 +8,25 @@ import argparse import json import logging +import os import shutil import sys -from datetime import datetime +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime, timezone from logging.handlers import RotatingFileHandler from pathlib import Path -from typing import NoReturn +from typing import Any, NoReturn -from . import core, ecmd, ini_tools, report, scenarios, validation +from . import ( + __version__, + analysis, + core, + ecmd, + ini_tools, + report, + scenarios, + validation, +) def setup_logging( @@ -71,6 +81,12 @@ def create_parser() -> argparse.ArgumentParser: ) # Global options + parser.add_argument( + "--version", + action="version", + version=f"eddypro-batch {__version__}", + help="Show the installed package version and exit", + ) parser.add_argument( "--config", type=str, @@ -128,8 +144,22 @@ def create_parser() -> argparse.ArgumentParser: run_parser.add_argument( "--metrics-interval", type=float, - default=0.5, - help="Performance monitoring sampling interval in seconds (default: 0.5)", + default=None, + help="Performance monitoring sampling interval in seconds " + "(default: metrics_interval_seconds from config, or 0.5)", + ) + run_parser.add_argument( + "--monitor", + dest="monitor", + action="store_true", + default=None, + help="Enable performance monitoring (overrides monitoring_enabled in config)", + ) + run_parser.add_argument( + "--no-monitor", + dest="monitor", + action="store_false", + help="Disable performance monitoring; no metrics files are written", ) run_parser.add_argument( "--reports-dir", @@ -238,8 +268,27 @@ def create_parser() -> argparse.ArgumentParser: scenarios_parser.add_argument( "--metrics-interval", type=float, - default=0.5, - help="Performance monitoring sampling interval in seconds (default: 0.5)", + default=None, + help="Performance monitoring sampling interval in seconds " + "(default: metrics_interval_seconds from config, or 0.5)", + ) + scenarios_parser.add_argument( + "--monitor", + dest="monitor", + action="store_true", + default=None, + help="Enable performance monitoring (overrides monitoring_enabled in config)", + ) + scenarios_parser.add_argument( + "--no-monitor", + dest="monitor", + action="store_false", + help="Disable performance monitoring; no metrics files are written", + ) + scenarios_parser.add_argument( + "--reports-dir", + type=str, + help="Custom reports directory (default: {output_dir}/reports)", ) # Validate command @@ -264,6 +313,214 @@ def create_parser() -> argparse.ArgumentParser: return parser +def _raise_missing_ecmd(site: str, path: Path | None) -> NoReturn: + raise ecmd.ECMDError(f"ECMD file not found for site {site}: {path}") + + +def _prepare_year_project( + *, + year: int, + site_id: str, + config: dict[str, Any], + template_path: Path, + ini_parameters: dict[str, int], + dry_run: bool, +) -> Path: + """ + Build the EddyPro project file for one year. + + Returns: + Path to the generated ``.eddypro`` project file. + + Raises: + Exception: Any failure to assemble the project, metadata, or inputs. + """ + output_pattern = config["output_dir_pattern"] + output_dir = Path(output_pattern.format(year=year, site_id=site_id)) + output_dir.mkdir(parents=True, exist_ok=True) + + project_file = output_dir / f"{site_id}.eddypro" + + ini_config = ini_tools.read_ini_template(template_path) + if ini_parameters: + validated_params = ini_tools.validate_parameters(ini_parameters) + ini_tools.patch_ini_parameters(ini_config, validated_params) + + ecmd_file_pattern = config.get("ecmd_file", "") + if "{site_id}" in ecmd_file_pattern: + ecmd_file = Path(ecmd_file_pattern.format(site_id=site_id)) + else: + ecmd_file = Path(ecmd_file_pattern) + + # Materialize metadata files early (idempotent) + try: + # Copy generic metadata template -> {site}.metadata + # All sites use the same template; ECMD values populate it. + metadata_template = Path("config") / "metadata_template.ini" + + if metadata_template.exists(): + shutil.copyfile(metadata_template, output_dir / f"{site_id}.metadata") + else: + logging.warning( + f"No metadata template found for site {site_id}, " + "skipping .metadata file generation" + ) + + # Generate dynamic metadata from ECMD CSV (all years included) + dyn_metadata_filename = f"{site_id}_dynamic_metadata.txt" + if ecmd_file.exists(): + ecmd.generate_dynamic_metadata( + ecmd_path=ecmd_file, + output_path=output_dir / dyn_metadata_filename, + site_id=site_id, + ) + else: + logging.warning( + f"ECMD file not found at {ecmd_file}, " + "skipping dynamic metadata generation" + ) + + except Exception as meta_err: + logging.warning(f"Failed to materialize metadata files: {meta_err}") + + if not ecmd_file.exists(): + _raise_missing_ecmd(site_id, ecmd_file) + + ecmd_row = ecmd.select_ecmd_row_for_year( + ecmd_path=ecmd_file, + site_id=site_id, + year=year, + ) + + # Patch path fields; input path comes from the configured pattern + input_pattern = config.get("input_dir_pattern", "") + data_path_value = input_pattern.format(year=year, site_id=site_id) + ini_tools.patch_ini_paths( + ini_config, + site_id=site_id, + proj_file=str(output_dir / f"{site_id}.metadata"), + dyn_metadata_file=str(output_dir / f"{site_id}_dynamic_metadata.txt"), + data_path=data_path_value, + out_path=str(output_dir), + ) + + # Patch Project metadata fields (creation_date, project_title, etc.) + ini_tools.patch_project_metadata( + ini_config, + site_id=site_id, + year=year, + scenario_suffix="", + ) + + ini_tools.write_project_file_with_metadata( + ini_config, + project_file, + metadata_path=output_dir / f"{site_id}.metadata", + site_id=site_id, + output_dir=output_dir, + ecmd_row=ecmd_row, + ) + + logging.info(f"Created project file: {project_file}") + + # Preflight validation: check data_path and file availability + if not dry_run: + ini_tools.validate_eddypro_inputs(ini_config) + ini_tools.validate_eddypro_metadata(ini_config) + + return project_file + + +def process_year(job: dict[str, Any]) -> dict[str, Any]: + """ + Process a single year end to end. + + Defined at module level and taking a plain dict so that it can be dispatched + to a :class:`~concurrent.futures.ProcessPoolExecutor` worker. Never raises: + every failure is captured into the returned record so that one bad year does + not abort the whole run and, crucially, remains visible in the run manifest. + + Returns: + ``{year, status, duration_seconds, error, output_dir}`` where status is + one of ``"success"``, ``"failed"``, or ``"dry_run"``. + """ + year = job["year"] + site_id = job["site_id"] + config = job["config"] + dry_run = job["dry_run"] + + # Worker processes start with a bare logging config; re-establish it so that + # parallel years still write to the configured log file. + if job.get("configure_logging"): + setup_logging( + job.get("log_level", "INFO"), + config.get("log_file"), + config.get("log_max_bytes"), + config.get("log_backup_count"), + ) + + output_dir = Path(config["output_dir_pattern"].format(year=year, site_id=site_id)) + started = datetime.now(timezone.utc) + record: dict[str, Any] = { + "year": year, + "status": "failed", + "duration_seconds": 0.0, + "error": None, + "output_dir": str(output_dir), + } + + logging.info(f"Processing year {year} for site {site_id}") + + try: + project_file = _prepare_year_project( + year=year, + site_id=site_id, + config=config, + template_path=Path(job["template_path"]), + ini_parameters=job["ini_parameters"], + dry_run=dry_run, + ) + except Exception as exc: + logging.exception(f"Failed to prepare project file for year {year}") + record["error"] = str(exc) + record["duration_seconds"] = ( + datetime.now(timezone.utc) - started + ).total_seconds() + return record + + if dry_run: + logging.info(f"Dry run: skipped EddyPro execution for year {year}") + record["status"] = "dry_run" + record["duration_seconds"] = ( + datetime.now(timezone.utc) - started + ).total_seconds() + return record + + try: + success = core.run_eddypro_with_monitoring( + project_file=project_file, + eddypro_executable=Path(config["eddypro_executable"]), + stream_output=job["stream_output"], + metrics_interval=job["metrics_interval"], + scenario_suffix="", + log_output=job["log_eddypro_output"], + monitoring_enabled=job["monitoring_enabled"], + ) + except Exception as exc: + logging.exception(f"EddyPro execution raised for year {year}") + record["error"] = str(exc) + else: + if success: + record["status"] = "success" + logging.info(f"EddyPro processing completed successfully for year {year}") + else: + record["error"] = "EddyPro returned a non-zero exit status" + logging.error(f"EddyPro processing failed for year {year}") + + record["duration_seconds"] = (datetime.now(timezone.utc) - started).total_seconds() + return record + + def cmd_run(args: argparse.Namespace) -> int: # noqa: PLR0912, PLR0915 """Execute the run command. @@ -281,7 +538,7 @@ def cmd_run(args: argparse.Namespace) -> int: # noqa: PLR0912, PLR0915 Exit code (0 for success, 1 for failure) """ logging.info("Starting EddyPro batch processing run...") - start_time = datetime.now() + start_time = datetime.now(timezone.utc) # Load configuration config_path = Path(args.config) @@ -340,8 +597,12 @@ def cmd_run(args: argparse.Namespace) -> int: # noqa: PLR0912, PLR0915 config["multiprocessing"] = True if getattr(args, "max_proc", None): config["max_processes"] = args.max_proc - if getattr(args, "metrics_interval", None): + # `is not None` matters: the parser default used to be 0.5, which is truthy, + # so metrics_interval_seconds from config.yaml was always silently overwritten. + if getattr(args, "metrics_interval", None) is not None: config["metrics_interval_seconds"] = args.metrics_interval + if getattr(args, "monitor", None) is not None: + config["monitoring_enabled"] = args.monitor if getattr(args, "reports_dir", None): config["reports_dir"] = args.reports_dir if getattr(args, "report_charts", None): @@ -350,13 +611,20 @@ def cmd_run(args: argparse.Namespace) -> int: # noqa: PLR0912, PLR0915 # Extract key settings site_id = config["site_id"] years = config["years_to_process"] - eddypro_exe = Path(config["eddypro_executable"]) + # The executable is read from config inside process_year so that the job dict + # stays picklable for the ProcessPoolExecutor workers. stream_output = config.get("stream_output", True) log_eddypro_output = config.get("log_eddypro_output", True) metrics_interval = config.get("metrics_interval_seconds", 0.5) + monitoring_enabled = config.get("monitoring_enabled", True) dry_run = args.dry_run config["dry_run"] = dry_run # Store in config for manifest + if not monitoring_enabled: + logging.info( + "Performance monitoring disabled; no metrics files will be written" + ) + if dry_run: logging.info("Dry run mode enabled - EddyPro will not be executed") @@ -383,168 +651,109 @@ def cmd_run(args: argparse.Namespace) -> int: # noqa: PLR0912, PLR0915 return 1 # Process each year - overall_success = True - years_processed = [] - - def _raise_missing_ecmd(site: str, path: Path | None) -> NoReturn: - raise ecmd.ECMDError(f"ECMD file not found for site {site}: {path}") - - for year in years: - logging.info(f"Processing year {year} for site {site_id}") - - # Determine paths - output_pattern = config["output_dir_pattern"] - output_dir = Path(output_pattern.format(year=year, site_id=site_id)) - - # Create output directory - output_dir.mkdir(parents=True, exist_ok=True) + use_mp = bool(config.get("multiprocessing", False)) + max_processes = int(config.get("max_processes", 1) or 1) + # Never spin up more workers than there is work, or than the machine has cores. + worker_count = max(1, min(max_processes, len(years), os.cpu_count() or 1)) + parallel = use_mp and worker_count > 1 and len(years) > 1 - # Generate project file with parameter overrides and patched paths - project_file = output_dir / f"{site_id}.eddypro" - try: - ini_config = ini_tools.read_ini_template(template_path) - if ini_parameters: - validated_params = ini_tools.validate_parameters(ini_parameters) - ini_tools.patch_ini_parameters(ini_config, validated_params) - - ecmd_file_pattern = config.get("ecmd_file", "") - if "{site_id}" in ecmd_file_pattern: - ecmd_file = Path(ecmd_file_pattern.format(site_id=site_id)) - else: - ecmd_file = Path(ecmd_file_pattern) - - # Materialize metadata files early (idempotent) - try: - # Copy generic metadata template -> {site}.metadata - # All sites use the same template; ECMD values populate it. - metadata_template = Path("config") / "metadata_template.ini" - - if metadata_template.exists(): - shutil.copyfile( - metadata_template, output_dir / f"{site_id}.metadata" - ) - else: - logging.warning( - f"No metadata template found for site {site_id}, " - "skipping .metadata file generation" - ) - - # Generate dynamic metadata from ECMD CSV (all years included) - dyn_metadata_filename = f"{site_id}_dynamic_metadata.txt" - if ecmd_file.exists(): - ecmd.generate_dynamic_metadata( - ecmd_path=ecmd_file, - output_path=output_dir / dyn_metadata_filename, - site_id=site_id, - ) - else: - logging.warning( - f"ECMD file not found at {ecmd_file}, " - "skipping dynamic metadata generation" - ) - - except Exception as meta_err: - logging.warning(f"Failed to materialize metadata files: {meta_err}") - - if not ecmd_file.exists(): - _raise_missing_ecmd(site_id, ecmd_file) - - ecmd_row = ecmd.select_ecmd_row_for_year( - ecmd_path=ecmd_file, - site_id=site_id, - year=year, - ) - - # Patch path fields - # Input path from configured pattern - input_pattern = config.get("input_dir_pattern", "") - data_path_value = input_pattern.format(year=year, site_id=site_id) - ini_tools.patch_ini_paths( - ini_config, - site_id=site_id, - proj_file=str(output_dir / f"{site_id}.metadata"), - dyn_metadata_file=str(output_dir / f"{site_id}_dynamic_metadata.txt"), - data_path=data_path_value, - out_path=str(output_dir), - ) - - # Patch Project metadata fields (creation_date, project_title, etc.) - ini_tools.patch_project_metadata( - ini_config, - site_id=site_id, - year=year, - scenario_suffix="", - ) - - ini_tools.write_project_file_with_metadata( - ini_config, - project_file, - metadata_path=output_dir / f"{site_id}.metadata", - site_id=site_id, - output_dir=output_dir, - ecmd_row=ecmd_row, - ) + if use_mp and not parallel: + logging.info( + "Multiprocessing requested but only one worker is useful " + f"({len(years)} year(s), max_processes={max_processes}); " + "running sequentially" + ) - logging.info(f"Created project file: {project_file}") + # Interleaved stdout from several concurrent EddyPro processes is unreadable, + # so streaming is suppressed when running in parallel. Output still reaches + # the log file when log_eddypro_output is enabled. + effective_stream_output = stream_output and not parallel + if parallel and stream_output: + logging.info( + "Output streaming disabled while running years in parallel; " + "EddyPro output is still captured in the log" + ) - # Preflight validation: check data_path and file availability - if not dry_run: + jobs = [ + { + "year": year, + "site_id": site_id, + "config": config, + "template_path": str(template_path), + "ini_parameters": ini_parameters, + "dry_run": dry_run, + "stream_output": effective_stream_output, + "metrics_interval": metrics_interval, + "log_eddypro_output": log_eddypro_output, + "monitoring_enabled": monitoring_enabled, + "configure_logging": parallel, + "log_level": getattr(args, "log_level", "INFO"), + } + for year in years + ] + + year_records: list[dict[str, Any]] = [] + + if parallel: + logging.info( + f"Processing {len(years)} years with {worker_count} parallel workers" + ) + results_by_year: dict[int, dict[str, Any]] = {} + with ProcessPoolExecutor(max_workers=worker_count) as executor: + futures = {executor.submit(process_year, job): job["year"] for job in jobs} + for future in as_completed(futures): + year = futures[future] try: - ini_tools.validate_eddypro_inputs(ini_config) - ini_tools.validate_eddypro_metadata(ini_config) - except ini_tools.INIParameterError: - logging.exception(f"Preflight validation failed for year {year}") - overall_success = False - continue - - except Exception: - logging.exception("Failed to create project file") - overall_success = False - continue - - # Execute EddyPro (or skip in dry-run mode) - if not dry_run: - success = core.run_eddypro_with_monitoring( - project_file=project_file, - eddypro_executable=eddypro_exe, - stream_output=stream_output, - metrics_interval=metrics_interval, - scenario_suffix="", - log_output=log_eddypro_output, - ) - - if not success: - logging.error(f"EddyPro processing failed for year {year}") - overall_success = False - else: - msg = f"EddyPro processing completed successfully for year {year}" - logging.info(msg) - years_processed.append(year) - else: - logging.info(f"Dry run: skipped EddyPro execution for year {year}") - years_processed.append(year) - - end_time = datetime.now() + results_by_year[year] = future.result() + except Exception as exc: + logging.exception(f"Worker for year {year} crashed") + results_by_year[year] = { + "year": year, + "status": "failed", + "duration_seconds": 0.0, + "error": f"worker crashed: {exc}", + "output_dir": str( + Path( + config["output_dir_pattern"].format( + year=year, site_id=site_id + ) + ) + ), + } + # Restore the requested order; completion order is nondeterministic. + year_records = [results_by_year[year] for year in years] + else: + year_records = [process_year(job) for job in jobs] + + years_processed = [ + r["year"] for r in year_records if r["status"] in ("success", "dry_run") + ] + run_errors = [f"{r['year']}: {r['error']}" for r in year_records if r.get("error")] + overall_success = all(r["status"] in ("success", "dry_run") for r in year_records) + end_time = datetime.now(timezone.utc) duration = (end_time - start_time).total_seconds() - # Generate reports - if years_processed: - try: - output_pattern = config["output_dir_pattern"] - first_year_dir = output_pattern.format(year=years[0], site_id=site_id) - output_base = Path(first_year_dir) - core.generate_run_report( - config=config, - site_id=site_id, - years_processed=years_processed, - output_base_dir=output_base, - start_time=start_time, - end_time=end_time, - overall_success=overall_success, - ) - logging.info("Reports generated successfully") - except Exception as e: - logging.warning(f"Failed to generate reports: {e}") + # Generate reports. This runs unconditionally: a run in which every year + # failed is exactly the run whose manifest matters most, and the old + # `if years_processed:` guard meant no record was written at all. + try: + output_pattern = config["output_dir_pattern"] + first_year_dir = output_pattern.format(year=years[0], site_id=site_id) + output_base = Path(first_year_dir) + core.generate_run_report( + config=config, + site_id=site_id, + years_processed=years_processed, + output_base_dir=output_base, + start_time=start_time, + end_time=end_time, + overall_success=overall_success, + year_records=year_records, + errors=run_errors, + ) + logging.info("Reports generated successfully") + except Exception as e: + logging.warning(f"Failed to generate reports: {e}") # Final summary logging.info(f"Processing completed in {duration:.1f}s") @@ -629,7 +838,21 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 eddypro_exe = Path(config["eddypro_executable"]) stream_output = config.get("stream_output", True) log_eddypro_output = config.get("log_eddypro_output", True) - metrics_interval = args.metrics_interval + # Honour config.yaml; the CLI flag only wins when explicitly supplied. + metrics_interval = ( + args.metrics_interval + if args.metrics_interval is not None + else config.get("metrics_interval_seconds", 0.5) + ) + monitoring_enabled = ( + args.monitor + if getattr(args, "monitor", None) is not None + else config.get("monitoring_enabled", True) + ) + if not monitoring_enabled: + logging.info( + "Performance monitoring disabled; no metrics files will be written" + ) if not site_id: logging.error("Site ID not provided via CLI or config") @@ -640,7 +863,7 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 return 1 # Process each year with all scenarios - start_time = datetime.now() + start_time = datetime.now(timezone.utc) all_scenario_results = [] for year in years: @@ -703,6 +926,7 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 ecmd_file=ecmd_file_path, dry_run=hasattr(args, "dry_run") and args.dry_run, log_output=log_eddypro_output, + monitoring_enabled=monitoring_enabled, ) # Collect results for reporting @@ -713,7 +937,7 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 failed = len(scenario_results) - successful logging.info(f"Year {year}: {successful} scenarios successful, {failed} failed") - end_time = datetime.now() + end_time = datetime.now(timezone.utc) duration = (end_time - start_time).total_seconds() logging.info(f"Scenario processing completed in {duration:.1f}s") @@ -746,24 +970,109 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 if output_dir_path.exists(): output_dirs.append(output_dir_path) - # Compute config checksum - config_checksum = str(hash(json.dumps(config, sort_keys=True))) + # Stable SHA256; str(hash(...)) changed on every process. + config_checksum = report.compute_config_checksum(config) - # Build scenario list for manifest + # Build scenario list for manifest. "scenario_name" is included so + # that `status` renders a name rather than "unknown". manifest_scenarios = [] for result in all_scenario_results: + suffix = result.get("scenario_suffix", "") manifest_scenarios.append( { + "scenario_name": result.get( + "scenario_name", f"scenario{suffix}" + ), + "year": result.get("year"), "scenario_index": result.get("scenario_index", 0), - "scenario_suffix": result.get("scenario_suffix", ""), + "scenario_suffix": suffix, "scenario_params": result.get("scenario_params", {}), "start_time": result.get("start_time", start_time.isoformat()), "end_time": result.get("end_time", end_time.isoformat()), "duration_seconds": result.get("duration_seconds", 0), "success": result.get("success", False), + "error": result.get("error"), + "output_dir": result.get("output_dir"), } ) + # Analyse each scenario's metrics and emit a per-scenario HTML report. + analyzer = analysis.BottleneckAnalyzer(config.get("performance_thresholds")) + analyses = [] + scenario_metrics: dict[str, list[dict[str, Any]]] = {} + for result in all_scenario_results: + out_dir = result.get("output_dir") + if not out_dir: + continue + out_path = Path(out_dir) + name = f"scenario{result.get('scenario_suffix', '')}" + metrics_files = sorted( + out_path.glob("metrics_*.csv"), + key=lambda f: f.stat().st_mtime, + ) + if not metrics_files: + continue + scenario_analyses = [ + analyzer.analyze(mf, scenario_name=f"{name}_{mf.stem}") + for mf in metrics_files + ] + analyses.extend(scenario_analyses) + for mf in metrics_files: + scenario_metrics[f"{name}_{mf.stem}"] = ( + report.load_metrics_from_csv(mf) + ) + + # Per-scenario report, promised by docs but never generated + # before: {output_dir}/reports/run_report.html + try: + sc_reports_dir = out_path / "reports" + sc_reports_dir.mkdir(parents=True, exist_ok=True) + sc_manifest = dict(result) + sc_manifest.update( + { + "run_id": f"{run_id}_{name}", + "site_id": site_id, + "scenarios": [ + s + for s in manifest_scenarios + if s["scenario_suffix"] + == result.get("scenario_suffix", "") + ], + "metrics_summary": { + "schema_version": 2, + "scenarios": [a.to_dict() for a in scenario_analyses], + "primary_bottleneck": ( + scenario_analyses[0].primary_bottleneck + if scenario_analyses + else "UNKNOWN" + ), + }, + } + ) + report.generate_html_report( + run_manifest=sc_manifest, + scenario_metrics={ + f"{name}_{mf.stem}": scenario_metrics[f"{name}_{mf.stem}"] + for mf in metrics_files + }, + chart_engine=config.get("report_charts", "plotly"), + output_path=sc_reports_dir / "run_report.html", + ) + except Exception as sc_err: + logging.warning(f"Failed to generate report for {name}: {sc_err}") + + metrics_summary = ( + { + "schema_version": 2, + "scenarios": [a.to_dict() for a in analyses], + "primary_bottleneck": analysis.dominant_bottleneck(analyses), + } + if analyses + else None + ) + + overall_success = all(r["success"] for r in all_scenario_results) + # Generate and write manifest manifest = report.generate_run_manifest( run_id=run_id, @@ -774,17 +1083,33 @@ def cmd_scenarios(args: argparse.Namespace) -> int: # noqa: PLR0911 scenarios=manifest_scenarios, start_time=start_time, end_time=end_time, - overall_success=all(r["success"] for r in all_scenario_results), + overall_success=overall_success, output_dirs=output_dirs, + provenance=report.get_provenance(config), + errors=[ + f"{s['scenario_name']}: {s['error']}" + for s in manifest_scenarios + if s.get("error") + ], + status="completed" if overall_success else "failed", + metrics_summary=metrics_summary, ) - # Add dry_run flag to config for manifest - manifest["dry_run"] = hasattr(args, "dry_run") and args.dry_run - # Write manifest manifest_path = reports_dir / "run_manifest.json" report.write_run_manifest(manifest, manifest_path) + # Aggregate comparison report across all scenarios + try: + report.generate_html_report( + run_manifest=manifest, + scenario_metrics=scenario_metrics or None, + chart_engine=config.get("report_charts", "plotly"), + output_path=reports_dir / "run_report.html", + ) + except Exception as agg_err: + logging.warning(f"Failed to generate aggregate report: {agg_err}") + logging.info("Reports generated successfully") except Exception: logging.exception("Failed to generate reports") @@ -923,6 +1248,23 @@ def cmd_status(args: argparse.Namespace) -> int: print(f"End Time: {end_time_str}") print(f"Duration: {duration:.1f} seconds") print(f"Mode: {'Dry Run' if dry_run else 'Production'}") + print(f"Status: {manifest.get('status', 'unknown')}") + + # Per-year outcome, including years that failed. `years_processed` lists only + # successes, so a failed year would otherwise be invisible in this output. + year_records = manifest.get("years", []) + if year_records: + print("\n" + "-" * 70) + print(f"{'Year':<8} {'Status':<12} {'Duration (s)':<15} {'Error'}") + print("-" * 70) + for rec in year_records: + err = rec.get("error") or "" + if len(err) > 30: + err = err[:27] + "..." + print( + f"{rec.get('year', '?'):<8} {rec.get('status', '?'):<12} " + f"{rec.get('duration_seconds', 0):<15.1f} {err}" + ) # Scenarios summary scenarios_data = manifest.get("scenarios", []) @@ -940,20 +1282,29 @@ def cmd_status(args: argparse.Namespace) -> int: print(f"{scenario_name:<25} {scenario_duration:<15.1f} {status:<10}") - # Metrics summary - metrics_summary = manifest.get("metrics_summary", {}) + # Performance summary and bottleneck verdict + metrics_summary = manifest.get("metrics_summary") or {} if metrics_summary: print("\n" + "-" * 70) - print("Performance Metrics") + print("Performance") print("-" * 70) - for key, value in metrics_summary.items(): - if isinstance(value, int | float): - print(f"{key}: {value:.2f}") - else: - print(f"{key}: {value}") + print(f"Primary bottleneck: {metrics_summary.get('primary_bottleneck')}") + for entry in metrics_summary.get("scenarios", []): + cpu = entry.get("cpu", {}) + print( + f" {entry.get('scenario_name', '?'):<18} " + f"{entry.get('primary_bottleneck', '?'):<18} " + f"CPU p95 {cpu.get('p95', 0):>6.1f}% " + f"peak RAM {entry.get('peak_memory_mb', 0):>7.0f} MB " + f"read {entry.get('total_read_mb', 0):>7.0f} MB " + f"write {entry.get('total_write_mb', 0):>7.0f} MB" + ) + if entry.get("explanation"): + print(f" {entry['explanation']}") - # Output paths - outputs = manifest.get("outputs", []) + # Output paths. The manifest key is "output_dirs"; reading "outputs" meant + # this section never rendered. + outputs = manifest.get("output_dirs", []) if outputs: print("\n" + "-" * 70) print("Output Directories") diff --git a/src/eddypro_batch_processor/core.py b/src/eddypro_batch_processor/core.py index 37b8d65..2cf68ab 100644 --- a/src/eddypro_batch_processor/core.py +++ b/src/eddypro_batch_processor/core.py @@ -7,17 +7,18 @@ import json import logging +import os import platform import shutil import subprocess import sys -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, NoReturn import yaml -from . import ecmd, ini_tools, report +from . import analysis, ecmd, ini_tools, report from .monitor import MonitoredOperation from .scenarios import Scenario @@ -137,27 +138,33 @@ def validate_config(self, config: dict[str, Any] | None = None) -> None: def run_subprocess_with_monitoring( - command: str, + command: list[str], working_dir: Path, stream_output: bool = True, metrics_interval: float = 0.5, output_dir: Path | None = None, scenario_suffix: str = "", log_output: bool = True, + monitoring_enabled: bool = True, ) -> int: """ Execute a subprocess command with performance monitoring. - This function runs the given command in a subprocess, optionally streams output, - and monitors performance metrics during execution. + The command is passed as an argv list and launched WITHOUT ``shell=True``. + That matters for more than quoting: under a shell, ``Popen.pid`` is the PID of + the intermediate ``cmd.exe`` wrapper rather than EddyPro itself, so the monitor + would sample an idle shell and report 0% CPU and no disk I/O for the whole run. Args: - command: The command line string to execute + command: The command as an argv list, e.g. ``[exe, "-s", "win", project]`` working_dir: Directory to execute the command in stream_output: Whether to stream output in real-time metrics_interval: Sampling interval for performance monitoring output_dir: Directory to write metrics files (defaults to working_dir) scenario_suffix: Suffix for metrics files in scenario runs + log_output: Whether to mirror subprocess output into the log + monitoring_enabled: When False, no monitor is started and no metrics files + are written Returns: Subprocess return code, or -1 if an exception occurs @@ -170,22 +177,19 @@ def run_subprocess_with_monitoring( interval_seconds=metrics_interval, output_dir=metrics_output_dir, scenario_suffix=scenario_suffix, + enabled=monitoring_enabled, ) as monitor: - # Start the subprocess - process = subprocess.Popen( # nosec B602 + process = subprocess.Popen( # nosec B603 command, - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=working_dir, ) - # Start monitoring the specific process if monitor is available + # Point the already-running monitor at the real EddyPro process tree. if monitor and process.pid: - # Stop current monitoring and restart with process PID - monitor.stop_monitoring() - monitor.start_monitoring(process_pid=process.pid) + monitor.attach_process(process.pid) # Handle output streaming if stream_output and process.stdout: @@ -205,7 +209,7 @@ def run_subprocess_with_monitoring( return return_code except Exception: - logging.exception(f"Failed to execute command '{command}'") + logging.exception(f"Failed to execute command {command!r}") return -1 @@ -216,6 +220,7 @@ def run_eddypro_with_monitoring( metrics_interval: float = 0.5, scenario_suffix: str = "", log_output: bool = True, + monitoring_enabled: bool = True, ) -> bool: """ Run EddyPro processing with performance monitoring. @@ -226,6 +231,8 @@ def run_eddypro_with_monitoring( stream_output: Whether to stream subprocess output metrics_interval: Performance monitoring sampling interval scenario_suffix: Suffix for scenario-specific metrics files + log_output: Whether to mirror EddyPro output into the log + monitoring_enabled: When False, no performance metrics are collected Returns: True if both eddypro_rp and eddypro_fcc succeed, False otherwise @@ -266,9 +273,8 @@ def run_eddypro_with_monitoring( # Construct commands os_suffix = "win" if platform.system() == "Windows" else "linux" - command_sys = f" -s {os_suffix} " - rp_command = f'"{rp_executable}"{command_sys}"{project_file}"' - fcc_command = f'"{fcc_executable}"{command_sys}"{project_file}"' + rp_command = [str(rp_executable), "-s", os_suffix, str(project_file)] + fcc_command = [str(fcc_executable), "-s", os_suffix, str(project_file)] success = True @@ -282,6 +288,7 @@ def run_eddypro_with_monitoring( output_dir=output_dir, scenario_suffix=f"{scenario_suffix}_rp" if scenario_suffix else "rp", log_output=log_output, + monitoring_enabled=monitoring_enabled, ) if rp_return_code != 0: logging.error(f"eddypro_rp failed with return code {rp_return_code}") @@ -298,6 +305,7 @@ def run_eddypro_with_monitoring( output_dir=output_dir, scenario_suffix=f"{scenario_suffix}_fcc" if scenario_suffix else "fcc", log_output=log_output, + monitoring_enabled=monitoring_enabled, ) if fcc_return_code != 0: logging.error(f"eddypro_fcc failed with return code {fcc_return_code}") @@ -342,6 +350,8 @@ def generate_run_report( start_time: datetime, end_time: datetime, overall_success: bool = True, + year_records: list[dict[str, Any]] | None = None, + errors: list[str] | None = None, ) -> None: """ Generate run manifest and HTML report after processing completes. @@ -349,11 +359,13 @@ def generate_run_report( Args: config: Configuration dictionary site_id: Site identifier - years_processed: List of years that were processed + years_processed: Years that completed successfully output_base_dir: Base output directory (parent of year-specific dirs) start_time: Processing start time (datetime) end_time: Processing end time (datetime) overall_success: Whether all processing succeeded + year_records: Per-year status records, including failed years + errors: Run-level error messages """ # Determine reports directory reports_dir_config = config.get("reports_dir") @@ -377,8 +389,8 @@ def generate_run_report( if year_dir.exists(): output_dirs.append(year_dir) - # Compute config checksum (simple hash of sorted config JSON) - config_checksum = str(hash(json.dumps(config, sort_keys=True))) + # Stable SHA256 -- the previous str(hash(...)) changed on every process. + config_checksum = report.compute_config_checksum(config) # Collect scenarios (single baseline scenario for now) scenario_list = [ @@ -392,18 +404,33 @@ def generate_run_report( } ] - # Load metrics if available + # Load metrics and analyse the bottleneck for each year that produced them. scenario_metrics = {} + analyses: list[analysis.ScenarioAnalysis] = [] + analyzer = analysis.BottleneckAnalyzer(config.get("performance_thresholds")) for year in years_processed: year_dir = Path( config.get("output_dir_pattern", "").format(year=year, site_id=site_id) ) - metrics_files = list(year_dir.glob("metrics_*.csv")) - if metrics_files: - # Load the most recent metrics file - metrics_file = sorted(metrics_files)[-1] - metrics = report.load_metrics_from_csv(metrics_file) - scenario_metrics[f"{year}_baseline"] = metrics + # Sort by modification time: lexicographic ordering put "_rp" after + # "_fcc" regardless of which actually ran last. + metrics_files = sorted( + year_dir.glob("metrics_*.csv"), key=lambda f: f.stat().st_mtime + ) + for metrics_file in metrics_files: + label = f"{year}_{metrics_file.stem.replace('metrics_', '')}" + scenario_metrics[label] = report.load_metrics_from_csv(metrics_file) + analyses.append(analyzer.analyze(metrics_file, scenario_name=label)) + + metrics_summary = ( + { + "schema_version": 2, + "scenarios": [a.to_dict() for a in analyses], + "primary_bottleneck": analysis.dominant_bottleneck(analyses), + } + if analyses + else None + ) # Generate run manifest run_manifest = report.generate_run_manifest( @@ -417,6 +444,11 @@ def generate_run_report( end_time=end_time, overall_success=overall_success, output_dirs=output_dirs, + provenance=report.get_provenance(config), + years=year_records, + errors=errors, + status="completed" if overall_success else "failed", + metrics_summary=metrics_summary, ) # Write run manifest @@ -437,6 +469,21 @@ def generate_run_report( logging.info(f"Run manifest generated: {manifest_path}") +def _write_scenario_manifest( + scenario_output_dir: Path, suffix: str, metadata: dict[str, Any] +) -> None: + """Write a scenario manifest atomically, tolerating a failed write.""" + manifest_path = scenario_output_dir / f"scenario_manifest{suffix}.json" + try: + scenario_output_dir.mkdir(parents=True, exist_ok=True) + tmp_path = manifest_path.with_suffix(".json.tmp") + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2) + os.replace(tmp_path, manifest_path) + except Exception: + logging.exception(f"Failed to write scenario manifest to {manifest_path}") + + def run_single_scenario( scenario: Scenario, template_path: Path, @@ -451,6 +498,7 @@ def run_single_scenario( input_dir: Path, ecmd_file: Path | None = None, dry_run: bool = False, + monitoring_enabled: bool = True, ) -> dict[str, Any]: """ Execute a single scenario with patched parameters. @@ -467,7 +515,7 @@ def run_single_scenario( Returns: Dictionary containing scenario execution metadata """ - start_time = datetime.now() + start_time = datetime.now(timezone.utc) # Create scenario-specific output directory scenario_output_dir = output_base_dir / f"scenario{scenario.suffix}" @@ -601,6 +649,7 @@ def run_single_scenario( metrics_interval=metrics_interval, scenario_suffix=scenario.suffix, log_output=log_output, + monitoring_enabled=monitoring_enabled, ) return_code = 0 if success else 1 else: @@ -608,11 +657,14 @@ def run_single_scenario( f"Scenario {scenario.index}: Dry run mode - skipping execution" ) - end_time = datetime.now() + end_time = datetime.now(timezone.utc) duration = (end_time - start_time).total_seconds() - # Build scenario metadata + # Build scenario metadata. "scenario_name" is emitted here and by the + # regular `run` path so that `status` can read one consistent shape. metadata = { + "scenario_name": f"{year}_scenario{scenario.suffix}", + "year": year, "scenario_index": scenario.index, "scenario_suffix": scenario.suffix, "scenario_params": scenario.parameters, @@ -623,13 +675,11 @@ def run_single_scenario( "duration_seconds": duration, "success": success, "return_code": return_code, + "error": None, "dry_run": dry_run, } - # Write scenario manifest - manifest_path = scenario_output_dir / f"scenario_manifest{scenario.suffix}.json" - with open(manifest_path, "w", encoding="utf-8") as f: - json.dump(metadata, f, indent=2) + _write_scenario_manifest(scenario_output_dir, scenario.suffix, metadata) logging.info( f"Scenario {scenario.index} {'completed' if success else 'failed'} " @@ -637,12 +687,14 @@ def run_single_scenario( ) except Exception as e: - end_time = datetime.now() + end_time = datetime.now(timezone.utc) duration = (end_time - start_time).total_seconds() logging.exception(f"Scenario {scenario.index} failed with exception") metadata = { + "scenario_name": f"{year}_scenario{scenario.suffix}", + "year": year, "scenario_index": scenario.index, "scenario_suffix": scenario.suffix, "scenario_params": scenario.parameters, @@ -659,6 +711,10 @@ def run_single_scenario( "dry_run": dry_run, } + # A failing scenario must still leave a manifest behind; previously this + # metadata was built and then silently discarded. + _write_scenario_manifest(scenario_output_dir, scenario.suffix, metadata) + return metadata @@ -676,6 +732,7 @@ def run_scenario_batch( input_dir: Path, ecmd_file: Path | None = None, dry_run: bool = False, + monitoring_enabled: bool = True, ) -> list[dict[str, Any]]: """ Execute a batch of scenarios sequentially. @@ -709,6 +766,7 @@ def run_scenario_batch( input_dir=input_dir, ecmd_file=ecmd_file, dry_run=dry_run, + monitoring_enabled=monitoring_enabled, ) scenario_results.append(result) diff --git a/src/eddypro_batch_processor/ecmd.py b/src/eddypro_batch_processor/ecmd.py index 2e68953..aa0a391 100644 --- a/src/eddypro_batch_processor/ecmd.py +++ b/src/eddypro_batch_processor/ecmd.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ ECMD utilities for dynamic metadata generation. diff --git a/src/eddypro_batch_processor/ini_tools.py b/src/eddypro_batch_processor/ini_tools.py index e53af25..533709b 100644 --- a/src/eddypro_batch_processor/ini_tools.py +++ b/src/eddypro_batch_processor/ini_tools.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ INI Tools for EddyPro Batch Processor. @@ -797,34 +796,3 @@ def get_parameter_info() -> dict[str, dict[str, Any]]: Dictionary with parameter information """ return PARAMETER_VALIDATION.copy() - - -def generate_scenario_suffix(parameters: dict[str, int]) -> str: - """ - Generate a deterministic suffix for scenario identification. - - Args: - parameters: Dictionary of parameter name -> value pairs - - Returns: - String suffix like "_rot1_tlag2_det0_spk0" - """ - if not parameters: - return "" - - # Create suffix in consistent order - suffix_parts = [] - for param_name in sorted(parameters.keys()): - value = parameters[param_name] - # Use short names for suffix - short_names = { - "rot_meth": "rot", - "tlag_meth": "tlag", - "detrend_meth": "det", - "despike_meth": "spk", - "hf_meth": "hf", - } - short_name = short_names.get(param_name, param_name) - suffix_parts.append(f"{short_name}{value}") - - return "_" + "_".join(suffix_parts) diff --git a/src/eddypro_batch_processor/monitor.py b/src/eddypro_batch_processor/monitor.py index 7e00d20..6c8c149 100644 --- a/src/eddypro_batch_processor/monitor.py +++ b/src/eddypro_batch_processor/monitor.py @@ -3,6 +3,19 @@ This module provides performance monitoring capabilities using psutil to track CPU, memory, and I/O metrics during EddyPro subprocess execution. + +Design notes +------------ +The monitored unit is a **process tree**, not a single process. EddyPro is launched +as a child process and may itself spawn workers, so every sample walks +``root.children(recursive=True)`` and aggregates across the whole tree. Sampling a +single PID was the historical cause of all-zero metrics. + +Disk I/O is exposed two ways: cumulative totals since monitoring started +(``read_mb`` / ``write_mb``) and instantaneous rates derived from the delta between +consecutive samples divided by the *actual* elapsed wall time (``read_mb_per_s`` / +``write_mb_per_s``). Raw psutil counters are monotonic since boot and are never +reported directly, because aggregate statistics over them are meaningless. """ import csv @@ -24,14 +37,41 @@ logger = logging.getLogger(__name__) +#: Bumped whenever the CSV column set or summary JSON structure changes. +METRICS_SCHEMA_VERSION = 2 + +#: Canonical column order for ``metrics.csv``. The first six columns are the +#: contract consumed by :mod:`eddypro_batch_processor.report`. +METRICS_FIELDNAMES = [ + "timestamp", + "relative_time", + "cpu_percent", + "cpu_percent_of_core", + "memory_mb", + "read_mb", + "write_mb", + "read_mb_per_s", + "write_mb_per_s", + "read_iops", + "write_iops", + "num_processes", + "system_cpu_percent", + "system_memory_percent", + "system_memory_used_mb", + "system_read_mb_per_s", + "system_write_mb_per_s", +] + +_BYTES_PER_MB = 1024.0 * 1024.0 + class PerformanceMonitor: """ - Monitor system and process performance metrics during operations. + Monitor system and process-tree performance metrics during operations. - Tracks CPU utilization, memory usage (RSS/peak), disk I/O, and wall-clock time - with configurable sampling intervals. Produces both time series (CSV) and - summary (JSON) outputs. + Tracks CPU utilization, memory usage, and disk I/O (both cumulative and as + rates) with a configurable sampling interval. Produces a time series (CSV) and + a summary (JSON). """ def __init__( @@ -61,16 +101,33 @@ def __init__( self.output_dir = Path(output_dir) if output_dir else Path.cwd() self.scenario_suffix = scenario_suffix + # Number of logical CPUs, used to normalise process CPU onto a 0-100 scale + # so it is directly comparable with psutil.cpu_percent(). + self._cpu_count = psutil.cpu_count() or 1 + # Monitoring state self._monitoring = False self._monitor_thread: threading.Thread | None = None self._start_time: float | None = None self._end_time: float | None = None - # Data storage + # Data storage. _samples is touched by both the sampler thread and the + # writer on the main thread, so it is guarded by _lock. self._samples: list[dict[str, Any]] = [] + self._lock = threading.Lock() self._process: psutil.Process | None = None + # Per-PID cumulative I/O counters, retained after a child exits so the + # tree total never goes backwards when a worker finishes. + self._io_by_pid: dict[int, tuple[float, float, float, float]] = {} + self._io_baseline: tuple[float, float, float, float] | None = None + self._primed_pids: set[int] = set() + + # Previous sample state, for delta-based rate computation + self._prev_time: float | None = None + self._prev_io: tuple[float, float, float, float] | None = None + self._prev_system_io: tuple[float, float] | None = None + # Output file paths self._metrics_csv_path = self._get_output_path("metrics.csv") self._summary_json_path = self._get_output_path("metrics_summary.json") @@ -82,12 +139,18 @@ def _get_output_path(self, filename: str) -> Path: filename = f"{name}_{self.scenario_suffix}.{ext}" return self.output_dir / filename + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + def start_monitoring(self, process_pid: int | None = None) -> None: """ Start performance monitoring. Args: - process_pid: PID of specific process to monitor. If None, monitors system. + process_pid: PID of the root process to monitor, including all of its + descendants. If None, only system-wide metrics are recorded until + :meth:`attach_process` is called. """ if self._monitoring: logger.warning("Monitoring already active") @@ -99,19 +162,22 @@ def start_monitoring(self, process_pid: int | None = None) -> None: self._monitoring = True self._start_time = time.time() - self._samples.clear() + with self._lock: + self._samples.clear() + self._io_by_pid.clear() + self._io_baseline = None + self._primed_pids.clear() + self._prev_time = None + self._prev_io = None + self._prev_system_io = None + + # Prime the system-wide CPU counter. The first call after import always + # returns 0.0 because there is no previous measurement to diff against. + psutil.cpu_percent(interval=None) - # Set up process monitoring if PID provided if process_pid: - try: - self._process = psutil.Process(process_pid) - except Exception: - logger.warning( - f"Process {process_pid} not found, monitoring system instead" - ) - self._process = None + self.attach_process(process_pid) - # Start monitoring thread self._monitor_thread = threading.Thread( target=self._monitor_loop, daemon=True, name="PerformanceMonitor" ) @@ -122,6 +188,43 @@ def start_monitoring(self, process_pid: int | None = None) -> None: f"process: {process_pid or 'system'})" ) + def attach_process(self, process_pid: int) -> bool: + """ + Attach to a process tree after monitoring has already started. + + This lets the caller spawn the subprocess first and then point the running + monitor at it, without the stop/restart cycle that used to truncate the + metrics files. + + Args: + process_pid: PID of the root process to monitor. + + Returns: + True if the process was found and attached. + """ + try: + self._process = psutil.Process(process_pid) + except Exception: + logger.warning( + f"Process {process_pid} not found, monitoring system-wide only" + ) + self._process = None + return False + + # Prime per-process CPU so the first real sample is not 0.0. + for proc in self._iter_tracked(): + try: + proc.cpu_percent(None) + self._primed_pids.add(proc.pid) + # nosec B112 - a process that vanishes or denies access between the + # tree walk and this call must be skipped, not allowed to abort + # priming for the rest of the tree. + except Exception: # noqa: PERF203 # nosec B112 + continue + + logger.debug(f"Attached performance monitor to PID {process_pid}") + return True + def stop_monitoring(self) -> dict[str, Any]: """ Stop performance monitoring and return summary. @@ -136,203 +239,318 @@ def stop_monitoring(self) -> dict[str, Any]: self._monitoring = False self._end_time = time.time() - # Wait for monitor thread to finish + # Wait for the sampler thread. The timeout must exceed one full sampling + # period or a slow interval would race the CSV writer. if self._monitor_thread and self._monitor_thread.is_alive(): - self._monitor_thread.join(timeout=2.0) + self._monitor_thread.join(timeout=max(5.0, self.interval_seconds * 2)) - # Generate summary summary = self._generate_summary() - # Write outputs self._write_metrics_csv() self._write_summary_json(summary) + duration = summary.get("timing", {}).get("duration_seconds", 0.0) logger.info( f"Stopped performance monitoring. " - f"Duration: {summary.get('duration_seconds', 0):.2f}s, " - f"Samples: {len(self._samples)}" + f"Duration: {duration:.2f}s, Samples: {len(self._samples)}" ) return summary + # ------------------------------------------------------------------ + # Sampling + # ------------------------------------------------------------------ + def _monitor_loop(self) -> None: """Main monitoring loop running in background thread.""" while self._monitoring: try: sample = self._collect_sample() if sample: - self._samples.append(sample) + with self._lock: + self._samples.append(sample) except Exception as e: logger.warning(f"Error collecting performance sample: {e}") time.sleep(self.interval_seconds) + def _iter_tracked(self) -> list[Any]: + """ + Return the root process plus every living descendant. + + Re-walked on every sample so workers spawned mid-run are picked up. A + vanished root yields an empty list but never clears ``self._process`` -- + transient errors must not permanently disable process monitoring. + """ + if not self._process: + return [] + try: + return [self._process, *self._process.children(recursive=True)] + except Exception: + return [] + def _collect_sample(self) -> dict[str, Any] | None: """ Collect a single performance sample. Returns: - Dictionary with timestamp and performance metrics, or None on error + Dictionary of canonical metrics, or None on error. """ try: timestamp = time.time() - sample = { + elapsed = ( + timestamp - self._prev_time if self._prev_time is not None else None + ) + + sample: dict[str, Any] = { "timestamp": timestamp, "relative_time": timestamp - (self._start_time or timestamp), } + sample.update(self._collect_system_metrics(elapsed)) + sample.update(self._collect_process_metrics(elapsed)) - # System-wide metrics - sample.update(self._collect_system_metrics()) - - # Process-specific metrics if available - if self._process: - process_metrics = self._collect_process_metrics() - if process_metrics: - sample.update(process_metrics) - + self._prev_time = timestamp except Exception as e: logger.debug(f"Failed to collect sample: {e}") return None else: return sample - def _collect_system_metrics(self) -> dict[str, Any]: - """Collect system-wide performance metrics.""" - metrics = {} + def _collect_system_metrics(self, elapsed: float | None) -> dict[str, Any]: + """Collect system-wide metrics, converting disk counters into rates.""" + metrics: dict[str, Any] = { + "system_cpu_percent": 0.0, + "system_memory_percent": 0.0, + "system_memory_used_mb": 0.0, + "system_read_mb_per_s": 0.0, + "system_write_mb_per_s": 0.0, + } try: - # CPU utilization - cpu_percent = psutil.cpu_percent(interval=0) - metrics["system_cpu_percent"] = cpu_percent + metrics["system_cpu_percent"] = psutil.cpu_percent(interval=None) - # Memory usage memory = psutil.virtual_memory() - metrics["system_memory_total"] = memory.total - metrics["system_memory_available"] = memory.available metrics["system_memory_percent"] = memory.percent + metrics["system_memory_used_mb"] = ( + memory.total - memory.available + ) / _BYTES_PER_MB - # Disk I/O disk_io = psutil.disk_io_counters() if disk_io: - metrics["system_disk_read_bytes"] = disk_io.read_bytes - metrics["system_disk_write_bytes"] = disk_io.write_bytes - metrics["system_disk_read_count"] = disk_io.read_count - metrics["system_disk_write_count"] = disk_io.write_count - - # Network I/O (optional) - try: - network_io = psutil.net_io_counters() - if network_io: - metrics["system_network_bytes_sent"] = network_io.bytes_sent - metrics["system_network_bytes_recv"] = network_io.bytes_recv - except AttributeError: - pass # Network counters not available on all systems + current = (float(disk_io.read_bytes), float(disk_io.write_bytes)) + if self._prev_system_io is not None and elapsed and elapsed > 0: + metrics["system_read_mb_per_s"] = max( + 0.0, (current[0] - self._prev_system_io[0]) + ) / (_BYTES_PER_MB * elapsed) + metrics["system_write_mb_per_s"] = max( + 0.0, (current[1] - self._prev_system_io[1]) + ) / (_BYTES_PER_MB * elapsed) + self._prev_system_io = current except Exception as e: logger.debug(f"Error collecting system metrics: {e}") return metrics - def _collect_process_metrics(self) -> dict[str, Any] | None: - """Collect process-specific performance metrics.""" - if not self._process: - return None + def _collect_process_metrics(self, elapsed: float | None) -> dict[str, Any]: + """ + Aggregate CPU, memory, and I/O across the monitored process tree. - try: - # Check if process still exists - if not self._process.is_running(): - logger.debug("Monitored process no longer running") - self._process = None - return None + Every psutil call is guarded per-process: a child that exits between the + tree walk and the read is skipped, and an ``AccessDenied`` on one process + never aborts the sample or disables monitoring. + """ + metrics: dict[str, Any] = { + "cpu_percent": 0.0, + "cpu_percent_of_core": 0.0, + "memory_mb": 0.0, + "read_mb": 0.0, + "write_mb": 0.0, + "read_mb_per_s": 0.0, + "write_mb_per_s": 0.0, + "read_iops": 0.0, + "write_iops": 0.0, + "num_processes": 0, + } - metrics = {} + tracked = self._iter_tracked() + if not tracked: + # The tree has exited. Cumulative totals must hold their final value + # rather than snapping back to zero -- read_mb/write_mb are + # monotonic by contract, and consumers chart them as such. + self._fill_cumulative_io(metrics) + return metrics - # CPU usage - try: - cpu_percent = self._process.cpu_percent() - metrics["process_cpu_percent"] = cpu_percent - except Exception: # nosec B110 - pass + total_cpu = 0.0 + total_rss = 0.0 + alive = 0 - # Memory usage + for proc in tracked: try: - memory_info = self._process.memory_info() - metrics["process_memory_rss"] = memory_info.rss - metrics["process_memory_vms"] = memory_info.vms + with proc.oneshot(): + # A process seen for the first time must be primed; its first + # cpu_percent() reading is meaningless and is discarded. + if proc.pid not in self._primed_pids: + proc.cpu_percent(None) + self._primed_pids.add(proc.pid) + else: + total_cpu += proc.cpu_percent(None) + + total_rss += float(proc.memory_info().rss) + alive += 1 + + try: + io = proc.io_counters() + except Exception: + io = None + + if io is not None: + # Retain the last known counters per PID so a finished child + # keeps contributing to the tree total. + self._io_by_pid[proc.pid] = ( + float(io.read_bytes), + float(io.write_bytes), + float(io.read_count), + float(io.write_count), + ) + # nosec B112 - NoSuchProcess / AccessDenied / ZombieProcess: skip + # this process only. self._process is deliberately left intact, + # because nulling it on the first transient error is what used to + # silently disable process monitoring for the rest of the run. + except Exception: # noqa: PERF203 # nosec B112 + continue - # Memory percent - memory_percent = self._process.memory_percent() - metrics["process_memory_percent"] = memory_percent - except Exception: # nosec B110 - pass + metrics["num_processes"] = alive + # Normalise onto 0-100 so this column is comparable with system_cpu_percent. + metrics["cpu_percent"] = round(total_cpu / self._cpu_count, 3) + # Un-normalised: 100 means "one core fully busy", 200 means two, and so on. + # EddyPro is largely single-threaded, so this is the column that reveals a + # saturated single core on a many-core machine. + metrics["cpu_percent_of_core"] = round(total_cpu, 3) + metrics["memory_mb"] = round(total_rss / _BYTES_PER_MB, 3) + + totals = self._fill_cumulative_io(metrics) + if totals is not None: + if self._prev_io is not None and elapsed and elapsed > 0: + metrics["read_mb_per_s"] = round( + max(0.0, totals[0] - self._prev_io[0]) / (_BYTES_PER_MB * elapsed), + 3, + ) + metrics["write_mb_per_s"] = round( + max(0.0, totals[1] - self._prev_io[1]) / (_BYTES_PER_MB * elapsed), + 3, + ) + metrics["read_iops"] = round( + max(0.0, totals[2] - self._prev_io[2]) / elapsed, 2 + ) + metrics["write_iops"] = round( + max(0.0, totals[3] - self._prev_io[3]) / elapsed, 2 + ) + self._prev_io = totals - # I/O counters - try: - io_counters = self._process.io_counters() - metrics["process_io_read_bytes"] = io_counters.read_bytes - metrics["process_io_write_bytes"] = io_counters.write_bytes - metrics["process_io_read_count"] = io_counters.read_count - metrics["process_io_write_count"] = io_counters.write_count - except Exception: # nosec B110 - pass # I/O counters not available on all platforms + return metrics - except Exception: - logger.debug("Error collecting process metrics") - self._process = None + def _fill_cumulative_io( + self, metrics: dict[str, Any] + ) -> tuple[float, float, float, float] | None: + """ + Set read_mb/write_mb from retained per-PID counters. + + Returns the raw aggregate totals so the caller can derive rates, or None + if no I/O counters have ever been observed. + """ + totals = self._aggregate_io() + if totals is None: return None - else: - return metrics + if self._io_baseline is None: + self._io_baseline = totals + base = self._io_baseline + metrics["read_mb"] = round(max(0.0, totals[0] - base[0]) / _BYTES_PER_MB, 3) + metrics["write_mb"] = round(max(0.0, totals[1] - base[1]) / _BYTES_PER_MB, 3) + return totals + + def _aggregate_io(self) -> tuple[float, float, float, float] | None: + """Sum retained per-PID I/O counters across the whole tree.""" + if not self._io_by_pid: + return None + read = write = rcount = wcount = 0.0 + for r, w, rc, wc in self._io_by_pid.values(): + read += r + write += w + rcount += rc + wcount += wc + return (read, write, rcount, wcount) + + # ------------------------------------------------------------------ + # Summary and output + # ------------------------------------------------------------------ def _generate_summary(self) -> dict[str, Any]: """Generate summary statistics from collected samples.""" - if not self._samples: - return {"error": "No samples collected"} + with self._lock: + samples = list(self._samples) + + duration = (self._end_time or 0) - (self._start_time or 0) + + if not samples: + return { + "schema_version": METRICS_SCHEMA_VERSION, + "error": "No samples collected", + "timing": { + "start_time": self._start_time, + "end_time": self._end_time, + "duration_seconds": duration, + }, + } - summary = { + summary: dict[str, Any] = { + "schema_version": METRICS_SCHEMA_VERSION, "monitoring_config": { "interval_seconds": self.interval_seconds, "scenario_suffix": self.scenario_suffix, "output_dir": str(self.output_dir), + "cpu_count": self._cpu_count, }, "timing": { "start_time": self._start_time, "end_time": self._end_time, - "duration_seconds": (self._end_time or 0) - (self._start_time or 0), + "duration_seconds": duration, }, "samples": { - "count": len(self._samples), - "first_timestamp": self._samples[0]["timestamp"], - "last_timestamp": self._samples[-1]["timestamp"], + "count": len(samples), + "first_timestamp": samples[0]["timestamp"], + "last_timestamp": samples[-1]["timestamp"], }, "metrics": {}, } - # Calculate statistics for each numeric metric - numeric_fields = self._get_numeric_fields() metrics_dict: dict[str, dict[str, float]] = {} - for field in numeric_fields: - values = [ - s[field] for s in self._samples if field in s and s[field] is not None - ] + for field in self._get_numeric_fields(samples): + values = [s[field] for s in samples if field in s and s[field] is not None] if values: metrics_dict[field] = self._calculate_stats(values) summary["metrics"] = metrics_dict + # Totals are the tail of the cumulative series, not an average. + summary["totals"] = { + "read_mb": samples[-1].get("read_mb", 0.0), + "write_mb": samples[-1].get("write_mb", 0.0), + "peak_memory_mb": max(s.get("memory_mb", 0.0) for s in samples), + } + return summary - def _get_numeric_fields(self) -> list[str]: + def _get_numeric_fields(self, samples: list[dict[str, Any]]) -> list[str]: """Get list of numeric field names from samples.""" - if not self._samples: + if not samples: return [] - numeric_fields = [] - for key, value in self._samples[0].items(): - if key in ["timestamp", "relative_time"]: - continue - if isinstance(value, int | float): - numeric_fields.append(key) - - return numeric_fields + return [ + key + for key, value in samples[0].items() + if key not in ("timestamp", "relative_time") + and isinstance(value, int | float) + ] def _calculate_stats(self, values: list[int | float]) -> dict[str, float]: """Calculate min, max, mean, and percentiles for a list of values.""" @@ -349,7 +567,6 @@ def _calculate_stats(self, values: list[int | float]) -> dict[str, float]: "count": n, } - # Percentiles if n >= 2: stats["p50"] = self._percentile(values, 0.5) stats["p90"] = self._percentile(values, 0.9) @@ -365,40 +582,34 @@ def _percentile(self, values: list[int | float], p: float) -> float: index = p * (len(values) - 1) if index.is_integer(): return float(values[int(index)]) - else: - lower = int(index) - upper = lower + 1 - weight = index - lower - return float(values[lower] * (1 - weight) + values[upper] * weight) + lower = int(index) + upper = lower + 1 + weight = index - lower + return float(values[lower] * (1 - weight) + values[upper] * weight) def _write_metrics_csv(self) -> None: - """Write time series metrics to CSV file.""" - if not self._samples: + """Write time series metrics to CSV using the canonical column order.""" + with self._lock: + samples = list(self._samples) + + if not samples: logger.warning("No samples to write to CSV") return try: - # Ensure output directory exists self.output_dir.mkdir(parents=True, exist_ok=True) - # Get all possible field names - all_fields: set[str] = set() - for sample in self._samples: - all_fields.update(sample.keys()) - - # Sort fields for consistent output - fieldnames = sorted(all_fields) - with open( self._metrics_csv_path, "w", newline="", encoding="utf-8" ) as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer = csv.DictWriter( + csvfile, fieldnames=METRICS_FIELDNAMES, extrasaction="ignore" + ) writer.writeheader() - writer.writerows(self._samples) + for sample in samples: + writer.writerow({k: sample.get(k, 0) for k in METRICS_FIELDNAMES}) - logger.info( - f"Wrote {len(self._samples)} samples to {self._metrics_csv_path}" - ) + logger.info(f"Wrote {len(samples)} samples to {self._metrics_csv_path}") except Exception: logger.exception("Failed to write metrics CSV") @@ -406,7 +617,6 @@ def _write_metrics_csv(self) -> None: def _write_summary_json(self, summary: dict[str, Any]) -> None: """Write summary statistics to JSON file.""" try: - # Ensure output directory exists self.output_dir.mkdir(parents=True, exist_ok=True) with open(self._summary_json_path, "w", encoding="utf-8") as jsonfile: @@ -417,6 +627,10 @@ def _write_summary_json(self, summary: dict[str, Any]) -> None: except Exception: logger.exception("Failed to write summary JSON") + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + @property def metrics_csv_path(self) -> Path: """Path to the metrics CSV file.""" @@ -435,7 +649,8 @@ def is_monitoring(self) -> bool: @property def sample_count(self) -> int: """Number of samples collected so far.""" - return len(self._samples) + with self._lock: + return len(self._samples) def create_monitor( @@ -472,15 +687,18 @@ def create_monitor( return None -# Context manager for convenient monitoring class MonitoredOperation: """ Context manager for monitoring operations. + When ``enabled`` is False the context manager is inert: no monitor is created + and no metrics files are written. + Example: with MonitoredOperation(output_dir="./metrics") as monitor: - # ... run expensive operation ... - pass + proc = subprocess.Popen(...) + monitor.attach_process(proc.pid) + proc.wait() # Metrics are automatically saved """ @@ -490,9 +708,15 @@ def __init__( output_dir: str | Path | None = None, scenario_suffix: str = "", process_pid: int | None = None, + enabled: bool = True, ): """Initialize monitored operation context.""" - self.monitor = create_monitor(interval_seconds, output_dir, scenario_suffix) + self.enabled = enabled + self.monitor = ( + create_monitor(interval_seconds, output_dir, scenario_suffix) + if enabled + else None + ) self.process_pid = process_pid self.summary: dict[str, Any] = {} diff --git a/src/eddypro_batch_processor/report.py b/src/eddypro_batch_processor/report.py index 74ef17f..2d251a2 100644 --- a/src/eddypro_batch_processor/report.py +++ b/src/eddypro_batch_processor/report.py @@ -9,7 +9,9 @@ import hashlib import json import logging +import os import platform +import subprocess import sys from datetime import datetime from pathlib import Path @@ -35,6 +37,95 @@ logger.debug("Plotly not available; charts will fall back to SVG or none") +#: Bumped when the run manifest structure changes in a way consumers must notice. +MANIFEST_SCHEMA_VERSION = 2 + + +def compute_config_checksum(config: dict[str, Any]) -> str: + """ + Compute a stable SHA256 checksum of a configuration mapping. + + Replaces the previous ``str(hash(json.dumps(...)))``, which used Python's + built-in ``hash()``. That is salted per-process by ``PYTHONHASHSEED``, so the + same configuration produced a different checksum on every run, making the + field useless for comparing runs. + """ + canonical = json.dumps(config, sort_keys=True, default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def get_git_provenance(repo_root: Path | None = None) -> dict[str, Any]: + """ + Capture the git commit the code was run from. + + Degrades gracefully: outside a git repository, or without git installed, the + fields are simply reported as unavailable rather than raising. + """ + cwd = repo_root or Path(__file__).resolve().parent + info: dict[str, Any] = {"git_sha": None, "git_dirty": None, "git_branch": None} + try: + info["git_sha"] = subprocess.check_output( # nosec B603 B607 + ["git", "rev-parse", "HEAD"], + cwd=cwd, + text=True, + stderr=subprocess.DEVNULL, + ).strip() + info["git_branch"] = subprocess.check_output( # nosec B603 B607 + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=cwd, + text=True, + stderr=subprocess.DEVNULL, + ).strip() + status = subprocess.check_output( # nosec B603 B607 + ["git", "status", "--porcelain"], + cwd=cwd, + text=True, + stderr=subprocess.DEVNULL, + ) + info["git_dirty"] = bool(status.strip()) + except Exception: + logger.debug("Git provenance unavailable") + return info + + +def get_provenance(config: dict[str, Any]) -> dict[str, Any]: + """ + Assemble the reproducibility block recorded in the run manifest. + + Captures the tool version, git state, the invoking command line, and the + EddyPro executable actually used (path plus checksum, so a silent upgrade of + the binary is detectable after the fact). + """ + # Imported lazily: the package __init__ imports core, which imports this + # module, so a top-level import here would be circular. + from . import __version__ # noqa: PLC0415 + + provenance: dict[str, Any] = { + "tool_version": __version__, + "command_line": list(sys.argv), + **get_git_provenance(), + } + + exe = config.get("eddypro_executable") + eddypro: dict[str, Any] = {"path": str(exe) if exe else None, "checksum": None} + if exe: + try: + exe_path = Path(exe) + if exe_path.exists(): + eddypro["checksum"] = compute_file_checksum(exe_path) + # EddyPro embeds its version in the install directory name, + # e.g. .../EddyPro-7.0.9/bin/eddypro_rp.exe + for part in exe_path.parts: + if part.lower().startswith("eddypro-"): + eddypro["version"] = part.split("-", 1)[1] + break + except Exception: + logger.debug("Could not checksum EddyPro executable") + provenance["eddypro"] = eddypro + + return provenance + + def compute_file_checksum(file_path: Path, algorithm: str = "sha256") -> str: """ Compute checksum of a file. @@ -222,6 +313,10 @@ def generate_run_manifest( overall_success: bool, output_dirs: list[Path], provenance: dict[str, Any] | None = None, + years: list[dict[str, Any]] | None = None, + errors: list[str] | None = None, + status: str = "completed", + metrics_summary: dict[str, Any] | None = None, ) -> dict[str, Any]: """ Generate a run-level manifest capturing all scenarios and metadata. @@ -229,15 +324,23 @@ def generate_run_manifest( Args: run_id: Unique identifier for this run config: Configuration dictionary used for the run - config_checksum: Checksum of the config file + config_checksum: Stable SHA256 checksum of the config site_id: Site identifier - years_processed: List of years processed + years_processed: Years that completed successfully (derived, kept for + backwards compatibility -- prefer ``years`` for full detail) scenarios: List of scenario manifests start_time: Run start timestamp end_time: Run end timestamp overall_success: Whether all scenarios succeeded output_dirs: List of output directories created - provenance: Optional provenance information (git SHA, etc.) + provenance: Provenance information (git SHA, tool version, EddyPro build) + years: Per-year records ``{year, status, duration_seconds, error, + output_dir}``. A failed year appears here even though it is absent + from ``years_processed``. + errors: Run-level error messages + status: ``"running"`` for the manifest written at run start, then + ``"completed"`` or ``"failed"`` + metrics_summary: Bottleneck analysis and performance summary Returns: Dictionary containing run manifest data @@ -252,44 +355,80 @@ def generate_run_manifest( output_files[str(output_dir)] = collected manifest = { + "manifest_schema_version": MANIFEST_SCHEMA_VERSION, "run_id": run_id, + "status": status, "timestamp": start_time.isoformat(), "start_time": start_time.isoformat(), "end_time": end_time.isoformat(), "duration_seconds": duration_seconds, "site_id": site_id, "years_processed": years_processed, + "years": years if years is not None else [], "config_checksum": config_checksum, "config_snapshot": config, # Include full config for reproducibility "overall_success": overall_success, "scenarios": scenarios, + "errors": errors if errors is not None else [], "output_dirs": [str(d) for d in output_dirs], - "output_files": output_files, # New: detailed file inventory + "output_files": output_files, # Detailed file inventory "environment": get_python_environment_info(), "dry_run": config.get("dry_run", False), # Track if this was a dry run } if provenance: manifest["provenance"] = provenance + if metrics_summary: + manifest["metrics_summary"] = metrics_summary return manifest -def write_run_manifest(manifest: dict[str, Any], output_path: Path) -> None: +def write_run_manifest(manifest: dict[str, Any], output_path: Path) -> bool: """ - Write run manifest to JSON file. + Write the run manifest to JSON atomically. + + The manifest is serialised to a temporary file in the destination directory + and then moved into place with :func:`os.replace`. Writing directly to the + destination meant that a crash mid-write destroyed the previous good manifest + and left invalid JSON behind. + + A copy is also archived under ``manifests/run_manifest_{run_id}.json`` so that + consecutive runs against the same output tree do not erase each other's + provenance. Args: manifest: Run manifest dictionary output_path: Path to write manifest JSON + + Returns: + True if the manifest was written successfully. """ try: output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open("w") as f: - json.dump(manifest, f, indent=2) + tmp_path = output_path.with_name(output_path.name + ".tmp") + with tmp_path.open("w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2, default=str) + os.replace(tmp_path, output_path) logger.info(f"Run manifest written to {output_path}") except Exception: logger.exception(f"Failed to write run manifest to {output_path}") + return False + + # Archive a per-run copy. Failure here is not fatal: the canonical manifest + # is already safely in place. + try: + run_id = manifest.get("run_id") + if run_id: + archive_dir = output_path.parent / "manifests" + archive_dir.mkdir(parents=True, exist_ok=True) + archive_path = archive_dir / f"run_manifest_{run_id}.json" + with archive_path.open("w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2, default=str) + except Exception: + logger.debug("Could not archive per-run manifest copy") + + return True def load_metrics_from_csv(metrics_csv_path: Path) -> list[dict[str, Any]]: @@ -306,15 +445,31 @@ def load_metrics_from_csv(metrics_csv_path: Path) -> list[dict[str, Any]]: try: with metrics_csv_path.open("r") as f: reader = csv.DictReader(f) + numeric_fields = ( + "cpu_percent", + "memory_mb", + "read_mb", + "write_mb", + "read_mb_per_s", + "write_mb_per_s", + "read_iops", + "write_iops", + "relative_time", + "system_cpu_percent", + "system_memory_percent", + ) for row in reader: - # Convert numeric fields - try: - row["cpu_percent"] = float(row.get("cpu_percent", 0)) - row["memory_mb"] = float(row.get("memory_mb", 0)) - row["read_mb"] = float(row.get("read_mb", 0)) - row["write_mb"] = float(row.get("write_mb", 0)) - except (ValueError, KeyError): - pass + # Coerce per field rather than in one block: a single blank cell + # must not abandon the remaining conversions for that row. + for name in numeric_fields: + if name not in row: + continue + try: + row[name] = ( + float(row[name]) if row[name] not in ("", None) else 0.0 + ) + except (TypeError, ValueError): + row[name] = 0.0 metrics.append(row) except Exception: logger.warning(f"Failed to load metrics from {metrics_csv_path}") @@ -448,8 +603,7 @@ def generate_html_report( html_parts = [] # HTML header - html_parts.append( - """ + html_parts.append(""" @@ -512,8 +666,7 @@ def generate_html_report(
-""" - ) +""") # Report title and summary run_id = run_manifest.get("run_id", "unknown") @@ -525,8 +678,7 @@ def generate_html_report( status_class = "success" if overall_success else "failure" status_text = "SUCCESS" if overall_success else "FAILURE" - html_parts.append( - f""" + html_parts.append(f"""

EddyPro Batch Processing Report

Run Summary

@@ -537,14 +689,82 @@ def generate_html_report(

Years Processed: {", ".join(map(str, years))}

Overall Status: {status_text}

-""" - ) +""") + + # Performance health check: traffic-light bottleneck summary. Placed directly + # under the run summary because it is the first thing a user wants to know. + metrics_summary = run_manifest.get("metrics_summary") or {} + perf_entries = metrics_summary.get("scenarios", []) + if perf_entries: + colours = { + "RED": "#e74c3c", + "YELLOW": "#f39c12", + "GREEN": "#27ae60", + "UNKNOWN": "#95a5a6", + } + + def _dot(status: str) -> str: + colour = colours.get(status, colours["UNKNOWN"]) + return ( + f'' + f"{status}" + ) + + html_parts.append(f""" +

Performance Health Check

+

Primary bottleneck: + {metrics_summary.get("primary_bottleneck", "UNKNOWN")}

+ + + + + + +""") + for entry in perf_entries: + cpu = entry.get("cpu", {}) or {} + html_parts.append(f""" + + + + + + + + + + +""") + html_parts.append("
RunCPUMemoryDiskBottleneckCPU p95Peak RAM (MB)Read (MB)Write (MB)
{entry.get("scenario_name", "?")}{_dot(entry.get("cpu_status", "UNKNOWN"))}{_dot(entry.get("memory_status", "UNKNOWN"))}{_dot(entry.get("disk_status", "UNKNOWN"))}{entry.get("primary_bottleneck", "?")}{cpu.get("p95", 0):.1f}%{entry.get("peak_memory_mb", 0):.0f}{entry.get("total_read_mb", 0):.0f}{entry.get("total_write_mb", 0):.0f}
\n") + for entry in perf_entries: + if entry.get("explanation"): + html_parts.append( + f'

{entry.get("scenario_name", "?")}: ' + f'{entry["explanation"]}

\n' + ) + + # Per-year results, including years that failed + year_records = run_manifest.get("years", []) + if year_records: + html_parts.append(""" +

Per-Year Results

+ + +""") + for rec in year_records: + html_parts.append( + f" " + f"" + f"" + f"\n" + ) + html_parts.append("
YearStatusDuration (s)Error
{rec.get('year', '?')}{rec.get('status', '?')}{rec.get('duration_seconds', 0):.1f}{rec.get('error') or ''}
\n") # Scenario summary table scenarios = run_manifest.get("scenarios", []) if scenarios: - html_parts.append( - """ + html_parts.append("""

Scenario Results

@@ -553,8 +773,7 @@ def generate_html_report( -""" - ) +""") for scenario in scenarios: name = scenario.get("scenario_name", "unknown") params = scenario.get("scenario_params", {}) @@ -564,16 +783,14 @@ def generate_html_report( status_class = "success" if success else "failure" status_text = "SUCCESS" if success else "FAILURE" - html_parts.append( - f""" + html_parts.append(f""" -""" - ) +""") html_parts.append("
Duration (s) Status
{name} {params_str or "baseline"} {duration_s:.2f} {status_text}
\n") # Performance charts (if available and requested) @@ -596,48 +813,40 @@ def generate_html_report( # Environment information env_info = run_manifest.get("environment", {}) - html_parts.append( - f""" + html_parts.append(f"""

Environment

Python Version: {env_info.get("python_version", "unknown")}

Platform: {env_info.get("platform", "unknown")}

Processor: {env_info.get("processor", "unknown")}

-""" - ) +""") package_versions = env_info.get("package_versions", {}) if package_versions: - html_parts.append( - """ + html_parts.append("""

Package Versions

-""" - ) +""") for pkg, version in package_versions.items(): - html_parts.append( - f""" + html_parts.append(f""" -""" - ) +""") html_parts.append("
Package Version
{pkg} {version}
\n") # HTML footer - html_parts.append( - """ + html_parts.append("""
-""" - ) +""") html_content = "".join(html_parts) diff --git a/src/eddypro_batch_processor/scenarios.py b/src/eddypro_batch_processor/scenarios.py index 0b9d717..2e948f7 100644 --- a/src/eddypro_batch_processor/scenarios.py +++ b/src/eddypro_batch_processor/scenarios.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Scenario Generation for EddyPro Batch Processor. diff --git a/src/eddypro_batch_processor/validation.py b/src/eddypro_batch_processor/validation.py index c32034e..490d5c7 100644 --- a/src/eddypro_batch_processor/validation.py +++ b/src/eddypro_batch_processor/validation.py @@ -81,6 +81,14 @@ def validate_config_structure(config: dict[str, Any]) -> list[str]: f"'max_processes' must be an integer, got {type(config['max_processes'])}" ) + if "monitoring_enabled" in config and not isinstance( + config["monitoring_enabled"], bool + ): + errors.append( + f"'monitoring_enabled' must be a boolean, got " + f"{type(config['monitoring_enabled'])}" + ) + if "metrics_interval_seconds" in config and not isinstance( config["metrics_interval_seconds"], int | float ): @@ -408,12 +416,14 @@ def validate_config_sanity(config: dict[str, Any]) -> list[str]: f"enabled, got {max_proc}" ) - # Check metrics_interval_seconds is positive - metrics_interval = config.get("metrics_interval_seconds", 0) - if metrics_interval <= 0: - errors.append( - f"'metrics_interval_seconds' must be positive, got {metrics_interval}" - ) + # Check metrics_interval_seconds is positive. Only meaningful when monitoring + # is enabled -- a disabled monitor never reads the interval. + if config.get("monitoring_enabled", True): + metrics_interval = config.get("metrics_interval_seconds", 0) + if metrics_interval <= 0: + errors.append( + f"'metrics_interval_seconds' must be positive, got {metrics_interval}" + ) return errors diff --git a/tests/.coverage b/tests/.coverage deleted file mode 100644 index ca8b4fd..0000000 Binary files a/tests/.coverage and /dev/null differ diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 0000000..8e2767e --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,341 @@ +""" +Tests for the bottleneck analysis module. + +Includes an intentionally un-mocked integration test that runs a real subprocess +under the real monitor. Every other monitor test mocks psutil wholesale, which is +precisely why the original "monitor reports 0.0 for everything" bug survived in +CI for so long. +""" + +import csv +import sys +import textwrap +from pathlib import Path + +import pytest + +from eddypro_batch_processor import core +from eddypro_batch_processor.analysis import ( + DEFAULT_THRESHOLDS, + BottleneckAnalyzer, + dominant_bottleneck, +) + +FIELDNAMES = [ + "timestamp", + "relative_time", + "cpu_percent", + "cpu_percent_of_core", + "memory_mb", + "read_mb", + "write_mb", + "read_mb_per_s", + "write_mb_per_s", + "read_iops", + "write_iops", + "num_processes", + "system_cpu_percent", + "system_memory_percent", + "system_memory_used_mb", + "system_read_mb_per_s", + "system_write_mb_per_s", +] + + +def write_metrics(path: Path, rows: list[dict]) -> Path: + """Write a metrics CSV in the canonical schema.""" + with path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) + writer.writeheader() + for i, row in enumerate(rows): + full = dict.fromkeys(FIELDNAMES, 0.0) + full["timestamp"] = 1000.0 + i + full["relative_time"] = float(i) + full.update(row) + writer.writerow(full) + return path + + +def make_series(path: Path, count: int = 20, **values) -> Path: + """Write `count` identical samples with the given metric values.""" + return write_metrics(path, [dict(values) for _ in range(count)]) + + +class TestBottleneckClassification: + """The analyzer must name the right limiting resource.""" + + def test_cpu_bound(self, tmp_path): + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=95.0, + memory_mb=500.0, + system_memory_percent=40.0, + read_mb_per_s=2.0, + write_mb_per_s=1.0, + ) + result = BottleneckAnalyzer().analyze(csv_path) + assert result.primary_bottleneck == "CPU" + assert result.cpu_status == "RED" + assert result.disk_status == "GREEN" + assert "saturated" in result.explanation.lower() + + def test_disk_throughput_bound(self, tmp_path): + # Idle CPU paired with heavy sustained I/O is the disk-bound signature. + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=8.0, + memory_mb=300.0, + system_memory_percent=35.0, + read_mb_per_s=180.0, + write_mb_per_s=20.0, + ) + result = BottleneckAnalyzer().analyze(csv_path) + assert result.primary_bottleneck == "DISK_THROUGHPUT" + assert result.disk_status == "RED" + assert result.cpu_status == "GREEN" + + def test_disk_iops_bound(self, tmp_path): + # Low throughput but very high operation count: many small files. + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=10.0, + memory_mb=200.0, + system_memory_percent=30.0, + read_mb_per_s=5.0, + write_mb_per_s=2.0, + read_iops=3000.0, + write_iops=500.0, + ) + result = BottleneckAnalyzer().analyze(csv_path) + assert result.primary_bottleneck == "DISK_IOPS" + + def test_memory_bound(self, tmp_path): + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=30.0, + memory_mb=24000.0, + system_memory_percent=93.0, + read_mb_per_s=5.0, + write_mb_per_s=5.0, + ) + result = BottleneckAnalyzer().analyze(csv_path) + assert result.primary_bottleneck == "MEMORY" + assert result.memory_status == "RED" + + def test_no_bottleneck(self, tmp_path): + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=15.0, + memory_mb=250.0, + system_memory_percent=30.0, + read_mb_per_s=3.0, + write_mb_per_s=2.0, + ) + result = BottleneckAnalyzer().analyze(csv_path) + assert result.primary_bottleneck == "NONE" + assert result.cpu_status == "GREEN" + assert "headroom" in result.explanation.lower() + + def test_cpu_takes_priority_over_disk(self, tmp_path): + # A saturated CPU outranks a busy disk: the CPU is the binding limit. + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=96.0, + system_memory_percent=40.0, + read_mb_per_s=200.0, + write_mb_per_s=200.0, + ) + assert BottleneckAnalyzer().analyze(csv_path).primary_bottleneck == "CPU" + + +class TestThresholdOverrides: + """Thresholds must be tunable for different hardware.""" + + def test_custom_disk_threshold_changes_verdict(self, tmp_path): + csv_path = make_series( + tmp_path / "metrics.csv", + cpu_percent=8.0, + system_memory_percent=30.0, + read_mb_per_s=180.0, + ) + assert BottleneckAnalyzer().analyze(csv_path).primary_bottleneck == ( + "DISK_THROUGHPUT" + ) + # On an NVMe drive 180 MB/s is unremarkable, so raise the ceiling. + relaxed = BottleneckAnalyzer( + {"disk_high_mb_per_s": 2000.0, "disk_moderate_mb_per_s": 1000.0} + ) + assert relaxed.analyze(csv_path).primary_bottleneck == "NONE" + + def test_defaults_are_not_mutated(self, tmp_path): + original = dict(DEFAULT_THRESHOLDS) + BottleneckAnalyzer({"cpu_high_percent": 10.0}) + assert dict(DEFAULT_THRESHOLDS) == original + + +class TestRobustness: + """Bad input must degrade, not raise.""" + + def test_missing_file(self, tmp_path): + result = BottleneckAnalyzer().analyze(tmp_path / "nope.csv") + assert result.primary_bottleneck == "UNKNOWN" + assert result.sample_count == 0 + + def test_empty_file(self, tmp_path): + path = write_metrics(tmp_path / "metrics.csv", []) + assert BottleneckAnalyzer().analyze(path).primary_bottleneck == "UNKNOWN" + + def test_blank_and_malformed_cells(self, tmp_path): + path = tmp_path / "metrics.csv" + with path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) + writer.writeheader() + for i in range(5): + row = dict.fromkeys(FIELDNAMES, "") + row["relative_time"] = i + row["cpu_percent"] = "" if i % 2 else 95.0 + row["system_memory_percent"] = "n/a" + writer.writerow(row) + result = BottleneckAnalyzer().analyze(path) + # Parses what it can rather than raising on the junk cells. + assert result.sample_count == 5 + assert result.cpu.max == 95.0 + + +class TestDominantBottleneck: + """Aggregation across several analyses.""" + + def test_empty(self): + assert dominant_bottleneck([]) == "UNKNOWN" + + def test_real_bottleneck_outranks_none(self, tmp_path): + cpu = make_series( + tmp_path / "a.csv", cpu_percent=95.0, system_memory_percent=30.0 + ) + idle = make_series( + tmp_path / "b.csv", cpu_percent=5.0, system_memory_percent=30.0 + ) + analyzer = BottleneckAnalyzer() + analyses = [ + analyzer.analyze(idle), + analyzer.analyze(idle), + analyzer.analyze(cpu), + ] + # NONE is more common, but the saturated run is the actionable finding. + assert dominant_bottleneck(analyses) == "CPU" + + +@pytest.mark.slow +@pytest.mark.integration +class TestRealWorkloadMonitoring: + """ + End-to-end regression guard for the original defect. + + The monitor used to be pointed at the `cmd.exe` wrapper created by + `shell=True` rather than at the real workload, so every CPU and disk figure + was 0.0. These tests use no mocks at all: if that regression returns, they + fail. + """ + + @pytest.fixture + def burner_script(self, tmp_path): + script = tmp_path / "burner.py" + script.write_text( + textwrap.dedent(""" + import os, sys, tempfile, time + end = time.time() + 3.0 + d = tempfile.mkdtemp() + payload = b"x" * (1024 * 1024) + i = 0 + while time.time() < end: + total = 0 + for j in range(150000): + total += j * j + path = os.path.join(d, "chunk%d.bin" % (i % 4)) + with open(path, "wb") as fh: + fh.write(payload * 4) + fh.flush() + os.fsync(fh.fileno()) + with open(path, "rb") as fh: + fh.read() + i += 1 + """), + encoding="utf-8", + ) + return script + + def test_monitor_captures_real_numbers(self, tmp_path, burner_script): + out_dir = tmp_path / "metrics" + rc = core.run_subprocess_with_monitoring( + command=[sys.executable, str(burner_script)], + working_dir=tmp_path, + stream_output=False, + metrics_interval=0.25, + output_dir=out_dir, + scenario_suffix="real", + log_output=False, + ) + assert rc == 0 + + csv_path = out_dir / "metrics_real.csv" + assert csv_path.exists(), "monitor wrote no metrics file" + rows = list(csv.DictReader(csv_path.open(encoding="utf-8"))) + assert len(rows) >= 3, f"expected several samples, got {len(rows)}" + + def col(name): + return [float(r[name]) for r in rows if r.get(name) not in ("", None)] + + # The load is single-threaded, so normalised CPU is small on a many-core + # box; cpu_percent_of_core is the column that must show real work. + assert max(col("cpu_percent_of_core")) > 20.0, ( + "process-tree CPU never rose above 20% of a core -- the monitor is " + "measuring the wrong process again" + ) + assert max(col("memory_mb")) > 1.0, "process memory looks like a shell" + + # Assert on combined I/O rather than reads specifically. On Linux + # io_counters().read_bytes counts only what was actually fetched from + # the storage layer, so reading back a file that is still in the page + # cache correctly reports zero reads. Writes are fsync'd, so the + # combined total is non-zero on every platform. + total_io = max(col("read_mb")) + max(col("write_mb")) + assert total_io > 0.0, "no disk I/O recorded at all" + total_rate = max(col("read_mb_per_s")) + max(col("write_mb_per_s")) + assert total_rate > 0.0, "no I/O rate derived from the counter deltas" + assert max(col("num_processes")) >= 1 + + # Cumulative counters must never decrease. + reads = col("read_mb") + assert reads == sorted(reads), "cumulative read_mb went backwards" + + def test_analysis_of_real_run_is_not_unknown(self, tmp_path, burner_script): + out_dir = tmp_path / "metrics" + core.run_subprocess_with_monitoring( + command=[sys.executable, str(burner_script)], + working_dir=tmp_path, + stream_output=False, + metrics_interval=0.25, + output_dir=out_dir, + scenario_suffix="real", + log_output=False, + ) + result = BottleneckAnalyzer().analyze(out_dir / "metrics_real.csv") + assert result.primary_bottleneck != "UNKNOWN" + assert result.sample_count > 0 + assert result.total_read_mb + result.total_write_mb > 0 + + def test_monitoring_disabled_writes_nothing(self, tmp_path, burner_script): + out_dir = tmp_path / "metrics" + rc = core.run_subprocess_with_monitoring( + command=[sys.executable, "-c", "print('quick')"], + working_dir=tmp_path, + stream_output=False, + metrics_interval=0.25, + output_dir=out_dir, + scenario_suffix="off", + log_output=False, + monitoring_enabled=False, + ) + assert rc == 0 + assert not (out_dir / "metrics_off.csv").exists() + assert not (out_dir / "metrics_summary_off.json").exists() diff --git a/tests/test_cli_functions.py b/tests/test_cli_functions.py index 4c502e8..cff3b7b 100644 --- a/tests/test_cli_functions.py +++ b/tests/test_cli_functions.py @@ -42,8 +42,7 @@ def test_cmd_run_basic(self, tmp_path): # Create a complete config file with all required fields ecmd_file = _write_ecmd_file(tmp_path, "test-site") config_file = tmp_path / "test_config.yaml" - config_file.write_text( - f""" + config_file.write_text(f""" site_id: test-site years_to_process: [2021] eddypro_executable: /fake/eddypro.exe @@ -57,8 +56,7 @@ def test_cmd_run_basic(self, tmp_path): metrics_interval_seconds: 0.5 reports_dir: null report_charts: none -""" - ) +""") args = argparse.Namespace( config=str(config_file), @@ -80,8 +78,7 @@ def test_cmd_run_with_site_override(self, tmp_path): """Test cmd_run with site override.""" ecmd_file = _write_ecmd_file(tmp_path, "TEST-SITE") config_file = tmp_path / "test_config.yaml" - config_file.write_text( - f""" + config_file.write_text(f""" site_id: original-site years_to_process: [2021] eddypro_executable: /fake/eddypro.exe @@ -95,8 +92,7 @@ def test_cmd_run_with_site_override(self, tmp_path): metrics_interval_seconds: 0.5 reports_dir: null report_charts: none -""" - ) +""") args = argparse.Namespace( config=str(config_file), @@ -117,8 +113,7 @@ def test_cmd_run_with_years_override(self, tmp_path): """Test cmd_run with years override.""" ecmd_file = _write_ecmd_file(tmp_path, "test-site") config_file = tmp_path / "test_config.yaml" - config_file.write_text( - f""" + config_file.write_text(f""" site_id: test-site years_to_process: [2020] eddypro_executable: /fake/eddypro.exe @@ -132,8 +127,7 @@ def test_cmd_run_with_years_override(self, tmp_path): metrics_interval_seconds: 0.5 reports_dir: null report_charts: none -""" - ) +""") args = argparse.Namespace( config=str(config_file), @@ -154,8 +148,7 @@ def test_cmd_run_with_dry_run(self, tmp_path): """Test cmd_run with dry run enabled.""" ecmd_file = _write_ecmd_file(tmp_path, "test-site") config_file = tmp_path / "test_config.yaml" - config_file.write_text( - f""" + config_file.write_text(f""" site_id: test-site years_to_process: [2021] eddypro_executable: /fake/eddypro.exe @@ -169,8 +162,7 @@ def test_cmd_run_with_dry_run(self, tmp_path): metrics_interval_seconds: 0.5 reports_dir: null report_charts: none -""" - ) +""") args = argparse.Namespace( config=str(config_file), @@ -197,8 +189,7 @@ def test_cmd_run_executes_via_core_runner(self, tmp_path: Path): ecmd_file = _write_ecmd_file(tmp_path, site_id) config_file = tmp_path / "test_config.yaml" - config_file.write_text( - f""" + config_file.write_text(f""" site_id: {site_id} years_to_process: [{year}] eddypro_executable: /fake/eddypro.exe @@ -212,8 +203,7 @@ def test_cmd_run_executes_via_core_runner(self, tmp_path: Path): metrics_interval_seconds: 0.5 reports_dir: null report_charts: none -""" - ) +""") args = argparse.Namespace( config=str(config_file), @@ -390,8 +380,7 @@ def test_cmd_status_basic(self, tmp_path): reports_dir = tmp_path / "reports" reports_dir.mkdir() manifest_file = reports_dir / "run_manifest.json" - manifest_file.write_text( - """ + manifest_file.write_text(""" { "run_id": "test-run-123", "config_snapshot": {}, @@ -400,8 +389,7 @@ def test_cmd_status_basic(self, tmp_path): "end_time": "2024-01-01T01:00:00", "dry_run": true } -""" - ) +""") args = argparse.Namespace( reports_dir=str(reports_dir), @@ -416,8 +404,7 @@ def test_cmd_status_with_reports_dir(self, tmp_path): reports_dir = tmp_path / "custom_reports" reports_dir.mkdir() manifest_file = reports_dir / "run_manifest.json" - manifest_file.write_text( - """ + manifest_file.write_text(""" { "run_id": "custom-run-456", "config_snapshot": {}, @@ -426,8 +413,7 @@ def test_cmd_status_with_reports_dir(self, tmp_path): "end_time": "2024-02-01T02:00:00", "dry_run": false } -""" - ) +""") args = argparse.Namespace( reports_dir=str(reports_dir), diff --git a/tests/test_core.py b/tests/test_core.py index 5635410..6ec906a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -258,8 +258,13 @@ def _mock_copytree(src: Path, dst: Path, dirs_exist_ok: bool = True) -> Path: rp_call = mock_run.call_args_list[0].kwargs fcc_call = mock_run.call_args_list[1].kwargs - assert "eddypro_rp.exe" in rp_call["command"] - assert "eddypro_fcc.exe" in fcc_call["command"] + # The command must be an argv list, not a shell string: with shell=True + # the monitored PID would be cmd.exe rather than EddyPro. + assert isinstance(rp_call["command"], list) + assert isinstance(fcc_call["command"], list) + assert "eddypro_rp.exe" in rp_call["command"][0] + assert "eddypro_fcc.exe" in fcc_call["command"][0] + assert str(project_file) in rp_call["command"] assert rp_call["working_dir"] == project_dir.parent assert fcc_call["working_dir"] == project_dir.parent assert rp_call["scenario_suffix"] == "rp" diff --git a/tests/test_ini_tools.py b/tests/test_ini_tools.py index 95160d4..015f140 100644 --- a/tests/test_ini_tools.py +++ b/tests/test_ini_tools.py @@ -564,67 +564,6 @@ def test_write_project_file_with_metadata(self): assert parser.get("Site", "site_id") == "SITE" -class TestScenarioSuffixGeneration(unittest.TestCase): - """Test scenario suffix generation functionality.""" - - def test_generate_scenario_suffix_empty(self): - """Test suffix generation with empty parameters.""" - result = ini_tools.generate_scenario_suffix({}) - self.assertEqual(result, "") - - def test_generate_scenario_suffix_single_parameter(self): - """Test suffix generation with single parameter.""" - parameters = {"rot_meth": 1} - result = ini_tools.generate_scenario_suffix(parameters) - self.assertEqual(result, "_rot1") - - def test_generate_scenario_suffix_multiple_parameters(self): - """Test suffix generation with multiple parameters.""" - parameters = { - "rot_meth": 3, - "tlag_meth": 4, - "detrend_meth": 1, - "despike_meth": 0, - } - result = ini_tools.generate_scenario_suffix(parameters) - - # Should be sorted by parameter name alphabetically - # despike_meth, detrend_meth, rot_meth, tlag_meth - expected = "_spk0_det1_rot3_tlag4" - self.assertEqual(result, expected) - - def test_generate_scenario_suffix_with_hf_meth(self): - """Suffix should include hf when hf_meth provided (alphabetical order).""" - parameters = { - "rot_meth": 1, - "hf_meth": 4, - "tlag_meth": 2, - } - result = ini_tools.generate_scenario_suffix(parameters) - # Alphabetical order of keys: hf_meth, rot_meth, tlag_meth -> hf, rot, tlag - self.assertEqual(result, "_hf4_rot1_tlag2") - - def test_generate_scenario_suffix_deterministic(self): - """Test that suffix generation is deterministic.""" - parameters = {"despike_meth": 1, "rot_meth": 1, "tlag_meth": 2} - - # Generate suffix multiple times - results = [ini_tools.generate_scenario_suffix(parameters) for _ in range(5)] - - # All results should be identical - self.assertTrue(all(r == results[0] for r in results)) - # Should be sorted alphabetically: despike_meth, rot_meth, tlag_meth - self.assertEqual(results[0], "_spk1_rot1_tlag2") - - def test_generate_scenario_suffix_unknown_parameter(self): - """Test suffix generation with unknown parameter name.""" - parameters = {"unknown_param": 1} - result = ini_tools.generate_scenario_suffix(parameters) - - # Should fallback to original parameter name - self.assertEqual(result, "_unknown_param1") - - class TestConditionalDateRanges(unittest.TestCase): """Test conditional date/time range population.""" diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 9797eec..e51ed95 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -148,7 +148,10 @@ def test_sample_collection(self, mock_time, temp_dir, mock_psutil): assert sample["relative_time"] == 0.0 assert sample["system_cpu_percent"] == 50.0 assert sample["system_memory_percent"] == 50.0 - assert sample["system_disk_read_bytes"] == 1000000 + # Raw since-boot counters are no longer emitted; rates are derived from + # deltas instead, and the first sample has no previous value to diff. + assert "system_disk_read_bytes" not in sample + assert sample["system_read_mb_per_s"] == 0.0 @patch("time.time") def test_process_monitoring(self, mock_time, temp_dir, mock_psutil): @@ -169,9 +172,12 @@ def test_process_monitoring(self, mock_time, temp_dir, mock_psutil): # Collect sample with process metrics sample = monitor._collect_sample() assert sample is not None - assert "process_cpu_percent" in sample - assert "process_memory_rss" in sample - assert sample["process_cpu_percent"] == 25.0 + # Canonical schema: the process tree is reported as cpu_percent (core + # normalised), cpu_percent_of_core (raw) and memory_mb. + assert "cpu_percent" in sample + assert "memory_mb" in sample + assert sample["memory_mb"] == 100.0 + assert sample["num_processes"] == 1 def test_process_not_found(self, temp_dir, mock_psutil): """Test handling of non-existent process.""" @@ -420,8 +426,9 @@ def test_deterministic_sample_generation(self, temp_dir): # Verify deterministic values assert sample["system_cpu_percent"] == 42.0 - assert sample["system_memory_total"] == 8589934592 - assert sample["system_disk_read_bytes"] == 1048576 + assert sample["system_memory_percent"] == 50.0 + # 8 GiB total - 4 GiB available = 4096 MiB used + assert sample["system_memory_used_mb"] == 4096.0 class TestErrorHandling: @@ -487,8 +494,11 @@ class MockAccessDeniedError(Exception): sample = monitor._collect_sample() assert sample is not None assert "timestamp" in sample - # System metrics should be missing due to exceptions - assert "system_cpu_percent" not in sample + # The canonical schema is fixed-width: on failure the keys are still + # present but hold defaults, so the CSV never gains ragged columns. + assert sample["system_cpu_percent"] == 0.0 + assert sample["system_memory_percent"] == 0.0 + assert sample["cpu_percent"] == 0.0 def test_thread_safety(self, temp_dir): """Test thread safety of monitoring operations.""" diff --git a/tests/test_report.py b/tests/test_report.py index 96b20a8..784f8ea 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,12 +1,14 @@ """Tests for the report module.""" +import csv import json -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path import pytest from eddypro_batch_processor import report +from eddypro_batch_processor.monitor import METRICS_FIELDNAMES def test_compute_file_checksum(tmp_path): @@ -338,3 +340,111 @@ def test_generate_plotly_charts(): # Test with empty metrics result = report.generate_plotly_charts([], scenario_name="test") assert result is None + + +class TestPerformanceAndManifestV2: + """Regression cover for the manifest and report fixes.""" + + def test_config_checksum_is_stable_and_order_independent(self): + """The old str(hash(...)) was salted per process and useless.""" + a = report.compute_config_checksum({"a": 1, "b": [2, 3]}) + b = report.compute_config_checksum({"b": [2, 3], "a": 1}) + assert a == b + assert len(a) == 64 # sha256 hex + + def test_manifest_has_schema_version_and_year_records(self, tmp_path): + now = datetime.now(timezone.utc) + manifest = report.generate_run_manifest( + run_id="r1", + config={"x": 1}, + config_checksum="abc", + site_id="S", + years_processed=[2021], + scenarios=[], + start_time=now, + end_time=now, + overall_success=False, + output_dirs=[], + years=[ + {"year": 2021, "status": "success", "error": None}, + {"year": 2022, "status": "failed", "error": "boom"}, + ], + errors=["2022: boom"], + status="failed", + ) + assert manifest["manifest_schema_version"] == 2 + assert manifest["status"] == "failed" + # A failed year must remain visible even though it is absent from + # years_processed. + assert [y["year"] for y in manifest["years"]] == [2021, 2022] + assert manifest["errors"] == ["2022: boom"] + assert "+00:00" in manifest["start_time"] + + def test_manifest_write_is_atomic_and_archived(self, tmp_path): + out = tmp_path / "reports" / "run_manifest.json" + ok = report.write_run_manifest({"run_id": "abc", "v": 1}, out) + assert ok is True + assert out.exists() + assert json.loads(out.read_text())["run_id"] == "abc" + # No temp file left behind + assert not list(out.parent.glob("*.tmp")) + # Per-run archive copy preserves prior provenance + assert (out.parent / "manifests" / "run_manifest_abc.json").exists() + + def test_report_renders_traffic_light_table(self): + entry = { + "scenario_name": "2021_rp", + "cpu_status": "RED", + "memory_status": "GREEN", + "disk_status": "YELLOW", + "primary_bottleneck": "CPU", + "cpu": {"p95": 94.2}, + "peak_memory_mb": 1234.0, + "total_read_mb": 500.0, + "total_write_mb": 250.0, + "explanation": "CPU saturated.", + } + html = report.generate_html_report( + run_manifest={ + "run_id": "r", + "timestamp": "t", + "duration_seconds": 1.0, + "site_id": "S", + "years_processed": [2021], + "overall_success": True, + "environment": {}, + "metrics_summary": { + "scenarios": [entry], + "primary_bottleneck": "CPU", + }, + }, + scenario_metrics=None, + chart_engine="none", + ) + assert "Performance Health Check" in html + assert "94.2" in html + assert "CPU saturated." in html + + def test_metrics_loader_reads_monitor_schema(self, tmp_path): + """The loader and the monitor must agree on column names.""" + csv_path = tmp_path / "metrics.csv" + with csv_path.open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=METRICS_FIELDNAMES) + w.writeheader() + w.writerow( + dict.fromkeys(METRICS_FIELDNAMES, 0) + | { + "cpu_percent": 55.5, + "memory_mb": 128.0, + "read_mb": 10.0, + "write_mb": 5.0, + } + ) + rows = report.load_metrics_from_csv(csv_path) + assert len(rows) == 1 + # These four are exactly what the chart generator reads; a mismatch here + # is what previously made every chart a flat zero line. + assert rows[0]["cpu_percent"] == 55.5 + assert rows[0]["memory_mb"] == 128.0 + assert rows[0]["read_mb"] == 10.0 + assert rows[0]["write_mb"] == 5.0