From d3256331a7c92c6e9d618aa0f8d13f5c1403de27 Mon Sep 17 00:00:00 2001 From: ArturSepp Date: Mon, 21 Sep 2026 10:50:42 +0200 Subject: [PATCH 1/3] ci: add Desktop preflight and required pull-request checks --- .gitattributes | 1 + .githooks/Run-Check.ps1 | 5 + .githooks/pre-commit | 11 + .github/GITHUB_DESKTOP.md | 13 + .github/oss-checks-requirements.txt | 3 + .github/oss-checks.json | 102 ++++++++ .github/oss_checks.py | 375 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 17 +- .github/workflows/docs.yml | 36 +-- .github/workflows/link-health.yml | 16 ++ .github/workflows/required.yml | 66 +++++ 11 files changed, 611 insertions(+), 34 deletions(-) create mode 100644 .githooks/Run-Check.ps1 create mode 100644 .githooks/pre-commit create mode 100644 .github/GITHUB_DESKTOP.md create mode 100644 .github/oss-checks-requirements.txt create mode 100644 .github/oss-checks.json create mode 100644 .github/oss_checks.py create mode 100644 .github/workflows/link-health.yml create mode 100644 .github/workflows/required.yml diff --git a/.gitattributes b/.gitattributes index d7c79ff..560a711 100644 --- a/.gitattributes +++ b/.gitattributes @@ -21,3 +21,4 @@ *.pkl binary *.pickle binary *.parquet binary +.githooks/* text eol=lf diff --git a/.githooks/Run-Check.ps1 b/.githooks/Run-Check.ps1 new file mode 100644 index 0000000..7e381c8 --- /dev/null +++ b/.githooks/Run-Check.ps1 @@ -0,0 +1,5 @@ +param([Parameter(Mandatory=$true)][string]$RepoPath) +$ErrorActionPreference = 'Stop' +$hub = if ($env:OSS_GOVERNANCE_ROOT) { $env:OSS_GOVERNANCE_ROOT } else { Join-Path $env:USERPROFILE 'OneDrive\analytics\my_github\ArturSepp\scripts\repo_governance' } +& (Join-Path $hub 'Invoke-CommitCheck.ps1') -RepoPath $RepoPath -Task preflight +exit $LASTEXITCODE diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100644 index 0000000..d0dd7cd --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,11 @@ +#!/bin/sh +# GitHub Desktop invokes this through its bundled Git; no activated shell is needed. +set -eu +repo=$(git rev-parse --show-toplevel) +if [ "$(uname -s | cut -c1-5)" = "MINGW" ] || [ "$(uname -s | cut -c1-4)" = "MSYS" ]; then + powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "$repo/.githooks/Run-Check.ps1" -RepoPath "$repo" +else + echo "OSS hooks require the Windows setup on this checkout. CI still enforces Required checks." + echo "Run python .github/oss_checks.py preflight using the pinned tooling on other hosts." + exit 1 +fi diff --git a/.github/GITHUB_DESKTOP.md b/.github/GITHUB_DESKTOP.md new file mode 100644 index 0000000..b3ad443 --- /dev/null +++ b/.github/GITHUB_DESKTOP.md @@ -0,0 +1,13 @@ +# Committing with GitHub Desktop + +*Author: [Artur Sepp](https://github.com/ArturSepp)* + +Create a working branch, select the intended changes, and commit normally. The installed +hook checks the selected contents. Push the branch, open a pull request, and merge after +**Required checks** passes. Main is protected; a failing branch does not change main. + +The [shared Desktop guide](https://github.com/ArturSepp/ArturSepp/blob/main/docs/github_desktop.md) +explains setup, repair messages, partial commits, and the longer local checks. +GitHub Desktop can bypass a local hook for a work-in-progress commit; remote checks still apply. +No hook automatically stages or changes files. External-link and live-dependency maintenance +runs are labelled separately from the required checks on a proposed change. diff --git a/.github/oss-checks-requirements.txt b/.github/oss-checks-requirements.txt new file mode 100644 index 0000000..dd4201d --- /dev/null +++ b/.github/oss-checks-requirements.txt @@ -0,0 +1,3 @@ +ruff==0.16.2 +PyYAML==6.0.3 +uv==0.12.13 diff --git a/.github/oss-checks.json b/.github/oss-checks.json new file mode 100644 index 0000000..309e6ad --- /dev/null +++ b/.github/oss-checks.json @@ -0,0 +1,102 @@ +{ + "version": "1.0.0", + "repository": "OptionChainAnalytics", + "package": "option_chain_analytics", + "python_environment": "OptionChainAnalytics312", + "lint_paths": [ + "src/option_chain_analytics/*.py", + "tests/*.py", + "examples/*.py" + ], + "preflight": [], + "docs": [ + [ + "-m", + "sphinx", + "-E", + "-W", + "--keep-going", + "-b", + "html", + "docs", + "{output}/html" + ] + ], + "tests": [ + [ + "-m", + "pytest", + "-o", + "cache_dir={output}/pytest" + ] + ], + "remote_only": [ + "supported Python/OS matrix", + "clean core/extras environments", + "wheel and sdist", + "coverage gates", + "dependency floors and live compatibility", + "online link and security health" + ], + "required_jobs": [ + "preflight", + "ci", + "docs" + ], + "workflow_inventory": { + "ci": { + "test": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "Install uv and Python ${{ matrix.python-version }}", + "Sync the test environment", + "Run tests", + "Run offline first success" + ], + "lint": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "Install uv and Python", + "Check lint" + ], + "wheel": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "Install uv and Python", + "Build the wheel from the source distribution", + "Assert distribution contents and metadata", + "Install the wheel into a clean environment", + "Exercise the installed wheel from outside the checkout" + ], + "stack-policy": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d", + "Exercise policy regressions", + "Check stack imports and optional adapter isolation" + ], + "lowest-dependencies": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d", + "Keep the test environment outside the checkout", + "Resolve a fresh lowest-direct test environment", + "Install the Bloomberg SDK for the provider extra", + "Record installed versions", + "Check optional dependencies stay outside package-root import", + "Test declared dependency floors without live provider credentials" + ] + }, + "docs": { + "docs": [ + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "Install uv and Python", + "Sync documentation environment", + "Verify redirect path and fragment handling", + "Build HTML with warnings as errors", + "Check documentation links", + "Build legacy documentation redirects", + "Configure GitHub Pages", + "Upload GitHub Pages artifact" + ], + "deploy": [ + "Deploy GitHub Pages" + ] + } + } +} diff --git a/.github/oss_checks.py b/.github/oss_checks.py new file mode 100644 index 0000000..7ab9253 --- /dev/null +++ b/.github/oss_checks.py @@ -0,0 +1,375 @@ +"""Portable OSS checks, shared by GitHub Actions and the Desktop commit hook. + +Canonical source: ArturSepp/scripts/repo_governance/oss_checks.py. +Copies are versioned in each package; the adoption audit detects drift. +""" + +from __future__ import annotations + +import argparse +import ast +import fnmatch +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path, PurePosixPath + +import tomllib + +VERSION = "1.0.0" +PROFILE = ".github/oss-checks.json" +GUIDE = "https://github.com/ArturSepp/ArturSepp/blob/main/docs/github_desktop.md" + + +class CheckFailure(Exception): + """An actionable verification failure, not a Python traceback.""" + + +def run(args, cwd, *, capture=False, env=None): + """Run an argument vector without a shell.""" + result = subprocess.run( + [str(a) for a in args], cwd=cwd, env=env, capture_output=capture, check=False + ) + if result.returncode: + details = result.stderr.decode("utf-8", "replace") if capture else "" + raise CheckFailure( + f"Command failed ({result.returncode}): {' '.join(map(str, args))}\n{details}" + ) + return result.stdout if capture else b"" + + +def git(root, *args): + """Read Git state from the owning checkout.""" + return run(["git", "-c", "core.quotepath=false", *args], root, capture=True) + + +def entries(root, revision=None): + """Return exact blob identities from the index or a committed tree.""" + if revision: + raw = git(root, "ls-tree", "-rz", "--full-tree", revision) + else: + raw = git(root, "ls-files", "--stage", "-z") + result = [] + for record in raw.split(b"\0"): + if not record: + continue + metadata, name = record.split(b"\t", 1) + fields = metadata.decode("ascii").split() + mode, oid = (fields[0], fields[2]) if revision else fields[:2] + path = name.decode("utf-8") + if not revision and fields[2] != "0": + raise CheckFailure(f"Resolve the staged merge conflict first: {path}") + safe_path(path) + if mode not in {"100644", "100755"}: + raise CheckFailure(f"Snapshot does not support mode {mode}: {path}") + result.append((mode, oid, path)) + return result + + +def safe_path(name): + """Reject paths that could write outside a source export.""" + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "\\" in name or ":" in name: + raise CheckFailure(f"Unsafe repository path: {name}") + return path + + +def fingerprint(items): + """Identify the selected files including modes and deletions.""" + return hashlib.sha256(json.dumps(items, ensure_ascii=False).encode()).hexdigest() + + +def export(root, items, destination): + """Export all tracked blobs, including export-ignore verification inputs. + + This reads existing objects only. No temporary Git index, clone, or object store + is created outside the original repository. + """ + payload = "".join(f"{oid}\n" for _, oid, _ in items).encode("ascii") + result = subprocess.run( + ["git", "cat-file", "--batch"], input=payload, cwd=root, capture_output=True, check=False + ) + if result.returncode: + raise CheckFailure(result.stderr.decode("utf-8", "replace")) + offset = 0 + for mode, oid, name in items: + end = result.stdout.index(b"\n", offset) + header = result.stdout[offset:end].decode().split() + if len(header) != 3 or header[0] != oid or header[1] != "blob": + raise CheckFailure(f"Expected a Git blob for {name}: {header}") + length = int(header[2]) + start = end + 1 + target = destination.joinpath(*safe_path(name).parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout[start : start + length]) + if os.name != "nt" and mode == "100755": + target.chmod(0o755) + offset = start + length + 1 + + +def diff(root, revision, base): + """Return changed paths and added line ranges for the actual selected tree.""" + spec = [base, revision] if revision else ["--cached", base] + names = git(root, "diff", "--name-only", "-z", *spec).decode().split("\0") + patch = git(root, "diff", "--no-ext-diff", "--unified=0", *spec).decode("utf-8", "replace") + return [name for name in names if name], added_lines(patch) + + +def added_lines(patch): + """Parse zero-context hunks without treating unchanged legacy lines as new.""" + result = {} + current = None + for line in patch.splitlines(): + if line.startswith("+++ b/"): + current = line[6:] + match = re.match(r"@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", line) + if match and current: + start, count = int(match[1]), int(match[2] or "1") + result.setdefault(current, set()).update(range(start, start + count)) + return result + + +def load_yaml(text): + """Use YAML's BaseLoader so GitHub's 'on' key is not converted to True.""" + import yaml + + try: + return yaml.load(text, Loader=yaml.BaseLoader) + except yaml.YAMLError as error: + raise ValueError(str(error)) from error + + +def source_checks(root, changed): + """Check affected syntax and known documentation contracts without imports.""" + errors = [] + for name in changed: + path = root / name + if not path.is_file(): + continue + suffix = path.suffix.lower() + if suffix not in {".py", ".json", ".toml", ".yml", ".yaml", ".md", ".rst", ".cff"}: + continue + try: + text = path.read_text(encoding="utf-8-sig") + if re.search(r"^(?:<{7} |={7}$|>{7} )", text, re.MULTILINE): + raise ValueError("unresolved merge-conflict markers") + if suffix == ".py": + ast.parse(text, filename=name) + elif suffix == ".json": + json.loads(text) + elif suffix == ".toml": + tomllib.loads(text) + elif suffix in {".yml", ".yaml", ".cff"}: + document = load_yaml(text) + if name.startswith(".github/workflows/") and ( + not isinstance(document, dict) + or not document.get("on") + or not document.get("jobs") + ): + raise ValueError("workflow needs nonempty 'on' and 'jobs' mappings") + elif suffix in {".md", ".rst"}: + bad = re.findall( + r"https://github\.com/ArturSepp/ArturSepp/blob/main/docs/" + r"documentation_standard\.md#(?!user-content-)([\w-]+)", + text, + ) + if bad: + raise ValueError( + f"shared-guide anchors need '#user-content-': {', '.join(bad)}" + ) + except (ValueError, SyntaxError, UnicodeError) as exc: + errors.append(f"{name}: {exc}") + if errors: + raise CheckFailure("\n".join(errors)) + + +def metadata_checks(root, changed): + """Enforce the version contracts that must be changed together.""" + if not set(changed).intersection({"pyproject.toml", "CITATION.cff", "README.md"}): + return + if not (root / "pyproject.toml").exists(): + return + project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))["project"] + version = str(project["version"]) + if (root / "CITATION.cff").exists(): + citation = load_yaml((root / "CITATION.cff").read_text(encoding="utf-8")) + if str(citation.get("version")) != version: + raise CheckFailure(f"CITATION.cff version must match pyproject.toml ({version}).") + if (root / "README.md").exists(): + readme = (root / "README.md").read_text(encoding="utf-8") + for block in re.findall(r"@software\{.*?(?=\n\})", readme, re.DOTALL | re.IGNORECASE): + match = re.search(r"\bversion\s*=\s*[\{\"]([^}\"]+)", block, re.IGNORECASE) + if match and match[1] != version: + raise CheckFailure( + f"README.md software citation version {match[1]} must be {version}." + ) + + +def lint(root, changed, lines, config): + """Run the pinned Ruff, retaining each package's existing lint scope.""" + paths = [ + p + for p in changed + if p.endswith(".py") + and (root / p).is_file() + and any(fnmatch.fnmatch(p, pattern) for pattern in config["lint_paths"]) + ] + if not paths: + return + command = [sys.executable, "-m", "ruff", "check", "--output-format", "json"] + if config.get("lint_select"): + command += ["--select", config["lint_select"]] + result = subprocess.run(command + paths, cwd=root, capture_output=True, check=False) + if result.returncode not in {0, 1}: + raise CheckFailure(result.stderr.decode("utf-8", "replace")) + findings = json.loads(result.stdout or b"[]") + errors = [] + for finding in findings: + relative = Path(finding["filename"]).relative_to(root).as_posix() + row = finding["location"]["row"] + if not config.get("lint_changed_lines") or row in lines.get(relative, set()): + errors.append(f"{relative}:{row}: {finding['code']} {finding['message']}") + if errors: + raise CheckFailure("\n".join(errors)) + + +def command_profile(root, config, phase, python, output): + """Execute the same explicit documentation/test command vectors as CI.""" + environment = os.environ.copy() + environment["PYTHONPATH"] = str(root / "src") + os.pathsep + str(root) + environment["MPLBACKEND"] = "Agg" + for step in config.get(phase, []): + args = [arg.replace("{output}", str(output)) for arg in step] + print(f"[{phase}] {' '.join(args)}", flush=True) + run([python, *args], root, env=environment) + + +def validate_gate(needs, required, optional=()): + """Fail closed if an expected job is absent, cancelled, failed or skipped.""" + errors = [] + for name in required: + result = needs.get(name, {}).get("result", "missing") + if result != "success": + errors.append(f"{name}: {result}") + for name in optional: + result = needs.get(name, {}).get("result", "missing") + if result not in {"success", "skipped"}: + errors.append(f"{name}: {result}") + if errors: + raise CheckFailure("Required checks did not pass: " + "; ".join(errors)) + + +def main(): + """Run a profile; hooks default to validating the exact staged index.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("profile", choices=["preflight", "docs", "ci", "doctor", "gate"]) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--revision") + parser.add_argument("--base", default="HEAD") + parser.add_argument("--working-tree", action="store_true") + parser.add_argument("--python", default=sys.executable) + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + started = time.monotonic() + root = args.repo.resolve() + if args.profile == "gate": + config = json.loads((root / PROFILE).read_text()) + needs = json.loads(os.environ["OSS_NEEDS"]) + required = list(config["required_jobs"]) + optional = [] + if "audit" in needs: + if needs.get("preflight", {}).get("outputs", {}).get("dependencies") == "true": + required.append("audit") + else: + optional.append("audit") + validate_gate(needs, required, optional) + print("All required checks passed.") + return + if args.profile == "doctor": + config = json.loads((root / PROFILE).read_text()) + print(f"OSS checks {VERSION}; package={config['repository']}; Python={args.python}") + print(f"Hook path: {git(root, 'config', '--get', 'core.hooksPath').decode().strip()}") + run([sys.executable, "-m", "ruff", "--version"], root) + run([sys.executable, "-m", "uv", "--version"], root) + print("Remote-only coverage: " + ", ".join(config["remote_only"])) + print(f"Guide: {GUIDE}") + return + output = ( + args.output_dir + or Path(os.environ.get("AGENT_LOCAL_ROOT", tempfile.gettempdir())) / "checks" + ) + output = output.resolve() + if output == root or root in output.parents: + raise CheckFailure("Check output must be outside the source checkout.") + output.mkdir(parents=True, exist_ok=True) + revision = args.revision + changed, lines = diff(root, revision, args.base) + identities = entries(root, revision) + digest = fingerprint(identities) + with tempfile.TemporaryDirectory(prefix="source-", dir=output) as temporary: + source = root if args.working_tree else Path(temporary) + if not args.working_tree: + export(root, identities, source) + config = json.loads((source / PROFILE).read_text(encoding="utf-8")) + if config["version"] != VERSION: + raise CheckFailure( + "Checker/profile versions differ; rerun the reviewed tooling update." + ) + if args.profile in {"preflight", "ci"}: + source_checks(source, changed) + metadata_checks(source, changed) + lint(source, changed, lines, config) + if any(p.startswith(".github/workflows/") for p in changed): + validator = os.environ.get("OSS_ACTIONLINT") + if not validator: + raise CheckFailure( + "Actionlint is missing. Run Install-CommitHooks.ps1 -All -SetupTools." + ) + # Explicit files work on source exports without a .git directory. + workflows = sorted((source / ".github/workflows").glob("*.y*ml")) + run([validator, "-shellcheck=", "-pyflakes=", *workflows], source) + if ( + set(changed).intersection({"pyproject.toml", "uv.lock"}) + and (source / "uv.lock").exists() + ): + run([sys.executable, "-m", "uv", "lock", "--check", "--offline"], source) + for check in config.get("preflight", []): + if any(fnmatch.fnmatch(p, pattern) for p in changed for pattern in check["paths"]): + command_profile( + source, {"check": [check["command"]]}, "check", args.python, output + ) + if args.profile in {"docs", "ci"}: + command_profile(source, config, "docs", args.python, output) + if args.profile == "ci": + command_profile(source, config, "tests", args.python, output) + if not args.working_tree and fingerprint(entries(root, revision)) != digest: + raise CheckFailure( + "Git selection changed during verification; commit again to check it." + ) + if os.environ.get("GITHUB_OUTPUT"): + dependency_change = bool( + set(changed).intersection({"pyproject.toml", "uv.lock", ".github/workflows/audit.yml"}) + ) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as handle: + handle.write(f"dependencies={str(dependency_change).lower()}\n") + elapsed = time.monotonic() - started + print( + f"PASS {args.profile}: {len(changed)} changed paths, tree {digest[:12]}, {elapsed:.1f}s", + flush=True, + ) + + +if __name__ == "__main__": + try: + main() + except (CheckFailure, OSError, KeyError, ValueError) as error: + print( + f"\nOSS check failed: {error}\nRepair the selected files and retry. Help: {GUIDE}", + file=sys.stderr, + ) + sys.exit(1) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e99fba8..3ad25d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,15 @@ # SHARED CI CORE v1.0 — T1 — synced 2026-08-22 name: CI +run-name: ${{ github.event_name == 'schedule' && 'Dependency compatibility (live resolution)' || 'CI' }} permissions: contents: read on: - push: - branches: [main] - pull_request: - branches: [main] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + schedule: + - cron: "37 3 * * *" + workflow_dispatch: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true @@ -47,7 +44,7 @@ jobs: # Retried because observed failures here are transport failures, not test failures. shell: bash run: | - sync_cmd=(uv sync --group test ${{ matrix.primary && '--locked' || '--upgrade' }}) + sync_cmd=(uv sync --group test ${{ github.event_name == 'schedule' && '--upgrade' || '--locked' }}) for attempt in 1 2 3; do if "${sync_cmd[@]}"; then exit 0; fi if [ "$attempt" -eq 3 ]; then echo "::error::uv sync failed after 3 attempts"; exit 1; fi @@ -128,6 +125,8 @@ jobs: run: python .github/scripts/check_stack_imports.py lowest-dependencies: + # Floor resolution is compatibility maintenance, not a verdict on an unrelated edit. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' name: lowest direct dependencies (Python 3.10, ${{ matrix.extras || 'core' }}) runs-on: ubuntu-latest timeout-minutes: 45 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e01f7be..6ace222 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,30 +5,13 @@ permissions: contents: read on: - push: - branches: [main] - paths: - - .github/workflows/docs.yml - - .github/scripts/*docs_redirects.py - - docs/** - - src/option_chain_analytics/** - - pyproject.toml - - uv.lock - pull_request: - branches: [main] - paths: - - .github/workflows/docs.yml - - .github/scripts/*docs_redirects.py - - docs/** - - src/option_chain_analytics/** - - pyproject.toml - - uv.lock + workflow_call: + inputs: + online: + type: boolean + default: false workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true MPLBACKEND: Agg @@ -51,17 +34,20 @@ jobs: - name: Verify redirect path and fragment handling run: uv run --no-sync python .github/scripts/test_docs_redirects.py - - name: Build HTML with warnings as errors - run: uv run --no-sync python -m sphinx -E -W --keep-going -b html docs docs/_build/html + - name: Build documentation with the shared profile + + run: uv run --no-sync python .github/oss_checks.py docs --working-tree --output-dir "$RUNNER_TEMP/oss-docs" - name: Check documentation links + + if: inputs.online || github.event_name == 'workflow_dispatch' run: uv run --no-sync python -m sphinx -E -W -b linkcheck docs docs/_build/linkcheck # Publish only redirects on GitHub Pages; Read the Docs serves the full site. - name: Build legacy documentation redirects run: >- uv run --no-sync python .github/scripts/build_docs_redirects.py - --source docs/_build/html + --source "$RUNNER_TEMP/oss-docs/html" --output docs/_build/redirects --base-url https://optionchainanalytics.readthedocs.io/en/latest/ --project-prefix /OptionChainAnalytics/ diff --git a/.github/workflows/link-health.yml b/.github/workflows/link-health.yml new file mode 100644 index 0000000..aa42afe --- /dev/null +++ b/.github/workflows/link-health.yml @@ -0,0 +1,16 @@ +name: External documentation health +on: + schedule: + - cron: "23 6 * * *" + workflow_dispatch: +permissions: + contents: read +jobs: + links: + uses: ./.github/workflows/docs.yml + with: + online: true + permissions: + contents: read + pages: write + id-token: write diff --git a/.github/workflows/required.yml b/.github/workflows/required.yml new file mode 100644 index 0000000..f25559b --- /dev/null +++ b/.github/workflows/required.yml @@ -0,0 +1,66 @@ +name: Required checks +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: required-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +jobs: + preflight: + name: Staged-source contracts + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + dependencies: ${{ steps.check.outputs.dependencies }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + python-version: "3.12" + version: "0.12.13" + - name: Install the checksum-verified workflow validator + run: | + curl --fail --location --retry 3 https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz -o "$RUNNER_TEMP/actionlint.tar.gz" + echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $RUNNER_TEMP/actionlint.tar.gz" | sha256sum --check + tar -xzf "$RUNNER_TEMP/actionlint.tar.gz" -C "$RUNNER_TEMP" actionlint + echo "OSS_ACTIONLINT=$RUNNER_TEMP/actionlint" >> "$GITHUB_ENV" + - name: Prepare the locked package interpreter for package-owned source checks + run: uv sync --locked --group test + - name: Check selected source and metadata + id: check + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then + BASE=$(git rev-parse HEAD^ 2>/dev/null || git rev-parse HEAD) + fi + uv run --no-project --with-requirements .github/oss-checks-requirements.txt python .github/oss_checks.py preflight --working-tree --revision HEAD --base "$BASE" --python "$GITHUB_WORKSPACE/.venv/bin/python" --output-dir "$RUNNER_TEMP/oss-checks" + ci: + uses: ./.github/workflows/ci.yml + docs: + uses: ./.github/workflows/docs.yml + permissions: + contents: read + pages: write + id-token: write + required: + name: Required checks + if: always() + needs: [preflight, ci, docs] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Fail closed unless every applicable component passed + env: + OSS_NEEDS: ${{ toJSON(needs) }} + run: python .github/oss_checks.py gate From 1393f38c21ff26e195ad87364901f2d7968f51a0 Mon Sep 17 00:00:00 2001 From: ArturSepp Date: Mon, 21 Sep 2026 11:16:52 +0200 Subject: [PATCH 2/3] ci: complete reference and consumer validation for Desktop workflow --- .githooks/pre-commit | 0 .github/GITHUB_DESKTOP.md | 3 + .github/check_new_references.py | 117 +++++++++++++++++++++++++++++++ .github/oss-checks.json | 15 +++- .github/oss_checks.py | 31 ++++++-- .github/workflows/downstream.yml | 68 ++++++++++++++++++ .github/workflows/required.yml | 30 +++++++- 7 files changed, 255 insertions(+), 9 deletions(-) mode change 100644 => 100755 .githooks/pre-commit create mode 100644 .github/check_new_references.py create mode 100644 .github/workflows/downstream.yml diff --git a/.githooks/pre-commit b/.githooks/pre-commit old mode 100644 new mode 100755 diff --git a/.github/GITHUB_DESKTOP.md b/.github/GITHUB_DESKTOP.md index b3ad443..9f801fc 100644 --- a/.github/GITHUB_DESKTOP.md +++ b/.github/GITHUB_DESKTOP.md @@ -2,6 +2,9 @@ *Author: [Artur Sepp](https://github.com/ArturSepp)* +Project: [option_chain_analytics](https://github.com/ArturSepp/OptionChainAnalytics). +Software citation: [CITATION.cff](https://github.com/ArturSepp/OptionChainAnalytics/blob/main/CITATION.cff). + Create a working branch, select the intended changes, and commit normally. The installed hook checks the selected contents. Push the branch, open a pull request, and merge after **Required checks** passes. Main is protected; a failing branch does not change main. diff --git a/.github/check_new_references.py b/.github/check_new_references.py new file mode 100644 index 0000000..334a1f6 --- /dev/null +++ b/.github/check_new_references.py @@ -0,0 +1,117 @@ +"""Check newly added public references without gating on unrelated server outages.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import time +from html.parser import HTMLParser +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urldefrag +from urllib.request import Request, urlopen + +MAX_BODY = 4 * 1024 * 1024 + + +class Anchors(HTMLParser): + """Collect static anchors, using the same rendered IDs a link checker can see.""" + + def __init__(self): + super().__init__() + self.names = set() + + def handle_starttag(self, tag, attrs): + for name, value in attrs: + if name in {"id", "name"} and value: + self.names.add(value) + + +def added_urls(patch): + """Extract external URLs only from added Markdown/RST lines.""" + urls = set() + for line in patch.splitlines(): + if not line.startswith("+") or line.startswith("+++"): + continue + for candidate in re.findall(r'https?://[^\s<>"\x27]+', line[1:]): + # Preserve balanced parentheses inside URLs, removing Markdown's closing wrapper. + candidate = candidate.rstrip(".,;]}") + while candidate.endswith(")") and candidate.count(")") > candidate.count("("): + candidate = candidate[:-1] + if ( + "{" not in candidate + and "PACKAGE" not in candidate + and "REPOSITORY" not in candidate + ): + urls.add(candidate) + return sorted(urls) + + +def inspect_url(url): + """Classify a confirmed broken reference separately from temporary unavailability.""" + address, fragment = urldefrag(url) + last = "unavailable" + for attempt in range(2): + try: + request = Request(address, headers={"User-Agent": "OSS-documentation-check/1.0"}) + with urlopen(request, timeout=15) as response: + kind = response.headers.get("Content-Type", "") + if not fragment or "html" not in kind: + return {"url": url, "status": "ok"} + body = response.read(MAX_BODY + 1) + if len(body) > MAX_BODY: + return {"url": url, "status": "deferred", "reason": "large HTML requires review"} + parser = Anchors() + parser.feed(body.decode("utf-8", "replace")) + if unquote(fragment) in parser.names: + return {"url": url, "status": "ok"} + last = f"static anchor #{fragment} not found" + except HTTPError as error: + if error.code not in {404, 410}: + return {"url": url, "status": "deferred", "reason": f"HTTP {error.code}"} + last = f"HTTP {error.code}" + except (URLError, TimeoutError, OSError) as error: + return {"url": url, "status": "deferred", "reason": str(error)} + if attempt == 0: + time.sleep(0.5) + return {"url": url, "status": "broken", "reason": last} + + +def main(): + """Check introduced references; emit a machine-readable maintenance report.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True) + parser.add_argument("--head", default="HEAD") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + patch = subprocess.check_output( + [ + "git", + "diff", + "--no-ext-diff", + "--unified=0", + args.base, + args.head, + "--", + "*.md", + "*.rst", + ], + text=True, + encoding="utf-8", + ) + results = [inspect_url(url) for url in added_urls(patch)] + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + for result in results: + print(f"{result['status']}: {result['url']} {result.get('reason', '')}") + deferred = sum(item["status"] == "deferred" for item in results) + print( + f"Checked {len(results)} introduced references; {deferred} need external-health follow-up." + ) + raise SystemExit(1 if any(item["status"] == "broken" for item in results) else 0) + + +if __name__ == "__main__": + main() diff --git a/.github/oss-checks.json b/.github/oss-checks.json index 309e6ad..2030e0e 100644 --- a/.github/oss-checks.json +++ b/.github/oss-checks.json @@ -41,7 +41,8 @@ "required_jobs": [ "preflight", "ci", - "docs" + "docs", + "references" ], "workflow_inventory": { "ci": { @@ -98,5 +99,15 @@ "Deploy GitHub Pages" ] } - } + }, + "audit": false, + "consumers": [ + { + "repository": "ArturSepp/StochVolModels", + "module": "stochvolmodels", + "extra": "research", + "sdk": false, + "commit": "2d90efba42ed9c8db71c59844208ca7898554686" + } + ] } diff --git a/.github/oss_checks.py b/.github/oss_checks.py index 7ab9253..c6c58c9 100644 --- a/.github/oss_checks.py +++ b/.github/oss_checks.py @@ -156,7 +156,7 @@ def source_checks(root, changed): continue try: text = path.read_text(encoding="utf-8-sig") - if re.search(r"^(?:<{7} |={7}$|>{7} )", text, re.MULTILINE): + if re.search(r"^(?:<{7} |>{7} )", text, re.MULTILINE): raise ValueError("unresolved merge-conflict markers") if suffix == ".py": ast.parse(text, filename=name) @@ -202,7 +202,14 @@ def metadata_checks(root, changed): raise CheckFailure(f"CITATION.cff version must match pyproject.toml ({version}).") if (root / "README.md").exists(): readme = (root / "README.md").read_text(encoding="utf-8") + repositories = [ + url.rstrip("/").lower() + for url in project.get("urls", {}).values() + if "github.com/" in url + ] for block in re.findall(r"@software\{.*?(?=\n\})", readme, re.DOTALL | re.IGNORECASE): + if repositories and not any(url in block.lower() for url in repositories): + continue match = re.search(r"\bversion\s*=\s*[\{\"]([^}\"]+)", block, re.IGNORECASE) if match and match[1] != version: raise CheckFailure( @@ -282,11 +289,15 @@ def main(): needs = json.loads(os.environ["OSS_NEEDS"]) required = list(config["required_jobs"]) optional = [] - if "audit" in needs: - if needs.get("preflight", {}).get("outputs", {}).get("dependencies") == "true": - required.append("audit") - else: - optional.append("audit") + for name, enabled, output_name in ( + ("audit", config.get("audit", False), "dependencies"), + ("downstream", bool(config.get("consumers")), "api"), + ): + if enabled: + if needs.get("preflight", {}).get("outputs", {}).get(output_name) == "true": + required.append(name) + else: + optional.append(name) validate_gate(needs, required, optional) print("All required checks passed.") return @@ -357,6 +368,14 @@ def main(): ) with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as handle: handle.write(f"dependencies={str(dependency_change).lower()}\n") + api_change = any( + path.startswith("src/") + and path.endswith(".py") + and "/tests/" not in path + and "/run_local/" not in path + for path in changed + ) + handle.write(f"api={str(api_change).lower()}\n") elapsed = time.monotonic() - started print( f"PASS {args.profile}: {len(changed)} changed paths, tree {digest[:12]}, {elapsed:.1f}s", diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml new file mode 100644 index 0000000..a1261cf --- /dev/null +++ b/.github/workflows/downstream.yml @@ -0,0 +1,68 @@ +name: Consumer compatibility +on: + workflow_call: + workflow_dispatch: +permissions: + contents: read +jobs: + plan: + runs-on: ubuntu-latest + outputs: + consumers: ${{ steps.matrix.outputs.consumers }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - id: matrix + run: | + python - <<'PY' + import json, os + from pathlib import Path + consumers = json.loads(Path('.github/oss-checks.json').read_text())['consumers'] + assert consumers, 'Expected at least one registered consumer' + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write('consumers=' + json.dumps(consumers) + '\n') + PY + smoke: + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.consumers) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: candidate + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.commit }} + path: consumer + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + python-version: "3.12" + version: "0.12.13" + - name: Use the pinned consumer environment + env: + EXTRA: ${{ matrix.extra }} + run: | + echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/consumer-env" >> "$GITHUB_ENV" + export UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/consumer-env" + if [ -n "$EXTRA" ]; then + uv sync --project consumer --locked --group test --extra "$EXTRA" + else + uv sync --project consumer --locked --group test + fi + - name: Install the terminal-free SDK for the Bloomberg edge + if: matrix.sdk + run: uv pip install --python "$UV_PROJECT_ENVIRONMENT/bin/python" --index-url https://blpapi.bloomberg.com/repository/releases/python/simple blpapi + - name: Install the proposed package over the pinned baseline + run: | + uv pip install --python "$UV_PROJECT_ENVIRONMENT/bin/python" --reinstall --no-deps ./candidate + uv pip check --python "$UV_PROJECT_ENVIRONMENT/bin/python" + - name: Import the actual consumer + env: + IMPORT_NAME: ${{ matrix.module }} + run: | + cd "$RUNNER_TEMP" + "$UV_PROJECT_ENVIRONMENT/bin/python" -c 'import importlib, os; module = importlib.import_module(os.environ["IMPORT_NAME"]); print(module.__file__)' diff --git a/.github/workflows/required.yml b/.github/workflows/required.yml index f25559b..fcb7451 100644 --- a/.github/workflows/required.yml +++ b/.github/workflows/required.yml @@ -19,6 +19,7 @@ jobs: timeout-minutes: 10 outputs: dependencies: ${{ steps.check.outputs.dependencies }} + api: ${{ steps.check.outputs.api }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -52,10 +53,37 @@ jobs: contents: read pages: write id-token: write + downstream: + needs: preflight + if: needs.preflight.outputs.api == 'true' + uses: ./.github/workflows/downstream.yml + references: + name: Introduced documentation references + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + - name: Reject confirmed broken new references + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then + BASE=$(git rev-parse HEAD^ 2>/dev/null || git rev-parse HEAD) + fi + python .github/check_new_references.py --base "$BASE" --head HEAD --output "$RUNNER_TEMP/reference-health.json" + - name: Retain reference-health evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: introduced-reference-health + path: ${{ runner.temp }}/reference-health.json + if-no-files-found: ignore required: name: Required checks if: always() - needs: [preflight, ci, docs] + needs: [references, downstream, preflight, ci, docs] runs-on: ubuntu-latest timeout-minutes: 5 steps: From 937a42a31af386e86d3c7655dd3e5796a8393ed9 Mon Sep 17 00:00:00 2001 From: ArturSepp Date: Mon, 21 Sep 2026 11:26:36 +0200 Subject: [PATCH 3/3] Handle Markdown emphasis when checking new references --- .github/check_new_references.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/check_new_references.py b/.github/check_new_references.py index 334a1f6..9b91bab 100644 --- a/.github/check_new_references.py +++ b/.github/check_new_references.py @@ -35,11 +35,18 @@ def added_urls(patch): for line in patch.splitlines(): if not line.startswith("+") or line.startswith("+++"): continue - for candidate in re.findall(r'https?://[^\s<>"\x27]+', line[1:]): - # Preserve balanced parentheses inside URLs, removing Markdown's closing wrapper. - candidate = candidate.rstrip(".,;]}") - while candidate.endswith(")") and candidate.count(")") > candidate.count("("): - candidate = candidate[:-1] + for candidate in re.findall(r'https?://[^\s<>`"\x27]+', line[1:]): + # Stop at the link's closing parenthesis, including when emphasis follows it. + depth = 0 + for index, character in enumerate(candidate): + if character == "(": + depth += 1 + elif character == ")": + if depth == 0: + candidate = candidate[:index] + break + depth -= 1 + candidate = candidate.rstrip(".,;]}*") if ( "{" not in candidate and "PACKAGE" not in candidate