diff --git a/.github/workflows/jvm-diagnostic.yml b/.github/workflows/jvm-diagnostic.yml new file mode 100644 index 0000000..1d0b63e --- /dev/null +++ b/.github/workflows/jvm-diagnostic.yml @@ -0,0 +1,41 @@ +name: JVM warm-up diagnostic +on: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: jvm-diagnostic + cancel-in-progress: false +jobs: + diagnose: + if: >- + github.repository == 'tappe9/simple-api-benchmark' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && + github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + timeout-minutes: 3 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + timeout-minutes: 3 + with: + python-version-file: apps/python-fastapi/.python-version + - name: Run one fixed JVM diagnostic, reserving cleanup and upload time + run: timeout --signal=TERM --kill-after=120s 104m python -m benchmark.jvm_diagnostic + - name: Retain complete JVM diagnostic evidence, including failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + timeout-minutes: 7 + with: + name: jvm-diagnostic-${{ github.run_id }}-${{ github.run_attempt }} + path: .cache/jvm-diagnostic/ + include-hidden-files: true + retention-days: 90 + if-no-files-found: error diff --git a/benchmark/jvm_diagnostic.py b/benchmark/jvm_diagnostic.py new file mode 100644 index 0000000..73890ef --- /dev/null +++ b/benchmark/jvm_diagnostic.py @@ -0,0 +1,268 @@ +"""One fixed, non-publishing study of the five-second warm-up budget.""" + +import argparse +import copy +import hashlib +import os +import re +import signal +import sys +import uuid +from pathlib import Path +from statistics import median + +from .contract_runner import protect_cleanup +from .contract_test import load_cases, run_contract +from .definition import PROFILE +from .healthcheck import EXTERNAL_READINESS +from .results import atomic_json, number, require +from .run import now + +ROOT = Path(__file__).resolve().parents[1] +CACHE_ROOT = ROOT / ".cache" / "jvm-diagnostic" +MEMBERS = ("java-spring-boot", "node-fastify", "go-gin") +ORDERS = tuple(MEMBERS[i:] + MEMBERS[:i] for i in range(3)) +RULES = { + "late_rps_range_over_median_max": 0.05, + "late_p95_range_over_median_max": 0.10, + "first_rps_relative_difference_max": 0.05, + "reject_strict_monotonic_late_trend": True, + "all_three_repeats_required": True, +} + + +def validate_directory(path: Path) -> Path: + candidate = path.absolute() + root = CACHE_ROOT.absolute() + require( + candidate != root and candidate.is_relative_to(root), + "output must be below diagnostic cache", + ) + require(candidate.resolve().is_relative_to(root.resolve()), "output escapes diagnostic cache") + require(".." not in candidate.parts, "parent traversal in diagnostic path") + for component in (candidate, *candidate.parents): + require(not component.is_symlink(), "symlink in diagnostic output path") + require(not candidate.exists(), "diagnostic attempt already exists; never overwrite or resume") + return candidate + + +def analyze_windows(windows: list[dict]) -> dict: + require(len(windows) == 4, "exactly four consecutive windows required") + values = {} + for field in ("requests_per_second", "p95_response_time_ms"): + values[field] = [number(w[field], field, positive=True) for w in windows] + rps = values["requests_per_second"] + p95 = values["p95_response_time_ms"] + + def spread(series): + return (max(series[1:]) - min(series[1:])) / median(series[1:]) + + def trend(series): + a, b, c = series[1:] + return a < b < c or a > b > c + + rps_spread, latency_spread = spread(rps), spread(p95) + late_trend = trend(rps) or trend(p95) + stable = rps_spread <= 0.05 and latency_spread <= 0.10 and not late_trend + difference = abs(rps[0] - median(rps[1:])) / median(rps[1:]) + return { + "late_rps_range_over_median": rps_spread, + "late_p95_range_over_median": latency_spread, + "strict_monotonic_late_trend": late_trend, + "first_rps_relative_difference": difference, + "late_stable": stable, + "five_second_supported": stable and difference <= 0.05, + } + + +def write_checksums(directory: Path) -> None: + entries = {} + for path in sorted(directory.rglob("*")): + require(not path.is_symlink(), "symlink in diagnostic evidence") + if path.is_file() and path.name != "checksums.json": + entries[str(path.relative_to(directory))] = hashlib.sha256( + path.read_bytes() + ).hexdigest() + atomic_json(directory / "checksums.json", {"algorithm": "sha256", "files": entries}) + + +def run_study(directory: Path, factory, *, metadata: dict, contract=run_contract) -> dict: + directory = validate_directory(directory) + for key in ("source_commit", "source_tree"): + require( + type(metadata.get(key)) is str and re.fullmatch(r"[0-9a-f]{40}", metadata[key]), + "missing source identity: " + key, + ) + directory.mkdir(parents=True) + conditions = copy.deepcopy(PROFILE) + conditions.update(runs=4) + result = { + "schema_version": 1, + "mode": "jvm-warmup-diagnostic", + "official": False, + "publishable": False, + "status": "running", + "started_at": now(), + "conditions": conditions, + "criteria": copy.deepcopy(RULES), + "metadata": copy.deepcopy(metadata), + "attempts": [ + { + "round": round_index, + "implementation": identifier, + "status": "not-started", + "endpoints": [], + } + for round_index, order in enumerate(ORDERS, 1) + for identifier in order + ], + } + output = directory / "diagnostic.json" + + def persist(): + atomic_json(output, result) + + persist() + try: + for index, attempt in enumerate(result["attempts"], 1): + identifier = attempt["implementation"] + attempt.update(status="running", started_at=now()) + persist() + environment = None + failure = None + try: + attempt_root = directory / f"{index:02d}-{identifier}" + environment = factory(attempt_root) + require( + environment.health_policy == EXTERNAL_READINESS, "external readiness required" + ) + require( + environment.connections == 50 and environment.request_timeout == 15, + "fixed load configuration required", + ) + attempt["artifact_directory"] = str(environment.artifacts.relative_to(directory)) + persist() + environment.build(identifier) + attempt["container"] = environment.start(identifier) + attempt["readiness"] = copy.deepcopy(environment.readiness) + checks = contract("http://127.0.0.1:8080", implementation=identifier) + require( + type(checks) is int and checks == 2 * len(load_cases()), + "shared contract incomplete", + ) + attempt["contract_checks"] = checks + persist() + environment.check() + for endpoint in PROFILE["endpoints"]: + entry = {"endpoint": endpoint, "windows": []} + attempt["endpoints"].append(entry) + persist() + entry["warmup"] = environment.measure(endpoint, 5, 0) + persist() + for window in range(1, 5): + entry["windows"].append(environment.measure(endpoint, 30, window)) + persist() + entry["analysis"] = analyze_windows(entry["windows"]) + persist() + environment.check() + except BaseException as error: + failure = error + attempt.update(status="failed", error=f"{type(error).__name__}: {error}") + persist() + raise + finally: + try: + if environment is not None: + with protect_cleanup(): + environment.cleanup() + attempt["cleanup"] = "passed" + except BaseException as cleanup_error: + attempt.update( + status="failed", cleanup="failed", cleanup_error=str(cleanup_error) + ) + if failure is None: + raise + finally: + attempt["completed_at"] = now() + persist() + attempt["status"] = "completed" + persist() + result["assessment"] = { + identifier: { + "five_second_supported_for_all_observed_endpoints_and_repeats": all( + entry["analysis"]["five_second_supported"] + for attempt in result["attempts"] + if attempt["implementation"] == identifier + for entry in attempt["endpoints"] + ) + } + for identifier in MEMBERS + } + result["status"] = "completed" + except BaseException as error: + result.update(status="failed", error=f"{type(error).__name__}: {error}") + raise + finally: + result["completed_at"] = now() + persist() + write_checksums(directory) + return result + + +def main(argv=None) -> int: + from .environment import provenance, registered_pinned_versions + from .install_oha import ensure_oha + from .jvm_environment import DiagnosticEnvironment + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compose", default="docker compose") + args = parser.parse_args(argv) + run_directory = CACHE_ROOT / ("study-" + uuid.uuid4().hex) + # Separate bootstrap evidence survives dependency/provenance failures before any load. + CACHE_ROOT.mkdir(parents=True, exist_ok=True) + bootstrap = CACHE_ROOT / (run_directory.name + "-bootstrap.json") + context = { + "started_at": now(), + "status": "starting", + "official": False, + "publishable": False, + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + "github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + } + atomic_json(bootstrap, context) + + def interrupted(signum, _frame): + raise KeyboardInterrupt(f"received signal {signum}") + + previous = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + oha = ensure_oha() + metadata = provenance(oha) + versions = registered_pinned_versions() + metadata["versions"] = {identifier: versions[identifier] for identifier in MEMBERS} + metadata.update( + github_run_id=context["github_run_id"], + github_run_attempt=context["github_run_attempt"], + jit_gc_evidence="Unavailable: production JRE unchanged; no instrumentation or extra load runs.", + ) + run_study( + run_directory, + lambda root: DiagnosticEnvironment(oha, root, compose=args.compose), + metadata=metadata, + ) + context["status"] = "completed" + print(f"Diagnostic completed: {run_directory}; no official results published.") + return 0 + except (Exception, KeyboardInterrupt) as error: + context.update(status="failed", error=f"{type(error).__name__}: {error}") + print(f"Diagnostic failed without retry: {error}", file=sys.stderr) + return 1 + finally: + context["completed_at"] = now() + atomic_json(bootstrap, context) + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmark/jvm_environment.py b/benchmark/jvm_environment.py new file mode 100644 index 0000000..8705455 --- /dev/null +++ b/benchmark/jvm_environment.py @@ -0,0 +1,273 @@ +"""Non-publishing JVM diagnostic evidence without changing official measurements. + +The inherited project lifecycle and resource checks remain unchanged. The only +load-window observer is the same sequential Docker stats command used by the +common environment, retaining its CPU field as well as its memory field. +""" + +import contextlib +import json +import math +import re +import time +from datetime import datetime, timezone +from pathlib import Path + +from .environment import DockerEnvironment, memory_bytes +from .healthcheck import EXTERNAL_READINESS +from .process import execute +from .results import parse_oha, require, strict_json + + +class DiagnosticEnvironment(DockerEnvironment): + """Retain raw window evidence and scoped logs, including failed attempts.""" + + def __init__(self, oha: Path, artifacts: Path, **kwargs): + kwargs.setdefault("health_policy", EXTERNAL_READINESS) + require(kwargs.get("connections", 50) == 50, "diagnostic connections must remain 50") + require( + kwargs.get("request_timeout", 15) == 15, "diagnostic request timeout must remain 15" + ) + require( + kwargs["health_policy"] == EXTERNAL_READINESS, "diagnostic requires external readiness" + ) + require(kwargs.get("audit_health_events", False) is False, "diagnostic adds no event audit") + super().__init__(oha, artifacts, **kwargs) + + def _failure(self, stage: str, error: BaseException) -> None: + """Best-effort evidence must not replace the original failure. + + execute exposes the final 4,000 output characters for a nonzero exit; + timeout/interruption output is not available from the shared helper. + """ + try: + (self.artifacts / f"{self.implementation}-{stage}-failure.log").write_text( + f"{type(error).__name__}: {error}\n", encoding="utf-8" + ) + try: + output = execute( + self.prefix + + [ + "logs", + "--no-color", + "--timestamps", + "--tail", + "200", + self.implementation, + "postgres", + ], + timeout=15, + ) + except Exception as log_error: + output = f"Container logs unavailable: {type(log_error).__name__}: {log_error}\n" + (self.artifacts / f"{self.implementation}-{stage}-container.log").write_text( + output, encoding="utf-8" + ) + except Exception: + # A full/unwritable disk must not mask the failure that stopped load. + pass + + def build(self, implementation: str) -> None: + self.implementation = implementation + try: + super().build(implementation) + except BaseException as error: + with contextlib.suppress(OSError): + (self.artifacts / f"{implementation}-build.log").write_text( + f"{type(error).__name__}: {error}\n", encoding="utf-8" + ) + self._failure("build", error) + raise + + def start(self, implementation: str) -> dict: + self.implementation = implementation + try: + context = super().start(implementation) + state = self.inspect() + # Never persist full inspect or Config.Env, which includes credentials. + context.update( + limits={ + "nano_cpus": state["HostConfig"]["NanoCpus"], + "memory_bytes": state["HostConfig"]["Memory"], + }, + container_started_at=state["State"]["StartedAt"], + restart_count=state["RestartCount"], + health_policy=self.health_policy, + connections=self.connections, + request_timeout_seconds=self.request_timeout, + ) + (self.artifacts / f"{implementation}-context.json").write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + return context + except BaseException as error: + self._failure("start", error) + raise + + def measure(self, endpoint: str, duration: int, index: int) -> dict: + """Use the common oha contract for each diagnostic window, including 0/4.""" + require(type(index) is int and 0 <= index <= 4, "window index must be 0 through 4") + require(type(duration) is int and duration > 0, "window duration must be positive") + require( + endpoint in ("/json", "/db/42", "/cpu"), + "diagnostic endpoint must be /json, /db/42 or /cpu", + ) + label = endpoint.strip("/").replace("/", "-") + f"-window-{index}" + path = self.artifacts / f"{self.implementation}-{label}.json" + samples = [] + cpu_samples = [] + started = datetime.now(timezone.utc) + clock_started = time.monotonic() + completed = None + elapsed = None + status = "failed" + failure = None + result = None + try: + self.check() + with ( + path.with_suffix(".stats.jsonl").open("w", encoding="utf-8") as stats_log, + path.with_suffix(".memory.jsonl").open("w", encoding="utf-8") as memory_log, + ): + + def sample(): + self.check() + sample_started = datetime.now(timezone.utc) + raw = execute( + [ + "docker", + "stats", + "--no-stream", + "--no-trunc", + "--format", + "{{json .}}", + self.container, + ], + timeout=8, + ) + sample_completed = datetime.now(timezone.utc) + # Write before parsing so malformed samples survive a failure. + stats_log.write( + json.dumps( + { + "started_at": sample_started.isoformat(), + "completed_at": sample_completed.isoformat(), + "raw": raw, + } + ) + + "\n" + ) + stats_log.flush() + memory = memory_bytes(raw, self.container) + cpu_text = strict_json(raw.encode()).get("CPUPerc") + require( + type(cpu_text) is str + and re.fullmatch(r"[0-9]+(?:\.[0-9]+)?%", cpu_text) is not None, + "Docker CPU sample has missing/invalid CPUPerc", + ) + cpu = float(cpu_text[:-1]) + require(math.isfinite(cpu), "Docker CPU sample is not finite") + samples.append(memory) + cpu_samples.append(cpu) + memory_log.write( + json.dumps( + { + "at": sample_completed.isoformat(), + "bytes": memory, + "cpu_percent": cpu, + "container_id": self.container, + } + ) + + "\n" + ) + memory_log.flush() + + started = datetime.now(timezone.utc) + clock_started = time.monotonic() + try: + output = execute( + [ + str(self.oha), + "--no-tui", + "--output-format", + "json", + "--output", + str(path), + "--http-version", + "1.1", + "--redirect", + "0", + "--disable-compression", + "-c", + str(self.connections), + "-z", + f"{duration}s", + "-w", + "-t", + f"{self.request_timeout}s", + "--connect-timeout", + "5s", + "http://127.0.0.1:8080" + endpoint, + ], + timeout=duration + self.request_timeout + 15, + tick=sample, + ) + except BaseException as error: + with contextlib.suppress(OSError): + path.with_suffix(".load.log").write_text( + f"{type(error).__name__}: {error}\n", encoding="utf-8" + ) + raise + finally: + completed = datetime.now(timezone.utc) + elapsed = time.monotonic() - clock_started + path.with_suffix(".load.log").write_text(output, encoding="utf-8") + self.check() + require(bool(samples), "memory/CPU collection produced no samples") + require( + path.is_file() and path.stat().st_size <= 1024 * 1024, + "oha result missing or oversized", + ) + raw = path.read_bytes() + result = parse_oha(raw, duration=duration, request_timeout=self.request_timeout) + result.update( + run=max(index, 1), + window_index=index, + p95_response_time_ms=strict_json(raw)["latencyPercentiles"]["p95"] * 1000, + peak_memory_bytes=max(samples), + memory_samples=len(samples), + cpu_percent_samples=cpu_samples, + started_at=started.isoformat(), + completed_at=completed.isoformat(), + duration_wall_seconds=elapsed, + ) + status = "complete" + return result + except BaseException as error: + failure = f"{type(error).__name__}: {error}" + self._failure(label, error) + raise + finally: + if completed is None: + completed = datetime.now(timezone.utc) + elapsed = time.monotonic() - clock_started + window = { + "implementation": self.implementation, + "endpoint": endpoint, + "window_index": index, + "requested_duration_seconds": duration, + "started_at": started.isoformat(), + "completed_at": completed.isoformat(), + "duration_wall_seconds": elapsed, + "status": status, + "error": failure, + "memory_samples": len(samples), + "cpu_percent_samples": cpu_samples, + } + try: + path.with_suffix(".window.json").write_text( + json.dumps(window, indent=2) + "\n", encoding="utf-8" + ) + except OSError: + if failure is None: + raise diff --git a/docs/JAVA-SPRING-BOOT.md b/docs/JAVA-SPRING-BOOT.md index c50a69f..47cec85 100644 --- a/docs/JAVA-SPRING-BOOT.md +++ b/docs/JAVA-SPRING-BOOT.md @@ -127,3 +127,6 @@ five-second warm-up or change existing eight-stack results. Any justified real official run, automatic audited publication and Pages update still require explicit authorization. Publication/protection redesign and Issue #59's live producer-to-Pages verification remain separate work. + +The bounded [JVM warm-up diagnostic](JVM-DIAGNOSTIC.md) records the predeclared +Issue #73 experiment, interpretation rules and remaining decision boundary. diff --git a/docs/JVM-DIAGNOSTIC.md b/docs/JVM-DIAGNOSTIC.md new file mode 100644 index 0000000..76eee8e --- /dev/null +++ b/docs/JVM-DIAGNOSTIC.md @@ -0,0 +1,131 @@ +# JVM warm-up diagnostic (Issue #73) + +This is a predeclared, bounded diagnostic of the current five-second warm-up +budget, not an official comparison or permission to admit Java to a cohort. +Java remains registered-only. Existing profiles, cohort definitions, applications, +dependency pins, published reports and historical results are unchanged. + +## Question and fixed plan + +Does throughput after the fixed warm-up resemble later windows on the same +process, across three clean starts, under this particular host and endpoint order? +The controls are Node / Fastify (another JIT runtime) and Go / Gin (a non-JIT +runtime). They help identify host/common effects but cannot identify a JVM cause +by themselves. Never compare these rows directly with the old eight-stack report. + +The three rounds use these orders, all sequentially in one job: + +1. Java / Spring Boot → Node / Fastify → Go / Gin +2. Node / Fastify → Go / Gin → Java / Spring Boot +3. Go / Gin → Java / Spring Boot → Node / Fastify + +Each of the nine attempts owns a fresh Compose project, API process and PostgreSQL +tmpfs database initialized from the committed fixture. Images use committed pins +and Docker build caching may be reused; this is not a cold image-build benchmark. +The API has 1 CPU and 512 MiB, the pool maximum is 10, and load uses 50 connections, +HTTP/1.1, the pinned oha and the existing 15-second request timeout. The API's +recurring healthcheck is disabled only in the owned measurement project. External +readiness stops before the shared two-round correctness contract and load. + +Within each process the endpoint order is `/json`, `/db/42`, `/cpu`. Each endpoint +gets a five-second warm-up, followed by four consecutive 30-second sending windows. +All windows, including warm-up, are retained. No median-winning run is selected +or discarded. Each oha invocation drains outstanding requests; state checks, +metrics and orchestration introduce gaps. Saved wall-clock timestamps expose +these gaps. Thus the windows are successive invocations, not a single gapless load. +The shared contract itself exercises the process before the five-second warm-up: +this diagnoses the existing contract-first sequence, not untouched cold startup. +Later endpoints inherit prior endpoint activity; a result cannot establish +independent cold-start behavior for every endpoint. + +## Predeclared provisional interpretation + +For each endpoint in each attempt, the last three windows must satisfy all of: + +- throughput range divided by its median is at most 5%; +- p95 latency range divided by its median is at most 10%; +- neither throughput nor p95 has a strictly increasing or strictly decreasing + sequence across those last three windows; +- every window and warm-up has only successful HTTP 200 responses, valid raw oha + evidence, valid resource identity and no detected restart/OOM/error. + +Only then compare the first window's throughput to the later-three median. A +relative difference at most 5% supports the five-second budget for that observed +endpoint/attempt. An implementation must pass every endpoint in every one of its +three attempts to receive the limited overall support flag. Missing, failed or +inconclusive attempts never become a passing result. + +These are provisional engineering thresholds fixed before data collection, not a +statistical confidence test or a universal steady-state guarantee. The strict +trend rule is deliberately conservative and can reject random monotonic noise. +A failed stability rule means insufficient evidence within the fixed window, not +proof that Java needs a particular longer warm-up. No threshold, endpoint order +or duration is adjusted after seeing results, and no automatic retry is permitted. +No single attractive sample can establish adequacy. Inspect all repeated/control +windows and timing before recommending a future policy. + +## Concurrency and instrumentation limits + +One JVM process does not mean one request-processing thread. Spring MVC uses +embedded Tomcat's platform-thread request handling with virtual threads disabled; +this repository does not pin a custom servlet thread-pool size. JDBC calls and +direct recursive CPU work run synchronously on request threads. HikariCP has a +maximum of ten connections and minimum idle zero. Node / Fastify serves requests +on its event loop without cluster/worker-thread offload, while Go / Gin uses the +Go runtime's concurrent request handling. These are whole-stack concurrency +models under common resource caps, not identical thread counts. No runtime is +tuned for this diagnostic. See the implementation guides and pinned source. + +The diagnostic records the raw Docker stats snapshots (including CPU percentage +and memory), UTC window boundaries, resource/image identities, source commit/tree, +versions, lock checksums, host information, raw oha output and errors. Docker +stats sampling and state checks run during load and consume host resources; +Docker's sampled memory is not kernel peak RSS. Instrumentation overhead is not +quantified by a paired uninstrumented run. Same-host controls share that method. + +No JIT/GC instrumentation flags, attach tools or extra instrumented load runs are +added. JIT compilation/GC events are therefore unavailable, explicitly recorded +as such; a changing window cannot be attributed specifically to either mechanism. +A later causality investigation would need a separate approved budget and should +keep its instrumented observations distinct. + +## Execution, evidence and stopping + +The **JVM warm-up diagnostic** Actions workflow accepts manual dispatch only on +the trusted default branch. It has read-only repository permissions and no +publication job. Adding the workflow is not permission for unattended recurring +runs. Run it only for an explicitly authorized investigation; do not rerun a +failed or inconclusive run without approval. CI runs synthetic harness and +workflow tests, never this study's real load. + +The job cap is 120 minutes. Checkout/setup each have a three-minute cap, the +entire harness (installation, builds, load and cleanup) has a 104-minute timeout, +with TERM followed by a two-minute kill limit; artifact upload has a seven-minute +cap. There are 27 endpoint sequences × 125 sending seconds = 56 minutes 15 seconds +of configured load. Request drain, readiness, contract, sampling, build and cleanup +add time. This estimate is not a guarantee of completion within the cap; an +exhausted budget stops the experiment without a follow-up run. + +Each attempt journals progress before starting and after each retained window. +Failure aborts the study, records untouched attempts as not started, and attempts +scoped cleanup. A SIGTERM follows the same failure path. The workflow always +attempts evidence upload, including partial raw files and logs, under +`jvm-diagnostic--`, with normal repository artifact access and a +configured 90-day retention. Abrupt worker loss, SIGKILL or failed upload can still +leave missing evidence; never claim that a checksum list proves completeness. +An evidence SHA-256 manifest covers retained study files after orderly completion +or failure; it is an integrity aid, not an independent signature. The separate +bootstrap record covers failures before the study starts. + +Artifacts live under `.cache/jvm-diagnostic/`, never `results/`. A diagnostic has +`official: false`, `publishable: false` and its own schema. Official report +validation rejects it. Attempts use new directories and cannot overwrite/resume +old output. Scratch investigations and plans remain local-only per `AGENTS.md`. + +After the one approved run reaches a terminal state, audit all attempts and raw +files, then propose one of: keep Java registered-only; consider a separately +approved unchanged-profile cohort; or investigate a new common versioned profile. +A changed official warm-up would apply to every member, with the full selected +cohort measured together. This diagnostic neither chooses that rollout nor +satisfies it. Issue #73 stays open while evidence or the owner's decision remains +unresolved; preparing the harness alone does not complete it. diff --git a/tests/test_benchmark_jvm_diagnostic.py b/tests/test_benchmark_jvm_diagnostic.py new file mode 100644 index 0000000..c90fe79 --- /dev/null +++ b/tests/test_benchmark_jvm_diagnostic.py @@ -0,0 +1,167 @@ +"""Fixed, non-publishing JVM study: synthetic scheduling and failure evidence.""" + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from benchmark.contract_test import load_cases +from benchmark.healthcheck import EXTERNAL_READINESS +from benchmark.results import BenchmarkFailure + + +class Environment: + health_policy = EXTERNAL_READINESS + connections = 50 + request_timeout = 15 + + def __init__(self, root, fail=None): + self.artifacts = root / "raw" + self.artifacts.mkdir(parents=True) + self.events = [] + self.fail = fail + self.readiness = {"attempts": 1} + + def build(self, identifier): + self.events.append(("build", identifier)) + if self.fail == "build": + raise BenchmarkFailure("build failed") + + def start(self, identifier): + self.events.append(("start", identifier)) + return {"id": "a" * 64, "image_id": "sha256:" + "b" * 64} + + def check(self): + pass + + def measure(self, endpoint, duration, index): + self.events.append((endpoint, duration, index)) + if self.fail == "measure": + raise BenchmarkFailure("measure failed") + if self.fail == "interrupt": + raise KeyboardInterrupt() + return { + "requests_per_second": [100, 100, 101, 100, 101][index], + "p95_response_time_ms": 2, + "successful_requests": 100, + "elapsed_seconds": duration, + } + + def cleanup(self): + self.events.append("cleanup") + if self.fail == "cleanup": + raise BenchmarkFailure("cleanup failed") + + +class DiagnosticTests(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(importlib.util.find_spec("benchmark.jvm_diagnostic")) + self.module = importlib.import_module("benchmark.jvm_diagnostic") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.cache = Path(temporary.name) + patcher = patch.object(self.module, "CACHE_ROOT", self.cache) + patcher.start() + self.addCleanup(patcher.stop) + self.output = self.cache / "run" + self.environments = [] + self.metadata = {"source_commit": "a" * 40, "source_tree": "b" * 40} + + def run_study(self, fail=None, count=None): + def factory(root): + env = Environment(root, fail) + self.environments.append(env) + return env + + return self.module.run_study( + self.output, + factory, + metadata=self.metadata, + contract=lambda *a, **k: 2 * len(load_cases()) if count is None else count, + ) + + def test_predeclared_order_windows_and_clean_projects(self): + result = self.run_study() + expected = [ + "java-spring-boot", + "node-fastify", + "go-gin", + "node-fastify", + "go-gin", + "java-spring-boot", + "go-gin", + "java-spring-boot", + "node-fastify", + ] + self.assertEqual([a["implementation"] for a in result["attempts"]], expected) + self.assertEqual(len(self.environments), 9) + for env in self.environments: + self.assertEqual(env.events[-1], "cleanup") + self.assertEqual( + env.events[2:-1], + [ + (endpoint, 5 if i == 0 else 30, i) + for endpoint in ("/json", "/db/42", "/cpu") + for i in range(5) + ], + ) + self.assertEqual(result["status"], "completed") + self.assertFalse(result["official"]) + self.assertFalse(result["publishable"]) + self.assertEqual(result["mode"], "jvm-warmup-diagnostic") + self.assertEqual(result, json.loads((self.output / "diagnostic.json").read_text())) + self.assertTrue((self.output / "checksums.json").is_file()) + from benchmark.report import validate_report + + with self.assertRaises(BenchmarkFailure): + validate_report(result) + + def test_failure_and_interruption_journal_before_cleanup_no_retry(self): + for stage in ("build", "measure", "cleanup", "interrupt"): + with self.subTest(stage=stage): + self.output = self.cache / stage + self.environments = [] + with self.assertRaises((BenchmarkFailure, KeyboardInterrupt)): + self.run_study(fail=stage) + result = json.loads((self.output / "diagnostic.json").read_text()) + self.assertEqual(result["status"], "failed") + self.assertEqual(len(self.environments), 1) + self.assertEqual(self.environments[0].events[-1], "cleanup") + self.assertEqual(result["attempts"][0]["status"], "failed") + self.assertTrue(all(a["status"] == "not-started" for a in result["attempts"][1:])) + self.assertTrue((self.output / "checksums.json").is_file()) + + def test_contract_count_fail_closed_without_load(self): + with self.assertRaises(BenchmarkFailure): + self.run_study(count=True) + self.assertEqual(len(self.environments[0].events), 3) + + def test_no_overwrite_or_output_escape(self): + self.run_study() + before = (self.output / "diagnostic.json").read_bytes() + with self.assertRaises(BenchmarkFailure): + self.run_study() + self.assertEqual(before, (self.output / "diagnostic.json").read_bytes()) + for path in (self.cache / ".." / "escape", self.cache): + with self.assertRaises(BenchmarkFailure): + self.module.validate_directory(path) + (self.cache / "linked").symlink_to(self.cache, target_is_directory=True) + with self.assertRaises(BenchmarkFailure): + self.module.validate_directory(self.cache / "linked" / "run2") + + def test_provisional_stability_rules_include_trend_and_missing_data(self): + def windows(rps, p95=(2, 2, 2, 2)): + return [{"requests_per_second": r, "p95_response_time_ms": p} for r, p in zip(rps, p95)] + + analyze = self.module.analyze_windows + self.assertTrue(analyze(windows([100, 101, 100, 101]))["five_second_supported"]) + self.assertFalse(analyze(windows([90, 101, 100, 101]))["five_second_supported"]) + self.assertFalse(analyze(windows([100, 100, 101, 102]))["late_stable"]) + self.assertFalse(analyze(windows([100, 90, 110, 100]))["late_stable"]) + self.assertFalse(analyze(windows([100, 101, 100, 101], [2, 2, 3, 2]))["late_stable"]) + with self.assertRaises(BenchmarkFailure): + analyze(windows([100, 100, 100])) + with self.assertRaises(BenchmarkFailure): + analyze(windows([100, float("nan"), 100, 100])) diff --git a/tests/test_benchmark_jvm_environment.py b/tests/test_benchmark_jvm_environment.py new file mode 100644 index 0000000..1571226 --- /dev/null +++ b/tests/test_benchmark_jvm_environment.py @@ -0,0 +1,307 @@ +"""Diagnostic-only JVM window evidence, with bounded command doubles.""" + +import importlib +import importlib.util +import json +import math +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +from benchmark import environment +from benchmark.healthcheck import EXTERNAL_READINESS +from benchmark.results import BenchmarkFailure + +CID = "a" * 64 +FIXTURE = Path(__file__).parent / "fixtures" / "oha-1.16.0" / "timed.json" + + +class DiagnosticEnvironmentTests(unittest.TestCase): + def setUp(self): + self.assertIsNotNone( + importlib.util.find_spec("benchmark.jvm_environment"), + "diagnostic-only environment module is required", + ) + self.module = importlib.import_module("benchmark.jvm_environment") + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.env = self.module.DiagnosticEnvironment(Path("/fake/oha"), Path(self.directory.name)) + self.env.container = CID + self.env.implementation = "java-spring-boot" + self.commands = [] + self.payload = FIXTURE.read_bytes() + self.stats = json.dumps({"ID": CID, "CPUPerc": "73.25%", "MemUsage": "12MiB / 512MiB"}) + self.load_failure = None + + def execute(self, arguments, *, timeout, tick=None, **kwargs): + self.commands.append((arguments, timeout)) + if arguments[:2] == ["docker", "stats"]: + return self.stats + if "--output" in arguments: + output = Path(arguments[arguments.index("--output") + 1]) + output.write_bytes(self.payload) + tick() + if self.load_failure: + # Evidence must already be flushed before a command fails. + self.assertTrue(list(self.env.artifacts.glob("*.stats.jsonl"))[0].read_text()) + raise self.load_failure + return "oha stdout\n" + if "logs" in arguments: + return "owned container log\n" + return "" + + def measure(self, index=4, endpoint="/json"): + with ( + patch.object(self.env, "check"), + patch.object(self.module, "execute", side_effect=self.execute), + ): + return self.env.measure(endpoint, 1, index) + + def window(self, index=4): + return self.env.artifacts / f"java-spring-boot-json-window-{index}.window.json" + + def test_inherits_scoped_cleanup_and_external_readiness_without_tuning(self): + self.assertIs( + self.module.DiagnosticEnvironment.cleanup, environment.DockerEnvironment.cleanup + ) + self.assertEqual(self.env.health_policy, EXTERNAL_READINESS) + self.assertEqual((self.env.connections, self.env.request_timeout), (50, 15)) + self.assertFalse(self.env.audit_health_events) + + def test_validated_p95_cpu_and_raw_stats_are_preserved_for_window_four(self): + result = self.measure() + raw = json.loads(self.payload) + self.assertEqual(result["p95_response_time_ms"], raw["latencyPercentiles"]["p95"] * 1000) + self.assertEqual(result["run"], 4) + self.assertEqual(result["window_index"], 4) + self.assertEqual(result["peak_memory_bytes"], 12 * 1024**2) + self.assertEqual(result["memory_samples"], 1) + self.assertEqual(result["cpu_percent_samples"], [73.25]) + base = self.env.artifacts / "java-spring-boot-json-window-4" + self.assertEqual(base.with_suffix(".json").read_bytes(), self.payload) + sample = json.loads(base.with_suffix(".stats.jsonl").read_text()) + self.assertEqual(sample["raw"], self.stats) + self.assertLessEqual( + datetime.fromisoformat(sample["started_at"]), + datetime.fromisoformat(sample["completed_at"]), + ) + memory = json.loads(base.with_suffix(".memory.jsonl").read_text()) + self.assertEqual(memory["bytes"], 12 * 1024**2) + self.assertEqual(memory["cpu_percent"], 73.25) + + def test_window_timing_and_status_are_persisted(self): + result = self.measure(0) + window = json.loads(self.window(0).read_text()) + self.assertEqual(window["status"], "complete") + self.assertEqual(window["window_index"], 0) + self.assertEqual(window["started_at"], result["started_at"]) + self.assertEqual(window["completed_at"], result["completed_at"]) + self.assertGreaterEqual(window["duration_wall_seconds"], 0) + self.assertIsNotNone(datetime.fromisoformat(window["started_at"]).tzinfo) + self.assertLessEqual( + datetime.fromisoformat(window["started_at"]), + datetime.fromisoformat(window["completed_at"]), + ) + + def test_oha_arguments_match_common_load_contract(self): + self.measure() + command, timeout = next( + (args, timeout) for args, timeout in self.commands if "--output" in args + ) + expected = [ + "/fake/oha", + "--no-tui", + "--output-format", + "json", + "--output", + str(self.env.artifacts / "java-spring-boot-json-window-4.json"), + "--http-version", + "1.1", + "--redirect", + "0", + "--disable-compression", + "-c", + "50", + "-z", + "1s", + "-w", + "-t", + "15s", + "--connect-timeout", + "5s", + "http://127.0.0.1:8080/json", + ] + self.assertEqual(command, expected) + self.assertEqual(timeout, 31) + self.assertEqual( + len([args for args, _ in self.commands if args[:2] == ["docker", "stats"]]), 1 + ) + + def test_failed_load_retains_partial_stats_raw_oha_logs_and_window_timing(self): + self.load_failure = BenchmarkFailure("command exited 1: oha\npartial failure output") + with self.assertRaisesRegex(BenchmarkFailure, "partial failure output"): + self.measure() + self.assertEqual(json.loads(self.window().read_text())["status"], "failed") + self.assertIn( + "partial failure output", + (self.env.artifacts / "java-spring-boot-json-window-4.load.log").read_text(), + ) + self.assertEqual( + (self.env.artifacts / "java-spring-boot-json-window-4.json").read_bytes(), self.payload + ) + self.assertTrue(list(self.env.artifacts.glob("*.stats.jsonl"))[0].read_text()) + self.assertIn( + "owned container log", + (self.env.artifacts / "java-spring-boot-json-window-4-container.log").read_text(), + ) + logs, timeout = next((args, timeout) for args, timeout in self.commands if "logs" in args) + self.assertIn(self.env.project, logs) + self.assertIn("--tail", logs) + self.assertLessEqual(timeout, 15) + + def test_invalid_cpu_retains_raw_sample_but_invalidates_window(self): + for cpu in (None, "NaN%", "-1%", "inf%", "1", True): + self.stats = json.dumps({"ID": CID, "CPUPerc": cpu, "MemUsage": "12MiB / 512MiB"}) + with self.subTest(cpu=cpu), self.assertRaises(BenchmarkFailure): + self.measure() + self.assertEqual( + json.loads(list(self.env.artifacts.glob("*.stats.jsonl"))[0].read_text())["raw"], + self.stats, + ) + self.assertEqual(json.loads(self.window().read_text())["status"], "failed") + + def test_p95_missing_or_nonfinite_is_rejected_by_common_parser(self): + for value in (None, math.inf, math.nan, "missing"): + data = json.loads(FIXTURE.read_bytes()) + if value == "missing": + del data["latencyPercentiles"]["p95"] + else: + data["latencyPercentiles"]["p95"] = value + self.payload = json.dumps(data).encode() + with self.subTest(value=value), self.assertRaises(BenchmarkFailure): + self.measure() + self.assertEqual(json.loads(self.window().read_text())["status"], "failed") + + def test_oha_error_response_is_retained_but_never_validated_as_success(self): + self.payload = FIXTURE.with_name("http-error.json").read_bytes() + with self.assertRaises(BenchmarkFailure): + self.measure() + self.assertEqual( + (self.env.artifacts / "java-spring-boot-json-window-4.json").read_bytes(), self.payload + ) + self.assertEqual(json.loads(self.window().read_text())["status"], "failed") + + def test_build_failure_preserves_existing_command_error_output(self): + failure = BenchmarkFailure("command exited 1: docker compose build\ncompiler details") + with ( + patch.object(environment, "execute", side_effect=failure), + patch.object(self.module, "execute", side_effect=self.execute), + ): + with self.assertRaisesRegex(BenchmarkFailure, "compiler details"): + self.env.build("java-spring-boot") + self.assertIn( + "compiler details", (self.env.artifacts / "java-spring-boot-build.log").read_text() + ) + + def test_start_context_is_allowlisted_and_has_no_environment_secrets(self): + public = { + "id": CID, + "image_id": "sha256:123", + "command": ["java", "-jar", "app.jar"], + "postgresql_version": "PostgreSQL 18.6", + } + state = { + "Id": CID, + "Image": "sha256:123", + "Config": {"Env": ["DATABASE_PASSWORD=secret-never-persist"]}, + "State": {"StartedAt": "2026-09-30T01:00:00Z"}, + "RestartCount": 0, + "HostConfig": {"NanoCpus": 1000000000, "Memory": 536870912}, + } + with ( + patch.object(environment.DockerEnvironment, "start", return_value=public), + patch.object(self.env, "inspect", return_value=state), + ): + result = self.env.start("java-spring-boot") + self.assertEqual(result["limits"], {"nano_cpus": 1000000000, "memory_bytes": 536870912}) + self.assertEqual(result["command"], public["command"]) + saved = (self.env.artifacts / "java-spring-boot-context.json").read_text() + self.assertNotIn("secret-never-persist", saved) + self.assertNotIn("DATABASE_PASSWORD", saved) + self.assertEqual(json.loads(saved), result) + + def test_start_failure_captures_scoped_logs_without_masking_cause(self): + failure = BenchmarkFailure("readiness failed") + with ( + patch.object(environment.DockerEnvironment, "start", side_effect=failure), + patch.object(self.module, "execute", side_effect=self.execute), + ): + with self.assertRaisesRegex(BenchmarkFailure, "readiness failed"): + self.env.start("java-spring-boot") + self.assertIn( + "readiness failed", + (self.env.artifacts / "java-spring-boot-start-failure.log").read_text(), + ) + self.assertIn( + "owned container log", + (self.env.artifacts / "java-spring-boot-start-container.log").read_text(), + ) + + def test_evidence_write_failure_does_not_mask_failed_load(self): + self.load_failure = BenchmarkFailure("original load failure") + original = Path.write_text + + def fail_evidence(path, text, **kwargs): + if path.suffix == ".log" or path.name.endswith(".window.json"): + raise OSError("disk full") + return original(path, text, **kwargs) + + with patch.object(Path, "write_text", fail_evidence): + with self.assertRaisesRegex(BenchmarkFailure, "original load failure"): + self.measure() + + def test_evidence_write_failure_does_not_mask_failed_build(self): + failure = BenchmarkFailure("original build failure") + with ( + patch.object(environment, "execute", side_effect=failure), + patch.object(Path, "write_text", side_effect=OSError("disk full")), + ): + with self.assertRaisesRegex(BenchmarkFailure, "original build failure"): + self.env.build("java-spring-boot") + + def test_constructor_rejects_measurement_policy_changes(self): + for change in ( + {"connections": 1}, + {"request_timeout": 30}, + {"health_policy": "container-healthcheck"}, + {"audit_health_events": True}, + ): + with self.subTest(change=change), self.assertRaises(BenchmarkFailure): + self.module.DiagnosticEnvironment( + Path("/fake/oha"), Path(self.directory.name), **change + ) + + def test_all_approved_endpoints_have_safe_evidence_paths(self): + for endpoint, label in (("/json", "json"), ("/db/42", "db-42"), ("/cpu", "cpu")): + with self.subTest(endpoint=endpoint): + self.measure(endpoint=endpoint) + self.assertTrue( + (self.env.artifacts / f"java-spring-boot-{label}-window-4.json").is_file() + ) + self.assertTrue( + ( + self.env.artifacts / f"java-spring-boot-{label}-window-4.window.json" + ).is_file() + ) + + def test_invalid_window_indexes_do_not_start_load(self): + for index in (-1, 5, True, 1.0): + with self.subTest(index=index), self.assertRaises(BenchmarkFailure): + self.measure(index) + self.assertEqual(self.commands, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmark_jvm_workflow.py b/tests/test_benchmark_jvm_workflow.py new file mode 100644 index 0000000..0c4c136 --- /dev/null +++ b/tests/test_benchmark_jvm_workflow.py @@ -0,0 +1,78 @@ +"""Keep JVM diagnostic execution separate from automatic CI, without extra dependencies.""" + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/jvm-diagnostic.yml" + + +class JvmDiagnosticWorkflowSourceTests(unittest.TestCase): + def setUp(self): + self.assertTrue(WORKFLOW.is_file(), "JVM diagnosis needs its own manual-only workflow") + self.source = WORKFLOW.read_text() + + def test_one_bounded_diagnostic_command_leaves_time_for_failure_artifacts(self): + commands = re.findall(r"^\s+run: (.+)$", self.source, flags=re.MULTILINE) + self.assertEqual( + commands, + ["timeout --signal=TERM --kill-after=120s 104m python -m benchmark.jvm_diagnostic"], + ) + self.assertIn(" timeout-minutes: 120\n", self.source) + self.assertNotIn("continue-on-error", self.source) + + def test_kill_grace_covers_cleanup_within_the_total_job_budget(self): + command = re.search( + r"^\s+run: timeout --signal=TERM --kill-after=(\d+)s (\d+)m ", + self.source, + flags=re.MULTILINE, + ) + self.assertIsNotNone(command) + kill_seconds, run_minutes = map(int, command.groups()) + # Failure logs (15s), Compose down (60s), and three resource checks (10s each). + cleanup_seconds = 15 + 60 + 3 * 10 + self.assertGreater(kill_seconds, cleanup_seconds, "leave time to journal cleanup") + step_caps = list( + map(int, re.findall(r"^ timeout-minutes: (\d+)$", self.source, re.MULTILINE)) + ) + self.assertEqual(step_caps, [3, 3, 7]) + job_minutes = re.search(r"^ timeout-minutes: (\d+)$", self.source, re.MULTILINE) + self.assertIsNotNone(job_minutes) + total_seconds = (sum(step_caps) + run_minutes) * 60 + kill_seconds + self.assertEqual(total_seconds, 119 * 60) + self.assertLess(total_seconds, int(job_minutes.group(1)) * 60) + + def test_diagnostic_has_no_automatic_trigger_or_user_controlled_profile(self): + trigger = re.search(r"^on:\n(.*?)(?=^\S)", self.source, flags=re.MULTILINE | re.DOTALL) + self.assertIsNotNone(trigger) + self.assertEqual(trigger.group(1), " workflow_dispatch:\n") + + def test_automatic_workflows_never_invoke_the_jvm_load_diagnostic(self): + for workflow in (ROOT / ".github/workflows").glob("*.yml"): + if workflow == WORKFLOW: + continue + with self.subTest(workflow=workflow.name): + source = workflow.read_text() + self.assertNotIn("benchmark.jvm_diagnostic", source) + self.assertNotIn("make jvm-diagnostic", source) + self.assertNotIn("./.github/workflows/jvm-diagnostic.yml", source) + + def test_diagnostic_cannot_publish_or_write_repository_credentials(self): + for forbidden in ( + "contents: write", + "pages: write", + "id-token: write", + "secrets.", + "github.token", + "benchmark.official", + "benchmark.publish", + "deploy-pages", + "git push", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, self.source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflows.py b/tests/test_workflows.py index a9a0076..f1cebbf 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -22,10 +22,11 @@ def load(name): class WorkflowTests(unittest.TestCase): - def test_only_four_permanent_workflows_use_pinned_actions_and_safe_checkouts(self): + def test_only_approved_workflows_use_pinned_actions_and_safe_checkouts(self): paths = list((ROOT / ".github/workflows").glob("*.yml")) self.assertEqual( - {p.name for p in paths}, {"ci.yml", "benchmark.yml", "pages.yml", "pages-deploy.yml"} + {p.name for p in paths}, + {"ci.yml", "benchmark.yml", "jvm-diagnostic.yml", "pages.yml", "pages-deploy.yml"}, ) for path in paths: workflow = load(path.name) @@ -44,6 +45,62 @@ def test_only_four_permanent_workflows_use_pinned_actions_and_safe_checkouts(sel self.assertEqual(step["with"]["persist-credentials"], "false") self.assertNotIn("${{", step.get("run", ""), "pass expression data through env") + def test_jvm_diagnostic_is_manual_trusted_read_only_and_sequential(self): + workflow = load("jvm-diagnostic.yml") + self.assertEqual(workflow.get("on"), {"workflow_dispatch": ""}) + self.assertEqual(workflow["permissions"], {"contents": "read"}) + self.assertEqual( + workflow["concurrency"], {"group": "jvm-diagnostic", "cancel-in-progress": "false"} + ) + self.assertEqual(set(workflow["jobs"]), {"diagnose"}) + job = workflow["jobs"]["diagnose"] + self.assertEqual( + job["if"], + "github.repository == 'tappe9/simple-api-benchmark' && " + "github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && " + "github.event_name == 'workflow_dispatch'", + ) + self.assertEqual(job["runs-on"], "ubuntu-24.04") + self.assertEqual(job["timeout-minutes"], "120") + self.assertNotIn("strategy", job) + self.assertNotIn("permissions", job) + self.assertNotIn("continue-on-error", job) + self.assertEqual(job["env"], {"PYTHONDONTWRITEBYTECODE": "1"}) + + def test_jvm_diagnostic_actions_match_existing_pins_and_upload_failures(self): + workflow = load("jvm-diagnostic.yml") + self.assertIn("jobs", workflow, "the manual JVM diagnostic workflow is required") + steps = workflow["jobs"]["diagnose"]["steps"] + self.assertEqual(len(steps), 4) + official_steps = load("benchmark.yml")["jobs"]["measure"]["steps"] + self.assertEqual(steps[0]["uses"], official_steps[0]["uses"]) + self.assertEqual(steps[0]["with"], official_steps[0]["with"]) + self.assertEqual(steps[1]["uses"], official_steps[1]["uses"]) + self.assertEqual(steps[1]["with"], official_steps[1]["with"]) + for step in steps[:2]: + self.assertGreater(int(step["timeout-minutes"]), 0) + self.assertLessEqual(int(step["timeout-minutes"]), 3) + self.assertEqual( + steps[2]["run"], + "timeout --signal=TERM --kill-after=120s 104m python -m benchmark.jvm_diagnostic", + ) + self.assertNotIn("if", steps[2]) + self.assertEqual(steps[3]["if"], "always()") + self.assertEqual(steps[3]["uses"], official_steps[3]["uses"]) + self.assertEqual(steps[3]["timeout-minutes"], "7") + self.assertEqual( + steps[3]["with"], + { + "name": "jvm-diagnostic-${{ github.run_id }}-${{ github.run_attempt }}", + "path": ".cache/jvm-diagnostic/", + "include-hidden-files": "true", + "retention-days": "90", + "if-no-files-found": "error", + }, + ) + for step in steps: + self.assertNotIn("continue-on-error", step) + def test_pr_ci_is_split_read_only_and_never_publishes(self): ci = load("ci.yml") self.assertEqual(ci["permissions"], {"contents": "read"})