diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 2c941983f9..834706ad7c 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -36,6 +36,13 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-24.04 + # Bound this job well short of GitHub's 360-minute platform default. Its + # only step is a single-repository, status-filtered gh api --paginate + # list-and-cancel sweep (up to 3 passes x 5 statuses), no branch update + # or merge -- lighter than pr-review-merge-scheduler.yml's scan-pr-queue + # job (PR #1702), which got timeout-minutes: 30 for a comparable + # single-repo scan that also dispatches a review and updates a branch. + timeout-minutes: 20 permissions: actions: write contents: read @@ -180,6 +187,21 @@ jobs: noema-review: name: noema-review runs-on: ubuntu-24.04 + # Bound this job well short of GitHub's 360-minute platform default. Its + # "Prepare Noema model verdict" step calls into two_phase.py's call_llm + # via the same contextual-orchestrator gateway whose unbounded wait was + # confirmed to stall runs for 7-20 hours in opencode-review.yml before + # PR #1707's fix -- and noema_review_gate.py's own comment says that + # step "remains governed by contextual-orchestrator rather than a fixed + # inference timeout", so nothing upstream of this job bounds it either. + # 210 minutes gives that step the same ~180-minute (3-hour) allowance + # PR #1707 set for its analogous model-wait deadline -- comfortably + # above this org's documented "accommodate over 2 hours per model" + # policy (docs/product-goal-directive.md #8) -- plus a 30-minute buffer + # for this job's other steps (tarball fetch, credential mint, the + # superseded-run cleanup sweep, visibility-lookup retries, sidecar + # provisioning, publication), while staying well under GitHub's default. + timeout-minutes: 210 if: >- github.event_name == 'repository_dispatch' || ( diff --git a/.github/workflows/source-fix-pr1715-no-model-job-timeout.yml b/.github/workflows/source-fix-pr1715-no-model-job-timeout.yml new file mode 100644 index 0000000000..0d733b2b72 --- /dev/null +++ b/.github/workflows/source-fix-pr1715-no-model-job-timeout.yml @@ -0,0 +1,105 @@ +name: Source Fix PR 1715 No Model Job Timeout + +on: + push: + branches: + - fix/noema-review-job-timeout-minutes + paths: + - scripts/ci/source_fix_pr1715_no_model_job_timeout.py + - .github/workflows/source-fix-pr1715-no-model-job-timeout.yml + +concurrency: + group: source-fix-pr1715-${{ github.repository }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + repair: + runs-on: ubuntu-slim + steps: + - name: Checkout exact writer head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Revalidate exact remote head + shell: bash + run: | + set -euo pipefail + remote_head="$(git ls-remote origin refs/heads/fix/noema-review-job-timeout-minutes | cut -f1)" + test -n "$remote_head" + test "$remote_head" = "$GITHUB_SHA" + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + cache: pip + + - name: Install exact test toolchain + shell: bash + run: | + set -euo pipefail + python -m pip install --require-hashes -r requirements-opencode-review-ci-hashes.txt + + - name: Apply causal-owner repair + shell: bash + run: | + set -euo pipefail + python scripts/ci/source_fix_pr1715_no_model_job_timeout.py + python -m py_compile scripts/ci/source_fix_pr1715_no_model_job_timeout.py + git diff --check + + - name: Verify Noema timeout authority contract + shell: bash + run: | + set -euo pipefail + python -m pytest \ + tests/test_noema_orchestrator_workflow_contract.py \ + tests/test_required_workflow_queue_contract.py \ + tests/test_noema_review_gate.py \ + tests/test_noema_review_handoff.py \ + tests/test_noema_two_phase_handoff.py \ + -q + python -m compileall -q scripts tests .github/actions/noema-review + git diff --check + + - name: Retire one-shot repair artifacts and verify scope + shell: bash + run: | + set -euo pipefail + rm scripts/ci/source_fix_pr1715_no_model_job_timeout.py + rm .github/workflows/source-fix-pr1715-no-model-job-timeout.yml + allowed='^(.github/workflows/noema-review.yml|tests/test_noema_orchestrator_workflow_contract.py|CHANGELOG.md|docs/product-technical-gap-baseline.md|scripts/ci/source_fix_pr1715_no_model_job_timeout.py|.github/workflows/source-fix-pr1715-no-model-job-timeout.yml)$' + bad="$(git status --short | sed -E 's/^.. //' | grep -Ev "$allowed" || true)" + test -z "$bad" + test ! -e scripts/ci/source_fix_pr1715_no_model_job_timeout.py + test ! -e .github/workflows/source-fix-pr1715-no-model-job-timeout.yml + remote_head="$(git ls-remote origin refs/heads/fix/noema-review-job-timeout-minutes | cut -f1)" + test "$remote_head" = "$GITHUB_SHA" + + - name: Publish normal non-force repair commit + env: + PRIMARY_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }} + FALLBACK_PUSH_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }} + shell: bash + run: | + set -euo pipefail + workflow_push_token="${PRIMARY_PUSH_TOKEN:-${FALLBACK_PUSH_TOKEN:-}}" + if [ -z "$workflow_push_token" ]; then + echo "::error::No workflow-starting mutation credential is configured; refusing github.token publication." + exit 1 + fi + remote_head="$(git ls-remote origin refs/heads/fix/noema-review-job-timeout-minutes | cut -f1)" + test "$remote_head" = "$GITHUB_SHA" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git diff --cached --check + git commit -m "fix(noema-review): remove model wall-clock termination" + git remote set-url origin "https://x-access-token:${workflow_push_token}@github.com/${GITHUB_REPOSITORY}.git" + git push origin HEAD:fix/noema-review-job-timeout-minutes diff --git a/scripts/ci/source_fix_pr1715_no_model_job_timeout.py b/scripts/ci/source_fix_pr1715_no_model_job_timeout.py new file mode 100644 index 0000000000..497d109678 --- /dev/null +++ b/scripts/ci/source_fix_pr1715_no_model_job_timeout.py @@ -0,0 +1,110 @@ +"""One-shot exact-head repair for PR #1715's Noema model timeout contract.""" + +from __future__ import annotations + +import re +from pathlib import Path + +WORKFLOW = Path(".github/workflows/noema-review.yml") +TEST = Path("tests/test_noema_orchestrator_workflow_contract.py") +CHANGELOG = Path("CHANGELOG.md") +BASELINE = Path("docs/product-technical-gap-baseline.md") + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + """Replace one literal block and fail closed when branch contents moved.""" + count = text.count(old) + if count != 1: + raise SystemExit(f"PR1715 {label}: expected one literal block, found {count}") + return text.replace(old, new, 1) + + +def patch_workflow() -> None: + """Keep bounded cleanup but remove elapsed-time authority from model work.""" + text = WORKFLOW.read_text(encoding="utf-8") + old = ''' # Bound this job well short of GitHub's 360-minute platform default. Its + # "Prepare Noema model verdict" step calls into two_phase.py's call_llm + # via the same contextual-orchestrator gateway whose unbounded wait was + # confirmed to stall runs for 7-20 hours in opencode-review.yml before + # PR #1707's fix -- and noema_review_gate.py's own comment says that + # step "remains governed by contextual-orchestrator rather than a fixed + # inference timeout", so nothing upstream of this job bounds it either. + # 210 minutes gives that step the same ~180-minute (3-hour) allowance + # PR #1707 set for its analogous model-wait deadline -- comfortably + # above this org's documented "accommodate over 2 hours per model" + # policy (docs/product-goal-directive.md #8) -- plus a 30-minute buffer + # for this job's other steps (tarball fetch, credential mint, the + # superseded-run cleanup sweep, visibility-lookup retries, sidecar + # provisioning, publication), while staying well under GitHub's default. + timeout-minutes: 210 +''' + new = ''' # Model-backed Noema intentionally has no job-level wall-clock timeout. + # contextual-orchestrator/orchestrator/free owns provider termination; + # GitHub admission must not stop reasoning, streaming, or tool work only + # because elapsed time crossed a repository-side deadline. Stale heads, + # closed/draft PRs, provider completion, and explicit cancellation remain + # authoritative termination signals. The non-model cleanup job above is + # independently bounded because it performs only GitHub API housekeeping. +''' + WORKFLOW.write_text( + replace_once(text, old, new, "model job timeout block"), encoding="utf-8" + ) + + +def patch_test() -> None: + """Replace the stale timeout-positive assertion with the owner contract.""" + text = TEST.read_text(encoding="utf-8") + marker = "def test_noema_review_job_has_a_bounded_runtime_above_the_two_hour_model_allowance() -> None:\n" + start = text.find(marker) + if start < 0 or text.find(marker, start + 1) >= 0: + raise SystemExit("PR1715 stale model-timeout test marker moved or duplicated") + replacement = '''def test_noema_review_model_job_has_no_elapsed_time_termination() -> None: + """Model-backed Noema delegates termination to orchestrator/provider authority.""" + workflow = workflow_text("noema-review.yml") + job = workflow.split(" noema-review:\\n", 1)[1] + + assert re.search(r"^ timeout-minutes:", job, flags=re.MULTILINE) is None + assert "contextual-orchestrator/orchestrator/free" in workflow + assert "Model-backed Noema intentionally has no job-level wall-clock timeout" in job + assert "timeout-minutes: 20" in workflow.split( + " cancel-closed-pr-runs:\\n", 1 + )[1].split("\\n noema-review:\\n", 1)[0] +''' + TEST.write_text(text[:start] + replacement, encoding="utf-8") + + +def append_traceability() -> None: + """Record why support housekeeping may be bounded while model work may not.""" + changelog_note = ( + "\n- PR #1715: keep the non-model Noema close-cleanup job bounded, but remove " + "the proposed 210-minute job timeout from model-backed `noema-review`; " + "`orchestrator/free`/provider completion, live PR/head state, or explicit " + "cancellation are the termination authorities rather than elapsed time.\n" + ) + changelog = CHANGELOG.read_text(encoding="utf-8") + if "PR #1715: keep the non-model Noema close-cleanup job bounded" not in changelog: + CHANGELOG.write_text(changelog + changelog_note, encoding="utf-8") + + baseline_note = ''' + +### Noema model-job timeout authority — PR #1715 + +- **Root cause:** a queue-operability repair proposed `timeout-minutes: 210` on the model-backed `noema-review` job, turning elapsed wall time into an admission/model termination authority. +- **Contract:** the lightweight closed-PR Actions cleanup remains bounded, while Noema model work has no repository-owned wall-clock cutoff. `orchestrator/free` and its upstream provider own normal model completion; live PR/head validation, provider end, or explicit cancellation remain authoritative stop conditions. +- **Regression:** `test_noema_review_model_job_has_no_elapsed_time_termination` rejects a job-level timeout on the model job while retaining the 20-minute bound on non-model cleanup. +- **Status:** Implemented on the PR #1715 writer branch; exact-head CI/review must be regenerated after the one-shot repair commit. +''' + baseline = BASELINE.read_text(encoding="utf-8") + if "### Noema model-job timeout authority — PR #1715" not in baseline: + BASELINE.write_text(baseline + baseline_note, encoding="utf-8") + + +def main() -> None: + """Apply the minimal owner repair and its permanent regression/docs.""" + patch_workflow() + patch_test() + append_traceability() + + +if __name__ == "__main__": + main() diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 3f6116caf4..f6e97f745b 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -4,6 +4,7 @@ import os import json +import re import shutil import subprocess import textwrap @@ -368,3 +369,52 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> ) assert noema.returncode == 1 assert "sidecar must be provisioned before Noema LLM review" in noema.stdout + + +def test_cancel_closed_pr_runs_has_a_bounded_runtime() -> None: + """cancel-closed-pr-runs must not fall back to GitHub's 360-minute default. + + Its only step is a single-repository, status-filtered gh api --paginate + list-and-cancel sweep (up to 3 passes x 5 statuses) with no branch update + or merge -- comparable to, or lighter than, pr-review-merge-scheduler.yml's + scan-pr-queue job, which PR #1702 bounded to timeout-minutes: 30 for a + single-repository scan that also dispatches a review and updates a branch. + """ + workflow = workflow_text("noema-review.yml") + job = workflow.split(" cancel-closed-pr-runs:\n", 1)[1].split("\n noema-review:\n", 1)[0] + + match = re.search(r"^ timeout-minutes: (\d+)$", job, flags=re.MULTILINE) + assert match is not None, "cancel-closed-pr-runs must declare a job-level timeout-minutes" + timeout = int(match.group(1)) + assert 1 <= timeout <= 30 + assert timeout < 360 + + +def test_noema_review_job_has_a_bounded_runtime_above_the_two_hour_model_allowance() -> None: + """noema-review must not fall back to GitHub's 360-minute platform default. + + Its "Prepare Noema model verdict" step calls into two_phase.py's + call_llm via the contextual-orchestrator gateway, which + noema_review_gate.py's own module comment says "remains governed by + contextual-orchestrator rather than a fixed inference timeout" -- so + nothing upstream of this job bounds that call. docs/product-goal-directive.md + section 8 documents that "중앙 OpenCode, Strix, Noema는 모델당 두 시간 + 이상 걸릴 수 있음을 수용한다" (central OpenCode, Strix, and Noema accept + that a model call may legitimately take over two hours), so the bound + must clear two hours (120 minutes) without falling back to GitHub's + 360-minute job default. + """ + workflow = workflow_text("noema-review.yml") + job = workflow.split(" noema-review:\n", 1)[1] + + match = re.search(r"^ timeout-minutes: (\d+)$", job, flags=re.MULTILINE) + assert match is not None, "noema-review must declare a job-level timeout-minutes" + timeout = int(match.group(1)) + assert 120 < timeout < 360 + + assert ( + "모델당 두 시간 이상 걸릴 수 있음을 수용한다" + in (Path(__file__).resolve().parents[1] / "docs" / "product-goal-directive.md").read_text( + encoding="utf-8" + ) + ), "the two-hour-per-model allowance this bound relies on must still be documented"