From dac6db91ef1edffced7d4efe2e0bbd4fadf330ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 15 Oct 2026 03:50:52 -0700 Subject: [PATCH] ci(noema): stage fixed-caller self-hosted reviewer quality supplier --- .github/workflows/noema-reviewer-quality.yml | 251 +++++++++++++++++++ tests/test_noema_reviewer_quality.py | 126 ++++++++++ 2 files changed, 377 insertions(+) create mode 100644 .github/workflows/noema-reviewer-quality.yml create mode 100644 tests/test_noema_reviewer_quality.py diff --git a/.github/workflows/noema-reviewer-quality.yml b/.github/workflows/noema-reviewer-quality.yml new file mode 100644 index 0000000000..6033e7f2d9 --- /dev/null +++ b/.github/workflows/noema-reviewer-quality.yml @@ -0,0 +1,251 @@ +name: Noema reviewer quality + +on: + workflow_call: {} + +permissions: + contents: read + +jobs: + reviewer: + name: reviewer + if: >- + github.repository == 'ContextualWisdomLab/noema' && + ((github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.base.repo.full_name == github.repository) || + (github.event_name == 'push' && github.ref == 'refs/heads/main')) + runs-on: + group: CWL CI isolated + labels: [self-hosted, linux, x64, cwlab-ci-isolated] + timeout-minutes: 30 + env: + NOEMA_CODEGRAPH_SANDBOX_SOURCE_IMAGE: gcr.io/distroless/java-base-debian13:nonroot + PYTHONPATH: ${{ github.workspace }}/reviewer:${{ github.workspace }}/packages/noema-core/src + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: init.defaultBranch + GIT_CONFIG_VALUE_0: main + defaults: + run: + working-directory: reviewer + steps: + - name: Validate fixed caller and required guest capabilities + id: guest_preflight + working-directory: ${{ runner.temp }} + shell: bash + env: + CALLER_REPOSITORY: ${{ github.repository }} + CALLER_EVENT: ${{ github.event_name }} + CALLER_REF: ${{ github.ref }} + PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + PR_BASE_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name }} + run: | + set -euo pipefail + test "$CALLER_REPOSITORY" = ContextualWisdomLab/noema + case "$CALLER_EVENT" in + pull_request) + test "$PR_HEAD_REPOSITORY" = "$CALLER_REPOSITORY" + test "$PR_BASE_REPOSITORY" = "$CALLER_REPOSITORY" ;; + push) test "$CALLER_REF" = refs/heads/main ;; + *) exit 1 ;; + esac + test "$(uname -s)" = Linux + test "$(uname -m)" = x86_64 + . /etc/os-release + test "$ID" = ubuntu + test "$VERSION_ID" = 24.04 + command -v git + command -v docker + docker info >/dev/null + NOEMA_JOB_TEMP="$RUNNER_TEMP/noema-reviewer-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-reviewer" + umask 077 + mkdir -- "$NOEMA_JOB_TEMP" + printf 'NOEMA_JOB_TEMP=%s\n' "$NOEMA_JOB_TEMP" >> "$GITHUB_ENV" + printf 'created=true\n' >> "$GITHUB_OUTPUT" + printf 'NPM_CONFIG_CACHE=%s/npm-cache\n' "$NOEMA_JOB_TEMP" >> "$GITHUB_ENV" + printf 'PIP_CACHE_DIR=%s/pip-cache\n' "$NOEMA_JOB_TEMP" >> "$GITHUB_ENV" + + - name: checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: verify exact checkout + shell: bash + env: + NOEMA_EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + if [[ ! "$NOEMA_EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + printf '::error::Invalid expected head SHA.\n' + exit 1 + fi + test "$(git rev-parse HEAD)" = "$NOEMA_EXPECTED_HEAD_SHA" + + - name: setup node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24.19.0" + + - name: setup python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: install (hash-pinned dependencies) + shell: bash + run: | + set -euo pipefail + python -m venv "$NOEMA_JOB_TEMP/quality-venv" + "$NOEMA_JOB_TEMP/quality-venv/bin/python" -m pip install --require-hashes --no-deps -r requirements-ci-hashes.txt + printf '%s/quality-venv/bin\n' "$NOEMA_JOB_TEMP" >> "$GITHUB_PATH" + + - name: test noema-core (100% line+branch coverage gate) + working-directory: packages/noema-core + run: python -m pytest + + - name: docstring coverage noema-core (100% gate) + working-directory: packages/noema-core + run: python -m interrogate -c pyproject.toml src/noema_core + + - name: test (100% line+branch coverage gate) + run: python -m pytest + + - name: docstring coverage (100% gate) + run: python -m interrogate -c pyproject.toml noema_reviewer + + - name: smoke-test installed reviewer wheel and sdist-to-wheel path + run: | + set -euo pipefail + wheel_dir="$NOEMA_JOB_TEMP/noema-reviewer-wheel" + sdist_dir="$NOEMA_JOB_TEMP/noema-reviewer-sdist" + sdist_wheel_dir="$NOEMA_JOB_TEMP/noema-reviewer-sdist-wheel" + direct_venv="$NOEMA_JOB_TEMP/noema-reviewer-install-smoke" + sdist_venv="$NOEMA_JOB_TEMP/noema-reviewer-sdist-install-smoke" + mkdir -p "$wheel_dir" "$sdist_dir" "$sdist_wheel_dir" + + python -m pip wheel . --no-deps --no-build-isolation --wheel-dir "$wheel_dir" + direct_wheel="$(find "$wheel_dir" -maxdepth 1 -type f -name 'noema_reviewer-*.whl' -print -quit)" + test -n "$direct_wheel" + + SDIST_DIR="$sdist_dir" SDIST_NAME_FILE="$NOEMA_JOB_TEMP/noema-reviewer-sdist-name" python - <<'PY' + import os + from pathlib import Path + from build_backend import build_sdist + + sdist_name = build_sdist(os.environ["SDIST_DIR"]) + Path(os.environ["SDIST_NAME_FILE"]).write_text(sdist_name, encoding="utf-8") + PY + sdist="$sdist_dir/$(cat "$NOEMA_JOB_TEMP/noema-reviewer-sdist-name")" + test -f "$sdist" + python -m pip wheel "$sdist" --no-deps --no-build-isolation --wheel-dir "$sdist_wheel_dir" + sdist_wheel="$(find "$sdist_wheel_dir" -maxdepth 1 -type f -name 'noema_reviewer-*.whl' -print -quit)" + test -n "$sdist_wheel" + + for contract in direct sdist; do + if [ "$contract" = direct ]; then + wheel="$direct_wheel" + venv_dir="$direct_venv" + else + wheel="$sdist_wheel" + venv_dir="$sdist_venv" + fi + python -m venv --system-site-packages "$venv_dir" + ( + cd "$NOEMA_JOB_TEMP" + PYTHONPATH='' "$venv_dir/bin/python" -m pip install --no-deps "$wheel" + PYTHONPATH='' "$venv_dir/bin/python" - <<'PY' + import hashlib + import os + from pathlib import Path + + import noema_core + import noema_core.agent + import noema_reviewer + from noema_reviewer.cli import parse_args + + canonical_agent = Path(os.environ["GITHUB_WORKSPACE"]) / "packages" / "noema-core" / "src" / "noema_core" / "agent.py" + installed_agent = Path(noema_core.agent.__file__) + assert hashlib.sha256(installed_agent.read_bytes()).digest() == hashlib.sha256(canonical_agent.read_bytes()).digest() + assert noema_core.NOEMA_PERSONA + assert noema_reviewer.build_agent is not None + assert parse_args([]).repo == "" + PY + ) + done + + - name: smoke-test isolated editable reviewer with locked runtime dependencies + run: | + set -euo pipefail + editable_venv="$NOEMA_JOB_TEMP/noema-reviewer-editable-smoke" + python -m venv "$editable_venv" + "$editable_venv/bin/python" -m pip install --require-hashes --no-deps -r requirements-ci-hashes.txt + "$editable_venv/bin/python" -m pip install --no-deps -e . + ( + cd "$NOEMA_JOB_TEMP" + PYTHONPATH='' "$editable_venv/bin/python" - <<'PY' + import noema_core + import noema_reviewer + + assert noema_core.build_agent is not None + assert noema_reviewer.build_agent is not None + PY + ) + + - name: install lock-pinned CodeGraph tooling for sandbox smoke test + env: + NPM_CONFIG_IGNORE_SCRIPTS: "true" + run: npm ci --prefix ../.github/codegraph --ignore-scripts --omit=dev --no-audit --no-fund + + - name: install Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: install Trivy + uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 + with: + version: v0.74.0 + cache: false + + - name: prepare authenticated and scanned CodeGraph sandbox image + run: bash ../scripts/prepare-codegraph-sandbox.sh + + - name: smoke-test the real no-network CodeGraph sandbox + run: | + set -euo pipefail + source_root="$NOEMA_JOB_TEMP/noema-codegraph-smoke" + mkdir -p "$source_root" + printf 'export const commercialReadiness = true;\n' >"$source_root/example.ts" + python - <<'PY' + import os + from noema_reviewer.sandbox import DockerCodeGraphRunner + + runner = DockerCodeGraphRunner() + source_root = os.path.join(os.environ["NOEMA_JOB_TEMP"], "noema-codegraph-smoke") + assert runner(["codegraph", "init", "-i"], source_root) == "" + assert runner(["codegraph", "sync"], source_root) == "" + assert runner(["codegraph", "status"], source_root) == "" + output = runner( + ["codegraph", "explore", "commercialReadiness"], + source_root, + ) + if ( + "Sandbox copied 1 files" not in output + or "## codegraph explore" not in output + or "No relevant code found" in output + or "export const commercialReadiness = true;" not in output + ): + raise SystemExit("CodeGraph sandbox smoke did not retrieve the indexed fixture") + print(output[:2000]) + PY + + + - name: Clean up job-private temporary state + if: ${{ always() && steps.guest_preflight.outputs.created == 'true' }} + working-directory: ${{ runner.temp }} + shell: bash + run: | + set -euo pipefail + expected="$RUNNER_TEMP/noema-reviewer-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-reviewer" + test "$NOEMA_JOB_TEMP" = "$expected" + rm -rf -- "$NOEMA_JOB_TEMP" diff --git a/tests/test_noema_reviewer_quality.py b/tests/test_noema_reviewer_quality.py new file mode 100644 index 0000000000..da40c5f757 --- /dev/null +++ b/tests/test_noema_reviewer_quality.py @@ -0,0 +1,126 @@ +"""Contract for the fixed-caller Noema reviewer quality supplier.""" +from pathlib import Path + +import yaml + +PATH = Path('.github/workflows/noema-reviewer-quality.yml') + + +def workflow(): + """Read the supplier as data; no workflow command is executed here.""" + return yaml.safe_load(PATH.read_text()) + + +def test_supplier_is_fixed_call_only_and_isolated(): + """Reject hosted/privileged fallbacks and caller-selected execution knobs.""" + data = workflow() + assert data.get('on', data.get(True)) == {'workflow_call': {}} + assert data['permissions'] == {'contents': 'read'} + job = data['jobs']['reviewer'] + assert job['runs-on'] == {'group': 'CWL CI isolated', 'labels': ['self-hosted', 'linux', 'x64', 'cwlab-ci-isolated']} + assert "github.repository == 'ContextualWisdomLab/noema'" in job['if'] + assert "github.event.pull_request.head.repo.full_name == github.repository" in job['if'] + assert "github.event_name == 'pull_request'" in job['if'] + assert "github.ref == 'refs/heads/main'" in job['if'] + assert 'secrets.' not in PATH.read_text() + + +def test_original_quality_gates_remain(): + """Keep Python, 100 percent gates, package-install and actual sandbox smoke.""" + job = workflow()['jobs']['reviewer'] + assert job['timeout-minutes'] == 30 + steps = {s['name']: s for s in job['steps']} + assert steps['setup python']['with']['python-version'] == '3.11' + assert steps['checkout']['with']['persist-credentials'] is False + assert 'github.event.pull_request.head.sha' in steps['checkout']['with']['ref'] + assert steps['test noema-core (100% line+branch coverage gate)']['run'] == 'python -m pytest' + assert steps['test (100% line+branch coverage gate)']['run'] == 'python -m pytest' + assert 'interrogate' in steps['docstring coverage (100% gate)']['run'] + assert '--require-hashes --no-deps' in steps['install (hash-pinned dependencies)']['run'] + for name in ['smoke-test installed reviewer wheel and sdist-to-wheel path', + 'smoke-test isolated editable reviewer with locked runtime dependencies', + 'prepare authenticated and scanned CodeGraph sandbox image', + 'smoke-test the real no-network CodeGraph sandbox']: + assert steps[name]['run'] + assert 'continue-on-error' not in steps[name] + assert 'DockerCodeGraphRunner' in steps['smoke-test the real no-network CodeGraph sandbox']['run'] + + +def test_isolated_tool_and_temporary_state_contract(): + """Require explicit Node/tool availability and job-scoped temporary paths.""" + steps = workflow()['jobs']['reviewer']['steps'] + names = [s['name'] for s in steps] + assert names.index('Validate fixed caller and required guest capabilities') < names.index('checkout') + assert names.index('setup node') < names.index('install lock-pinned CodeGraph tooling for sandbox smoke test') + node = next(s for s in steps if s['name'] == 'setup node') + assert node['with']['node-version'] == '24.19.0' + assert 'cache' not in node['with'] + assert 'if' not in node + assert steps[-1]['if'] == "${{ always() && steps.guest_preflight.outputs.created == 'true' }}" + assert steps[-1]['working-directory'] == '${{ runner.temp }}' + for step in steps: + if 'uses' in step: + assert len(step['uses'].split('@')[1]) == 40 + + +ORIGINAL_RUN_SHA256 = {'verify exact checkout': 'be4904078076d80c17548dd2dfe04ac8cf920754bb55aed58573c3a688b9c3d4', 'test noema-core (100% line+branch coverage gate)': '88cc0cc55eff4bf0ccdbc9b3d04ecefe2f8d4e10f4aafca9dfd5f467e78e53df', 'docstring coverage noema-core (100% gate)': '0b27ac45cf8a26c48e0491991e03fc314caef56ebe6e8f887b5909b73a63f27b', 'test (100% line+branch coverage gate)': '88cc0cc55eff4bf0ccdbc9b3d04ecefe2f8d4e10f4aafca9dfd5f467e78e53df', 'docstring coverage (100% gate)': 'd0a929f4820c4a1e4b5ab5e11ccdcb02bffd36941cda419887557e3096f62c1a', 'smoke-test installed reviewer wheel and sdist-to-wheel path': 'f620ec6efd06519941b58946a066c2ec826ea676515d76c4a83f9216b18f3936', 'smoke-test isolated editable reviewer with locked runtime dependencies': '6491c954049d08784fe3685760a67b00f3a371d3e57a5e53d0cc423143008273', 'install lock-pinned CodeGraph tooling for sandbox smoke test': '4951c132eec7f7723ce9420b0c7e866bbbb00433cdaa4abffe83e6036eefab72', 'prepare authenticated and scanned CodeGraph sandbox image': '63258b0d6b245ed2a2eeb9557c1202a2d015e656cfe218409c5fd91f4816e9dc', 'smoke-test the real no-network CodeGraph sandbox': '0374b422afe3462054024d793908c2d138547cb6fa3db9d6421a574d7fa1ef4a'} + + +def test_original_run_bodies_are_preserved(): + """Bind original execution bodies, allowing only job-private temp relocation.""" + import hashlib + + steps = {step["name"]: step for step in workflow()["jobs"]["reviewer"]["steps"]} + for name, digest in ORIGINAL_RUN_SHA256.items(): + assert hashlib.sha256(steps[name]["run"].encode()).hexdigest() == digest + + +def test_foreign_or_unsupported_caller_stops_before_tools(tmp_path): + """Execute the actual preflight and deny callers before any tool invocation.""" + import os + import subprocess + + script = workflow()['jobs']['reviewer']['steps'][0]['run'] + tool_dir = tmp_path / 'bin' + tool_dir.mkdir() + marker = tmp_path / 'tool-invoked' + for name in ['uname', 'git', 'docker', 'mkdir']: + tool = tool_dir / name + tool.write_text(f'#!/bin/sh\n/usr/bin/touch "{marker}"\nexit 97\n') + tool.chmod(0o700) + baseline = dict(os.environ, PATH=str(tool_dir), + CALLER_REPOSITORY='ContextualWisdomLab/noema', + CALLER_EVENT='pull_request', CALLER_REF='refs/pull/1/merge', + PR_HEAD_REPOSITORY='ContextualWisdomLab/noema', + PR_BASE_REPOSITORY='ContextualWisdomLab/noema') + for change in [dict(CALLER_REPOSITORY='other/repo'), + dict(PR_HEAD_REPOSITORY='fork/noema'), + dict(PR_BASE_REPOSITORY='other/repo'), + dict(CALLER_EVENT='workflow_dispatch'), + dict(CALLER_EVENT='pull_request_target'), + dict(CALLER_EVENT='push', CALLER_REF='refs/heads/topic')]: + result = subprocess.run(['/bin/bash', '-c', script], + env=dict(baseline, **change), capture_output=True) + assert result.returncode != 0 + assert not marker.exists() + + +def test_cleanup_refuses_unrelated_directory(tmp_path): + """Run the actual cleaner and preserve paths outside its exact job identity.""" + import os + import subprocess + + script = workflow()['jobs']['reviewer']['steps'][-1]['run'] + owned = tmp_path / 'noema-reviewer-123-2-reviewer' + foreign = tmp_path / 'another-job' + owned.mkdir() + foreign.mkdir() + for directory in [owned, foreign]: + (directory / 'sentinel').write_text('retain unless owned') + env = dict(os.environ, RUNNER_TEMP=str(tmp_path), GITHUB_RUN_ID='123', + GITHUB_RUN_ATTEMPT='2', NOEMA_JOB_TEMP=str(foreign)) + denied = subprocess.run(['/bin/bash', '-c', script], env=env, capture_output=True) + assert denied.returncode != 0 and foreign.exists() and owned.exists() + accepted = subprocess.run(['/bin/bash', '-c', script], + env=dict(env, NOEMA_JOB_TEMP=str(owned)), capture_output=True) + assert accepted.returncode == 0 and not owned.exists() and foreign.exists()