From b67eda5c41945539d9393f30005f31e211819c83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:02:47 +0900 Subject: [PATCH 1/3] fix(ci): keep metadata reconciliation tests focused Signed-off-by: Seongho Bae --- .../repository-metadata-reconcile.yml | 4 +- tests/test_repository_metadata_workflow.py | 75 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repository-metadata-reconcile.yml b/.github/workflows/repository-metadata-reconcile.yml index e05a8b155a..d8fa5eef3c 100644 --- a/.github/workflows/repository-metadata-reconcile.yml +++ b/.github/workflows/repository-metadata-reconcile.yml @@ -73,8 +73,10 @@ jobs: --include=scripts/ci/reconcile_repository_metadata.py \ -m pytest -q \ tests/test_repository_metadata_reconciliation.py \ + tests/test_repository_metadata_convergence.py \ tests/test_repository_metadata_identity.py \ tests/test_repository_metadata_live_verification.py \ + tests/test_repository_metadata_workflow.py \ tests/test_repository_metadata_workflow_pages.py python -m coverage report \ --fail-under=100 \ @@ -85,6 +87,7 @@ jobs: --branch \ --include=scripts/ci/reconcile_repository_labels.py \ -m pytest -q \ + tests/test_repository_label_taxonomy.py \ tests/test_repository_label_reconciliation.py \ tests/test_repository_label_convergence.py \ tests/test_repository_label_identity.py \ @@ -97,7 +100,6 @@ jobs: --fail-under 100 \ scripts/ci/reconcile_repository_metadata.py \ scripts/ci/reconcile_repository_labels.py - python -m pytest -q git diff --check apply: diff --git a/tests/test_repository_metadata_workflow.py b/tests/test_repository_metadata_workflow.py index 7b41a667d9..ee32a6a71c 100644 --- a/tests/test_repository_metadata_workflow.py +++ b/tests/test_repository_metadata_workflow.py @@ -1,12 +1,35 @@ """Static contracts for the privileged repository metadata workflow.""" +import os from pathlib import Path +import stat +import subprocess +import textwrap ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "repository-metadata-reconcile.yml" +def _metadata_test_script() -> str: + """Extract the metadata test step's executable shell body.""" + + source = WORKFLOW.read_text(encoding="utf-8") + step = source.split( + " - name: Run metadata contract tests at repository quality gates\n", 1 + )[1].split("\n\n apply:\n", 1)[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +def _write_command_recorder(path: Path) -> None: + """Create a fake executable that records argv without running project code.""" + + path.write_text( + '#!/bin/sh\nprintf "%s\\n" "$*" >> "$COMMAND_LOG"\n', encoding="utf-8" + ) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + def test_metadata_apply_uses_dedicated_least_privilege_credential() -> None: """Repository settings writes must not reuse the review/merge credential.""" source = WORKFLOW.read_text(encoding="utf-8") @@ -16,3 +39,55 @@ def test_metadata_apply_uses_dedicated_least_privilege_credential() -> None: assert "secrets.PR_REVIEW_MERGE_TOKEN" not in apply_source assert "Require dedicated repository settings credential" in apply_source assert 'test -n "${GH_TOKEN}"' in apply_source + + +def test_metadata_quality_step_runs_every_owned_test_without_the_full_suite( + tmp_path: Path, +) -> None: + """The hourly job must run every metadata-owner test and no unrelated suite.""" + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + for command_name in ("python", "git"): + _write_command_recorder(fake_bin / command_name) + command_log = tmp_path / "commands.log" + environment = os.environ.copy() + environment.update( + PATH=f"{fake_bin}{os.pathsep}{environment['PATH']}", + COMMAND_LOG=str(command_log), + ) + + result = subprocess.run( + ["bash", "-c", _metadata_test_script()], + cwd=ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + commands = command_log.read_text(encoding="utf-8").splitlines() + pytest_commands = [command for command in commands if "-m pytest -q" in command] + assert pytest_commands + assert "-m pytest -q" not in pytest_commands + owner_tests = { + "tests/test_repository_metadata_reconciliation.py", + "tests/test_repository_metadata_convergence.py", + "tests/test_repository_metadata_identity.py", + "tests/test_repository_metadata_live_verification.py", + "tests/test_repository_metadata_workflow.py", + "tests/test_repository_metadata_workflow_pages.py", + "tests/test_repository_label_taxonomy.py", + "tests/test_repository_label_reconciliation.py", + "tests/test_repository_label_convergence.py", + "tests/test_repository_label_identity.py", + "tests/test_repository_label_live_verification.py", + } + invoked_tests = { + argument + for command in pytest_commands + for argument in command.split() + if argument.startswith("tests/") + } + assert invoked_tests == owner_tests From 8a3c2bb95ee6065c8a15515de3f909d5c476c885 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:12:45 +0900 Subject: [PATCH 2/3] test(ci): reject disguised full metadata suites Signed-off-by: Seongho Bae --- tests/test_repository_metadata_workflow.py | 131 ++++++++++++++------- 1 file changed, 86 insertions(+), 45 deletions(-) diff --git a/tests/test_repository_metadata_workflow.py b/tests/test_repository_metadata_workflow.py index ee32a6a71c..f3f7f1c071 100644 --- a/tests/test_repository_metadata_workflow.py +++ b/tests/test_repository_metadata_workflow.py @@ -4,21 +4,36 @@ from pathlib import Path import stat import subprocess -import textwrap + +import pytest + +from tests.test_opencode_workflow_shell_syntax import _extract_run_block ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "repository-metadata-reconcile.yml" +OWNER_TESTS = { + "tests/test_repository_metadata_reconciliation.py", + "tests/test_repository_metadata_convergence.py", + "tests/test_repository_metadata_identity.py", + "tests/test_repository_metadata_live_verification.py", + "tests/test_repository_metadata_workflow.py", + "tests/test_repository_metadata_workflow_pages.py", + "tests/test_repository_label_taxonomy.py", + "tests/test_repository_label_reconciliation.py", + "tests/test_repository_label_convergence.py", + "tests/test_repository_label_identity.py", + "tests/test_repository_label_live_verification.py", +} def _metadata_test_script() -> str: """Extract the metadata test step's executable shell body.""" - source = WORKFLOW.read_text(encoding="utf-8") - step = source.split( - " - name: Run metadata contract tests at repository quality gates\n", 1 - )[1].split("\n\n apply:\n", 1)[0] - return textwrap.dedent(step.split(" run: |\n", 1)[1]) + return _extract_run_block( + WORKFLOW.read_text(encoding="utf-8"), + "Run metadata contract tests at repository quality gates", + ) def _write_command_recorder(path: Path) -> None: @@ -30,21 +45,8 @@ def _write_command_recorder(path: Path) -> None: path.chmod(path.stat().st_mode | stat.S_IXUSR) -def test_metadata_apply_uses_dedicated_least_privilege_credential() -> None: - """Repository settings writes must not reuse the review/merge credential.""" - source = WORKFLOW.read_text(encoding="utf-8") - - assert "secrets.CWL_REPOSITORY_METADATA_TOKEN" in source - apply_source = source.split(" apply:", 1)[1] - assert "secrets.PR_REVIEW_MERGE_TOKEN" not in apply_source - assert "Require dedicated repository settings credential" in apply_source - assert 'test -n "${GH_TOKEN}"' in apply_source - - -def test_metadata_quality_step_runs_every_owned_test_without_the_full_suite( - tmp_path: Path, -) -> None: - """The hourly job must run every metadata-owner test and no unrelated suite.""" +def _record_metadata_commands(tmp_path: Path) -> list[str]: + """Execute the workflow shell with inert commands and return recorded argv.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -56,7 +58,6 @@ def test_metadata_quality_step_runs_every_owned_test_without_the_full_suite( PATH=f"{fake_bin}{os.pathsep}{environment['PATH']}", COMMAND_LOG=str(command_log), ) - result = subprocess.run( ["bash", "-c", _metadata_test_script()], cwd=ROOT, @@ -65,29 +66,69 @@ def test_metadata_quality_step_runs_every_owned_test_without_the_full_suite( capture_output=True, text=True, ) - assert result.returncode == 0, result.stderr - commands = command_log.read_text(encoding="utf-8").splitlines() + return command_log.read_text(encoding="utf-8").splitlines() + + +def _assert_metadata_commands(commands: list[str]) -> None: + """Require the exact metadata-owner test set and no bare full-suite command.""" + pytest_commands = [command for command in commands if "-m pytest -q" in command] assert pytest_commands - assert "-m pytest -q" not in pytest_commands - owner_tests = { - "tests/test_repository_metadata_reconciliation.py", - "tests/test_repository_metadata_convergence.py", - "tests/test_repository_metadata_identity.py", - "tests/test_repository_metadata_live_verification.py", - "tests/test_repository_metadata_workflow.py", - "tests/test_repository_metadata_workflow_pages.py", - "tests/test_repository_label_taxonomy.py", - "tests/test_repository_label_reconciliation.py", - "tests/test_repository_label_convergence.py", - "tests/test_repository_label_identity.py", - "tests/test_repository_label_live_verification.py", - } - invoked_tests = { - argument - for command in pytest_commands - for argument in command.split() - if argument.startswith("tests/") - } - assert invoked_tests == owner_tests + invoked_tests: set[str] = set() + for command in pytest_commands: + targets = { + argument + for argument in command.split() + if argument.startswith("tests/") and argument.endswith(".py") + } + assert targets + assert targets <= OWNER_TESTS + invoked_tests.update(targets) + assert invoked_tests == OWNER_TESTS + assert sum(command.startswith("-m coverage run --branch") for command in commands) == 2 + assert sum( + command.startswith("-m coverage report --fail-under=100") + for command in commands + ) == 2 + assert commands.count("-m coverage erase") == 1 + interrogate = next( + command for command in commands if command.startswith("-m interrogate ") + ) + assert "--fail-under 100" in interrogate + assert "scripts/ci/reconcile_repository_metadata.py" in interrogate + assert "scripts/ci/reconcile_repository_labels.py" in interrogate + assert commands.count("diff --check") == 1 + + +def test_metadata_apply_uses_dedicated_least_privilege_credential() -> None: + """Repository settings writes must not reuse the review/merge credential.""" + source = WORKFLOW.read_text(encoding="utf-8") + + assert "secrets.CWL_REPOSITORY_METADATA_TOKEN" in source + apply_source = source.split(" apply:", 1)[1] + assert "secrets.PR_REVIEW_MERGE_TOKEN" not in apply_source + assert "Require dedicated repository settings credential" in apply_source + assert 'test -n "${GH_TOKEN}"' in apply_source + + +def test_metadata_quality_step_runs_every_owned_test_without_the_full_suite( + tmp_path: Path, +) -> None: + """The hourly job must run every metadata-owner test and no unrelated suite.""" + + commands = _record_metadata_commands(tmp_path) + _assert_metadata_commands(commands) + + +def test_metadata_quality_step_rejects_flagged_bare_full_suite(tmp_path: Path) -> None: + """A pytest flag must not disguise an untargeted repository-wide invocation.""" + + commands = _record_metadata_commands(tmp_path) + pytest_index = next( + index for index, command in enumerate(commands) if "-m pytest -q" in command + ) + commands[pytest_index] = "-m pytest -q -W error" + + with pytest.raises(AssertionError): + _assert_metadata_commands(commands) From bf5dfef9dea70ce85391200a4fcae1ffe4fb1688 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:32:13 +0900 Subject: [PATCH 3/3] docs(ci): separate queue observations from causal claims --- ...tions-plan-concurrency-ceiling-20260903.md | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/doctoring/actions-plan-concurrency-ceiling-20260903.md b/docs/doctoring/actions-plan-concurrency-ceiling-20260903.md index 39796beb44..1a9806401b 100644 --- a/docs/doctoring/actions-plan-concurrency-ceiling-20260903.md +++ b/docs/doctoring/actions-plan-concurrency-ceiling-20260903.md @@ -1,11 +1,16 @@ -# Doctoring record: the org's GitHub Actions concurrency ceiling is a plan-level quota, not a workflow defect (2026-09-03) +# Actions 용량 한도와 workflow 부하: 관측 및 추론의 경계 + +2026-09-07 정정: 아래 2026-09-03 관측은 역사적 기록으로 보존한다. +용량 한도와 workflow 부하를 배타적 원인으로 단정했던 해석만 좁혔다. +현재 운영 상태나 이번 수리의 효과를 증명하는 문서가 아니다. - **Date:** 2026-09-03 - **Subject:** two peer sessions independently observed the org's GitHub Actions run queue growing rather than shrinking this week and, in that tick, proposed auditing/consolidating/centralizing workflow files - across the org as the fix. Before either session sank time into that plan, this root cause needed a - durable record: the actual bottleneck this session identified is a **plan-level concurrent-job quota**, - not workflow duplication, and consolidating workflow files cannot lift it. + across the org as the fix. 당시 사용자 보고는 동시 job 한도에 가까운 사용량을 + 뒷받침한다. 그러나 한도 포화와 중복 실행·장시간 점유·admission 결함은 함께 + 존재할 수 있다. 이 기록만으로 지배적 원인을 확정하거나 workflow 부하 원인을 + 배제할 수 없다. 파일 통합은 계약상 한도를 높이지 않지만 실행량은 줄일 수 있다. - **Decision record:** none in `docs/adr/` — this is a diagnostic/root-cause finding for the org owner's awareness and eventual plan-tier decision, not an architecture decision this repository can make. - **PR:** see the PR that carries this commit. @@ -34,29 +39,24 @@ gh api "repos/ContextualWisdomLab//actions/runs?status=in_progress&per_pag gh api "repos/ContextualWisdomLab//actions/runs?status=queued&per_page=1" --jq '.total_count' ``` -| Repository | `in_progress` | `queued` | +| Repository | `in_progress` workflow runs | `queued` workflow runs | |---|---|---| | `.github` | 5 | 1,877 | | `contextual-orchestrator` | 0 | 727 | | `naruon` | 5 | 416 | | **Total (3-repo sample)** | **10** | **3,020** | -This is a deliberately small sample, not a full 63-repo census — an attempted full sweep across every -non-archived, non-fork repository (the same corpus as the 2026-09-02 workflow-duplication audit) hung -indefinitely on this run and was aborted; a post-hoc `gh api rate_limit` check immediately after showed -5,000/5,000 REST calls remaining, so the hang was not caused by hitting the org's shared REST rate limit -(consistent with this session's standing practice of preferring REST over GraphQL to avoid that limit) — -its actual cause is undetermined and not investigated further here, since the 3-repo sample already -establishes the pattern this record needs. - -The pattern itself is the useful signal: single-digit `in_progress` counts (5, 0, 5) against -quadruple-digit `queued` counts (1,877; 727; 416) in the same moment, across independently-owned -repositories, each triggering its own workflows on its own schedule. That shape — many jobs queued, -very few ever concurrently running — is exactly what a hard, roughly-constant, **org-wide** (not -per-repository) concurrent-job ceiling produces, and is hard to explain by per-repository causes alone -(each repository's own workflow volume, trigger frequency, and CI design differ substantially). It is -consistent with, though does not by itself prove, the specific 58-60/60 figure from the primary evidence -above. +이 표는 당시 63개 저장소 전수가 아닌 세 저장소 표본이다. 당시 전체 조회는 +멈춘 상태가 지속돼 중단했고, 이후 `gh api rate_limit`은 5,000/5,000을 반환했다. +실패 당시 응답과 헤더가 보존되지 않아 원인은 미확정이다. 사후 잔여량만으로 +같은 인증 주체·resource의 실패 당시 primary quota, reset 또는 secondary 제한을 +배제할 수 없다. + +이 API가 세는 대상은 job이나 점유 runner가 아니라 workflow run이다. Run 하나에 +여러 job이 있을 수 있고, in-progress run 안에서도 job이 대기할 수 있다. +세 저장소의 비원자적 표본만으로 조직 전체 동시 job 수, 실제 한도 포화 또는 +저장소별 원인과 조직 공통 원인의 우선순위를 식별할 수 없다. 사용자 보고의 +58–60/60과 양립하는 관측이지만 그 수치를 독립적으로 증명하지는 않는다. ## Relationship to other queue-related findings already in this repository @@ -86,29 +86,37 @@ on this tick as *the* fix for the growing queue — is real hygiene and can redu runs triggered* (fewer redundant CI paths competing for the same slots), which helps the queue drain somewhat faster once jobs are submitted. It does **not** change how many jobs GitHub will run concurrently for this organization at once: that number is set by the plan tier, not by how many `.yml` files exist or -how many of them are centralized versus per-repository. A large cross-repo consolidation-and-deletion -effort undertaken on the theory that it would resolve the backlog would be solving the wrong layer of the -problem, at real cost (each deletion needs branch-protection `required_status_checks` re-verified per -repo, and any repo-specific `with:` tuning preserved or intentionally dropped). +how many of them are centralized versus per-repository. + +저장소 간 workflow 통합·삭제는 파일 재배치와 실제 중복 trigger·job·점유시간 감소를 구분해야 한다. +후자는 한도를 바꾸지 않고도 backlog를 줄일 수 있다. 삭제할 때는 저장소별 +branch-protection required checks와 입력 조정값을 재검증하고 보존해야 한다. ## Recommendation -This is a plan/billing decision, not a code change either agent session can make: raising the concurrent-job -ceiling (a higher GitHub plan tier, purchasing additional included concurrency, or provisioning -self-hosted/larger runners with their own separate capacity pool) is the org owner's call to make with the -actual billing page in front of them, not something to infer further from repository-side evidence. -Workflow consolidation remains worth pursuing for its own, independent hygiene reasons (see -`docs/doctoring/ci-workflow-duplication-audit-20260902.md` for what is and is not already duplicated -org-wide) — but should not be scoped or prioritized as *the* fix for the current backlog growth. +용량 확대와 workflow 부하 감소는 별도 선택지다. 요금제·유료 용량·runner 추가는 +실제 설정과 비용을 확인한 조직 소유자의 결정이 필요하다. 그 판단과 별개로 중복 +실행, 불필요한 대기, stale-head 실행과 admission 결함은 기존 권한 안에서 수리한다. +정확한 전후 revision, trigger, job 실행시간, 취소 원인과 queue 표본 범위를 남겨 +효과를 검증한다. 로컬 테스트 통과나 파일 수 감소만으로 운영 적체 해소를 주장하지 않는다. ## Audit trail - User-reported screenshot of the organization's Actions usage view, shared earlier in this session (primary source for the 58-60/60 figure; not independently re-verifiable from this record alone). - Live `gh api` sample gathered 2026-09-03 for this record (table above); `gh api rate_limit` confirmed - 5,000/5,000 REST calls remaining immediately after the aborted full-org sweep, ruling out rate-limiting - as the sweep's failure cause. + 5,000/5,000 REST calls remaining immediately after the aborted full-org sweep. 사후 해당 + primary quota 소진이 관측되지 않았다는 뜻이며 실패 당시 rate-limit을 배제하지 않는다. - `docs/product-technical-gap-baseline.md` — the 2026-08-31 chained-poller-removal entry and the `ubuntu-latest` starved-image entry, both cross-referenced above. - `docs/doctoring/ci-workflow-duplication-audit-20260902.md` — the org-wide workflow-duplication sweep this record's "Implication" section points back to. + +## 정정 근거 + +- GitHub. (n.d.). *Using jobs in a workflow*. Retrieved September 7, 2026, from + https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-jobs + — 하나의 workflow에 여러 job이 존재할 수 있으므로 두 개수를 구분한다. +- GitHub. (n.d.). *Rate limits for the REST API*. Retrieved September 7, 2026, from + https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api + — 인증 방식과 resource별 primary 제한, 별도 secondary 제한 및 조회 한계를 구분한다.