diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 71049694..7e628c3c 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -111,13 +111,29 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Compute gate deadline + # Keep the gate deadline 15 minutes under this job's + # timeout-minutes so a starved job fails at the gate with a + # queue/ledger verdict instead of at the workflow cancel with + # nothing. Floor of 300s. + id: gate + run: | + deadline=$(( (${{ inputs.timeout }} - 15) * 60 )) + [ "$deadline" -ge 300 ] || deadline=300 + echo "deadline=$deadline" >> "$GITHUB_OUTPUT" + - name: Acquire memory slot - # Coordinated admission gate. Takes an flock'd reservation on the - # runner host's RAM budget so the pool can scale past the point where - # every concurrent slot fits at peak memory_mb. Times out and proceeds - # without a reservation rather than failing — better one OOM than a - # stuck queue. - run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + # Single-authority admission gate: MemTotal − reserve − committed + # limits.memory − pending reservations, FIFO with bounded + # overtakes. create.yml converts the reservation into committed + # capacity right after the containers launch. + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: acquire + molecule-scenario: ${{ matrix.scenario }} + deadline-seconds: ${{ steps.gate.outputs.deadline }} + incus-host: ${{ secrets.INCUS_HOST }} + ssh-key: ${{ runner.temp }}/molecule_id_ed25519 - name: Converge run: molecule converge -s ${{ matrix.scenario }} @@ -158,11 +174,13 @@ jobs: DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - name: Release memory slot - # Runs whether converge/verify/idempotence passed or failed, so the - # reservation is freed promptly. Stale reservations (>1h) are also - # GC'd by the next acquire under the same lock as a safety net. + # Runs whether converge/verify/idempotence passed or failed, so + # the reservation is freed promptly. Normally a no-op: create.yml + # already converted the reservation at container launch. if: always() - run: bash scripts/wait-for-memory.sh release + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: release - name: Collect and upload diagnostics if: failure() diff --git a/.github/workflows/test_elasticsearch_upgrade.yml b/.github/workflows/test_elasticsearch_upgrade.yml index 7189f28f..0fc4a84c 100644 --- a/.github/workflows/test_elasticsearch_upgrade.yml +++ b/.github/workflows/test_elasticsearch_upgrade.yml @@ -113,7 +113,18 @@ jobs: ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - name: Acquire memory slot - run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + # Single-authority admission gate: MemTotal − reserve − committed + # limits.memory − pending reservations, FIFO with bounded + # overtakes. Fails after its 45-min deadline instead of barging + # in — a starved job is an explicit retryable failure, not an + # OOM risk. create.yml converts the reservation into committed + # capacity right after the containers launch. + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: acquire + molecule-scenario: ${{ matrix.scenario }} + incus-host: ${{ secrets.INCUS_HOST }} + ssh-key: ${{ runner.temp }}/molecule_id_ed25519 - name: Test with molecule run: molecule test -s ${{ matrix.scenario }} --destroy=never @@ -129,8 +140,13 @@ jobs: DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - name: Release memory slot + # Runs whether converge/verify/idempotence passed or failed, so + # the reservation is freed promptly. Normally a no-op: create.yml + # already converted the reservation at container launch. if: always() - run: bash scripts/wait-for-memory.sh release + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: release - name: Collect and upload diagnostics if: failure() @@ -235,7 +251,18 @@ jobs: ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - name: Acquire memory slot - run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + # Single-authority admission gate: MemTotal − reserve − committed + # limits.memory − pending reservations, FIFO with bounded + # overtakes. Fails after its 45-min deadline instead of barging + # in — a starved job is an explicit retryable failure, not an + # OOM risk. create.yml converts the reservation into committed + # capacity right after the containers launch. + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: acquire + molecule-scenario: ${{ matrix.scenario }} + incus-host: ${{ secrets.INCUS_HOST }} + ssh-key: ${{ runner.temp }}/molecule_id_ed25519 - name: Test with molecule run: molecule test -s ${{ matrix.scenario }} --destroy=never @@ -251,8 +278,13 @@ jobs: DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - name: Release memory slot + # Runs whether converge/verify/idempotence passed or failed, so + # the reservation is freed promptly. Normally a no-op: create.yml + # already converted the reservation at container launch. if: always() - run: bash scripts/wait-for-memory.sh release + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: release - name: Collect and upload diagnostics if: failure() diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index 393778bc..382014d6 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -91,18 +91,15 @@ jobs: strategy: fail-fast: false - # Each scenario/distro/release combo needs 15-25 GB of container RAM on - # the shared incus-ci host. The memory-capacity gate in shared/create.yml - # eventually admits every job, but without a matrix-level cap the full - # 16-combo PR matrix would swamp the 131 GB host and starve the biggest - # scenarios (elasticstack_default, cert_renewal). 6 concurrent slots was - # the previous target, sized for the intended 90-100 GB steady state. - # Observed reality after landing several PRs concurrently: with all six - # slots filled, the 20 GB + 13.8 GB scenarios still lose the memory - # race and time out at the 45-min gate deadline. Halving to 3 keeps - # committed memory well under host capacity and lets the heavy jobs - # actually acquire without starvation. Wall clock roughly doubles. - max-parallel: 3 + # Each scenario/distro/release combo needs 15-25 GB of container RAM + # on the shared incus-ci host. The admission gate (FIFO with bounded + # overtakes, fail-fast) now guarantees heavy scenarios can't be + # starved by smaller jobs, so the cap is back to the throughput + # target it was originally sized for. The 3-slot value was a + # workaround for gate starvation that no longer exists; if the + # nightly storm shows starved jobs or OOMs, revert this commit + # first and investigate the gate second. + max-parallel: 6 matrix: # Standardise on rockylinux10 (not 9) for PR runs to match the rest # of the workflows. Rocky 10 exercises the EL≥9 branch in @@ -164,12 +161,18 @@ jobs: ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - name: Acquire memory slot - # Coordinated admission gate — reserves the scenario's memory_mb on - # the shared incus-ci host so the biggest scenarios don't trample - # each other or the parallel test_role_* / test_elasticsearch_* - # workflows. This step was missing from the standalone workflow; - # the reusable molecule.yml wrapper has always had it. - run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + # Single-authority admission gate: MemTotal − reserve − committed + # limits.memory − pending reservations, FIFO with bounded + # overtakes. Fails after its 45-min deadline instead of barging + # in — a starved job is an explicit retryable failure, not an + # OOM risk. create.yml converts the reservation into committed + # capacity right after the containers launch. + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: acquire + molecule-scenario: ${{ matrix.scenario }} + incus-host: ${{ secrets.INCUS_HOST }} + ssh-key: ${{ runner.temp }}/molecule_id_ed25519 - name: Converge run: molecule converge -s ${{ matrix.scenario }} @@ -210,11 +213,13 @@ jobs: DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - name: Release memory slot - # Runs whether converge/verify/idempotence passed or failed, so the - # reservation is freed promptly. Stale reservations (>1h) are also - # GC'd by the next acquire under the same lock as a safety net. + # Runs whether converge/verify/idempotence passed or failed, so + # the reservation is freed promptly. Normally a no-op: create.yml + # already converted the reservation at container launch. if: always() - run: bash scripts/wait-for-memory.sh release + uses: Oddly/incus-memory-gate@ce1c0240b0076db36b0b5b7c439690a7076d9de7 # v1.0.1 + with: + mode: release - name: Collect and upload diagnostics if: failure() diff --git a/CLAUDE.md b/CLAUDE.md index d55da48d..c4fb92bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,14 +149,16 @@ per-OS variable files live at with a fallback to `{{ ansible_facts.os_family }}.yml`, loaded via `include_vars` with `with_first_found` (see `roles/elasticsearch/tasks/main.yml`). -The heaviest scenarios by declared per-scenario memory in -`scripts/wait-for-memory.sh`: `elasticstack_default` (20 GB), +The heaviest scenarios by summed platform `memory_mb` in +`molecule//molecule.yml` (which is exactly what the +Oddly/incus-memory-gate action admits on — there is no separate table +to keep in sync): `elasticstack_default` (20 GB), `elasticsearch_roles_calculation` (16 GB), `es_kibana` (13.8 GB), -`cert_renewal` (10.5 GB). `test_full_stack.yml` uses `max-parallel: 3` -because 6-way concurrency across the top four starved the biggest -scenarios on the shared incus-ci host — the reasoning and memory -arithmetic are in the concurrency-drop commit message (`git log -.github/workflows/test_full_stack.yml`). +`cert_renewal` (10.5 GB). `test_full_stack.yml` uses `max-parallel: 6`; the admission gate (the +Oddly/incus-memory-gate action: FIFO, bounded overtakes, fail-fast) +keeps the heaviest scenarios from being starved on the shared +incus-ci host — the history of the 6→3→6 moves is in `git log +.github/workflows/test_full_stack.yml`. ## Reviewing PRs diff --git a/docs/superpowers/plans/2026-08-12-memory-gate-redesign.md b/docs/superpowers/plans/2026-08-12-memory-gate-redesign.md new file mode 100644 index 00000000..d633feca --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-memory-gate-redesign.md @@ -0,0 +1,1353 @@ +# Memory-Gate Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the memory gate as a reusable public repo (`Oddly/incus-memory-gate`: script + composite action + hermetic test suite + own CI), and convert oddly/elasticstack into its first consumer (create.yml reservation conversion, container limit fixes, thin `uses:` wiring, max-parallel back to 6). + +**Architecture:** The gate is a single-authority admission decision — `MemTotal − reserve − Σ committed limits.memory − Σ pending reservations` — with FIFO queueing, bounded overtakes (K=10), and fail-fast at a deadline. Reservations convert into committed capacity when the consumer's launch step deletes the reservation file right after `incus launch` (contract: file `r.` in the gate dir, content `