From 3e6081ce1f74e8892400380e4d3a3e91e24becdf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:25:17 +0000 Subject: [PATCH 1/5] ci(conformance): retry prod lane once (GHA-native), flag retried passes Co-Authored-By: bot_apk --- .github/workflows/conformance.yml | 46 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1f1b600..aab28bf 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -240,8 +240,10 @@ jobs: prod-lane: runs-on: ubuntu-latest # Prod scenarios are dominated by the ~30s duplicate-settle window (~40s - # each), so allow generous headroom over the fault lane. - timeout-minutes: 15 + # each), so allow generous headroom over the fault lane. Sized to fit TWO + # prod attempts (the runner step retries once, per-attempt timeout_minutes + # below) plus checkout/build setup. + timeout-minutes: 25 steps: - name: Gate (secrets + fork-safety) id: gate @@ -318,22 +320,40 @@ jobs: # with RAINDROP_WRITE_KEY; the runner polls RAINDROP_QUERY_URL with # RAINDROP_QUERY_API_KEY. Endpoints + credentials are scoped to this # step so no other step in the job can read them. + # + # GHA-native retry (nick-fields/retry): the prod lane crosses the real + # network (ingest + Query API readback), so a transient blip can fail an + # otherwise-conformant run. Retry the prod lane ONCE — max 2 attempts, + # green on a single pass — while the fault lane (a separate job) is never + # retried. The runner's `exit_code`/`total_attempts` outputs drive the + # summary + the flake annotation below; a retried pass is never silent. + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 env: RAINDROP_SINK_URL: https://api.raindrop.ai RAINDROP_WRITE_KEY: ${{ secrets.RAINDROP_WRITE_KEY }} RAINDROP_QUERY_URL: https://query.raindrop.ai RAINDROP_QUERY_API_KEY: ${{ secrets.RAINDROP_QUERY_API_KEY }} - run: | - set +e - node harness/runner/dist/src/index.js \ - --driver "$DRIVER_BIN" \ - --failures conformance/failures.txt \ - --lane prod \ - --report "$RUNNER_TEMP/report.json" \ - 2>&1 | tee "$RUNNER_TEMP/runner-output.txt" - code=${PIPESTATUS[0]} - echo "exit_code=$code" >> "$GITHUB_OUTPUT" - exit "$code" + with: + max_attempts: 2 + # Per-attempt cap; the job timeout-minutes above is sized for 2 of them. + timeout_minutes: 10 + retry_wait_seconds: 15 + shell: bash + command: | + set +e + node harness/runner/dist/src/index.js \ + --driver "$DRIVER_BIN" \ + --failures conformance/failures.txt \ + --lane prod \ + --report "$RUNNER_TEMP/report.json" \ + 2>&1 | tee "$RUNNER_TEMP/runner-output.txt" + exit "${PIPESTATUS[0]}" + + - name: Flag prod-lane flake (passed on retry) + # A prod pass that needed attempt 2 is a flake — surface it loudly so a + # retried green is visible in the run summary, never silent. + if: steps.gate.outputs.run == 'true' && steps.runner.outputs.total_attempts == '2' && steps.runner.outputs.exit_code == '0' + run: echo "::warning::raindrop-rust prod lane passed on retry (flake)" - name: Report result if: always() From 7fcd1f598877a721a99bfae61880972a3832f484 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:41:19 +0000 Subject: [PATCH 2/5] ci(conformance): raise prod retry per-attempt cap to 20m and job cap to 45m Co-Authored-By: bot_apk --- .github/workflows/conformance.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index aab28bf..ffa1ea0 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -241,9 +241,9 @@ jobs: runs-on: ubuntu-latest # Prod scenarios are dominated by the ~30s duplicate-settle window (~40s # each), so allow generous headroom over the fault lane. Sized to fit TWO - # prod attempts (the runner step retries once, per-attempt timeout_minutes - # below) plus checkout/build setup. - timeout-minutes: 25 + # full 20-min prod attempts (the runner step retries once, per-attempt + # timeout_minutes below) plus the retry wait and checkout/build setup. + timeout-minutes: 45 steps: - name: Gate (secrets + fork-safety) id: gate @@ -335,8 +335,9 @@ jobs: RAINDROP_QUERY_API_KEY: ${{ secrets.RAINDROP_QUERY_API_KEY }} with: max_attempts: 2 - # Per-attempt cap; the job timeout-minutes above is sized for 2 of them. - timeout_minutes: 10 + # Per-attempt cap sized above the measured prod runtime (~8.4 min) with + # headroom for scenario growth; the job timeout-minutes above fits 2. + timeout_minutes: 20 retry_wait_seconds: 15 shell: bash command: | From 16e00b94456dc29e7888c4b3b550a33aa37159ba Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:00:43 +0000 Subject: [PATCH 3/5] ci(conformance): raise prod job cap 45->50m to fit worst-case two-attempt run Co-Authored-By: bot_apk --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index ffa1ea0..b1eff56 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -243,7 +243,7 @@ jobs: # each), so allow generous headroom over the fault lane. Sized to fit TWO # full 20-min prod attempts (the runner step retries once, per-attempt # timeout_minutes below) plus the retry wait and checkout/build setup. - timeout-minutes: 45 + timeout-minutes: 50 steps: - name: Gate (secrets + fork-safety) id: gate From 1c8f90146a64680ccb115312698ec0e53b8e08e4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:19:15 +0000 Subject: [PATCH 4/5] ci(conformance): replace nick-fields/retry with an inline bash retry loop (no third-party action; 2 attempts, per-attempt timeout 1200s) Co-Authored-By: bot_apk --- .github/workflows/conformance.yml | 44 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index b1eff56..c15f3ac 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -241,8 +241,8 @@ jobs: runs-on: ubuntu-latest # Prod scenarios are dominated by the ~30s duplicate-settle window (~40s # each), so allow generous headroom over the fault lane. Sized to fit TWO - # full 20-min prod attempts (the runner step retries once, per-attempt - # timeout_minutes below) plus the retry wait and checkout/build setup. + # full 20-min prod attempts (the runner step's inline retry loop caps each + # attempt at `timeout 1200`) plus the retry wait and checkout/build setup. timeout-minutes: 50 steps: - name: Gate (secrets + fork-safety) @@ -321,34 +321,40 @@ jobs: # RAINDROP_QUERY_API_KEY. Endpoints + credentials are scoped to this # step so no other step in the job can read them. # - # GHA-native retry (nick-fields/retry): the prod lane crosses the real - # network (ingest + Query API readback), so a transient blip can fail an - # otherwise-conformant run. Retry the prod lane ONCE — max 2 attempts, - # green on a single pass — while the fault lane (a separate job) is never - # retried. The runner's `exit_code`/`total_attempts` outputs drive the - # summary + the flake annotation below; a retried pass is never silent. - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + # Inline retry loop (repo-controlled shell, no third-party action): the + # prod lane crosses the real network (ingest + Query API readback), so a + # transient blip can fail an otherwise-conformant run. Run the prod lane + # up to twice — green on a single pass — while the fault lane (a separate + # job) is never retried; each attempt is capped at 20 min by + # `timeout 1200`. We write `exit_code`/`total_attempts` ourselves so the + # summary + the flake annotation below are unchanged; a retried pass is + # never silent. + shell: bash env: RAINDROP_SINK_URL: https://api.raindrop.ai RAINDROP_WRITE_KEY: ${{ secrets.RAINDROP_WRITE_KEY }} RAINDROP_QUERY_URL: https://query.raindrop.ai RAINDROP_QUERY_API_KEY: ${{ secrets.RAINDROP_QUERY_API_KEY }} - with: - max_attempts: 2 - # Per-attempt cap sized above the measured prod runtime (~8.4 min) with - # headroom for scenario growth; the job timeout-minutes above fits 2. - timeout_minutes: 20 - retry_wait_seconds: 15 - shell: bash - command: | + run: | + attempts=0 + code=1 + for i in 1 2; do + attempts=$i set +e - node harness/runner/dist/src/index.js \ + timeout 1200 node harness/runner/dist/src/index.js \ --driver "$DRIVER_BIN" \ --failures conformance/failures.txt \ --lane prod \ --report "$RUNNER_TEMP/report.json" \ 2>&1 | tee "$RUNNER_TEMP/runner-output.txt" - exit "${PIPESTATUS[0]}" + code=${PIPESTATUS[0]} + set -e + [ "$code" -eq 0 ] && break + [ "$i" -eq 1 ] && { echo "raindrop-rust prod lane attempt 1 exited $code; retrying in 15s"; sleep 15; } + done + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + echo "total_attempts=$attempts" >> "$GITHUB_OUTPUT" + exit "$code" - name: Flag prod-lane flake (passed on retry) # A prod pass that needed attempt 2 is a flake — surface it loudly so a From 22f3f51864060f8fe3d9ee48ec151b03c04f9b26 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:26:09 +0000 Subject: [PATCH 5/5] ci(conformance): label prod-lane exit 124 as TIMEOUT in the result summary (timeout 1200 makes 124 a real outcome) Co-Authored-By: bot_apk --- .github/workflows/conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index c15f3ac..1c9673e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -379,6 +379,7 @@ jobs: 0) echo "- Result: **PASS** (warnings allowed)" ;; 1) echo "- Result: **FAIL** — scenario/ratchet failures (non-blocking for now)" ;; 2) echo "- Result: **ERROR** — infra/config problem (non-blocking for now)" ;; + 124) echo "- Result: **TIMEOUT** — an attempt exceeded the 20-min \`timeout 1200\` cap and was killed (recorded as a failure)" ;; *) echo "- Result: **UNKNOWN** (runner did not report an exit code)" ;; esac echo