diff --git a/.github/workflows/supporter-loop-reusable.yml b/.github/workflows/supporter-loop-reusable.yml index b1d4e17..ccef941 100644 --- a/.github/workflows/supporter-loop-reusable.yml +++ b/.github/workflows/supporter-loop-reusable.yml @@ -1,5 +1,5 @@ # supporter-loop — record-only first; caller activation belongs to checkpoint #4. -# Contract: caty-ai/x-collector docs/supporter-loop/CONTRACT.md v1.11 (frozen) +# Contract: caty-ai/x-collector docs/supporter-loop/CONTRACT.md v2.0 (frozen) # SECURITY INVARIANTS — §13, do not weaken: # 1. Never check out or execute code from the triggering ref, including in callers. # 2. Actor strings enter scripts only through env or GITHUB_EVENT_PATH. @@ -17,7 +17,7 @@ # 8. Fixed comment templates: login/reward_repo_url substitutions plus numeric marker. # Caller concurrency, token registration/expiry variables, baseline JSON and # the Contributors wall/release notes are owned by child #3 / the owner. -# Backfill input is deliberately absent: add only at checkpoint #4 (§7 v1.9). +# Backfill is retired in v2.0; historical ok-backfill lines retain their meaning. name: supporter-loop-reusable on: workflow_call: @@ -63,7 +63,7 @@ jobs: stars: if: ${{ inputs.sweep == true }} runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 permissions: contents: write outputs: @@ -85,8 +85,9 @@ jobs: || ! jq -e 'type=="array" and length>0 and all(.[]; type=="array") and all(.[][]; .id|type=="number" and .>0 and floor==.)' "$work/pages" >/dev/null; then ok=false; printf '[]\n' > "$work/pages" fi + # jq builtin unique_by sorts: the local definition preserves first page occurrence. # IDs only, never logins or author text. Fail before GitHub's output-size truncation. - stars=$(jq -c '[.[][].id] | unique' "$work/pages") + stars=$(jq -c 'def stable_unique_by(f): reduce .[] as $x ({seen:{},out:[]}; ($x|f|tojson) as $k | if .seen[$k] then . else .seen[$k]=true | .out += [$x] end) | .out; [.[][].id] | stable_unique_by(.)' "$work/pages") [ "${#stars}" -lt 400000 ] || { echo '::error::stargazer handoff exceeds safe output size'; exit 1; } printf 'ok=%s\n' "$ok" >> "$GITHUB_OUTPUT" printf 'stargazers=%s\n' "$stars" >> "$GITHUB_OUTPUT" @@ -107,6 +108,9 @@ jobs: has_live_actions: ${{ steps.decide.outputs.has_live_actions }} sweep_gate: ${{ steps.decide.outputs.sweep_gate }} stargazers: ${{ steps.decide.outputs.stargazers }} + cancelled_event: ${{ steps.decide.outputs.cancelled_event }} + cancelled_schedule: ${{ steps.decide.outputs.cancelled_schedule }} + red_run: ${{ steps.decide.outputs.red_run }} steps: - name: Validate mode before any network access env: @@ -166,6 +170,45 @@ jobs: ([.[] | select(.action=="cancel-invite" and .result=="expired") | [.ts,.__order]] | max // ["",-1]) as $expired | any(.[]; .tier==$tier and .dedup_key==($repo+":"+($tier|tostring)+":"+($id|tostring)) and .action==$action and success and [.ts,.__order]>$expired) else any(.[]; .tier==$tier and .dedup_key==($repo+":"+($tier|tostring)+":"+($id|tostring)) and .action==$action and success) end); + # Shared key rule: a sweep delivery is a tier delivery, never a closure. + def key_namespace($action;$event;$tier): + if $event=="sweep" and ($action|IN("skip","revoke","cancel-invite","would-revoke","would-cancel-invite")) then "sweep" else ($tier|tostring) end; + # Rehearsal also honors live deliveries to avoid predicting actions already performed. + def invite_delivered($repo;$id): + delivered($repo;$id;1;"invite") or + (if $mode=="record-only" then delivered($repo;$id;1;"would-invite") else false end) or + (current($repo;$id) | to_entries | map(.value+{__order:.key}) | + ([.[]|select(.action=="cancel-invite" and .result=="expired")|[.ts,.__order]]|max // ["",-1]) as $expired | + any(.[]; .action=="invite" and .result=="ok-backfill" and .dedup_key==($repo+":sweep:"+($id|tostring)) and [.ts,.__order]>$expired)); + def append_delivered($repo;$id): delivered($repo;$id;1;"supporters-append") or + (if $mode=="record-only" then delivered($repo;$id;1;"would-supporters-append") else false end); + def excluded($repo;$id): current($repo;$id) | any(.[]; .action=="skip" and (.result|startswith("excluded-"))); + def exhausted($repo;$id): + # Count completed invite/expiry cycles, not concurrent PUTs before one expiry. + current($repo;$id) | to_entries | unique_by(.value) | sort_by(.value.ts,.key) | map(.value) | + reduce .[] as $r ({pending:false,expired:0}; + if $r.action=="invite" and ($r.result|IN("ok","ok-backfill","ok-catchup")) then .pending=true + elif $r.action=="cancel-invite" and $r.result=="expired" and .pending then .expired+=1 | .pending=false else . end) | .expired>=2; + def catchup($repo;$stars;$tiers): . as $all | + if ($tiers|split(",")|index("1"))==null then [] else + [$stars[] as $id | ($all|current($repo;$id)) as $history | + select($all|excluded($repo;$id)|not) | + ($all|invite_delivered($repo;$id)) as $invited | ($all|append_delivered($repo;$id)) as $appended | + select(($invited and $appended)|not) | + {actor_id:$id,gen:($all|generation($repo;$id)),invite:$invited,append:$appended, + actor:($history|sort_by(.ts)|last|.actor // ""), + rearmed:($history|any(.[];.action=="invite" and success)),exhausted:($all|exhausted($repo;$id))}] | + map(select(.rearmed|not)) + map(select(.rearmed)) end; + # Rolling window is (now-86400, now]: exactly 24h old has aged out. + # Exact duplicate rotation rows count once; already-1 never consumed a PUT. + # Only current-generation successful PUTs suppress a pending invitation. + def quota($repo;$pending;$now): . as $all | + [unique[] | select(.repo==$repo and .action=="invite" and (.result|IN("ok","ok-backfill","ok-catchup"))) | + {id:.actor_id,gen:.gen,ts:(.ts|fromdateiso8601)} | select(.ts>$now-86400 and .ts<=$now)] as $sent | + ($sent + [$pending[] | {id:.invitee.id,ts:(.created_at|fromdateiso8601)} | + select(.ts>$now-86400 and .ts<=$now) | . as $p | select(all($sent[];.id!=$p.id or .gen!=($all|generation($repo;$p.id))))]) as $counted | + {B:($counted|length),Q:([0,45-($counted|length)]|max), + age_out:([$counted[].ts+86400-$now]|min // 0)}; JQ model() { jq -s -L "$work" --arg mode "$MODE" "include \"model\"; $1" "$ledger"; } # Read-shaped requests; HTTP errors are classified without printing response bodies. @@ -206,14 +249,12 @@ jobs: } line() { local action="$1" result="$2" tier="$3" subject="$4" event="$5" - local key="$GITHUB_REPOSITORY:$tier:$actor_id" - [ "$event" != sweep ] || key="$GITHUB_REPOSITORY:sweep:$actor_id" - jq -cn --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg run_id "$RUN_KEY" \ + jq -cn -L "$work" --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg run_id "$RUN_KEY" \ --arg repo "$GITHUB_REPOSITORY" --arg event "$event" --arg actor "$actor" \ --arg actor_id "$actor_id" --arg tier "$tier" --arg subject "$subject" \ --arg action "$action" --arg mode "$MODE" --arg result "$result" \ - --arg dedup_key "$key" --arg gen "$gen" \ - '{schema:1,ts:$ts,run_id:$run_id,repo:$repo,event:$event,actor:$actor,actor_id:($actor_id|tonumber),tier:($tier|tonumber),subject:$subject,action:$action,mode:$mode,result:$result,dedup_key:$dedup_key,gen:($gen|tonumber)}' + --arg gen "$gen" \ + 'include "model"; key_namespace($action;$event;$tier) as $ns | {schema:1,ts:$ts,run_id:$run_id,repo:$repo,event:$event,actor:$actor,actor_id:($actor_id|tonumber),tier:($tier|tonumber),subject:$subject,action:$action,mode:$mode,result:$result,dedup_key:($repo+":"+$ns+":"+$actor_id),gen:($gen|tonumber)}' } append() { local entry="$1" attempt=0 sha content status @@ -248,6 +289,30 @@ jobs: done echo '::error::ledger CAS retries exhausted'; return 1 } + identity() { + get "$GH_TOKEN" "user/$actor_id" "$work/identity.json" || { code=transport; return 1; } + [ "$code" = 200 ] || return 1 + actor=$(jq -er --argjson id "$actor_id" 'select(.id==$id) | .login | select(type=="string" and test("^[A-Za-z0-9-]{1,39}(\\[bot\\])?$"))' "$work/identity.json") || { code=identity; return 1; } + user_type=$(jq -er '.type|select(type=="string")' "$work/identity.json") || { code=identity; return 1; } + } + eligibility() { + actor_lc=$(printf '%s' "$actor" | tr '[:upper:]' '[:lower:]') + # Exclusion precedence: later matches win (member > org > self > bot > family). + reason='' + # Twin external-input-watch.yml fam_roster: case-insensitive; identical token set. + # Existing production workflow is intentionally untouched; copied roster is noted in README. + fam_roster="sho""jikumaru caty2 alec-bridge alpha-mbp-bridge cero-bridge" + for fam in $fam_roster; do [ "$actor_lc" != "$fam" ] || reason=excluded-family; done + if [ "$user_type" = Bot ] || [[ "$actor_lc" == *'[bot]' ]]; then reason=excluded-bot; fi + if [ "$actor_lc" = "$(printf '%s' "${GITHUB_REPOSITORY%/*}" | tr '[:upper:]' '[:lower:]')" ] || [ "$actor_lc" = 'github-actions[bot]' ]; then reason=excluded-self; fi + [ "$user_type" != Organization ] || reason=excluded-org + case "$assoc" in OWNER|MEMBER) reason=excluded-member ;; esac + if [ -z "$reason" ]; then + if ! { [ "$user_type" = User ] && [[ "$actor" =~ ^[A-Za-z0-9-]{1,39}$ ]]; }; then + echo '::error::invalid actor type/login'; return 1 + fi + fi + } # Crash-safe §6.3 copy, verify byte equality, then delete the exact base SHA. rotate() { local archive target_sha='' base_sha content status @@ -401,8 +466,9 @@ jobs: jq '[.[].workflow_runs[] | select(.path==".github/workflows/supporter-loop.yml")]' "$target.pages" > "$target" } cutoff='' - if runs 'event=schedule&status=success' "$work/success-runs.json"; then - cutoff=$(jq -r 'sort_by(.created_at) | last | .created_at // empty' "$work/success-runs.json") + if runs 'event=schedule&status=success' "$work/success-runs.json" && + runs 'event=workflow_dispatch&status=success' "$work/dispatch-runs.json"; then + cutoff=$(jq -sr 'add | sort_by(.created_at) | last | .created_at // empty' "$work/success-runs.json" "$work/dispatch-runs.json") else gate=false; fi if [ -z "$cutoff" ]; then if runs '' "$work/all-runs.json"; then cutoff=$(jq -r 'sort_by(.created_at) | first | .created_at // empty' "$work/all-runs.json"); else gate=false; fi @@ -410,18 +476,28 @@ jobs: if [ -z "$cutoff" ]; then gate=false elif ! runs "status=failure&created=%3E%3D$cutoff" "$work/failed-runs.json"; then gate=false elif [ "$(jq length "$work/failed-runs.json")" -ne 0 ]; then gate=false; fi + printf 'red_run=%s\n' "$([ "$gate" = true ] && echo false || echo true)" >> "$GITHUB_OUTPUT" + # Cancelled runs are observability only: they NEVER change any gate. + cancelled_event=0; cancelled_schedule=0 + if [ -n "$cutoff" ] && runs "status=cancelled&created=%3E%3D$cutoff" "$work/cancelled-runs.json"; then + cancelled_event=$(jq '[.[]|select(.event|IN("watch","issues","discussion","pull_request_target"))]|length' "$work/cancelled-runs.json") + cancelled_schedule=$(jq '[.[]|select(.event|IN("schedule","workflow_dispatch"))]|length' "$work/cancelled-runs.json") + else echo '::warning::cancelled-run listing unavailable; counts default to zero'; fi + printf 'cancelled_event=%s\ncancelled_schedule=%s\n' "$cancelled_event" "$cancelled_schedule" >> "$GITHUB_OUTPUT" + stars_ok=false # §11(b): pagination errors do not hide expiry/audit work in live act. # v1.11: the list comes from the sweep-only stars job (§4.4) — decide never lists # stargazers itself (GITHUB_TOKEN gets 403 without contents: write; caty-ai/.github#89). # A skipped/failed handoff or a non-numeric entry fails closed here. if [ "$STAR_IDS_OK" = true ] && printf '%s' "$STAR_IDS" | jq -e 'type=="array" and all(.[]; type=="number" and .>0 and floor==.)' >/dev/null 2>&1; then + stars_ok=true printf '%s' "$STAR_IDS" | jq '[.[] | {id: .}]' > "$work/stargazers.json" else gate=false; printf '[]\n' > "$work/stargazers.json" fi printf 'sweep_gate=%s\n' "$gate" >> "$GITHUB_OUTPUT" # IDs only, never secrets or author text. Fail before GitHub's output-size truncation. - stars=$(jq -c '[.[].id] | unique' "$work/stargazers.json") + stars=$(jq -c 'def stable_unique_by(f): reduce .[] as $x ({seen:{},out:[]}; ($x|f|tojson) as $k | if .seen[$k] then . else .seen[$k]=true | .out += [$x] end) | .out; [.[].id] | stable_unique_by(.)' "$work/stargazers.json") [ "${#stars}" -lt 400000 ] || { echo '::error::stargazer handoff exceeds safe output size'; exit 1; } printf 'stargazers=%s\n' "$stars" >> "$GITHUB_OUTPUT" printf 'actions=[]\n' >> "$GITHUB_OUTPUT" @@ -453,6 +529,33 @@ jobs: append "$(line would-revoke noop 0 '' sweep)" done < <(jq -r '.[]' "$work/ids.json") fi + # §11(4b) rehearsal is gated ONLY by exhaustive stars. Pending quota is + # unavailable without Administration; B is ledger-only (intentional undercount). + if [ "$stars_ok" = true ]; then + jq -s -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" --arg tiers "$TIERS_ENABLED" --argjson stars "$stars" \ + 'include "model"; catchup($repo;$stars;$tiers)' "$ledger" > "$work/catchup.json" + rehearsal_count=0 + while IFS= read -r item; do + actor_id=$(jq -r .actor_id <<< "$item"); actor=$(jq -r .actor <<< "$item"); gen=$(jq -r .gen <<< "$item") + append "$(line would-supporters-append ok 1 '' sweep)" + done < <(jq -c '.[]|select(.invite and (.append|not))' "$work/catchup.json") + while IFS= read -r item; do + quota=$(jq -s -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" --argjson now "$(date -u +%s)" 'include "model"; quota($repo;[];$now)' "$ledger") + [ "$(jq -r .Q <<< "$quota")" -gt "$rehearsal_count" ] || break + actor_id=$(jq -r .actor_id <<< "$item"); gen=$(jq -r .gen <<< "$item"); actor=$(jq -r .actor <<< "$item") + if [ "$(jq -r .exhausted <<< "$item")" = true ]; then append "$(line skip excluded-unresponsive 0 '' sweep)"; continue; fi + assoc=NONE; reason='' + if ! { identity && eligibility; }; then + if [ -n "$reason" ]; then append "$(line skip "$reason" 0 '' sweep)" + else echo "::warning::rehearsal identity unavailable for actor_id=$actor_id"; fi + continue + fi + if [ -n "$reason" ]; then append "$(line skip "$reason" 0 '' sweep)"; continue; fi + append "$(line would-invite ok 1 '' sweep)" + rehearsal_count=$((rehearsal_count+1)) + if [ "$(jq -r .append <<< "$item")" = false ]; then append "$(line would-supporters-append ok 1 '' sweep)"; fi + done < <(jq -c '.[]|select(.invite|not)' "$work/catchup.json") + fi # §11 steps 5–7: rotate, omit derived-public write, warn on ledger-token expiry. rotate reminder "$LEDGER_EXPIRES" 'ledger token' @@ -473,22 +576,7 @@ jobs: actor=$(jq -r "$actor_path.login" "$GITHUB_EVENT_PATH") actor_id=$(jq -er "$actor_path.id | select(type==\"number\" and .>0 and floor==.)" "$GITHUB_EVENT_PATH") user_type=$(jq -r "$actor_path.type" "$GITHUB_EVENT_PATH") - actor_lc=$(printf '%s' "$actor" | tr '[:upper:]' '[:lower:]') - # Exclusion precedence: later matches win (member > org > self > bot > family). - reason='' - # Twin external-input-watch.yml fam_roster: case-insensitive; identical token set. - # Existing production workflow is intentionally untouched; copied roster is noted in README. - fam_roster="sho""jikumaru caty2 alec-bridge alpha-mbp-bridge cero-bridge" - for fam in $fam_roster; do [ "$actor_lc" != "$fam" ] || reason=excluded-family; done - if [ "$user_type" = Bot ] || [[ "$actor_lc" == *'[bot]' ]]; then reason=excluded-bot; fi - if [ "$actor_lc" = "$(printf '%s' "${GITHUB_REPOSITORY%/*}" | tr '[:upper:]' '[:lower:]')" ] || [ "$actor_lc" = 'github-actions[bot]' ]; then reason=excluded-self; fi - [ "$user_type" != Organization ] || reason=excluded-org - case "$assoc" in OWNER|MEMBER) reason=excluded-member ;; esac - if [ -z "$reason" ]; then - if ! { [ "$user_type" = User ] && [[ "$actor" =~ ^[A-Za-z0-9-]{1,39}$ ]]; }; then - echo '::error::invalid actor type/login'; exit 1 - fi - fi + eligibility gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") if [ -n "$reason" ]; then append "$(line skip "$reason" 0 "$subject" "$EVENT_NAME")" @@ -547,6 +635,7 @@ jobs: [ -n "$ADMIN_TOKEN" ] || { echo '::error::SUPPORTER_LOOP_TOKEN is empty'; exit 1; } umask 077 mkdir -p "$RUNNER_TEMP/supporter-loop" + date -u +%s > "$RUNNER_TEMP/supporter-loop/job-start" probe() { [ "$MODE" = live ] || return 1 code=$(curl -sS --connect-timeout 10 --max-time 45 -o "$RUNNER_TEMP/supporter-loop/preflight.json" -w '%{http_code}' \ @@ -582,6 +671,10 @@ jobs: ACTIONS: ${{ needs.decide.outputs.actions }} SWEEP_GATE: ${{ needs.decide.outputs.sweep_gate }} STARGAZERS: ${{ needs.decide.outputs.stargazers }} + TIERS_ENABLED: ${{ inputs.tiers_enabled }} + CANCELLED_EVENT: ${{ needs.decide.outputs.cancelled_event }} + CANCELLED_SCHEDULE: ${{ needs.decide.outputs.cancelled_schedule }} + RED_RUN: ${{ needs.decide.outputs.red_run }} LEDGER_EXPIRES: ${{ vars.SUPPORTER_LEDGER_TOKEN_EXPIRES }} ADMIN_EXPIRES: ${{ vars.SUPPORTER_LOOP_TOKEN_EXPIRES }} run: | @@ -593,6 +686,23 @@ jobs: mkdir -p "$work" base="${GITHUB_REPOSITORY//\//--}" ledger="$work/ledger.ndjson" + # §11 work deadline includes preflight. Tests can inject an epoch clock file. + started=$(date -u +%s) + if [ -f "$work/job-start" ]; then started=$(cat "$work/job-start"); fi + elapsed_offset=$(( $(date -u +%s) - started )); SECONDS=0 + writeCount=0; reservedCount=0; waitCount=0; budget_active=false + if [ "$SWEEP" = true ]; then reservedCount=2; fi # final projection + state + elapsed() { + if [ -n "${SWEEP_CLOCK_FILE:-}" ]; then echo "$(( $(cat "$SWEEP_CLOCK_FILE") - started ))" + else echo "$((elapsed_offset + SECONDS))"; fi + } + within_deadline() { [ "$(elapsed)" -le 360 ]; } + work_budget() { within_deadline && [ "$((writeCount + reservedCount + ${1:-0}))" -le 150 ]; } + pace_write() { + # Full second after the previous attempted write completed, even across helpers. + if [ "$writeCount" -gt 0 ]; then sleep 1; fi + writeCount=$((writeCount+1)) + } # .publication-denylist option a: split committer email preserves contract §6 identity. # Model is embedded in the reviewed central workflow, never loaded from source code. cat > "$work/model.jq" <<'JQ' @@ -607,11 +717,51 @@ jobs: ([.[] | select(.action=="cancel-invite" and .result=="expired") | [.ts,.__order]] | max // ["",-1]) as $expired | any(.[]; .tier==$tier and .dedup_key==($repo+":"+($tier|tostring)+":"+($id|tostring)) and .action==$action and success and [.ts,.__order]>$expired) else any(.[]; .tier==$tier and .dedup_key==($repo+":"+($tier|tostring)+":"+($id|tostring)) and .action==$action and success) end); + # Shared key rule: a sweep delivery is a tier delivery, never a closure. + def key_namespace($action;$event;$tier): + if $event=="sweep" and ($action|IN("skip","revoke","cancel-invite","would-revoke","would-cancel-invite")) then "sweep" else ($tier|tostring) end; + # Rehearsal also honors live deliveries to avoid predicting actions already performed. + def invite_delivered($repo;$id): + delivered($repo;$id;1;"invite") or + (if $mode=="record-only" then delivered($repo;$id;1;"would-invite") else false end) or + (current($repo;$id) | to_entries | map(.value+{__order:.key}) | + ([.[]|select(.action=="cancel-invite" and .result=="expired")|[.ts,.__order]]|max // ["",-1]) as $expired | + any(.[]; .action=="invite" and .result=="ok-backfill" and .dedup_key==($repo+":sweep:"+($id|tostring)) and [.ts,.__order]>$expired)); + def append_delivered($repo;$id): delivered($repo;$id;1;"supporters-append") or + (if $mode=="record-only" then delivered($repo;$id;1;"would-supporters-append") else false end); + def excluded($repo;$id): current($repo;$id) | any(.[]; .action=="skip" and (.result|startswith("excluded-"))); + def exhausted($repo;$id): + # Count completed invite/expiry cycles, not concurrent PUTs before one expiry. + current($repo;$id) | to_entries | unique_by(.value) | sort_by(.value.ts,.key) | map(.value) | + reduce .[] as $r ({pending:false,expired:0}; + if $r.action=="invite" and ($r.result|IN("ok","ok-backfill","ok-catchup")) then .pending=true + elif $r.action=="cancel-invite" and $r.result=="expired" and .pending then .expired+=1 | .pending=false else . end) | .expired>=2; + def catchup($repo;$stars;$tiers): . as $all | + if ($tiers|split(",")|index("1"))==null then [] else + [$stars[] as $id | ($all|current($repo;$id)) as $history | + select($all|excluded($repo;$id)|not) | + ($all|invite_delivered($repo;$id)) as $invited | ($all|append_delivered($repo;$id)) as $appended | + select(($invited and $appended)|not) | + {actor_id:$id,gen:($all|generation($repo;$id)),invite:$invited,append:$appended, + actor:($history|sort_by(.ts)|last|.actor // ""), + rearmed:($history|any(.[];.action=="invite" and success)),exhausted:($all|exhausted($repo;$id))}] | + map(select(.rearmed|not)) + map(select(.rearmed)) end; + # Rolling window is (now-86400, now]: exactly 24h old has aged out. + # Exact duplicate rotation rows count once; already-1 never consumed a PUT. + # Only current-generation successful PUTs suppress a pending invitation. + def quota($repo;$pending;$now): . as $all | + [unique[] | select(.repo==$repo and .action=="invite" and (.result|IN("ok","ok-backfill","ok-catchup"))) | + {id:.actor_id,gen:.gen,ts:(.ts|fromdateiso8601)} | select(.ts>$now-86400 and .ts<=$now)] as $sent | + ($sent + [$pending[] | {id:.invitee.id,ts:(.created_at|fromdateiso8601)} | + select(.ts>$now-86400 and .ts<=$now) | . as $p | select(all($sent[];.id!=$p.id or .gen!=($all|generation($repo;$p.id))))]) as $counted | + {B:($counted|length),Q:([0,45-($counted|length)]|max), + age_out:([$counted[].ts+86400-$now]|min // 0)}; JQ model() { jq -s -L "$work" --arg mode "$MODE" "include \"model\"; $1" "$ledger"; } # Read-shaped requests; HTTP errors are classified without printing response bodies. get() { local token="$1" endpoint="$2" target="$3" + if [ "${budget_active:-false}" = true ]; then within_deadline || return 1; fi [ "$MODE" = live ] || return 1 code=$(curl -sS --connect-timeout 10 --max-time 45 -o "$target" -w '%{http_code}' \ -H "Authorization: Bearer $token" -H 'Accept: application/vnd.github+json' \ @@ -619,8 +769,13 @@ jobs: } pages() { local token="$1" endpoint="$2" target="$3" + if [ "${budget_active:-false}" = true ]; then within_deadline || return 1; fi [ "$MODE" = live ] || return 1 - GH_TOKEN="$token" gh api --hostname github.com --method GET --paginate --slurp "$endpoint" > "$target.pages" || return 1 + if [ "${budget_active:-false}" = true ]; then + timeout "$((361 - $(elapsed)))" env GH_TOKEN="$token" gh api --hostname github.com --method GET --paginate --slurp "$endpoint" > "$target.pages" || return 1 + else + GH_TOKEN="$token" gh api --hostname github.com --method GET --paginate --slurp "$endpoint" > "$target.pages" || return 1 + fi jq -e 'type=="array" and length>0 and all(.[]; type=="array")' "$target.pages" >/dev/null || return 1 jq 'add // []' "$target.pages" > "$target" } @@ -649,14 +804,12 @@ jobs: } line() { local action="$1" result="$2" tier="$3" subject="$4" event="$5" - local key="$GITHUB_REPOSITORY:$tier:$actor_id" - [ "$event" != sweep ] || key="$GITHUB_REPOSITORY:sweep:$actor_id" - jq -cn --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg run_id "$RUN_KEY" \ + jq -cn -L "$work" --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg run_id "$RUN_KEY" \ --arg repo "$GITHUB_REPOSITORY" --arg event "$event" --arg actor "$actor" \ --arg actor_id "$actor_id" --arg tier "$tier" --arg subject "$subject" \ --arg action "$action" --arg mode "$MODE" --arg result "$result" \ - --arg dedup_key "$key" --arg gen "$gen" \ - '{schema:1,ts:$ts,run_id:$run_id,repo:$repo,event:$event,actor:$actor,actor_id:($actor_id|tonumber),tier:($tier|tonumber),subject:$subject,action:$action,mode:$mode,result:$result,dedup_key:$dedup_key,gen:($gen|tonumber)}' + --arg gen "$gen" \ + 'include "model"; key_namespace($action;$event;$tier) as $ns | {schema:1,ts:$ts,run_id:$run_id,repo:$repo,event:$event,actor:$actor,actor_id:($actor_id|tonumber),tier:($tier|tonumber),subject:$subject,action:$action,mode:$mode,result:$result,dedup_key:($repo+":"+$ns+":"+$actor_id),gen:($gen|tonumber)}' } append() { local entry="$1" attempt=0 sha content status @@ -678,6 +831,7 @@ jobs: --arg message "supporter-loop: $(jq -r '.event+" "+.actor+" "+.action' <<< "$entry")" \ '{message:$message,content:$content,committer:{name:"supporter-loop[bot]",email:("supporter-loop@" + "caty-ai" + ".noreply")}} + (if $sha=="" then {} else {sha:$sha} end)' > "$work/put.json" [ "$MODE" = live ] || return 1 + pace_write status=$(curl -sS --connect-timeout 10 --max-time 45 -o "$work/put-response.json" -w '%{http_code}' \ -X PUT -H "Authorization: Bearer $LEDGER_TOKEN" -H 'Accept: application/vnd.github+json' \ --data-binary @"$work/put.json" "https://api.github.com/repos/$REWARD_REPO/contents/ledger/$base.ndjson") || return 1 @@ -721,6 +875,7 @@ jobs: jq -cn --arg content "$content" --arg sha "$target_sha" \ '{message:"supporter-loop: sweep ledger rotate",content:$content,committer:{name:"supporter-loop[bot]",email:("supporter-loop@" + "caty-ai" + ".noreply")}} + (if $sha=="" then {} else {sha:$sha} end)' > "$work/rotate.json" [ "$MODE" = live ] || return 1 + pace_write status=$(curl -sS --connect-timeout 10 --max-time 45 -o "$work/rotate-response" -w '%{http_code}' -X PUT \ -H "Authorization: Bearer $LEDGER_TOKEN" --data-binary @"$work/rotate.json" \ "https://api.github.com/repos/$REWARD_REPO/contents/ledger/$archive") || return 1 @@ -731,6 +886,7 @@ jobs: cmp -s "$work/rotation-base" "$work/verified" || { echo '::error::rotation byte verification failed'; return 1; } jq -cn --arg sha "$base_sha" '{message:"supporter-loop: sweep ledger rotate",sha:$sha,committer:{name:"supporter-loop[bot]",email:("supporter-loop@" + "caty-ai" + ".noreply")}}' > "$work/delete.json" [ "$MODE" = live ] || return 1 + pace_write status=$(curl -sS --connect-timeout 10 --max-time 45 -o "$work/delete-response" -w '%{http_code}' -X DELETE \ -H "Authorization: Bearer $LEDGER_TOKEN" --data-binary @"$work/delete.json" \ "https://api.github.com/repos/$REWARD_REPO/contents/ledger/$base.ndjson") || return 1 @@ -739,6 +895,7 @@ jobs: # GraphQL is read-only here. Explicit cursors exhaust comments AND every replies connection. graph() { local query="$1" variables="$2" target="$3" entry key value kind + if [ "${budget_active:-false}" = true ]; then within_deadline || return 1; fi [ "$MODE" = live ] || return 1 # Only query operations; gh field flags select POST for GraphQL transport. case "$query" in query*) ;; *) return 1 ;; esac @@ -834,6 +991,7 @@ jobs: mutate() { local token="$1" method="$2" endpoint="$3" payload="$4" [ "$MODE" = live ] || return 1 + pace_write code=$(curl -sS --connect-timeout 10 --max-time 45 -o "$work/mutation.json" -w '%{http_code}' \ -X "$method" -H "Authorization: Bearer $token" -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' --data-binary "@$payload" "https://api.github.com/$endpoint") || code=transport @@ -843,17 +1001,60 @@ jobs: entry=$(line "$1" "$2" "$3" "$4" "$5") # Independent owner leg survives a failed ledger append; no tokens/text enter this file. printf '%s\n' "$entry" >> "$work/notify.ndjson" - append "$entry" || { echo '::error::action returned but ledger append failed'; return 1; } + append "$entry" || { echo '::error::action returned but ledger append failed'; failed=1; return 1; } } identity() { - get "$GH_TOKEN" "user/$actor_id" "$work/identity.json" || return 1 + get "$GH_TOKEN" "user/$actor_id" "$work/identity.json" || { code=transport; return 1; } [ "$code" = 200 ] || return 1 - actor=$(jq -er --argjson id "$actor_id" 'select(.id==$id and .type=="User") | .login | select(test("^[A-Za-z0-9-]{1,39}$"))' "$work/identity.json") + actor=$(jq -er --argjson id "$actor_id" 'select(.id==$id) | .login | select(type=="string" and test("^[A-Za-z0-9-]{1,39}(\\[bot\\])?$"))' "$work/identity.json") || { code=identity; return 1; } + user_type=$(jq -er '.type|select(type=="string")' "$work/identity.json") || { code=identity; return 1; } + } + eligibility() { + actor_lc=$(printf '%s' "$actor" | tr '[:upper:]' '[:lower:]') + # Exclusion precedence: later matches win (member > org > self > bot > family). + reason='' + # Twin external-input-watch.yml fam_roster: case-insensitive; identical token set. + # Existing production workflow is intentionally untouched; copied roster is noted in README. + fam_roster="sho""jikumaru caty2 alec-bridge alpha-mbp-bridge cero-bridge" + for fam in $fam_roster; do [ "$actor_lc" != "$fam" ] || reason=excluded-family; done + if [ "$user_type" = Bot ] || [[ "$actor_lc" == *'[bot]' ]]; then reason=excluded-bot; fi + if [ "$actor_lc" = "$(printf '%s' "${GITHUB_REPOSITORY%/*}" | tr '[:upper:]' '[:lower:]')" ] || [ "$actor_lc" = 'github-actions[bot]' ]; then reason=excluded-self; fi + [ "$user_type" != Organization ] || reason=excluded-org + case "$assoc" in OWNER|MEMBER) reason=excluded-member ;; esac + if [ -z "$reason" ]; then + if ! { [ "$user_type" = User ] && [[ "$actor" =~ ^[A-Za-z0-9-]{1,39}$ ]]; }; then + echo '::error::invalid actor type/login'; return 1 + fi + fi + } + # PROVISIONAL §14 launch gate: owner must probe and pin exact GitHub text before merge. + INVITE_LIMIT_PATTERN='invit.*(limit|exceed|too many|24 ?h|per day)|too many invit' + invitation_limit() { + jq -e --arg pattern "$INVITE_LIMIT_PATTERN" '[.message, .errors[]? | if type=="object" then (.message,.code) else . end] | any(.[]; type=="string" and test($pattern;"i"))' "$work/mutation.json" >/dev/null + } + quota_now() { + jq -s -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" --argjson now "$(date -u +%s)" --slurpfile pending "$work/pending.json" \ + 'include "model"; quota($repo;$pending[0];$now)' "$ledger" > "$work/quota.json" + } + quota_room() { + local age_out + while :; do + work_budget || return 1 + quota_now || return 2 + [ "$(jq -r .Q "$work/quota.json")" -eq 0 ] || return 0 + age_out=$(jq -r .age_out "$work/quota.json") + [ "$age_out" -gt 0 ] && [ "$age_out" -le 300 ] && [ "$((waitCount+age_out))" -le 180 ] || return 1 + [ "$(( $(elapsed) + age_out ))" -le 360 ] || return 1 + sleep "$age_out" + waitCount=$((waitCount+age_out)) + done } invite() { local pending collaborator_status result=ok - if ! identity; then result="error-$code"; return 0; fi + if [ "${1:-false}" != true ]; then + if ! identity; then result="error-$code"; return 0; fi + fi if ! get "$ADMIN_TOKEN" "repos/$REWARD_REPO/collaborators/$actor" "$work/collaborator.json"; then result=error-transport; return 0; fi collaborator_status="$code" case "$collaborator_status" in 204|404) ;; *) result="error-$code"; return 0 ;; esac @@ -861,10 +1062,21 @@ jobs: if ! jq -e 'all(.[]; (.id|type=="number" and .>0 and floor==.) and (.invitee.id|type=="number" and .>0 and floor==.))' "$work/pending.json" >/dev/null; then result=error-preflight; return 0; fi pending=$(jq --argjson id "$actor_id" 'any(.[];.invitee.id==$id)' "$work/pending.json") if [ "$collaborator_status" = 204 ] || [ "$pending" = true ]; then result=already-1; return 0; fi + # Recompute immediately before every catch-up PUT, after fresh pending preflight. + if [ "$SWEEP" = true ]; then + if quota_room; then :; else + case "$?" in 1) result=budget-stop ;; *) result=error-quota ;; esac + return 0 + fi + fi # §2: explicit REST enum pull; omitting this grants push by platform default. printf '{"permission":"pull"}\n' > "$work/invite.json" mutate "$ADMIN_TOKEN" PUT "repos/$REWARD_REPO/collaborators/$actor" "$work/invite.json" - case "$code" in 200|201|204) result=ok ;; *) result="error-$code" ;; esac + case "$code" in + 200|201|204) result=ok ;; + 422) if invitation_limit; then result=deferred-quota; else result=error-422; fi ;; + *) result="error-$code" ;; + esac } comment() { local tier="$1" subject="$2" event="$3" number marker body template node vars query matched @@ -909,41 +1121,97 @@ jobs: fi } regenerate() { - local attempt=0 sha content header projected="${1:-}" derive="$ledger" + local code="" attempt=0 sha content header projected="${1:-}" original_ledger="$ledger" + local ledger="$work/fresh-ledger.ndjson" derive="$work/projected.ndjson" + regenerate_status="" [ "$MODE" = live ] || return 1 # §6.5: child #1 owns the header; never publish a headerless projection. if ! get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/SUPPORTERS.header.md" "$work/supporters-header.json" || [ "$code" != 200 ] || ! header=$(jq -er '.content | select(type=="string" and length>0)' "$work/supporters-header.json" | base64 -d) || [ -z "$header" ] || [ "${header##*$'\n'}" != '' ]; then + case "$code" in [45][0-9][0-9]) regenerate_status="$code" ;; esac echo '::error::SUPPORTERS.header.md missing or malformed in reward repo (child #1); regeneration skipped' return 1 fi - # Command substitution strips only trailing LF bytes; restore one plus the separator. - printf '%s\n\n' "$header" > "$work/SUPPORTERS.md" || return 1 - # Project only the intended success into a temporary render input. - # The actual ledger still changes only AFTER the Contents API returns. - if [ -n "$projected" ]; then - derive="$work/projected.ndjson" - cat "$ledger" > "$derive" - printf '%s\n' "$projected" >> "$derive" - fi - # §6.5: stable numeric identity, latest login, achieved success tier, first date. - # This is a whole-file derivation; the reward-repo child owns its presentation. - jq -sr -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" ' - include "model"; - . as $all | [.[]|select(.repo==$repo)|.actor_id]|unique | - map(. as $id | ($all|current($repo;$id)) as $history | ($all|achieved($repo;$id)) as $tier | - select($tier>0) | ($all|map(select(.repo==$repo and .actor_id==$id))|sort_by(.ts)) as $ever | - {id:$id,actor:($ever|last|.actor),tier:$tier,date:($ever|first|.ts|split("T")[0])}) | - map(select((.actor|ascii_downcase) as $a | (["sho"+"jikumaru","caty2","alec-bridge","alpha-mbp-bridge","cero-bridge","github-actions[bot]"]|index($a))==null and ($a|endswith("[bot]")|not))) | - sort_by(.date,.id) | "| Supporter | Tier | Since |\n| --- | --- | --- |", (.[]|"| @\(.actor) | \(.tier) | \(.date) |")' "$derive" >> "$work/SUPPORTERS.md" || return 1 - content=$(base64 < "$work/SUPPORTERS.md" | tr -d '\n') while [ "$attempt" -le 5 ]; do - get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/SUPPORTERS.md" "$work/supporters.json" || return 1 - case "$code" in 200) sha=$(jq -er .sha "$work/supporters.json") ;; 404) sha='' ;; *) return 1 ;; esac + regenerate_status="" + get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/SUPPORTERS.md" "$work/supporters.json" || { case "$code" in [1-5][0-9][0-9]) regenerate_status="$code" ;; esac; return 1; } + case "$code" in 200) sha=$(jq -er .sha "$work/supporters.json") ;; 404) sha='' ;; *) regenerate_status="$code"; return 1 ;; esac + # §6.5 CAS restart starts at GET, then reads fresh API ledger before rendering. + read_ledger || { code="ledger-read"; return 1; } + cp "$ledger" "$original_ledger" || return 1 + # Command substitution strips only trailing LF bytes; restore one plus the separator. + printf '%s\n\n' "$header" > "$work/SUPPORTERS.md" || return 1 + # Project only the intended success into a temporary render input. + # The actual ledger still changes only AFTER the Contents API returns. + cat "$ledger" > "$derive" || return 1 + if [ -n "$projected" ]; then printf '%s\n' "$projected" >> "$derive"; fi + # §6.5: stable numeric identity, latest login, achieved success tier, first date. + # This is a whole-file derivation; the reward-repo child owns its presentation. + jq -sr -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" ' + include "model"; + . as $all | [.[]|select(.repo==$repo)|.actor_id]|unique | + map(. as $id | ($all|current($repo;$id)) as $history | ($all|achieved($repo;$id)) as $tier | + select($tier>0 and ($tier>1 or ($all|invite_delivered($repo;$id)))) | ($all|map(select(.repo==$repo and .actor_id==$id))|sort_by(.ts)) as $ever | + {id:$id,actor:($ever|last|.actor),tier:$tier,date:($ever|first|.ts|split("T")[0])}) | + map(select((.actor|ascii_downcase) as $a | (["sho"+"jikumaru","caty2","alec-bridge","alpha-mbp-bridge","cero-bridge","github-actions[bot]"]|index($a))==null and ($a|endswith("[bot]")|not))) | + sort_by(.date,.id) | "| Supporter | Tier | Since |\n| --- | --- | --- |", (.[]|"| @\(.actor) | \(.tier) | \(.date) |")' "$derive" >> "$work/SUPPORTERS.md" || return 1 + content=$(base64 < "$work/SUPPORTERS.md" | tr -d '\n') jq -cn --arg sha "$sha" --arg content "$content" '{message:"supporter-loop: regenerate supporters",content:$content,committer:{name:"supporter-loop[bot]",email:("supporter-loop@" + "caty-ai" + ".noreply")}}+(if $sha=="" then {} else {sha:$sha} end)' > "$work/supporters-put.json" mutate "$LEDGER_TOKEN" PUT "repos/$REWARD_REPO/contents/SUPPORTERS.md" "$work/supporters-put.json" + case "$code" in [1-5][0-9][0-9]) regenerate_status="$code" ;; esac + case "$code" in 200|201) regenerate_status=""; return 0 ;; 409) ;; 422) jq -e '.message|test("sha|does not match";"i")' "$work/mutation.json" >/dev/null || return 1 ;; *) return 1 ;; esac + attempt=$((attempt+1)); [ "$attempt" -gt 5 ] || sleep "$((1+RANDOM%4))" + done + return 1 + } + sweep_state() { + local attempt=0 sha content previous current backlog stars_count md_bytes B state_path + state_path="ledger/$base.sweep-state.json" + while [ "$attempt" -le 5 ]; do + get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/$state_path" "$work/state-blob.json" || return 1 + case "$code" in + 200) sha=$(jq -er .sha "$work/state-blob.json") || return 1 + previous=$(jq -er '.content|gsub("[\\n\\r]"; "") | @base64d|fromjson|.alarm_state|select(type=="object")' "$work/state-blob.json" 2>/dev/null) || previous='{}' ;; + 404) sha=''; previous='{}' ;; + *) return 1 ;; + esac + read_ledger || return 1 + candidates_now > "$work/backlog.json" || return 1 + backlog=$(jq length "$work/backlog.json") + stars_count=$(jq length <<< "$STARGAZERS") + # Measure the actual published projection, not a failed render. + get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/SUPPORTERS.md" "$work/published-supporters.json" || return 1 + case "$code" in + 200) md_bytes=$(jq -er '.size|select(type=="number")' "$work/published-supporters.json") || return 1 ;; + 404) md_bytes=0 ;; + *) return 1 ;; + esac + quota_now || return 1 + B=$(jq -r .B "$work/quota.json") + jq -cn --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson backlog "$backlog" \ + --argjson events "${CANCELLED_EVENT:-0}" --argjson schedules "${CANCELLED_SCHEDULE:-0}" \ + --argjson stars "$stars_count" --argjson bytes "$md_bytes" --argjson B "$B" \ + --argjson red "${RED_RUN:-false}" --argjson deferred "$deferred_quota" ' + {stars:($stars/40000),supporters_md:($bytes/1048576),invitations:($B/45)} as $capacity | + {last_sweep_ts:$ts,backlog:$backlog,cancelled_runs:{event:$events,schedule:$schedules},capacity_pct:$capacity, + alarm_state:{stars_70:($capacity.stars>=0.7),supporters_md_70:($capacity.supporters_md>=0.7), + invitations_backlog:($backlog>0),red_run:$red,deferred_quota:$deferred}}' > "$work/sweep-state.json" || return 1 + # Prepare the independent notification leg even if state PUT ultimately fails. + jq -r --arg repo "$GITHUB_REPOSITORY" --slurpfile entries "$work/notify.ndjson" ' + (.capacity_pct|to_entries|max_by(.value)) as $max | + ([$entries[]|select(.event=="sweep" and (.action|IN("invite","supporters-append")) and (.result|IN("ok","ok-catchup","already-1")))|.actor_id]|unique|length) as $delivered | + ([$entries[]|select(.event=="sweep" and (.action|IN("revoke","cancel-invite")) and (.result|IN("ok","noop")))]|length) as $revoked | + "supporter-loop [live] \($repo) · sweep · delivered \($delivered) · backlog \(.backlog) · cancelled runs \(.cancelled_runs.event)/\(.cancelled_runs.schedule) · capacity \(($max.value*100)|ceil)% of \($max.key) · revoked \($revoked)"' "$work/sweep-state.json" > "$work/sweep-messages" || return 1 + current=$(jq -c .alarm_state "$work/sweep-state.json") + jq -nr --arg repo "$GITHUB_REPOSITORY" --argjson previous "$previous" --argjson current "$current" ' + $current|to_entries[]|select(.value!=($previous[.key] // false)) | + "supporter-loop [live] \($repo) · \(.key) · \(if .value then "raised" else "cleared" end)"' >> "$work/sweep-messages" || return 1 + echo "::notice::catch-up backlog=$backlog" + content=$(base64 < "$work/sweep-state.json" | tr -d '\n') + jq -cn --arg sha "$sha" --arg content "$content" '{message:"supporter-loop: sweep state",content:$content,committer:{name:"supporter-loop[bot]",email:("supporter-loop@" + "caty-ai" + ".noreply")}}+(if $sha=="" then {} else {sha:$sha} end)' > "$work/state-put.json" + mutate "$LEDGER_TOKEN" PUT "repos/$REWARD_REPO/contents/$state_path" "$work/state-put.json" case "$code" in 200|201) return 0 ;; 409) ;; 422) jq -e '.message|test("sha|does not match";"i")' "$work/mutation.json" >/dev/null || return 1 ;; *) return 1 ;; esac attempt=$((attempt+1)); [ "$attempt" -gt 5 ] || sleep "$((1+RANDOM%4))" done @@ -961,13 +1229,13 @@ jobs: invite) invite emit invite "$result" "$tier" "$subject" "$event" - case "$result" in error-*) invite_failure="$result"; failed=1 ;; *) changed=true ;; esac ;; + case "$result" in deferred-quota) invite_failure="$result" ;; error-*) invite_failure="$result"; failed=1 ;; *) changed=true ;; esac ;; comment) comment "$tier" "$subject" "$event" emit comment "$result" "$tier" "$subject" "$event" case "$result" in error-*) failed=1 ;; *) changed=true ;; esac ;; supporters-append) - if regenerate "$(line supporters-append ok "$tier" "$subject" "$event")"; then result=ok; else result="error-$code"; failed=1; fi + if regenerate "$(line supporters-append ok "$tier" "$subject" "$event")"; then result=ok; else result="error-${regenerate_status:-regenerate}"; failed=1; fi emit supporters-append "$result" "$tier" "$subject" "$event" ;; *) echo '::error::unexpected live plan action'; exit 1 ;; esac @@ -978,209 +1246,296 @@ jobs: [ "$failed" -eq 0 ] || { echo '::error::one or more live actions failed'; exit 1; } exit 0 fi - # §11 step 2 (ungated, ledger only): expire the invite delivery, not its generation. - # Preflight (c) already passed; negative (a)/(b) still permits recovery and audit. - gate="$SWEEP_GATE" - : > "$work/cancel-queue.ndjson" - pages "$LEDGER_TOKEN" "repos/$REWARD_REPO/collaborators?per_page=100" "$work/collaborators.json" - pages "$ADMIN_TOKEN" "repos/$REWARD_REPO/invitations?per_page=100" "$work/pending.json" - jq -e 'all(.[];.id|type=="number" and .>0 and floor==.)' "$work/collaborators.json" >/dev/null - jq -e 'all(.[]; (.id|type=="number" and .>0 and floor==.) and (.invitee.id|type=="number" and .>0 and floor==.))' "$work/pending.json" >/dev/null - model '[.[].actor_id]|unique' > "$work/ids.json" - while IFS= read -r actor_id; do - gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") - [ "$(model "achieved(\"$GITHUB_REPOSITORY\";$actor_id)")" -gt 0 ] || continue - recent=$(model "current(\"$GITHUB_REPOSITORY\";$actor_id)|[.[]|select(.action==\"invite\" and success)]|sort_by(.ts)|last") - [ "$recent" != null ] || continue - # Already re-armed invitations are not re-armed on every weekly sweep. - [ "$(model "delivered(\"$GITHUB_REPOSITORY\";$actor_id;1;\"invite\")")" = true ] || continue - invited=$(jq -r .ts <<< "$recent") - [ $(( $(date -u +%s) - $(date -u -d "$invited" +%s) )) -gt 604800 ] || continue - jq -e --argjson id "$actor_id" 'any(.[];.id==$id)' "$work/collaborators.json" >/dev/null && continue - actor=$(jq -r .actor <<< "$recent") - pending=$(jq -c --argjson id "$actor_id" '[.[]|select(.invitee.id==$id)]' "$work/pending.json") - if [ "$(jq length <<< "$pending")" = 0 ]; then - emit cancel-invite expired 0 '' sweep; changed=true - else - # Only genuinely old pending entries are defensive candidates; a new invite - # created manually or by a recovered run is never cancelled on an old timestamp. - while IFS= read -r invitation; do - created=$(jq -r .created_at <<< "$invitation") - if [ $(( $(date -u +%s) - $(date -u -d "$created" +%s) )) -gt 604800 ]; then - jq -cn --arg actor "$actor" --argjson id "$actor_id" --argjson gen "$gen" --argjson invitation "$invitation" '{actor:$actor,actor_id:$id,gen:$gen,invitation_id:$invitation.id}' >> "$work/cancel-queue.ndjson" - fi - done < <(jq -c '.[]' <<< "$pending") - fi - done < <(jq -r '.[]' "$work/ids.json") - # §11 step 3: exact set-based audit after re-arm; baseline and manual allowlists. - baseline - all_markers - for allow in manual-allowlist manual-markers-allowlist; do - get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/ledger/$allow.json" "$work/allow-blob.json" - case "$code" in - 200) jq -er '.content|gsub("[\\n\\r]"; "") | @base64d|fromjson|select(type=="array")' "$work/allow-blob.json" > "$work/$allow.json" ;; - 404) printf '[]\n' > "$work/$allow.json" ;; - *) echo '::error::manual allowlist unreadable'; exit 1 ;; - esac - done - : > "$work/threads.ndjson" - jq -sr '[.[]|select(.action=="comment")|.subject]|unique|.[]' "$ledger" > "$work/subjects" - while IFS= read -r subject; do - number="${subject##*/}" - [[ "$number" =~ ^[0-9]+$ ]] || { echo '::error::invalid ledger subject'; exit 1; } - exists=true - case "$subject" in - "https://github.com/$GITHUB_REPOSITORY/issues/$number"|"https://github.com/$GITHUB_REPOSITORY/pull/$number") - get "$GH_TOKEN" "repos/$GITHUB_REPOSITORY/issues/$number" "$work/thread.json" - case "$code" in 200) ;; 404|410) exists=false ;; *) echo '::error::thread audit read failed'; exit 1 ;; esac ;; - "https://github.com/$GITHUB_REPOSITORY/discussions/$number") - vars=$(jq -cn --arg owner "${GITHUB_REPOSITORY%/*}" --arg name "${GITHUB_REPOSITORY#*/}" --argjson number "$number" '{owner:$owner,name:$name,number:$number}') - # shellcheck disable=SC2016 - if ! graph 'query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){discussion(number:$number){id}}}' "$vars" "$work/thread.json"; then - jq -e '(.errors|length)>0 and all(.errors[];.type=="NOT_FOUND")' "$work/thread.json" >/dev/null || { echo '::error::thread audit read failed'; exit 1; } - exists=false - fi ;; - *) echo '::error::ledger subject outside source'; exit 1 ;; - esac - # The ledger has no comment ID: never guess that a missing marker was deleted. - # Owner-accepted residuals are explicit in manual-markers-allowlist (§5.3). - jq -cn --arg subject "$subject" --argjson exists "$exists" '{subject:$subject,exists:$exists,readable:$exists,marker_deleted:false}' >> "$work/threads.ndjson" - done < "$work/subjects" - jq -sn --arg source "$GITHUB_REPOSITORY" --slurpfile ledger "$ledger" \ - --slurpfile baseline "$work/baseline.json" --slurpfile collaborators "$work/collaborators.json" \ - --slurpfile invitations "$work/pending.json" --slurpfile markers "$work/markers.json" \ - --slurpfile threads "$work/threads.ndjson" --slurpfile manual "$work/manual-allowlist.json" \ - --slurpfile manual_markers "$work/manual-markers-allowlist.json" \ - '{source_repo:$source,ledger:$ledger,baseline:$baseline[0],collaborators:$collaborators[0],invitations:$invitations[0],invitations_status:200,markers:$markers[0],threads:$threads,manual_allowlist:$manual[0],manual_markers_allowlist:$manual_markers[0],complete:true}' > "$work/audit.json" - divergences=$(jq -r --arg mode live ' - - def success: .result|(.=="ok" or startswith("ok-") or startswith("already-") or .=="noop" or .=="expired"); - def closes: (.action|IN("revoke","would-revoke","cancel-invite","would-cancel-invite")) and success and .result!="expired"; - def ids: map(if type=="number" then . else .id end)|unique; - def markkey: [.actor_id,.tier,.subject]; - . as $s | [.markers[]|select(.author=="github-actions[bot]")] as $markers | - if $mode=="record-only" then - (if (.collaborators|ids)!=(.baseline.collaborators|ids) then "reduced-collaborators: baseline differs" else empty end), - (if .invitations_status!=403 then "reduced-probe: expected 403" else empty end), - (if ($markers|length)>0 then "reduced-markers: workflow markers exist" else empty end) - else - # File-order duplicates cannot increase the closure count. Latest means ts, - # with input order as the tie breaker for same-second action attempts. - [.ledger|to_entries[]|.value+{_order:.key}|select(.repo==$s.source_repo)] as $ledger | - ([$ledger|group_by(.actor_id)[]| - . as $history | (1+([$history[]|select(closes)|.run_id]|unique|length)) as $gen | - [$history[]|select(.gen==$gen)] as $current | - [$current[]|select(.action=="invite" and success)]|sort_by(.ts,._order)|last as $invite | - select($invite!=null) | - select(all($current[]; (.action!="cancel-invite" or .result!="expired") or ([.ts,._order] <= [$invite.ts,$invite._order]))) | - $invite.actor_id] | unique) as $predicted | - ([.collaborators[].id,.invitations[].invitee.id]|unique) as $actual | - ([.baseline.collaborators[].id,.baseline.invitations[].invitee.id]+(.manual_allowlist|ids)|unique) as $allowed | - (($predicted-$actual)[]|"access: missing actor_id=\(.)"), - (($actual-$predicted-$allowed)[]|"strangers: unexpected actor_id=\(.)"), - [$ledger[]|select(.action=="comment" and success)] as $comments | - ($markers[] as $m|select(all($comments[];.actor_id!=$m.actor_id or .tier!=$m.tier))|"markers-A: unledgered actor_id=\($m.actor_id) tier=\($m.tier) subject=\($m.subject)"), - ($comments|unique_by(markkey)[] as $c| - [$s.threads[]|select(.subject==$c.subject)] as $thread | - if ($thread|length)==0 then "markers-B-state: absent thread evidence subject=\($c.subject)" - elif any($thread[];.exists and .readable and (.marker_deleted!=true)) and - all($markers[];markkey!=($c|markkey)) and - all($s.manual_markers_allowlist[];markkey!=($c|markkey)) - then "markers-B: missing actor_id=\($c.actor_id) tier=\($c.tier) subject=\($c.subject)" else empty end) - end - ' "$work/audit.json") - if [ -n "$divergences" ]; then - while IFS= read -r divergence; do printf '::error::%s\n' "$divergence"; done <<< "$divergences" - gate=false - fi - # §11(e), before any destructive call: recover own markers missing from the ledger. - # This runs even if (a)/(d) is false, so a red run can be reconciled safely. - printf '[]\n' > "$work/protected.json" - while IFS= read -r actor_id; do - [ "$(model "achieved(\"$GITHUB_REPOSITORY\";$actor_id)")" = 1 ] || continue - jq -e --argjson id "$actor_id" 'index($id)!=null' <<< "$STARGAZERS" >/dev/null && continue - gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") - actor=$(model "[.[]|select(.repo==\"$GITHUB_REPOSITORY\" and .actor_id==$actor_id)]|sort_by(.ts)|last|.actor" | jq -r .) - while IFS= read -r evidence; do - tier=$(jq -r .tier <<< "$evidence"); subject=$(jq -r .subject <<< "$evidence") - jq --argjson id "$actor_id" '.+[$id]|unique' "$work/protected.json" > "$work/protected-next.json" - mv "$work/protected-next.json" "$work/protected.json" - if ! jq -se -L "$work" --arg mode "$MODE" --argjson id "$actor_id" --argjson tier "$tier" --arg repo "$GITHUB_REPOSITORY" \ - 'include "model"; any(.[];.repo==$repo and .actor_id==$id and .tier==$tier and .action=="comment" and success)' "$ledger" >/dev/null; then - case "$subject" in - "https://github.com/$GITHUB_REPOSITORY/discussions/"*) event=discussion ;; - "https://github.com/$GITHUB_REPOSITORY/pull/"*) event=pull_request_target ;; - "https://github.com/$GITHUB_REPOSITORY/issues/"*) event=issues ;; - *) echo '::error::marker subject outside source'; gate=false; continue ;; - esac - # A recovered comment uses its original tier key, not a sweep closure key. - emit comment "already-$tier" "$tier" "$subject" "$event"; changed=true + sweep_cleanup() { + # §11 step 2 (ungated, ledger only): expire the invite delivery, not its generation. + # Preflight (c) already passed; negative (a)/(b) still permits recovery and audit. + gate="$SWEEP_GATE" + audit_complete=false + : > "$work/cancel-queue.ndjson" + pages "$LEDGER_TOKEN" "repos/$REWARD_REPO/collaborators?per_page=100" "$work/collaborators.json" || return 1 + pages "$ADMIN_TOKEN" "repos/$REWARD_REPO/invitations?per_page=100" "$work/pending.json" || return 1 + jq -e 'all(.[];.id|type=="number" and .>0 and floor==.)' "$work/collaborators.json" >/dev/null || return 1 + jq -e 'all(.[]; (.id|type=="number" and .>0 and floor==.) and (.invitee.id|type=="number" and .>0 and floor==.))' "$work/pending.json" >/dev/null || return 1 + # order-insensitive by design (builtin sort is fine here) + model '[.[].actor_id]|unique' > "$work/ids.json" + while IFS= read -r actor_id; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") + [ "$(model "achieved(\"$GITHUB_REPOSITORY\";$actor_id)")" -gt 0 ] || continue + recent=$(model "current(\"$GITHUB_REPOSITORY\";$actor_id)|[.[]|select(.action==\"invite\" and success)]|sort_by(.ts)|last") + [ "$recent" != null ] || continue + # Already re-armed invitations are not re-armed on every weekly sweep. + [ "$(model "delivered(\"$GITHUB_REPOSITORY\";$actor_id;1;\"invite\")")" = true ] || continue + invited=$(jq -r .ts <<< "$recent") + [ $(( $(date -u +%s) - $(date -u -d "$invited" +%s) )) -gt 604800 ] || continue + jq -e --argjson id "$actor_id" 'any(.[];.id==$id)' "$work/collaborators.json" >/dev/null && continue + actor=$(jq -r .actor <<< "$recent") + pending=$(jq -c --argjson id "$actor_id" '[.[]|select(.invitee.id==$id)]' "$work/pending.json") + if [ "$(jq length <<< "$pending")" = 0 ]; then + emit cancel-invite expired 0 '' sweep; changed=true + else + # Only genuinely old pending entries are defensive candidates; a new invite + # created manually or by a recovered run is never cancelled on an old timestamp. + while IFS= read -r invitation; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + created=$(jq -r .created_at <<< "$invitation") + if [ $(( $(date -u +%s) - $(date -u -d "$created" +%s) )) -gt 604800 ]; then + jq -cn --arg actor "$actor" --argjson id "$actor_id" --argjson gen "$gen" --argjson invitation "$invitation" '{actor:$actor,actor_id:$id,gen:$gen,invitation_id:$invitation.id}' >> "$work/cancel-queue.ndjson" + fi + done < <(jq -c '.[]' <<< "$pending") fi - done < <(jq -c --argjson id "$actor_id" '.[]|select(.actor_id==$id)' "$work/markers.json") - done < <(jq -r '.[]' "$work/ids.json") - # §11 step 4: ALL destructive effects are behind the combined (a)–(e) gate. - if [ "$gate" = true ]; then - # Defensive pending cancels queued at step 2 execute before unstar cleanup. - while IFS= read -r queued; do - actor_id=$(jq -r .actor_id <<< "$queued"); actor=$(jq -r .actor <<< "$queued"); gen=$(jq -r .gen <<< "$queued") - invitation_id=$(jq -r .invitation_id <<< "$queued") - jq -e --argjson id "$actor_id" 'index($id)!=null' "$work/protected.json" >/dev/null && continue - printf '{}\n' > "$work/empty.json" - mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/invitations/$invitation_id" "$work/empty.json" - case "$code" in 200|204) result=ok; changed=true ;; *) result="error-$code"; failed=1 ;; esac - emit cancel-invite "$result" 0 '' sweep - done < "$work/cancel-queue.ndjson" + done < <(jq -r '.[]' "$work/ids.json") + # §11 step 3: exact set-based audit after re-arm; baseline and manual allowlists. + baseline || return 1 + all_markers || return 1 + for allow in manual-allowlist manual-markers-allowlist; do + get "$LEDGER_TOKEN" "repos/$REWARD_REPO/contents/ledger/$allow.json" "$work/allow-blob.json" || return 1 + case "$code" in + 200) jq -er '.content|gsub("[\\n\\r]"; "") | @base64d|fromjson|select(type=="array")' "$work/allow-blob.json" > "$work/$allow.json" ;; + 404) printf '[]\n' > "$work/$allow.json" ;; + *) echo '::error::manual allowlist unreadable'; return 1 ;; + esac + done + : > "$work/threads.ndjson" + jq -sr '[.[]|select(.action=="comment")|.subject]|unique|.[]' "$ledger" > "$work/subjects" + while IFS= read -r subject; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + number="${subject##*/}" + [[ "$number" =~ ^[0-9]+$ ]] || { echo '::error::invalid ledger subject'; return 1; } + exists=true + case "$subject" in + "https://github.com/$GITHUB_REPOSITORY/issues/$number"|"https://github.com/$GITHUB_REPOSITORY/pull/$number") + get "$GH_TOKEN" "repos/$GITHUB_REPOSITORY/issues/$number" "$work/thread.json" || return 1 + case "$code" in 200) ;; 404|410) exists=false ;; *) echo '::error::thread audit read failed'; return 1 ;; esac ;; + "https://github.com/$GITHUB_REPOSITORY/discussions/$number") + vars=$(jq -cn --arg owner "${GITHUB_REPOSITORY%/*}" --arg name "${GITHUB_REPOSITORY#*/}" --argjson number "$number" '{owner:$owner,name:$name,number:$number}') + # shellcheck disable=SC2016 + if ! graph 'query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){discussion(number:$number){id}}}' "$vars" "$work/thread.json"; then + jq -e '(.errors|length)>0 and all(.errors[];.type=="NOT_FOUND")' "$work/thread.json" >/dev/null || { echo '::error::thread audit read failed'; return 1; } + exists=false + fi ;; + *) echo '::error::ledger subject outside source'; return 1 ;; + esac + # The ledger has no comment ID: never guess that a missing marker was deleted. + # Owner-accepted residuals are explicit in manual-markers-allowlist (§5.3). + jq -cn --arg subject "$subject" --argjson exists "$exists" '{subject:$subject,exists:$exists,readable:$exists,marker_deleted:false}' >> "$work/threads.ndjson" + done < "$work/subjects" + jq -sn --arg source "$GITHUB_REPOSITORY" --slurpfile ledger "$ledger" \ + --slurpfile baseline "$work/baseline.json" --slurpfile collaborators "$work/collaborators.json" \ + --slurpfile invitations "$work/pending.json" --slurpfile markers "$work/markers.json" \ + --slurpfile threads "$work/threads.ndjson" --slurpfile manual "$work/manual-allowlist.json" \ + --slurpfile manual_markers "$work/manual-markers-allowlist.json" \ + '{source_repo:$source,ledger:$ledger,baseline:$baseline[0],collaborators:$collaborators[0],invitations:$invitations[0],invitations_status:200,markers:$markers[0],threads:$threads,manual_allowlist:$manual[0],manual_markers_allowlist:$manual_markers[0],complete:true}' > "$work/audit.json" + divergences=$(jq -r --arg mode live ' + + def success: .result|(.=="ok" or startswith("ok-") or startswith("already-") or .=="noop" or .=="expired"); + def closes: (.action|IN("revoke","would-revoke","cancel-invite","would-cancel-invite")) and success and .result!="expired"; + def ids: map(if type=="number" then . else .id end)|unique; + def markkey: [.actor_id,.tier,.subject]; + . as $s | [.markers[]|select(.author=="github-actions[bot]")] as $markers | + if $mode=="record-only" then + (if (.collaborators|ids)!=(.baseline.collaborators|ids) then "reduced-collaborators: baseline differs" else empty end), + (if .invitations_status!=403 then "reduced-probe: expected 403" else empty end), + (if ($markers|length)>0 then "reduced-markers: workflow markers exist" else empty end) + else + # File-order duplicates cannot increase the closure count. Latest means ts, + # with input order as the tie breaker for same-second action attempts. + [.ledger|to_entries[]|.value+{_order:.key}|select(.repo==$s.source_repo)] as $ledger | + # order-insensitive by design (builtin sort is fine here) + ([$ledger|group_by(.actor_id)[]| + . as $history | (1+([$history[]|select(closes)|.run_id]|unique|length)) as $gen | + [$history[]|select(.gen==$gen)] as $current | + [$current[]|select(.action=="invite" and success)]|sort_by(.ts,._order)|last as $invite | + select($invite!=null) | + select(all($current[]; (.action!="cancel-invite" or .result!="expired") or ([.ts,._order] <= [$invite.ts,$invite._order]))) | + $invite.actor_id] | unique) as $predicted | + ([.collaborators[].id,.invitations[].invitee.id]|unique) as $actual | + ([.baseline.collaborators[].id,.baseline.invitations[].invitee.id]+(.manual_allowlist|ids)|unique) as $allowed | + (($predicted-$actual)[]|"access: missing actor_id=\(.)"), + (($actual-$predicted-$allowed)[]|"strangers: unexpected actor_id=\(.)"), + [$ledger[]|select(.action=="comment" and success)] as $comments | + ($markers[] as $m|select(all($comments[];.actor_id!=$m.actor_id or .tier!=$m.tier))|"markers-A: unledgered actor_id=\($m.actor_id) tier=\($m.tier) subject=\($m.subject)"), + # order-insensitive by design (builtin sort is fine here) + ($comments|unique_by(markkey)[] as $c| + [$s.threads[]|select(.subject==$c.subject)] as $thread | + if ($thread|length)==0 then "markers-B-state: absent thread evidence subject=\($c.subject)" + elif any($thread[];.exists and .readable and (.marker_deleted!=true)) and + all($markers[];markkey!=($c|markkey)) and + all($s.manual_markers_allowlist[];markkey!=($c|markkey)) + then "markers-B: missing actor_id=\($c.actor_id) tier=\($c.tier) subject=\($c.subject)" else empty end) + end + ' "$work/audit.json") + if [ -n "$divergences" ]; then + while IFS= read -r divergence; do printf '::error::%s\n' "$divergence"; done <<< "$divergences" + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + gate=false + fi + # §11(e), before any destructive call: recover own markers missing from the ledger. + # This runs even if (a)/(d) is false, so a red run can be reconciled safely. + printf '[]\n' > "$work/protected.json" while IFS= read -r actor_id; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi [ "$(model "achieved(\"$GITHUB_REPOSITORY\";$actor_id)")" = 1 ] || continue jq -e --argjson id "$actor_id" 'index($id)!=null' <<< "$STARGAZERS" >/dev/null && continue - jq -e --argjson id "$actor_id" 'index($id)!=null' "$work/protected.json" >/dev/null && continue gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") actor=$(model "[.[]|select(.repo==\"$GITHUB_REPOSITORY\" and .actor_id==$actor_id)]|sort_by(.ts)|last|.actor" | jq -r .) - if ! identity; then emit revoke "error-$code" 0 '' sweep; failed=1; continue; fi - # Match numeric IDs; resolving the current login prevents rename confusion (§11). - if ! pages "$ADMIN_TOKEN" "repos/$REWARD_REPO/invitations?per_page=100" "$work/current-pending.json"; then - emit revoke error-preflight 0 '' sweep; failed=1; continue - fi - get "$ADMIN_TOKEN" "repos/$REWARD_REPO/collaborators/$actor" "$work/current-collaborator.json" - pending=$(jq -c --argjson id "$actor_id" '[.[]|select(.invitee.id==$id)]' "$work/current-pending.json") - case "$code" in - 404) - if [ "$(jq length <<< "$pending")" = 0 ]; then result=noop - else + while IFS= read -r evidence; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + tier=$(jq -r .tier <<< "$evidence"); subject=$(jq -r .subject <<< "$evidence") + jq --argjson id "$actor_id" '.+[$id]|unique' "$work/protected.json" > "$work/protected-next.json" + mv "$work/protected-next.json" "$work/protected.json" + if ! jq -se -L "$work" --arg mode "$MODE" --argjson id "$actor_id" --argjson tier "$tier" --arg repo "$GITHUB_REPOSITORY" \ + 'include "model"; any(.[];.repo==$repo and .actor_id==$id and .tier==$tier and .action=="comment" and success)' "$ledger" >/dev/null; then + case "$subject" in + "https://github.com/$GITHUB_REPOSITORY/discussions/"*) event=discussion ;; + "https://github.com/$GITHUB_REPOSITORY/pull/"*) event=pull_request_target ;; + "https://github.com/$GITHUB_REPOSITORY/issues/"*) event=issues ;; + *) echo '::error::marker subject outside source'; gate=false; continue ;; + esac + # A recovered comment uses its original tier key, not a sweep closure key. + emit comment "already-$tier" "$tier" "$subject" "$event"; changed=true + fi + done < <(jq -c --argjson id "$actor_id" '.[]|select(.actor_id==$id)' "$work/markers.json") + done < <(jq -r '.[]' "$work/ids.json") + audit_complete=true + # §11 step 4: ALL destructive effects are behind the combined (a)–(e) gate. + if [ "$gate" = true ]; then + # Defensive pending cancels queued at step 2 execute before unstar cleanup. + while IFS= read -r queued; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + actor_id=$(jq -r .actor_id <<< "$queued"); actor=$(jq -r .actor <<< "$queued"); gen=$(jq -r .gen <<< "$queued") + invitation_id=$(jq -r .invitation_id <<< "$queued") + jq -e --argjson id "$actor_id" 'index($id)!=null' "$work/protected.json" >/dev/null && continue + printf '{}\n' > "$work/empty.json" + mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/invitations/$invitation_id" "$work/empty.json" + case "$code" in 200|204) result=ok; changed=true ;; *) result="error-$code"; failed=1 ;; esac + emit cancel-invite "$result" 0 '' sweep || return 1 + done < "$work/cancel-queue.ndjson" + while IFS= read -r actor_id; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi + [ "$(model "achieved(\"$GITHUB_REPOSITORY\";$actor_id)")" = 1 ] || continue + jq -e --argjson id "$actor_id" 'index($id)!=null' <<< "$STARGAZERS" >/dev/null && continue + jq -e --argjson id "$actor_id" 'index($id)!=null' "$work/protected.json" >/dev/null && continue + gen=$(model "generation(\"$GITHUB_REPOSITORY\";$actor_id)") + actor=$(model "[.[]|select(.repo==\"$GITHUB_REPOSITORY\" and .actor_id==$actor_id)]|sort_by(.ts)|last|.actor" | jq -r .) + if ! identity; then emit revoke "error-$code" 0 '' sweep; failed=1; continue; fi + # Match numeric IDs; resolving the current login prevents rename confusion (§11). + if ! pages "$ADMIN_TOKEN" "repos/$REWARD_REPO/invitations?per_page=100" "$work/current-pending.json"; then + emit revoke error-preflight 0 '' sweep; failed=1; continue + fi + get "$ADMIN_TOKEN" "repos/$REWARD_REPO/collaborators/$actor" "$work/current-collaborator.json" || return 1 + pending=$(jq -c --argjson id "$actor_id" '[.[]|select(.invitee.id==$id)]' "$work/current-pending.json") + case "$code" in + 404) + if [ "$(jq length <<< "$pending")" = 0 ]; then result=noop + else + printf '{}\n' > "$work/empty.json" + mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/collaborators/$actor" "$work/empty.json" + case "$code" in 200|204|404) result=noop ;; *) result="error-$code"; failed=1 ;; esac + fi ;; + 204) printf '{}\n' > "$work/empty.json" mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/collaborators/$actor" "$work/empty.json" - case "$code" in 200|204|404) result=noop ;; *) result="error-$code"; failed=1 ;; esac - fi ;; - 204) + case "$code" in 200|204) result=ok ;; *) result="error-$code"; failed=1 ;; esac ;; + *) result="error-$code"; failed=1 ;; + esac + emit revoke "$result" 0 '' sweep || return 1 + case "$result" in error-*) continue ;; *) changed=true ;; esac + # The pair deliberately shares the pre-closure gen and run_id: one closure (§7). + while IFS= read -r invitation_id; do + if ! within_deadline; then [ "$audit_complete" = true ] && return 0; echo '::error::audit deadline exceeded'; return 1; fi printf '{}\n' > "$work/empty.json" - mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/collaborators/$actor" "$work/empty.json" - case "$code" in 200|204) result=ok ;; *) result="error-$code"; failed=1 ;; esac ;; - *) result="error-$code"; failed=1 ;; + mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/invitations/$invitation_id" "$work/empty.json" + case "$code" in 200|204) result=ok ;; 404) result=noop ;; *) result="error-$code"; failed=1 ;; esac + emit cancel-invite "$result" 0 '' sweep || return 1 + done < <(jq -r '.[].id' <<< "$pending") + done < <(jq -r '.[]' "$work/ids.json") + else + echo '::error::sweep preconditions (a)-(e) failed; all API cleanup skipped' + failed=1 + fi + } + gate="$SWEEP_GATE" + budget_active=true + if ! sweep_cleanup; then + echo '::error::sweep recovery/audit incomplete; destructive work stopped' + gate=false; failed=1 + fi + budget_active=false + # §11 step 4b: only (a)(b)(c), independently of cleanup/audit (d)(e). + budget_active=true + : > "$work/supporters-queue.ndjson" + deferred_quota=false + catchup_gate="$SWEEP_GATE" + if ! jq -e 'type=="array" and all(.[]; type=="number" and .>0 and floor==.)' <<< "$STARGAZERS" >/dev/null; then + echo '::error::stargazer handoff invalid in act' + catchup_gate=false; STARGAZERS='[]'; failed=1 + fi + candidates_now() { + jq -s -L "$work" --arg mode "$MODE" --arg repo "$GITHUB_REPOSITORY" --arg tiers "$TIERS_ENABLED" --argjson stars "$STARGAZERS" \ + 'include "model"; catchup($repo;$stars;$tiers)' "$ledger" + } + queue_supporters() { + reservedCount=$((reservedCount+1)) + line supporters-append ok 1 '' sweep >> "$work/supporters-queue.ndjson" + } + candidates_now > "$work/catchup.json" + if [ "$catchup_gate" = true ]; then + while IFS= read -r item; do + work_budget 1 || break + actor_id=$(jq -r .actor_id <<< "$item"); actor=$(jq -r .actor <<< "$item"); gen=$(jq -r .gen <<< "$item") + queue_supporters + done < <(jq -c '.[]|select(.invite and (.append|not))' "$work/catchup.json") + while IFS= read -r item; do + # PUT + invite line + optional append reservation; repairs already counted. + needed=$(jq 'if .append then 2 else 3 end' <<< "$item") + work_budget "$needed" || break + if quota_room; then :; else + [ "$?" -eq 1 ] || failed=1 + break + fi + actor_id=$(jq -r .actor_id <<< "$item"); actor=$(jq -r .actor <<< "$item"); gen=$(jq -r .gen <<< "$item") + if [ "$(jq -r .exhausted <<< "$item")" = true ]; then emit skip excluded-unresponsive 0 '' sweep || failed=1; continue; fi + if ! identity; then emit invite "error-$code" 1 '' sweep || failed=1; failed=1; continue; fi + assoc=NONE + if ! eligibility; then emit invite error-identity 1 '' sweep || failed=1; failed=1; continue; fi + if [ -n "$reason" ]; then emit skip "$reason" 0 '' sweep || failed=1; continue; fi + work_budget "$needed" || break + invite true + case "$result" in + budget-stop) break ;; + deferred-quota) deferred_quota=true; emit skip deferred-quota 0 '' sweep || failed=1; break ;; + error-*) emit invite "$result" 1 '' sweep || failed=1; failed=1; continue ;; + ok) result=ok-catchup ;; esac - emit revoke "$result" 0 '' sweep - case "$result" in error-*) continue ;; *) changed=true ;; esac - # The pair deliberately shares the pre-closure gen and run_id: one closure (§7). - while IFS= read -r invitation_id; do - printf '{}\n' > "$work/empty.json" - mutate "$ADMIN_TOKEN" DELETE "repos/$REWARD_REPO/invitations/$invitation_id" "$work/empty.json" - case "$code" in 200|204) result=ok ;; 404) result=noop ;; *) result="error-$code"; failed=1 ;; esac - emit cancel-invite "$result" 0 '' sweep - done < <(jq -r '.[].id' <<< "$pending") - done < <(jq -r '.[]' "$work/ids.json") + if ! emit invite "$result" 1 '' sweep; then failed=1; break; fi + if [ "$(jq -r .append <<< "$item")" = false ]; then queue_supporters; fi + done < <(jq -c '.[]|select(.invite|not)' "$work/catchup.json") else - echo '::error::sweep preconditions (a)-(e) failed; all API cleanup skipped' + echo '::error::catch-up preconditions (a)(b)(c) failed; catch-up skipped' failed=1 fi - # §11 steps 5, 6, 7 must still run after a red-run/audit gate failure. + # §11 steps 5–8 always run, including after a red-run/audit gate failure. + budget_active=false rotate || failed=1 - if [ "$changed" = true ]; then regenerate || failed=1; fi + reservedCount=$((reservedCount-1)) + if regenerate "$(cat "$work/supporters-queue.ndjson")"; then projection_result=ok + else projection_result="error-${regenerate_status:-regenerate}"; failed=1; fi + while IFS= read -r queued; do + actor=$(jq -r .actor <<< "$queued"); actor_id=$(jq -r .actor_id <<< "$queued"); gen=$(jq -r .gen <<< "$queued") + reservedCount=$((reservedCount-1)) + emit supporters-append "$projection_result" 1 '' sweep || failed=1 + done < "$work/supporters-queue.ndjson" reminder "$LEDGER_EXPIRES" 'ledger token' reminder "$ADMIN_EXPIRES" 'invitation token' + reservedCount=$((reservedCount-1)) + sweep_state || { echo '::error::sweep state/summary failed'; failed=1; } [ "$failed" -eq 0 ] || { echo '::error::live sweep failed closed'; exit 1; } - name: Notify owner independently of ledger and delivery outcome if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} env: MODE: ${{ inputs.mode }} + SWEEP: ${{ inputs.sweep }} TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} run: | @@ -1199,7 +1554,13 @@ jobs: fi if [ "$status" != 200 ] || ! jq -e '.ok==true' "$work/telegram.json" >/dev/null; then echo '::warning::Telegram rejected owner notification'; fi } - if [ -f "$work/notify.ndjson" ]; then + if [ "$SWEEP" = true ]; then + if [ -f "$work/sweep-messages" ]; then + while IFS= read -r text; do notify "$text"; done < "$work/sweep-messages" + else + notify "supporter-loop [live] $GITHUB_REPOSITORY · sweep · summary unavailable (failed)" + fi + elif [ -f "$work/notify.ndjson" ]; then while IFS= read -r entry; do case "$(jq -r .result <<< "$entry")" in already-*) continue ;; esac text=$(jq -r '"supporter-loop [\(.mode)] \(.repo) · \(.event) · @\(.actor) · tier \(.tier) · \(.action) · \(.result)" + (if .subject=="" then "" else "\n"+.subject end)' <<< "$entry") diff --git a/supporter-loop/README.md b/supporter-loop/README.md index eb34f87..91f63bf 100644 --- a/supporter-loop/README.md +++ b/supporter-loop/README.md @@ -1,6 +1,6 @@ # Supporter loop -The reusable workflow implements the frozen [Supporter Reward Loop contract](https://github.com/caty-ai/x-collector/blob/epic/119/docs/supporter-loop/CONTRACT.md) (moves to `main` when the epic merges), v1.11. The caller initially selects `record-only`; enabling live delivery and approving the placeholder Japanese/English comment templates belong to owner checkpoint #4. The optional backfill input is deliberately absent until that checkpoint. +The reusable workflow implements the frozen [Supporter Reward Loop contract](https://github.com/caty-ai/x-collector/blob/epic/119/docs/supporter-loop/CONTRACT.md) (moves to `main` when the epic merges), v2.0. The caller initially selects `record-only`; enabling live delivery and approving the placeholder Japanese/English comment templates belong to owner checkpoint #4. The retired `backfill` input is absent; historical `ok-backfill` lines remain successful deliveries. - `check-contents-decode.sh ` requires CR/LF stripping at every base64 decode, byte-compares wrapped Contents API fixtures, and verifies that plain decoding fails. - `check-ledger.sh ` prints the number of mode/action violations and succeeds only at zero. @@ -17,12 +17,84 @@ The reusable workflow implements the frozen [Supporter Reward Loop contract](htt - Backslash-newline continuations inside a shell word join without an inserted space, preserving curl option clusters, methods and endpoint/path words. An odd trailing backslash count escapes the newline; an even count leaves literal backslashes and a real statement boundary. Shell single quotes do not enable continuation. Whitespace before the continuation backslash keeps the word separator. Unsupported multiline/folded YAML and escaped double-quoted run scalars still fail closed under rule `d`; use literal `|` blocks for executable multiline scripts. - `reconcile-audit.sh --mode record-only|live` reads JSON state from stdin and reports divergence. Its header documents the fixture schema and `--live-gh` collection interface. Missing or incomplete evidence fails closed. Record-only checks the reduced audit; live checks access, strangers, and both marker directions, including the manual allowlists. The owner-only checkpoint #4 zero-send audit uses `checkpoint4: true` in the stdin JSON to compare the baseline invitation set too; with `--live-gh` this additionally requires the invitation credential and still checks that the ledger credential receives 403. -`supporter-loop-selftest.yml` runs these scripts against clean and deliberately violating fixtures on pull requests touching this directory or either supporter-loop workflow. Fixtures are offline: no real credentials or GitHub writes are needed. The reusable workflow executes inline trusted code and never checks out the triggering ref. Additional fixture tests extract its actual jq transition, audit and SUPPORTERS.md models, verify the frozen interface, and execute only `record-only` decision/sweep code with fail-closed local API doubles. The live workflow path is not executed by these tests. +`supporter-loop-selftest.yml` runs these scripts against clean and deliberately violating fixtures on pull requests touching this directory or either supporter-loop workflow. Fixtures are offline: no real credentials or GitHub writes are needed. The reusable workflow executes inline trusted code and never checks out the triggering ref. Additional fixture tests extract its actual jq transition, audit and SUPPORTERS.md models, verify the frozen interface, and execute decision/sweep code and extracted live helpers with fail-closed local API doubles. These tests never contact GitHub or use real credentials; they do not prove live API permissions. The family roster is copied from `external-input-watch.yml`: **case-insensitive; identical token set**. The twin reference is in the new workflow only because issue #75 forbids changing existing workflows. Keep these copies synchronized in a separately authorized change. Caller concurrency, caller secrets registration, baseline creation, Contributors wall and release notes belong to child #3/the owner. Baselines and manual allowlists live under `ledger/` in the reward repository; this change does not create them. Live API permissions, owner baselines, checkpoint approvals and actual zero-send evidence require owner verification outside fixture CI. -The append-only schema has no comment ID. Live collection cannot prove that an absent marker was removed by moderation rather than edited. For a still-readable thread it conservatively blocks cleanup until the owner records the accepted exception in `manual-markers-allowlist.json`; fixture audits also accept explicit `marker_deleted` evidence. Deleted/unreadable threads self-exempt. The Actions API 1,000-result cap also blocks cleanup conservatively, including when unrelated workflows fill the repository-wide listing. +The append-only schema has no comment ID. Live collection cannot prove that an absent marker was removed by moderation rather than edited. For a still-readable thread it conservatively blocks cleanup until the owner records the accepted exception in `manual-markers-allowlist.json`; fixture audits also accept explicit `marker_deleted` evidence. Deleted/unreadable threads self-exempt. The Actions API 1,000-result cap also blocks cleanup conservatively, using only the per-workflow supporter-loop listing; unrelated workflows do not consume that cap. Regeneration prepends the reward repository’s `SUPPORTERS.header.md` (owned by child #1) and fails closed if that header is missing or malformed. + +## Contract v2.0 + +Every live sweep repairs stranded tier-1 projections first, then invites undelivered +stargazers in page order (never-invited before re-armed). All current-generation +`excluded-*` skips count regardless of key; `deferred-quota` never excludes. +Catch-up requires tier 1 enabled. Delivery lines use `:1:` even on a +sweep; closures and sweep skips use `:sweep:` with tier 0. Two expired +invitation cycles yield one `excluded-unresponsive` sweep skip. The actor's next +tier event can still re-invite normally. + +The rolling quota counts actual successful invitation PUTs (`ok`, `ok-backfill`, +`ok-catchup`) plus recent pending invitations not represented by such a line. +`already-1` does not count. The window is `(now - 24 h, now]`, and `Q = max(0, +45 - B)` is re-evaluated before every sweep PUT. Waiting is limited to three +minutes cumulatively, with five-minute age-out lookahead. Event runs use the +platform quota directly. Record-only predicts the same batch using ledger-only B +(the unledgered pending term needs Administration and is unavailable). + +Catch-up stops taking work at six minutes elapsed or 150 attempted/reserved +writes; queue entries reserve their closing ledger writes immediately. Audit, +marker and revoke loops also honor the elapsed deadline. Rotation, unconditional +regeneration, reminders and state/summary still run. All act content writes are +paced at least one second apart. Regeneration retries GET sha → fresh ledger read +→ render → PUT after each CAS conflict. Queued append lines follow that PUT; +a failed projection records errors and keeps those actors in backlog. An expired +tier-1 invitation disappears from the projection until re-invited; tier-2/3 +recognition stays. + +`ledger/--.sweep-state.json` is a derived owner latch: + +```json +{"last_sweep_ts":"2026-09-08T00:00:00Z","backlog":0, + "cancelled_runs":{"event":0,"schedule":0}, + "capacity_pct":{"stars":0,"supporters_md":0,"invitations":0}, + "alarm_state":{"stars_70":false,"supporters_md_70":false, + "invitations_backlog":false,"red_run":false,"deferred_quota":false}} +``` + +Capacity values are fractions (0.7 = 70%): stars / 40000, published bytes / 1048576, +and B / 45. The watermark includes successful scheduled and manual sweeps. +Cancelled counts are informational and never gate delivery. Live sweeps send one +summary with delivered/backlog/cancelled/capacity/revoked counts and alarm +transitions only; missing/corrupt state starts from false. Record-only never +reads or writes this latch and never sends Telegram. + +**422 launch gate:** `INVITE_LIMIT_PATTERN` is defined once in the reusable +workflow and is provisional: `invit.*(limit|exceed|too many|24 ?h|per day)|too many invit`. +Precondition: the reward repository must contain a valid `SUPPORTERS.header.md`; missing or malformed headers fail regeneration and the live sweep closed. + +Before merge, the owner must capture a real invitation-limit response from the +approved probe, remove sensitive data, replace +`fixtures/workflow/invitation-limit-422.json` with that body, and pin the pattern +to its exact `.message`/`.errors[].message`/`.errors[].code` wording. Rerun +`python3 -B supporter-loop/fixtures/workflow/test-live-contract.py`; the real +limit body must match while `sha does not match` and generic `Validation Failed` +422 bodies remain errors. Events defer the entire invitation/comment/projection +bundle, while sweeps write one deferred skip and retain a green backlog. + +New v2 fixtures: `test-catchup-model.py` covers candidate ordering, namespaces, +quota window edges, historical values, mode parity and projection re-arm; +`test-live-contract.py` runs actual catch-up/event/regeneration/quota/state helpers +for 422 bundles, exclusions, queue ordering, CAS re-read, pacing, budget and +alarm transitions. `test-decide.py`, `test-model.py`, and `test-interface.py` +retain the existing execution/model/interface checks. The four `clean-*` ledger +fixtures cover the new results and tier-1 sweep deliveries; the vocabulary +checker already accepts these values, so its implementation needs no change. +The stars scanner reuses rule `a` for PAT/secret references, rule `b` for a second +step, noncanonical GET or write, and rule `d` for nonliteral run blocks. The three +`violating-*-stars-*.yml` fixtures pin those boundaries. The 400000-character +handoff guard is unchanged; stars now has a ten-minute timeout. A local jq +`stable_unique_by` definition preserves page order because jq's builtin sorts. diff --git a/supporter-loop/check-decide-static.sh b/supporter-loop/check-decide-static.sh index 1f3f712..55463ee 100755 --- a/supporter-loop/check-decide-static.sh +++ b/supporter-loop/check-decide-static.sh @@ -244,7 +244,8 @@ function inspect(s,n,feeding,replacement, gh,curl,explicit,implicit,get,write,pa if (unresolved || (write && facts["target_unknown"] && (!facts["method_seen"] || !facts["ledger_only"] || facts["non_ledger_method"]))) hit("b",n,"unresolvable gh/curl target in decide") if (s ~ /(^|[^A-Za-z_])mutation([^A-Za-z_]|$)/) hit("b",n,"GraphQL mutation in decide") - if (write && (s ~ /\/(collaborators|invitations|comments|graphql)([^A-Za-z_-]|$)|\/contents\/SUPPORTERS[.]md/ || (gh && s ~ /api[ \t]+graphql([ \t]|$)/))) hit("b",n,"write to delivery endpoint") + if (facts["forbidden_host"]) hit("a",n,"forbidden Telegram host in resolved target") + if (write && (facts["delivery_endpoint"] || s ~ /\/(collaborators|invitations|comments|graphql)([^A-Za-z_-]|$)|\/contents\/SUPPORTERS[.]md/ || (gh && s ~ /api[ \t]+graphql([ \t]|$)/))) hit("b",n,"write to delivery endpoint") if (write && facts["contents_bad"]) hit("a-prime",n,"Contents write outside provable ledger/ prefix") if (write && match(s,/\/contents\//)) { tail=substr(s,RSTART+RLENGTH) @@ -255,6 +256,9 @@ function inspect(s,n,feeding,replacement, gh,curl,explicit,implicit,get,write,pa # Inspect actual endpoint words, never a ledger-looking header or payload. function target(t,facts, path,tail,literal) { facts["target_seen"]=1 + # request() joins adjacent quoted/unquoted shell fragments before this match. + if (t ~ /api[.]telegram[.]org/) facts["forbidden_host"]=1 + if (t ~ /\/(collaborators|invitations|comments|graphql)([^A-Za-z_-]|$)|\/contents\/SUPPORTERS[.]md/ || t=="graphql") facts["delivery_endpoint"]=1 if (t=="" || t ~ /[{}]|[$][@*1-9]/ || (facts["replacement"]!="" && index(t,facts["replacement"]))) facts["target_placeholder"]=1 if (match(t,/\/contents\//)) { tail=substr(t,RSTART+RLENGTH) @@ -278,11 +282,27 @@ function target(t,facts, path,tail,literal) { } facts["ledger_only"]=0 } +# Decode one shell assignment word before inserting it into a quoted reference. +# Shell separators end the value only outside quotes; quoted spaces stay data. +function assignment_value(s, j,ch,q,escaped,w,offset) { + s=trim(s) + for (j=1;j<=length(s);j++) { + ch=substr(s,j,1) + if (escaped) { escaped=0; continue } + if (ch=="\\" && q!="\047") { escaped=1; continue } + if (ch=="\047" || ch=="\"") { + if (q=="") q=ch + else if (q==ch) q="" + } else if (q=="" && ch ~ /[ \t;|&]/) break + } + shell_words(substr(s,1,j-1),w,offset) + return w[1] +} # Assignments are collected before any invocation is inspected. Conflicts and # dynamic assignments permanently invalidate a name, independent of key order. function collect(key,value,yaml, expression,token) { if (yaml && trim(value) ~ /^"/ && index(value,"\\")) escaped_scalars[key]=1 - value=unquote(value) + value=(yaml ? unquote(value) : assignment_value(value)) while (match(value,/\$\{\{[ \t]*(inputs|github|secrets|vars)[.][A-Za-z_0-9.]+[ \t]*\}\}/)) { expression=substr(value,RSTART,RLENGTH) if (!(expression in opaque)) opaque[expression]="OPAQUE_CONTEXT_" ++opaque_count @@ -491,6 +511,32 @@ function nested_yaml(first,owner, j,t,d,root,seen) { return seen } +# v2 stars boundary: optional in legacy decide-only fixtures; when present it is +# a single secret-free GET step. Reuse a/b/d so the frozen selftest rule map holds. +/^ [A-Za-z_][A-Za-z_0-9-]*:/ { + in_stars=($0 ~ /^ stars:/) + if (in_stars) { stars_seen++; stars_line=NR } +} +in_stars { + star=trim($0) + if (star !~ /^#/) { + if (star ~ /secrets[.[]|SUPPORTER_.*TOKEN|TELEGRAM_/) hit("a",NR,"secret reference in stars") + if (star ~ /^- /) stars_steps++ + if (star ~ /^run:/) { + stars_runs++ + if (star !~ /^run: [|]$/) hit("d",NR,"stars requires a literal run block") + } + if (star ~ /(^|[^A-Za-z_])(curl|wget|ssh|nc|python[0-9]*|node|eval|bash|sh)([^A-Za-z_]|$)/ || star ~ /^uses:/) + hit("b",NR,"other executable or network call in stars") + if (star ~ /(^|[^A-Za-z_])gh[ \t]+api/) { + stars_gets++ + if (star !~ /gh api --hostname github.com --method GET --paginate --slurp "repos\/[$]GITHUB_REPOSITORY\/stargazers[?]per_page=100" > "[$]work\/pages"/) + hit("b",NR,"stars must use the single canonical stargazer GET") + } + if (star ~ /--method (PUT|POST|PATCH|DELETE)|-X (PUT|POST|PATCH|DELETE)|star[+]json/) + hit("b",NR,"write or unsupported media type in stars") + } +} # Workflow-level env is outside the decide slice but participates in shell # expansion there. Collect only its mapping entries, including a name key. /^env:[ \t]*(#.*)?$/ { workflow_env=1; next } @@ -519,6 +565,7 @@ inside { lines[++count]=$0; numbers[count]=NR } END { + if (stars_seen && (stars_seen!=1 || stars_steps!=1 || stars_runs!=1 || stars_gets!=1)) hit("b",stars_line,"stars must contain exactly one run step and one GET") if (!found) { hit("structure",1,"decide job missing"); exit 1 } command=""; start=0; quote="" for (i=1;i<=count;i++) { diff --git a/supporter-loop/fixtures/ledger/clean-deferred-quota.ndjson b/supporter-loop/fixtures/ledger/clean-deferred-quota.ndjson new file mode 100644 index 0000000..6c97355 --- /dev/null +++ b/supporter-loop/fixtures/ledger/clean-deferred-quota.ndjson @@ -0,0 +1 @@ +{"schema": 1, "ts": "2026-09-08T00:00:00Z", "run_id": "91-1", "repo": "caty-ai/x-collector", "event": "sweep", "actor": "supporter", "actor_id": 42, "tier": 0, "subject": "", "action": "skip", "mode": "live", "result": "deferred-quota", "dedup_key": "caty-ai/x-collector:sweep:42", "gen": 1} diff --git a/supporter-loop/fixtures/ledger/clean-excluded-unresponsive.ndjson b/supporter-loop/fixtures/ledger/clean-excluded-unresponsive.ndjson new file mode 100644 index 0000000..e70d919 --- /dev/null +++ b/supporter-loop/fixtures/ledger/clean-excluded-unresponsive.ndjson @@ -0,0 +1 @@ +{"schema": 1, "ts": "2026-09-08T00:00:00Z", "run_id": "91-1", "repo": "caty-ai/x-collector", "event": "sweep", "actor": "supporter", "actor_id": 42, "tier": 0, "subject": "", "action": "skip", "mode": "live", "result": "excluded-unresponsive", "dedup_key": "caty-ai/x-collector:sweep:42", "gen": 1} diff --git a/supporter-loop/fixtures/ledger/clean-ok-catchup.ndjson b/supporter-loop/fixtures/ledger/clean-ok-catchup.ndjson new file mode 100644 index 0000000..085c4cc --- /dev/null +++ b/supporter-loop/fixtures/ledger/clean-ok-catchup.ndjson @@ -0,0 +1 @@ +{"schema": 1, "ts": "2026-09-08T00:00:00Z", "run_id": "91-1", "repo": "caty-ai/x-collector", "event": "sweep", "actor": "supporter", "actor_id": 42, "tier": 1, "subject": "", "action": "invite", "mode": "live", "result": "ok-catchup", "dedup_key": "caty-ai/x-collector:1:42", "gen": 1} diff --git a/supporter-loop/fixtures/ledger/clean-sweep-delivery.ndjson b/supporter-loop/fixtures/ledger/clean-sweep-delivery.ndjson new file mode 100644 index 0000000..f89df8f --- /dev/null +++ b/supporter-loop/fixtures/ledger/clean-sweep-delivery.ndjson @@ -0,0 +1 @@ +{"schema": 1, "ts": "2026-09-08T00:00:00Z", "run_id": "91-1", "repo": "caty-ai/x-collector", "event": "sweep", "actor": "supporter", "actor_id": 42, "tier": 1, "subject": "", "action": "supporters-append", "mode": "live", "result": "ok", "dedup_key": "caty-ai/x-collector:1:42", "gen": 1} diff --git a/supporter-loop/fixtures/static/violating-a-stars-pat.yml b/supporter-loop/fixtures/static/violating-a-stars-pat.yml new file mode 100644 index 0000000..996248f --- /dev/null +++ b/supporter-loop/fixtures/static/violating-a-stars-pat.yml @@ -0,0 +1,33 @@ +jobs: + stars: + if: ${{ inputs.sweep == true }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + outputs: + ok: ${{ steps.list.outputs.ok }} + stargazers: ${{ steps.list.outputs.stargazers }} + steps: + - name: List stargazers of the source repo (ids only) + id: list + env: + GH_TOKEN: ${{ secrets.SUPPORTER_LEDGER_TOKEN }} + run: "set -euo pipefail\numask 077\nwork=\"$RUNNER_TEMP/stars\"\nmkdir -p \"\ + $work\"\nok=true\n# \xA711(b): every page or nothing; any shape defect fails\ + \ closed in decide.\nif ! gh api --hostname github.com --method GET --paginate\ + \ --slurp \"repos/$GITHUB_REPOSITORY/stargazers?per_page=100\" > \"$work/pages\"\ + \ \\\n || ! jq -e 'type==\"array\" and length>0 and all(.[]; type==\"array\"\ + ) and all(.[][]; .id|type==\"number\" and .>0 and floor==.)' \"$work/pages\"\ + \ >/dev/null; then\n ok=false; printf '[]\\n' > \"$work/pages\"\nfi\n# jq\ + \ builtin unique_by sorts: the local definition preserves first page occurrence.\n\ + # IDs only, never logins or author text. Fail before GitHub's output-size\ + \ truncation.\nstars=$(jq -c 'def unique_by(f): reduce .[] as $x ({seen:{},out:[]};\ + \ ($x|f|tojson) as $k | if .seen[$k] then . else .seen[$k]=true | .out +=\ + \ [$x] end) | .out; [.[][].id] | unique_by(.)' \"$work/pages\")\n[ \"${#stars}\"\ + \ -lt 400000 ] || { echo '::error::stargazer handoff exceeds safe output size';\ + \ exit 1; }\nprintf 'ok=%s\\n' \"$ok\" >> \"$GITHUB_OUTPUT\"\nprintf 'stargazers=%s\\\ + n' \"$stars\" >> \"$GITHUB_OUTPUT\"\n" + decide: + steps: + - run: echo safe diff --git a/supporter-loop/fixtures/static/violating-b-split-endpoint-concat.yml b/supporter-loop/fixtures/static/violating-b-split-endpoint-concat.yml new file mode 100644 index 0000000..331c25e --- /dev/null +++ b/supporter-loop/fixtures/static/violating-b-split-endpoint-concat.yml @@ -0,0 +1,5 @@ +jobs: + decide: + steps: + - run: | + u="https://api.github.com/repos/o/r/colla""borators/x"; curl -sf -X PUT "$u" -d "{}" diff --git a/supporter-loop/fixtures/static/violating-b-split-single-endpoint-concat.yml b/supporter-loop/fixtures/static/violating-b-split-single-endpoint-concat.yml new file mode 100644 index 0000000..fc9c91b --- /dev/null +++ b/supporter-loop/fixtures/static/violating-b-split-single-endpoint-concat.yml @@ -0,0 +1,5 @@ +jobs: + decide: + steps: + - run: | + u='https://api.github.com/repos/o/r/colla''borators/x'; curl -sf -X PUT "$u" -d "{}" diff --git a/supporter-loop/fixtures/static/violating-b-split-unquoted-endpoint-concat.yml b/supporter-loop/fixtures/static/violating-b-split-unquoted-endpoint-concat.yml new file mode 100644 index 0000000..706bf8c --- /dev/null +++ b/supporter-loop/fixtures/static/violating-b-split-unquoted-endpoint-concat.yml @@ -0,0 +1,5 @@ +jobs: + decide: + steps: + - run: | + u=https://api.github.com/repos/o/r/colla"borators/x"; curl -sf -X PUT "$u" -d "{}" diff --git a/supporter-loop/fixtures/static/violating-b-stars-second-step.yml b/supporter-loop/fixtures/static/violating-b-stars-second-step.yml new file mode 100644 index 0000000..da19954 --- /dev/null +++ b/supporter-loop/fixtures/static/violating-b-stars-second-step.yml @@ -0,0 +1,34 @@ +jobs: + stars: + if: ${{ inputs.sweep == true }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + outputs: + ok: ${{ steps.list.outputs.ok }} + stargazers: ${{ steps.list.outputs.stargazers }} + steps: + - name: List stargazers of the source repo (ids only) + id: list + env: + GH_TOKEN: ${{ github.token }} + run: "set -euo pipefail\numask 077\nwork=\"$RUNNER_TEMP/stars\"\nmkdir -p \"\ + $work\"\nok=true\n# \xA711(b): every page or nothing; any shape defect fails\ + \ closed in decide.\nif ! gh api --hostname github.com --method GET --paginate\ + \ --slurp \"repos/$GITHUB_REPOSITORY/stargazers?per_page=100\" > \"$work/pages\"\ + \ \\\n || ! jq -e 'type==\"array\" and length>0 and all(.[]; type==\"array\"\ + ) and all(.[][]; .id|type==\"number\" and .>0 and floor==.)' \"$work/pages\"\ + \ >/dev/null; then\n ok=false; printf '[]\\n' > \"$work/pages\"\nfi\n# jq\ + \ builtin unique_by sorts: the local definition preserves first page occurrence.\n\ + # IDs only, never logins or author text. Fail before GitHub's output-size\ + \ truncation.\nstars=$(jq -c 'def unique_by(f): reduce .[] as $x ({seen:{},out:[]};\ + \ ($x|f|tojson) as $k | if .seen[$k] then . else .seen[$k]=true | .out +=\ + \ [$x] end) | .out; [.[][].id] | unique_by(.)' \"$work/pages\")\n[ \"${#stars}\"\ + \ -lt 400000 ] || { echo '::error::stargazer handoff exceeds safe output size';\ + \ exit 1; }\nprintf 'ok=%s\\n' \"$ok\" >> \"$GITHUB_OUTPUT\"\nprintf 'stargazers=%s\\\ + n' \"$stars\" >> \"$GITHUB_OUTPUT\"\n" + - run: echo extra + decide: + steps: + - run: echo safe diff --git a/supporter-loop/fixtures/static/violating-b-stars-write.yml b/supporter-loop/fixtures/static/violating-b-stars-write.yml new file mode 100644 index 0000000..04a9658 --- /dev/null +++ b/supporter-loop/fixtures/static/violating-b-stars-write.yml @@ -0,0 +1,33 @@ +jobs: + stars: + if: ${{ inputs.sweep == true }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + outputs: + ok: ${{ steps.list.outputs.ok }} + stargazers: ${{ steps.list.outputs.stargazers }} + steps: + - name: List stargazers of the source repo (ids only) + id: list + env: + GH_TOKEN: ${{ github.token }} + run: "set -euo pipefail\numask 077\nwork=\"$RUNNER_TEMP/stars\"\nmkdir -p \"\ + $work\"\nok=true\n# \xA711(b): every page or nothing; any shape defect fails\ + \ closed in decide.\nif ! gh api --hostname github.com --method PUT --paginate\ + \ --slurp \"repos/$GITHUB_REPOSITORY/stargazers?per_page=100\" > \"$work/pages\"\ + \ \\\n || ! jq -e 'type==\"array\" and length>0 and all(.[]; type==\"array\"\ + ) and all(.[][]; .id|type==\"number\" and .>0 and floor==.)' \"$work/pages\"\ + \ >/dev/null; then\n ok=false; printf '[]\\n' > \"$work/pages\"\nfi\n# jq\ + \ builtin unique_by sorts: the local definition preserves first page occurrence.\n\ + # IDs only, never logins or author text. Fail before GitHub's output-size\ + \ truncation.\nstars=$(jq -c 'def unique_by(f): reduce .[] as $x ({seen:{},out:[]};\ + \ ($x|f|tojson) as $k | if .seen[$k] then . else .seen[$k]=true | .out +=\ + \ [$x] end) | .out; [.[][].id] | unique_by(.)' \"$work/pages\")\n[ \"${#stars}\"\ + \ -lt 400000 ] || { echo '::error::stargazer handoff exceeds safe output size';\ + \ exit 1; }\nprintf 'ok=%s\\n' \"$ok\" >> \"$GITHUB_OUTPUT\"\nprintf 'stargazers=%s\\\ + n' \"$stars\" >> \"$GITHUB_OUTPUT\"\n" + decide: + steps: + - run: echo safe diff --git a/supporter-loop/fixtures/workflow/invitation-limit-422.json b/supporter-loop/fixtures/workflow/invitation-limit-422.json new file mode 100644 index 0000000..114972b --- /dev/null +++ b/supporter-loop/fixtures/workflow/invitation-limit-422.json @@ -0,0 +1,8 @@ +{ + "message": "Invitation limit exceeded: too many invitations in 24 hours.", + "errors": [ + { + "code": "invitation_limit_exceeded" + } + ] +} diff --git a/supporter-loop/fixtures/workflow/test-catchup-model.py b/supporter-loop/fixtures/workflow/test-catchup-model.py new file mode 100644 index 0000000..be7db37 --- /dev/null +++ b/supporter-loop/fixtures/workflow/test-catchup-model.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Contract v2 §7/11: exercise the workflow model and actual page-order expressions.""" +import json +import re +import subprocess +import tempfile +import os +from pathlib import Path +import yaml + +wf = yaml.safe_load(Path('.github/workflows/supporter-loop-reusable.yml').read_text()) +decide = wf['jobs']['decide']['steps'][1]['run'] +act = wf['jobs']['act']['steps'][1]['run'] +model = re.search(r"cat > .*model.jq.*<<'JQ'\n(.*?)\nJQ", decide, re.S).group(1) +assert model == re.search(r"cat > .*model.jq.*<<'JQ'\n(.*?)\nJQ", act, re.S).group(1) +repo = 'caty-ai/x-collector' +now = 1788825600 # fixed clock, independent of host date + +def row(id=42, action='invite', result='ok', ts='2026-09-08T00:00:00Z', tier=1, gen=1, run='1-1', key=None): + return dict(schema=1, repo=repo, actor_id=id, actor='user-'+str(id), action=action, result=result, + ts=ts, tier=tier, gen=gen, run_id=run, event='watch', mode='live', subject='', + dedup_key=key or f'{repo}:{tier}:{id}') + +def evaluate(rows, expression, mode='live'): + r = subprocess.run(['jq', '--arg', 'mode', mode, '--arg', 'repo', repo, model+'\n'+expression], + input=json.dumps(rows), capture_output=True, text=True) + assert r.returncode == 0, r.stderr + return json.loads(r.stdout) + +def ids(rows, stars=(99, 42, 7), tiers='1,2,3', mode='live'): + return evaluate(rows, f'catchup($repo;{json.dumps(stars)};"{tiers}")|map(.actor_id)', mode) + +cases = [ + ('empty burst is recovered in page order', [], [99,42,7]), + ('delivered invite is projection repair', [row()], [99,7,42]), + ('whole bundle already delivered', [row(),row(action='supporters-append')], [99,7]), + ('tier key exclusion', [row(action='skip',result='excluded-family')], [99,7]), + ('tier2 member exclusion', [row(action='skip',result='excluded-member',tier=2)], [99,7]), + ('tier3 member exclusion', [row(action='skip',result='excluded-member',tier=3)], [99,7]), + ('sweep key exclusion', [row(action='skip',result='excluded-bot',tier=0,key=repo+':sweep:42')], [99,7]), + ('stop line does not self exclude', [row(action='skip',result='deferred-quota',tier=0,key=repo+':sweep:42')], [99,42,7]), + ('error invitation retries', [row(result='error-422')], [99,42,7]), + ('re-star after closure recovers empty generation', [row(action='revoke',result='noop',tier=0)], [99,42,7]), + ('new event reopens after closure', [row(action='revoke',tier=0),row(gen=2,result='deferred-quota')], [99,42,7]), + ('old generation exclusion no longer applies', [row(action='skip',result='excluded-family'),row(action='revoke',tier=0),row(gen=2,result='deferred-quota')], [99,42,7]), + ('disabled skip is not exclusion', [row(action='skip',result='tier-disabled')], [99,42,7]), +] +for name, rows, expected in cases: + assert ids(rows) == expected, (name,ids(rows),expected) + print('PASS catchup '+name) +assert ids([],tiers='2,3') == [] +print('PASS tier1 must be enabled') +closed=[row(action='revoke',result='noop',tier=0)] +reopened=evaluate(closed,'catchup($repo;[42];"1")') +assert len(reopened)==1 and reopened[0]['gen']==2 and reopened[0]['invite'] is False and reopened[0]['append'] is False,reopened +assert ids(closed,stars=(99,7))==[99,7] +print('PASS closure noop + re-star + empty gen2 is candidate; absent star is not') +# Two distinct expired cycles, across event and sweep paths, with rotation duplicate safety. +cycles=[row(ts='2026-08-01T00:00:00Z'), row(action='cancel-invite',tier=0,result='expired',ts='2026-08-09T00:00:00Z',run='2-1'), + row(result='ok-catchup',ts='2026-08-10T00:00:00Z',run='3-1'),row(action='cancel-invite',tier=0,result='expired',ts='2026-08-18T00:00:00Z',run='4-1')] +assert evaluate(cycles,'exhausted($repo;42)') is True +assert evaluate(cycles[:2]*2,'exhausted($repo;42)') is False +assert evaluate([cycles[0],dict(cycles[0],run_id='other-put'),cycles[1]],'exhausted($repo;42)') is False, 'one expiry is not two completed cycles' +assert evaluate(cycles,'invite_delivered($repo;42)') is False +assert ids(cycles) == [99,7,42] +assert evaluate(cycles+[row(ts='2026-09-08T00:00:00Z')],'invite_delivered($repo;42)') is True +assert evaluate([row(result='ok-backfill',key=repo+':sweep:42')],'invite_delivered($repo;42)') is True +print('PASS rearm ordering / two-invitation bound / historical sweep-key backfill') +# Quota window fixtures: only actual successful PUT lines count, strict lower boundary. +from datetime import datetime, timezone +now=int(datetime(2026,9,8,tzinfo=timezone.utc).timestamp()) +quota_rows=[row(id=1),row(id=2,result='ok-backfill'),row(id=3,result='ok-catchup'),row(id=4,result='already-1'), + row(id=5,ts='2026-09-07T00:00:00Z'),row(id=6,ts='2026-09-07T00:00:01Z'),row(id=7,result='deferred-quota'),row(id=8,ts='2026-09-09T00:00:00Z')] +pending=[dict(invitee=dict(id=1),created_at='2026-09-08T00:00:00Z'),dict(invitee=dict(id=9),created_at='2026-09-08T00:00:00Z')] +q=evaluate(quota_rows,f'quota($repo;{json.dumps(pending)};{now})') +assert q == dict(B=5,Q=40,age_out=1),q +assert evaluate([row(id=i) for i in range(50)],f'quota($repo;[];{now})')['Q']==0 +assert evaluate(quota_rows*2,f'quota($repo;{json.dumps(pending)};{now})')==q +pending42=[dict(invitee=dict(id=42),created_at='2026-09-08T00:00:00Z')] +old_generation=[row(),row(action='revoke',result='noop',tier=0)] +assert evaluate(old_generation,f'quota($repo;{json.dumps(pending42)};{now})')['B']==2 +assert evaluate([row()],f'quota($repo;{json.dumps(pending42)};{now})')['B']==1 +for result in ('error-422','deferred-quota','already-1'): + assert evaluate([row(result=result)],f'quota($repo;{json.dumps(pending42)};{now})')['B']==1,result +print('PASS pending quota: old generation cannot suppress; current success dedups; error/deferred/already cannot suppress') +print('PASS quota actual PUT results / already-1 excluded / pending dedup / 24h boundary / nonnegative Q / rotation copies') +# Actual stars job and decide re-emit jq source; builtin unique would sort this fixture. +pages=[[dict(id=90),dict(id=2)],[dict(id=90),dict(id=17),dict(id=2),dict(id=1)]] +for name, script, data in [('stars',wf['jobs']['stars']['steps'][0]['run'],pages),('decide',decide,[v for page in pages for v in page])]: + expression=re.search(r"stars=\$\(jq -c '(.*?)'",script).group(1) + r=subprocess.run(['jq','-c',expression],input=json.dumps(data),capture_output=True,text=True,check=True) + assert json.loads(r.stdout)==[90,2,17,1],(name,r.stdout) + assert "def stable_unique_by(f):" in script and "| stable_unique_by(.)" in script + print('PASS ordering across pages with duplicates: '+name) +# Real line helper shares the same namespace rule in both jobs. +for action,event,tier,key in [('invite','sweep',1,'1'),('supporters-append','sweep',1,'1'),('would-invite','sweep',1,'1'), + ('would-supporters-append','sweep',1,'1'),('revoke','sweep',0,'sweep'),('cancel-invite','sweep',0,'sweep'), + ('skip','sweep',0,'sweep'),('comment','issues',2,'2'),('skip','watch',0,'0')]: + assert evaluate([],f'key_namespace("{action}";"{event}";{tier})')==key +print('PASS shared action-key namespace table') +# Run the actual rehearsal loop and live pass-1/pass-2 blocks, including their +# real line/key helper and queue closure. Only API/ledger transport and clock are +# doubled; comparison includes actor, action, result, tier, key, event and generation. +def shell_function(script, name): + return re.search(r'^'+name+r'\(\) \{.*?^\}',script,re.S|re.M).group(0) + +def decisions(rows, mode, stars=(99,42,7)): + script=decide if mode=='record-only' else act + if mode=='record-only': + block=script.split(' rehearsal_count=0',1)[1].split('\n fi\n # §11 steps 5',1)[0] + block='rehearsal_count=0'+block + else: + block=script.split('if [ "$catchup_gate" = true ]; then',1)[1].split('\nelse\n echo \'::error::catch-up',1)[0] + # Execute actual queue closure after a successful projection transport. + queue_body=script.rsplit('while IFS= read -r queued; do',1)[1].split('done < "$work/supporters-queue.ndjson"',1)[0] + block+='\nprojection_result=ok\nwhile IFS= read -r queued; do'+queue_body+'done < "$work/supporters-queue.ndjson"' + with tempfile.TemporaryDirectory(prefix='catchup-parity-') as directory: + root=Path(directory) + (root/'model.jq').write_text(model) + (root/'ledger').write_text(''.join(json.dumps(r)+'\n' for r in rows)) + (root/'catchup.json').write_text(json.dumps(evaluate(rows,f'catchup($repo;{json.dumps(stars)};"1")',mode))) + (root/'supporters-queue.ndjson').touch() + harness=r''' +set -euo pipefail +ledger="$work/ledger" +failed=0; reservedCount=0 +append() { printf '%s\n' "$1" >> "$ledger"; printf '%s\n' "$1" >> "$work/emitted"; } +emit() { append "$(line "$@")"; } +identity() { actor="user-$actor_id"; user_type=User; } +work_budget() { return 0; } +quota_room() { return 0; } +invite() { result=ok; } +date() { if [ "$*" = '-u +%s' ]; then echo 1788825600; else echo 2026-09-08T00:00:00Z; fi; } +''' + harness+='\n'+shell_function(script,'line')+'\n'+shell_function(script,'eligibility')+'\n' + if mode=='live': harness+=shell_function(act,'queue_supporters')+'\n' + result=subprocess.run(['/bin/bash','-c',harness+block],env=dict(os.environ,work=directory,MODE=mode,GITHUB_REPOSITORY=repo,RUN_KEY='parity'),text=True,capture_output=True) + assert result.returncode==0,(mode,result.stdout,result.stderr,block) + emitted=root/'emitted' + actual=[json.loads(line) for line in emitted.read_text().splitlines()] if emitted.exists() else [] + normalized=[] + for line in actual: + line['action']=line['action'].removeprefix('would-') + if line['result']=='ok-catchup': line['result']='ok' + normalized.append({k:v for k,v in line.items() if k not in ('mode','ts','run_id')}) + return sorted(normalized,key=lambda r:(r['actor_id'],r['action'])) + +# Restricted parity fixture (i): no delivery history in either vocabulary, +# no pending invitations, identical identities and ample quota/budget. +for history in ([],[row(result='deferred-quota')],[row(result='error-422')],closed): + rehearsal=decisions(history,'record-only') + live=decisions(history,'live') + assert rehearsal==live,(rehearsal,live) + assert len(live)==6,live +print('PASS normalized actual rehearsal/live decisions: empty, deferred, failed and re-starred generation') +# Also drive pass 1 on a stranded append, outside fixture (i)'s restricted set. +assert decisions([row()],'record-only')==decisions([row()],'live') +print('PASS actual pass-1 repair / pass-2 invitation decision equality') +# Migration fixture (ii): successful rehearsal vocabulary suppresses itself only. +rehearsed=[dict(row(action='would-invite'),mode='record-only'),dict(row(action='would-supporters-append'),mode='record-only')] +assert decisions(rehearsed,'record-only',stars=(42,))==[] +migrated=decisions(rehearsed,'live',stars=(42,)) +assert [r['action'] for r in migrated]==['invite','supporters-append'],migrated +print('PASS would-* history suppresses repeated rehearsal but not actual live catch-up') +# Projection omission and restoration after rearm, with higher-tier recognition preserved. +render=re.search(r"jq -sr -L \"\$work\" [^\n]*'(.*?)' \"\$derive\"",act,re.S).group(1).replace('include "model";','') +for name,rows,present in [('expired tier1 omitted',cycles,False),('excluded unresponsive omitted',cycles+[row(action='skip',result='excluded-unresponsive',tier=0)],False), + ('reinvite restores without append',cycles+[row()],True),('tier2 recognition stays',cycles+[row(action='comment',tier=2)],True)]: + r=subprocess.run(['jq','-r','--arg','mode','live','--arg','repo',repo,model+'\n'+render],input=json.dumps(rows),text=True,capture_output=True,check=True) + assert ('@user-42' in r.stdout)==present,(name,r.stdout) + print('PASS projection '+name) diff --git a/supporter-loop/fixtures/workflow/test-decide.py b/supporter-loop/fixtures/workflow/test-decide.py index 7ace785..5cd7c4e 100644 --- a/supporter-loop/fixtures/workflow/test-decide.py +++ b/supporter-loop/fixtures/workflow/test-decide.py @@ -11,11 +11,15 @@ import yaml # Invoked through temporary curl/gh symlinks. Unknown requests are fatal. -if Path(sys.argv[0]).name in ('curl', 'gh', 'date'): +if Path(sys.argv[0]).name in ('curl', 'gh', 'date', 'timeout', 'sleep'): args = sys.argv[1:] root = Path(os.environ['MOCK_STATE']) name = Path(sys.argv[0]).name config = json.loads((root / 'mock-config.json').read_text()) + if name == 'timeout': + os.execvp(args[1], args[1:]) + if name == 'sleep': + raise SystemExit(0) if name == 'date': if '-d' in args: instant = datetime.datetime.fromisoformat(args[args.index('-d') + 1].replace('Z', '+00:00')) @@ -65,7 +69,15 @@ data = [dict(name=p.name, type='file') for p in root.iterdir() if p.suffix == '.ndjson' or p.name.startswith('baseline-')] elif '/actions/workflows/supporter-loop.yml/runs?' in endpoint: assert os.environ['GH_TOKEN'] == 'fixture-source', 'precondition (a) runs listing stays on GITHUB_TOKEN' - runs = [] if 'status=failure' in endpoint else [dict(path='.github/workflows/supporter-loop.yml', created_at='2026-01-01T00:00:00Z')] + runs = [] if 'status=failure' in endpoint or 'status=cancelled' in endpoint else [dict(path='.github/workflows/supporter-loop.yml', event='schedule', created_at='2026-01-01T00:00:00Z')] + if config.get('runs_case'): + if 'event=workflow_dispatch&status=success' in endpoint: + runs = [dict(path='.github/workflows/supporter-loop.yml', event='workflow_dispatch', created_at='2026-09-07T00:00:00Z')] + if 'status=failure' in endpoint or 'status=cancelled' in endpoint: + assert 'created=%3E%3D2026-09-07T00:00:00Z' in endpoint, endpoint + if 'status=cancelled' in endpoint: + runs = [dict(path='.github/workflows/supporter-loop.yml',event=e,created_at='2026-09-08T00:00:00Z') + for e in ('watch','issues','discussion','pull_request_target','schedule','workflow_dispatch')] data = dict(total_count=len(runs), workflow_runs=runs) elif '/stargazers?' in endpoint: raise SystemExit('UNEXPECTED: decide must not list stargazers itself (v1.11 stars job)') @@ -92,7 +104,7 @@ output.write_text(json.dumps(dict(content=base64.b64encode(header.read_bytes()).decode()))) print('200', end=''); raise SystemExit(0) if config['live_act'] and url.endswith('/contents/SUPPORTERS.md'): - code, response = '404', dict(message='Not Found') + code, response = ('200', dict(sha='fixture-sha', size=(root / 'SUPPORTERS.md').stat().st_size)) if (root / 'SUPPORTERS.md').exists() else ('404', dict(message='Not Found')) if method == 'PUT': body = json.loads(Path(args[args.index('--data-binary') + 1].lstrip('@')).read_text()) (root / 'SUPPORTERS.md').write_bytes(base64.b64decode(body['content'])) @@ -103,8 +115,13 @@ if method == 'DELETE': (root / 'revoked').touch() output.write_text('{}'); print('204', end=''); raise SystemExit(0) - if config['live_act'] and url == 'https://api.github.com/user/42': - output.write_text(json.dumps(dict(id=42, login='external-supporter', type='User'))) + if method == 'GET' and url.startswith('https://api.github.com/user/'): + assert 'Authorization: Bearer fixture-source' in args + actor_id = int(url.rsplit('/', 1)[1]) + if actor_id in config.get('identity_missing', []): + output.write_text(json.dumps(dict(message='Not Found'))) + print('404', end=''); raise SystemExit(0) + output.write_text(json.dumps(dict(id=actor_id, login='external-supporter' if actor_id == 42 else 'user-'+str(actor_id), type='User'))) print('200', end=''); raise SystemExit(0) if method != 'GET': raise SystemExit('UNEXPECTED nonledger mutation: ' + method + ' ' + url) @@ -129,11 +146,11 @@ code = '200' if path.exists() else '404' response = dict(sha='fixture-sha', content=base64.b64encode(path.read_bytes()).decode()) if path.exists() else dict(message='Not Found') elif method == 'PUT': - assert relative.endswith('.ndjson'), 'Only ledger writes allowed' + assert relative.endswith('.ndjson') or (config['live_act'] and relative.endswith('.sweep-state.json')), 'Only ledger writes allowed in decide' payload = json.loads(Path(args[args.index('--data-binary') + 1].lstrip('@')).read_text()) response = dict(content=dict(sha='fixture-sha')) code = '200' if path.exists() else '201' - assert payload['message'].startswith('supporter-loop: ') and len(payload['message'].split()) == 4, payload['message'] + assert payload['message'].startswith('supporter-loop: ') and (len(payload['message'].split()) == 4 or (config['live_act'] and relative.endswith('.sweep-state.json'))), payload['message'] # .publication-denylist option a; contract §6 freezes the runtime identity. assert payload['committer'] == dict(name='supporter-loop[bot]', email='supporter-loop@' + 'caty-ai' + '.noreply') if os.environ.get('MOCK_CAS_CONFLICT') == '1' and not (root / 'conflict-used').exists(): @@ -154,20 +171,20 @@ REPO = 'caty-ai/x-collector' KEYS = ['schema', 'ts', 'run_id', 'repo', 'event', 'actor', 'actor_id', 'tier', 'subject', 'action', 'mode', 'result', 'dedup_key', 'gen'] -def execute(event, payload, mode='record-only', tiers='1,2,3', prior=None, cas_conflict=False, sweep=False, stargazers=None, stars_ok='true', expected_error=False, discussions=False, live_act=False, missing_ledger=False): - with tempfile.TemporaryDirectory(prefix='supporter-decide-', dir=Path(__file__).resolve().parent) as directory: +def execute(event, payload, mode='record-only', tiers='1,2,3', prior=None, cas_conflict=False, sweep=False, stargazers=None, stars_ok='true', expected_error=False, discussions=False, live_act=False, missing_ledger=False, runs_case=False, identity_missing=None): + with tempfile.TemporaryDirectory(prefix='supporter-decide-') as directory: root = Path(directory) mock_bin = root / 'bin' mock_bin.mkdir() state = root / 'state' state.mkdir() - (state / 'mock-config.json').write_text(json.dumps(dict(stargazers=stargazers or [], discussions=discussions, live_act=live_act, missing_ledger=missing_ledger, collaborators=[dict(id=42)] if live_act else []))) + (state / 'mock-config.json').write_text(json.dumps(dict(stargazers=stargazers or [], discussions=discussions, live_act=live_act, missing_ledger=missing_ledger, runs_case=runs_case, identity_missing=identity_missing or [], collaborators=[dict(id=42)] if live_act else []))) if sweep: (state / 'baseline-2026-01-01.json').write_text(json.dumps(dict(collaborators=[], invitations=[]))) launcher = mock_bin / 'double' launcher.write_text('#!' + sys.executable + '\n' + Path(__file__).read_text()) launcher.chmod(0o700) - for command in ('curl', 'gh', 'date'): + for command in ('curl', 'gh', 'date', 'timeout', 'sleep'): (mock_bin / command).symlink_to(launcher) ledger = state / 'caty-ai--x-collector.ndjson' if prior: @@ -193,6 +210,8 @@ def execute(event, payload, mode='record-only', tiers='1,2,3', prior=None, cas_c if bool(result.returncode) != expected_error: raise AssertionError(f'{event}: exit {result.returncode}\n{result.stdout}\n{result.stderr}') assert not result.stderr, result.stderr + for missing_id in identity_missing or []: + assert f'::warning::rehearsal identity unavailable for actor_id={missing_id}' in result.stdout,result.stdout if expected_error: assert '::error::' in result.stdout, result.stdout if missing_ledger: @@ -210,6 +229,8 @@ def execute(event, payload, mode='record-only', tiers='1,2,3', prior=None, cas_c assert 'fixture-' not in json.dumps(row) if discussions: assert (state / 'graphql-calls').read_text().splitlines() == ['listing:POST', 'comments:POST', 'replies:POST'] + if not live_act: + assert not list(state.glob('*.sweep-state.json')) if live_act: assert (state / 'revoked').exists() == any(r['action'] == 'revoke' for r in fresh) if (state / 'revoked').exists(): @@ -298,3 +319,25 @@ def payload(event, actor=None, assoc='NONE', merged=True): execute('watch', payload('watch'), missing_ledger=True, expected_error=True) print('PASS actual decide: missing ledger directory has actionable contract diagnostic') + +# Execute the new rehearsal path, including its read-shaped identity call. +fresh,_,output=execute('workflow_dispatch',dict(repository=repo),sweep=True,stargazers=[dict(id=42)],runs_case=True) +assert [(r['action'],r['tier'],r['dedup_key'],r['event']) for r in fresh]==[ + ('would-invite',1,REPO+':1:42','sweep'),('would-supporters-append',1,REPO+':1:42','sweep')],fresh +assert 'cancelled_event=4' in output and 'cancelled_schedule=2' in output and 'sweep_gate=true' in output +print('PASS actual rehearsal identity/tier-key bundle; dispatch watermark; cancelled counts never gate') +prior=[dict(live_invite,actor_id=i,actor='user-'+str(i),dedup_key=REPO+':1:'+str(i)) for i in range(100,144)] +fresh,_,_=execute('schedule',dict(repository=repo),sweep=True,stargazers=[dict(id=42),dict(id=99)],prior=prior) +assert [r['actor_id'] for r in fresh]==[42,42],fresh +print('PASS actual record-only quota reserves one of 45 remaining slots, same as live') +fresh,_,_=execute('schedule',dict(repository=repo),sweep=True,stargazers=[dict(id=42)],tiers='2,3') +assert fresh==[],fresh +print('PASS actual record-only tier1-disabled produces no catch-up or state') + +# A deleted identity must neither fabricate a delivery nor consume the next slot. +fresh,_,_=execute('schedule',dict(repository=repo),sweep=True, + stargazers=[dict(id=99),dict(id=42)],identity_missing=[99],prior=prior) +assert [(r['actor_id'],r['action']) for r in fresh]==[ + (42,'would-invite'),(42,'would-supporters-append')],fresh +assert all(r['actor'] for r in fresh),fresh +print('PASS actual rehearsal identity 404 ledgers nothing; next actor retains quota; no empty actor') diff --git a/supporter-loop/fixtures/workflow/test-graphql.py b/supporter-loop/fixtures/workflow/test-graphql.py index b41fb1a..5a17fff 100644 --- a/supporter-loop/fixtures/workflow/test-graphql.py +++ b/supporter-loop/fixtures/workflow/test-graphql.py @@ -7,7 +7,7 @@ import sys import tempfile -with tempfile.TemporaryDirectory(prefix='supporter-graphql-', dir=Path(__file__).resolve().parent) as directory: +with tempfile.TemporaryDirectory(prefix='supporter-graphql-') as directory: root = Path(directory) binary = root / 'bin' binary.mkdir() diff --git a/supporter-loop/fixtures/workflow/test-interface.py b/supporter-loop/fixtures/workflow/test-interface.py index c85a18e..cbd72be 100644 --- a/supporter-loop/fixtures/workflow/test-interface.py +++ b/supporter-loop/fixtures/workflow/test-interface.py @@ -68,6 +68,8 @@ # the 1,000-result cap counts supporter-loop runs only (repo-wide was 1,691). stars = workflow['jobs']['stars'] assert stars['if'] == '${{ inputs.sweep == true }}', 'stars runs only in sweeps' +assert stars['timeout-minutes'] == 10 +assert act['timeout-minutes'] == 10 assert stars['permissions'] == {'contents': 'write'} assert len(stars['steps']) == 1 and stars['steps'][0]['id'] == 'list' assert set(stars['steps'][0]['env']) == {'GH_TOKEN'} and 'secrets.' not in yaml.safe_dump(stars), 'stars never sees a secret' diff --git a/supporter-loop/fixtures/workflow/test-live-contract.py b/supporter-loop/fixtures/workflow/test-live-contract.py new file mode 100644 index 0000000..31bc31b --- /dev/null +++ b/supporter-loop/fixtures/workflow/test-live-contract.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""Run actual act helpers and event/catch-up blocks with local, fail-closed API doubles.""" +import base64 +import json +import os +import re +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path +import yaml + +source = Path('.github/workflows/supporter-loop-reusable.yml').read_text() +wf = yaml.safe_load(source) +act = wf['jobs']['act']['steps'][1]['run'] +prefix, main = act.split('read_ledger\nif [ "$SWEEP" != true ]; then', 1) +event_block = 'if [ "$SWEEP" != true ]; then' + main.split('sweep_cleanup() {', 1)[0] +catchup_block = '# §11 step 4b:' + act.split('# §11 step 4b:', 1)[1] +repo = 'caty-ai/x-collector' +now = int(datetime(2026,9,8,tzinfo=timezone.utc).timestamp()) +fixture_dir = Path(__file__).resolve().parent + +def row(id=42, action='invite', result='ok', ts='2026-09-08T00:00:00Z', tier=1, run='1-1', key=None): + return dict(schema=1, ts=ts, run_id=run, repo=repo, event='watch', actor='user-'+str(id), actor_id=id, + tier=tier, subject='', action=action, mode='live', result=result, dedup_key=key or f'{repo}:{tier}:{id}', gen=1) + +clock = r''' +date() { + case "$*" in + '-u +%s') cat "$SWEEP_CLOCK_FILE" ;; + '-u +%Y-%m-%dT%H:%M:%SZ') echo 2026-09-08T00:00:00Z ;; + '-u -d '*) python3 -B -c 'import datetime,sys; print(int(datetime.datetime.fromisoformat(sys.argv[1].replace("Z","+00:00")).timestamp()))' "$3" ;; + *) command date "$@" ;; + esac +} +sleep() { + printf 'WAIT %s\n' "$1" >> "$RUNNER_TEMP/trace" + echo "$(( $(cat "$SWEEP_CLOCK_FILE") + $1 ))" > "$SWEEP_CLOCK_FILE" +} +''' +doubles = r''' +read_ledger() { + echo READ_LEDGER >> "$RUNNER_TEMP/trace" + cp "$RUNNER_TEMP/api-ledger" "$ledger" +} +get() { + local endpoint="$2" target="$3" id + echo "GET $endpoint" >> "$RUNNER_TEMP/trace" + code=200 + case "$endpoint" in + user/*) + id="${endpoint#user/}" + jq --arg id "$id" '.[$id] // {id:($id|tonumber),login:("user-"+$id),type:"User"}' "$RUNNER_TEMP/identities.json" > "$target" ;; + "repos/$REWARD_REPO/collaborators/"*) code=404; echo '{}' > "$target" ;; + "repos/$REWARD_REPO/contents/SUPPORTERS.header.md") + jq -n --rawfile header "$RUNNER_TEMP/header" '{content:($header|@base64)}' > "$target" ;; + "repos/$REWARD_REPO/contents/SUPPORTERS.md") + if [ -f "$RUNNER_TEMP/published" ]; then + jq -n --rawfile body "$RUNNER_TEMP/published" '{sha:"supporter-sha",size:($body|utf8bytelength),content:($body|@base64)}' > "$target" + else code=404; echo '{}' > "$target"; fi ;; + "repos/$REWARD_REPO/contents/ledger/manual-allowlist.json"|"repos/$REWARD_REPO/contents/ledger/manual-markers-allowlist.json") + code=404; echo '{}' > "$target" ;; + "repos/$REWARD_REPO/contents/ledger/$base.sweep-state.json") + if [ -f "$RUNNER_TEMP/previous-state" ]; then + jq -n --rawfile state "$RUNNER_TEMP/previous-state" '{sha:"state-sha",content:($state|@base64)}' > "$target" + else code=404; echo '{}' > "$target"; fi ;; + *) echo "UNEXPECTED GET $endpoint" >&2; return 1 ;; + esac +} +pages() { + [ "$1" = "$ADMIN_TOKEN" ] && [ "$2" = "repos/$REWARD_REPO/invitations?per_page=100" ] || return 1 + cp "$RUNNER_TEMP/pending.json" "$3" +} +append() { + pace_write + echo "WRITE $(cat "$SWEEP_CLOCK_FILE") APPEND $(jq -r '.action+" "+.result+" "+(.actor_id|tostring)' <<< "$1")" >> "$RUNNER_TEMP/trace" + printf '%s\n' "$1" >> "$RUNNER_TEMP/api-ledger" + printf '%s\n' "$1" >> "$ledger" +} +mutate() { + pace_write + echo "WRITE $(cat "$SWEEP_CLOCK_FILE") $2 $3" >> "$RUNNER_TEMP/trace" + code=200 + case "$3" in + "repos/$REWARD_REPO/collaborators/"*) + [ "$1" = "$ADMIN_TOKEN" ] && [ "$2" = PUT ] || return 1 + jq -e '.permission=="pull"' "$4" >/dev/null || return 1 + code="$INVITE_STATUS" + cp "$RUNNER_TEMP/invite-response" "$work/mutation.json" ;; + "repos/$REWARD_REPO/contents/SUPPORTERS.md") + [ "$1" = "$LEDGER_TOKEN" ] && [ "$2" = PUT ] || return 1 + if [ "$RENDER_CONFLICT" = true ] && [ ! -f "$RUNNER_TEMP/conflict-used" ]; then + touch "$RUNNER_TEMP/conflict-used" + cat "$RUNNER_TEMP/concurrent-row" >> "$RUNNER_TEMP/api-ledger" + echo '{"message":"sha does not match"}' > "$work/mutation.json" + code="$CONFLICT_STATUS" + elif [ "$RENDER_FAIL" = true ]; then code="$RENDER_STATUS" + else jq -jr '.content|@base64d' "$4" > "$RUNNER_TEMP/published"; fi ;; + "repos/$REWARD_REPO/contents/ledger/$base.sweep-state.json") + [ "$1" = "$LEDGER_TOKEN" ] && [ "$2" = PUT ] || return 1 + jq -jr '.content|@base64d' "$4" > "$RUNNER_TEMP/new-state" ;; + *) echo "UNEXPECTED MUTATION $3" >&2; return 1 ;; + esac +} +rotate() { echo ROTATE >> "$RUNNER_TEMP/trace"; } +reminder() { :; } +comment() { echo COMMENT_SENT >> "$RUNNER_TEMP/trace"; result=ok; } +''' + +def execute(rows=(), stars=(42,), identities=None, pending=(), sweep=True, invite_status=201, body=None, + conflict=False, render_fail=False, gate=True, setup='', previous=None, block=None, conflict_status=409, tiers='1,2,3', render_status=500): + with tempfile.TemporaryDirectory(prefix='v2-live-') as directory: + root=Path(directory) + (root/'api-ledger').write_text(''.join(json.dumps(r)+'\n' for r in rows)) + (root/'clock').write_text(str(now)) + (root/'trace').touch() + (root/'identities.json').write_text(json.dumps(identities or {})) + (root/'pending.json').write_text(json.dumps(pending)) + (root/'header').write_bytes((fixture_dir/'SUPPORTERS.header.sample.md').read_bytes()) + (root/'invite-response').write_text(json.dumps(body or {})) + (root/'concurrent-row').write_text(json.dumps(row(id=200,action='comment',tier=2))+'\n') + if previous is not None: (root/'previous-state').write_text(previous if isinstance(previous,str) else json.dumps(previous)) + actions=[dict(actor='user-42',actor_id=42,gen=1,tier=t,subject='',event='issues',action=a) + for a,t in [('invite',1),('comment',2),('supporters-append',2)]] + env=dict(os.environ, RUNNER_TEMP=str(root), SWEEP_CLOCK_FILE=str(root/'clock'), MODE='live', + SWEEP=str(sweep).lower(), GITHUB_REPOSITORY=repo, REWARD_REPO='caty-ai/ask-ai-widget', + GH_TOKEN='fixture-source',LEDGER_TOKEN='fixture-ledger',ADMIN_TOKEN='fixture-admin',RUN_KEY='91-1', + STARGAZERS=json.dumps(stars),TIERS_ENABLED=tiers,SWEEP_GATE=str(gate).lower(), + RED_RUN=str(not gate).lower(),CANCELLED_EVENT='3',CANCELLED_SCHEDULE='2', + INVITE_STATUS=str(invite_status),CONFLICT_STATUS=str(conflict_status),RENDER_CONFLICT=str(conflict).lower(),RENDER_FAIL=str(render_fail).lower(),RENDER_STATUS=str(render_status), + LEDGER_EXPIRES='',ADMIN_EXPIRES='',ACTIONS=json.dumps(actions)) + script=clock+prefix+doubles+'\nread_ledger\ncp "$RUNNER_TEMP/pending.json" "$work/pending.json"\n'+setup+'\n'+(block if block is not None else catchup_block if sweep else event_block) + result=subprocess.run(['/bin/bash','-c',script],env=env,capture_output=True,text=True) + assert not result.stderr,(result.stdout,result.stderr) + files={p.name:p.read_text() for p in root.iterdir() if p.is_file()} + all_rows=[json.loads(v) for v in files['api-ledger'].splitlines()] + state=json.loads(files['new-state']) if 'new-state' in files else None + messages=(root/'supporter-loop/sweep-messages').read_text() if (root/'supporter-loop/sweep-messages').exists() else '' + return result,all_rows[len(rows):],state,files,messages + +limit=json.loads((fixture_dir/'invitation-limit-422.json').read_text()) +for body,expected,red in [(limit,'deferred-quota',False),({'message':'sha does not match'},'error-422',True),({'message':'Validation Failed'},'error-422',True)]: + r,rows,state,files,_=execute(sweep=False,invite_status=422,body=body) + assert [v['result'] for v in rows]==[expected]*3, rows + assert bool(r.returncode)==red,(r.stdout,r.returncode) + assert 'COMMENT_SENT' not in files['trace'] and 'published' not in files + assert all('invitations_backlog' not in v for v in rows) +print('PASS invitation-limit 422 / CAS and generic negatives / whole event bundle declines siblings / green deferral') + +r,rows,state,files,_=execute() +assert r.returncode==0,r.stdout +assert [(v['action'],v['result'],v['tier'],v['dedup_key']) for v in rows]==[ + ('invite','ok-catchup',1,repo+':1:42'),('supporters-append','ok',1,repo+':1:42')],rows +assert set(state)=={'last_sweep_ts','backlog','cancelled_runs','capacity_pct','alarm_state'} +assert state['backlog']==0 and state['cancelled_runs']==dict(event=3,schedule=2) +assert state['capacity_pct']['invitations']==1/45 +assert files['trace'].index('PUT repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md') < files['trace'].index('APPEND supporters-append') +assert '@user-42' in files['published'] +write_times=[int(line.split()[1]) for line in files['trace'].splitlines() if line.startswith('WRITE ')] +assert all(b-a>=1 for a,b in zip(write_times,write_times[1:])),write_times +print('PASS actual catchup ok-catchup tier key / PUT precedes append queue / state derivation / <=1 write per second') + +r,rows,state,files,_=execute(rows=[row()],pending=[dict(invitee=dict(id=42),created_at='2026-09-08T00:00:00Z')]) +assert r.returncode==0 and [v['action'] for v in rows]==['supporters-append'] +assert 'GET user/' not in files['trace'] and 'PUT repos/caty-ai/ask-ai-widget/collaborators/' not in files['trace'] +print('PASS crash-between-actions stranded append repaired in pass1 without identity or invitation') + +for login,kind,reason in [('CaTy2','User','family'),('helper','Bot','bot'),('helper[bot]','User','bot'), + ('caty-ai','User','self'),('github-actions[bot]','Bot','self'),('some-org','Organization','org')]: + r,rows,state,files,_=execute(stars=[42,99],identities={'42':dict(id=42,login=login,type=kind)}) + assert r.returncode==0,r.stdout + assert rows[0]['action']=='skip' and rows[0]['result']=='excluded-'+reason and rows[0]['dedup_key']==repo+':sweep:42',rows + assert rows[1]['actor_id']==99 and rows[1]['result']=='ok-catchup' + assert state['backlog']==0 + print('PASS actual catchup exclusion / slot refill: '+login) +# MEMBER is only observable from tier2/3 events; current-generation ledger exclusion suppresses identity. +r,rows,state,files,_=execute(rows=[row(action='skip',tier=2,result='excluded-member')]) +assert r.returncode==0 and rows==[] and 'GET user/' not in files['trace'] +print('PASS catchup excluded-member from event ledger suppresses identity') + +cycles=[row(ts='2026-08-01T00:00:00Z'),row(action='cancel-invite',result='expired',tier=0,ts='2026-08-09T00:00:00Z',run='2-1'), + row(ts='2026-08-10T00:00:00Z',run='3-1'),row(action='cancel-invite',result='expired',tier=0,ts='2026-08-18T00:00:00Z',run='4-1')] +r,rows,state,files,_=execute(rows=cycles) +assert r.returncode==0 and len(rows)==1 and rows[0]['result']=='excluded-unresponsive',rows +assert 'GET user/' not in files['trace'] and state['backlog']==0 +print('PASS actual two-invitation bound produces excluded-unresponsive once') + +r,rows,state,files,_=execute(stars=[42,99,7],invite_status=422,body=limit) +assert r.returncode==0 and len(rows)==1 and rows[0]['action']=='skip' and rows[0]['result']=='deferred-quota' +assert state['backlog']==3 and state['alarm_state']['deferred_quota'] +assert files['trace'].count('GET user/')==1 +print('PASS sweep invitation-limit stop emits exactly one skip, remains green, reports every unserved candidate') + +r,rows,state,files,_=execute(conflict=True) +assert r.returncode==0,r.stdout +assert '@user-200' in files['published'] +operations=[v for v in files['trace'].splitlines() if v=='READ_LEDGER' or v=='GET repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md' or 'PUT repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md' in v] +first_put=next(i for i,v in enumerate(operations) if 'PUT ' in v) +assert operations[first_put-2:first_put]==['GET repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md','READ_LEDGER'],operations +assert operations[first_put+1:first_put+3]==['GET repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md','READ_LEDGER'],operations +print('PASS two-lane regeneration conflict restarts GET -> read ledger -> render -> PUT; concurrent actor preserved') + +r,rows,state,files,_=execute(render_fail=True, stars=[42,99,7]) +queued=[v for v in rows if v['action']=='supporters-append'] +assert r.returncode!=0 and len(queued)==3 and all(v['result']=='error-500' for v in queued) and state['backlog']==3 +print('PASS failed projection closes queued lines with error and keeps backlog red') + +r,rows,state,files,_=execute(sweep=False, render_fail=True) +assert r.returncode!=0 and rows[-1]['action']=='supporters-append' and rows[-1]['result']=='error-500' +print('PASS event projection failure records exact PUT status') + +r,rows,state,files,_=execute(render_fail=True, render_status='transport') +assert r.returncode!=0 and rows[-1]['result']=='error-regenerate' +print('PASS projection transport failure without HTTP status uses error-regenerate') + + +r,rows,state,files,_=execute(setup='code=404; : > "$RUNNER_TEMP/header"') +assert r.returncode!=0 and rows[-1]['result']=='error-regenerate' and state['backlog']==1 +print('PASS malformed header without HTTP failure uses error-regenerate, never stale status') + +r,rows,state,files,_=execute(stars=['invalid']) +assert r.returncode!=0 and '::error::stargazer handoff invalid in act' in r.stdout +assert rows==[] and 'published' in files and state is not None +print('PASS defensive invalid stars handoff is loud and red; reserve still runs') + + +r,rows,state,files,_=execute(gate=False) +assert r.returncode!=0 and rows==[] and 'published' in files and state['backlog']==1 and state['alarm_state']['red_run'] +print('PASS red-run sweep skips catchup but unconditionally regenerates and writes state') + +for setup in ['writeCount=148; reservedCount=2', 'echo "$((started+361))" > "$SWEEP_CLOCK_FILE"']: + r,rows,state,files,_=execute(setup=setup) + assert r.returncode==0 and rows==[] and state['backlog']==1 and 'ROTATE' in files['trace'] and 'published' in files +print('PASS whole-sweep write/time budget stops green and still executes reserve') +# Pass1 reservations cap 148 queued lines with two final writes reserved; no identities. +r,rows,state,files,_=execute(rows=[row(id=i) for i in range(1,5)],stars=[1,2,3,4],setup='writeCount=146') +assert r.returncode==0 and len(rows)==2 and state['backlog']==2 +print('PASS pass1 queued repairs reserve writes at enqueue time') + +# End-of-sweep latch only fires edges, including clear; corrupt latch self heals. +_,_,state,_,_=execute() +r,rows,state2,files,messages=execute(previous=state) +assert len(messages.splitlines())==1 and state2==state +r,rows,state2,files,messages=execute(previous={'alarm_state':{'stars_70':True,'red_run':True,'deferred_quota':True,'invitations_backlog':True}}) +assert messages.count('cleared')==4 +r,rows,state2,files,messages=execute(previous='not JSON') +assert r.returncode==0 and len(messages.splitlines())==1 +print('PASS sweep-state latch repeat / raised-to-cleared transitions / unparsable recovery') + +# Dynamic quota wait helper with the actual rolling model, injectable clock and sleep. +quota_rows=[row(id=i,ts='2026-09-07T00:02:00Z') for i in range(45)] +block='quota_room\necho "WAIT_TOTAL=$waitCount"\n' +r,_,_,files,_=execute(rows=quota_rows,block=block) +assert r.returncode==0 and 'WAIT_TOTAL=120' in r.stdout +quota_rows=[row(id=i,ts='2026-09-07T00:04:00Z') for i in range(45)] +r,_,_,files,_=execute(rows=quota_rows,block=block) +assert r.returncode!=0 and 'WAIT ' not in files['trace'] +quota_rows=[row(id=i,ts='2026-09-07T00:02:00Z') for i in range(45)] +r,_,_,files,_=execute(rows=quota_rows,setup='waitCount=100',block=block) +assert r.returncode!=0 and 'WAIT ' not in files['trace'] +print('PASS quota ages out dynamically / 5min lookahead / cumulative wait <=3min') +# Reserve quota before identities, re-evaluate per PUT, lazy selection at capacity. +r,rows,state,files,_=execute(rows=[row(id=i) for i in range(1,45)],stars=[90,91,92]) +assert r.returncode==0 and state['backlog']==2 and files['trace'].count('GET user/')==1 +assert [v['actor_id'] for v in rows]==[90,90] +print('PASS per-PUT quota reevaluation and identity calls bounded by available capacity') + +# A sha-mismatch 422 follows the same restart path as 409, never quota deferral. +r,rows,state,files,_=execute(conflict=True,conflict_status=422) +assert r.returncode==0 and '@user-200' in files['published'],(r.stdout,rows) +assert all(v['result']!='deferred-quota' for v in rows) +operations=[v for v in files['trace'].splitlines() if v=='READ_LEDGER' or v=='GET repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md' or 'PUT repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md' in v] +first_put=next(i for i,v in enumerate(operations) if 'PUT ' in v) +assert operations[first_put+1:first_put+3]==['GET repos/caty-ai/ask-ai-widget/contents/SUPPORTERS.md','READ_LEDGER'],operations +print('PASS 422 catch-up CAS restarts GET and fresh ledger read without quota deferral') + +# One expired cycle keeps the existing append delivered: only re-invite is ledgered. +r,rows,state,files,_=execute(rows=cycles[:2]+[row(action='supporters-append',ts='2026-08-01T00:00:00Z')]) +assert r.returncode==0 and [(v['action'],v['result']) for v in rows]==[('invite','ok-catchup')],rows +assert '@user-42' in files['published'] and state['backlog']==0 +print('PASS rearmed invite restores projection without queue or redundant append line') +r,rows,state,files,_=execute(tiers='2,3') +assert r.returncode==0 and rows==[] and state['backlog']==0 and 'GET user/' not in files['trace'] +print('PASS actual catch-up is empty when tier1 disabled') + +# Error fields can identify the limit, but generic validation must never be swallowed. +for field in ('message','code'): + r,rows,_,files,_=execute(sweep=False,invite_status=422,body={'message':'Validation Failed','errors':[{field:'too many invitations'}]}) + assert r.returncode==0 and all(v['result']=='deferred-quota' for v in rows) + assert 'COMMENT_SENT' not in files['trace'] +print('PASS invitation-limit nested errors message/code recognition') + +# Exercise the real steps 2-4 loops with a fake clock, then the real reserve tail. +cleanup_block='sweep_cleanup() {'+main.split('sweep_cleanup() {',1)[1] +cleanup_doubles=r''' +# Only these two exhaustive list reads are allowed during cleanup. +pages() { + case "$2" in + "repos/$REWARD_REPO/collaborators?per_page=100") echo '[{"id":42}]' > "$3" ;; + "repos/$REWARD_REPO/invitations?per_page=100") echo '[]' > "$3" ;; + *) return 1 ;; + esac +} +baseline() { echo '{"collaborators":[],"invitations":[]}' > "$work/baseline.json"; } +all_markers() { echo '[]' > "$work/markers.json"; } +''' +r,rows,state,files,_=execute(rows=[row()],setup=cleanup_doubles+'\necho "$((started+361))" > "$SWEEP_CLOCK_FILE"',block=cleanup_block) +assert r.returncode!=0 and rows==[] and 'audit deadline exceeded' in r.stdout +assert 'published' in files and state is not None and 'ROTATE' in files['trace'] +print('PASS step2 deadline fails incomplete audit closed and still executes steps5-8') +# Advance while collecting marker evidence: the per-subject loop must not start reads. +setup=cleanup_doubles+r''' +all_markers() { + echo '[]' > "$work/markers.json" + echo "$((started+361))" > "$SWEEP_CLOCK_FILE" +} +''' +comment_row=dict(row(action='comment',tier=2),subject='https://github.com/'+repo+'/issues/5') +r,rows,state,files,_=execute(rows=[row(),comment_row],setup=setup,block=cleanup_block) +assert r.returncode!=0 and 'audit deadline exceeded' in r.stdout and 'published' in files and state is not None +assert 'GET repos/'+repo+'/issues/5' not in files['trace'] +print('PASS per-subject audit deadline stops reads, blocks cleanup, preserves reserve') +# With no subjects, the same boundary is checked inside marker reconciliation. +r,rows,state,files,_=execute(rows=[row()],stars=[],setup=setup,block=cleanup_block) +assert r.returncode!=0 and 'audit deadline exceeded' in r.stdout and 'published' in files and state is not None +assert 'DELETE ' not in files['trace'] +print('PASS marker reconciliation deadline blocks destructive cleanup and preserves reserve') +# Once the audit is complete, unrevoked actors wait; this budget stop stays green. +setup=cleanup_doubles+r''' +within_deadline() { + if [ "${audit_complete:-false}" = true ]; then echo "$((started+361))" > "$SWEEP_CLOCK_FILE"; fi + [ "$(elapsed)" -le 360 ] +} +''' +r,rows,state,files,_=execute(rows=[row()],stars=[],setup=setup,block=cleanup_block) +assert r.returncode==0 and rows==[] and 'DELETE ' not in files['trace'] and 'published' in files and state is not None,(r.stdout,rows) +print('PASS revoke deadline defers unrevoked actors green and executes steps5-8') diff --git a/supporter-loop/fixtures/workflow/test-model.py b/supporter-loop/fixtures/workflow/test-model.py index 3634f01..11ba414 100644 --- a/supporter-loop/fixtures/workflow/test-model.py +++ b/supporter-loop/fixtures/workflow/test-model.py @@ -55,7 +55,7 @@ def row(action, tier=1, result='ok', gen=1, run='1-1', ts='2026-01-01T00:00:00Z' ('other actor closure does not affect generation', [dict(row('revoke'), actor_id=43)], 'generation($repo;$id)', 1), ('other repository closure does not affect generation', [dict(row('revoke'), repo='caty-ai/other')], 'generation($repo;$id)', 1), ] -with tempfile.TemporaryDirectory(prefix='supporter-model-', dir=pathlib.Path(__file__).resolve().parent) as scratch: +with tempfile.TemporaryDirectory(prefix='supporter-model-') as scratch: model = pathlib.Path(scratch, 'test.jq') mode_cases = [(name, ledger, expression, expected, 'live') for name, ledger, expression, expected in cases] mode_cases += [('record-only ignores live upgrade', [row('would-invite', mode='record-only'), row('comment', 2)], 'achieved($repo;$id)', 1, 'record-only'), @@ -104,6 +104,7 @@ def row(action, tier=1, result='ok', gen=1, run='1-1', ts='2026-01-01T00:00:00Z' header_sample = pathlib.Path(__file__).with_name('SUPPORTERS.header.sample.md').read_bytes() error = '::error::SUPPORTERS.header.md missing or malformed in reward repo (child #1); regeneration skipped' double = r''' +read_ledger() { cp "$work/api-ledger.ndjson" "$ledger"; } get() { [ "$1" = "$LEDGER_TOKEN" ] || return 1 printf '%s\n' "$2" >> "$work/gets" @@ -121,7 +122,7 @@ def row(action, tier=1, result='ok', gen=1, run='1-1', ts='2026-01-01T00:00:00Z' code=200 } ''' -with tempfile.TemporaryDirectory(prefix='supporter-render-', dir=pathlib.Path(__file__).resolve().parent) as directory: +with tempfile.TemporaryDirectory(prefix='supporter-render-') as directory: root = pathlib.Path(directory) (root / 'model.jq').write_text(models[0]) script = 'set -euo pipefail\n' + double + textwrap.dedent(function.group()) + '\nregenerate\n' @@ -143,7 +144,7 @@ def row(action, tier=1, result='ok', gen=1, run='1-1', ts='2026-01-01T00:00:00Z' for output in ('SUPPORTERS.md', 'published.json', 'gets'): (root / output).unlink(missing_ok=True) (root / 'header.json').write_text(json.dumps(dict(content=base64.b64encode(header).decode()))) - (root / 'ledger.ndjson').write_text(json.dumps(sample_row) + '\n') + (root / 'api-ledger.ndjson').write_text(json.dumps(sample_row) + '\n') env = dict(os.environ, work=str(root), ledger=str(root / 'ledger.ndjson'), MODE='live', GITHUB_REPOSITORY=REPO, REWARD_REPO='caty-ai/ask-ai-widget', LEDGER_TOKEN='fixture-ledger', HEADER_STATUS=str(status)) @@ -169,7 +170,7 @@ def row(action, tier=1, result='ok', gen=1, run='1-1', ts='2026-01-01T00:00:00Z' # All existing projection cases also pass through the complete header + table render. for name, ledger_rows, fragment, present in render_cases: (root / 'header.json').write_text(json.dumps(dict(content=base64.b64encode(header_sample).decode()))) - (root / 'ledger.ndjson').write_text('\n'.join(json.dumps(item) for item in ledger_rows)) + (root / 'api-ledger.ndjson').write_text('\n'.join(json.dumps(item) for item in ledger_rows)) env['HEADER_STATUS'] = '200' result = subprocess.run(['/bin/bash', '-c', script], env=env, capture_output=True, text=True) assert result.returncode == 0, (name, result.stdout, result.stderr)