Repository navigation
Build ghrunner EBS snapshot & open PR #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build ghrunner EBS snapshot & open PR | |
| # Builds a pre-warmed Bottlerocket data-volume EBS snapshot (cached container | |
| # images) for the github-actions Karpenter NodePool, then opens a SIGNED pull | |
| # request in analyticsMD/devops-multiaccount bumping `ebs_snapshot_id` in | |
| # stacks/internal/eks/uw2-prod-eks.yaml. A human reviews, runs `atmos plan`, | |
| # and applies via the normal TACOS flow. | |
| # | |
| # The snapshot ID is NOT written to SSM / read via Atmos `!store` on purpose: | |
| # we keep it a versioned, reviewable value in git (signed-commit policy). | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| images: | |
| description: "Comma-separated container images to cache into the snapshot" | |
| required: true | |
| default: "351480950201.dkr.ecr.us-west-2.amazonaws.com/github-runner:latest" | |
| region: | |
| description: "AWS region to build the snapshot in" | |
| required: true | |
| default: "us-west-2" | |
| bottlerocket_ami_param: | |
| description: "SSM param path for the Bottlerocket AMI (must match the cluster k8s version)" | |
| required: true | |
| default: "/aws/service/bottlerocket/aws-k8s-1.33/x86_64/latest/image_id" | |
| snapshot_size: | |
| description: "Builder data-volume size in GiB (must fit all cached images)" | |
| required: true | |
| default: "200" | |
| instance_role: | |
| description: "Existing IAM role for the builder instance (-R); reused so no role is created" | |
| required: true | |
| default: "KarpenterNodeRole-prod-eks-cluster" | |
| subnet_id: | |
| description: "Subnet ID for the builder instance (private subnet, no public IP)" | |
| required: true | |
| default: "subnet-071a8bf1ba92508f3" | |
| security_group_id: | |
| description: "Security group ID for the builder instance (blank = default VPC SG)" | |
| required: false | |
| default: "" | |
| # Auto-rebuild when the runner image changes on main. "Build and Push Docker | |
| # Image" runs on a 1st/15th cron (~2x/month) + manual dispatch; chaining off its | |
| # completion keeps the cached snapshot in sync with the image it caches. The | |
| # `branches: [main]` filter restricts this to image builds that ran on main. | |
| workflow_run: | |
| workflows: ["Build and Push Docker Image"] | |
| types: [completed] | |
| branches: [main] | |
| # Never run two builders concurrently — each spins up a CFN stack and opens a PR. | |
| concurrency: | |
| group: ghrunner-snapshot-build | |
| cancel-in-progress: false | |
| env: | |
| TARGET_REPO: analyticsMD/devops-multiaccount | |
| TARGET_FILE: stacks/internal/eks/uw2-prod-eks.yaml | |
| SLACK_CHANNEL_ID: C0830SADR0X # devops-github-actions | |
| # Snapshot builder is vendored at snapshot-tool/ (see snapshot-tool/UPSTREAM). | |
| # Build-parameter defaults. workflow_dispatch overrides these via inputs; the | |
| # workflow_run (image-rebuild) trigger carries NO inputs, so these are used. | |
| DEFAULT_IMAGES: "351480950201.dkr.ecr.us-west-2.amazonaws.com/github-runner:latest" | |
| DEFAULT_REGION: "us-west-2" | |
| DEFAULT_BR_AMI_PARAM: "/aws/service/bottlerocket/aws-k8s-1.33/x86_64/latest/image_id" | |
| DEFAULT_SNAPSHOT_SIZE: "200" | |
| DEFAULT_INSTANCE_ROLE: "KarpenterNodeRole-prod-eks-cluster" | |
| DEFAULT_SUBNET_ID: "subnet-071a8bf1ba92508f3" | |
| jobs: | |
| build-snapshot: | |
| # Manual dispatch always runs. On workflow_run, only proceed when the image | |
| # build on main succeeded AND was cron-triggered (the scheduled rebuild) — so | |
| # manual/one-off image rebuilds don't also kick off a snapshot build. | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.event == 'schedule') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write # AWS OIDC for snapshot.sh | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Resolve build parameters | |
| # workflow_run carries no inputs — fall back to the DEFAULT_* env so the | |
| # image-triggered rebuild uses the same parameters as a default dispatch. | |
| run: | | |
| { | |
| echo "REGION=${{ github.event.inputs.region || env.DEFAULT_REGION }}" | |
| echo "IMAGES=${{ github.event.inputs.images || env.DEFAULT_IMAGES }}" | |
| echo "BR_AMI_PARAM=${{ github.event.inputs.bottlerocket_ami_param || env.DEFAULT_BR_AMI_PARAM }}" | |
| echo "SNAPSHOT_SIZE=${{ github.event.inputs.snapshot_size || env.DEFAULT_SNAPSHOT_SIZE }}" | |
| echo "INSTANCE_ROLE=${{ github.event.inputs.instance_role || env.DEFAULT_INSTANCE_ROLE }}" | |
| echo "SUBNET_ID=${{ github.event.inputs.subnet_id || env.DEFAULT_SUBNET_ID }}" | |
| echo "SECURITY_GROUP_ID=${{ github.event.inputs.security_group_id }}" | |
| } >> "$GITHUB_ENV" | |
| - name: Authenticate via GitHub Actions OIDC (auth role) | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: arn:aws:iam::351480950201:role/qv-internal-github-actions-auth-role | |
| aws-region: ${{ env.REGION }} | |
| - name: Chain into snapshot-build role with repository session tag | |
| # Mirrors buid-push-image.yaml's role-chaining pattern. The role name ends | |
| # in -gha-deploy-role so the shared auth role's PermitAssumeDeployRoles policy | |
| # already allows chaining into it (defined in devops-multiaccount | |
| # stacks/internal/common/uw2-github-actions.yaml). | |
| run: | | |
| CREDS=$(aws sts assume-role \ | |
| --role-arn arn:aws:iam::351480950201:role/qv-internal-github-actions-snapshot-gha-deploy-role \ | |
| --role-session-name github-runner-snapshot-build \ | |
| --tags "Key=repository,Value=${{ github.repository }}") | |
| KEY=$(echo "$CREDS" | jq -r .Credentials.AccessKeyId) | |
| SECRET=$(echo "$CREDS" | jq -r .Credentials.SecretAccessKey) | |
| TOKEN=$(echo "$CREDS" | jq -r .Credentials.SessionToken) | |
| echo "::add-mask::$KEY" | |
| echo "::add-mask::$SECRET" | |
| echo "::add-mask::$TOKEN" | |
| { | |
| echo "AWS_ACCESS_KEY_ID=$KEY" | |
| echo "AWS_SECRET_ACCESS_KEY=$SECRET" | |
| echo "AWS_SESSION_TOKEN=$TOKEN" | |
| } >> "$GITHUB_ENV" | |
| - name: Build EBS snapshot | |
| id: snapshot | |
| working-directory: snapshot-tool | |
| run: | | |
| ARGS=(-q -r "$REGION" \ | |
| -a "$BR_AMI_PARAM" \ | |
| -s "$SNAPSHOT_SIZE" \ | |
| -R "$INSTANCE_ROLE" \ | |
| -sn "$SUBNET_ID" \ | |
| -p false) | |
| [ -n "$SECURITY_GROUP_ID" ] && ARGS+=(-sg "$SECURITY_GROUP_ID") | |
| # -q prints ONLY the snapshot id to stdout; logs go to stderr. | |
| SNAP_ID=$(./snapshot.sh "${ARGS[@]}" "$IMAGES") | |
| echo "Built snapshot: $SNAP_ID" | |
| echo "snap_id=$SNAP_ID" >> "$GITHUB_OUTPUT" | |
| - name: Generate GitHub App token (scoped to devops-multiaccount) | |
| id: app-token | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.GH_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_PEM }} | |
| owner: analyticsMD | |
| repositories: devops-multiaccount | |
| - name: Checkout devops-multiaccount | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.TARGET_REPO }} | |
| token: ${{ steps.app-token.outputs.token }} | |
| path: devops-multiaccount | |
| - name: Update ebs_snapshot_id | |
| working-directory: devops-multiaccount | |
| env: | |
| SNAP_ID: ${{ steps.snapshot.outputs.snap_id }} | |
| run: | | |
| # Surgical single-line replace keyed off the `managed-by` marker comment on | |
| # the target line. `yq -i` would rewrite the file in canonical form (stripping | |
| # blank lines between mappings and expanding YAML anchors), producing a noisy | |
| # diff that fails the repo's "scoped specifically" review gate. Keying the sed | |
| # off the marker keeps the diff to exactly one line and stays correct if another | |
| # Karpenter NodePool later adds its own ebs_snapshot_id. | |
| MARKER='managed-by: build-ghrunner-snapshot' | |
| matches=$(grep -cE "ebs_snapshot_id:.*# *${MARKER}" "$TARGET_FILE" || true) | |
| if [ "$matches" -ne 1 ]; then | |
| echo "::error::expected exactly 1 ebs_snapshot_id line marked '${MARKER}' in $TARGET_FILE, found $matches" | |
| exit 1 | |
| fi | |
| # Capture the currently-live id (pre-bump) from the marked line so the prune | |
| # step never deletes it. | |
| PREV_SNAP_ID=$(grep -E "ebs_snapshot_id:.*# *${MARKER}" "$TARGET_FILE" | grep -oE 'snap-[0-9a-f]+') | |
| echo "PREV_SNAP_ID=$PREV_SNAP_ID" >> "$GITHUB_ENV" | |
| # Replace only the marked line, preserving the marker comment. | |
| sed -i -E "s|^([[:space:]]*ebs_snapshot_id:[[:space:]]*)\"?snap-[0-9a-f]+\"?([[:space:]]*#.*${MARKER}.*)$|\1\"${SNAP_ID}\"\2|" "$TARGET_FILE" | |
| git --no-pager diff -- "$TARGET_FILE" | |
| - name: Open signed PR | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} | |
| path: devops-multiaccount | |
| sign-commits: true # API-created commit → signed by GitHub → passes branch protection | |
| branch: chore/ghrunners-ebs-snapshot | |
| delete-branch: true | |
| commit-message: "chore: bump ghrunners EBS snapshot to ${{ steps.snapshot.outputs.snap_id }}" | |
| title: "chore: bump ghrunners EBS snapshot to ${{ steps.snapshot.outputs.snap_id }}" | |
| body: | | |
| Automated bump of the github-actions Karpenter NodePool pre-warmed | |
| EBS snapshot. | |
| - New snapshot: `${{ steps.snapshot.outputs.snap_id }}` | |
| - Built from: `${{ github.repository }}` run ${{ github.run_id }} | |
| - Images cached: `${{ env.IMAGES }}` | |
| Review, then run `atmos plan` and apply via TACOS. | |
| - name: Prune old snapshots (keep newest 3) | |
| # Best-effort cleanup so snapshots don't accumulate (~2/month, 200 GiB each). | |
| # Keeps the newest 3, and always protects the just-built id and the | |
| # currently-live id. Bounded by the role's DeleteSnapshot tag/region condition. | |
| continue-on-error: true | |
| env: | |
| KEEP: "3" | |
| NEW_ID: ${{ steps.snapshot.outputs.snap_id }} | |
| run: | | |
| set -euo pipefail | |
| mapfile -t SNAPS < <(aws ec2 describe-snapshots --owner-ids self --region "$REGION" \ | |
| --filters 'Name=tag:Name,Values=Bottlerocket Data Volume' \ | |
| --query 'reverse(sort_by(Snapshots,&StartTime))[].SnapshotId' --output text | tr '\t' '\n') | |
| echo "Found ${#SNAPS[@]} snapshots; keeping newest $KEEP + live ($PREV_SNAP_ID) + new ($NEW_ID)" | |
| i=0 | |
| for s in "${SNAPS[@]}"; do | |
| [ -z "$s" ] && continue | |
| i=$((i+1)) | |
| if [ "$i" -le "$KEEP" ] || [ "$s" = "$NEW_ID" ] || [ "$s" = "$PREV_SNAP_ID" ]; then | |
| echo "keep $s"; continue | |
| fi | |
| echo "prune $s" | |
| aws ec2 delete-snapshot --region "$REGION" --snapshot-id "$s" \ | |
| || echo "::warning::failed to delete $s" | |
| done | |
| - name: Slack notify (success) | |
| if: success() | |
| uses: slackapi/slack-github-action@v1.24.0 | |
| with: | |
| channel-id: ${{ env.SLACK_CHANNEL_ID }} | |
| payload: | | |
| { | |
| "text": "ghrunner EBS snapshot PR opened", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "✅ *ghrunner EBS snapshot built:* `${{ steps.snapshot.outputs.snap_id }}`\nPR opened in *${{ env.TARGET_REPO }}* — review + `atmos plan` to roll out." | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| - name: Slack notify (failure) | |
| if: failure() | |
| uses: slackapi/slack-github-action@v1.24.0 | |
| with: | |
| channel-id: ${{ env.SLACK_CHANNEL_ID }} | |
| payload: | | |
| { | |
| "text": "ghrunner EBS snapshot build failed", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "❌ *ghrunner EBS snapshot workflow failed*\n*Repo:* ${{ github.repository }}\n*Logs:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n<!channel>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} |