From e176b8a8937d72287f11e8ea0ef9f61ce0534648 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Mon, 20 Jul 2026 14:21:50 +0100 Subject: [PATCH 1/3] chore: cp latest starter changes --- .github/actions/terragrunt/README.md | 214 ------------ .github/actions/terragrunt/action.yml | 247 ------------- .github/docs/README.md | 68 ++-- .github/workflows/destroy.yml | 116 ++----- .../workflows/dev_infra_apply_from_plan.yml | 22 -- .github/workflows/dev_infra_apply_no_plan.yml | 17 - .github/workflows/dev_infra_plan.yml | 17 - .github/workflows/infra_apply.yml | 190 ++++++++++ .github/workflows/infra_bootstrap.yml | 80 +++++ .github/workflows/infra_plan.yml | 156 +++++++++ .github/workflows/prod_code_deploy.yml | 3 +- .../workflows/prod_infra_apply_from_plan.yml | 22 -- .../workflows/prod_infra_apply_no_plan.yml | 23 -- .github/workflows/prod_infra_plan.yml | 23 -- .github/workflows/pull_request.yml | 66 +--- .github/workflows/shared_build.yml | 42 ++- .github/workflows/shared_build_get.yml | 47 ++- .github/workflows/shared_code_deploy.yml | 77 +++-- .github/workflows/shared_get_modules.yml | 215 ------------ .../shared_infra_apply_from_plan.yml | 274 --------------- .../workflows/shared_infra_apply_no_plan.yml | 115 ------ .github/workflows/shared_infra_plan.yml | 268 -------------- README.md | 12 +- infra/README.md | 59 ++++ infra/root.hcl | 49 ++- justfile | 15 - scripts/ci/justfile | 327 ++++-------------- 27 files changed, 780 insertions(+), 1984 deletions(-) delete mode 100644 .github/actions/terragrunt/README.md delete mode 100644 .github/actions/terragrunt/action.yml delete mode 100644 .github/workflows/dev_infra_apply_from_plan.yml delete mode 100644 .github/workflows/dev_infra_apply_no_plan.yml delete mode 100644 .github/workflows/dev_infra_plan.yml create mode 100644 .github/workflows/infra_apply.yml create mode 100644 .github/workflows/infra_bootstrap.yml create mode 100644 .github/workflows/infra_plan.yml delete mode 100644 .github/workflows/prod_infra_apply_from_plan.yml delete mode 100644 .github/workflows/prod_infra_apply_no_plan.yml delete mode 100644 .github/workflows/prod_infra_plan.yml delete mode 100644 .github/workflows/shared_get_modules.yml delete mode 100644 .github/workflows/shared_infra_apply_from_plan.yml delete mode 100644 .github/workflows/shared_infra_apply_no_plan.yml delete mode 100644 .github/workflows/shared_infra_plan.yml diff --git a/.github/actions/terragrunt/README.md b/.github/actions/terragrunt/README.md deleted file mode 100644 index 05f3623..0000000 --- a/.github/actions/terragrunt/README.md +++ /dev/null @@ -1,214 +0,0 @@ -# Execute Terraform & Terragrunt - -This GitHub Action sets up **Terraform** and **Terragrunt** and runs a specified `terragrunt` action. When the action needs AWS, the workflow job should configure credentials first. - ---- - -## πŸš€ Features - -- Installs pinned versions of Terraform and Terragrunt -- Installs Terragrunt through `jdx/mise-action@v4` -- Uses AWS credentials already configured earlier in the same job when needed -- Optionally passes Terragrunt variables via JSON tfvars -- Supports `apply`, `plan`, `apply_plan`, `destroy`, `init`, and `graph` -- Supports `plan` mode for producing local saved plan files -- Supports `init` mode for outputs-only reads -- Supports `graph` mode for raw `terragrunt run-all graph-dependencies` output capture -- Writes saved plan files into the live stack directory so workflows can upload and download them with GitHub artifacts -- Exports Terragrunt outputs as compact JSON when state exists -- Refuses to run against `infra/live/_catalog`; create a real environment under `infra/live/` from the catalog before planning or applying - -The Terragrunt install step is kept in this repo-local action rather than hidden behind a third-party Terragrunt wrapper action so the repo can control the exact setup-action revision and react quickly to GitHub Actions runtime deprecations or nested dependency warnings. - ---- - -## πŸ“₯ Inputs - -| Name | Description | Required | Default | -|--------------------|---------------------------------------------------------------------------------------------|----------|--------------| -| `tf_version` | Version of Terraform to install | ❌ | `1.13.3` | -| `tg_version` | Version of Terragrunt to install | ❌ | `0.72.6` | -| `aws_region` | AWS region | ❌ | `eu-west-2` | -| `override_tg_vars` | Override or additional Terragrunt variables in JSON format | ❌ | `{}` | -| `tg_directory` | Directory containing the Terragrunt config | βœ… | β€” | -| `tg_action` | Terragrunt action: `apply`, `plan`, `apply_plan`, `destroy`, `init`, or `graph` | βœ… | `apply` | - -`override_tg_vars` is written for `apply`, `plan`, and `destroy`, but not for `init`. - ---- - -## πŸ“€ Outputs - -| Name | Description | -|---------------------------|-------------------------------------------------------------------------------| -| `tg_outputs` | All Terraform outputs in compact JSON. If no state exists, returns `{}` | -| `tg_graph_output` | Raw Terragrunt `run-all graph-dependencies` output. Set only for `graph` | -| `plan_has_changes` | Whether the saved plan contains changes | -| `plan_artifact_directory` | Directory containing the saved plan artifact bundle | - ---- - -## βš™οΈ Behavior - -- `apply` - Runs `terragrunt apply -auto-approve`. -- `plan` - Runs `terragrunt plan -detailed-exitcode -out=/terragrunt.tfplan`. The action writes `terragrunt.plan.meta.json` for every plan run, including `has_changes` and `contains_mocked_outputs`, and writes `terragrunt.plan.txt` alongside the binary plan when the plan has changes. -- `apply_plan` - Runs `terragrunt apply /terragrunt.tfplan`. The calling workflow must download that stack's saved plan artifact into the live stack directory before invoking `apply_plan`. The action requires `terragrunt.plan.meta.json` to be present there. If metadata is missing, or if it says `contains_mocked_outputs: true`, the action fails before apply and tells the operator to regenerate the plan from real upstream outputs. -- `destroy` - Runs `terragrunt destroy -auto-approve`. -- `init` - Runs `terragrunt init -input=false -reconfigure` and then captures outputs. -- `graph` - Runs `terragrunt run-all graph-dependencies --terragrunt-non-interactive --terragrunt-include-external-dependencies --terragrunt-log-level error` and exposes the raw output as `tg_graph_output`. - ---- - -## πŸ—‚ Saved Plan Layout - -One run-level metadata file is stored separately by the shared infra wrapper as a GitHub Actions artifact: - -- artifact name: `infra-plan-metadata` -- file: `plan-metadata.json` containing the frozen workflow inputs and derived `waves` - -Each Terragrunt stack or module stores its own plan bundle as a GitHub Actions artifact named `terragrunt-plan--`: - -- `terragrunt.plan.meta.json` -- `terragrunt.tfplan` only when changes exist -- `terragrunt.plan.txt` only when changes exist - ---- - -## πŸ” AWS Credentials - -Configure AWS credentials in the workflow job before calling this action. The action then reuses those ambient credentials for Terragrunt itself and for any Terragrunt-hook-driven saved-plan upload or download steps. - ---- - -## πŸ›  Example Usage - -### Reuse AWS credentials already configured in the job - -```yaml -jobs: - read-outputs: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - uses: actions/checkout@v6 - - - name: Configure AWS credentials once - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - - - name: Read Terragrunt outputs - id: tg - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/dev/aws/frontend - tg_action: init - - - name: Use outputs - run: | - echo '${{ steps.tg.outputs.tg_outputs }}' | jq . -``` - -### Minimal apply - -```yaml -jobs: - apply: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - uses: actions/checkout@v6 - - - name: Configure AWS credentials once - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - - - name: Apply infrastructure - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/dev/aws/frontend - tg_action: apply - override_tg_vars: '{"example":"value"}' -``` - -### Plan - -```yaml -jobs: - plan: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - uses: actions/checkout@v6 - - - name: Configure AWS credentials once - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-dev-github-oidc-role - - - name: Plan infrastructure - id: tg - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/dev/aws/frontend - tg_action: plan - - - name: Show plan result - run: | - echo "has_changes=${{ steps.tg.outputs.plan_has_changes }}" - echo "artifact_dir=${{ steps.tg.outputs.plan_artifact_directory }}" -``` - -### Apply from downloaded GitHub artifact - -```yaml -jobs: - apply: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - actions: read - - steps: - - uses: actions/checkout@v6 - - - name: Configure AWS credentials once - uses: aws-actions/configure-aws-credentials@v6 - with: - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-dev-github-oidc-role - - - name: Download saved plan - uses: actions/download-artifact@v7 - with: - name: terragrunt-plan-dev-frontend - path: infra/live/dev/aws/frontend - - - name: Apply infrastructure from uploaded plan - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/dev/aws/frontend - tg_action: apply_plan -``` - -This action expects the workflow to download the matching per-stack plan artifact into the live stack directory before using `tg_action: apply_plan`. diff --git a/.github/actions/terragrunt/action.yml b/.github/actions/terragrunt/action.yml deleted file mode 100644 index e0cb726..0000000 --- a/.github/actions/terragrunt/action.yml +++ /dev/null @@ -1,247 +0,0 @@ -name: "Execute Terraform & Terragrunt" -description: "Sets up Terraform and Terragrunt and runs a terragrunt action using AWS credentials already configured in the job when needed" - -inputs: - tf_version: - description: "Version of Terraform to install" - required: false - default: "1.13.3" - tg_version: - description: "Version of Terragrunt to install" - required: false - default: "0.72.6" - aws_region: - description: "AWS Region" - default: "eu-west-2" - override_tg_vars: - description: "Override or additional Terragrunt variables in JSON format" - required: false - default: "{}" - tg_directory: - description: "Module directory to perform action upon" - required: true - tg_action: - description: "Terragrunt action to perform (`apply`, `plan`, `apply_plan`, `destroy`, `init` for outputs-only, or `graph` for raw run-all graph output)" - required: true - default: apply - -outputs: - tg_outputs: - description: "All Terraform outputs in JSON format" - value: ${{ steps.tg_outputs.outputs.terraform_json }} - tg_graph_output: - description: "Raw Terragrunt run-all dependency graph output" - value: ${{ steps.tg_graph.outputs.graph_output }} - plan_has_changes: - description: "Whether the saved plan contains changes" - value: ${{ steps.terragrunt_action.outputs.plan_has_changes }} - plan_artifact_directory: - description: "Directory containing the saved plan artifact bundle" - value: ${{ steps.terragrunt_action.outputs.plan_artifact_directory }} - -runs: - using: "composite" - steps: - - name: Install Terraform - uses: hashicorp/setup-terraform@v4 - with: - terraform_version: ${{ inputs.tf_version }} - terraform_wrapper: false - - # Install Terragrunt through mise directly so this repo controls the exact - # setup-action revision and can move faster than third-party wrapper actions - # when GitHub Action runtime deprecations or nested dependency warnings land. - - name: Install Terragrunt - uses: jdx/mise-action@v4 - with: - tool_versions: | - terragrunt ${{ inputs.tg_version }} - github_token: ${{ github.token }} - - - name: Verify Terragrunt installation - shell: bash - run: terragrunt --version - - - name: Normalize and write override_tg_vars - if: inputs.tg_action == 'apply' || inputs.tg_action == 'plan' || inputs.tg_action == 'destroy' - shell: bash - env: - OVERRIDE_TG_VARS: ${{ inputs.override_tg_vars }} - run: | - echo "$OVERRIDE_TG_VARS" | jq -c . > ${{ inputs.tg_directory }}/override_tg_vars.tfvars.json - - - name: Action Terragrunt - id: terragrunt_action - shell: bash - env: - TF_IN_AUTOMATION: true - TG_PLAN_LOG_FILENAME: terragrunt.plan.log - TG_PLAN_LOG_ABS_PATH: ${{ github.workspace }}/${{ inputs.tg_directory }}/terragrunt.plan.log - working-directory: ${{ inputs.tg_directory }} - run: | - PLAN_DIR="${{ github.workspace }}/${{ inputs.tg_directory }}" - PLAN_PATH="${PLAN_DIR}/terragrunt.tfplan" - PLAN_TEXT_PATH="${PLAN_DIR}/terragrunt.plan.txt" - PLAN_META_PATH="${PLAN_DIR}/terragrunt.plan.meta.json" - PLAN_JSON_PATH="${PLAN_DIR}/terragrunt.plan.json" - PLAN_LOG_PATH="${PLAN_DIR}/terragrunt.plan.log" - PLAN_ARTIFACT_DIR="${PLAN_DIR}/terragrunt-plan-artifact" - PLAN_ARTIFACT_DIR_REL="${{ inputs.tg_directory }}/terragrunt-plan-artifact" - - case "${{ inputs.tg_action }}" in - apply) - terragrunt --terragrunt-non-interactive apply -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json - ;; - plan) - set +e - terragrunt --terragrunt-non-interactive plan -input=false -lock=false -detailed-exitcode -compact-warnings -out="$PLAN_PATH" -var-file=override_tg_vars.tfvars.json 2>&1 | tee "$PLAN_LOG_PATH" - plan_exit_code=${PIPESTATUS[0]} - set -e - - if [ "$plan_exit_code" -eq 1 ]; then - exit 1 - fi - - plan_has_changes=false - if [ "$plan_exit_code" -eq 2 ]; then - plan_has_changes=true - fi - - plan_contains_mocked_outputs=false - if grep -Fq "mock outputs provided and returning those in dependency output" "$PLAN_LOG_PATH"; then - plan_contains_mocked_outputs=true - echo "::warning title=Mock outputs used during plan::Terragrunt used dependency mock outputs while creating a saved plan. This plan artifact should not be used with apply_plan until a fresh plan is created from real upstream outputs." - fi - - rm -rf "$PLAN_ARTIFACT_DIR" - mkdir -p "$PLAN_ARTIFACT_DIR" - - if [ "$plan_has_changes" = "true" ]; then - terragrunt show -no-color "$PLAN_PATH" > "$PLAN_TEXT_PATH" - jq -n \ - --arg tg_directory "${{ inputs.tg_directory }}" \ - --argjson has_changes "$plan_has_changes" \ - --argjson contains_mocked_outputs "$plan_contains_mocked_outputs" \ - --argjson artifact_files '["terragrunt.plan.meta.json","terragrunt.tfplan","terragrunt.plan.txt"]' \ - '{tg_directory: $tg_directory, has_changes: $has_changes, contains_mocked_outputs: $contains_mocked_outputs, artifact_files: $artifact_files}' \ - > "$PLAN_META_PATH" - cp "$PLAN_META_PATH" "$PLAN_ARTIFACT_DIR/" - cp "$PLAN_PATH" "$PLAN_ARTIFACT_DIR/" - cp "$PLAN_TEXT_PATH" "$PLAN_ARTIFACT_DIR/" - else - jq -n \ - --arg tg_directory "${{ inputs.tg_directory }}" \ - --argjson has_changes "$plan_has_changes" \ - --argjson contains_mocked_outputs "$plan_contains_mocked_outputs" \ - --argjson artifact_files '["terragrunt.plan.meta.json"]' \ - '{tg_directory: $tg_directory, has_changes: $has_changes, contains_mocked_outputs: $contains_mocked_outputs, artifact_files: $artifact_files}' \ - > "$PLAN_META_PATH" - cp "$PLAN_META_PATH" "$PLAN_ARTIFACT_DIR/" - rm -f "$PLAN_PATH" "$PLAN_TEXT_PATH" "$PLAN_JSON_PATH" - fi - - echo "=== terragrunt.plan.meta.json ===" - cat "$PLAN_META_PATH" - - echo "=== plan module directory ===" - ls -la "$PLAN_DIR" - echo "=== plan artifact directory ===" - ls -la "$PLAN_ARTIFACT_DIR" - - echo "plan_exit_code=$plan_exit_code" >> "$GITHUB_OUTPUT" - echo "plan_has_changes=$plan_has_changes" >> "$GITHUB_OUTPUT" - echo "plan_contains_mocked_outputs=$plan_contains_mocked_outputs" >> "$GITHUB_OUTPUT" - echo "plan_artifact_directory=$PLAN_ARTIFACT_DIR_REL" >> "$GITHUB_OUTPUT" - echo "Terragrunt binary plan path: $PLAN_PATH" - - ;; - apply_plan) - if [ ! -f "$PLAN_PATH" ]; then - echo "::error title=Missing saved plan artifact::Expected '${PLAN_PATH}' to exist before apply_plan." - exit 1 - fi - - if [ ! -f "$PLAN_META_PATH" ]; then - echo "::error title=Missing saved plan metadata::Expected '${PLAN_META_PATH}' to exist before apply_plan." - exit 1 - fi - - plan_contains_mocked_outputs="$(jq -r '.contains_mocked_outputs // false' "$PLAN_META_PATH")" - - if [ "$plan_contains_mocked_outputs" = "true" ]; then - echo "::error title=Saved plan contains mocked outputs::Saved plan metadata indicates mocked outputs were used. Regenerate it after upstream real outputs exist." - exit 1 - fi - - set +e - APPLY_LOG_PATH="$(pwd)/terragrunt.apply.log" - terragrunt --terragrunt-non-interactive apply -auto-approve "$PLAN_PATH" 2>&1 | tee "$APPLY_LOG_PATH" - apply_exit_code=${PIPESTATUS[0]} - set -e - - if [ "$apply_exit_code" -ne 0 ]; then - emit_error() { - local title="$1" - local pattern="$2" - - if grep -Fq "$pattern" "$APPLY_LOG_PATH"; then - local err_line - err_line="$(grep -F "$pattern" "$APPLY_LOG_PATH" | head -n 1)" - echo "::error title=${title}::${err_line}" - fi - } - - emit_error "Saved plan contains mocked outputs" "contains mocked outputs. Regenerate it after upstream real outputs exist." - - if grep -Fq "Saved plan is stale" "$APPLY_LOG_PATH"; then - emit_error "Saved plan is stale" "Saved plan is stale" - fi - - exit "$apply_exit_code" - fi - ;; - destroy) - terragrunt --terragrunt-non-interactive destroy -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json - ;; - init) - echo "Running init only (no infra changes)..." - terragrunt --terragrunt-non-interactive init -input=false -reconfigure - ;; - graph) - echo "Graph mode runs in the dedicated graph step." - ;; - *) - echo "Unknown tg_action: '${{ inputs.tg_action }}' (expected: apply|plan|apply_plan|destroy|init|graph)" >&2 - exit 2 - ;; - esac - - - name: Capture Terragrunt graph JSON - if: inputs.tg_action == 'graph' - id: tg_graph - shell: bash - working-directory: ${{ inputs.tg_directory }} - run: | - echo "πŸ•ΈοΈ Rendering Terragrunt run-all dependency graph..." - { - echo "graph_output<> "$GITHUB_OUTPUT" - echo "βœ… Terragrunt graph captured." - - - name: Capture Terraform Outputs - if: inputs.tg_action != 'destroy' && inputs.tg_action != 'graph' - id: tg_outputs - shell: bash - working-directory: ${{ inputs.tg_directory }} - run: | - echo "πŸ“€ Capturing all Terraform outputs..." - if terragrunt output -json >/dev/null 2>&1; then - TERRAGRUNT_OUTPUTS=$(terragrunt output -json | jq -c) - else - echo "ℹ️ No outputs available (no state found)." - TERRAGRUNT_OUTPUTS="{}" - fi - echo "terraform_json=$TERRAGRUNT_OUTPUTS" >> "$GITHUB_OUTPUT" - echo "βœ… Terraform outputs captured successfully." diff --git a/.github/docs/README.md b/.github/docs/README.md index 7e873a4..1f1a41b 100644 --- a/.github/docs/README.md +++ b/.github/docs/README.md @@ -7,25 +7,23 @@ workflows, or workflow-owned `just` behavior. | Workflow | Purpose | | --- | --- | -| `pull_request.yml` | Runs change-filtered PR validation for title/version preview, wrapper sync, workflow linting, repo-local action tests, Terraform/Terragrunt formatting, Terragrunt wave shape, TFLint, frontend builds, and lambda builds. | +| `pull_request.yml` | Runs change-filtered PR validation for title/version preview, wrapper sync, workflow linting, repo-local action tests, Terraform/Terragrunt formatting, TFLint, frontend builds, and lambda builds. | | `release.yml` | Tags versioned releases from `main`, publishes frontend and lambda artifacts to the CI code bucket, and creates GitHub releases. | -| `dev_infra_plan.yml` | Plans the ordered dev infra graph. | -| `dev_infra_apply_no_plan.yml` | Applies dev infrastructure using the current commit as the infra ref. | -| `dev_infra_apply_from_plan.yml` | Applies dev infra from a prior saved-plan run using `plan_artifact_run_id`. | +| `infra_bootstrap.yml` | Bootstraps the selected environment by applying `aws/code_bucket` first, then the full environment. | +| `infra_plan.yml` | Plans the selected environment with `terragrunt run-all` and saves reusable plan artifacts. | +| `infra_apply.yml` | Applies a prior saved-plan run for the selected environment using `plan_artifact_run_id`. | | `dev_code_deploy.yml` | Builds fresh frontend and lambda artifacts and deploys to dev. | -| `prod_infra_plan.yml` | Plans the ordered prod infra graph for the requested infra ref. | -| `prod_infra_apply_no_plan.yml` | Applies prod infrastructure using the pinned infra ref. | -| `prod_infra_apply_from_plan.yml` | Applies prod infra from a prior saved-plan run. | | `prod_code_deploy.yml` | Deploys existing frontend and lambda artifacts to prod. | -| `destroy.yml` | Tears down non-shared infrastructure through the Terragrunt graph in reverse wave order. | +| `destroy.yml` | Tears down infrastructure by running `terragrunt run-all destroy`, excluding `aws/oidc`. | ## Build And Deploy `shared_build.yml` builds and publishes `frontend.zip` under `frontend//` and `log_processor.zip` under `lambdas//` in the selected environment code bucket. -The selected environment's `aws/code_bucket` stack must already have a real -Terraform output named `bucket`; otherwise the build fails before upload. +The selected environment's `aws/code_bucket` stack is applied before build +upload so the bucket and artifact-prefix inputs are current when artifacts are +published. On the first release, `release.yml` has no prior tag to diff against, so release notes are generated from the full history up to the new tag. @@ -33,8 +31,9 @@ notes are generated from the full history up to the new tag. `shared_build_get.yml` resolves an existing frontend artifact from the selected environment code bucket. Prod deploys use `environment: ci` so production promotes frontend artifacts already present in the shared CI artifact bucket. -The Lambda artifact version is passed separately to -`shared_code_deploy.yml`. +It also validates that the requested `log_processor.zip` and +`cost_explorer.zip` artifacts exist for the selected Lambda version before the +deploy wrapper continues. `shared_code_deploy.yml` rolls out frontend code and the `log_processor` Lambda: @@ -52,40 +51,36 @@ Lambda: - starts the CodeDeploy deployment and prunes old versions - runs a separate Lambda invoke job after the Lambda deploy completes -## Infra Waves +## Shared Infra Wrappers -`shared_get_modules.yml` renders the Terragrunt graph and exposes static wave -outputs consumed by shared plan/apply/destroy wrappers. +The infra plan/apply/destroy wrappers install Terraform and Terragrunt first, +then execute Terragrunt across the whole environment. -The current graph is sized for four static wave jobs: +- They follow the same Terragrunt setup pattern as other AWS workflows. +- `infra_bootstrap.yml` first applies `aws/code_bucket`, because that stack + publishes the shared bootstrap Lambda zip consumed by `log_processor` and + `cost_explorer`, and then runs the full bootstrap apply. +- `infra_plan.yml` runs `terragrunt run-all plan`, then + `terragrunt run-all show`. +- `infra_apply.yml` downloads the saved plan metadata, checks out the planned + infra ref, and then runs `terragrunt run-all apply`. +- `destroy.yml` runs `terragrunt run-all destroy`, excluding `aws/oidc`. -- wave 0: roots such as `oidc` -- wave 1: dependents such as `frontend` and `code_bucket` -- wave 2: deeper dependents such as `log_processor` -- wave 3: deepest dependents when an environment adds another dependency layer +Shared infra wrappers must forward permissions required by the nested reusable +call chain: -If the live graph grows deeper, update these workflows together: - -- `shared_infra_plan.yml` -- `shared_infra_apply_no_plan.yml` -- `shared_infra_apply_from_plan.yml` -- `destroy.yml` - -Then run: - -```sh -just tg-graph-waves prod -``` +- `id-token: write` everywhere AWS OIDC is used +- `contents: read` for checkout ## Saved Plans -`shared_infra_plan.yml` writes: +`infra_plan.yml` writes: - run-level artifact: `infra-plan-metadata` -- per-stack artifact: `terragrunt-plan--` +- run-level artifact: `infra-plan-files` -`shared_infra_apply_from_plan.yml` downloads those artifacts and applies only -modules whose saved plan metadata reported changes. +`infra_apply.yml` downloads those artifacts and applies only modules whose +saved plan metadata reported changes. Saved plans are time-limited by GitHub artifact retention. @@ -96,7 +91,6 @@ references point at local paths instead of external action tags. - [get-changes](../actions/get-changes/README.md) - [just](../actions/just/README.md) -- [terragrunt](../actions/terragrunt/README.md) When a repo-local action needs AWS, configure credentials in the workflow job before calling the action. The local action should reuse that ambient AWS diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 620aa21..aaa15cc 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -11,7 +11,7 @@ on: - dev - prod -concurrency: # only run one infra apply or destroy at a time for the same environment +concurrency: group: infra-mutate-${{ inputs.environment }} cancel-in-progress: false @@ -24,73 +24,13 @@ env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role AWS_REGION: ${{ vars.AWS_REGION }} + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 jobs: - generate_waves: - name: Waves - uses: ./.github/workflows/shared_get_modules.yml - with: - environment: ${{ inputs.environment }} - infra_version: ${{ github.sha }} - ignore_shared_artifact_modules: true - ignore_oidc_module: true - wave_summary_title: Destroy Wave Summary - wave_summary_note: Destroy runs dependency waves in reverse order. OIDC and shared artifact modules are excluded. - wave_summary_order: reverse - - waves: - name: Destroy Waves - needs: generate_waves + destroy: + name: Destroy Infra runs-on: ubuntu-latest - outputs: - wave_0_modules: ${{ steps.emit.outputs.wave_0_modules }} - wave_1_modules: ${{ steps.emit.outputs.wave_1_modules }} - wave_2_modules: ${{ steps.emit.outputs.wave_2_modules }} - steps: - - name: Derive destroy waves - id: emit - shell: bash - env: - RAW_WAVES_JSON: ${{ needs.generate_waves.outputs.waves_json }} - run: | - set -euo pipefail - destroy_waves_json="$(jq -c ' - reverse - | to_entries - | map({wave: .key, modules: .value.modules}) - ' <<<"$RAW_WAVES_JSON")" - - echo "wave_0_modules=$(jq -c '.[0].modules // []' <<<"$destroy_waves_json")" >> "$GITHUB_OUTPUT" - echo "wave_1_modules=$(jq -c '.[1].modules // []' <<<"$destroy_waves_json")" >> "$GITHUB_OUTPUT" - echo "wave_2_modules=$(jq -c '.[2].modules // []' <<<"$destroy_waves_json")" >> "$GITHUB_OUTPUT" - { - echo "## Destroy Wave Summary" - echo - echo "| Wave | Modules |" - echo "| --- | --- |" - jq -r ' - if length == 0 then - ["| none | No modules selected |"] - else - map( - .wave as $wave - | (.modules | map("`" + . + "`") | join(", ")) as $modules - | "| \($wave) | \($modules) |" - ) - end - | .[] - ' <<<"$destroy_waves_json" - } >> "$GITHUB_STEP_SUMMARY" - - wave_0: - name: 0 / ${{ matrix.module }} - needs: waves - if: ${{ needs.waves.outputs.wave_0_modules != '[]' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.waves.outputs.wave_0_modules) }} steps: - uses: actions/checkout@v6 @@ -99,33 +39,27 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Destroy ${{ matrix.module }} infra - uses: ./.github/actions/terragrunt + - uses: hashicorp/setup-terraform@v4 with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: destroy + terraform_version: ${{ env.TF_VERSION }} - wave_1: - name: 1 / ${{ matrix.module }} - needs: - - waves - - wave_0 - if: ${{ always() && needs.waves.outputs.wave_1_modules != '[]' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.waves.outputs.wave_1_modules) }} - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 + - uses: gruntwork-io/terragrunt-action@v3 with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform - - name: Destroy ${{ matrix.module }} infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: destroy + - name: Destroy all infra + working-directory: infra/live/${{ inputs.environment }} + shell: bash + run: terragrunt run-all destroy --terragrunt-non-interactive --terragrunt-exclude-dir aws/oidc + + - name: Write destroy summary + shell: bash + run: | + { + echo "## Destroy Summary" + echo + echo "- \`environment\`: \`${{ inputs.environment }}\`" + echo "- command: \`terragrunt run-all destroy --terragrunt-non-interactive --terragrunt-exclude-dir aws/oidc\`" + echo "- retained stack: \`aws/oidc\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/dev_infra_apply_from_plan.yml b/.github/workflows/dev_infra_apply_from_plan.yml deleted file mode 100644 index 5c18b61..0000000 --- a/.github/workflows/dev_infra_apply_from_plan.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Dev Infra Apply From Plan - -on: - workflow_dispatch: - inputs: - plan_artifact_run_id: - description: "Workflow run ID from the earlier dev infra plan run" - required: true - type: string - -permissions: - id-token: write - contents: read - actions: read - -jobs: - infra: - name: Apply Planned Infra - uses: ./.github/workflows/shared_infra_apply_from_plan.yml - with: - environment: dev - plan_artifact_run_id: ${{ inputs.plan_artifact_run_id }} diff --git a/.github/workflows/dev_infra_apply_no_plan.yml b/.github/workflows/dev_infra_apply_no_plan.yml deleted file mode 100644 index e5a1e13..0000000 --- a/.github/workflows/dev_infra_apply_no_plan.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Dev Infra Apply (no plan) - -on: - workflow_dispatch: - -permissions: - id-token: write - contents: write - actions: read - -jobs: - infra: - name: Apply - uses: ./.github/workflows/shared_infra_apply_no_plan.yml - with: - environment: dev - infra_version: ${{ github.sha }} diff --git a/.github/workflows/dev_infra_plan.yml b/.github/workflows/dev_infra_plan.yml deleted file mode 100644 index ddd307c..0000000 --- a/.github/workflows/dev_infra_plan.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Dev Infra Plan - -on: - workflow_dispatch: - -permissions: - id-token: write - contents: write - actions: read - -jobs: - infra: - name: Plan - uses: ./.github/workflows/shared_infra_plan.yml - with: - environment: dev - infra_version: ${{ github.sha }} diff --git a/.github/workflows/infra_apply.yml b/.github/workflows/infra_apply.yml new file mode 100644 index 0000000..ea5ea89 --- /dev/null +++ b/.github/workflows/infra_apply.yml @@ -0,0 +1,190 @@ +name: Infra Apply From Plan + +on: + workflow_dispatch: + inputs: + environment: + description: environment reference i.e. 'prod' or 'dev' + required: true + type: choice + options: + - dev + - prod + plan_artifact_run_id: + description: "Workflow run ID used to recover plan metadata" + required: true + type: string + +concurrency: + group: infra-mutate-${{ inputs.environment }} + cancel-in-progress: false + +permissions: + id-token: write + contents: read + actions: read + +env: + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + TARGET_PLAN_ARTIFACT_RUN_ID: ${{ inputs.plan_artifact_run_id }} + AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + TF_VAR_bootstrap: "true" + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 + +jobs: + apply: + name: Apply Infra From Saved Plans + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Download plan metadata artifact + uses: actions/download-artifact@v7 + with: + name: infra-plan-metadata + github-token: ${{ github.token }} + run-id: ${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }} + path: . + + - name: Check plan metadata artifact + shell: bash + run: | + if [ ! -f plan-metadata.json ]; then + echo "::error title=Missing plan metadata artifact::Expected artifact 'infra-plan-metadata' containing 'plan-metadata.json' from workflow run '${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }}', but it was not downloaded." + exit 1 + fi + + - name: Read plan metadata + id: metadata + shell: bash + run: | + echo "infra_version=$(jq -r '.infra_version' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "changed_modules=$(jq -c '.changed_modules // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "changed_module_paths=$(jq -c '.changed_module_paths // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + + - name: Exit early when no saved changes were recorded + id: change_check + shell: bash + env: + CHANGED_MODULE_PATHS_JSON: ${{ steps.metadata.outputs.changed_module_paths }} + run: | + set -euo pipefail + + changed_count="$(jq 'length' <<<"$CHANGED_MODULE_PATHS_JSON")" + + if [[ "$changed_count" == "0" ]]; then + echo "::warning title=No applicable saved plan changes::The saved plan metadata contains no changed modules. Skipping apply." + echo "should_apply=false" >> "$GITHUB_OUTPUT" + { + echo "## Apply From Plan Summary" + echo + echo "- \`environment\`: \`${{ inputs.environment }}\`" + echo "- \`infra_version\`: \`${{ steps.metadata.outputs.infra_version }}\`" + echo "- \`plan_artifact_run_id\`: \`${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }}\`" + echo "- result: no saved changes detected; apply skipped" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "should_apply=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v6 + if: steps.change_check.outputs.should_apply == 'true' + with: + ref: ${{ steps.metadata.outputs.infra_version }} + + - name: Download plan files artifact + if: steps.change_check.outputs.should_apply == 'true' + uses: actions/download-artifact@v7 + with: + name: infra-plan-files + github-token: ${{ github.token }} + run-id: ${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }} + path: infra/live/${{ inputs.environment }} + + - name: Check restored plan files + if: steps.change_check.outputs.should_apply == 'true' + shell: bash + run: | + plan_files="$(find "infra/live/${{ inputs.environment }}" -name terragrunt.tfplan -print | sort)" + + if [[ -z "$plan_files" ]]; then + echo "::error title=Missing saved plan files::Expected artifact 'infra-plan-files' containing terragrunt.tfplan files from workflow run '${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }}', but none were restored." + exit 1 + fi + + printf '%s\n' "$plan_files" + + - uses: aws-actions/configure-aws-credentials@v6 + if: steps.change_check.outputs.should_apply == 'true' + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - uses: hashicorp/setup-terraform@v4 + if: steps.change_check.outputs.should_apply == 'true' + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + if: steps.change_check.outputs.should_apply == 'true' + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + + - name: Build terragrunt include-dir args + id: include_dirs + if: steps.change_check.outputs.should_apply == 'true' + shell: bash + env: + CHANGED_MODULE_PATHS_JSON: ${{ steps.metadata.outputs.changed_module_paths }} + run: | + set -euo pipefail + + include_dir_args="$(jq -r ' + [.[] | "--terragrunt-include-dir", .] + | @sh + ' <<<"$CHANGED_MODULE_PATHS_JSON")" + + echo "include_dir_args=$include_dir_args" >> "$GITHUB_OUTPUT" + + - name: Apply changed saved plans + if: steps.change_check.outputs.should_apply == 'true' + working-directory: infra/live/${{ inputs.environment }} + shell: bash + env: + INCLUDE_DIR_ARGS: ${{ steps.include_dirs.outputs.include_dir_args }} + TG_USE_SAVED_PLAN: "true" + run: | + set -euo pipefail + eval "terragrunt run-all apply --terragrunt-non-interactive ${INCLUDE_DIR_ARGS}" + + - name: Write apply summary + if: steps.change_check.outputs.should_apply == 'true' + env: + CHANGED_MODULES_JSON: ${{ steps.metadata.outputs.changed_modules }} + CHANGED_MODULE_PATHS_JSON: ${{ steps.metadata.outputs.changed_module_paths }} + shell: bash + run: | + { + echo "## Apply From Plan Summary" + echo + echo "- \`environment\`: \`${{ inputs.environment }}\`" + echo "- \`infra_version\`: \`${{ steps.metadata.outputs.infra_version }}\`" + echo "- \`plan_artifact_run_id\`: \`${{ env.TARGET_PLAN_ARTIFACT_RUN_ID }}\`" + echo "- command: \`TG_USE_SAVED_PLAN=true terragrunt run-all apply --terragrunt-non-interactive --terragrunt-include-dir ...\`" + echo + echo 'Changed module paths used for include-dir selection:' + echo + echo '```json' + jq -c . <<<"$CHANGED_MODULE_PATHS_JSON" + echo '```' + echo + echo 'Full saved plan summary:' + echo + echo '```json' + jq -c . <<<"$CHANGED_MODULES_JSON" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/infra_bootstrap.yml b/.github/workflows/infra_bootstrap.yml new file mode 100644 index 0000000..27c48bc --- /dev/null +++ b/.github/workflows/infra_bootstrap.yml @@ -0,0 +1,80 @@ +name: Infra Bootstrap + +on: + workflow_dispatch: + inputs: + environment: + description: environment reference i.e. 'prod' or 'dev' + required: true + type: choice + options: + - dev + - prod + infra_version: + description: "Version of infrastructure (terraform) to be applied" + required: true + default: main + type: string + +concurrency: + group: infra-mutate-${{ inputs.environment }} + cancel-in-progress: false + +permissions: + id-token: write + contents: read + actions: read + +env: + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + TF_VAR_bootstrap: "true" + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 + +jobs: + bootstrap_infra: + name: Bootstrap Infra + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + + - name: Apply code bucket infra + working-directory: infra/live/${{ inputs.environment }} + shell: bash + run: terragrunt run-all apply --terragrunt-non-interactive --terragrunt-include-dir aws/code_bucket + + - name: Apply all bootstrap infra + working-directory: infra/live/${{ inputs.environment }} + shell: bash + run: terragrunt run-all apply --terragrunt-non-interactive + + - name: Write bootstrap summary + shell: bash + run: | + { + echo "## Infra Bootstrap Summary" + echo + echo "- \`environment\`: \`${{ inputs.environment }}\`" + echo "- \`infra_version\`: \`${{ inputs.infra_version }}\`" + echo "- first command: \`terragrunt run-all apply --terragrunt-non-interactive --terragrunt-include-dir aws/code_bucket\`" + echo "- second command: \`terragrunt run-all apply --terragrunt-non-interactive\`" + echo "- reason: \`aws/code_bucket\` publishes the shared bootstrap Lambda zip used by `log_processor` and `cost_explorer`." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/infra_plan.yml b/.github/workflows/infra_plan.yml new file mode 100644 index 0000000..fc4a013 --- /dev/null +++ b/.github/workflows/infra_plan.yml @@ -0,0 +1,156 @@ +name: Infra Plan + +on: + workflow_dispatch: + inputs: + environment: + description: environment reference i.e. 'prod' or 'dev' + required: true + type: choice + options: + - dev + - prod + infra_version: + description: "Version of infrastructure (terraform) to be planned" + required: true + default: main + type: string + +concurrency: + group: infra-plan-${{ inputs.environment }} + cancel-in-progress: false + +permissions: + id-token: write + contents: read + actions: read + +env: + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + TF_VAR_bootstrap: "true" + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 + +jobs: + plan: + name: Plan Infra + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + + - name: Remove stale saved plan artifacts + uses: ./.github/actions/just + with: + justfile_path: scripts/ci/justfile + just_action: plan-json-files-clean ${{ inputs.environment }} + + - name: Plan all infra + working-directory: infra/live/${{ inputs.environment }} + shell: bash + run: terragrunt run-all plan --terragrunt-non-interactive + + - name: Render saved plan JSON files + working-directory: infra/live/${{ inputs.environment }} + shell: bash + run: terragrunt run-all show --terragrunt-non-interactive + + - name: List changed-plan modules + id: changed_module_paths + uses: ./.github/actions/just + with: + justfile_path: scripts/ci/justfile + just_action: plan-json-files-list-modules ${{ inputs.environment }} + + - name: Summarize changed modules + id: changed_modules + uses: ./.github/actions/just + env: + MODULE_PATHS_JSON: ${{ steps.changed_module_paths.outputs.just_outputs }} + with: + justfile_path: scripts/ci/justfile + just_action: plan-json-files-to-change-summary ${{ inputs.environment }} + + - name: Build changed module path list + id: changed_module_paths_only + env: + CHANGED_MODULES_JSON: ${{ steps.changed_modules.outputs.just_outputs }} + shell: bash + run: | + echo "changed_module_paths=$(jq -c '[.[] | select(.has_changes == true) | .module]' <<<"$CHANGED_MODULES_JSON")" >> "$GITHUB_OUTPUT" + + - name: Write plan metadata from workflow inputs + env: + CHANGED_MODULES_JSON: ${{ steps.changed_modules.outputs.just_outputs }} + CHANGED_MODULE_PATHS_JSON: ${{ steps.changed_module_paths_only.outputs.changed_module_paths }} + shell: bash + run: | + cat > plan-metadata.json <> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/prod_code_deploy.yml b/.github/workflows/prod_code_deploy.yml index 3816390..b3f622c 100644 --- a/.github/workflows/prod_code_deploy.yml +++ b/.github/workflows/prod_code_deploy.yml @@ -23,6 +23,7 @@ jobs: uses: ./.github/workflows/shared_build_get.yml with: environment: ci + lambda_version: ${{ inputs.lambda_version }} frontend_version: ${{ inputs.frontend_version }} deploy: @@ -33,5 +34,5 @@ jobs: with: environment: prod frontend_version: ${{ needs.get_build.outputs.frontend_version }} - lambda_version: ${{ inputs.lambda_version }} + lambda_version: ${{ needs.get_build.outputs.lambda_version }} code_bucket: ${{ needs.get_build.outputs.code_bucket }} diff --git a/.github/workflows/prod_infra_apply_from_plan.yml b/.github/workflows/prod_infra_apply_from_plan.yml deleted file mode 100644 index 403a89b..0000000 --- a/.github/workflows/prod_infra_apply_from_plan.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Prod Infra Apply From Plan - -on: - workflow_dispatch: - inputs: - plan_artifact_run_id: - description: "Workflow run ID from the earlier prod_infra_plan run" - required: true - type: string - -permissions: - id-token: write - contents: read - actions: read - -jobs: - infra: - name: Apply Planned Infra - uses: ./.github/workflows/shared_infra_apply_from_plan.yml - with: - environment: prod - plan_artifact_run_id: ${{ inputs.plan_artifact_run_id }} diff --git a/.github/workflows/prod_infra_apply_no_plan.yml b/.github/workflows/prod_infra_apply_no_plan.yml deleted file mode 100644 index 9c2d89f..0000000 --- a/.github/workflows/prod_infra_apply_no_plan.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Prod Infra Apply (no plan) - -on: - workflow_dispatch: - inputs: - infra_version: - description: "Infrastructure ref to apply" - required: true - default: "main" - type: string - -permissions: - id-token: write - contents: write - actions: read - -jobs: - infra: - name: Apply - uses: ./.github/workflows/shared_infra_apply_no_plan.yml - with: - environment: prod - infra_version: ${{ inputs.infra_version }} diff --git a/.github/workflows/prod_infra_plan.yml b/.github/workflows/prod_infra_plan.yml deleted file mode 100644 index c2f6412..0000000 --- a/.github/workflows/prod_infra_plan.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Prod Infra Plan - -on: - workflow_dispatch: - inputs: - infra_version: - description: "Infrastructure ref to plan" - required: true - default: "main" - type: string - -permissions: - id-token: write - contents: read - actions: read - -jobs: - infra: - name: Plan - uses: ./.github/workflows/shared_infra_plan.yml - with: - environment: prod - infra_version: ${{ inputs.infra_version }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index db0278b..f3c7f88 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -19,6 +19,10 @@ on: type: string default: "fix: manual workflow dispatch" +env: + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 + jobs: check-pr-title: name: PR Title @@ -205,69 +209,9 @@ jobs: working-directory: infra - name: Terragrunt hclfmt check - run: terragrunt hclfmt --check + run: terragrunt hclfmt --terragrunt-check working-directory: infra - validate-tg-waves: - needs: - - check - - format-github - - format-terragrunt - runs-on: ubuntu-latest - if: ${{ always() && needs.check.result == 'success' && (needs.check.outputs.github == 'true' || needs.check.outputs.terraform == 'true' || needs.check.outputs.terragrunt == 'true') && (needs.format-github.result == 'success' || needs.format-github.result == 'skipped') && (needs.format-terragrunt.result == 'success' || needs.format-terragrunt.result == 'skipped') }} - name: TG Waves - timeout-minutes: 5 - permissions: - contents: read - env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_REGION: ${{ vars.AWS_REGION }} - steps: - - uses: actions/checkout@v6 - - - uses: hashicorp/setup-terraform@v4 - - - uses: jdx/mise-action@v4 - with: - tool_versions: | - terragrunt 0.72.6 - github_token: ${{ github.token }} - - - name: Build prod Terragrunt waves - id: waves - uses: ./.github/actions/just - with: - aws_region: ${{ env.AWS_REGION }} - just_action: tg-graph-waves prod - - - name: Build infra wave JSON - id: infra_waves - uses: ./.github/actions/just - env: - RAW_WAVES_JSON: ${{ steps.waves.outputs.just_outputs }} - with: - justfile_path: scripts/ci/justfile - just_action: tg-waves-to-infra-waves - - - name: Build destroy wave JSON - id: destroy_waves - uses: ./.github/actions/just - env: - RAW_WAVES_JSON: ${{ steps.waves.outputs.just_outputs }} - with: - justfile_path: scripts/ci/justfile - just_action: tg-waves-to-destroy-waves - - - name: Validate static wave jobs - uses: ./.github/actions/just - env: - RAW_WAVES_JSON: ${{ steps.waves.outputs.just_outputs }} - INFRA_WAVES_JSON: ${{ steps.infra_waves.outputs.just_outputs }} - DESTROY_WAVES_JSON: ${{ steps.destroy_waves.outputs.just_outputs }} - with: - justfile_path: scripts/ci/justfile - just_action: tg-validate-static-wave-jobs - lint-terraform: needs: check runs-on: ubuntu-latest diff --git a/.github/workflows/shared_build.yml b/.github/workflows/shared_build.yml index 25f1bf6..6a08661 100644 --- a/.github/workflows/shared_build.yml +++ b/.github/workflows/shared_build.yml @@ -35,6 +35,8 @@ env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role AWS_REGION: ${{ vars.AWS_REGION }} + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 jobs: bucket: @@ -50,21 +52,45 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Get build artifact bucket - id: deploy_bucket - uses: ./.github/actions/terragrunt + - uses: hashicorp/setup-terraform@v4 with: - tg_directory: infra/live/${{ inputs.environment }}/aws/code_bucket + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + + - name: Read Lambda artifact dir + id: lambda_artifact_dir + uses: ./.github/actions/just + with: + justfile_path: scripts/ci/justfile + just_action: code-bucket-get-lambda-artifact-dir + + - name: Read AppSpec artifact dir + id: appspec_artifact_dir + uses: ./.github/actions/just + with: + justfile_path: scripts/ci/justfile + just_action: code-bucket-get-appspec-artifact-dir + + - name: Apply build artifact bucket + env: + TF_VAR_lambda_artifact_dir: ${{ steps.lambda_artifact_dir.outputs.just_outputs }} + TF_VAR_appspec_artifact_dir: ${{ steps.appspec_artifact_dir.outputs.just_outputs }} + working-directory: infra/live/${{ inputs.environment }}/aws/code_bucket + shell: bash + run: terragrunt apply --terragrunt-non-interactive -auto-approve - name: Get bucket name id: get_bucket_name - env: - TG_OUTPUTS: ${{ steps.deploy_bucket.outputs.tg_outputs }} + working-directory: infra/live/${{ inputs.environment }}/aws/code_bucket + shell: bash run: | - bucket=$(echo "$TG_OUTPUTS" | jq -er '.bucket.value // empty' || true) + bucket="$(terragrunt output -raw bucket || true)" if [[ -z "$bucket" ]]; then echo "::error title=Missing code bucket output::infra/live/${{ inputs.environment }}/aws/code_bucket did not expose Terraform output 'bucket'. Apply that stack before publishing build artifacts." - echo "$TG_OUTPUTS" | jq . exit 1 fi echo "bucket=$bucket" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/shared_build_get.yml b/.github/workflows/shared_build_get.yml index e2cfd75..c8ab7d9 100644 --- a/.github/workflows/shared_build_get.yml +++ b/.github/workflows/shared_build_get.yml @@ -6,6 +6,9 @@ on: environment: required: true type: string + lambda_version: + required: true + type: string frontend_version: required: true type: string @@ -13,6 +16,9 @@ on: code_bucket: description: "Bucket containing build artifacts" value: ${{ jobs.bucket.outputs.code_bucket_name }} + lambda_version: + description: "Resolved lambda version" + value: ${{ inputs.lambda_version }} frontend_version: description: "Resolved frontend version" value: ${{ inputs.frontend_version }} @@ -25,6 +31,8 @@ env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role AWS_REGION: ${{ vars.AWS_REGION }} + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 jobs: bucket: @@ -42,22 +50,23 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Get build artifact bucket - id: deploy_bucket - uses: ./.github/actions/terragrunt + - uses: hashicorp/setup-terraform@v4 with: - tg_directory: infra/live/${{ inputs.environment }}/aws/code_bucket - tg_action: init + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform - name: Get bucket name id: get_bucket_name - env: - TG_OUTPUTS: ${{ steps.deploy_bucket.outputs.tg_outputs }} + working-directory: infra/live/${{ inputs.environment }}/aws/code_bucket + shell: bash run: | - bucket=$(echo "$TG_OUTPUTS" | jq -er '.bucket.value // empty' || true) + bucket="$(terragrunt output -raw bucket || true)" if [[ -z "$bucket" ]]; then echo "::error title=Missing code bucket output::infra/live/${{ inputs.environment }}/aws/code_bucket did not expose Terraform output 'bucket'. Apply that stack before publishing build artifacts." - echo "$TG_OUTPUTS" | jq . exit 1 fi echo "bucket=$bucket" >> "$GITHUB_OUTPUT" @@ -70,3 +79,23 @@ jobs: with: justfile_path: scripts/ci/justfile just_action: frontend-check-version + + - name: Check log processor artifact exists + uses: ./.github/actions/just + env: + BUCKET_NAME: ${{ steps.get_bucket_name.outputs.bucket }} + VERSION: ${{ inputs.lambda_version }} + LAMBDA_ARTIFACT_NAME: log_processor + with: + justfile_path: scripts/ci/justfile + just_action: lambda-check-deploy-artifacts + + - name: Check cost explorer artifact exists + uses: ./.github/actions/just + env: + BUCKET_NAME: ${{ steps.get_bucket_name.outputs.bucket }} + VERSION: ${{ inputs.lambda_version }} + LAMBDA_ARTIFACT_NAME: cost_explorer + with: + justfile_path: scripts/ci/justfile + just_action: lambda-check-deploy-artifacts diff --git a/.github/workflows/shared_code_deploy.yml b/.github/workflows/shared_code_deploy.yml index 88a4c6e..289b1db 100644 --- a/.github/workflows/shared_code_deploy.yml +++ b/.github/workflows/shared_code_deploy.yml @@ -37,6 +37,8 @@ env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role AWS_REGION: ${{ vars.AWS_REGION }} + TF_VERSION: 1.12.2 + TG_VERSION: 0.72.6 jobs: summary: @@ -80,21 +82,30 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} + - uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + - name: Get frontend infra - uses: ./.github/actions/terragrunt id: get-infra - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/frontend - tg_action: init + working-directory: infra/live/${{ inputs.environment }}/aws/frontend + shell: bash + run: | + echo "website_bucket=$(terragrunt output -raw bucket_name)" >> "$GITHUB_OUTPUT" + echo "cloudfront_distribution_id=$(terragrunt output -raw cloudfront_distribution_id)" >> "$GITHUB_OUTPUT" + echo "website_url=$(terragrunt output -raw website_url)" >> "$GITHUB_OUTPUT" - name: Get frontend outputs id: get_frontend_outputs - env: - TG_OUTPUTS: ${{ steps.get-infra.outputs.tg_outputs }} run: | - echo "website_bucket=$(echo "$TG_OUTPUTS" | jq -r '.bucket_name.value')" >> "$GITHUB_OUTPUT" - echo "cloudfront_distribution_id=$(echo "$TG_OUTPUTS" | jq -r '.cloudfront_distribution_id.value')" >> "$GITHUB_OUTPUT" - echo "website_url=$(echo "$TG_OUTPUTS" | jq -r '.website_url.value')" >> "$GITHUB_OUTPUT" + echo "website_bucket=${{ steps.get-infra.outputs.website_bucket }}" >> "$GITHUB_OUTPUT" + echo "cloudfront_distribution_id=${{ steps.get-infra.outputs.cloudfront_distribution_id }}" >> "$GITHUB_OUTPUT" + echo "website_url=${{ steps.get-infra.outputs.website_url }}" >> "$GITHUB_OUTPUT" - name: Deploy frontend uses: ./.github/actions/just @@ -127,6 +138,15 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} + - uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + - name: Set Lambda deploy paths id: lambda_paths shell: bash @@ -138,19 +158,18 @@ jobs: echo "lambda_appspec_key=$lambda_appspec_key" >> $GITHUB_OUTPUT - name: Get log processor infra - uses: ./.github/actions/terragrunt id: get-infra - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/log_processor - tg_action: init + working-directory: infra/live/${{ inputs.environment }}/aws/log_processor + shell: bash + run: | + echo "lambda_function_name=$(terragrunt output -raw lambda_function_name)" >> "$GITHUB_OUTPUT" + echo "lambda_alias_name=$(terragrunt output -raw lambda_alias_name)" >> "$GITHUB_OUTPUT" - name: Get infra detail id: get_infra_detail - env: - TG_OUTPUTS: ${{ steps.get-infra.outputs.tg_outputs }} run: | - echo "lambda_function_name=$(echo $TG_OUTPUTS | jq -r '.lambda_function_name.value')" >> $GITHUB_OUTPUT - echo "lambda_alias_name=$(echo $TG_OUTPUTS | jq -r '.lambda_alias_name.value')" >> $GITHUB_OUTPUT + echo "lambda_function_name=${{ steps.get-infra.outputs.lambda_function_name }}" >> $GITHUB_OUTPUT + echo "lambda_alias_name=${{ steps.get-infra.outputs.lambda_alias_name }}" >> $GITHUB_OUTPUT - name: Publish new lambda version id: publish @@ -202,6 +221,15 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} + - uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TF_VERSION }} + + - uses: gruntwork-io/terragrunt-action@v3 + with: + tg_version: ${{ env.TG_VERSION }} + tf_path: terraform + - name: Set Lambda deploy paths id: lambda_paths shell: bash @@ -213,19 +241,18 @@ jobs: echo "lambda_appspec_key=$lambda_appspec_key" >> $GITHUB_OUTPUT - name: Get cost explorer infra - uses: ./.github/actions/terragrunt id: get-infra - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/cost_explorer - tg_action: init + working-directory: infra/live/${{ inputs.environment }}/aws/cost_explorer + shell: bash + run: | + echo "lambda_function_name=$(terragrunt output -raw lambda_function_name)" >> "$GITHUB_OUTPUT" + echo "lambda_alias_name=$(terragrunt output -raw lambda_alias_name)" >> "$GITHUB_OUTPUT" - name: Get infra detail id: get_infra_detail - env: - TG_OUTPUTS: ${{ steps.get-infra.outputs.tg_outputs }} run: | - echo "lambda_function_name=$(echo $TG_OUTPUTS | jq -r '.lambda_function_name.value')" >> $GITHUB_OUTPUT - echo "lambda_alias_name=$(echo $TG_OUTPUTS | jq -r '.lambda_alias_name.value')" >> $GITHUB_OUTPUT + echo "lambda_function_name=${{ steps.get-infra.outputs.lambda_function_name }}" >> $GITHUB_OUTPUT + echo "lambda_alias_name=${{ steps.get-infra.outputs.lambda_alias_name }}" >> $GITHUB_OUTPUT - name: Publish new lambda version id: publish diff --git a/.github/workflows/shared_get_modules.yml b/.github/workflows/shared_get_modules.yml deleted file mode 100644 index a2f49cc..0000000 --- a/.github/workflows/shared_get_modules.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: Shared Get Modules - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - infra_version: - description: "Version of infrastructure (terraform) to inspect" - required: true - type: string - ignore_shared_artifact_modules: - description: "Whether to exclude shared artifact modules such as code_bucket from emitted waves" - required: false - type: boolean - default: false - ignore_oidc_module: - description: "Whether to exclude the oidc module from emitted waves" - required: false - type: boolean - default: false - show_wave_summary: - description: "Whether to write a Terragrunt wave overview to the GitHub step summary" - required: false - type: boolean - default: true - wave_summary_title: - description: "Title for the Terragrunt wave step summary" - required: false - type: string - default: "Terragrunt Wave Summary" - wave_summary_note: - description: "Optional note to include below the Terragrunt wave step summary heading" - required: false - type: string - default: "" - wave_summary_order: - description: "Display order for wave summary rows: forward or reverse" - required: false - type: string - default: "forward" - show_wave_json: - description: "Whether to include the raw wave JSON in the GitHub step summary" - required: false - type: boolean - default: false - outputs: - waves_json: - description: "Dependency-safe Terragrunt wave matrix JSON" - value: ${{ jobs.generate_waves.outputs.waves_json }} - wave_0_modules: - description: "Wave 0 module array" - value: ${{ jobs.generate_waves.outputs.wave_0_modules }} - wave_1_modules: - description: "Wave 1 module array" - value: ${{ jobs.generate_waves.outputs.wave_1_modules }} - wave_2_modules: - description: "Wave 2 module array" - value: ${{ jobs.generate_waves.outputs.wave_2_modules }} - wave_3_modules: - description: "Wave 3 module array" - value: ${{ jobs.generate_waves.outputs.wave_3_modules }} - -permissions: - id-token: write - contents: read - actions: read - -env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role - AWS_REGION: ${{ vars.AWS_REGION }} - -jobs: - generate_waves: - name: Summary - runs-on: ubuntu-latest - outputs: - waves_json: ${{ steps.filtered_waves.outputs.waves_json }} - wave_0_modules: ${{ steps.wave_outputs.outputs.wave_0_modules }} - wave_1_modules: ${{ steps.wave_outputs.outputs.wave_1_modules }} - wave_2_modules: ${{ steps.wave_outputs.outputs.wave_2_modules }} - wave_3_modules: ${{ steps.wave_outputs.outputs.wave_3_modules }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Render Terragrunt graph - id: tg_graph - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws - tg_action: graph - - - name: Convert graph output to compact JSON - id: tg_graph_json - uses: ./.github/actions/just - env: - TG_GRAPH_OUTPUT: ${{ steps.tg_graph.outputs.tg_graph_output }} - with: - justfile_path: scripts/ci/justfile - just_action: tg-graph-output-to-json ${{ inputs.environment }} - - - name: Build wave matrix JSON - id: waves_json - uses: ./.github/actions/just - env: - TG_GRAPH_JSON: ${{ steps.tg_graph_json.outputs.just_outputs }} - with: - justfile_path: scripts/ci/justfile - just_action: tg-graph-json-to-waves - - - name: Optionally exclude modules from wave matrix - id: filtered_waves - shell: bash - env: - RAW_WAVES_JSON: ${{ steps.waves_json.outputs.just_outputs }} - IGNORE_SHARED_ARTIFACT_MODULES: ${{ inputs.ignore_shared_artifact_modules }} - IGNORE_OIDC_MODULE: ${{ inputs.ignore_oidc_module }} - run: | - echo "waves_json=$(jq -c ' - if $ignore_shared_artifact_modules then - map(.modules |= map(select(. != "code_bucket"))) - else - . - end - | if $ignore_oidc_module then - map(.modules |= map(select(. != "oidc"))) - else - . - end - | map(select(.modules | length > 0)) - | to_entries - | map({wave: .key, modules: .value.modules}) - ' --argjson ignore_shared_artifact_modules "$IGNORE_SHARED_ARTIFACT_MODULES" --argjson ignore_oidc_module "$IGNORE_OIDC_MODULE" <<<"$RAW_WAVES_JSON")" >> "$GITHUB_OUTPUT" - - - name: Expose wave module arrays - id: wave_outputs - shell: bash - env: - WAVES_JSON: ${{ steps.filtered_waves.outputs.waves_json }} - run: | - echo "wave_0_modules=$(jq -c '.[0].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" - echo "wave_1_modules=$(jq -c '.[1].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" - echo "wave_2_modules=$(jq -c '.[2].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" - echo "wave_3_modules=$(jq -c '.[3].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" - - - name: Summary - if: ${{ inputs.show_wave_summary }} - shell: bash - env: - ENVIRONMENT: ${{ inputs.environment }} - INFRA_VERSION: ${{ inputs.infra_version }} - SUMMARY_TITLE: ${{ inputs.wave_summary_title }} - SUMMARY_NOTE: ${{ inputs.wave_summary_note }} - SUMMARY_ORDER: ${{ inputs.wave_summary_order }} - SHOW_WAVE_JSON: ${{ inputs.show_wave_json }} - WAVES_JSON: ${{ steps.filtered_waves.outputs.waves_json }} - run: | - set -euo pipefail - - waves_json="${WAVES_JSON:-[]}" - if [[ "$SUMMARY_ORDER" == "reverse" ]]; then - summary_waves_json="$(jq -c 'reverse' <<<"$waves_json")" - else - summary_waves_json="$waves_json" - fi - - { - echo "## $SUMMARY_TITLE" - echo - echo "- \`environment\`: \`$ENVIRONMENT\`" - echo "- \`infra_version\`: \`$INFRA_VERSION\`" - echo "- \`waves\`: \`$(jq -r 'length' <<<"$waves_json")\`" - echo "- \`modules\`: \`$(jq -r 'map(.modules | length) | add // 0' <<<"$waves_json")\`" - echo - if [[ -n "$SUMMARY_NOTE" ]]; then - echo "$SUMMARY_NOTE" - echo - fi - echo "| Wave | Modules |" - echo "| --- | --- |" - jq -r ' - if length == 0 then - ["| none | No modules selected |"] - else - map( - .wave as $wave - | (.modules | map("`" + . + "`") | join(", ")) as $modules - | "| \($wave) | \($modules) |" - ) - end - | .[] - ' <<<"$summary_waves_json" - - if [[ "$SHOW_WAVE_JSON" == "true" ]]; then - echo - echo "
" - echo "Wave JSON" - echo - echo '```json' - jq . <<<"$waves_json" - echo '```' - echo - echo "
" - fi - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/shared_infra_apply_from_plan.yml b/.github/workflows/shared_infra_apply_from_plan.yml deleted file mode 100644 index 18c3f07..0000000 --- a/.github/workflows/shared_infra_apply_from_plan.yml +++ /dev/null @@ -1,274 +0,0 @@ -name: Shared Infra Apply From Plan - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - plan_artifact_run_id: - description: "Workflow run ID used to recover metadata and plan artifacts" - required: true - type: string - -concurrency: - group: infra-mutate-${{ inputs.environment }} - cancel-in-progress: false - -permissions: - id-token: write - contents: read - actions: read - -env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role - AWS_REGION: ${{ vars.AWS_REGION }} - -jobs: - metadata: - name: Saved Plan - runs-on: ubuntu-latest - outputs: - infra_version: ${{ steps.read_metadata.outputs.infra_version }} - waves_json: ${{ steps.read_metadata.outputs.waves_json }} - wave_0_modules: ${{ steps.read_metadata.outputs.wave_0_modules }} - wave_1_modules: ${{ steps.read_metadata.outputs.wave_1_modules }} - wave_2_modules: ${{ steps.read_metadata.outputs.wave_2_modules }} - steps: - - uses: actions/checkout@v6 - - - name: Download plan metadata artifact - uses: actions/download-artifact@v7 - with: - name: infra-plan-metadata - github-token: ${{ github.token }} - run-id: ${{ inputs.plan_artifact_run_id }} - path: . - - - name: Download saved plan metadata artifacts - uses: actions/download-artifact@v7 - with: - pattern: terragrunt-plan-${{ inputs.environment }}-* - github-token: ${{ github.token }} - run-id: ${{ inputs.plan_artifact_run_id }} - path: plan-artifacts - - - name: Check plan metadata artifact - shell: bash - run: | - if [ ! -f plan-metadata.json ]; then - echo "::error title=Missing plan metadata artifact::Expected artifact 'infra-plan-metadata' containing 'plan-metadata.json' from workflow run '${{ inputs.plan_artifact_run_id }}', but it was not downloaded." - exit 1 - fi - - - name: Print recovered plan metadata JSON - shell: bash - run: | - echo "=== plan-metadata.json ===" - cat plan-metadata.json - echo "=== downloaded plan metadata files ===" - find plan-artifacts -name terragrunt.plan.meta.json -print | sort - - - name: Filter saved plan waves to changed modules - id: filter_waves - uses: ./.github/actions/just - env: - PLAN_METADATA_FILE: plan-metadata.json - PLAN_ARTIFACTS_DIR: plan-artifacts - with: - justfile_path: scripts/ci/justfile - just_action: infra-plan-filter-waves-by-changes - - - name: Read plan metadata - id: read_metadata - shell: bash - env: - FILTERED_WAVES_JSON: ${{ steps.filter_waves.outputs.just_outputs }} - run: | - set -euo pipefail - - echo "infra_version=$(jq -r '.infra_version' plan-metadata.json)" >> "$GITHUB_OUTPUT" - filtered_waves_json="$(printf '%s\n' "$FILTERED_WAVES_JSON" | jq -c .)" - - echo "waves_json=$filtered_waves_json" >> "$GITHUB_OUTPUT" - echo "wave_0_modules=$(jq -c '.[0].modules // []' <<<"$filtered_waves_json")" >> "$GITHUB_OUTPUT" - echo "wave_1_modules=$(jq -c '.[1].modules // []' <<<"$filtered_waves_json")" >> "$GITHUB_OUTPUT" - echo "wave_2_modules=$(jq -c '.[2].modules // []' <<<"$filtered_waves_json")" >> "$GITHUB_OUTPUT" - - - name: Print filtered apply waves and write summary - shell: bash - env: - ENVIRONMENT: ${{ inputs.environment }} - PLAN_ARTIFACT_RUN_ID: ${{ inputs.plan_artifact_run_id }} - INFRA_VERSION: ${{ steps.read_metadata.outputs.infra_version }} - WAVES_JSON: ${{ steps.read_metadata.outputs.waves_json }} - WAVE_0_MODULES: ${{ steps.read_metadata.outputs.wave_0_modules }} - WAVE_1_MODULES: ${{ steps.read_metadata.outputs.wave_1_modules }} - WAVE_2_MODULES: ${{ steps.read_metadata.outputs.wave_2_modules }} - run: | - set -euo pipefail - - echo "=== filtered apply waves ===" - printf '%s\n' "$WAVES_JSON" | jq . - echo "wave_0_modules=$WAVE_0_MODULES" - echo "wave_1_modules=$WAVE_1_MODULES" - echo "wave_2_modules=$WAVE_2_MODULES" - - { - echo "## Apply From Plan Summary" - echo - echo "- \`environment\`: \`$ENVIRONMENT\`" - echo "- \`infra_version\`: \`$INFRA_VERSION\`" - echo "- \`plan_artifact_run_id\`: \`$PLAN_ARTIFACT_RUN_ID\`" - echo "- \`waves\`: \`$(jq -r 'length' <<<"$WAVES_JSON")\`" - echo "- \`modules_to_apply\`: \`$(jq -r 'map(.modules | length) | add // 0' <<<"$WAVES_JSON")\`" - echo - echo "Filtered to modules whose saved plan metadata reported changes." - echo - echo "| Wave | Modules To Apply |" - echo "| --- | --- |" - jq -r ' - if length == 0 then - ["| none | No modules selected |"] - else - map( - .wave as $wave - | (.modules | map("`" + . + "`") | join(", ")) as $modules - | "| \($wave) | \($modules) |" - ) - end - | .[] - ' <<<"$WAVES_JSON" - } >> "$GITHUB_STEP_SUMMARY" - - wave_0: - needs: metadata - name: 0 / ${{ matrix.module }} - if: ${{ needs.metadata.outputs.wave_0_modules != '[]' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.metadata.outputs.wave_0_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ needs.metadata.outputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Download saved plan artifact - uses: actions/download-artifact@v7 - with: - name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} - github-token: ${{ github.token }} - run-id: ${{ inputs.plan_artifact_run_id }} - path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - - - name: Print saved plan files - shell: bash - run: | - module_dir="infra/live/${{ inputs.environment }}/aws/${{ matrix.module }}" - echo "=== downloaded saved plan files ===" - ls -la "$module_dir" - echo "=== saved plan metadata ===" - cat "$module_dir/terragrunt.plan.meta.json" - - - name: Apply ${{ matrix.module }} infra from saved plan - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply_plan - - wave_1: - needs: - - metadata - - wave_0 - name: 1 / ${{ matrix.module }} - if: ${{ always() && needs.metadata.outputs.wave_1_modules != '[]' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.metadata.outputs.wave_1_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ needs.metadata.outputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Download saved plan artifact - uses: actions/download-artifact@v7 - with: - name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} - github-token: ${{ github.token }} - run-id: ${{ inputs.plan_artifact_run_id }} - path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - - - name: Print saved plan files - shell: bash - run: | - module_dir="infra/live/${{ inputs.environment }}/aws/${{ matrix.module }}" - echo "=== downloaded saved plan files ===" - ls -la "$module_dir" - echo "=== saved plan metadata ===" - cat "$module_dir/terragrunt.plan.meta.json" - - - name: Apply ${{ matrix.module }} infra from saved plan - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply_plan - - wave_2: - needs: - - metadata - - wave_1 - name: 2 / ${{ matrix.module }} - if: ${{ always() && needs.metadata.outputs.wave_2_modules != '[]' && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.metadata.outputs.wave_2_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ needs.metadata.outputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Download saved plan artifact - uses: actions/download-artifact@v7 - with: - name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} - github-token: ${{ github.token }} - run-id: ${{ inputs.plan_artifact_run_id }} - path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - - - name: Print saved plan files - shell: bash - run: | - module_dir="infra/live/${{ inputs.environment }}/aws/${{ matrix.module }}" - echo "=== downloaded saved plan files ===" - ls -la "$module_dir" - echo "=== saved plan metadata ===" - cat "$module_dir/terragrunt.plan.meta.json" - - - name: Apply ${{ matrix.module }} infra from saved plan - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply_plan diff --git a/.github/workflows/shared_infra_apply_no_plan.yml b/.github/workflows/shared_infra_apply_no_plan.yml deleted file mode 100644 index 1a88568..0000000 --- a/.github/workflows/shared_infra_apply_no_plan.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Shared Infra Apply No Plan - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - infra_version: - description: "Version of infrastructure (terraform) to be applied" - required: true - type: string - -concurrency: - group: infra-mutate-${{ inputs.environment }} - cancel-in-progress: false - -permissions: - id-token: write - contents: read - actions: read - -env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role - AWS_REGION: ${{ vars.AWS_REGION }} - TG_ACTION_LABEL: "Apply" - -jobs: - generate_waves: - name: Waves - uses: ./.github/workflows/shared_get_modules.yml - with: - environment: ${{ inputs.environment }} - infra_version: ${{ inputs.infra_version }} - - wave_0: - needs: generate_waves - name: 0 / ${{ matrix.module }} - if: ${{ needs.generate_waves.outputs.wave_0_modules != '[]' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.generate_waves.outputs.wave_0_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply - - wave_1: - needs: - - generate_waves - - wave_0 - name: 1 / ${{ matrix.module }} - if: ${{ always() && needs.generate_waves.outputs.wave_1_modules != '[]' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.generate_waves.outputs.wave_1_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply - - wave_2: - needs: - - generate_waves - - wave_1 - name: 2 / ${{ matrix.module }} - if: ${{ always() && needs.generate_waves.outputs.wave_2_modules != '[]' && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - module: ${{ fromJson(needs.generate_waves.outputs.wave_2_modules) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - tg_action: apply diff --git a/.github/workflows/shared_infra_plan.yml b/.github/workflows/shared_infra_plan.yml deleted file mode 100644 index d5903b7..0000000 --- a/.github/workflows/shared_infra_plan.yml +++ /dev/null @@ -1,268 +0,0 @@ -name: Shared Infra Plan - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - infra_version: - description: "Version of infrastructure (terraform) to be planned" - required: true - type: string - outputs: - plan_artifact_run_id: - description: "Workflow run ID for the saved plan artifacts and metadata" - value: ${{ jobs.plan_context.outputs.plan_artifact_run_id }} - -concurrency: - group: infra-plan-${{ inputs.environment }} - cancel-in-progress: false - -permissions: - id-token: write - contents: read - actions: read - -env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role - AWS_REGION: ${{ vars.AWS_REGION }} - TG_ACTION_LABEL: "Plan" - -jobs: - generate_waves: - name: Waves - uses: ./.github/workflows/shared_get_modules.yml - with: - environment: ${{ inputs.environment }} - infra_version: ${{ inputs.infra_version }} - show_wave_summary: false - - metadata: - name: Save Plan Metadata - needs: generate_waves - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - name: Write plan metadata from workflow inputs - shell: bash - run: | - cat > plan-metadata.json <> "$GITHUB_OUTPUT" - echo "plan_artifact_run_id=$PLAN_ARTIFACT_RUN_ID" - { - echo "## Infra Plan Context" - echo - echo "- \`plan_artifact_run_id\`: \`$PLAN_ARTIFACT_RUN_ID\`" - echo "- Saved plan artifacts are time-limited. Apply from this plan before the metadata artifact expires after 14 days." - } >> "$GITHUB_STEP_SUMMARY" - - plan_summary: - name: Plan Summary - needs: - - generate_waves - - metadata - - wave_0 - - wave_1 - - wave_2 - if: ${{ always() && needs.generate_waves.result == 'success' && needs.metadata.result == 'success' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') && (needs.wave_2.result == 'success' || needs.wave_2.result == 'skipped') }} - runs-on: ubuntu-latest - steps: - - name: Download saved plan metadata artifacts - if: ${{ needs.generate_waves.outputs.waves_json != '[]' }} - uses: actions/download-artifact@v7 - with: - pattern: terragrunt-plan-${{ inputs.environment }}-* - path: plan-artifacts - - - name: Write detected change summary - shell: bash - env: - ENVIRONMENT: ${{ inputs.environment }} - WAVES_JSON: ${{ needs.generate_waves.outputs.waves_json }} - run: | - set -euo pipefail - - changed_modules_json='[]' - if [[ -d plan-artifacts ]]; then - changed_modules_json="$( - find plan-artifacts -name terragrunt.plan.meta.json -print \ - | sort \ - | while IFS= read -r meta; do - has_changes="$(jq -r '.has_changes // false' "$meta")" - if [[ "$has_changes" == "true" ]]; then - jq -r '.tg_directory | split("/") | last' "$meta" - fi - done \ - | jq -Rsc 'split("\n") | map(select(length > 0)) | sort' - )" - fi - - { - echo "## Infra Plan Summary" - echo - echo "- \`environment\`: \`$ENVIRONMENT\`" - echo "- \`waves\`: \`$(jq -r 'length' <<<"$WAVES_JSON")\`" - echo "- \`planned_modules\`: \`$(jq -r 'map(.modules | length) | add // 0' <<<"$WAVES_JSON")\`" - echo "- \`modules_to_apply\`: \`$(jq -r 'length' <<<"$changed_modules_json")\`" - echo "- Saved plan artifacts are time-limited. Apply from this plan before the metadata artifact expires after 14 days." - echo - echo "| Wave | Planned Modules |" - echo "| --- | --- |" - jq -r ' - if length == 0 then - ["| none | No modules selected |"] - else - map( - .wave as $wave - | (.modules | map("`" + . + "`") | join(", ")) as $modules - | "| \($wave) | \($modules) |" - ) - end - | .[] - ' <<<"$WAVES_JSON" - echo - - if [[ "$(jq -r 'length' <<<"$changed_modules_json")" -eq 0 ]]; then - echo "No modules have detected plan changes." - else - echo "Modules that will be applied by apply-from-plan:" - echo - jq -r '.[] | "- `" + . + "`"' <<<"$changed_modules_json" - fi - } >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 6ff2aa0..a824533 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # chrispsheehan.com [![Release](https://img.shields.io/github/v/release/chrispsheehan/chrispsheehan.com?display_name=tag&label=Release)](https://github.com/chrispsheehan/chrispsheehan.com/releases) -[![Infra Plan](https://img.shields.io/github/actions/workflow/status/chrispsheehan/chrispsheehan.com/prod_infra_plan.yml?label=Infra%20Plan)](https://github.com/chrispsheehan/chrispsheehan.com/actions/workflows/prod_infra_plan.yml) -[![Infra Apply](https://img.shields.io/github/actions/workflow/status/chrispsheehan/chrispsheehan.com/prod_infra_apply_from_plan.yml?label=Infra%20Apply)](https://github.com/chrispsheehan/chrispsheehan.com/actions/workflows/prod_infra_apply_from_plan.yml) +[![Infra Plan](https://img.shields.io/github/actions/workflow/status/chrispsheehan/chrispsheehan.com/infra_plan.yml?label=Infra%20Plan)](https://github.com/chrispsheehan/chrispsheehan.com/actions/workflows/infra_plan.yml) +[![Infra Apply](https://img.shields.io/github/actions/workflow/status/chrispsheehan/chrispsheehan.com/infra_apply.yml?label=Infra%20Apply)](https://github.com/chrispsheehan/chrispsheehan.com/actions/workflows/infra_apply.yml) [![Code Deploy](https://img.shields.io/github/actions/workflow/status/chrispsheehan/chrispsheehan.com/prod_code_deploy.yml?label=Code%20Deploy)](https://github.com/chrispsheehan/chrispsheehan.com/actions/workflows/prod_code_deploy.yml) CloudFront-backed static frontend for [`chrispsheehan.com`](https://chrispsheehan.com), scaffolded from the [`aws-terragrunt-starter`](https://github.com/chrispsheehan/aws-terragrunt-starter) @@ -84,8 +84,12 @@ Workflows assume roles named: Development deploys target `dev.chrispsheehan.com` and build from the current commit. -- `Dev Infra Plan` and `Dev Infra Apply No Plan` create or update the AWS - infrastructure. +- `Infra Bootstrap` is the first-time environment bring-up path for `dev` and + `prod`. It applies `aws/code_bucket` first so the shared bootstrap Lambda zip + exists before the Lambda stacks are created. +- `Infra Plan` writes saved Terragrunt plans for the selected environment. +- `Infra Apply From Plan` reuses a prior plan run and applies only modules + whose saved plan JSON reported changes. - `Dev Code Deploy` builds `frontend.zip`, `log_processor.zip`, and `cost_explorer.zip`, uploads them to the dev code bucket, syncs the frontend artifact to the S3 origin bucket, refreshes CloudFront in a separate CI job, diff --git a/infra/README.md b/infra/README.md index 86fafdb..23218fb 100644 --- a/infra/README.md +++ b/infra/README.md @@ -64,9 +64,68 @@ and `lambdas//*.zip` artifacts later. ```sh terraform fmt -recursive terragrunt hclfmt +just tg dev aws/code_bucket apply just tg dev aws/frontend plan just tg-all dev plan ``` Plans require AWS credentials, access to the configured remote state bucket, and Route53/CloudFront permissions. + +## Terragrunt Graph Helpers + +Use these commands when debugging stack ordering or local Terragrunt graph +output. + +Terragrunt derives account-scoped names from `AWS_ACCOUNT_ID`. The repo-root +`just tg`, `just tg-all`, and `just tg-graph` recipes resolve it with +`aws sts get-caller-identity`. Set it yourself only when running Terragrunt +directly or using non-root justfiles: + +```sh +export AWS_ACCOUNT_ID= +``` + +If you only need the raw Terragrunt graph output: + +```sh +just tg-graph dev > graph.txt +``` + +That runs: + +```sh +cd infra/live/dev/aws +terragrunt run-all graph-dependencies \ + --terragrunt-non-interactive \ + --terragrunt-include-external-dependencies +``` + +For saved-plan runs: + +```sh +just tg-all dev plan +just tg-all dev show +``` + +That writes one `terragrunt.tfplan` file and one `terragrunt.plan.json` file +per live stack directory under `infra/live//**`. + +To list the modules that produced `terragrunt.plan.json` for one environment: + +```sh +just --justfile scripts/ci/justfile plan-json-files-list-modules dev +``` + +To build the per-module `has_changes` summary from that list: + +```sh +MODULE_PATHS_JSON="$(just --justfile scripts/ci/justfile plan-json-files-list-modules dev)" \ +just --justfile scripts/ci/justfile plan-json-files-to-change-summary dev +``` + +To apply that same saved plan later, restore the plan artifacts and run with: + +```sh +TG_USE_SAVED_PLAN=true terragrunt run-all apply --terragrunt-non-interactive +``` diff --git a/infra/root.hcl b/infra/root.hcl index da7a13b..8a6755a 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -16,16 +16,17 @@ locals { project_name = element(split("/", local.github_repo), 1) project_slug = replace(local.project_name, ".", "-") - aws_region = local.global_vars.inputs.aws_region - base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_slug}" - deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" - deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" - state_bucket = "${local.base_reference}-tfstate" - state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" - state_locking_mode = "s3" + aws_region = local.global_vars.inputs.aws_region + base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_slug}" + deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" + deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" + state_bucket = "${local.base_reference}-tfstate" + state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" # separate shared artifact resources when dev, otherwise ci - artifact_base = local.environment == "dev" ? "${local.base_reference}-${local.environment}" : "${local.base_reference}-ci" - code_bucket = "${local.artifact_base}-code" + artifact_base = local.environment == "dev" ? "${local.base_reference}-${local.environment}" : "${local.base_reference}-ci" + code_bucket = "${local.artifact_base}-code" + use_saved_plan = get_env("TG_USE_SAVED_PLAN", "false") == "true" + saved_plan_path = "${get_terragrunt_dir()}/terragrunt.tfplan" } terraform { @@ -35,6 +36,35 @@ terraform { "bash", "-c", "echo STATE:${local.state_bucket}/${local.state_key} LOCKFILE:${local.state_key}.tflock" ] } + + extra_arguments "saved_plan_output" { + commands = ["plan"] + arguments = [ + "-out=${local.saved_plan_path}" + ] + } + + extra_arguments "saved_plan_show_json" { + commands = ["show"] + arguments = [ + "-json", + local.saved_plan_path + ] + } + + extra_arguments "apply_saved_plan" { + commands = ["apply"] + arguments = local.use_saved_plan ? [local.saved_plan_path] : [] + } + + after_hook "write_show_json_file" { + commands = ["show"] + execute = [ + "bash", + "-lc", + "terraform show -json \"${local.saved_plan_path}\" > \"${get_terragrunt_dir()}/terragrunt.plan.json\"" + ] + } } remote_state { @@ -102,7 +132,6 @@ inputs = merge( deploy_role_name = local.deploy_role_name deploy_role_arn = local.deploy_role_arn state_bucket = local.state_bucket - state_locking_mode = local.state_locking_mode code_bucket = local.code_bucket } ) diff --git a/justfile b/justfile index ba9f1e1..8a0a5f1 100644 --- a/justfile +++ b/justfile @@ -185,18 +185,3 @@ tg-graph env provider='aws': --terragrunt-include-external-dependencies \ --terragrunt-log-level error - -# Run tg-graph once locally and feed the raw output through the CI graph and -# wave processors. -tg-graph-waves env provider='aws': - #!/usr/bin/env bash - set -euo pipefail - cd {{justfile_directory()}} - - tg_graph_json="$( - TG_GRAPH_OUTPUT="$(just tg-graph "{{env}}" "{{provider}}")" \ - just --justfile "{{justfile_directory()}}/scripts/ci/justfile" tg-graph-output-to-json "{{env}}" "{{provider}}" - )" - - TG_GRAPH_JSON="$tg_graph_json" \ - just --justfile "{{justfile_directory()}}/scripts/ci/justfile" tg-graph-json-to-waves diff --git a/scripts/ci/justfile b/scripts/ci/justfile index 0753097..f476e7a 100644 --- a/scripts/ci/justfile +++ b/scripts/ci/justfile @@ -1,5 +1,5 @@ # Read-only CI helpers only. -# This file is for discovery, validation, artifact existence checks, and matrix shaping. +# This file is for discovery, validation, and artifact existence checks. # Do not put AWS-mutating rollout or deploy commands here. PROJECT_DIR := `cd ../.. && pwd` @@ -8,302 +8,97 @@ APPSPEC_DIR := `just --justfile ../../justfile --evaluate APPSPEC_DIR` FRONTEND_DIR := `just --justfile ../../justfile --evaluate FRONTEND_DIR` -# Convert raw Terragrunt graph output from TG_GRAPH_OUTPUT into compact JSON. -tg-graph-output-to-json environment provider='aws': +# Build a compact per-module change summary from terragrunt.plan.json files for one environment. +plan-json-files-list-modules environment provider='aws': #!/usr/bin/env bash set -euo pipefail cd "{{PROJECT_DIR}}" - if [[ -z "${TG_GRAPH_OUTPUT:-}" ]]; then - echo "❌ TG_GRAPH_OUTPUT is required for tg-graph-output-to-json." - exit 1 - fi + env_dir="infra/live/{{environment}}/{{provider}}" - tmp_nodes="$(mktemp)" - tmp_edges="$(mktemp)" - trap 'rm -f "$tmp_nodes" "$tmp_edges"' EXIT - - awk -F'"' ' - /->/ && NF >= 4 { - from = $2 - to = $4 - sub(".*/", "", from) - sub(".*/", "", to) - print from "\t" to - next - } - /^[[:space:]]*"/ && /;[[:space:]]*$/ && NF >= 2 { - node = $2 - sub(".*/", "", node) - print node - } - ' <(printf '%s\n' "$TG_GRAPH_OUTPUT") \ - | while IFS= read -r line; do - if [[ "$line" == *$'\t'* ]]; then - printf '%s\n' "$line" >> "$tmp_edges" - elif [[ -n "$line" ]]; then - printf '%s\n' "$line" >> "$tmp_nodes" - fi - done - - { - cat "$tmp_nodes" - awk -F'\t' 'NF >= 2 { print $1; print $2 }' "$tmp_edges" - } \ - | sort -u \ - | while IFS= read -r node; do - [[ -n "$node" ]] || continue - deps="$( - awk -F'\t' -v target="$node" '$1 == target { print $2 }' "$tmp_edges" \ - | sort -u \ - | jq -R . \ - | jq -s -c . - )" - printf '%s\t%s\n' "$node" "$deps" - done \ - | jq -R -s --arg environment "{{environment}}" --arg provider "{{provider}}" ' - split("\n") - | map(select(length > 0)) - | map(split("\t")) - | map(select(length == 2)) - | { - environment: $environment, - provider: $provider, - dependencies: ( - reduce .[] as $pair - ({}; - .[$pair[0]] = ($pair[1] | fromjson) - ) - ) - } - ' \ - | jq -c ' - .dependencies as $deps - | . + { - nodes: ($deps | keys | sort), - edges: ( - $deps - | to_entries - | map(.key as $from | .value[]? | {from: $from, to: .}) - | sort_by(.from, .to) - ) - } - ' - - -# Convert compact Terragrunt graph JSON from TG_GRAPH_JSON into a dependency-safe -# wave matrix JSON array for GitHub Actions. -tg-graph-json-to-waves: - #!/usr/bin/env bash - set -euo pipefail - cd "{{PROJECT_DIR}}" - - if [[ -z "${TG_GRAPH_JSON:-}" ]]; then - echo "❌ TG_GRAPH_JSON is required for tg-graph-json-to-waves." - exit 1 - fi - - jq -cn --argjson graph "$TG_GRAPH_JSON" ' - def build_waves($remaining): - if ($remaining | length) == 0 then - [] - else - ($remaining | to_entries | map(select((.value | length) == 0) | .key) | sort) as $ready - | if ($ready | length) == 0 then - error("Cycle detected in Terragrunt dependency graph.") - else - [{modules: $ready}] + build_waves( - reduce ($remaining | to_entries[]) as $entry - ({}; - if ($ready | index($entry.key)) != null then - . - else - .[$entry.key] = ($entry.value - $ready) - end - ) - ) - end - end; - - ($graph.dependencies // {}) as $deps - | build_waves($deps) - | to_entries - | map({wave: .key, modules: .value.modules}) - ' | jq -c . - - -# Normalize raw Terragrunt waves to the infra plan/apply wave shape. -tg-waves-to-infra-waves: - #!/usr/bin/env bash - set -euo pipefail - cd "{{PROJECT_DIR}}" - - if [[ -z "${RAW_WAVES_JSON:-}" ]]; then - echo "❌ RAW_WAVES_JSON is required for tg-waves-to-infra-waves." + if [[ ! -d "$env_dir" ]]; then + echo "❌ Environment directory not found: $env_dir" exit 1 fi - jq -c ' - map(select(.modules | length > 0)) - | to_entries - | map({wave: .key, modules: .value.modules}) - ' <<<"$RAW_WAVES_JSON" - - -# Filter raw Terragrunt waves to the destroy wave shape. -tg-waves-to-destroy-waves: - #!/usr/bin/env bash - set -euo pipefail - cd "{{PROJECT_DIR}}" - - if [[ -z "${RAW_WAVES_JSON:-}" ]]; then - echo "❌ RAW_WAVES_JSON is required for tg-waves-to-destroy-waves." - exit 1 - fi - - jq -c ' - map(.modules |= map(select(. != "oidc" and . != "code_bucket"))) - | map(select(.modules | length > 0)) - | to_entries - | map({wave: .key, modules: .value.modules}) - ' <<<"$RAW_WAVES_JSON" + find "$env_dir" -mindepth 1 -maxdepth 1 -type d \ + -exec test -f '{}/terragrunt.plan.json' ';' -print \ + | sed "s#^infra/live/{{environment}}/##" \ + | sort \ + | jq -Rsc 'split("\n") | map(select(length > 0))' -# Validate static GitHub workflow wave jobs match generated wave depth. -tg-validate-static-wave-jobs: +# Remove stale saved-plan artifacts for one environment before generating fresh plan/show output. +plan-json-files-clean environment provider='aws': #!/usr/bin/env bash set -euo pipefail cd "{{PROJECT_DIR}}" - if [[ -z "${RAW_WAVES_JSON:-}" ]]; then - echo "❌ RAW_WAVES_JSON is required for tg-validate-static-wave-jobs." - exit 1 - fi + env_dir="infra/live/{{environment}}/{{provider}}" - if [[ -z "${INFRA_WAVES_JSON:-}" ]]; then - echo "❌ INFRA_WAVES_JSON is required for tg-validate-static-wave-jobs." + if [[ ! -d "$env_dir" ]]; then + echo "❌ Environment directory not found: $env_dir" exit 1 fi - if [[ -z "${DESTROY_WAVES_JSON:-}" ]]; then - echo "❌ DESTROY_WAVES_JSON is required for tg-validate-static-wave-jobs." - exit 1 - fi + find "$env_dir" -type f \( -name terragrunt.tfplan -o -name terragrunt.plan.json \) -delete + echo "Cleaned saved plan artifacts under $env_dir" - raw_waves_json="$(jq -c . <<<"$RAW_WAVES_JSON")" - infra_waves_json="$(jq -c . <<<"$INFRA_WAVES_JSON")" - destroy_waves_json="$(jq -c . <<<"$DESTROY_WAVES_JSON")" - - indexes_for_waves() { - jq -c 'length as $n | [range(0; $n)]' <<<"$1" - } - - indexes_for_workflow() { - grep -E '^ wave_[0-9]+:' "$1" \ - | sed -E 's/^ wave_([0-9]+):.*/\1/' \ - | sort -n \ - | jq -Rsc 'split("\n") | map(select(length > 0) | tonumber)' - } - - validate_workflow() { - local label="$1" - local workflow="$2" - local waves_json="$3" - local expected actual missing extra reason - - expected="$(indexes_for_waves "$waves_json")" - actual="$(indexes_for_workflow "$workflow")" - missing="$(jq -cn --argjson expected "$expected" --argjson actual "$actual" '$expected - $actual')" - extra="$(jq -cn --argjson expected "$expected" --argjson actual "$actual" '$actual - $expected')" - - echo "$label expected wave jobs: $expected" - echo "$label actual wave jobs: $actual" - - if [[ "$actual" != "$expected" ]]; then - reason="Wave jobs do not match generated graph." - if [[ "$extra" != "[]" && "$missing" == "[]" ]]; then - reason="Too many static wave jobs." - elif [[ "$missing" != "[]" && "$extra" == "[]" ]]; then - reason="Not enough static wave jobs." - elif [[ "$missing" != "[]" && "$extra" != "[]" ]]; then - reason="Static wave jobs are not contiguous." - fi - { - echo "::error file=$workflow,title=Terragrunt wave jobs out of date::$label: $reason" - echo "Expected contiguous jobs: $expected" - echo "Actual static jobs: $actual" - echo "Not enough wave jobs: $missing" - echo "Too many wave jobs: $extra" - echo - echo "Fix $workflow so its top-level wave_N jobs exactly match $expected." - if [[ "$extra" != "[]" ]]; then - echo "Remove stale wave job(s): $(jq -r 'map("wave_" + tostring) | join(", ")' <<<"$extra")" - fi - if [[ "$missing" != "[]" ]]; then - echo "Add missing wave job(s): $(jq -r 'map("wave_" + tostring) | join(", ")' <<<"$missing")" - fi - echo "Local validation commands: infra/README.md#terragrunt-graph-helpers" - } >&2 - exit 1 - fi - } - - echo "Raw prod waves: $raw_waves_json" - echo "Infra plan/apply waves: $infra_waves_json" - echo "Destroy waves: $destroy_waves_json" - - validate_workflow "Shared infra plan" ".github/workflows/shared_infra_plan.yml" "$infra_waves_json" - validate_workflow "Shared infra apply" ".github/workflows/shared_infra_apply_no_plan.yml" "$infra_waves_json" - validate_workflow "Shared infra apply from plan" ".github/workflows/shared_infra_apply_from_plan.yml" "$infra_waves_json" - validate_workflow "Destroy" ".github/workflows/destroy.yml" "$destroy_waves_json" - - -# Filter saved infra-plan waves down to modules whose terragrunt.plan.meta.json -# reports has_changes=true. -infra-plan-filter-waves-by-changes: +# Build a compact per-module change summary from terragrunt.plan.json files for one environment. +plan-json-files-to-change-summary environment provider='aws': #!/usr/bin/env bash set -euo pipefail cd "{{PROJECT_DIR}}" - if [[ -z "${PLAN_METADATA_FILE:-}" ]]; then - echo "❌ PLAN_METADATA_FILE is required for infra-plan-filter-waves-by-changes." - exit 1 - fi + env_dir="infra/live/{{environment}}/{{provider}}" - if [[ -z "${PLAN_ARTIFACTS_DIR:-}" ]]; then - echo "❌ PLAN_ARTIFACTS_DIR is required for infra-plan-filter-waves-by-changes." + if [[ ! -d "$env_dir" ]]; then + echo "❌ Environment directory not found: $env_dir" exit 1 fi - changed_modules_file="$(mktemp)" - trap 'rm -f "$changed_modules_file"' EXIT - - while IFS= read -r meta; do - has_changes="$(jq -r '.has_changes // false' "$meta")" - module_name="$(jq -r '.tg_directory | split("/") | last' "$meta")" - - if [[ "$has_changes" == "true" && -n "$module_name" && "$module_name" != "null" ]]; then - printf '%s\n' "$module_name" >> "$changed_modules_file" - fi - done < <(find "$PLAN_ARTIFACTS_DIR" -name terragrunt.plan.meta.json -print | sort) - - if [[ -s "$changed_modules_file" ]]; then - changed_modules_json="$( - sort -u "$changed_modules_file" | jq -Rsc 'split("\n") | map(select(length > 0))' - )" + if [[ -n "${MODULE_PATHS_JSON:-}" ]]; then + module_paths_json="$(jq -c . <<<"$MODULE_PATHS_JSON")" else - changed_modules_json='[]' + module_paths_json="$( + just --justfile "{{justfile_directory()}}/justfile" plan-json-files-list-modules "{{environment}}" "{{provider}}" + )" fi - jq -c \ - --argjson changed "$changed_modules_json" \ - ' - (.waves // []) - | map(.modules |= map(select(. as $item | $changed | index($item) != null))) - | map(select(.modules | length > 0)) - ' \ - "$PLAN_METADATA_FILE" + jq -r '.[]' <<<"$module_paths_json" \ + | while IFS= read -r module; do + plan_file="infra/live/{{environment}}/${module}/terragrunt.plan.json" + + jq -c --arg module_name "$module" ' + def is_change_action($action): + $action != "no-op"; + + def has_resource_changes: + any( + (.resource_changes // [])[]; + any( + (.change.actions // [])[]; + is_change_action(.) + ) + ); + + def has_output_changes: + any( + ((.output_changes // {}) | to_entries)[]; + any( + (.value.actions // [])[]; + is_change_action(.) + ) + ); + + { + module: $module_name, + has_changes: (has_resource_changes or has_output_changes) + } + ' "$plan_file" + done \ + | jq -s -c . # Run `tflint` across Terraform module directories. From 244fb6ff3ea3c831618ab7078788e32fb3cb7d86 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Mon, 20 Jul 2026 15:08:01 +0100 Subject: [PATCH 2/3] chore: set s3 state lock mode --- .gitignore | 1 + infra/root.hcl | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 394f2da..8c574ea 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ docker/ # ignore generated files infra/live/*.tf +terragrunt.tfplan dist .astro/ node_modules/ diff --git a/infra/root.hcl b/infra/root.hcl index 8a6755a..3296c18 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -16,12 +16,13 @@ locals { project_name = element(split("/", local.github_repo), 1) project_slug = replace(local.project_name, ".", "-") - aws_region = local.global_vars.inputs.aws_region - base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_slug}" - deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" - deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" - state_bucket = "${local.base_reference}-tfstate" - state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" + aws_region = local.global_vars.inputs.aws_region + base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_slug}" + deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" + deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" + state_bucket = "${local.base_reference}-tfstate" + state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" + state_locking_mode = "s3" # separate shared artifact resources when dev, otherwise ci artifact_base = local.environment == "dev" ? "${local.base_reference}-${local.environment}" : "${local.base_reference}-ci" code_bucket = "${local.artifact_base}-code" @@ -132,6 +133,7 @@ inputs = merge( deploy_role_name = local.deploy_role_name deploy_role_arn = local.deploy_role_arn state_bucket = local.state_bucket + state_locking_mode = local.state_locking_mode code_bucket = local.code_bucket } ) From 94e551f2c6d63ae179ef97d34d17e8fa8afe7f88 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Mon, 20 Jul 2026 15:08:51 +0100 Subject: [PATCH 3/3] fix: no need to deploy code bucker first --- .github/workflows/infra_bootstrap.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/infra_bootstrap.yml b/.github/workflows/infra_bootstrap.yml index 27c48bc..3fcedd0 100644 --- a/.github/workflows/infra_bootstrap.yml +++ b/.github/workflows/infra_bootstrap.yml @@ -56,11 +56,6 @@ jobs: tg_version: ${{ env.TG_VERSION }} tf_path: terraform - - name: Apply code bucket infra - working-directory: infra/live/${{ inputs.environment }} - shell: bash - run: terragrunt run-all apply --terragrunt-non-interactive --terragrunt-include-dir aws/code_bucket - - name: Apply all bootstrap infra working-directory: infra/live/${{ inputs.environment }} shell: bash