From e98ae863dcf42b06d953951bf8f9f5ce2ee83667 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:08:30 +0100 Subject: [PATCH 1/4] Add per-environment assets bucket Rolling ECS deploys leave a window where old and new tasks serve traffic together. Requests for fingerprinted assets from a new release can be routed to an old task, which cannot serve them and returns 404. Add an S3 bucket to each environment to hold assets from every release, along with an origin access control so CloudFront can read from it. The bucket name is exported so the deploy pipelines can sync assets into it before deploying to ECS. --- .../deployments/forms/environment/outputs.tf | 4 ++ infra/modules/cloudfront/outputs.tf | 4 ++ infra/modules/cloudfront/s3.tf | 43 +++++++++++++++++++ infra/modules/environment/outputs.tf | 4 ++ 4 files changed, 55 insertions(+) diff --git a/infra/deployments/forms/environment/outputs.tf b/infra/deployments/forms/environment/outputs.tf index 3fc681044..7d93ff624 100644 --- a/infra/deployments/forms/environment/outputs.tf +++ b/infra/deployments/forms/environment/outputs.tf @@ -51,6 +51,10 @@ output "cloudfront_secret" { sensitive = true } +output "assets_bucket_name" { + value = module.environment.assets_bucket_name +} + output "eventbridge_dead_letter_queue_arn" { value = module.environment.eventbridge_dead_letter_queue_arn } diff --git a/infra/modules/cloudfront/outputs.tf b/infra/modules/cloudfront/outputs.tf index 7e86332fd..51a23f619 100644 --- a/infra/modules/cloudfront/outputs.tf +++ b/infra/modules/cloudfront/outputs.tf @@ -22,3 +22,7 @@ output "cloudfront_secret" { value = random_password.cloudfront_secret.result sensitive = true } + +output "assets_bucket_name" { + value = module.assets_bucket.name +} diff --git a/infra/modules/cloudfront/s3.tf b/infra/modules/cloudfront/s3.tf index 4944e9e47..e2c86a6ba 100644 --- a/infra/modules/cloudfront/s3.tf +++ b/infra/modules/cloudfront/s3.tf @@ -5,6 +5,49 @@ module "error_page_bucket" { name = "govuk-forms-${var.env_name}-error-page" } +locals { + assets_bucket_name = "govuk-forms-${var.env_name}-assets" +} + +module "assets_bucket" { + source = "../secure-bucket" + + name = local.assets_bucket_name + access_logging_enabled = true + send_access_logs_to_cyber = var.send_logs_to_cyber + + extra_bucket_policies = [data.aws_iam_policy_document.assets_bucket_cloudfront_read.json] +} + +data "aws_iam_policy_document" "assets_bucket_cloudfront_read" { + statement { + sid = "AllowCloudFrontToReadAssets" + principals { + type = "Service" + identifiers = ["cloudfront.amazonaws.com"] + } + actions = [ + "s3:GetObject", + ] + resources = [ + "arn:aws:s3:::${local.assets_bucket_name}/*" + ] + condition { + test = "StringEquals" + variable = "AWS:SourceArn" + values = [aws_cloudfront_distribution.main.arn] + } + } +} + +resource "aws_cloudfront_origin_access_control" "assets" { + name = "assets-${var.env_name}" + description = "Allow the CloudFront distribution to access the assets bucket" + origin_access_control_origin_type = "s3" + signing_behavior = "always" + signing_protocol = "sigv4" +} + locals { content_type_map = { "js" = "application/json" diff --git a/infra/modules/environment/outputs.tf b/infra/modules/environment/outputs.tf index 280410e44..3310df861 100644 --- a/infra/modules/environment/outputs.tf +++ b/infra/modules/environment/outputs.tf @@ -75,6 +75,10 @@ output "cloudfront_secret" { sensitive = true } +output "assets_bucket_name" { + value = module.cloudfront[0].assets_bucket_name +} + output "eventbridge_dead_letter_queue_arn" { value = aws_sqs_queue.event_bridge_dlq.arn } From feb62a57172db26eb1fafeb6a53e245eb764f23e Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:09:23 +0100 Subject: [PATCH 2/4] Allow deployer to manage assets bucket and OAC The deployer role needs to create the per-environment assets bucket and CloudFront origin access control, and the deploy pipelines use the same role to sync assets into the bucket. --- infra/modules/deployer-access/policy.tf | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/infra/modules/deployer-access/policy.tf b/infra/modules/deployer-access/policy.tf index 87f76d876..7a10aba36 100644 --- a/infra/modules/deployer-access/policy.tf +++ b/infra/modules/deployer-access/policy.tf @@ -187,6 +187,17 @@ data "aws_iam_policy_document" "cloudfront" { ] sid = "ManageCloudfrontDistribution" } + + statement { + actions = [ + "cloudfront:*OriginAccessControl*" + ] + effect = "Allow" + resources = [ + "arn:aws:cloudfront::${var.account_id}:origin-access-control/*" + ] + sid = "ManageCloudfrontOriginAccessControl" + } } data "aws_iam_policy_document" "cloudwatch" { @@ -927,7 +938,9 @@ data "aws_iam_policy_document" "s3" { "arn:aws:s3:::*-alb-access-logs-access-logs/*", "arn:aws:s3:::govuk-forms-alb-logs-${var.environment_name}*", - "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*" + "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*", + + "arn:aws:s3:::govuk-forms-${var.environment_name}-assets*" ] sid = "ManageS3" } From 7efbfcb68aa4f910d0338d69d144590deb581c87 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:12:52 +0100 Subject: [PATCH 3/4] Sync assets to S3 during container deploys Copy the built assets out of the container image and upload them to the environment assets bucket before the new task definition is deployed to ECS. This makes the assets for a release available in S3 before any task serves pages referencing them. Assets from previous releases are left in the bucket so tasks which have not yet been replaced keep working during a deployment. --- .../buildspecs/sync-assets/sync-assets.yml | 30 +++++++++++++++++++ .../pipelines/deploy-forms-admin-container.tf | 21 +++++++++++++ .../deploy-forms-product-page-container.tf | 21 +++++++++++++ .../deploy-forms-runner-container.tf | 21 +++++++++++++ .../forms/pipelines/sync-assets.tf | 18 +++++++++++ 5 files changed, 111 insertions(+) create mode 100644 infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml create mode 100644 infra/deployments/forms/pipelines/sync-assets.tf diff --git a/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml b/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml new file mode 100644 index 000000000..3bdbd33db --- /dev/null +++ b/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml @@ -0,0 +1,30 @@ +version: 0.2 +phases: + pre_build: + commands: + # Check Image URI is not the default pipeline value + - | + if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then + echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\"" + exit 1 + fi + + - echo "Log in to Amazon ECR repository ${CONTAINER_REGISTRY}" + - aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin "${CONTAINER_REGISTRY}" + build: + commands: + # Copy the built assets out of the container image + - docker pull "${IMAGE_URI}" + - CONTAINER_ID=$(docker create "${IMAGE_URI}") + - docker cp "${CONTAINER_ID}:/app/public/assets" ./assets + - docker rm "${CONTAINER_ID}" + + # Upload the assets so they are available before the new tasks + # serve any pages which reference them. Assets from previous + # releases are deliberately left in place so tasks which have not + # yet been replaced keep working during the deployment. + - | + aws s3 sync ./assets "s3://${ASSETS_BUCKET}/assets/" \ + --exclude ".vite/*" \ + --cache-control "public, max-age=300" \ + --no-progress diff --git a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf index 468ce695e..d266f6f68 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf @@ -189,6 +189,27 @@ resource "aws_codepipeline" "deploy_admin_container" { } } + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf index 12d881630..9a56ec318 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf @@ -146,6 +146,27 @@ resource "aws_codepipeline" "deploy_product_pages_container" { stage { name = "deploy-to-ecs" + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf b/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf index 1b8fa6689..d80bf6c60 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf @@ -182,6 +182,27 @@ resource "aws_codepipeline" "deploy_runner_container" { } } + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/sync-assets.tf b/infra/deployments/forms/pipelines/sync-assets.tf new file mode 100644 index 000000000..b127d82f4 --- /dev/null +++ b/infra/deployments/forms/pipelines/sync-assets.tf @@ -0,0 +1,18 @@ +# A single project is shared by the app deploy pipelines. All of the +# apps store their assets in the same bucket: every file is fingerprinted +# with a content hash, so names cannot collide. +module "sync_assets" { + source = "../../../modules/code-build-build" + project_name = "sync_assets_${var.environment_name}" + project_description = "Sync assets from a container image to the assets bucket" + environment = var.environment_name + artifact_store_arn = module.artifact_bucket.arn + buildspec = file("${path.root}/buildspecs/sync-assets/sync-assets.yml") + log_group_name = "codebuild/sync_assets_${var.environment_name}" + codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn + + environment_variables = { + ASSETS_BUCKET = data.terraform_remote_state.forms_environment.outputs.assets_bucket_name + CONTAINER_REGISTRY = var.container_registry + } +} From 7c7845cbe2fdf575108372972f3c982df786beab Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:18:02 +0100 Subject: [PATCH 4/4] Add option to serve assets from S3 When enabled, CloudFront serves /assets/* from the assets bucket instead of the applications. The bucket holds assets from every release, so requests can no longer fail when a deployment leaves tasks from two releases serving traffic at once. The option is off by default and should only be enabled for an environment once its deploy pipelines have populated the bucket for all three applications. Serving from S3 cannot fall back to the ALB via an origin group: CloudFront must not forward the viewer Host header to an S3 origin, and without it the ALB listener rules cannot route the request. --- infra/deployments/forms/environment/main.tf | 2 ++ infra/deployments/forms/inputs.tf | 1 + infra/modules/cloudfront/cloudfront.tf | 18 ++++++++++++++++-- infra/modules/cloudfront/variables.tf | 6 ++++++ infra/modules/environment/cloudfront.tf | 1 + infra/modules/environment/variables.tf | 6 ++++++ infra/modules/secure-bucket/outputs.tf | 4 ++++ 7 files changed, 36 insertions(+), 2 deletions(-) diff --git a/infra/deployments/forms/environment/main.tf b/infra/deployments/forms/environment/main.tf index bcb755f7f..ec311c03c 100644 --- a/infra/deployments/forms/environment/main.tf +++ b/infra/deployments/forms/environment/main.tf @@ -18,4 +18,6 @@ module "environment" { enable_shield_advanced_healthchecks = var.environmental_settings.enable_shield_advanced_healthchecks scheduled_smoke_tests_settings = var.scheduled_smoke_tests_settings + + serve_assets_from_s3 = var.environmental_settings.serve_assets_from_s3 } diff --git a/infra/deployments/forms/inputs.tf b/infra/deployments/forms/inputs.tf index 20fb1dfe3..2861e47ee 100644 --- a/infra/deployments/forms/inputs.tf +++ b/infra/deployments/forms/inputs.tf @@ -204,6 +204,7 @@ variable "environmental_settings" { allow_pagerduty_alerts = bool redis_multi_az_enabled = bool enable_advanced_database_insights = bool + serve_assets_from_s3 = optional(bool, false) }) } diff --git a/infra/modules/cloudfront/cloudfront.tf b/infra/modules/cloudfront/cloudfront.tf index 611425b74..23d7b25d1 100644 --- a/infra/modules/cloudfront/cloudfront.tf +++ b/infra/modules/cloudfront/cloudfront.tf @@ -89,6 +89,12 @@ resource "aws_cloudfront_distribution" "main" { } } + origin { + domain_name = module.assets_bucket.regional_domain_name + origin_id = "assets_s3" + origin_access_control_id = aws_cloudfront_origin_access_control.assets.id + } + default_cache_behavior { viewer_protocol_policy = "redirect-to-https" allowed_methods = ["GET", "HEAD", "OPTIONS", "PUT", "POST", "PATCH", "DELETE"] @@ -143,15 +149,23 @@ resource "aws_cloudfront_distribution" "main" { origin_request_policy_id = data.aws_cloudfront_origin_request_policy.cors_s3_origin.id } + # The applications can serve their own assets, but doing so causes + # errors during deployments: old tasks cannot serve assets for a new + # release. The assets bucket holds the assets for every release, so + # prefer it once the deploy pipelines have started populating it. + # + # The all_viewer origin request policy must not be used with an S3 + # origin because forwarding the viewer Host header breaks the origin + # access control request signing. ordered_cache_behavior { path_pattern = "/assets/*" allowed_methods = ["GET", "HEAD", "OPTIONS"] cached_methods = ["GET", "HEAD"] - target_origin_id = "application_load_balancer" + target_origin_id = var.serve_assets_from_s3 ? "assets_s3" : "application_load_balancer" viewer_protocol_policy = "redirect-to-https" compress = true cache_policy_id = data.aws_cloudfront_cache_policy.caching_optimized.id - origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id + origin_request_policy_id = var.serve_assets_from_s3 ? data.aws_cloudfront_origin_request_policy.cors_s3_origin.id : data.aws_cloudfront_origin_request_policy.all_viewer.id } } diff --git a/infra/modules/cloudfront/variables.tf b/infra/modules/cloudfront/variables.tf index 33bdeb70b..184ef0fed 100644 --- a/infra/modules/cloudfront/variables.tf +++ b/infra/modules/cloudfront/variables.tf @@ -50,3 +50,9 @@ variable "kinesis_subscription_role_arn" { description = "The arn of the role that is allowed to subscribe to the kinesis stream" type = string } + +variable "serve_assets_from_s3" { + description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled." + type = bool + default = false +} diff --git a/infra/modules/environment/cloudfront.tf b/infra/modules/environment/cloudfront.tf index c69135987..02c8832e9 100644 --- a/infra/modules/environment/cloudfront.tf +++ b/infra/modules/environment/cloudfront.tf @@ -22,6 +22,7 @@ module "cloudfront" { subject_alternative_names = local.subject_alternative_names[var.env_name] kinesis_subscription_role_arn = var.kinesis_subscription_role_arn + serve_assets_from_s3 = var.serve_assets_from_s3 } resource "aws_ssm_parameter" "email_zendesk" { diff --git a/infra/modules/environment/variables.tf b/infra/modules/environment/variables.tf index 80fc3111a..a19498a4b 100644 --- a/infra/modules/environment/variables.tf +++ b/infra/modules/environment/variables.tf @@ -32,6 +32,12 @@ variable "enable_cloudfront" { default = true } +variable "serve_assets_from_s3" { + type = bool + description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled." + default = false +} + variable "enable_alert_actions" { type = bool description = "Whether any actions associated with CloudWatch alarms should be enabled" diff --git a/infra/modules/secure-bucket/outputs.tf b/infra/modules/secure-bucket/outputs.tf index 5099e3958..11633177d 100644 --- a/infra/modules/secure-bucket/outputs.tf +++ b/infra/modules/secure-bucket/outputs.tf @@ -5,3 +5,7 @@ output "name" { output "arn" { value = aws_s3_bucket.this.arn } + +output "regional_domain_name" { + value = aws_s3_bucket.this.bucket_regional_domain_name +}