From b8394c147da59a7c707480b0819fa50ec2a1f829 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 10:56:34 +0000 Subject: [PATCH 1/3] feat(pipedream): add edge pipeline group for edge/default and the 15 PoPs Adds an `edge` pipeline group covering the sentry-edge `primary` cluster and its 15 points-of-presence clusters, so edge workloads can be deployed by the same generated diff/apply/rollback pipelines every other region already has. The PoPs are modelled as pseudo-regions (`edge-pop-au`, ...) the way the uptime clusters already are, so each cluster gets its own diff/apply job. Every edge region is added to `default_excluded_regions`. Without that, adding the group would hand *every* pipedream service a 16-job edge pipeline for manifests it does not render; instead, reaching edge is opt-in per service via `include_regions`, exactly how `control` and `snty-tools` work today. The unchanged pipedream goldens are the evidence: only the group registry golden moved, no service gained a pipeline. `default_excluded_regions` is now exported so downstream renderers that reimplement the region filter can share it rather than copying the list and drifting. `edge` is placed after `snty-tools` in `group_order` so these ingest-path clusters trail the SaaS regions they front. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PLUQrwA7mZgNE45pp9DM3B --- libs/getsentry.libsonnet | 35 +- libs/pipedream.libsonnet | 12 +- .../fixtures/pipedream/include-edge.jsonnet | 39 ++ .../groups.jsonnet_output-files.golden | 19 + .../include-edge.jsonnet_output-files.golden | 360 ++++++++++++++++++ .../include-edge.jsonnet_single-file.golden | 333 ++++++++++++++++ 6 files changed, 795 insertions(+), 3 deletions(-) create mode 100644 test/testdata/fixtures/pipedream/include-edge.jsonnet create mode 100644 test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 7c0fa66..78160de 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -2,8 +2,40 @@ * sentry-specific helpers */ +// The edge region and its 15 points-of-presence clusters. The PoPs are +// modelled as pseudo-regions (like the uptime clusters) so each one gets its +// own diff/apply job; `edge` itself is the sentry-edge `primary` cluster. +// Every entry here is default-excluded in pipedream.libsonnet, so a service +// only deploys to edge if it opts in via include_regions. +// +// Kept as a file-level local rather than a field so that `pipeline_groups.edge` +// does not depend on `self` — pipeline_groups gets copied into other objects, +// which would rebind `self` and break the reference. +local edge_regions = [ + 'edge', + 'edge-pop-au', + 'edge-pop-br', + 'edge-pop-ca', + 'edge-pop-de', + 'edge-pop-fi', + 'edge-pop-in', + 'edge-pop-jp', + 'edge-pop-nl', + 'edge-pop-or', + 'edge-pop-qa', + 'edge-pop-rr', + 'edge-pop-sc', + 'edge-pop-sg', + 'edge-pop-tx', + 'edge-pop-va', +]; + { - group_order: ['s4s2', 'de', 'us', 'us2', 'control', 'prod-control', 'snty-tools', 'st'], + edge_regions:: edge_regions, + + // `edge` trails the SaaS regions: these are ingest-path clusters, so they + // should only move after the regions they front are known good. + group_order: ['s4s2', 'de', 'us', 'us2', 'control', 'prod-control', 'snty-tools', 'edge', 'st'], // Empty for now — add future test groups here test_group_order: [], // These groupings consist of user facing deployments @@ -15,6 +47,7 @@ control: ['control'], 'prod-control': ['prod-control'], 'snty-tools': ['snty-tools'], + edge: edge_regions, st: ['customer-1', 'customer-2', 'customer-7'], }, // Test groups will deploy in parallel to the groups above diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index d9a3afb..d9750de 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -53,8 +53,11 @@ local wrap_task(task) = local is_autodeploy(pipedream_config) = !std.objectHas(pipedream_config, 'auto_deploy') || pipedream_config.auto_deploy == true; -// Regions that are excluded by default and must be explicitly included -local default_excluded_regions = ['control', 'prod-control', 'snty-tools']; +// Regions that are excluded by default and must be explicitly included. +// The edge regions are listed here so that adding the `edge` pipeline group +// does not hand every pipedream service a 16-job edge pipeline for manifests +// it does not render. Opting in is per-service, via include_regions. +local default_excluded_regions = ['control', 'prod-control', 'snty-tools'] + getsentry.edge_regions; local is_excluded_region = function(region, config) std.objectHas(config, 'exclude_regions') && std.length(std.find(region, config.exclude_regions)) > 0; @@ -513,6 +516,11 @@ local pipeline_to_array(pipeline) = if pipeline == null then [] else [pipeline]; { + // Exported so downstream renderers that reimplement the region filter (e.g. + // ops' render_with_pre_diff) can share this list instead of copying it and + // drifting out of sync. + default_excluded_regions:: default_excluded_regions, + // render generates the trigger pipeline (if manual), group pipelines, and rollback pipeline. render(pipedream_config, pipeline_fn, parallel=false):: local groups_to_render = std.filter( diff --git a/test/testdata/fixtures/pipedream/include-edge.jsonnet b/test/testdata/fixtures/pipedream/include-edge.jsonnet new file mode 100644 index 0000000..bb8935a --- /dev/null +++ b/test/testdata/fixtures/pipedream/include-edge.jsonnet @@ -0,0 +1,39 @@ +// The edge regions are default-excluded, so a service reaches them only by +// naming them in include_regions. This fixture opts into `edge` itself plus two +// PoP pseudo-regions, and should render a single `deploy-example-edge` pipeline +// with one job per included region -- and no jobs for the 13 PoPs left out. +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +local pipedream_config = { + name: 'example', + auto_deploy: true, + include_regions: ['edge', 'edge-pop-au', 'edge-pop-va'], +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + ['deploy-' + region]: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index e3aa339..79b1d88 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -6,6 +6,24 @@ "de": [ "de" ], + "edge": [ + "edge", + "edge-pop-au", + "edge-pop-br", + "edge-pop-ca", + "edge-pop-de", + "edge-pop-fi", + "edge-pop-in", + "edge-pop-jp", + "edge-pop-nl", + "edge-pop-or", + "edge-pop-qa", + "edge-pop-rr", + "edge-pop-sc", + "edge-pop-sg", + "edge-pop-tx", + "edge-pop-va" + ], "prod-control": [ "prod-control" ], @@ -35,6 +53,7 @@ "control", "prod-control", "snty-tools", + "edge", "st" ] } diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden new file mode 100644 index 0000000..7918141 --- /dev/null +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden @@ -0,0 +1,360 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "de" + }, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-edge.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge": { + "display_order": 6, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge,edge-pop-au,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-us2-pipeline-complete": { + "pipeline": "deploy-example-us2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge" + } + ] + }, + "deploy-edge-pop-au": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-au" + } + ] + }, + "deploy-edge-pop-va": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-va" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "s4s2" + }, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 7, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + }, + "group": "example", + "materials": { + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us" + }, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us2": { + "display_order": 5, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us2" + }, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden new file mode 100644 index 0000000..91759da --- /dev/null +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden @@ -0,0 +1,333 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "de" + }, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-edge": { + "display_order": 6, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge,edge-pop-au,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-us2-pipeline-complete": { + "pipeline": "deploy-example-us2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge" + } + ] + }, + "deploy-edge-pop-au": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-au" + } + ] + }, + "deploy-edge-pop-va": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-va" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s2": { + "display_order": 2, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "s4s2" + }, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 7, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + }, + "group": "example", + "materials": { + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us" + }, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us2": { + "display_order": 5, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us2" + }, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} From c2e98858bbeace43220d3ddd2fe675fa23f070a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 13:23:37 +0000 Subject: [PATCH 2/3] ref(pipedream): split edge into primary, canary PoP and parallel PoP groups Addresses review on #101. Three changes, all to how the 16 edge clusters are grouped; the default-exclusion and the pseudo-region modelling are unchanged. The edge primary cluster is an ordinary region cluster, so it now gets an ordinary single-region group -- `edge: ['edge']`, exactly like us/de/s4s2/ control -- instead of being lumped into one 16-job pipeline with the PoPs. pop-rr is the canary cluster for the PoP fleet: the PoPs have no in-cluster canary deployments the way the SaaS regions do, one whole cluster plays that role. It therefore gets its own group ahead of the others, so group chaining makes it a real gate rather than just another parallel PoP job. The remaining 14 PoPs share a group, which makes them parallel jobs in one pipeline. Alphabetical order implied a deploy sequence that does not exist, and this way adding a PoP needs no ordering decision. It also means an operator can deploy a subset via PIPEDREAM_GROUP_REGIONS, which single-region groups skip. Resulting chain: ... -> snty-tools -> edge -> edge-pop-canary -> edge-pop -> st. Also exports edge_pop_regions (the 15 PoPs) alongside edge_regions (all 16), so downstream consumers don't have to filter 'edge' out by hand. --- libs/getsentry.libsonnet | 63 +++-- .../fixtures/pipedream/include-edge.jsonnet | 12 +- .../groups.jsonnet_output-files.golden | 10 +- .../include-edge.jsonnet_output-files.golden | 222 +++++++++++++++++- .../include-edge.jsonnet_single-file.golden | 200 +++++++++++++++- 5 files changed, 467 insertions(+), 40 deletions(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 78160de..f8ffc56 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -2,17 +2,21 @@ * sentry-specific helpers */ -// The edge region and its 15 points-of-presence clusters. The PoPs are -// modelled as pseudo-regions (like the uptime clusters) so each one gets its -// own diff/apply job; `edge` itself is the sentry-edge `primary` cluster. -// Every entry here is default-excluded in pipedream.libsonnet, so a service -// only deploys to edge if it opts in via include_regions. -// -// Kept as a file-level local rather than a field so that `pipeline_groups.edge` -// does not depend on `self` — pipeline_groups gets copied into other objects, -// which would rebind `self` and break the reference. -local edge_regions = [ - 'edge', +// `edge` is the sentry-edge `primary` cluster. It is an ordinary region cluster +// and gets an ordinary single-region group, exactly like us/de/s4s2/control. +local edge_region = 'edge'; + +// pop-rr is the canary cluster for the PoP fleet. The PoPs have no in-cluster +// canary deployments the way the SaaS regions do -- one whole cluster plays that +// role instead -- so it deploys as its own group, ahead of the rest, and acts as +// the gate for them. +local edge_pop_canary_regions = ['edge-pop-rr']; + +// The remaining PoPs. They share a group so they deploy as parallel jobs: there +// is no meaningful order between them, and alphabetical order in particular +// implies a sequencing that does not exist. Adding a PoP here needs no ordering +// decision. +local edge_pop_regions = [ 'edge-pop-au', 'edge-pop-br', 'edge-pop-ca', @@ -23,19 +27,44 @@ local edge_regions = [ 'edge-pop-nl', 'edge-pop-or', 'edge-pop-qa', - 'edge-pop-rr', 'edge-pop-sc', 'edge-pop-sg', 'edge-pop-tx', 'edge-pop-va', ]; +// All 16 edge clusters. The PoPs are modelled as pseudo-regions (like the uptime +// clusters) so each one gets its own diff/apply job. Every entry is +// default-excluded in pipedream.libsonnet, so a service reaches edge only by +// opting in via include_regions -- the same treatment control and snty-tools +// get, and for the same reason: most services render no manifests there. +// +// Kept as file-level locals rather than fields so that pipeline_groups does not +// depend on `self` -- pipeline_groups gets copied into other objects, which +// would rebind `self` and break the reference. +local edge_all_pop_regions = edge_pop_canary_regions + edge_pop_regions; +local edge_regions = [edge_region] + edge_all_pop_regions; + { edge_regions:: edge_regions, + edge_pop_regions:: edge_all_pop_regions, - // `edge` trails the SaaS regions: these are ingest-path clusters, so they - // should only move after the regions they front are known good. - group_order: ['s4s2', 'de', 'us', 'us2', 'control', 'prod-control', 'snty-tools', 'edge', 'st'], + // The edge groups trail the SaaS regions: these are ingest-path clusters, so + // they should only move after the regions they front are known good. Within + // edge, the primary cluster goes first, then the canary PoP gates the rest. + group_order: [ + 's4s2', + 'de', + 'us', + 'us2', + 'control', + 'prod-control', + 'snty-tools', + 'edge', + 'edge-pop-canary', + 'edge-pop', + 'st', + ], // Empty for now — add future test groups here test_group_order: [], // These groupings consist of user facing deployments @@ -47,7 +76,9 @@ local edge_regions = [ control: ['control'], 'prod-control': ['prod-control'], 'snty-tools': ['snty-tools'], - edge: edge_regions, + edge: [edge_region], + 'edge-pop-canary': edge_pop_canary_regions, + 'edge-pop': edge_pop_regions, st: ['customer-1', 'customer-2', 'customer-7'], }, // Test groups will deploy in parallel to the groups above diff --git a/test/testdata/fixtures/pipedream/include-edge.jsonnet b/test/testdata/fixtures/pipedream/include-edge.jsonnet index bb8935a..9c72d2e 100644 --- a/test/testdata/fixtures/pipedream/include-edge.jsonnet +++ b/test/testdata/fixtures/pipedream/include-edge.jsonnet @@ -1,13 +1,17 @@ // The edge regions are default-excluded, so a service reaches them only by -// naming them in include_regions. This fixture opts into `edge` itself plus two -// PoP pseudo-regions, and should render a single `deploy-example-edge` pipeline -// with one job per included region -- and no jobs for the 13 PoPs left out. +// naming them in include_regions. +// +// This fixture opts into all 16 and should render three chained pipelines: +// `edge` (the primary cluster, an ordinary single-region group), then +// `edge-pop-canary` holding only pop-rr, then `edge-pop` holding the remaining +// 14 as parallel jobs. The canary group deploys before the rest and gates them. +local getsentry = import '../../../../libs/getsentry.libsonnet'; local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: true, - include_regions: ['edge', 'edge-pop-au', 'edge-pop-va'], + include_regions: getsentry.edge_regions, }; local sample = { diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index 79b1d88..5b43a00 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -7,7 +7,9 @@ "de" ], "edge": [ - "edge", + "edge" + ], + "edge-pop": [ "edge-pop-au", "edge-pop-br", "edge-pop-ca", @@ -18,12 +20,14 @@ "edge-pop-nl", "edge-pop-or", "edge-pop-qa", - "edge-pop-rr", "edge-pop-sc", "edge-pop-sg", "edge-pop-tx", "edge-pop-va" ], + "edge-pop-canary": [ + "edge-pop-rr" + ], "prod-control": [ "prod-control" ], @@ -54,6 +58,8 @@ "prod-control", "snty-tools", "edge", + "edge-pop-canary", + "edge-pop", "st" ] } diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden index 7918141..9bcb61c 100644 --- a/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden @@ -54,18 +54,18 @@ } } }, - "deploy-example-edge.yaml": { + "deploy-example-edge-pop-canary.yaml": { "format_version": 10, "pipelines": { - "deploy-example-edge": { - "display_order": 6, + "deploy-example-edge-pop-canary": { + "display_order": 7, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "edge,edge-pop-au,edge-pop-va" + "PIPEDREAM_GROUP_REGIONS": "edge-pop-rr" }, "group": "example", "materials": { - "deploy-example-us2-pipeline-complete": { - "pipeline": "deploy-example-us2", + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", "stage": "pipeline-complete" }, "example_repo": { @@ -78,14 +78,61 @@ { "deploy": { "jobs": { - "deploy-edge": { + "deploy-edge-pop-rr": { "elastic_profile_id": "example", "tasks": [ { - "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge" + "script": "./deploy.sh --region=edge-pop-rr" } ] - }, + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-edge-pop.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge-pop": { + "display_order": 8, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-au,edge-pop-br,edge-pop-ca,edge-pop-de,edge-pop-fi,edge-pop-in,edge-pop-jp,edge-pop-nl,edge-pop-or,edge-pop-qa,edge-pop-sc,edge-pop-sg,edge-pop-tx,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-canary-pipeline-complete": { + "pipeline": "deploy-example-edge-pop-canary", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { "deploy-edge-pop-au": { "elastic_profile_id": "example", "tasks": [ @@ -94,6 +141,102 @@ } ] }, + "deploy-edge-pop-br": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-br" + } + ] + }, + "deploy-edge-pop-ca": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-ca" + } + ] + }, + "deploy-edge-pop-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-de" + } + ] + }, + "deploy-edge-pop-fi": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-fi" + } + ] + }, + "deploy-edge-pop-in": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-in" + } + ] + }, + "deploy-edge-pop-jp": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-jp" + } + ] + }, + "deploy-edge-pop-nl": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-nl" + } + ] + }, + "deploy-edge-pop-or": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-or" + } + ] + }, + "deploy-edge-pop-qa": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-qa" + } + ] + }, + "deploy-edge-pop-sc": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sc" + } + ] + }, + "deploy-edge-pop-sg": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sg" + } + ] + }, + "deploy-edge-pop-tx": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-tx" + } + ] + }, "deploy-edge-pop-va": { "elastic_profile_id": "example", "tasks": [ @@ -125,6 +268,61 @@ } } }, + "deploy-example-edge.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge": { + "display_order": 6, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge" + }, + "group": "example", + "materials": { + "deploy-example-us2-pipeline-complete": { + "pipeline": "deploy-example-us2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-s4s2.yaml": { "format_version": 10, "pipelines": { @@ -180,14 +378,14 @@ "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 7, + "display_order": 9, "environment_variables": { "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" }, "group": "example", "materials": { - "deploy-example-edge-pipeline-complete": { - "pipeline": "deploy-example-edge", + "deploy-example-edge-pop-pipeline-complete": { + "pipeline": "deploy-example-edge-pop", "stage": "pipeline-complete" }, "example_repo": { diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden index 91759da..c9504dd 100644 --- a/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden @@ -54,7 +54,7 @@ "deploy-example-edge": { "display_order": 6, "environment_variables": { - "PIPEDREAM_GROUP_REGIONS": "edge,edge-pop-au,edge-pop-va" + "PIPEDREAM_GROUP_REGIONS": "edge" }, "group": "example", "materials": { @@ -76,10 +76,52 @@ "elastic_profile_id": "example", "tasks": [ { - "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge" + "script": "./deploy.sh --region=edge" } ] - }, + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-edge-pop": { + "display_order": 8, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-au,edge-pop-br,edge-pop-ca,edge-pop-de,edge-pop-fi,edge-pop-in,edge-pop-jp,edge-pop-nl,edge-pop-or,edge-pop-qa,edge-pop-sc,edge-pop-sg,edge-pop-tx,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-canary-pipeline-complete": { + "pipeline": "deploy-example-edge-pop-canary", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { "deploy-edge-pop-au": { "elastic_profile_id": "example", "tasks": [ @@ -88,6 +130,102 @@ } ] }, + "deploy-edge-pop-br": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-br" + } + ] + }, + "deploy-edge-pop-ca": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-ca" + } + ] + }, + "deploy-edge-pop-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-de" + } + ] + }, + "deploy-edge-pop-fi": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-fi" + } + ] + }, + "deploy-edge-pop-in": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-in" + } + ] + }, + "deploy-edge-pop-jp": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-jp" + } + ] + }, + "deploy-edge-pop-nl": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-nl" + } + ] + }, + "deploy-edge-pop-or": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-or" + } + ] + }, + "deploy-edge-pop-qa": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-qa" + } + ] + }, + "deploy-edge-pop-sc": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sc" + } + ] + }, + "deploy-edge-pop-sg": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sg" + } + ] + }, + "deploy-edge-pop-tx": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-tx" + } + ] + }, "deploy-edge-pop-va": { "elastic_profile_id": "example", "tasks": [ @@ -117,6 +255,56 @@ } ] }, + "deploy-example-edge-pop-canary": { + "display_order": 7, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-rr" + }, + "group": "example", + "materials": { + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge-pop-rr": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge-pop-rr" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-s4s2": { "display_order": 2, "environment_variables": { @@ -164,14 +352,14 @@ ] }, "deploy-example-st": { - "display_order": 7, + "display_order": 9, "environment_variables": { "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" }, "group": "example", "materials": { - "deploy-example-edge-pipeline-complete": { - "pipeline": "deploy-example-edge", + "deploy-example-edge-pop-pipeline-complete": { + "pipeline": "deploy-example-edge-pop", "stage": "pipeline-complete" }, "example_repo": { From 79f24ead002165b66671bba1f0796e92a1a2e8e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 13:30:58 +0000 Subject: [PATCH 3/3] ref(pipedream): disambiguate the gated-PoP local from the exported PoP list Bugbot flagged that the exported `edge_pop_regions` (all 15 PoPs) collided with a file-level local of the same name holding only the 14 the canary gates. The contents of both were intentional, but binding one identifier to two different lists in the same file is a genuine trap for the next reader. Renames the local to `edge_pop_gated_regions`, which is also what the `edge-pop` group holds, and documents on the export that it deliberately covers all 15 including the canary: a service that runs on the PoPs wants all 15 in its include_regions, and pipedream splits them across the canary and gated groups on its behalf. Opting in per-group would defeat the gate. No behaviour change -- group contents are identical and every golden is untouched. --- libs/getsentry.libsonnet | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index f8ffc56..363d209 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -12,11 +12,11 @@ local edge_region = 'edge'; // the gate for them. local edge_pop_canary_regions = ['edge-pop-rr']; -// The remaining PoPs. They share a group so they deploy as parallel jobs: there -// is no meaningful order between them, and alphabetical order in particular -// implies a sequencing that does not exist. Adding a PoP here needs no ordering -// decision. -local edge_pop_regions = [ +// The PoPs the canary gates. They share a group so they deploy as parallel jobs: +// there is no meaningful order between them, and alphabetical order in +// particular implies a sequencing that does not exist. Adding a PoP here needs +// no ordering decision. +local edge_pop_gated_regions = [ 'edge-pop-au', 'edge-pop-br', 'edge-pop-ca', @@ -42,12 +42,17 @@ local edge_pop_regions = [ // Kept as file-level locals rather than fields so that pipeline_groups does not // depend on `self` -- pipeline_groups gets copied into other objects, which // would rebind `self` and break the reference. -local edge_all_pop_regions = edge_pop_canary_regions + edge_pop_regions; -local edge_regions = [edge_region] + edge_all_pop_regions; +local edge_pop_regions = edge_pop_canary_regions + edge_pop_gated_regions; +local edge_regions = [edge_region] + edge_pop_regions; { edge_regions:: edge_regions, - edge_pop_regions:: edge_all_pop_regions, + + // All 15 PoPs, canary included -- deliberately not the same set as the + // `edge-pop` group, which holds only the 14 the canary gates. A service that + // runs on the PoPs wants all 15 in its include_regions; pipedream then splits + // them across the canary and gated groups on its behalf. + edge_pop_regions:: edge_pop_regions, // The edge groups trail the SaaS regions: these are ingest-path clusters, so // they should only move after the regions they front are known good. Within @@ -78,7 +83,7 @@ local edge_regions = [edge_region] + edge_all_pop_regions; 'snty-tools': ['snty-tools'], edge: [edge_region], 'edge-pop-canary': edge_pop_canary_regions, - 'edge-pop': edge_pop_regions, + 'edge-pop': edge_pop_gated_regions, st: ['customer-1', 'customer-2', 'customer-7'], }, // Test groups will deploy in parallel to the groups above