feat(pipedream): add edge pipeline group for edge/default and the 15 PoPs

libs/getsentry.libsonnet

@@ -2,8 +2,74 @@
 * sentry-specific helpers
 */
 
+// `edge` is the sentry-edge `primary` cluster. It is an ordinary region cluster
+// and gets an ordinary single-region group, exactly like us/de/s4s2/control.
+local edge_region = 'edge';
+
+// pop-rr is the canary cluster for the PoP fleet. The PoPs have no in-cluster
+// canary deployments the way the SaaS regions do -- one whole cluster plays that
+// role instead -- so it deploys as its own group, ahead of the rest, and acts as
+// the gate for them.
+local edge_pop_canary_regions = ['edge-pop-rr'];
+
+// The PoPs the canary gates. They share a group so they deploy as parallel jobs:
+// there is no meaningful order between them, and alphabetical order in
+// particular implies a sequencing that does not exist. Adding a PoP here needs
+// no ordering decision.
+local edge_pop_gated_regions = [
+  'edge-pop-au',
+  'edge-pop-br',
+  'edge-pop-ca',
+  'edge-pop-de',
+  'edge-pop-fi',
+  'edge-pop-in',
+  'edge-pop-jp',
+  'edge-pop-nl',
+  'edge-pop-or',
+  'edge-pop-qa',
+  'edge-pop-sc',
+  'edge-pop-sg',
+  'edge-pop-tx',
+  'edge-pop-va',
+];
+
+// All 16 edge clusters. The PoPs are modelled as pseudo-regions (like the uptime
+// clusters) so each one gets its own diff/apply job. Every entry is
+// default-excluded in pipedream.libsonnet, so a service reaches edge only by
+// opting in via include_regions -- the same treatment control and snty-tools
+// get, and for the same reason: most services render no manifests there.
+//
+// Kept as file-level locals rather than fields so that pipeline_groups does not
+// depend on `self` -- pipeline_groups gets copied into other objects, which
+// would rebind `self` and break the reference.
+local edge_pop_regions = edge_pop_canary_regions + edge_pop_gated_regions;
+local edge_regions = [edge_region] + edge_pop_regions;
+
 {
-  group_order: ['s4s2', 'de', 'us', 'us2', 'control', 'prod-control', 'snty-tools', 'st'],
+  edge_regions:: edge_regions,
+
+  // All 15 PoPs, canary included -- deliberately not the same set as the
+  // `edge-pop` group, which holds only the 14 the canary gates. A service that
+  // runs on the PoPs wants all 15 in its include_regions; pipedream then splits
+  // them across the canary and gated groups on its behalf.
+  edge_pop_regions:: edge_pop_regions,
+
+  // The edge groups trail the SaaS regions: these are ingest-path clusters, so
+  // they should only move after the regions they front are known good. Within
+  // edge, the primary cluster goes first, then the canary PoP gates the rest.
+  group_order: [
+    's4s2',
+    'de',
+    'us',
+    'us2',
+    'control',
+    'prod-control',
+    'snty-tools',
+    'edge',
+    'edge-pop-canary',
+    'edge-pop',
+    'st',
+  ],
   // Empty for now — add future test groups here
   test_group_order: [],
   // These groupings consist of user facing deployments
@@ -15,6 +81,9 @@
     control: ['control'],
     'prod-control': ['prod-control'],
     'snty-tools': ['snty-tools'],
+    edge: [edge_region],
+    'edge-pop-canary': edge_pop_canary_regions,
+    'edge-pop': edge_pop_gated_regions,
     st: ['customer-1', 'customer-2', 'customer-7'],
   },
   // Test groups will deploy in parallel to the groups above

libs/pipedream.libsonnet

@@ -53,8 +53,11 @@ local wrap_task(task) =
 local is_autodeploy(pipedream_config) =
   !std.objectHas(pipedream_config, 'auto_deploy') || pipedream_config.auto_deploy == true;
 
-// Regions that are excluded by default and must be explicitly included
-local default_excluded_regions = ['control', 'prod-control', 'snty-tools'];
+// Regions that are excluded by default and must be explicitly included.
+// The edge regions are listed here so that adding the `edge` pipeline group
+// does not hand every pipedream service a 16-job edge pipeline for manifests
+// it does not render. Opting in is per-service, via include_regions.
+local default_excluded_regions = ['control', 'prod-control', 'snty-tools'] + getsentry.edge_regions;
 
 local is_excluded_region = function(region, config)
   std.objectHas(config, 'exclude_regions') && std.length(std.find(region, config.exclude_regions)) > 0;
@@ -513,6 +516,11 @@ local pipeline_to_array(pipeline) =
   if pipeline == null then [] else [pipeline];
 
 {
+  // Exported so downstream renderers that reimplement the region filter (e.g.
+  // ops' render_with_pre_diff) can share this list instead of copying it and
+  // drifting out of sync.
+  default_excluded_regions:: default_excluded_regions,
+
   // render generates the trigger pipeline (if manual), group pipelines, and rollback pipeline.
   render(pipedream_config, pipeline_fn, parallel=false)::
     local groups_to_render = std.filter(

test/testdata/fixtures/pipedream/include-edge.jsonnet

@@ -0,0 +1,43 @@
+// The edge regions are default-excluded, so a service reaches them only by
+// naming them in include_regions.
+//
+// This fixture opts into all 16 and should render three chained pipelines:
+// `edge` (the primary cluster, an ordinary single-region group), then
+// `edge-pop-canary` holding only pop-rr, then `edge-pop` holding the remaining
+// 14 as parallel jobs. The canary group deploys before the rest and gates them.
+local getsentry = import '../../../../libs/getsentry.libsonnet';
+local pipedream = import '../../../../libs/pipedream.libsonnet';
+
+local pipedream_config = {
+  name: 'example',
+  auto_deploy: true,
+  include_regions: getsentry.edge_regions,
+};
+
+local sample = {
+  pipeline(region):: {
+    materials: {
+      example_repo: {
+        git: 'git@github.com:getsentry/example.git',
+        branch: 'master',
+        destination: 'example',
+      },
+    },
+    stages: [
+      {
+        deploy: {
+          jobs: {
+            ['deploy-' + region]: {
+              elastic_profile_id: 'example',
+              tasks: [
+                { script: './deploy.sh --region=' + region },
+              ],
+            },
+          },
+        },
+      },
+    ],
+  },
+};
+
+pipedream.render(pipedream_config, sample.pipeline)

test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden

@@ -6,6 +6,28 @@
       "de": [
          "de"
       ],
+      "edge": [
+         "edge"
+      ],
+      "edge-pop": [
+         "edge-pop-au",
+         "edge-pop-br",
+         "edge-pop-ca",
+         "edge-pop-de",
+         "edge-pop-fi",
+         "edge-pop-in",
+         "edge-pop-jp",
+         "edge-pop-nl",
+         "edge-pop-or",
+         "edge-pop-qa",
+         "edge-pop-sc",
+         "edge-pop-sg",
+         "edge-pop-tx",
+         "edge-pop-va"
+      ],
+      "edge-pop-canary": [
+         "edge-pop-rr"
+      ],
       "prod-control": [
          "prod-control"
       ],
@@ -35,6 +57,9 @@
       "control",
       "prod-control",
       "snty-tools",
+      "edge",
+      "edge-pop-canary",
+      "edge-pop",
       "st"
    ]
 }