diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 7c0fa66..363d209 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -2,8 +2,74 @@ * sentry-specific helpers */ +// `edge` is the sentry-edge `primary` cluster. It is an ordinary region cluster +// and gets an ordinary single-region group, exactly like us/de/s4s2/control. +local edge_region = 'edge'; + +// pop-rr is the canary cluster for the PoP fleet. The PoPs have no in-cluster +// canary deployments the way the SaaS regions do -- one whole cluster plays that +// role instead -- so it deploys as its own group, ahead of the rest, and acts as +// the gate for them. +local edge_pop_canary_regions = ['edge-pop-rr']; + +// The PoPs the canary gates. They share a group so they deploy as parallel jobs: +// there is no meaningful order between them, and alphabetical order in +// particular implies a sequencing that does not exist. Adding a PoP here needs +// no ordering decision. +local edge_pop_gated_regions = [ + 'edge-pop-au', + 'edge-pop-br', + 'edge-pop-ca', + 'edge-pop-de', + 'edge-pop-fi', + 'edge-pop-in', + 'edge-pop-jp', + 'edge-pop-nl', + 'edge-pop-or', + 'edge-pop-qa', + 'edge-pop-sc', + 'edge-pop-sg', + 'edge-pop-tx', + 'edge-pop-va', +]; + +// All 16 edge clusters. The PoPs are modelled as pseudo-regions (like the uptime +// clusters) so each one gets its own diff/apply job. Every entry is +// default-excluded in pipedream.libsonnet, so a service reaches edge only by +// opting in via include_regions -- the same treatment control and snty-tools +// get, and for the same reason: most services render no manifests there. +// +// Kept as file-level locals rather than fields so that pipeline_groups does not +// depend on `self` -- pipeline_groups gets copied into other objects, which +// would rebind `self` and break the reference. +local edge_pop_regions = edge_pop_canary_regions + edge_pop_gated_regions; +local edge_regions = [edge_region] + edge_pop_regions; + { - group_order: ['s4s2', 'de', 'us', 'us2', 'control', 'prod-control', 'snty-tools', 'st'], + edge_regions:: edge_regions, + + // All 15 PoPs, canary included -- deliberately not the same set as the + // `edge-pop` group, which holds only the 14 the canary gates. A service that + // runs on the PoPs wants all 15 in its include_regions; pipedream then splits + // them across the canary and gated groups on its behalf. + edge_pop_regions:: edge_pop_regions, + + // The edge groups trail the SaaS regions: these are ingest-path clusters, so + // they should only move after the regions they front are known good. Within + // edge, the primary cluster goes first, then the canary PoP gates the rest. + group_order: [ + 's4s2', + 'de', + 'us', + 'us2', + 'control', + 'prod-control', + 'snty-tools', + 'edge', + 'edge-pop-canary', + 'edge-pop', + 'st', + ], // Empty for now — add future test groups here test_group_order: [], // These groupings consist of user facing deployments @@ -15,6 +81,9 @@ control: ['control'], 'prod-control': ['prod-control'], 'snty-tools': ['snty-tools'], + edge: [edge_region], + 'edge-pop-canary': edge_pop_canary_regions, + 'edge-pop': edge_pop_gated_regions, st: ['customer-1', 'customer-2', 'customer-7'], }, // Test groups will deploy in parallel to the groups above diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index d9a3afb..d9750de 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -53,8 +53,11 @@ local wrap_task(task) = local is_autodeploy(pipedream_config) = !std.objectHas(pipedream_config, 'auto_deploy') || pipedream_config.auto_deploy == true; -// Regions that are excluded by default and must be explicitly included -local default_excluded_regions = ['control', 'prod-control', 'snty-tools']; +// Regions that are excluded by default and must be explicitly included. +// The edge regions are listed here so that adding the `edge` pipeline group +// does not hand every pipedream service a 16-job edge pipeline for manifests +// it does not render. Opting in is per-service, via include_regions. +local default_excluded_regions = ['control', 'prod-control', 'snty-tools'] + getsentry.edge_regions; local is_excluded_region = function(region, config) std.objectHas(config, 'exclude_regions') && std.length(std.find(region, config.exclude_regions)) > 0; @@ -513,6 +516,11 @@ local pipeline_to_array(pipeline) = if pipeline == null then [] else [pipeline]; { + // Exported so downstream renderers that reimplement the region filter (e.g. + // ops' render_with_pre_diff) can share this list instead of copying it and + // drifting out of sync. + default_excluded_regions:: default_excluded_regions, + // render generates the trigger pipeline (if manual), group pipelines, and rollback pipeline. render(pipedream_config, pipeline_fn, parallel=false):: local groups_to_render = std.filter( diff --git a/test/testdata/fixtures/pipedream/include-edge.jsonnet b/test/testdata/fixtures/pipedream/include-edge.jsonnet new file mode 100644 index 0000000..9c72d2e --- /dev/null +++ b/test/testdata/fixtures/pipedream/include-edge.jsonnet @@ -0,0 +1,43 @@ +// The edge regions are default-excluded, so a service reaches them only by +// naming them in include_regions. +// +// This fixture opts into all 16 and should render three chained pipelines: +// `edge` (the primary cluster, an ordinary single-region group), then +// `edge-pop-canary` holding only pop-rr, then `edge-pop` holding the remaining +// 14 as parallel jobs. The canary group deploys before the rest and gates them. +local getsentry = import '../../../../libs/getsentry.libsonnet'; +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +local pipedream_config = { + name: 'example', + auto_deploy: true, + include_regions: getsentry.edge_regions, +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + ['deploy-' + region]: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index e3aa339..5b43a00 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -6,6 +6,28 @@ "de": [ "de" ], + "edge": [ + "edge" + ], + "edge-pop": [ + "edge-pop-au", + "edge-pop-br", + "edge-pop-ca", + "edge-pop-de", + "edge-pop-fi", + "edge-pop-in", + "edge-pop-jp", + "edge-pop-nl", + "edge-pop-or", + "edge-pop-qa", + "edge-pop-sc", + "edge-pop-sg", + "edge-pop-tx", + "edge-pop-va" + ], + "edge-pop-canary": [ + "edge-pop-rr" + ], "prod-control": [ "prod-control" ], @@ -35,6 +57,9 @@ "control", "prod-control", "snty-tools", + "edge", + "edge-pop-canary", + "edge-pop", "st" ] } diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden new file mode 100644 index 0000000..9bcb61c --- /dev/null +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_output-files.golden @@ -0,0 +1,558 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "de" + }, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-edge-pop-canary.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge-pop-canary": { + "display_order": 7, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-rr" + }, + "group": "example", + "materials": { + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge-pop-rr": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge-pop-rr" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-edge-pop.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge-pop": { + "display_order": 8, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-au,edge-pop-br,edge-pop-ca,edge-pop-de,edge-pop-fi,edge-pop-in,edge-pop-jp,edge-pop-nl,edge-pop-or,edge-pop-qa,edge-pop-sc,edge-pop-sg,edge-pop-tx,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-canary-pipeline-complete": { + "pipeline": "deploy-example-edge-pop-canary", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge-pop-au": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-au" + } + ] + }, + "deploy-edge-pop-br": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-br" + } + ] + }, + "deploy-edge-pop-ca": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-ca" + } + ] + }, + "deploy-edge-pop-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-de" + } + ] + }, + "deploy-edge-pop-fi": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-fi" + } + ] + }, + "deploy-edge-pop-in": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-in" + } + ] + }, + "deploy-edge-pop-jp": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-jp" + } + ] + }, + "deploy-edge-pop-nl": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-nl" + } + ] + }, + "deploy-edge-pop-or": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-or" + } + ] + }, + "deploy-edge-pop-qa": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-qa" + } + ] + }, + "deploy-edge-pop-sc": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sc" + } + ] + }, + "deploy-edge-pop-sg": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sg" + } + ] + }, + "deploy-edge-pop-tx": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-tx" + } + ] + }, + "deploy-edge-pop-va": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-va" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-edge.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-edge": { + "display_order": 6, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge" + }, + "group": "example", + "materials": { + "deploy-example-us2-pipeline-complete": { + "pipeline": "deploy-example-us2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "s4s2" + }, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 9, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-pipeline-complete": { + "pipeline": "deploy-example-edge-pop", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us" + }, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us2": { + "display_order": 5, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us2" + }, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden new file mode 100644 index 0000000..c9504dd --- /dev/null +++ b/test/testdata/goldens/pipedream/include-edge.jsonnet_single-file.golden @@ -0,0 +1,521 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "de" + }, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-edge": { + "display_order": 6, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge" + }, + "group": "example", + "materials": { + "deploy-example-us2-pipeline-complete": { + "pipeline": "deploy-example-us2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-edge-pop": { + "display_order": 8, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-au,edge-pop-br,edge-pop-ca,edge-pop-de,edge-pop-fi,edge-pop-in,edge-pop-jp,edge-pop-nl,edge-pop-or,edge-pop-qa,edge-pop-sc,edge-pop-sg,edge-pop-tx,edge-pop-va" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-canary-pipeline-complete": { + "pipeline": "deploy-example-edge-pop-canary", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge-pop-au": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-au" + } + ] + }, + "deploy-edge-pop-br": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-br" + } + ] + }, + "deploy-edge-pop-ca": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-ca" + } + ] + }, + "deploy-edge-pop-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-de" + } + ] + }, + "deploy-edge-pop-fi": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-fi" + } + ] + }, + "deploy-edge-pop-in": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-in" + } + ] + }, + "deploy-edge-pop-jp": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-jp" + } + ] + }, + "deploy-edge-pop-nl": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-nl" + } + ] + }, + "deploy-edge-pop-or": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-or" + } + ] + }, + "deploy-edge-pop-qa": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-qa" + } + ] + }, + "deploy-edge-pop-sc": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sc" + } + ] + }, + "deploy-edge-pop-sg": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-sg" + } + ] + }, + "deploy-edge-pop-tx": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-tx" + } + ] + }, + "deploy-edge-pop-va": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=edge-pop-va" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-edge-pop-canary": { + "display_order": 7, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "edge-pop-rr" + }, + "group": "example", + "materials": { + "deploy-example-edge-pipeline-complete": { + "pipeline": "deploy-example-edge", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-edge-pop-rr": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=edge-pop-rr" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s2": { + "display_order": 2, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "s4s2" + }, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 9, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "customer-1,customer-2,customer-7" + }, + "group": "example", + "materials": { + "deploy-example-edge-pop-pipeline-complete": { + "pipeline": "deploy-example-edge-pop", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us" + }, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us2": { + "display_order": 5, + "environment_variables": { + "PIPEDREAM_GROUP_REGIONS": "us2" + }, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "fetch_materials": false, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +}