From a75a38443770fcd72bf6bea11331d4ac576984a2 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 24 Sep 2026 13:08:08 +0200 Subject: [PATCH 1/7] target etl PR for demo --- .github/workflows/build-and-deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index b957e92c..91ec67c3 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -433,7 +433,10 @@ jobs: registry-password: ${{ secrets.GHCR_ETL_PULL_TOKEN }} with: deployment-lane: ${{ needs.resolve-context.outputs.deployment-lane }} - full-image-name: ghcr.io/rmi/stitch-etl-poc:${{ vars.ETL_IMAGE_TAG || 'main' }} + # demo/norway: pinned to stitch-etl-poc PR #38 (feat/norway-data-added-STIT-640) for this demo. + # Intentionally hardcoded instead of `vars.ETL_IMAGE_TAG || 'main'` so a lane + # variable can't override the pin. Revert to the vars-based line before merging. + full-image-name: ghcr.io/rmi/stitch-etl-poc:pr-38 container-app-name: ${{ format('{0}-etl', needs.resolve-context.outputs.deployment-name) }} registry-server: ghcr.io registry-username: ${{ needs.lane-config-validate.outputs.ghcr-etl-pull-username }} From a828f479bf27a26bb223579646628a243e45ffba Mon Sep 17 00:00:00 2001 From: MB <5711147+mbarlow12@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:30:53 -0600 Subject: [PATCH 2/7] Update deployment references for PR #39 --- .github/workflows/build-and-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 91ec67c3..7be2b770 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -433,10 +433,10 @@ jobs: registry-password: ${{ secrets.GHCR_ETL_PULL_TOKEN }} with: deployment-lane: ${{ needs.resolve-context.outputs.deployment-lane }} - # demo/norway: pinned to stitch-etl-poc PR #38 (feat/norway-data-added-STIT-640) for this demo. + # demo/norway: pinned to stitch-etl-poc PR #39 (demo/norway) for this demo. # Intentionally hardcoded instead of `vars.ETL_IMAGE_TAG || 'main'` so a lane # variable can't override the pin. Revert to the vars-based line before merging. - full-image-name: ghcr.io/rmi/stitch-etl-poc:pr-38 + full-image-name: ghcr.io/rmi/stitch-etl-poc:pr-39 container-app-name: ${{ format('{0}-etl', needs.resolve-context.outputs.deployment-name) }} registry-server: ghcr.io registry-username: ${{ needs.lane-config-validate.outputs.ghcr-etl-pull-username }} From e26a4cb5a7a6daf1f26aea2d0c0e1b5986eee393 Mon Sep 17 00:00:00 2001 From: Michael Barlow Date: Thu, 24 Sep 2026 11:06:43 -0600 Subject: [PATCH 3/7] redeploy From a947775a8db3bbd25071a8db8c1f64688dadd69c Mon Sep 17 00:00:00 2001 From: Michael Barlow Date: Thu, 24 Sep 2026 11:30:25 -0600 Subject: [PATCH 4/7] redeploy From 92c995b4ca44cf498f8d510e7e14310340283423 Mon Sep 17 00:00:00 2001 From: Michael Barlow Date: Thu, 24 Sep 2026 12:13:42 -0600 Subject: [PATCH 5/7] redeploy From b0b1e5192498cbf5c81bcd27baf5264a2b261e25 Mon Sep 17 00:00:00 2001 From: Michael Barlow Date: Thu, 24 Sep 2026 12:30:20 -0600 Subject: [PATCH 6/7] ci: gate ETL deploy on downstream token permissions --- .github/scripts/preflight_downstream_token.py | 84 +++++++++++++++++++ .github/workflows/build-and-deploy.yml | 15 +++- 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/preflight_downstream_token.py diff --git a/.github/scripts/preflight_downstream_token.py b/.github/scripts/preflight_downstream_token.py new file mode 100644 index 00000000..24c65171 --- /dev/null +++ b/.github/scripts/preflight_downstream_token.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Fail the deploy when the downstream ETL token cannot read every source. + +Auth0 freezes the RBAC permission array into an access token at issuance, so a +grant added after a token was minted never reaches it. The token keeps +authenticating and only the newest `source:read:` is absent, which the ETL +surfaces as dedupe re-posting every record for that one dataset rather than as +any error. Checking before the deploy turns that into a named CI failure. + +Reads CURRENT_TOKEN from the environment, never argv, and prints no token. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import os +import sys +from datetime import datetime, timezone + +# Source keys are defined upstream in packages/stitch-ogsi/src/stitch/ogsi/model +# as `_SRC` constants, and each ETL dataset imports its key from there. +# Listed by hand to keep this job dependency-free, so adding a dataset means +# adding its key here too. +REQUIRED_PERMISSIONS = frozenset( + { + "source:write", + "source:read:alb", + "source:read:bc", + "source:read:ccr", + "source:read:gem", + "source:read:nor", + "source:read:wm", + } +) + + +def claims_of(token: str) -> dict: + parts = token.split(".") + if len(parts) != 3: + raise ValueError("not a three-part JWT") + payload = parts[1] + "=" * (-len(parts[1]) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def main() -> int: + token = os.environ.get("CURRENT_TOKEN", "") + if not token: + sys.exit("CURRENT_TOKEN must be set") + + try: + claims = claims_of(token) + except (ValueError, json.JSONDecodeError, binascii.Error) as exc: + sys.exit(f"Downstream token is undecodable ({exc}).") + + expiry = claims.get("exp") + if isinstance(expiry, (int, float)): + expires_at = datetime.fromtimestamp(expiry, tz=timezone.utc) + hours = (expires_at - datetime.now(timezone.utc)).total_seconds() / 3600 + floor = float(os.environ.get("MIN_REMAINING_HOURS", "24")) + print(f"Token expires {expires_at:%Y-%m-%d %H:%M} UTC ({hours:.1f}h left)") + if hours <= 0: + sys.exit("Downstream token has already expired.") + if hours < floor: + print(f"::warning::Downstream token has {hours:.1f}h left, below the {floor:.0f}h floor") + else: + print("::warning::Downstream token carries no exp claim") + + missing = sorted(REQUIRED_PERMISSIONS - set(claims.get("permissions") or [])) + if missing: + sys.exit( + "Downstream token is missing " + + ", ".join(missing) + + ". An Auth0 grant only reaches tokens minted after it was added, so " + "re-mint the service account's token and update the environment secret." + ) + + print(f"Downstream token carries all {len(REQUIRED_PERMISSIONS)} required permissions.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 7be2b770..a5d857d7 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -419,10 +419,23 @@ jobs: # Only the production release stays warm; everything else scales to zero. min-replicas: ${{ needs.resolve-context.outputs.always-on }} + preflight-etl-token: + name: "Preflight ETL token" + if: ${{ needs.resolve-context.outputs.deployment-lane != 'development' }} + needs: [resolve-context] + runs-on: ubuntu-latest + environment: ${{ needs.resolve-context.outputs.deployment-lane }} + steps: + - uses: actions/checkout@v4 + - name: Check downstream token permissions + run: python .github/scripts/preflight_downstream_token.py + env: + CURRENT_TOKEN: ${{ secrets.STITCH_CLIENT_PRIVILEGED_BEARER_TOKEN }} + deploy-etl: name: "Deploy ETL" if: ${{ needs.resolve-context.outputs.deployment-lane != 'development' }} - needs: [resolve-context, lane-config-validate, deploy-api] + needs: [resolve-context, lane-config-validate, deploy-api, preflight-etl-token] uses: ./.github/workflows/deploy-container.yml secrets: azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} From 996467f8985651468fde480177093a0e7eb85535 Mon Sep 17 00:00:00 2001 From: Michael Barlow Date: Thu, 24 Sep 2026 18:08:10 -0600 Subject: [PATCH 7/7] redeploy