Staging

.github/workflows/main.yml

@@ -0,0 +1,146 @@
+name: Deploy Templatized Data Environment
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - staging
+      - main
+  push:
+    branches:
+      - staging
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  deploy_data_environment:
+    # Only run when PR is merged or on direct push to environment branches
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set deployment target
+        id: set_env
+        run: |
+          TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}"
+
+          if [[ "$TARGET_BRANCH" == "staging" ]]; then
+            echo "SNOWFLAKE_ENV=STAGING" >> "$GITHUB_ENV"
+            echo "DEPLOY_BRANCH=staging" >> "$GITHUB_ENV"
+            echo "Environment set to staging"
+          elif [[ "$TARGET_BRANCH" == "main" ]]; then
+            echo "SNOWFLAKE_ENV=PROD" >> "$GITHUB_ENV"
+            echo "DEPLOY_BRANCH=main" >> "$GITHUB_ENV"
+            echo "Environment set to production"
+          else
+            echo "Unexpected branch: $TARGET_BRANCH"
+            exit 1
+          fi
+
+      - name: Install Snowflake CLI
+        uses: snowflakedb/snowflake-cli-action@v1.5
+        with:
+          cli-version: "latest"
+          default-config-file-path: "config.toml"
+
+      - name: Configure Snowflake connection
+        env:
+          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+          SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+          SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+          SNOWFLAKE_PRIVATE_KEY_B64: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_B64 }}
+          SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
+        run: |
+          trim() { printf '%s' "$1" | tr -d '[:space:]'; }
+
+          ACCOUNT="$(trim "$SNOWFLAKE_ACCOUNT")"
+          USER="$(trim "$SNOWFLAKE_USER")"
+          WAREHOUSE="$(trim "$SNOWFLAKE_WAREHOUSE")"
+          ROLE="$(trim "$SNOWFLAKE_ROLE")"
+          KEY_B64="$(trim "$SNOWFLAKE_PRIVATE_KEY_B64")"
+          KEY_PATH="${RUNNER_TEMP}/snowflake_rsa_key.p8"
+
+          if [[ -z "$ACCOUNT" || -z "$USER" || -z "$WAREHOUSE" || -z "$ROLE" || -z "$KEY_B64" ]]; then
+            echo "Missing one or more required secrets."
+            echo "Required: SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_ROLE, SNOWFLAKE_PRIVATE_KEY_B64"
+            exit 1
+          fi
+
+          # JWT auth requires the account identifier (e.g. ORG-ACCOUNT), not the locator alone.
+          if [[ "$ACCOUNT" == "FD17944" ]]; then
+            echo "SNOWFLAKE_ACCOUNT is set to locator FD17944."
+            echo "Use the full account identifier instead, e.g. UTHZAZW-FJ87774"
+            exit 1
+          fi
+
+          if [[ "$ACCOUNT" != *"-"* ]]; then
+            echo "SNOWFLAKE_ACCOUNT should be the full account identifier (contains a hyphen)."
+            echo "Example: UTHZAZW-FJ87774"
+            exit 1
+          fi
+
+          printf '%s' "$KEY_B64" | base64 -d > "$KEY_PATH"
+          chmod 600 "$KEY_PATH"
+
+          if ! openssl pkey -in "$KEY_PATH" -noout >/dev/null 2>&1; then
+            echo "Invalid PEM after base64 decode."
+            echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 with:"
+            echo "base64 -i ~/.snowflake/keys/rsa_key.p8 | tr -d '\\n' | pbcopy"
+            exit 1
+          fi
+
+          KEY_FP="$(openssl pkey -in "$KEY_PATH" -pubout -outform DER | openssl dgst -sha256 -binary | openssl enc -base64)"
+          echo "Decoded key fingerprint: SHA256:${KEY_FP}="
+          echo "Expected fingerprint:      SHA256:cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM="
+
+          if [[ "$KEY_FP" != "cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" ]]; then
+            echo "Private key fingerprint does not match the public key on user SEBAUSTIN512."
+            echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 from ~/.snowflake/keys/rsa_key.p8"
+            exit 1
+          fi
+
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT=$ACCOUNT" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER=$USER" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_WAREHOUSE=$WAREHOUSE" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ROLE=$ROLE" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_DATABASE=COURSE_REPO" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_SCHEMA=PUBLIC" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PRIVATE_KEY_FILE=$KEY_PATH" >> "$GITHUB_ENV"
+
+          PASSPHRASE="$(trim "$SNOWFLAKE_PRIVATE_KEY_PASSPHRASE")"
+          if [[ -n "$PASSPHRASE" ]]; then
+            echo "PRIVATE_KEY_PASSPHRASE=$PASSPHRASE" >> "$GITHUB_ENV"
+          fi
+
+          echo "Account length: ${#ACCOUNT} (expected 17 for UTHZAZW-FJ87774)"
+          echo "User length: ${#USER} (expected 12 for SEBAUSTIN512)"
+
+      - name: Verify Snowflake connection
+        run: snow connection test
+
+      - name: Fetch latest changes to Snowflake
+        run: snow git fetch COURSE_REPO.PUBLIC.ADVANCED_DATA_ENGINEERING_SNOWFLAKE
+
+      - name: Deploy data pipeline
+        run: |
+          echo "Deploying branch: ${DEPLOY_BRANCH} with env: ${SNOWFLAKE_ENV}"
+          snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/data/" \
+            -D "env='${SNOWFLAKE_ENV}'" \
+            --database=COURSE_REPO \
+            --schema=PUBLIC
+
+      - name: Deploy pipeline objects
+        run: |
+          snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/objects/" \
+            -D "env='${SNOWFLAKE_ENV}'" \
+            --database=COURSE_REPO \
+            --schema=PUBLIC

module-1/hamburg_weather/main.yml

@@ -11,57 +11,136 @@ on:
       - staging
       - main
 
+permissions:
+  contents: read
+
 jobs:
   deploy_data_environment:
     # Only run when PR is merged or on direct push to environment branches
     if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true)
     runs-on: ubuntu-latest
 
-    env:
-      # Read connection secret
-      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
-      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
-      SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-
     steps:
-      # Checkout step is necessary to access repository files
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
-      # Determine environment based on target branch
-      - name: Set environment
+      - name: Set deployment target
         id: set_env
         run: |
           TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}"
-
-          if [[ $TARGET_BRANCH == "staging" ]]; then
-            echo "DEPLOY_ENV=staging" >> $GITHUB_ENV
+
+          if [[ "$TARGET_BRANCH" == "staging" ]]; then
+            echo "SNOWFLAKE_ENV=STAGING" >> "$GITHUB_ENV"
+            echo "DEPLOY_BRANCH=staging" >> "$GITHUB_ENV"
             echo "Environment set to staging"
-          elif [[ $TARGET_BRANCH == "main" ]]; then
-            echo "DEPLOY_ENV=prod" >> $GITHUB_ENV
+          elif [[ "$TARGET_BRANCH" == "main" ]]; then
+            echo "SNOWFLAKE_ENV=PROD" >> "$GITHUB_ENV"
+            echo "DEPLOY_BRANCH=main" >> "$GITHUB_ENV"
             echo "Environment set to production"
           else
             echo "Unexpected branch: $TARGET_BRANCH"
             exit 1
           fi
 
-      # Install Snowflake CLI GitHub Action and point to config file
-      - name: Install SnowflakeCLI
+      - name: Install Snowflake CLI
         uses: snowflakedb/snowflake-cli-action@v1.5
         with:
           cli-version: "latest"
           default-config-file-path: "config.toml"
 
-      # Fetch latest changes from the repository to Snowflake
+      - name: Configure Snowflake connection
+        env:
+          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+          SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+          SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+          SNOWFLAKE_PRIVATE_KEY_B64: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_B64 }}
+          SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
+        run: |
+          trim() { printf '%s' "$1" | tr -d '[:space:]'; }
+
+          ACCOUNT="$(trim "$SNOWFLAKE_ACCOUNT")"
+          USER="$(trim "$SNOWFLAKE_USER")"
+          WAREHOUSE="$(trim "$SNOWFLAKE_WAREHOUSE")"
+          ROLE="$(trim "$SNOWFLAKE_ROLE")"
+          KEY_B64="$(trim "$SNOWFLAKE_PRIVATE_KEY_B64")"
+          KEY_PATH="${RUNNER_TEMP}/snowflake_rsa_key.p8"
+
+          if [[ -z "$ACCOUNT" || -z "$USER" || -z "$WAREHOUSE" || -z "$ROLE" || -z "$KEY_B64" ]]; then
+            echo "Missing one or more required secrets."
+            echo "Required: SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_ROLE, SNOWFLAKE_PRIVATE_KEY_B64"
+            exit 1
+          fi
+
+          # JWT auth requires the account identifier (e.g. ORG-ACCOUNT), not the locator alone.
+          if [[ "$ACCOUNT" == "FD17944" ]]; then
+            echo "SNOWFLAKE_ACCOUNT is set to locator FD17944."
+            echo "Use the full account identifier instead, e.g. UTHZAZW-FJ87774"
+            exit 1
+          fi
+
+          if [[ "$ACCOUNT" != *"-"* ]]; then
+            echo "SNOWFLAKE_ACCOUNT should be the full account identifier (contains a hyphen)."
+            echo "Example: UTHZAZW-FJ87774"
+            exit 1
+          fi
+
+          printf '%s' "$KEY_B64" | base64 -d > "$KEY_PATH"
+          chmod 600 "$KEY_PATH"
+
+          if ! openssl pkey -in "$KEY_PATH" -noout >/dev/null 2>&1; then
+            echo "Invalid PEM after base64 decode."
+            echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 with:"
+            echo "base64 -i ~/.snowflake/keys/rsa_key.p8 | tr -d '\\n' | pbcopy"
+            exit 1
+          fi
+
+          KEY_FP="$(openssl pkey -in "$KEY_PATH" -pubout -outform DER | openssl dgst -sha256 -binary | openssl enc -base64)"
+          echo "Decoded key fingerprint: SHA256:${KEY_FP}="
+          echo "Expected fingerprint:      SHA256:cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM="
+
+          if [[ "$KEY_FP" != "cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" ]]; then
+            echo "Private key fingerprint does not match the public key on user SEBAUSTIN512."
+            echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 from ~/.snowflake/keys/rsa_key.p8"
+            exit 1
+          fi
+
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT=$ACCOUNT" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER=$USER" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_WAREHOUSE=$WAREHOUSE" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ROLE=$ROLE" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_DATABASE=COURSE_REPO" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_SCHEMA=PUBLIC" >> "$GITHUB_ENV"
+          echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PRIVATE_KEY_FILE=$KEY_PATH" >> "$GITHUB_ENV"
+
+          PASSPHRASE="$(trim "$SNOWFLAKE_PRIVATE_KEY_PASSPHRASE")"
+          if [[ -n "$PASSPHRASE" ]]; then
+            echo "PRIVATE_KEY_PASSPHRASE=$PASSPHRASE" >> "$GITHUB_ENV"
+          fi
+
+          echo "Account length: ${#ACCOUNT} (expected 17 for UTHZAZW-FJ87774)"
+          echo "User length: ${#USER} (expected 12 for SEBAUSTIN512)"
+
+      - name: Verify Snowflake connection
+        run: snow connection test
+
       - name: Fetch latest changes to Snowflake
-        run: snow git fetch course_repo.public.advanced_data_engineering_snowflake
+        run: snow git fetch COURSE_REPO.PUBLIC.ADVANCED_DATA_ENGINEERING_SNOWFLAKE
+
+      - name: Deploy data pipeline
+        run: |
+          echo "Deploying branch: ${DEPLOY_BRANCH} with env: ${SNOWFLAKE_ENV}"
+          snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/data/" \
+            -D "env='${SNOWFLAKE_ENV}'" \
+            --database=COURSE_REPO \
+            --schema=PUBLIC
 
-      # Deploy templates to the data environment -
-      - name: Deploy templates to data environment
+      - name: Deploy pipeline objects
         run: |
-          # Export TARGET_BRANCH for use in this step
-          echo "Using branch: ${GITHUB_REF_NAME}"
-          snow git execute @advanced_data_engineering_snowflake/branches/FIX-ME/module-1/hamburg_weather/pipeline/data/ \
-          -D "env='${{  }}'" \
-          --database=COURSE_REPO \
-          --schema=PUBLIC
+          snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/objects/" \
+            -D "env='${SNOWFLAKE_ENV}'" \
+            --database=COURSE_REPO \
+            --schema=PUBLIC

module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql

@@ -57,14 +57,15 @@ raw zone table build
 -- country table build
 
 -- todo: complete table build
-CREATE TABLE {{env}}_tasty_bytes.raw_pos.country
+CREATE OR ALTER TABLE {{env}}_tasty_bytes.raw_pos.country
 (
    country_id NUMBER(18,0),
    country VARCHAR(16777216),
    iso_currency VARCHAR(3),
    iso_country VARCHAR(2),
    city VARCHAR(16777216),
-   city_population VARCHAR(16777216)
+   city_population VARCHAR(16777216),
+   city_id NUMBER(19,0)
 );
 
 
@@ -293,17 +294,17 @@ USE WAREHOUSE demo_build_wh;
 
 
 -- country table load
--- COPY INTO {{env}}_tasty_bytes.raw_pos.country
--- (
---    country_id,
---    country,
---    iso_currency,
---    iso_country,
---    city_id,
---    city,
---    city_population
--- )
--- FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
+COPY INTO {{env}}_tasty_bytes.raw_pos.country
+(
+    country_id,
+   country,
+   iso_currency,
+  iso_country,
+    city_id,
+   city,
+   city_population
+)
+FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/;
 
 
 -- franchise table load