diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..25b7e14a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,146 @@ +name: Deploy Templatized Data Environment + +on: + pull_request: + types: [closed] + branches: + - staging + - main + push: + branches: + - staging + - main + +permissions: + contents: read + +jobs: + deploy_data_environment: + # Only run when PR is merged or on direct push to environment branches + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set deployment target + id: set_env + run: | + TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" + + if [[ "$TARGET_BRANCH" == "staging" ]]; then + echo "SNOWFLAKE_ENV=STAGING" >> "$GITHUB_ENV" + echo "DEPLOY_BRANCH=staging" >> "$GITHUB_ENV" + echo "Environment set to staging" + elif [[ "$TARGET_BRANCH" == "main" ]]; then + echo "SNOWFLAKE_ENV=PROD" >> "$GITHUB_ENV" + echo "DEPLOY_BRANCH=main" >> "$GITHUB_ENV" + echo "Environment set to production" + else + echo "Unexpected branch: $TARGET_BRANCH" + exit 1 + fi + + - name: Install Snowflake CLI + uses: snowflakedb/snowflake-cli-action@v1.5 + with: + cli-version: "latest" + default-config-file-path: "config.toml" + + - name: Configure Snowflake connection + env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_PRIVATE_KEY_B64: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_B64 }} + SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }} + run: | + trim() { printf '%s' "$1" | tr -d '[:space:]'; } + + ACCOUNT="$(trim "$SNOWFLAKE_ACCOUNT")" + USER="$(trim "$SNOWFLAKE_USER")" + WAREHOUSE="$(trim "$SNOWFLAKE_WAREHOUSE")" + ROLE="$(trim "$SNOWFLAKE_ROLE")" + KEY_B64="$(trim "$SNOWFLAKE_PRIVATE_KEY_B64")" + KEY_PATH="${RUNNER_TEMP}/snowflake_rsa_key.p8" + + if [[ -z "$ACCOUNT" || -z "$USER" || -z "$WAREHOUSE" || -z "$ROLE" || -z "$KEY_B64" ]]; then + echo "Missing one or more required secrets." + echo "Required: SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_ROLE, SNOWFLAKE_PRIVATE_KEY_B64" + exit 1 + fi + + # JWT auth requires the account identifier (e.g. ORG-ACCOUNT), not the locator alone. + if [[ "$ACCOUNT" == "FD17944" ]]; then + echo "SNOWFLAKE_ACCOUNT is set to locator FD17944." + echo "Use the full account identifier instead, e.g. UTHZAZW-FJ87774" + exit 1 + fi + + if [[ "$ACCOUNT" != *"-"* ]]; then + echo "SNOWFLAKE_ACCOUNT should be the full account identifier (contains a hyphen)." + echo "Example: UTHZAZW-FJ87774" + exit 1 + fi + + printf '%s' "$KEY_B64" | base64 -d > "$KEY_PATH" + chmod 600 "$KEY_PATH" + + if ! openssl pkey -in "$KEY_PATH" -noout >/dev/null 2>&1; then + echo "Invalid PEM after base64 decode." + echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 with:" + echo "base64 -i ~/.snowflake/keys/rsa_key.p8 | tr -d '\\n' | pbcopy" + exit 1 + fi + + KEY_FP="$(openssl pkey -in "$KEY_PATH" -pubout -outform DER | openssl dgst -sha256 -binary | openssl enc -base64)" + echo "Decoded key fingerprint: SHA256:${KEY_FP}=" + echo "Expected fingerprint: SHA256:cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" + + if [[ "$KEY_FP" != "cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" ]]; then + echo "Private key fingerprint does not match the public key on user SEBAUSTIN512." + echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 from ~/.snowflake/keys/rsa_key.p8" + exit 1 + fi + + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT=$ACCOUNT" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER=$USER" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_WAREHOUSE=$WAREHOUSE" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ROLE=$ROLE" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_DATABASE=COURSE_REPO" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_SCHEMA=PUBLIC" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PRIVATE_KEY_FILE=$KEY_PATH" >> "$GITHUB_ENV" + + PASSPHRASE="$(trim "$SNOWFLAKE_PRIVATE_KEY_PASSPHRASE")" + if [[ -n "$PASSPHRASE" ]]; then + echo "PRIVATE_KEY_PASSPHRASE=$PASSPHRASE" >> "$GITHUB_ENV" + fi + + echo "Account length: ${#ACCOUNT} (expected 17 for UTHZAZW-FJ87774)" + echo "User length: ${#USER} (expected 12 for SEBAUSTIN512)" + + - name: Verify Snowflake connection + run: snow connection test + + - name: Fetch latest changes to Snowflake + run: snow git fetch COURSE_REPO.PUBLIC.ADVANCED_DATA_ENGINEERING_SNOWFLAKE + + - name: Deploy data pipeline + run: | + echo "Deploying branch: ${DEPLOY_BRANCH} with env: ${SNOWFLAKE_ENV}" + snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/data/" \ + -D "env='${SNOWFLAKE_ENV}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC + + - name: Deploy pipeline objects + run: | + snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/objects/" \ + -D "env='${SNOWFLAKE_ENV}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC diff --git a/module-1/hamburg_weather/main.yml b/module-1/hamburg_weather/main.yml index bc93eac2..25b7e14a 100644 --- a/module-1/hamburg_weather/main.yml +++ b/module-1/hamburg_weather/main.yml @@ -11,57 +11,136 @@ on: - staging - main +permissions: + contents: read + jobs: deploy_data_environment: # Only run when PR is merged or on direct push to environment branches if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) runs-on: ubuntu-latest - env: - # Read connection secret - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - steps: - # Checkout step is necessary to access repository files - name: Checkout repository uses: actions/checkout@v4 + with: + persist-credentials: false - # Determine environment based on target branch - - name: Set environment + - name: Set deployment target id: set_env run: | TARGET_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}" - - if [[ $TARGET_BRANCH == "staging" ]]; then - echo "DEPLOY_ENV=staging" >> $GITHUB_ENV + + if [[ "$TARGET_BRANCH" == "staging" ]]; then + echo "SNOWFLAKE_ENV=STAGING" >> "$GITHUB_ENV" + echo "DEPLOY_BRANCH=staging" >> "$GITHUB_ENV" echo "Environment set to staging" - elif [[ $TARGET_BRANCH == "main" ]]; then - echo "DEPLOY_ENV=prod" >> $GITHUB_ENV + elif [[ "$TARGET_BRANCH" == "main" ]]; then + echo "SNOWFLAKE_ENV=PROD" >> "$GITHUB_ENV" + echo "DEPLOY_BRANCH=main" >> "$GITHUB_ENV" echo "Environment set to production" else echo "Unexpected branch: $TARGET_BRANCH" exit 1 fi - # Install Snowflake CLI GitHub Action and point to config file - - name: Install SnowflakeCLI + - name: Install Snowflake CLI uses: snowflakedb/snowflake-cli-action@v1.5 with: cli-version: "latest" default-config-file-path: "config.toml" - # Fetch latest changes from the repository to Snowflake + - name: Configure Snowflake connection + env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_PRIVATE_KEY_B64: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_B64 }} + SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }} + run: | + trim() { printf '%s' "$1" | tr -d '[:space:]'; } + + ACCOUNT="$(trim "$SNOWFLAKE_ACCOUNT")" + USER="$(trim "$SNOWFLAKE_USER")" + WAREHOUSE="$(trim "$SNOWFLAKE_WAREHOUSE")" + ROLE="$(trim "$SNOWFLAKE_ROLE")" + KEY_B64="$(trim "$SNOWFLAKE_PRIVATE_KEY_B64")" + KEY_PATH="${RUNNER_TEMP}/snowflake_rsa_key.p8" + + if [[ -z "$ACCOUNT" || -z "$USER" || -z "$WAREHOUSE" || -z "$ROLE" || -z "$KEY_B64" ]]; then + echo "Missing one or more required secrets." + echo "Required: SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_ROLE, SNOWFLAKE_PRIVATE_KEY_B64" + exit 1 + fi + + # JWT auth requires the account identifier (e.g. ORG-ACCOUNT), not the locator alone. + if [[ "$ACCOUNT" == "FD17944" ]]; then + echo "SNOWFLAKE_ACCOUNT is set to locator FD17944." + echo "Use the full account identifier instead, e.g. UTHZAZW-FJ87774" + exit 1 + fi + + if [[ "$ACCOUNT" != *"-"* ]]; then + echo "SNOWFLAKE_ACCOUNT should be the full account identifier (contains a hyphen)." + echo "Example: UTHZAZW-FJ87774" + exit 1 + fi + + printf '%s' "$KEY_B64" | base64 -d > "$KEY_PATH" + chmod 600 "$KEY_PATH" + + if ! openssl pkey -in "$KEY_PATH" -noout >/dev/null 2>&1; then + echo "Invalid PEM after base64 decode." + echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 with:" + echo "base64 -i ~/.snowflake/keys/rsa_key.p8 | tr -d '\\n' | pbcopy" + exit 1 + fi + + KEY_FP="$(openssl pkey -in "$KEY_PATH" -pubout -outform DER | openssl dgst -sha256 -binary | openssl enc -base64)" + echo "Decoded key fingerprint: SHA256:${KEY_FP}=" + echo "Expected fingerprint: SHA256:cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" + + if [[ "$KEY_FP" != "cLN7wwfsL1rWFglmlXVjNtHAp1xpleISJzHFpq0gEsM=" ]]; then + echo "Private key fingerprint does not match the public key on user SEBAUSTIN512." + echo "Re-create SNOWFLAKE_PRIVATE_KEY_B64 from ~/.snowflake/keys/rsa_key.p8" + exit 1 + fi + + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ACCOUNT=$ACCOUNT" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_USER=$USER" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_WAREHOUSE=$WAREHOUSE" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_ROLE=$ROLE" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_DATABASE=COURSE_REPO" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_SCHEMA=PUBLIC" >> "$GITHUB_ENV" + echo "SNOWFLAKE_CONNECTIONS_ADVANCED_DATA_ENGINEERING_SNOWFLAKE_PRIVATE_KEY_FILE=$KEY_PATH" >> "$GITHUB_ENV" + + PASSPHRASE="$(trim "$SNOWFLAKE_PRIVATE_KEY_PASSPHRASE")" + if [[ -n "$PASSPHRASE" ]]; then + echo "PRIVATE_KEY_PASSPHRASE=$PASSPHRASE" >> "$GITHUB_ENV" + fi + + echo "Account length: ${#ACCOUNT} (expected 17 for UTHZAZW-FJ87774)" + echo "User length: ${#USER} (expected 12 for SEBAUSTIN512)" + + - name: Verify Snowflake connection + run: snow connection test + - name: Fetch latest changes to Snowflake - run: snow git fetch course_repo.public.advanced_data_engineering_snowflake + run: snow git fetch COURSE_REPO.PUBLIC.ADVANCED_DATA_ENGINEERING_SNOWFLAKE + + - name: Deploy data pipeline + run: | + echo "Deploying branch: ${DEPLOY_BRANCH} with env: ${SNOWFLAKE_ENV}" + snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/data/" \ + -D "env='${SNOWFLAKE_ENV}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC - # Deploy templates to the data environment - - - name: Deploy templates to data environment + - name: Deploy pipeline objects run: | - # Export TARGET_BRANCH for use in this step - echo "Using branch: ${GITHUB_REF_NAME}" - snow git execute @advanced_data_engineering_snowflake/branches/FIX-ME/module-1/hamburg_weather/pipeline/data/ \ - -D "env='${{ }}'" \ - --database=COURSE_REPO \ - --schema=PUBLIC \ No newline at end of file + snow git execute "@advanced_data_engineering_snowflake/branches/${DEPLOY_BRANCH}/module-1/hamburg_weather/pipeline/objects/" \ + -D "env='${SNOWFLAKE_ENV}'" \ + --database=COURSE_REPO \ + --schema=PUBLIC diff --git a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql index 4565f538..993f7720 100644 --- a/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql +++ b/module-1/hamburg_weather/pipeline/data/load_tasty_bytes.sql @@ -57,14 +57,15 @@ raw zone table build -- country table build -- todo: complete table build -CREATE TABLE {{env}}_tasty_bytes.raw_pos.country +CREATE OR ALTER TABLE {{env}}_tasty_bytes.raw_pos.country ( country_id NUMBER(18,0), country VARCHAR(16777216), iso_currency VARCHAR(3), iso_country VARCHAR(2), city VARCHAR(16777216), - city_population VARCHAR(16777216) + city_population VARCHAR(16777216), + city_id NUMBER(19,0) ); @@ -293,17 +294,17 @@ USE WAREHOUSE demo_build_wh; -- country table load --- COPY INTO {{env}}_tasty_bytes.raw_pos.country --- ( --- country_id, --- country, --- iso_currency, --- iso_country, --- city_id, --- city, --- city_population --- ) --- FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/; +COPY INTO {{env}}_tasty_bytes.raw_pos.country +( + country_id, + country, + iso_currency, + iso_country, + city_id, + city, + city_population +) +FROM @{{env}}_tasty_bytes.public.s3load/raw_pos/country/; -- franchise table load