feat: Add new analytics models for F1 data transformation

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: ci
+on:
+  pull_request:
+  push:
+    branches: [main]
+jobs:
+  dbt-and-python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install dbt-duckdb pyarrow pytest
+      - name: Python unit tests (extractor, manifest, GCS loader)
+        run: python -m pytest -q airflow/tests
+      - name: Compile DAG and plugin code
+        run: python -m compileall -q airflow/dags airflow/plugins
+      - name: Build a synthetic OpenF1 dataset
+        run: python dbt_f1_transformations/ci/generate_fixtures.py "$RUNNER_TEMP/f1_raw"
+      - name: dbt deps
+        working-directory: dbt_f1_transformations
+        run: dbt deps --profiles-dir . --target ci
+      - name: dbt build (all models and all tests) on the synthetic data
+        working-directory: dbt_f1_transformations
+        env:
+          F1_RAW_DATA_PATH: ${{ runner.temp }}/f1_raw
+          F1_DUCKDB_PATH: ${{ runner.temp }}/f1_ci.duckdb
+        run: dbt build --profiles-dir . --target ci

.github/workflows/deploy_to_composer.yml

@@ -1,55 +1,23 @@
-name: Deploy to Cloud Composer
-
+name: deploy-to-composer
+# Manual trigger only until a Composer environment and credentials exist. NOT exercised in this repo.
 on:
-  push:
-    branches:
-      - main
   workflow_dispatch:
-
+    inputs:
+      dry_run:
+        description: "Dry run (list changes only)"
+        type: boolean
+        default: true
 jobs:
   deploy:
-    name: Deploy Airflow to Composer
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-
+    environment: production
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Google Cloud SDK
-        uses: google-github-actions/setup-gcloud@v1
+      - uses: actions/checkout@v4
+      - uses: google-github-actions/auth@v2
         with:
-          service_account_key: ${{ secrets.GCP_SA_KEY }}
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          export_default_credentials: true
-
-      - name: Configure gcloud project and region
-        run: |
-          gcloud config set project "${{ secrets.GCP_PROJECT_ID }}"
-          gcloud config set composer/location "${{ secrets.COMPOSER_REGION }}"
-          gcloud config set compute/region "${{ secrets.GCP_REGION }}"
-
-      - name: Validate Airflow Python files
-        run: |
-          find airflow/dags -name '*.py' -print0 | xargs -0 python -m py_compile
-          find airflow/plugins -name '*.py' -print0 | xargs -0 python -m py_compile
-        shell: bash
-
-      - name: Upload DAGs to Cloud Composer
-        run: |
-          gcloud composer environments storage dags import \
-            --environment="${{ secrets.COMPOSER_ENVIRONMENT }}" \
-            --location="${{ secrets.COMPOSER_REGION }}" \
-            --source="airflow/dags"
-
-      - name: Upload plugins to Cloud Composer
-        run: |
-          gcloud composer environments storage plugins import \
-            --environment="${{ secrets.COMPOSER_ENVIRONMENT }}" \
-            --location="${{ secrets.COMPOSER_REGION }}" \
-            --source="airflow/plugins"
-
-      - name: Confirm deployment
-        run: |
-          echo "Composer deployment completed for environment: ${{ secrets.COMPOSER_ENVIRONMENT }}"
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+      - uses: google-github-actions/setup-gcloud@v2
+      - name: Sync DAGs and plugins
+        env:
+          COMPOSER_BUCKET: ${{ secrets.COMPOSER_BUCKET }}
+        run: ./scripts/deploy_to_composer.sh ${{ inputs.dry_run && '--dry-run' || '' }}

.gitignore

@@ -222,9 +222,20 @@ dbt_modules/
 *.duckdb
 *.duckdb.wal
 *.parquet
-duckdb_tmp/
 output/
 
 opentofu/
 *.exe
 *.duckdb
+
+# --- added: secrets, local warehouse, CI scratch ---
+airflow/.env
+.env
+.ci_raw/
+duckdb_tmp/
+.duckdb_tmp/
+airflow/data/
+dbt_f1_transformations/target/
+dbt_f1_transformations/dbt_packages/
+dbt_f1_transformations/logs/
+logs/