Refactor dependencies and improve Docker integration

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+jobs:
+  # Run tests first
+  tests:
+    uses: ./.github/workflows/tests.yml
+
+  # Build and push Docker images (only on main or tags)
+  docker:
+    needs: tests
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    uses: ./.github/workflows/docker.yml
+    with:
+      push_images: ${{ github.event_name != 'pull_request' }}
+    secrets:
+      github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  # Publish to PyPI (only on main branch)
+  pypi:
+    needs: tests
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    uses: ./.github/workflows/publish-pypi.yml
+    secrets:
+      semantic_release_token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
+      pypi_token: ${{ secrets.PYPI_TOKEN }}

.github/workflows/docker.yml

@@ -0,0 +1,95 @@
+name: Build and Push Docker Images (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      push_images:
+        description: 'Whether to push images to registry'
+        required: false
+        type: boolean
+        default: true
+    secrets:
+      github_token:
+        description: 'GitHub token for container registry'
+        required: true
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Define dbt-core version range
+        # Update these versions as new releases become available
+        dbt_version:
+          - '1.8.0'
+          - '1.8.7'
+          - '1.9.0'
+          - '1.9.1'
+          - '1.10.0'
+          - '1.10.13'
+          - '1.11.0'
+          - '1.11.5'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        if: inputs.push_images
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.github_token }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=dbt-${{ matrix.dbt_version }}
+            type=raw,value=latest,enable=${{ matrix.dbt_version == '1.10.13' }}
+            type=sha,prefix=dbt-${{ matrix.dbt_version }}-
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ inputs.push_images }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            DBT_CORE_VERSION=${{ matrix.dbt_version }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Test Docker image
+        run: |
+          docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dbt-${{ matrix.dbt_version }} --help
+
+  summary:
+    runs-on: ubuntu-latest
+    needs: build-and-push
+    if: always()
+    steps:
+      - name: Check build status
+        run: |
+          if [ "${{ needs.build-and-push.result }}" == "success" ]; then
+            echo "✅ All Docker images built successfully!"
+          else
+            echo "❌ Some Docker image builds failed"
+            exit 1
+          fi

.github/workflows/publish.yml → .github/workflows/publish-pypi.yml

@@ -1,8 +1,14 @@
-name: Publish to PyPI
+name: Publish to PyPI (Reusable)
 
 on:
-  push:
-    branches: [ main ]
+  workflow_call:
+    secrets:
+      semantic_release_token:
+        description: 'Token for semantic release'
+        required: true
+      pypi_token:
+        description: 'Token for PyPI publishing'
+        required: true
 
 jobs:
   release:
@@ -16,7 +22,7 @@ jobs:
       with:
         fetch-depth: 0
         persist-credentials: true
-        token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
+        token: ${{ secrets.semantic_release_token }}
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -37,6 +43,6 @@ jobs:
       id: release
       uses: python-semantic-release/publish-action@v9.14.0
       with:
-        github_token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
+        github_token: ${{ secrets.semantic_release_token }}
         repository_username: __token__
-        repository_password: ${{ secrets.PYPI_TOKEN }}
+        repository_password: ${{ secrets.pypi_token }}

.github/workflows/tests.yml

@@ -1,6 +1,7 @@
-name: Tests
+name: Tests (Reusable)
 
 on:
+  workflow_call:
   pull_request:
     branches: [ main ]
 

Dockerfile

@@ -1,16 +1,15 @@
-FROM ghcr.io/dbt-labs/dbt-core:1.11.3
+FROM python:3.12-slim
 
-RUN pip install dbt-duckdb
+# Build argument for dbt-core version
+ARG DBT_CORE_VERSION=1.10.13
 
-# Copy dbt project (or mount at runtime with -v)
-COPY dbt /dbt
-WORKDIR /dbt
+WORKDIR /app
+COPY . /app
 
-# Install dbt dependencies
-RUN dbt deps
+# Install dependencies excluding dbt-core, then install specific dbt-core version
+RUN grep -v "^dbt-core" requirements.txt > /tmp/requirements.txt && \
+    pip install --no-cache-dir -r /tmp/requirements.txt && \
+    pip install --no-cache-dir dbt-core==${DBT_CORE_VERSION} && \
+    rm /tmp/requirements.txt
 
-# Set entrypoint to dbt command
-ENTRYPOINT ["dbt"]
-
-# Default to showing version
-CMD ["--version"]
+ENTRYPOINT ["python", "-m", "src.cli"]

Pipfile

@@ -4,23 +4,23 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-click = ">=8.0.0"
-pyyaml = ">=6.0"
-dbt-core = ">=1.10.13"
-docker = "*"
+click = "==8.1.8"
+pyyaml = "==6.0.3"
+dbt-core = "==1.10.13"
+docker = "==7.1.0"
+boto3 = "==1.35.0"
+google-cloud-bigquery = "==3.27.0"
+google-cloud-storage = "==2.19.0"
 
 [dev-packages]
-pytest = ">=7.0.0"
-black = ">=22.0.0"
-mypy = ">=0.950"
-flake8 = ">=6.0.0"
-duckdb = ">=0.8.0"
-google-cloud-bigquery = ">=3.0.0"
-google-cloud-storage = "*"
-boto3 = "*"
-slack-sdk = "*"
-pytest-cov = "*"
-pytest-mock = "*"
+pytest = "==8.3.5"
+black = "==24.10.0"
+mypy = "==1.14.1"
+flake8 = "==7.1.2"
+duckdb = "==1.2.0"
+slack-sdk = "==3.36.0"
+pytest-cov = "==6.0.0"
+pytest-mock = "==3.14.0"
 
 [requires]
 python_version = "3.12"