diff --git a/.dvc/config b/.dvc/config index 1d45389..f85dfd1 100644 --- a/.dvc/config +++ b/.dvc/config @@ -2,3 +2,6 @@ remote = dagshub ['remote "dagshub"'] url = https://dagshub.com/miles2542/rossmann-ops.dvc + auth = basic + user = ${DAGSHUB_USERNAME} + password = ${DAGSHUB_PAT} diff --git a/.env.example b/.env.example index dfe81b2..a9a8edd 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,5 @@ MLFLOW_TRACKING_URI=https://dagshub.com//.mlflow MLFLOW_TRACKING_USERNAME= MLFLOW_TRACKING_PASSWORD= GITHUB_PAT= +DAGSHUB_USERNAME= +DAGSHUB_PAT= diff --git a/.github/workflows/mlops_pipeline.yaml b/.github/workflows/mlops_pipeline.yaml index 143f86d..daa3f95 100644 --- a/.github/workflows/mlops_pipeline.yaml +++ b/.github/workflows/mlops_pipeline.yaml @@ -51,7 +51,7 @@ jobs: # Strict: a failing test DOES block train + build jobs - name: Run tests - run: uv run pytest tests/ -v + run: uv run pytest tests/ -v --cov=src --cov-report=term-missing # ─── Job 2: Train & Export ───────────────────────────────────────────────── # Runs on: push to main (or tag), repository_dispatch (CT from drift) @@ -60,8 +60,10 @@ jobs: name: Train Model runs-on: ubuntu-latest needs: ci - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'repository_dispatch' + if: (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))) || github.event_name == 'repository_dispatch' env: + DAGSHUB_USERNAME: ${{ secrets.DAGSHUB_USERNAME }} + DAGSHUB_PAT: ${{ secrets.DAGSHUB_PAT }} MLFLOW_TRACKING_URI: https://dagshub.com/${{ secrets.DAGSHUB_USERNAME }}/rossmann-ops.mlflow MLFLOW_TRACKING_USERNAME: ${{ secrets.DAGSHUB_USERNAME }} MLFLOW_TRACKING_PASSWORD: ${{ secrets.DAGSHUB_PAT }} @@ -75,12 +77,6 @@ jobs: - name: Install dependencies run: uv sync --frozen - - name: Configure DVC credentials - run: | - uv run dvc remote modify dagshub --local auth basic - uv run dvc remote modify dagshub --local user ${{ secrets.DAGSHUB_USERNAME }} - uv run dvc remote modify dagshub --local password ${{ secrets.DAGSHUB_PAT }} - - name: Pull data from DagsHub run: uv run dvc pull -r dagshub @@ -108,6 +104,8 @@ jobs: if: github.event_name == 'workflow_dispatch' env: GITHUB_PAT: ${{ secrets.GITHUB_PAT }} + DAGSHUB_USERNAME: ${{ secrets.DAGSHUB_USERNAME }} + DAGSHUB_PAT: ${{ secrets.DAGSHUB_PAT }} MLFLOW_TRACKING_URI: https://dagshub.com/${{ secrets.DAGSHUB_USERNAME }}/rossmann-ops.mlflow MLFLOW_TRACKING_USERNAME: ${{ secrets.DAGSHUB_USERNAME }} MLFLOW_TRACKING_PASSWORD: ${{ secrets.DAGSHUB_PAT }} @@ -121,12 +119,8 @@ jobs: - name: Install dependencies run: uv sync --frozen - - name: Configure DVC + pull data - run: | - uv run dvc remote modify dagshub --local auth basic - uv run dvc remote modify dagshub --local user ${{ secrets.DAGSHUB_USERNAME }} - uv run dvc remote modify dagshub --local password ${{ secrets.DAGSHUB_PAT }} - uv run dvc pull -r dagshub + - name: Pull data from DagsHub + run: uv run dvc pull -r dagshub # attack mode correctly exits 1 (detects poisoning) — mark continue-on-error # so the job passes and logs are visible, not swallowed in a red job failure @@ -208,21 +202,71 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Create release and attach k8s manifests + - name: Download model artifacts + uses: actions/download-artifact@v4 + with: + name: production-model + path: models/ + + - name: Rewrite manifests for DockerHub + run: | + sed -i "s|image: rossmann-api:latest|image: ${{ secrets.DOCKERHUB_USERNAME }}/rossmann-api:${{ github.ref_name }}|g" k8s/api.yaml + sed -i "s|image: rossmann-ui:latest|image: ${{ secrets.DOCKERHUB_USERNAME }}/rossmann-ui:${{ github.ref_name }}|g" k8s/ui.yaml + + - name: Create Deployment Bundle (ZIP) + run: | + zip -r deployment-bundle-${{ github.ref_name }}.zip \ + src/ \ + scripts/ \ + k8s/ \ + configs/ \ + Justfile \ + pyproject.toml \ + uv.lock \ + models/store_target_means.json + + - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: files: | + deployment-bundle-${{ github.ref_name }}.zip k8s/api.yaml k8s/ui.yaml body: | - ## Docker Images - ``` - ${{ secrets.DOCKERHUB_USERNAME }}/rossmann-api:${{ github.ref_name }} - ${{ secrets.DOCKERHUB_USERNAME }}/rossmann-ui:${{ github.ref_name }} - ``` - - ## Deploy to Kubernetes - ```bash - kubectl apply -f k8s/api.yaml - kubectl apply -f k8s/ui.yaml - ``` + # Release Specification: ${{ github.ref_name }} + + This release provides the finalized configuration and artifacts for the evaluation of the Rossmann Store Sales demand forecasting system. + + ## Asset Overview + + ### Optimized Kubernetes Manifests + - **api.yaml & ui.yaml**: Version-controlled manifests pre-configured to resolve container images directly from DockerHub. These files allow for immediate deployment to a Kubernetes cluster without requiring a local build environment or source code access. + + ### Deployment Bundle + - **deployment-bundle-${{ github.ref_name }}.zip**: A comprehensive archive containing the project source code, infrastructure automation logic, and pre-trained statistical artifacts. Required for local simulation, retraining, or a full system audit. + + ## Deployment Instructions + + ### Option 1: Direct Container Deployment + This method utilizes the pre-built images hosted on DockerHub. + + 1. Initialize the local cluster infrastructure: + ```bash + kind create cluster --name rossmann-cluster + ``` + 2. Apply the versioned manifests: + ```bash + kubectl apply -f api.yaml + kubectl apply -f ui.yaml + ``` + + ### Option 2: Full System Replication + This method initializes the complete MLOps environment. + + 1. Extract the deployment bundle ZIP. + 2. Ensure a Python 3.12 environment and the `uv` package manager are installed. + 3. Execute the setup sequence: + ```bash + just setup + ``` + 4. Verify cluster status or run local simulations as documented in the source `README.md`. diff --git a/.gitignore b/.gitignore index 4aaea45..9116b20 100644 --- a/.gitignore +++ b/.gitignore @@ -250,5 +250,7 @@ models/** *.pkl *.onnx *.ubj +mlflow.db +.coverage .ipynb_checkpoints/ diff --git a/mlruns/mlflow.db.dvc b/mlruns/mlflow.db.dvc new file mode 100644 index 0000000..89e3547 --- /dev/null +++ b/mlruns/mlflow.db.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 4aeac1783ef742745f35fc872c695bb5 + size: 663552 + hash: md5 + path: mlflow.db diff --git a/notebooks/03_optuna.ipynb b/notebooks/03_optuna.ipynb index bbec81f..ca9a0e4 100644 --- a/notebooks/03_optuna.ipynb +++ b/notebooks/03_optuna.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "0e99a74b", + "id": "0", "metadata": {}, "source": [ "# **Hyperparameter Tuning (Optuna)**\n", @@ -15,7 +15,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a43c71c3", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -58,7 +58,7 @@ }, { "cell_type": "markdown", - "id": "7e541216", + "id": "2", "metadata": {}, "source": [ "## **1. Data Setup & Split Logic**\n", @@ -68,7 +68,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4fb4af1a", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -94,7 +94,7 @@ }, { "cell_type": "markdown", - "id": "66c48248", + "id": "4", "metadata": {}, "source": [ "## **2. Feature & Folding Setup**\n", @@ -104,7 +104,7 @@ { "cell_type": "code", "execution_count": null, - "id": "70b16f90", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -168,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "82bfcffc", + "id": "6", "metadata": {}, "source": [ "## **3. Optuna Objective**\n", @@ -178,7 +178,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5298dd2a", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -235,7 +235,7 @@ }, { "cell_type": "markdown", - "id": "7adc4512", + "id": "8", "metadata": {}, "source": [ "## **4. Run Optimization (Example)**\n", @@ -245,7 +245,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3749c21b", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -263,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "7393e098", + "id": "10", "metadata": {}, "source": [ "## **5. Tuning Summary & Design Justfication**\n", diff --git a/scripts/check_docker.py b/scripts/check_docker.py index f0c092c..8642400 100644 --- a/scripts/check_docker.py +++ b/scripts/check_docker.py @@ -1,4 +1,5 @@ import logging +import os import subprocess import sys import time @@ -24,7 +25,8 @@ def start_docker_desktop(): logger.info("Docker not running. Attempting to start Docker Desktop...") if platform == "win32": - docker_path = Path(r"C:\Program Files\Docker\Docker\Docker Desktop.exe") + prog_files = os.environ.get("ProgramFiles", "C:\\Program Files") + docker_path = Path(prog_files) / "Docker" / "Docker" / "Docker Desktop.exe" if docker_path.exists(): subprocess.Popen([str(docker_path)]) else: