mock-db-refresh.yml

name: Refresh Mock Database

on:
  workflow_dispatch:
  push:
    branches:
      - main
      - develop
    paths:
      - 'backend/src/main/java/net/modtale/model/**'
      - 'backend/src/main/java/net/modtale/repository/**'
      - 'mock-db/package-lock.json'
      - 'mock-db/package.json'
      - 'mock-db/scripts/**'

permissions:
  contents: read
  actions: write

concurrency:
  group: refresh-mock-database
  cancel-in-progress: false

jobs:
  refresh:
    name: Generate Sanitized Mock DB
    if: github.repository == 'Modtale/modtale'
    runs-on: ubuntu-latest
    environment: mock-refresh

    env:
      MOCK_SOURCE_MONGODB_URI: ${{ secrets.MOCK_SOURCE_MONGODB_URI }}
      MOCK_SOURCE_DATABASE_NAME: ${{ vars.MOCK_SOURCE_DATABASE_NAME }}
      MOCK_TEMPLATE_MONGODB_URI: ${{ secrets.MOCK_TEMPLATE_MONGODB_URI }}
      MOCK_TEMPLATE_DATABASE_NAME: ${{ vars.MOCK_TEMPLATE_DATABASE_NAME }}
      MOCK_DB_OUTPUT_DIR: ${{ github.workspace }}/mock-db/generated/collections
      MOCK_DB_COLLECTION_DIR: ${{ github.workspace }}/mock-db/generated/collections
      MOCK_R2_OBJECT_KEYS_FILE: ${{ github.workspace }}/mock-db/generated/r2-object-keys.txt
      MOCK_SOURCE_R2_BUCKET_NAME: ${{ vars.MOCK_SOURCE_R2_BUCKET_NAME }}
      MOCK_TEMPLATE_R2_BUCKET_NAME: ${{ vars.MOCK_TEMPLATE_R2_BUCKET_NAME }}
      MOCK_SOURCE_R2_ACCESS_KEY: ${{ secrets.MOCK_SOURCE_R2_ACCESS_KEY }}
      MOCK_SOURCE_R2_SECRET_KEY: ${{ secrets.MOCK_SOURCE_R2_SECRET_KEY }}
      MOCK_SOURCE_R2_ENDPOINT: ${{ secrets.MOCK_SOURCE_R2_ENDPOINT }}
      MOCK_TEMPLATE_R2_ACCESS_KEY: ${{ secrets.MOCK_TEMPLATE_R2_ACCESS_KEY }}
      MOCK_TEMPLATE_R2_SECRET_KEY: ${{ secrets.MOCK_TEMPLATE_R2_SECRET_KEY }}
      MOCK_TEMPLATE_R2_ENDPOINT: ${{ secrets.MOCK_TEMPLATE_R2_ENDPOINT }}
      CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
      CLOUDFLARE_R2_PROVISIONER: ${{ secrets.CLOUDFLARE_R2_PROVISIONER }}
      CLOUDFLARE_R2_JURISDICTION: ${{ vars.CLOUDFLARE_R2_JURISDICTION }}

    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Validate secret configuration
        run: |
          : "${MOCK_SOURCE_MONGODB_URI:?Set secret MOCK_SOURCE_MONGODB_URI with a read-only source MongoDB user.}"
          : "${MOCK_SOURCE_R2_ACCESS_KEY:?Set secret MOCK_SOURCE_R2_ACCESS_KEY with read-only access to the production source R2 bucket.}"
          : "${MOCK_SOURCE_R2_SECRET_KEY:?Set secret MOCK_SOURCE_R2_SECRET_KEY with read-only access to the production source R2 bucket.}"
          : "${MOCK_SOURCE_R2_ENDPOINT:?Set secret MOCK_SOURCE_R2_ENDPOINT for the production source R2 account.}"
          : "${MOCK_TEMPLATE_R2_ACCESS_KEY:?Set secret MOCK_TEMPLATE_R2_ACCESS_KEY with write access to the sanitized preview template R2 bucket.}"
          : "${MOCK_TEMPLATE_R2_SECRET_KEY:?Set secret MOCK_TEMPLATE_R2_SECRET_KEY with write access to the sanitized preview template R2 bucket.}"
          : "${MOCK_TEMPLATE_R2_ENDPOINT:?Set secret MOCK_TEMPLATE_R2_ENDPOINT for the sanitized preview template R2 bucket.}"
          : "${CLOUDFLARE_ACCOUNT_ID:?Set secret CLOUDFLARE_ACCOUNT_ID.}"
          : "${CLOUDFLARE_R2_PROVISIONER:?Set secret CLOUDFLARE_R2_PROVISIONER.}"

          source_db="${MOCK_SOURCE_DATABASE_NAME:-modtale}"
          template_db="${MOCK_TEMPLATE_DATABASE_NAME:-modtale-mock-template}"
          source_r2_bucket="${MOCK_SOURCE_R2_BUCKET_NAME:-modtale-binaries}"
          template_r2_bucket="${MOCK_TEMPLATE_R2_BUCKET_NAME:-modtale-preview-template}"

          if [ "$source_db" = "$template_db" ]; then
            echo "::error::MOCK_TEMPLATE_DATABASE_NAME must differ from MOCK_SOURCE_DATABASE_NAME. Refusing to load mock data into the source database."
            exit 1
          fi

          if [ "$source_r2_bucket" = "$template_r2_bucket" ]; then
            echo "::error::MOCK_TEMPLATE_R2_BUCKET_NAME must differ from MOCK_SOURCE_R2_BUCKET_NAME."
            exit 1
          fi

          echo "MOCK_SOURCE_DATABASE_NAME=$source_db" >> "$GITHUB_ENV"
          echo "MOCK_TEMPLATE_DATABASE_NAME=$template_db" >> "$GITHUB_ENV"
          echo "MOCK_SOURCE_R2_BUCKET_NAME=$source_r2_bucket" >> "$GITHUB_ENV"
          echo "MOCK_TEMPLATE_R2_BUCKET_NAME=$template_r2_bucket" >> "$GITHUB_ENV"

          if [ -z "${MOCK_TEMPLATE_MONGODB_URI:-}" ]; then
            echo "MOCK_TEMPLATE_MONGODB_URI=$MOCK_SOURCE_MONGODB_URI" >> "$GITHUB_ENV"
            echo "TEMPLATE_URI_SOURCE=MOCK_SOURCE_MONGODB_URI" >> "$GITHUB_ENV"
          else
            echo "TEMPLATE_URI_SOURCE=MOCK_TEMPLATE_MONGODB_URI" >> "$GITHUB_ENV"
          fi

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22.12.0

      - name: Install mock DB tooling
        working-directory: mock-db
        run: npm ci

      - name: Generate sanitized fixtures
        working-directory: mock-db
        run: npm run generate

      - name: Validate sanitized fixtures
        working-directory: mock-db
        run: npm run validate

      - name: Collect sanitized R2 artifact keys
        working-directory: mock-db
        run: npm run collect-r2-keys

      - name: Delete previous public mock DB artifacts
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          artifact_ids="$(
            gh api --paginate \
              "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=modtale-mock-db-json&per_page=100" \
              --jq '.artifacts[] | select(.expired == false) | .id'
          )"

          if [ -z "$artifact_ids" ]; then
            echo "No previous modtale-mock-db-json artifacts found."
            exit 0
          fi

          for artifact_id in $artifact_ids; do
            echo "Deleting previous mock DB artifact $artifact_id"
            gh api --method DELETE "/repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
          done

      - name: Upload public mock DB artifact
        uses: actions/upload-artifact@v4
        with:
          name: modtale-mock-db-json
          path: mock-db/generated/collections/*.json
          if-no-files-found: error
          retention-days: 30

      - name: Load preview template database
        working-directory: mock-db
        run: npm run load-template

      - name: Sync preview template R2 bucket
        run: |
          set -euo pipefail

          source_endpoint="$(printf '%s' "$MOCK_SOURCE_R2_ENDPOINT" | sed -E 's#(https?://[^/]+).*#\1#')"
          template_endpoint="$(printf '%s' "$MOCK_TEMPLATE_R2_ENDPOINT" | sed -E 's#(https?://[^/]+).*#\1#')"

          echo "::add-mask::$MOCK_SOURCE_R2_ACCESS_KEY"
          echo "::add-mask::$MOCK_SOURCE_R2_SECRET_KEY"
          echo "::add-mask::$MOCK_TEMPLATE_R2_ACCESS_KEY"
          echo "::add-mask::$MOCK_TEMPLATE_R2_SECRET_KEY"

          if ! command -v aws >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y awscli
          fi

          R2_BUCKET_NAME="$MOCK_TEMPLATE_R2_BUCKET_NAME" \
            node .github/scripts/cloudflare-r2-preview.mjs ensure-bucket

          AWS_ACCESS_KEY_ID="$MOCK_TEMPLATE_R2_ACCESS_KEY" \
          AWS_SECRET_ACCESS_KEY="$MOCK_TEMPLATE_R2_SECRET_KEY" \
          AWS_DEFAULT_REGION=auto \
          aws s3api head-bucket \
            --endpoint-url "$template_endpoint" \
            --bucket "$MOCK_TEMPLATE_R2_BUCKET_NAME" >/dev/null

          copied=0
          missing=0
          failed=0

          while IFS= read -r key || [ -n "$key" ]; do
            [ -n "$key" ] || continue

            object_file="$(mktemp)"
            if AWS_ACCESS_KEY_ID="$MOCK_SOURCE_R2_ACCESS_KEY" \
               AWS_SECRET_ACCESS_KEY="$MOCK_SOURCE_R2_SECRET_KEY" \
               AWS_DEFAULT_REGION=auto \
               aws s3 cp \
                 --endpoint-url "$source_endpoint" \
                 --only-show-errors \
                 "s3://$MOCK_SOURCE_R2_BUCKET_NAME/$key" \
                 "$object_file"; then
              if AWS_ACCESS_KEY_ID="$MOCK_TEMPLATE_R2_ACCESS_KEY" \
                 AWS_SECRET_ACCESS_KEY="$MOCK_TEMPLATE_R2_SECRET_KEY" \
                 AWS_DEFAULT_REGION=auto \
                 aws s3 cp \
                   --endpoint-url "$template_endpoint" \
                   --only-show-errors \
                   "$object_file" \
                   "s3://$MOCK_TEMPLATE_R2_BUCKET_NAME/$key"; then
                copied=$((copied + 1))
              else
                echo "::warning::Failed to upload sanitized R2 artifact '$key' to '$MOCK_TEMPLATE_R2_BUCKET_NAME'."
                failed=$((failed + 1))
              fi
            else
              echo "::warning::Source R2 artifact '$key' was not found in '$MOCK_SOURCE_R2_BUCKET_NAME'."
              missing=$((missing + 1))
            fi

            rm -f "$object_file"
          done < "$MOCK_R2_OBJECT_KEYS_FILE"

          echo "Sanitized R2 template sync finished: copied=$copied missing=$missing failed=$failed"

          if [ "$missing" -gt 0 ] || [ "$failed" -gt 0 ]; then
            echo "::error::Sanitized R2 template sync was incomplete."
            exit 1
          fi

      - name: Write summary
        run: |
          {
            echo "## Mock DB Refresh"
            echo ""
            echo "- Generated sanitized JSON artifact: \`modtale-mock-db-json\`"
            echo "- Refreshed template database: \`${MOCK_TEMPLATE_DATABASE_NAME:-modtale-mock-template}\`"
            echo "- Synced sanitized R2 template bucket: \`${MOCK_TEMPLATE_R2_BUCKET_NAME:-modtale-preview-template}\`"
            echo "- Template Mongo URI source: \`${TEMPLATE_URI_SOURCE:-MOCK_TEMPLATE_MONGODB_URI}\`"
            echo "- Source privileged collections were not read; reports/admin logs/API keys/status/analytics are synthetic."
          } >> "$GITHUB_STEP_SUMMARY"