docker.yml

name: Container images
env:
  DOCKER_CLI_EXPERIMENTAL: enabled
  DOCKER_BUILDKIT: 1  # Enable BuildKit for better performance
  COMPOSE_DOCKER_CLI_BUILD: 0

# Two routes, never mixed:
# - A pull request to develop or main that touches the container templates or
#   the runtime configuration they render gets a build-only gate: the images
#   are built and started locally on the runner and nothing is pushed.
# - Publication to the registry happens only from an explicit dispatch that
#   says so. There is no tag trigger and no label shortcut: nobody publishes
#   an image by pushing a ref or adding a label to a pull request.
on:
  pull_request:
    branches: [develop, main]
    types: [opened, synchronize, reopened]
    paths:
      - 'docker/**'
      - 'production/mempool-config.*.json'
      - 'production/mempool-frontend-config.*.json'
      - 'backend/mempool-config.sample.json'
      - 'frontend/mempool-frontend-config.sample.json'
      - 'nginx.conf'
      - 'nginx-mempool.conf'
      - 'rust/gbt/rust-toolchain'
      - 'backend/package.json'
      - 'backend/package-lock.json'
      - 'frontend/package.json'
      - 'frontend/package-lock.json'
      - '.github/workflows/docker.yml'
  workflow_dispatch:
    inputs:
      tag:
        description: 'Image tag to publish (for example universe-2026.09.17)'
        required: true
        type: string
      publish:
        description: 'Push the built images to the registry'
        required: true
        type: boolean
        default: false
      latest:
        description: 'Also move the latest tag to this build (publish must be true)'
        required: false
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  test-images:
    env:
      COMPOSE_PROJECT_NAME: mempool-images-${{ github.run_id }}-${{ github.run_attempt }}
    # Build-only: runs for every pull request in scope and every dispatch.
    runs-on: [self-hosted, linux, x64, universe-super]
    timeout-minutes: 30
    name: Test built Docker images
    steps:
      - name: Checkout project
        uses: actions/checkout@v4

      - name: Add SHORT_SHA env property with commit short sha
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            SHA="${{ github.event.pull_request.head.sha }}"
          else
            SHA="${GITHUB_SHA}"
          fi
          echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV

      - name: Set TAG from the dispatch input or package.json
        env:
          DISPATCH_TAG: ${{ github.event.inputs.tag }}
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            case "$DISPATCH_TAG" in
              ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;;
            esac
            TAG="$DISPATCH_TAG"
          else
            FRONTEND_VERSION=$(jq -r '.version' frontend/package.json)
            BACKEND_VERSION=$(jq -r '.version' backend/package.json)
            if [ "$FRONTEND_VERSION" != "$BACKEND_VERSION" ]; then
              echo "Error: Frontend version ($FRONTEND_VERSION) and backend version ($BACKEND_VERSION) do not match"
              exit 1
            fi
            TAG="v${FRONTEND_VERSION}-${SHORT_SHA}"
          fi
          echo "TAG=${TAG}" >> $GITHUB_ENV

      - name: Show set environment variables
        run: |
          printf "      TAG: %s\n"  "$TAG"
          printf " SHORT_SHA: %s\n" "$SHORT_SHA"

      - name: Init repo for Dockerization
        run: docker/init.sh "$TAG"

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build frontend image locally
        run: |
          docker buildx build \
            --tag test-frontend:$TAG \
            --build-arg commitHash=$SHORT_SHA \
            --load \
            --platform linux/amd64 \
            ./frontend/

      - name: Build backend image locally
        run: |
          docker buildx build \
            --tag test-backend:$TAG \
            --build-context rustgbt=./rust \
            --build-context backend=./backend \
            --build-arg commitHash=$SHORT_SHA \
            --load \
            --platform linux/amd64 \
            ./backend/

      - name: Prepare docker-compose test file
        run: |
          cat > "$RUNNER_TEMP/modify-compose-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT.py" << 'SCRIPT_END'
          import re
          import os
          import sys
          
          # Read the base docker-compose file
          with open('docker/docker-compose.yml', 'r') as f:
              content = f.read()
          
          # Get TAG from environment
          tag = os.environ.get('TAG', '')
          
          # Replace image names with locally built test images
          content = content.replace('image: mempool/frontend:latest', f'image: test-frontend:{tag}')
          content = content.replace('image: mempool/backend:latest', f'image: test-backend:{tag}')
          
          # Let Docker reserve a loopback host port atomically for this job.
          content = content.replace('- 80:8080', '- 127.0.0.1::8080')
          
          # Remove volumes from api service
          content = re.sub(r'    volumes:\n      - \.\/data:\/backend\/cache\n', '', content)
          
          # For db service: remove user and volumes, add tmpfs and healthcheck
          # Remove user line from db service (only the one in db service)
          lines = content.split('\n')
          in_db_service = False
          new_lines = []
          for i, line in enumerate(lines):
              if line.strip().startswith('db:'):
                  in_db_service = True
              elif line.strip() and not line.startswith(' ') and not line.startswith('\t'):
                  in_db_service = False
              if in_db_service and line.strip() == 'user: "1000:1000"':
                  continue
              new_lines.append(line)
          content = '\n'.join(new_lines)
          
          # Remove volumes section from db service
          content = re.sub(r'    volumes:\n      - \.\/mysql\/data:\/var\/lib\/mysql\n', '', content)
          
          # Add tmpfs after stop_grace_period in db service (healthcheck already exists in base file)
          db_stop_grace = '    stop_grace_period: 1m'
          db_additions = '    stop_grace_period: 1m\n    tmpfs:\n      - /var/lib/mysql'
          before_db, db_section = content.split('  db:\n', 1)
          content = before_db + '  db:\n' + db_section.replace(db_stop_grace, db_additions, 1)
          
          # Add depends_on to web service after ports
          web_ports = '    ports:\n      - 127.0.0.1::8080'
          web_with_depends = '    ports:\n      - 127.0.0.1::8080\n    depends_on:\n      - api\n      - db'
          content = content.replace(web_ports, web_with_depends, 1)
          
          # Add depends_on to api service after command
          api_command = '    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"'
          api_with_depends = '    command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"\n    depends_on:\n      - db'
          content = content.replace(api_command, api_with_depends, 1)
          
          # Write the modified content
          with open('docker-compose.test.yml', 'w') as f:
              f.write(content)
          
          print("Generated docker-compose.test.yml")
          SCRIPT_END
          python3 "$RUNNER_TEMP/modify-compose-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT.py"
          cat docker-compose.test.yml

      - name: Start containers
        run: |
          docker compose -f docker-compose.test.yml up -d

      - name: Wait for services to be ready
        run: |
          echo "Waiting for all services (web, api, db) to be healthy..."
          timeout=120
          elapsed=0
          while [ $elapsed -lt $timeout ]; do
            # Check health status for all services
            PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
            HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
            if [ "$HEALTHY_COUNT" -ge 3 ]; then
              echo "All services are healthy!"
              echo "$PS_OUTPUT"
              break
            fi
            echo "Waiting for services to be healthy... (${elapsed}s/${timeout}s)"
            echo "$PS_OUTPUT"
            sleep 2
            elapsed=$((elapsed + 2))
          done
          if [ $elapsed -ge $timeout ]; then
            echo "Services did not become healthy in time"
            docker compose -f docker-compose.test.yml ps
            docker compose -f docker-compose.test.yml logs
            exit 1
          fi

      - name: Verify containers are healthy
        run: |
          echo "Checking container health status..."
          PS_OUTPUT=$(docker compose -f docker-compose.test.yml ps)
          echo "$PS_OUTPUT"
          
          # Check that all three services (web, api, db) are healthy
          HEALTHY_COUNT=$(echo "$PS_OUTPUT" | grep -c "(healthy)" || true)
          if [ "$HEALTHY_COUNT" -lt 3 ]; then
            echo "Not all containers are healthy. Expected 3 healthy services, found $HEALTHY_COUNT"
            docker compose -f docker-compose.test.yml logs
            exit 1
          fi
          
          # Verify each service individually for better error messages
          if ! echo "$PS_OUTPUT" | grep -q "web.*(healthy)"; then
            echo "Web service is not healthy"
            docker compose -f docker-compose.test.yml logs web
            exit 1
          fi
          if ! echo "$PS_OUTPUT" | grep -q "api.*(healthy)"; then
            echo "API service is not healthy"
            docker compose -f docker-compose.test.yml logs api
            exit 1
          fi
          if ! echo "$PS_OUTPUT" | grep -q "db.*(healthy)"; then
            echo "Database service is not healthy"
            docker compose -f docker-compose.test.yml logs db
            exit 1
          fi
          
          echo "All containers are healthy!"

      - name: Show container logs
        if: failure()
        run: |
          docker compose -f docker-compose.test.yml logs

      - name: Clean up containers
        if: always()
        run: |
          docker compose -f docker-compose.test.yml down -v

  build:
    needs: [test-images]
    # Publication: only an explicit dispatch that asked for it, and only after
    # the build-only gate passed on the same commit.
    if: |
      needs.test-images.result == 'success' &&
      github.event_name == 'workflow_dispatch' &&
      github.event.inputs.publish == 'true'
    strategy:
      matrix:
        service:
          - frontend
          - backend
    runs-on: [self-hosted, linux, x64, universe-super]
    timeout-minutes: 120
    name: Build and push to DockerHub
    outputs:
      image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }}
      image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }}
      tag: ${{ matrix.service == 'frontend' && steps.set-tag.outputs.tag || '' }}
    steps:
      # Shared runner capacity is provisioned outside jobs. Never replace host
      # swap, hide Docker data, or restart the shared daemon during a build.
      - name: Set TAG from the dispatch input
        id: set-tag
        env:
          DISPATCH_TAG: ${{ github.event.inputs.tag }}
        run: |
          case "$DISPATCH_TAG" in
            ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;;
          esac
          echo "TAG=${DISPATCH_TAG}" >> $GITHUB_ENV
          echo "tag=${DISPATCH_TAG}" >> $GITHUB_OUTPUT

      - name: Add SHORT_SHA env property with commit short sha
        run: |
          SHA="${GITHUB_SHA}"
          echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV


      - name: Login to Docker for building
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Checkout project
        uses: actions/checkout@v4

      - name: Show set environment variables
        run: |
          printf "      TAG: %s\n"  "$TAG"
          printf " SHORT_SHA: %s\n" "$SHORT_SHA"

      - name: Init repo for Dockerization
        run: docker/init.sh "$TAG"

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: linux/amd64,linux/arm64
        id: qemu

      - name: Setup Docker buildx action
        uses: docker/setup-buildx-action@v3
        with:
          platforms: linux/amd64,linux/arm64
          driver-opts: |
            network=host
        id: buildx

      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

      - name: Cache Docker layers
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ matrix.service }}-

      - name: Run Docker buildx for ${{ matrix.service }} against tag
        id: docker-build
        run: |
          docker buildx build \
          --cache-from "type=local,src=/tmp/.buildx-cache" \
          --cache-to "type=local,dest=/tmp/.buildx-cache,mode=max" \
          --platform linux/amd64,linux/arm64 \
          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
          --build-context rustgbt=./rust \
          --build-context backend=./backend \
          --output "type=registry,push=true" \
          --build-arg commitHash=$SHORT_SHA \
          ./${{ matrix.service }}/

  tag-latest:
    needs: [build]
    # Only when the same dispatch asked for it.
    if: ${{ needs.build.result == 'success' && github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.latest == 'true' }}
    runs-on: [self-hosted, linux, x64, universe-super]
    timeout-minutes: 30
    name: Tag release build as latest
    strategy:
      matrix:
        service:
          - frontend
          - backend
    steps:
      - name: Set env variables
        env:
          DISPATCH_TAG: ${{ github.event.inputs.tag }}
        run: |
          case "$DISPATCH_TAG" in
            ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;;
          esac
          echo "TAG=${DISPATCH_TAG}" >> $GITHUB_ENV

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: linux/amd64,linux/arm64

      - name: Setup Docker buildx action
        uses: docker/setup-buildx-action@v3
        with:
          platforms: linux/amd64,linux/arm64
          driver-opts: |
            network=host

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Tag as latest for ${{ matrix.service }}
        run: |
          docker buildx imagetools create \
            --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
            ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG