Tiler monitoring pipeline

.github/workflows/chartpress.yaml

@@ -4,7 +4,7 @@ on:
     branches:
       - 'main'
       - 'staging'
-      - 'k8s_deploy'
+      - 'tiler_monitoring'
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -71,7 +71,7 @@ jobs:
           OHM_SLACK_WEBHOOK_URL: ${{ secrets.OHM_SLACK_WEBHOOK_URL }}
       ################ Staging secrets ################ 
       - name: Staging - substitute secrets
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
         uses: bluwy/substitute-string-action@v1
         with:
           _input-file: 'values.staging.template.yaml'
@@ -189,46 +189,46 @@ jobs:
           PRODUCTION_OPENSTREETMAP_AUTH_SECRET: ${{ secrets.PRODUCTION_OPENSTREETMAP_AUTH_SECRET }}
 
       - name: AWS Credentials
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: Setup Kubectl and Helm Dependencies
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: |
           sudo pip install awscli --ignore-installed six
-          sudo curl -L -o /usr/bin/kubectl https://dl.k8s.io/release/v1.33.0/bin/linux/amd64/kubectl
+          sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl
           sudo chmod +x /usr/bin/kubectl
-          sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator
+          sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator
           sudo chmod +x /usr/bin/aws-iam-authenticator
-          curl -L https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz -o helm.tar.gz
+          curl -L https://get.helm.sh/helm-v3.14.4-linux-amd64.tar.gz -o helm.tar.gz
           tar -xvzf helm.tar.gz
           sudo mv linux-amd64/helm /usr/local/bin/
           sudo chmod +x /usr/local/bin/helm
           helm version
 
       - name: Update kube-config staging
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging
       - name: Update kube-config prod
         if: github.ref == 'refs/heads/main'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2
       - name: Add Helm repository
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: |
           helm repo add osm-seed https://osm-seed.github.io/osm-seed-chart/
           helm repo update
       - name: Install helm dependencies for
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: cd ohm && helm dep up
       # Staging
       - name: Staging - helm deploy
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
-        run: helm upgrade --install staging --wait --timeout 10m ohm/ -f values.staging.yaml -f ohm/values.yaml
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
+        run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
       # Production
       - name: Production - helm deploy
         if: github.ref == 'refs/heads/main'
-        run: helm upgrade --install production --wait --timeout 10m ohm/ -f values.production.yaml -f ohm/values.yaml
+        run: helm upgrade --install production --wait ohm/ -f values.production.yaml -f ohm/values.yaml
 

.gitignore

@@ -33,8 +33,8 @@ hetzner/*/.envs.*.production
 hetzner/traefik/cloudflare-ips.txt
 hetzner/traefik/traefik.yml
 .vscode/
+.claude/
 imposm3.json
-
 cachedir_reimport/
 config_reimport.json
 imposm3_reimport.json

compose/tiler.yml

@@ -71,21 +71,24 @@ services:
   #     - ohm_network
 
 
-  # tiler-monitor:
-  #   image: rub21/tiler-monitor:v1
-  #   build:
-  #     context: ../images/tiler-monitor
-  #     dockerfile: Dockerfile
-  #   volumes:
-  #     - /var/run/docker.sock:/var/run/docker.sock
-  #     - ../images/tiler-monitor:/app
-  #     - ../hetzner:/app/hetzner
-  #   environment:
-  #     - DOCKER_CONFIG_ENVIRONMENT=staging
-  #   env_file:
-  #     - ../envs/.env.tiler
-  #   stdin_open: true
-  #   tty: true
+  tiler-monitor:
+    image: rub21/tiler-monitor:v2
+    build:
+      context: ../images/tiler-monitor
+      dockerfile: Dockerfile
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ../hetzner:/app/hetzner
+      - tiler_monitor_data:/data
+    ports:
+      - "8001:8001"
+    environment:
+      - TILER_MONITORING_DOCKER_CONFIG_ENVIRONMENT=staging
+    env_file:
+      - ../envs/.env.tiler
+    restart: always
+    networks:
+      - ohm_network
 
 networks:
   ohm_network:
@@ -99,3 +102,7 @@ volumes:
   tiler_imposm_data:
     driver: local
     name: tiler_imposm
+
+  tiler_monitor_data:
+    driver: local
+    name: tiler_monitor

hetzner/tiler/tiler.production.yml

@@ -120,11 +120,12 @@ services:
 
   tiler_monitor:
     container_name: tiler_monitor
-    image: ghcr.io/openhistoricalmap/tiler-monitor:0.0.1-0.dev.git.2874.ha9bff68
+    image: ghcr.io/openhistoricalmap/tiler-monitor:0.0.1-0.dev.git.3353.h63df1944
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - ../../images/tiler-monitor:/app
+      # - ../../images/tiler-monitor:/app
       - ../../hetzner:/app/hetzner
+      - tiler_monitor_data:/data
     environment:
       - DOCKER_CONFIG_ENVIRONMENT=production
     stdin_open: true
@@ -141,6 +142,9 @@ volumes:
   tiler_imposm_data:
     driver: local
     name: tiler_imposm_17_03
+  tiler_monitor_data:
+    driver: local
+    name: tiler_monitor_data
 
 networks:
   ohm_network:

hetzner/traefik/traefik.template.yml

@@ -161,6 +161,14 @@ http:
       middlewares:
         - secure-headers
 
+    tiler-monitoring-router:
+      rule: Host(`tiler-monitoring.{{OHM_DOMAIN}}`)
+      entryPoints:
+        - port-web
+      service: tiler_monitor
+      middlewares:
+        - secure-headers
+
   services:
     tiler_server:
       loadBalancer:
@@ -207,6 +215,11 @@ http:
         servers:
           - url: http://cadvisor:8080
 
+    tiler_monitor:
+      loadBalancer:
+        servers:
+          - url: http://tiler-monitor:8001
+
 providers:
   file:
     filename: /etc/traefik/traefik.yml

images/tiler-monitor/Dockerfile

@@ -1,14 +1,42 @@
-FROM docker:cli
+FROM python:3.12-slim
 
-RUN apk add --no-cache \
-    bash \
-    curl \
-    postgresql-client \
-    docker-cli-compose
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        curl \
+        bash \
+        postgresql-client && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Docker CLI from official static binaries
+RUN ARCH=$(dpkg --print-architecture) && \
+    if [ "$ARCH" = "amd64" ]; then DOCKER_ARCH="x86_64"; else DOCKER_ARCH="aarch64"; fi && \
+    curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-27.5.1.tgz" | \
+    tar xz --strip-components=1 -C /usr/local/bin docker/docker
+
+# Install Docker Compose plugin
+RUN ARCH=$(dpkg --print-architecture) && \
+    if [ "$ARCH" = "amd64" ]; then COMPOSE_ARCH="x86_64"; else COMPOSE_ARCH="aarch64"; fi && \
+    mkdir -p /usr/local/lib/docker/cli-plugins && \
+    curl -fsSL "https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-${COMPOSE_ARCH}" \
+    -o /usr/local/lib/docker/cli-plugins/docker-compose && \
+    chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
 
 WORKDIR /app
 
-COPY monitor_languages.sh .
-RUN chmod +x monitor_languages.sh
+# Install Python dependencies for pipeline monitor
+COPY pipeline-monitor/requirements.txt /app/pipeline-monitor/requirements.txt
+RUN pip install --no-cache-dir -r /app/pipeline-monitor/requirements.txt
+
+# Copy application code
+COPY language-monitor/ /app/language-monitor/
+COPY pipeline-monitor/ /app/pipeline-monitor/
+COPY entrypoint.sh /app/entrypoint.sh
+
+RUN chmod +x /app/entrypoint.sh /app/language-monitor/monitor_languages.sh
+
+EXPOSE 8001
+
+HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=30s \
+    CMD curl -f http://localhost:8001/health || exit 1
 
-CMD ["bash", "monitor_languages.sh"]
+CMD ["/app/entrypoint.sh"]

images/tiler-monitor/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+echo "$(date +'%Y-%m-%d %H:%M:%S') - Starting tiler-monitor (combined)"
+
+# Start language monitor in background
+echo "$(date +'%Y-%m-%d %H:%M:%S') - Starting language monitor in background..."
+bash /app/language-monitor/monitor_languages.sh &
+
+# Start pipeline monitor in foreground
+echo "$(date +'%Y-%m-%d %H:%M:%S') - Starting pipeline monitor (FastAPI on port 8001)..."
+cd /app/pipeline-monitor
+exec python monitor.py

images/tiler-monitor/monitor_languages.sh → images/tiler-monitor/language-monitor/monitor_languages.sh

@@ -11,9 +11,10 @@ log_message() {
 
 PG_CONNECTION="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
-NIM_NUMBER_LANGUAGES="${NIM_NUMBER_LANGUAGES:-5}" # Default to 5 languages
-FORCE_LANGUAGES_GENERATION="${FORCE_LANGUAGES_GENERATION:-false}"
-EVALUATION_INTERVAL="${EVALUATION_INTERVAL:-3600}" # Default to 1 hour
+NIM_NUMBER_LANGUAGES="${TILER_MONITORING_NIM_NUMBER_LANGUAGES:-5}" # Default to 5 languages
+FORCE_LANGUAGES_GENERATION="${TILER_MONITORING_FORCE_LANGUAGES_GENERATION:-false}"
+EVALUATION_INTERVAL="${TILER_MONITORING_EVALUATION_INTERVAL:-3600}" # Default to 1 hour
+DOCKER_CONFIG_ENVIRONMENT="${TILER_MONITORING_DOCKER_CONFIG_ENVIRONMENT:-staging}"
 
 log_message "Configuration Summary:"
 log_message "  Environment:             $DOCKER_CONFIG_ENVIRONMENT"