docker-compose.yml

# #######################################################################
# Config snippets
# #######################################################################
x-restart-policy: &restart-policy unless-stopped

x-logging: &default-logging
  options:
    max-size: '100m'
    max-file: '3'

x-logging-extended: &extended-logging
  options:
    max-size: '1000m'
    max-file: '3'


services:

  # #######################################################################
  # Data services
  # #######################################################################
  data-postgis:
    build:
      context: .
      dockerfile: docker/postgis/Dockerfile.pg_parquet
    restart: *restart-policy
    profiles:
      - dev
      - prod
    environment:
      POSTGRES_USER: ${STORAGE__PSQL__USER}
      POSTGRES_PASSWORD: ${STORAGE__PSQL__PASSWORD}
      POSTGRES_DB: ${STORAGE__PSQL__DB}
      PAGER: "less -FRX"  # used for paging psql output
    volumes:
      - data-postgis:/var/lib/postgresql/data
    logging: *extended-logging
    ports:
      # Port must be external for host, because Airbyte spawns docker containers
      # when doing an EL-Task. These containers are not within the network odapi_public.
      - "127.0.0.1:4003:5432"
    networks:
      - odapi_private
      - odapi_public
    healthcheck:
      test: pg_isready -U ${STORAGE__PSQL__USER} || exit 1
      interval: 5s
      retries: 5
    shm_size: 24gb
    # For debugging queries, log all statements (only for DEV)
    command: >
      -c shared_buffers=16GB
      -c effective_cache_size=48GB
      -c work_mem=1GB
      -c maintenance_work_mem=2GB
      -c max_parallel_workers_per_gather=8
      -c max_worker_processes=16
      -c random_page_cost=1.1
      -c effective_io_concurrency=256
      -c checkpoint_timeout=30min
      -c max_wal_size=8GB
      -c min_wal_size=2GB
      -c wal_compression=on
      -c log_statement=all
      -c log_min_duration_statement=1000


  data-minio:
    image: minio/minio
    restart: *restart-policy
    volumes:
      - data-minio:/data
    profiles:
      - dev
      - prod
    environment:
      MINIO_ROOT_USER: ${MINIO__ROOT__USER}
      MINIO_ROOT_PASSWORD: ${MINIO__ROOT__PASSWORD}
    ports:
      - 4001:9000
      - 4002:9090
    networks:
      - odapi_private
      - odapi_public
    logging: *extended-logging
    command: server /data --console-address ":9090"


  data-redis:
    image: redis
    restart: *restart-policy
    logging: *extended-logging
    profiles:
      - dev
      - prod
    environment:
      - ALLOW_EMPTY_PASSWORD=yes
    ports:
      - "127.0.0.1:4004:6379"
    networks:
      - odapi_private
      - odapi_public
    healthcheck:
      test: ['CMD', 'redis-cli', 'ping']

  # #######################################################################
  # UTILS
  # #######################################################################
  utils-pgadmin:
    image: dpage/pgadmin4
    restart: *restart-policy
    ports:
      - "4005:80"
    profiles:
      - dev
      - prod
    environment:
      PGADMIN_DEFAULT_EMAIL: ${PGADMIN__DEFAULT_EMAIL}
      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN__DEFAULT_PASSWORD}
    volumes:
      - utils-pgadmin:/var/lib/pgadmin
    networks:
      - odapi_public
    extra_hosts:
      - "host.docker.internal:host-gateway"

  # #######################################################################
  # DAGSTER
  # #######################################################################
  dagster-webserver:
    build:
      context: .
      dockerfile: docker/dagster/Dockerfile
      secrets:
        - git_token
    restart: *restart-policy
    init: true
    profiles:
      - prod
    environment:
      - DAGSTER__PSQL_META__HOST=${DAGSTER__PSQL_META__HOST}
      - DAGSTER__PSQL_META__DB=${DAGSTER__PSQL_META__DB}
      - DAGSTER__PSQL_META__USER=${DAGSTER__PSQL_META__USER}
      - DAGSTER__PSQL_META__PASS=${DAGSTER__PSQL_META__PASS}
    expose:
      - "3000"
    ports:
      - "127.0.0.1:4006:3000"
    volumes:
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - odapi_public
      - odapi_private
    depends_on:
      - data-postgis

  dagster-daemon:
    build:
      context: .
      dockerfile: docker/dagster/Dockerfile
      secrets:
        - git_token
    restart: *restart-policy
    init: true
    entrypoint:
      - dagster-daemon
      - run
      - -f
      - "odapi/definitions.py"
    profiles:
      - prod
    environment:
      - DAGSTER__PSQL_META__HOST=${DAGSTER__PSQL_META__HOST}
      - DAGSTER__PSQL_META__DB=${DAGSTER__PSQL_META__DB}
      - DAGSTER__PSQL_META__USER=${DAGSTER__PSQL_META__USER}
      - DAGSTER__PSQL_META__PASS=${DAGSTER__PSQL_META__PASS}
    volumes:
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - odapi_public
      - odapi_private
    depends_on:
      - data-postgis

  # #######################################################################
  # API
  # #######################################################################
  api:
    build:
      context: .
      dockerfile: api/Dockerfile
    user: "1000:1000"                 # OWASP Rule #3
    security_opt:
      - no-new-privileges:true        # OWASP Rule #4
    restart: *restart-policy
    deploy:
      resources:
        limits:
          cpus: 8
          memory: 64g
    profiles:
      - dev
      - prod
    expose:
      - 80
    networks:
      - odapi_public
      - odapi_private
      - traefik_traefik_public
    depends_on:
      - data-postgis
    labels:
      traefik.enable: true
      traefik.http.routers.odapi.entrypoints: websecure
      traefik.http.routers.odapi.rule: Host(`odapi.bardos.dev`)
      traefik.http.routers.odapi.tls: true
      traefik.http.routers.odapi.tls.certresolver: myresolver
      traefik.docker.network: traefik_traefik_public

# #######################################################################
# VOLUMES
# #######################################################################
volumes:
  data-minio:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /mnt/storagebox/docker/volumes/odapi-data-minio/_data
  data-postgis:
  utils-pgadmin:

# #######################################################################
# Networks
# #######################################################################
networks:
  odapi_public:
  odapi_private:
  traefik_traefik_public:
    external: true

# #######################################################################
# Secrets
# #######################################################################
secrets:
  git_token:
    file: ./docker/.deploy/git_token.txt