docker-compose.dev.yml

#
# docker-compose file used ONLY for local development.
# For more info, see:
# https://posthog.com/handbook/engineering/developing-locally
#
# PostHog has sunset support for self-hosted K8s deployments.
# See: https://posthog.com/blog/sunsetting-helm-support-posthog
#
# Profile guidance:
#   New optional services should declare `profiles:` directly on the service
#   here (see capture, plugins, ingestion-* for examples).
#   docker-compose.profiles.yml is a legacy overlay that retroactively gates
#   older services; do not add new entries there.
#

services:
    proxy:
        extends:
            file: docker-compose.base.yml
            service: proxy
        ports:
            - 127.0.0.1:8010:8000
        extra_hosts:
            - 'web:host-gateway'
            - 'plugins:host-gateway'
            - 'capture:host-gateway'
            - 'capture-ai:host-gateway'
            - 'capture-logs:host-gateway'
            - 'replay-capture:host-gateway'
            - 'feature-flags:host-gateway'
            - 'hypercache-server:host-gateway'
        environment:
            CADDYFILE: |
                {
                    servers {
                        trusted_proxies static ${CADDY_TRUSTED_PROXIES:-127.0.0.1/32 ::1/128}
                    }
                }
                ${CADDY_HOST:-http://localhost:8000} {
                    @replay-capture {
                        path /s
                        path /s/*
                    }

                    @capture-ai {
                        path /i/v0/ai
                        path /i/v0/ai/*
                    }

                    @capture {
                        path /e
                        path /e/*
                        path /i/v0/*
                        path /i/v1/analytics/events
                        path /i/v1/analytics/events/
                        path /batch
                        path /batch*
                        path /capture
                        path /capture*
                    }

                    @capture-logs {
                        path /i/v1/logs
                        path /i/v1/traces
                        path /i/v1/metrics
                    }

                    @flags {
                        path /flags
                        path /flags*
                        # Local evaluation (flag definitions) — mirrors cloud Contour routing to Rust
                        path /api/feature_flag/local_evaluation
                        path /api/feature_flag/local_evaluation/
                        path /api/feature_flag/local_evaluation/*
                    }

                    @surveys {
                        path /surveys
                        path /surveys/
                        path /api/surveys
                        path /api/surveys/
                    }

                    @remote-config {
                        path /array/*
                    }

                    @webhooks {
                        path /public/webhooks
                        path /public/webhooks/*
                        path /public/m
                        path /public/m/*
                    }

                    handle @capture-ai {
                        reverse_proxy capture-ai:3309
                    }

                    handle @capture {
                        reverse_proxy capture:3307
                    }

                    handle @capture-logs {
                        reverse_proxy capture-logs:4320
                    }

                    handle @replay-capture {
                        reverse_proxy replay-capture:3306
                    }

                    handle @flags {
                        reverse_proxy feature-flags:3001
                    }

                    handle @surveys {
                        reverse_proxy hypercache-server:3002
                    }

                    handle @remote-config {
                        reverse_proxy hypercache-server:3002
                    }

                    handle @webhooks {
                        reverse_proxy plugins:6738
                    }

                    handle {
                        reverse_proxy web:8000
                    }
                }

    capture:
        extends:
            file: docker-compose.base.yml
            service: capture
        restart: on-failure
        ports:
            - '3307:3000'
        depends_on:
            kafka:
                condition: service_healthy
            redis7:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:3000/_liveness']
            interval: 5s
            timeout: 5s
            retries: 24
            start_period: 10s
        profiles:
            - capture

    plugins:
        extends:
            file: docker-compose.base.yml
            service: plugins
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6738:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
            PERSONS_DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_PERSONS_DB_NAME:-posthog}
            # The plugins container runs the CDP rerun worker (default capabilities), which
            # requires this. bin/start exports it for local dev; the value below is the default
            # for callers (like the E2E CI job) that launch this service without bin/start.
            CYCLOTRON_NODE_DATABASE_URL: ${CYCLOTRON_NODE_DATABASE_URL:-postgres://posthog:posthog@db:5432/cyclotron_node}
            CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-posthog}
            # CDP Redis defaults to 127.0.0.1 — must point to the Docker service
            CDP_REDIS_HOST: redis7
            # Encryption keys default is only set when NODE_ENV=dev, but the image uses production
            ENCRYPTION_SALT_KEYS: '00beef0000beef0000beef0000beef00'
            # PersonHog is required for CDP — point at the local gRPC router
            PERSONHOG_ADDR: 'personhog-router:50052'
            PERSONHOG_ENABLED: 'true'
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
            personhog-router:
                condition: service_started
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    ingestion-general:
        extends:
            file: docker-compose.base.yml
            service: ingestion-general
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6739:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
            PERSONS_DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_PERSONS_DB_NAME:-posthog}
            CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-posthog}
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    ingestion-sessionreplay:
        extends:
            file: docker-compose.base.yml
            service: ingestion-sessionreplay
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6740:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
            ENCRYPTION_SALT_KEYS: '00beef0000beef0000beef0000beef00'
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    recording-api:
        extends:
            file: docker-compose.base.yml
            service: recording-api
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6741:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
            CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-posthog}
            ENCRYPTION_SALT_KEYS: '00beef0000beef0000beef0000beef00'
        depends_on:
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    ingestion-error-tracking:
        extends:
            file: docker-compose.base.yml
            service: ingestion-error-tracking
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6742:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
            PERSONHOG_ADDR: 'personhog-router:50052'
            PERSONHOG_ENABLED: 'true'
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
            personhog-router:
                condition: service_started
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    ingestion-logs:
        extends:
            file: docker-compose.base.yml
            service: ingestion-logs
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6743:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    ingestion-traces:
        extends:
            file: docker-compose.base.yml
            service: ingestion-traces
        image: ghcr.io/posthog/posthog-node:${POSTHOG_NODE_TAG:-master}
        ports:
            - '6744:6738'
        environment:
            DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_DB_NAME:-posthog}
        depends_on:
            kafka:
                condition: service_started
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        healthcheck:
            test: ['CMD', 'curl', '-f', 'http://localhost:6738/_ready']
            interval: 5s
            timeout: 5s
            retries: 30
            start_period: 15s
        profiles:
            - ingestion

    browserless:
        # Renders heatmap screenshots and image exports — there is no local-browser fallback.
        # Optional: docker compose -f docker-compose.dev.yml --profile browserless up -d browserless
        # See the HEATMAP_BROWSERLESS_* / BROWSERLESS_* entries in .env.example for the app-side wiring.
        image: ghcr.io/browserless/chromium:v2.51.2
        restart: on-failure
        ports:
            - '127.0.0.1:4445:3000'
        environment:
            TOKEN: dev
        profiles:
            - browserless

    db:
        extends:
            file: docker-compose.base.yml
            service: db
        ports:
            - '5432:5432'
        volumes:
            - postgres-15-data:/var/lib/postgresql/data
        # something in the django app when running in dev mode
        # (maybe only in Pycharm) keeps many idle transactions open
        # and eventually kills postgres, these settings aim to stop that happening.
        # They are only for DEV and should not be used in production.
        command: postgres -c max_connections=1000 -c idle_in_transaction_session_timeout=300000 -c max_prepared_transactions=10

    redis7:
        extends:
            file: docker-compose.base.yml
            service: redis7
        ports:
            - '6379:6379'

    redis-cluster:
        image: redis:7.2
        restart: on-failure
        ports:
            - '127.0.0.1:6399:6379'
        command: >
            bash -c "
            redis-server
            --port 6379
            --cluster-enabled yes
            --cluster-config-file /data/nodes.conf
            --cluster-node-timeout 5000
            --cluster-announce-ip 127.0.0.1
            --cluster-announce-port 6399
            --appendonly yes &
            sleep 1 &&
            redis-cli -p 6379 cluster addslots $$(seq 0 16383) &&
            tail -f /dev/null
            "
        volumes:
            - redis-cluster-data:/data
        healthcheck:
            test: ['CMD', 'redis-cli', '-p', '6379', 'cluster', 'info']
            interval: 2s
            timeout: 3s
            retries: 10

    flower:
        extends:
            file: docker-compose.base.yml
            service: flower
        ports:
            - '5555:5555'

    clickhouse:
        extends:
            file: docker-compose.base.yml
            service: clickhouse
        hostname: clickhouse
        # ClickHouse writes ~1 GB/hour in dev and json-file driver has no default rotation
        logging:
            driver: json-file
            options:
                max-size: '50m'
                max-file: '3'
        # Development performance optimizations
        mem_limit: 8g
        cpus: 2
        environment:
            - AWS_ACCESS_KEY_ID=object_storage_root_user
            - AWS_SECRET_ACCESS_KEY=object_storage_root_password
            - CLICKHOUSE_SKIP_USER_SETUP=1
        ports:
            - '8123:8123'
            - '8443:8443'
            - '9000:9000'
            - '9440:9440'
            - '9009:9009'
        volumes:
            - ./posthog/idl:/idl
            - ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
            - ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
            - ./docker/clickhouse/config.d/default.xml:/etc/clickhouse-server/config.d/default.xml
            - ./docker/clickhouse/config.d/dev-memory.xml:/etc/clickhouse-server/config.d/dev-memory.xml
            - ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
            - ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
            - ./posthog/user_scripts:/var/lib/clickhouse/user_scripts
            - clickhouse-data:/var/lib/clickhouse
        extra_hosts:
            - 'host.docker.internal:host-gateway'
        depends_on:
            - kafka
            - zookeeper

    zookeeper:
        extends:
            file: docker-compose.base.yml
            service: zookeeper
        ports:
            - '2181:2181'
        # With persistent volumes, snapshots and txn logs accumulate forever unless
        # autopurge is on (the image default disables it). Matches hobby.
        environment:
            ZOO_AUTOPURGE_PURGEINTERVAL: 1
            ZOO_AUTOPURGE_SNAPRETAINCOUNT: 3
        volumes:
            - zookeeper-datalog:/datalog
            - zookeeper-data:/data
            - zookeeper-logs:/logs

    kafka:
        extends:
            file: docker-compose.base.yml
            service: kafka
        ports:
            - '9092:9092'
        depends_on:
            - zookeeper

    kafka-init:
        image: docker.io/redpandadata/redpanda:v25.1.9
        entrypoint: /bin/sh
        restart: 'no'
        command:
            - -c
            - |
                set -x
                # Redpanda auto_create_topics_enabled=true only fires on producer-first
                # writes. Consumer-only services (e.g. embedding-worker) error on missing
                # topics, so we pre-create them here. Idempotent — re-running is safe.
                TIMEOUT=60
                ELAPSED=0
                until rpk topic list --brokers kafka:9092 2>/dev/null; do
                    echo "Waiting for Kafka broker... (elapsed: ${ELAPSED}s)"
                    sleep 2
                    ELAPSED=$((ELAPSED + 2))
                    if [ $ELAPSED -ge $TIMEOUT ]; then
                        echo "Timeout waiting for Kafka broker after ${TIMEOUT}s"
                        exit 1
                    fi
                done
                # The combined analytics ingestion consumer checks topic existence at
                # startup for every output it produces to (see analytics/outputs/registry.ts).
                # Auto-create only fires on producer-first writes, so pre-create them here.
                TOPICS="document_embeddings_input \
                    document_embedding_results \
                    error_tracking_ingestion_notifications \
                    clickhouse_events_json \
                    clickhouse_ai_events_json \
                    clickhouse_heatmap_events \
                    clickhouse_ingestion_warnings \
                    events_plugin_ingestion_dlq \
                    events_plugin_ingestion_overflow \
                    events_plugin_ingestion_async \
                    ingestion-clientwarnings-main-1 \
                    heatmaps_ingestion \
                    clickhouse_groups \
                    clickhouse_person \
                    clickhouse_person_distinct_id \
                    clickhouse_app_metrics2 \
                    log_entries \
                    clickhouse_property_values \
                    clickhouse_tophog"
                for topic in $$TOPICS; do
                    if rpk topic create "$$topic" --brokers kafka:9092 -p 1 -r 1 2>&1; then
                        echo "Topic $$topic created"
                    elif rpk topic list --brokers kafka:9092 | grep -q "$$topic"; then
                        echo "Topic $$topic already exists"
                    else
                        echo "Failed to create topic $$topic"
                        exit 1
                    fi
                done
        depends_on:
            kafka:
                condition: service_healthy

    kafka_ui:
        extends:
            file: docker-compose.base.yml
            service: kafka_ui
        ports:
            - '9093:8080'
        environment:
            SERVER_MAX_HTTP_REQUEST_HEADER_SIZE: 65536
        depends_on:
            - kafka

    objectstorage:
        extends:
            file: docker-compose.base.yml
            service: objectstorage
        ports:
            - '19000:19000'
            - '19001:19001'
        # Fresh named volume so the swap starts clean; data on the previous volume
        # stays salvageable via `hogli deploy:upgrade-objectstorage`.
        volumes:
            - objectstorage-data:/data

    objectstorage-azure:
        extends:
            file: docker-compose.base.yml
            service: objectstorage-azure
        ports:
            - '10000:10000'

    maildev:
        extends:
            file: docker-compose.base.yml
            service: maildev
        ports:
            - '1080:1080'
            - '1025:1025'

    webhook-tester:
        image: tarampampam/webhook-tester:2.2.6
        restart: on-failure
        ports:
            - '2080:8080'
        environment:
            - AUTO_CREATE_SESSIONS=true

    personhog-replica:
        extends:
            file: docker-compose.base.yml
            service: personhog-replica
        environment:
            PRIMARY_DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_PERSONS_DB_NAME:-posthog}
        depends_on:
            db:
                condition: service_healthy
        profiles:
            - ingestion

    personhog-router:
        extends:
            file: docker-compose.base.yml
            service: personhog-router
        ports:
            - '50052:50052'
        depends_on:
            personhog-replica:
                condition: service_started
        profiles:
            - ingestion

    feature-flags:
        extends:
            file: docker-compose.base.yml
            service: feature-flags
        depends_on:
            redis7:
                condition: service_healthy
            db:
                condition: service_healthy
        environment:
            # Mode 2: Dual-write warming phase for developers
            FLAGS_REDIS_URL: 'redis://redis7:6379/1'
            FLAGS_REDIS_ENABLED: 'false' # Warming but not reading yet
            REDIS_COMPRESSION_ENABLED: 'true'
            INTERNAL_REQUEST_TOKEN: 'dev-internal-token'

    # Temporal containers
    elasticsearch:
        extends:
            file: docker-compose.base.yml
            service: elasticsearch
        expose:
            - 9200

    # OpenSearch (LLM trace reverse index). 9201 to avoid Elasticsearch's 9200.
    opensearch:
        extends:
            file: docker-compose.base.yml
            service: opensearch
        ports:
            - '127.0.0.1:9201:9200'
        profiles: [opensearch]
    opensearch-dashboards:
        extends:
            file: docker-compose.base.yml
            service: opensearch-dashboards
        ports:
            - '127.0.0.1:5601:5601'
        profiles: [opensearch]
    opensearch-init:
        image: curlimages/curl:8.10.1
        depends_on:
            opensearch:
                condition: service_healthy
        restart: 'no'
        volumes:
            - ./products/ai_observability/opensearch:/work:ro
        entrypoint: ['sh', '/work/init.sh']
        profiles: [opensearch]

    temporal:
        extends:
            file: docker-compose.base.yml
            service: temporal
        ports:
            - 7233:7233
        volumes:
            - ./docker/temporal/dynamicconfig:/etc/temporal/config/dynamicconfig

    temporal-admin-tools:
        extends:
            file: docker-compose.base.yml
            service: temporal-admin-tools
        depends_on:
            - temporal
    temporal-ui:
        extends:
            file: docker-compose.base.yml
            service: temporal-ui
        ports:
            - 8081:8080
        depends_on:
            temporal:
                condition: service_started
            db:
                condition: service_healthy

    otel-collector:
        extends:
            file: docker-compose.base.yml
            service: otel-collector
        container_name: otel-collector-local
        ports:
            - '4317:4317' # OTLP gRPC receiver
            - '4318:4318' # OTLP HTTP receiver
            - '13133:13133' # health_check extension
            - '55679:55679' # zpages extension
        volumes:
            - ./otel-collector-config.dev.yaml:/etc/otel-collector-config.yaml
        environment:
            # Used by otel-collector-config.dev.yaml to scope container log
            # collection to this compose project only.
            COMPOSE_PROJECT: posthog
        extra_hosts:
            - 'host.docker.internal:host-gateway'
        depends_on:
            - jaeger

    jaeger:
        extends:
            file: docker-compose.base.yml
            service: jaeger
        container_name: jaeger-local
        ports:
            - '16686:16686' # Jaeger UI
            - '14268:14268' # jaeger.thrift
            - '14250:14250' # model.proto

    localstack:
        extends:
            file: docker-compose.base.yml
            service: localstack
        container_name: '${LOCALSTACK_DOCKER_NAME:-localstack-main}'
        ports:
            - '127.0.0.1:4566:4566' # LocalStack Gateway
            - '127.0.0.1:4510-4559:4510-4559' # external services port range

    seaweedfs:
        extends:
            file: docker-compose.base.yml
            service: seaweedfs
        container_name: '${SEAWEEDFS_DOCKER_NAME:-seaweedfs-main}'
        command: server -s3 -s3.port=8333 -filer -dir=/data
        ports:
            - '127.0.0.1:8333:8333' # S3 API
            - '127.0.0.1:9333:9333' # Master server
            - '127.0.0.1:8888:8888' # SeaweedFS Filer UI
        volumes:
            - 'seaweedfs-data:/data'

    duckgres:
        extends:
            file: docker-compose.base.yml
            service: duckgres
        profiles: ['duckgres']
        ports:
            - '15432:15432'

    etcd:
        image: quay.io/coreos/etcd:v3.6.8
        restart: on-failure
        environment:
            ETCD_NAME: etcd0
            ETCD_ADVERTISE_CLIENT_URLS: http://etcd:2379
            ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
        ports:
            - '127.0.0.1:2379:2379'
        healthcheck:
            test: ['CMD', 'etcdctl', 'endpoint', 'health']
            interval: 5s
            timeout: 5s
            retries: 10

networks:
    otel_network:
        driver: bridge

volumes:
    postgres-15-data:
    redis-cluster-data:
    redpanda-data:
    seaweedfs-data:
    duckgres-data:
    objectstorage-data:
    clickhouse-data:
    zookeeper-data:
    zookeeper-datalog:
    zookeeper-logs: