Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .env.dfe-example
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
# docker compose --env-file .env --env-file .env.dfe \
# -f docker-compose.yml -f docker-compose.dfe.yml up -d

# PostgreSQL / FerretDB credentials (used by docker-compose.dfe.yml)
# DocumentDB login (used by docker-compose.dfe.yml), created on its first start
POSTGRES_USER=hyperdx
POSTGRES_PASSWORD=hyperdx

# FerretDB connection (replaces MongoDB)
# DocumentDB connection (replaces MongoDB), TLS with a SCRAM-SHA-256 login
# For the API running inside Docker (app service), this is set in docker-compose.dfe.yml
# For local development (API outside Docker), use:
# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?authMechanism=PLAIN
# For local development (API outside Docker, against docker-compose.dfe.dev.yml), use:
# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256

# DFE Auth Mode: 'oidc-proxy' enables OIDC via Envoy, '' or unset = upstream Passport.js
DFE_AUTH_MODE=oidc-proxy
Expand Down
6 changes: 4 additions & 2 deletions .env.dfe.example
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
# DFE dev overrides -- copy to .env.dfe (gitignored) and adjust.
# Consumed by `yarn dev:dfe` (compose --env-file + dotenvx -f).

# FerretDB backing store (required by docker-compose.dfe.dev.yml).
# FERRETDB_AUTH is false in dev, so any matching pair works.
# DocumentDB login (required by docker-compose.dfe.dev.yml), created on its
# first start. The gateway requires TLS with a SCRAM-SHA-256 login, on a
# self-signed certificate that tlsAllowInvalidCertificates accepts in dev only.
POSTGRES_USER=hyperdx
POSTGRES_PASSWORD=hyperdx
MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256

# DFE identity master switch: empty disables the /dfe/* identity layer,
# `oidc-proxy` verifies engine JWTs, `header-dev` trusts dev headers.
Expand Down
100 changes: 100 additions & 0 deletions .github/workflows/dfe-documentdb.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# Project: dfe-hyperdx
# File: .github/workflows/dfe-documentdb.yml
# Purpose: Run the api integration suite, dfe/ tests included, on DocumentDB.
#
# License: MIT - HYPERI PTY LIMITED
# Copyright: (c) 2026 HYPERI PTY LIMITED
#
# DFE keeps HyperDX's documents in DocumentDB, which speaks the MongoDB wire
# protocol over Postgres and implements a tested subset of MongoDB. Upstream's
# harness runs MongoDB, so this job layers dfe/test/docker-compose.documentdb.yml
# over docker-compose.ci.yml and leaves upstream's files as they ship.

name: dfe-documentdb

'on':
pull_request:
branches: [main]
push:
branches: [main]
workflow_dispatch: {}

permissions:
contents: read

concurrency:
group: dfe-documentdb-${{ github.ref }}
cancel-in-progress: true

env:
# TLS and SCRAM-SHA-256 as deployed. The harness certificate is self-signed,
# hence tlsAllowInvalidCertificates, which no deployment should carry.
DFE_INT_MONGO_URI: mongodb://hyperdx:hyperdx-int@localhost:39999/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256

jobs:
integration:
name: Integration on DocumentDB (shard ${{ matrix.shard }})
runs-on: ubuntu-24.04
# The whole suite took 949s serially on DocumentDB, so it is split like upstream's.
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Setup node
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version-file: '.nvmrc'
cache-dependency-path: 'yarn.lock'
cache: 'yarn'

- name: Install dependencies
run: yarn install --immutable

# The bake step's GHA cache needs the runtime token in the shell.
- name: Expose GitHub runtime
uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0

# The collector creates the ClickHouse schema the suite waits for. It is
# built from source, so the layer cache saves the OCB compile on every run
# after the first. The tag is the name compose gives it, so `up` reuses it.
- name: Build the collector
run: |
docker buildx create --use --driver=docker-container
docker buildx bake -f docker-compose.ci.yml \
--set 'otel-collector.tags=hdx-ci-otel-collector' \
--set '*.cache-to=type=gha,mode=max,scope=dfe-documentdb' \
--set '*.cache-from=type=gha,scope=dfe-documentdb' \
--load

- name: Build dependencies
run: yarn workspace @hyperdx/common-utils ci:build

- name: Start ClickHouse, the collector and DocumentDB
run: |
docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --quiet-pull
docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --wait --wait-timeout 300 db

# SHORTCUT: two upstream MCP assertions are skipped by name. They fail on
# MongoDB too: our lockfile carries @modelcontextprotocol/sdk 1.32.1 where
# upstream's has 1.29.0, and its validation message lacks the text they
# assert. Drop the pattern when either side moves.
- name: Run the api integration suite
env:
SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}
SKIP:
'^(?!.*(rejects a delta on a non-gauge metric|validate the
displayType allowlist)).*$'
run:
yarn workspace @hyperdx/api ci:int --shard="$SHARD"
--testNamePattern="$SKIP"

- name: DocumentDB log on failure
if: failure()
run:
docker compose -f docker-compose.ci.yml -f
dfe/test/docker-compose.documentdb.yml logs --tail 200 db
33 changes: 33 additions & 0 deletions dfe/test/docker-compose.documentdb.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# DocumentDB in place of MongoDB for the integration suite, layered over
# upstream's docker-compose.ci.yml so that file stays as upstream ships it.
#
# docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml \
# up -d --wait ch-server db
#
# The gateway requires TLS and authenticates with SCRAM-SHA-256, as a deployed
# store does. Its certificate is self-signed, so the suite connects with
# tlsAllowInvalidCertificates=true - acceptable here and nowhere else. The
# connection string is in .github/workflows/dfe-documentdb.yml.

services:
db:
# DocumentDB 1.0 on Postgres 18, the pairing DFE deploys.
image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5
# Upstream's `--port 29999` is a mongod flag the DocumentDB entrypoint refuses.
command: !reset []
environment:
USERNAME: hyperdx
PASSWORD: hyperdx-int
TLS_MODE: requireTLS
# Off, the gateway listens on the container's loopback only.
ALLOW_EXTERNAL_CONNECTIONS: 'true'
DOCUMENTDB_USAGE_TELEMETRY: 'false'
ports: !override
- '${HDX_CI_MONGO_PORT:-39999}:10260'
# The image's own probe, polled faster than its 30s default so `--wait` returns promptly.
healthcheck:
test: ['CMD', '/usr/local/bin/documentdb-healthcheck']
interval: 2s
timeout: 10s
start_period: 120s
retries: 30
50 changes: 26 additions & 24 deletions docker-compose.dfe.dev.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
# DFE Dev Override: FerretDB + PostgreSQL (no app — app runs locally)
# DFE Dev Override: DocumentDB (no app -- app runs locally)
#
# Usage (requires .env.dfe with POSTGRES_USER, POSTGRES_PASSWORD):
# docker compose --env-file .env.dfe -f docker-compose.dev.yml -f docker-compose.dfe.dev.yml up -d
#
# Use this with docker-compose.dev.yml. The app service is not defined in dev
# (it runs locally via yarn dev), so this override omits the app section.
# For production, use docker-compose.dfe.yml instead.
#
# The local app reaches the store with this in .env.dfe, the POSTGRES_ pair
# being the DocumentDB login created on its first start:
# MONGO_URI=mongodb://<POSTGRES_USER>:<POSTGRES_PASSWORD>@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256

services:
db:
Expand All @@ -28,48 +32,46 @@
# pulled or run and only lets compose skip the build.
otel-collector:
build: !reset null
image: ${CH_IMAGE_REPO:-hyperdx}/${NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB:-hyperdx-otel-collector}:${IMAGE_VERSION:-2}

Check notice on line 35 in docker-compose.dfe.dev.yml

View workflow job for this annotation

GitHub Actions / ci / Quality

hyperi-ci compose-pins: compose/undigested-image

`${CH_IMAGE_REPO:-hyperdx}/${NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB:-hyperdx-otel-collector}:${IMAGE_VERSION:-2}` resolves to the tag `2`, which the registry can move - a `@sha256:` digest is the stronger pin
deploy:
replicas: 0

otel-collector-json:
build: !reset null
image: ${CH_IMAGE_REPO:-hyperdx}/${NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB:-hyperdx-otel-collector}:${IMAGE_VERSION:-2}

Check notice on line 41 in docker-compose.dfe.dev.yml

View workflow job for this annotation

GitHub Actions / ci / Quality

hyperi-ci compose-pins: compose/undigested-image

`${CH_IMAGE_REPO:-hyperdx}/${NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB:-hyperdx-otel-collector}:${IMAGE_VERSION:-2}` resolves to the tag `2`, which the registry can move - a `@sha256:` digest is the stronger pin
deploy:
replicas: 0

postgres:
image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0
# The app runs on the host, which cannot read a CA held in a volume, so dev
# keeps the gateway's self-signed certificate and the MONGO_URI above sets
# tlsAllowInvalidCertificates. docker-compose.dfe.yml verifies the gateway.
documentdb:
image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: postgres
USERNAME: ${POSTGRES_USER}
PASSWORD: ${POSTGRES_PASSWORD}
TLS_MODE: requireTLS
# Off, the gateway listens on the container's loopback only.
ALLOW_EXTERNAL_CONNECTIONS: 'true'
# On by default upstream: it reports to a third party at start and hourly.
DOCUMENTDB_USAGE_TELEMETRY: 'false'
ports:
- '27017:10260'
volumes:
- .volumes/pg_data:/var/lib/postgresql/data
- documentdb-data:/data
networks:
- internal
restart: on-failure
healthcheck:
# -d names the real database: without it libpq dials one named after the user.
test: pg_isready -U ${POSTGRES_USER} -d postgres
# The image's own probe, polled faster than its 30s default.
test: ['CMD', '/usr/local/bin/documentdb-healthcheck']
interval: 5s
timeout: 3s
timeout: 10s
start_period: 60s
retries: 10

ferretdb:
image: ghcr.io/ferretdb/ferretdb:2.7.0
environment:
FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres
# Disable auth for local dev — allows createIndexes and other operations
FERRETDB_AUTH: 'false'
ports:
- '27017:27017'
networks:
- internal
depends_on:
postgres:
condition: service_healthy
restart: on-failure
volumes:
documentdb-data:

networks:
internal:
122 changes: 88 additions & 34 deletions docker-compose.dfe.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# DFE Override: Replace MongoDB with FerretDB + PostgreSQL (DocumentDB)
# DFE Override: Replace MongoDB with DocumentDB
#
# Usage (production):
# docker compose -f docker-compose.yml -f docker-compose.dfe.yml up -d
Expand All @@ -8,60 +8,114 @@
#
# This override:
# 1. Removes the upstream 'db' (mongo) service
# 2. Adds 'postgres' (PG 17 + DocumentDB extension) and 'ferretdb' services
# 3. Updates the 'app' service MONGO_URI to point at FerretDB
# 4. FerretDB exposes the MongoDB wire protocol on port 27017
# 2. Adds 'documentdb' (PostgreSQL 18 with the DocumentDB extensions and its
# Mongo-wire gateway in one container) and 'documentdb-tls', which mints the
# CA and the certificate the gateway serves
# 3. Points the 'app' service MONGO_URI at the gateway, over TLS checked against
# that CA and with a SCRAM-SHA-256 login
#
# The HyperDX application code is completely unchanged — Mongoose, connect-mongo,
# and passport-local-mongoose all work through FerretDB's MongoDB protocol proxy.
# The HyperDX application code is completely unchanged -- Mongoose, connect-mongo,
# and passport-local-mongoose all work through the DocumentDB gateway.

services:
# Disable upstream mongo service (empty profile prevents it from starting)
db:
profiles:
- disabled

# PostgreSQL 17 with DocumentDB extension (backing store for FerretDB)
postgres:
image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0
# Mints the CA and the gateway certificate once, into their own volume. The
# DocumentDB image is reused for its openssl. Root, because a fresh named volume
# is root-owned. The CA key is discarded once the certificate is signed.
documentdb-tls:
image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5
user: '0:0'
entrypoint: ['/bin/sh', '-c']
command:
- |
set -eu
umask 077
cd /tls
if [ ! -s ca/ca.crt ] || [ ! -s server/tls.crt ] || [ ! -s server/tls.key ]; then
rm -rf ca server
mkdir ca server
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 -days 3650 \
-subj /CN=documentdb-ca \
-addext basicConstraints=critical,CA:TRUE -addext keyUsage=critical,keyCertSign,cRLSign \
-keyout ca.key -out ca/ca.crt
openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 \
-subj /CN=documentdb -keyout server/tls.key -out server.csr
printf '%s\n' basicConstraints=critical,CA:FALSE keyUsage=critical,digitalSignature \
extendedKeyUsage=serverAuth subjectAltName=DNS:documentdb > server.ext
openssl x509 -req -in server.csr -CA ca/ca.crt -CAkey ca.key -CAcreateserial -CAserial ca.srl \
-days 3650 -sha384 -extfile server.ext -out server/tls.crt
rm -f ca.key ca.srl server.csr server.ext
fi
# The app reads the CA, and documentdb (uid 1000) the pair.
chmod 0755 ca
chmod 0644 ca/ca.crt
chown -R 1000:1000 server
volumes:
- documentdb-tls:/tls
network_mode: none
restart: 'no'

documentdb:
image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: postgres
USERNAME: ${POSTGRES_USER}
PASSWORD: ${POSTGRES_PASSWORD}
# The gateway refuses plain connections and serves the minted certificate.
TLS_MODE: requireTLS
CERT_PATH: /etc/documentdb/tls/tls.crt
KEY_FILE: /etc/documentdb/tls/tls.key
# Off, the gateway listens on the container's loopback only.
ALLOW_EXTERNAL_CONNECTIONS: 'true'
# On by default upstream: it reports to a third party at start and hourly.
DOCUMENTDB_USAGE_TELEMETRY: 'false'
volumes:
- .volumes/pg_data:/var/lib/postgresql/data
- documentdb-data:/data
- type: volume
source: documentdb-tls
target: /etc/documentdb/tls
read_only: true
volume:
subpath: server
networks:
- internal
restart: on-failure
depends_on:
documentdb-tls:
condition: service_completed_successfully
healthcheck:
# -d names the real database: without it libpq dials one named after the user.
test: pg_isready -U ${POSTGRES_USER} -d postgres
# The image's own probe, polled faster than its 30s default.
test: ['CMD', '/usr/local/bin/documentdb-healthcheck']
interval: 5s
timeout: 3s
timeout: 10s
start_period: 60s
retries: 10

# FerretDB: MongoDB wire protocol proxy → PostgreSQL
ferretdb:
image: ghcr.io/ferretdb/ferretdb:2.7.0
environment:
FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres
ports:
- '27017:27017'
networks:
- internal
depends_on:
postgres:
condition: service_healthy
restart: on-failure

# Override app service to use FerretDB instead of mongo
# Override app service to use DocumentDB instead of mongo
app:
environment:
MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ferretdb:27017/hyperdx?authMechanism=PLAIN'
MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@documentdb:10260/hyperdx?tls=true&tlsCAFile=/etc/dfe/documentdb-ca/ca.crt&authMechanism=SCRAM-SHA-256'
# The CA alone, never the gateway key beside it.
volumes:
- type: volume
source: documentdb-tls
target: /etc/dfe/documentdb-ca
read_only: true
volume:
subpath: ca
# Replaces the base list: a merge would keep `db`, which the profile above removes.
depends_on: !override
- ch-server
- ferretdb
ch-server:
condition: service_started
documentdb:
condition: service_healthy

volumes:
documentdb-data:
documentdb-tls:

networks:
internal:
Loading
Loading