From 08825ffb6b8afb94dfc0fdccbf65f88c42862bd0 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 17:19:22 +1100 Subject: [PATCH 1/4] chore(dev): run local dev, integration and e2e on DocumentDB instead of MongoDB Local development and CI now use open-source DocumentDB (Postgres plus a Mongo wire-protocol gateway, MIT) in place of MongoDB 5.0 (SSPL, EOL). The API still talks to it through Mongoose, so this is the first step towards dropping MongoDB from what we ship, tested daily by every developer and CI run before users see it. HDX-5709. - One image, built from source in docker/documentdb and shared by the dev, integration and e2e stacks; it handles root-owned volumes, interrupted first boots and clean shutdown, and pins the upstream commit. - e2e database helpers use the Node driver instead of mongosh, which the image doesn't ship; CI builds the image with the GHA cache and waits on its healthcheck. - Test teardown retries DocumentDB's intermittent dropDatabase failure. - Dev data moves to .volumes/docdb_dev_; old MongoDB volumes are not migrated. --- .github/actions/e2e-setup/action.yml | 36 +++--- .github/workflows/main.yml | 4 +- AGENTS.md | 7 +- Makefile | 8 +- agent_docs/development.md | 19 ++- docker-compose.ci.yml | 9 +- docker-compose.dev.yml | 12 +- docker/documentdb/Dockerfile | 48 ++++++++ docker/documentdb/entrypoint.sh | 49 ++++++++ packages/api/.env.development | 2 +- packages/api/.env.e2e | 4 +- packages/api/.env.test | 2 +- packages/api/scripts/seed-alerts.ts | 4 +- packages/api/src/fixtures.ts | 9 +- packages/app/package.json | 1 + packages/app/tests/e2e/docker-compose.yml | 8 +- .../app/tests/e2e/features/dashboard.spec.ts | 78 +++++++----- .../app/tests/e2e/features/sources.spec.ts | 8 +- .../app/tests/e2e/global-setup-fullstack.ts | 116 +++++++++--------- packages/app/tests/e2e/utils/db-helpers.ts | 91 +++++++------- scripts/dev-env.sh | 2 +- scripts/dev-portal/index.html | 2 +- scripts/dev-portal/server.js | 6 +- scripts/test-e2e.sh | 71 ++--------- yarn.lock | 1 + 25 files changed, 340 insertions(+), 257 deletions(-) create mode 100644 docker/documentdb/Dockerfile create mode 100755 docker/documentdb/entrypoint.sh diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index f4eaaf1ac9..3bcbfe1626 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -3,7 +3,7 @@ description: | Shared setup for E2E test workflows: sets the HDX_E2E_* port env vars and common Playwright env vars, then runs Node.js setup, dependency install, build, Playwright browser install, and starts the E2E Docker Compose stack - (MongoDB + ClickHouse) with health checks. The caller is responsible for + (DocumentDB + ClickHouse) with health checks. The caller is responsible for `actions/checkout`, running Playwright, uploading artifacts, and tearing down the Docker stack. runs: @@ -22,7 +22,7 @@ runs: echo "HDX_E2E_MONGO_PORT=$MONGO_PORT" >> $GITHUB_ENV echo "HDX_E2E_APP_LOCAL_PORT=21200" >> $GITHUB_ENV echo "HDX_E2E_APP_PORT=21300" >> $GITHUB_ENV - echo "MONGO_URI=mongodb://localhost:$MONGO_PORT/hyperdx-e2e" >> $GITHUB_ENV + echo "MONGO_URI=mongodb://hyperdx:hyperdx@localhost:$MONGO_PORT/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" >> $GITHUB_ENV echo "E2E_FULLSTACK=true" >> $GITHUB_ENV echo "E2E_UNIQUE_USER=true" >> $GITHUB_ENV echo "E2E_API_HEALTH_CHECK_MAX_RETRIES=60" >> $GITHUB_ENV @@ -66,12 +66,22 @@ runs: shell: bash run: cd packages/app && npx playwright install --with-deps chromium + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v4 + + # DocumentDB is built from source; the GHA cache keeps that to the first run. + - name: Build DocumentDB image + shell: bash + run: | + docker buildx create --use --driver=docker-container + docker buildx bake -f packages/app/tests/e2e/docker-compose.yml db --set *.cache-to="type=gha,mode=max,scope=documentdb" --set *.cache-from="type=gha,scope=documentdb" --load + - name: Start E2E Docker Compose shell: bash run: | # Pre-pull images with retries to handle transient Docker Hub timeouts for attempt in 1 2 3; do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull; then + if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull --ignore-buildable; then echo "Docker images pulled successfully on attempt $attempt" break fi @@ -82,20 +92,12 @@ runs: echo "Docker pull failed (attempt $attempt/3), retrying in 10s..." sleep 10 done - docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d - echo "Waiting for MongoDB..." - for i in $(seq 1 30); do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml exec -T db mongosh --quiet --eval "db.adminCommand({ping:1})" >/dev/null 2>&1; then - echo "MongoDB is ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "MongoDB failed to become ready after 30 seconds" - exit 1 - fi - echo "Waiting for MongoDB... ($i/30)" - sleep 1 - done + # --wait blocks on the DocumentDB healthcheck (gateway accepting connections). + if ! docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d --wait --wait-timeout 300; then + echo "E2E services failed to become healthy" + docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml logs db | tail -30 + exit 1 + fi echo "Waiting for ClickHouse..." for i in $(seq 1 60); do if curl -sf http://localhost:${HDX_E2E_CH_PORT}/ping >/dev/null 2>&1; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b4c3c94a5..912d7fe028 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,7 +125,7 @@ jobs: - name: Build dependencies run: make ci-build # Integration tests boot a MockServer on fixed ports and share a single - # MongoDB/ClickHouse database, so they must stay serial (--runInBand) + # DocumentDB/ClickHouse database, so they must stay serial (--runInBand) # within a runner. We shard ACROSS runners instead: each shard is its own # VM with its own Docker stack, and jest's --shard splits the test files # so the shards run in parallel. @@ -143,7 +143,7 @@ jobs: # Actions streams stdout. - name: Run integration tests run: | - docker compose -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 status=0 yarn workspace @hyperdx/common-utils ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 yarn workspace @hyperdx/api ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 diff --git a/AGENTS.md b/AGENTS.md index 9e5653ec04..3590fde95d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,8 @@ yarn dev # Start full stack with worktree-isolated ports ``` The project uses **Yarn 4.13.0** workspaces. Docker Compose manages ClickHouse, -MongoDB, and the OTel Collector. +DocumentDB (the local metadata store, which the API reaches over the Mongo wire +protocol), and the OTel Collector. **This repo is multi-agent friendly.** `yarn dev`, `make dev-int`, and `make dev-e2e` all use slot-based port isolation so multiple worktrees can run @@ -461,7 +462,7 @@ default `/workspace` directory, the slot is **76**, so services are at: - **App**: http://localhost:30276 - **API**: http://localhost:30176 - **ClickHouse**: http://localhost:30576 -- **MongoDB**: localhost:30476 +- **DocumentDB**: localhost:30476 ### Key commands reference @@ -475,7 +476,7 @@ reference. Quick summary: ### First-time registration -When the dev stack starts fresh (empty MongoDB), the app shows a registration +When the dev stack starts fresh (empty DocumentDB), the app shows a registration page. Create any account to get started — no external auth provider is needed. --- diff --git a/Makefile b/Makefile index 91896878ea..02caba3ab8 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ install-tools: # API server : 30100 + slot # App (Next.js) : 30200 + slot # OpAMP : 30300 + slot -# MongoDB : 30400 + slot +# DocumentDB : 30400 + slot # ClickHouse HTTP : 30500 + slot # ClickHouse Native : 30600 + slot # OTel health : 30700 + slot @@ -158,7 +158,7 @@ dev-int: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run @hyperdx/api:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/api-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -169,7 +169,7 @@ dev-int-common-utils: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run @hyperdx/common-utils:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/common-utils-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -178,7 +178,7 @@ dev-int-common-utils: .PHONY: ci-int ci-int: @mkdir -p $(HDX_CI_LOGS_DIR) - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 bash -c 'set -o pipefail; npx nx run-many -t ci:int --parallel=false --output-style=stream 2>&1 | tee $(HDX_CI_LOGS_DIR)/ci-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ diff --git a/agent_docs/development.md b/agent_docs/development.md index 305c0d9380..f3c281c4d2 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -30,7 +30,16 @@ yarn dev # or equivalently: make dev ## Environment Configuration - `.env.development`: Development environment variables -- Docker Compose manages ClickHouse, MongoDB, OTel Collector +- Docker Compose manages ClickHouse, DocumentDB, OTel Collector +- The metadata store is [DocumentDB](https://github.com/documentdb/documentdb) + (Postgres plus a Mongo wire-protocol gateway), built from + `docker/documentdb/Dockerfile`. The API still talks to it through Mongoose. + To browse it with MongoDB Compass or `mongosh`, connect to + `mongodb://hyperdx:hyperdx@localhost:/?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`. + Everything works except Compass's Performance tab (the gateway doesn't + implement `serverStatus` or `top`). + Dev data lives in `.volumes/docdb_dev_`; data from the old MongoDB + volumes (`.volumes/db_dev_`) is not migrated, so the dev stack starts empty. - Hot reload enabled for all services in development ## Worktree Isolation (Multi-Agent / Multi-Developer) @@ -59,7 +68,7 @@ integration tests (14320-40098) and E2E tests (20320-21399). | API server | 30100 | 30100 - 30199 | `HYPERDX_API_PORT` | | App (Next.js) | 30200 | 30200 - 30299 | `HYPERDX_APP_PORT` | | OpAMP | 30300 | 30300 - 30399 | `HYPERDX_OPAMP_PORT` | -| MongoDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | +| DocumentDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | | ClickHouse HTTP | 30500 | 30500 - 30599 | `HDX_DEV_CH_HTTP_PORT` | | ClickHouse Native | 30600 | 30600 - 30699 | `HDX_DEV_CH_NATIVE_PORT` | | OTel health | 30700 | 30700 - 30799 | `HDX_DEV_OTEL_HEALTH_PORT` | @@ -173,7 +182,7 @@ Port mapping (base + slot): | Service | Default port (slot 0) | Variable | | --------------- | --------------------- | ----------------- | | ClickHouse HTTP | 18123 | HDX_CI_CH_PORT | -| MongoDB | 39999 | HDX_CI_MONGO_PORT | +| DocumentDB | 39999 | HDX_CI_MONGO_PORT | | API test server | 19000 | HDX_CI_API_PORT | | OpAMP | 14320 | HDX_CI_OPAMP_PORT | @@ -181,7 +190,7 @@ Port mapping (base + slot): - Uses separate Docker Compose configuration (`docker-compose.ci.yml`) - Isolated test environment with unique `-p int-` project name -- Includes all necessary services (ClickHouse, MongoDB, OTel Collector) +- Includes all necessary services (ClickHouse, DocumentDB, OTel Collector) - Tests run against real database instances for accurate integration testing ### E2E Testing @@ -198,7 +207,7 @@ E2E port mapping (base + slot): | ClickHouse HTTP | 20500 | 20500 - 20599 | `HDX_E2E_CH_PORT` | | ClickHouse Native | 20600 | 20600 - 20699 | `HDX_E2E_CH_NATIVE_PORT` | | API server | 21000 | 21000 - 21099 | `HDX_E2E_API_PORT` | -| MongoDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | +| DocumentDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | | App (local) | 21200 | 21200 - 21299 | `HDX_E2E_APP_LOCAL_PORT` | | App (fullstack) | 21300 | 21300 - 21399 | `HDX_E2E_APP_PORT` | diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index bbc07b9799..f425f5d98a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -35,10 +35,13 @@ services: networks: - internal db: - image: mongo:5.0.32-focal - command: --port 29999 + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - '${HDX_CI_MONGO_PORT:-39999}:29999' + - '${HDX_CI_MONGO_PORT:-39999}:10260' networks: - internal networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9c3c0491ec..c93cc67a0b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -7,13 +7,17 @@ services: db: labels: <<: *hdx-labels - hdx.dev.service: mongodb + hdx.dev.service: documentdb hdx.dev.port: '${HDX_DEV_MONGO_PORT:-27017}' - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx volumes: - - .volumes/db_dev_${HDX_DEV_SLOT:-0}:/data/db + - .volumes/docdb_dev_${HDX_DEV_SLOT:-0}:/data/docdb ports: - - '${HDX_DEV_MONGO_PORT:-27017}:27017' + - '${HDX_DEV_MONGO_PORT:-27017}:10260' networks: - internal otel-collector: diff --git a/docker/documentdb/Dockerfile b/docker/documentdb/Dockerfile new file mode 100644 index 0000000000..a17c9c9b0b --- /dev/null +++ b/docker/documentdb/Dockerfile @@ -0,0 +1,48 @@ +# DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used +# in place of MongoDB (SSPL). Built from source because upstream ships glibc +# packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. +ARG DOCUMENTDB_VERSION=v1.0-RC2 +# Tags can move; the build fails unless the tag still resolves to this commit. +ARG DOCUMENTDB_COMMIT=b0cb0dae85e8746f4e0b79c847894179b2f80d64 + +FROM alpine:3.24 AS builder +ARG DOCUMENTDB_VERSION +ARG DOCUMENTDB_COMMIT +RUN apk add --no-cache build-base cmake git curl bash tar pkgconf perl linux-headers python3 \ + icu-dev krb5-dev openssl-dev rust cargo \ + postgresql18 postgresql18-dev postgresql18-contrib postgresql-pg_cron postgresql-pgvector postgis +ENV PG_CONFIG=/usr/libexec/postgresql18/pg_config INSTALL_DEPENDENCIES_ROOT=/deps +RUN git clone --depth 1 --branch ${DOCUMENTDB_VERSION} https://github.com/documentdb/documentdb.git /src \ + && test "$(git -C /src rev-parse HEAD)" = "${DOCUMENTDB_COMMIT}" \ + && mkdir -p /deps +# musl exports _init/_fini from every .so, which the duplicate-export check counts. +RUN sed -i "s#grep -vxE '_PG_init|Pg_magic_func'#grep -vxE '_PG_init|Pg_magic_func|_init|_fini'#g" \ + /src/pg_documentdb_extended_rum/validate_rum_core_exports.sh +RUN cd /src/scripts && MAKE_PROGRAM=cmake bash install_setup_libbson.sh \ + && bash install_setup_pcre2.sh && bash install_setup_intel_decimal_math_lib.sh +# Serial make: with -j the RUM export check can run before the library is linked. +RUN cd /src && make install-documentdb with_llvm=no PG_CONFIG=$PG_CONFIG +RUN cd /src/pg_documentdb_gw && cargo build --release -p documentdb_gateway \ + && install -s target/release/documentdb_gateway /usr/bin/documentdb-gateway +# Keep only what Postgres and DocumentDB link against: no GDAL, JIT or PostGIS raster/topology. +RUN set -e; PG=/usr/libexec/postgresql18; LIB=/usr/lib/postgresql18; SH=/usr/share/postgresql18; \ + rm -f $LIB/llvmjit*.so $LIB/postgis_raster-3.so $LIB/postgis_sfcgal-3.so $LIB/postgis_topology-3.so $LIB/address_standardizer*.so; rm -rf $LIB/bitcode; \ + rm -f $SH/extension/postgis_raster* $SH/extension/postgis_sfcgal* $SH/extension/postgis_topology* $SH/extension/postgis_tiger* $SH/extension/address_standardizer*; rm -rf $SH/contrib/postgis*; \ + for f in $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB/*.so; do echo $f; ldd $f 2>/dev/null | awk '/=> \//{print $3}'; done | sort -u > /tmp/files; \ + mkdir /out; tar -ch -T /tmp/files -f - | tar -x -C /out; \ + tar -ch $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql $LIB $SH /usr/share/proj /usr/share/icu -f - | tar -x -C /out; \ + install -D -m 644 /src/pg_documentdb_gw/SetupConfiguration.json /out/etc/documentdb/gateway/SetupConfiguration.json + +FROM alpine:3.24 AS runtime +RUN apk add --no-cache bash openssl su-exec \ + && addgroup -S postgres && adduser -S -G postgres -h /var/lib/postgresql postgres \ + && install -d -o postgres -g postgres /data/docdb +COPY --from=builder /out/ / +COPY --chmod=755 entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh +ENV PATH=/usr/libexec/postgresql18:$PATH PGDATA=/data/docdb/pgdata +EXPOSE 10260 +# The gateway only listens once Postgres is initialised and the user exists; the +# psql probe catches Postgres dying under a gateway that keeps its port open. +HEALTHCHECK --interval=2s --timeout=3s --start-period=60s --retries=30 \ + CMD nc -z 127.0.0.1 10260 && psql -h /tmp -U postgres -d postgres -qtAX -c 'SELECT 1' >/dev/null +ENTRYPOINT ["bash", "/usr/local/bin/documentdb-entrypoint.sh"] diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh new file mode 100755 index 0000000000..388c264294 --- /dev/null +++ b/docker/documentdb/entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Starts Postgres with the DocumentDB extensions, then the gateway on :10260. +# Run as root: fixes data dir ownership, then re-runs itself as postgres. +set -euo pipefail +: "${DOCDB_USER:?DOCDB_USER is required}" "${DOCDB_PASSWORD:?DOCDB_PASSWORD is required}" + +if [ "$(id -u)" = 0 ]; then + # Volumes may be root-owned or hold lost+found, and initdb needs an empty dir it owns. + install -d -o postgres -g postgres -m 700 "$PGDATA" + exec su-exec postgres "$0" "$@" +fi + +# The marker is written last, so a first boot killed part-way is redone, not left broken. +if [ ! -f "$PGDATA/.documentdb-initialised" ]; then + find "$PGDATA" -mindepth 1 -delete + initdb -D "$PGDATA" -U postgres --auth=trust >/dev/null + cat >> "$PGDATA/postgresql.conf" </dev/null + psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -c "CREATE EXTENSION documentdb CASCADE; CREATE EXTENSION documentdb_extended_rum;" + doc=$(printf '{"createUser":"%s","pwd":"%s","roles":[{"role":"readWriteAnyDatabase","db":"admin"},{"role":"clusterAdmin","db":"admin"}]}' "$DOCDB_USER" "$DOCDB_PASSWORD") + psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -v doc="$doc" <<< "SELECT documentdb_api.create_user(:'doc');" >/dev/null + touch "$PGDATA/.documentdb-initialised" +else + pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null +fi + +echo "postgresql://postgres@localhost:5432/postgres" > /tmp/pg-url +mkdir -p /tmp/tls +export DOCUMENTDB_PG_URL_FILE=/tmp/pg-url DOCUMENTDB_TLS_STATE_DIR=/tmp/tls + +documentdb-gateway run --config /etc/documentdb/gateway/SetupConfiguration.json & +gateway=$! +# Stop Postgres cleanly on docker stop so persistent volumes skip crash recovery. +trap 'kill "$gateway" 2>/dev/null; pg_ctl -D "$PGDATA" -m fast stop >/dev/null; exit 0' TERM INT +# Keep the gateway's exit code so a crash shows as a failed container, not a clean exit. +rc=0 +wait "$gateway" || rc=$? +pg_ctl -D "$PGDATA" -m fast stop >/dev/null || true +exit "$rc" diff --git a/packages/api/.env.development b/packages/api/.env.development index d4995195fd..9da78e573a 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -6,7 +6,7 @@ FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 HDX_NODE_BETA_MODE=1 HDX_NODE_CONSOLE_CAPTURE=1 -MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT}/hyperdx" +MONGO_URI="mongodb://hyperdx:hyperdx@localhost:${HDX_DEV_MONGO_PORT}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" OTEL_RESOURCE_ATTRIBUTES="service.version=dev" diff --git a/packages/api/.env.e2e b/packages/api/.env.e2e index 9db633c4d6..8b93a929c7 100644 --- a/packages/api/.env.e2e +++ b/packages/api/.env.e2e @@ -10,8 +10,8 @@ CLICKHOUSE_PASSWORD= CLICKHOUSE_USER=default RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://localhost:${HDX_E2E_APP_PORT:-21300} -# MongoDB connection string -MONGO_URI=mongodb://localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e +# DocumentDB (Mongo wire protocol) connection string +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_E2E_API_PORT:-21000} OPAMP_PORT=${HDX_E2E_OPAMP_PORT:-20320} diff --git a/packages/api/.env.test b/packages/api/.env.test index 45ee1e9b43..ddba188320 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -7,7 +7,7 @@ CLICKHOUSE_PASSWORD=api CLICKHOUSE_USER=api RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://app:8080 -MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} diff --git a/packages/api/scripts/seed-alerts.ts b/packages/api/scripts/seed-alerts.ts index 28220ab4ea..fa68ca4e18 100644 --- a/packages/api/scripts/seed-alerts.ts +++ b/packages/api/scripts/seed-alerts.ts @@ -120,7 +120,7 @@ function parseArgs(argv: string[]): Args { } /** - * Mongo URI of this worktree's dev stack. `yarn seed:alerts` runs through + * URI of this worktree's dev DocumentDB. `yarn seed:alerts` runs through * scripts/with-slots.sh, which sets HDX_DEV_MONGO_PORT. */ function devSlotMongoUri(): string { @@ -130,7 +130,7 @@ function devSlotMongoUri(): string { 'HDX_DEV_MONGO_PORT is not set. Run this with `yarn seed:alerts`, or pass --mongo-uri.', ); } - return `mongodb://localhost:${port}/hyperdx`; + return `mongodb://hyperdx:hyperdx@localhost:${port}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; } function randomInt(minInclusive: number, maxInclusive: number): number { diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 47d2edafd8..58a8678d53 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -22,6 +22,7 @@ import { AlertInterval, AlertSource } from '@/models/alert'; import Server from '@/server'; import logger from '@/utils/logger'; import { MetricModel } from '@/utils/logParser'; +import { withRetry } from '@/utils/retry'; import { ExternalDashboardTile } from './utils/zod'; @@ -141,7 +142,13 @@ export const closeDB = async () => { if (!config.IS_CI) { throw new Error('ONLY execute this in CI env 😈 !!!'); } - await mongooseConnection.dropDatabase(); + // DocumentDB intermittently fails dropDatabase while background index builds + // run (WriteConflict 112, or InternalError 1); a retry clears it. + await withRetry(() => mongooseConnection.dropDatabase(), { + maxRetries: 5, + initialDelayMs: 200, + retryOnlyOnStatus: [1, 112], + }); await mongoose.disconnect(); }; diff --git a/packages/app/package.json b/packages/app/package.json index c6fc18f131..334ab03e62 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -139,6 +139,7 @@ "identity-obj-proxy": "^3.0.0", "jest": "^30.2.0", "jest-environment-jsdom": "^30.2.0", + "mongodb": "^6.15.0", "msw": "^2.3.0", "msw-storybook-addon": "^2.0.2", "postcss": "^8.5.23", diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index b8152ccc6c..a7a2c3a2b1 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -1,9 +1,13 @@ name: hdx-e2e services: db: - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ../../../../docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - ${HDX_E2E_MONGO_PORT:-21100}:27017 + - ${HDX_E2E_MONGO_PORT:-21100}:10260 networks: - internal deploy: diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 924006f1a3..59f89e6c73 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -12,7 +12,7 @@ import { DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, } from '../utils/constants'; -import { runMongoshScript } from '../utils/db-helpers'; +import { ObjectId, withE2eDb } from '../utils/db-helpers'; import { getSqlEditor } from '../utils/locators'; test.describe('Dashboard', { tag: ['@dashboard'] }, () => { @@ -1158,17 +1158,16 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written straight to Mongo: the API and the form both refuse this // state now, so the only way to hold it is to predate that rule. const dashboardId = dashboardPage.getCurrentDashboardId(); - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { "filters.0.isVariableEnabled": false } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { 'filters.0.isVariableEnabled': false } }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); await dashboardPage.gotoDashboard(dashboardId); @@ -3351,19 +3350,18 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { await test.step('Mark it provisioned', async () => { // ProvisionDashboardsTask is the only thing that sets this in normal // operation, so write it directly rather than running that job. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { provisioned: true } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { provisioned: true } }, + ), ); // A zero-match write would make the assertion below pass for a reason // that has nothing to do with the gate. - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { @@ -3407,19 +3405,33 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written directly rather than built in the chart editor: the editor // flow for a PromQL tile is long and this test is about the gate, not // the editor. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - " { $set: { tiles: [{ id: 'promql-1', x: 0, y: 0, w: 4, h: 2,", - " config: { configType: 'promql', promqlExpression: 'up',", - " connection: 'c1', source: 's1', displayType: 'line' } }] } }", - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db.collection('dashboards').updateOne( + { _id: new ObjectId(dashboardId) }, + { + $set: { + tiles: [ + { + id: 'promql-1', + x: 0, + y: 0, + w: 4, + h: 2, + config: { + configType: 'promql', + promqlExpression: 'up', + connection: 'c1', + source: 's1', + displayType: 'line', + }, + }, + ], + }, + }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 0ee087888a..79cac5b20d 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -341,7 +341,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should show series Table field when isMetricsSeriesTableEnabled is on', { tag: ['@full-stack'] }, async () => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); try { await searchPage.goto(); await searchPage.sourceActionsMenu.click(); @@ -358,7 +358,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { await searchPage.page.keyboard.press('Escape'); } finally { - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); @@ -367,7 +367,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should warn when the configured series table does not match the series table schema', { tag: ['@full-stack'] }, async ({ page }) => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); const API_URL = getApiUrl(); const metricSources = await getSources(page, 'metric'); @@ -415,7 +415,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { ).toBeVisible({ timeout: 20000 }); } finally { await page.request.delete(`${API_URL}/sources/${createdSourceId}`); - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); diff --git a/packages/app/tests/e2e/global-setup-fullstack.ts b/packages/app/tests/e2e/global-setup-fullstack.ts index 9c196b6900..819d097789 100644 --- a/packages/app/tests/e2e/global-setup-fullstack.ts +++ b/packages/app/tests/e2e/global-setup-fullstack.ts @@ -2,13 +2,13 @@ * Global setup for full-stack E2E tests * * This setup: - * 1. Clears MongoDB database to ensure clean state + * 1. Clears the database to ensure clean state * 2. Creates a test user and team * 3. Applies DEFAULT_SOURCES from .env.e2e * 4. Saves authentication state for tests * * Full-stack mode uses: - * - MongoDB (local) for authentication, teams, users, persistence + * - DocumentDB (local) for authentication, teams, users, persistence * - API server (local) for backend logic * - Demo ClickHouse (remote) for telemetry data (logs, traces, metrics, K8s) */ @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import { chromium, FullConfig } from '@playwright/test'; -import { runMongoshScript } from './utils/db-helpers'; +import { dropE2eDatabase, ObjectId, withE2eDb } from './utils/db-helpers'; import { seedClickHouse } from './seed-clickhouse'; // Configuration constants @@ -63,14 +63,14 @@ export const SEEDED_ERROR_ALERT = { }; /** - * Clears the MongoDB database to ensure a clean slate for tests + * Clears the database to ensure a clean slate for tests */ -function clearDatabase() { - console.log('Clearing MongoDB database for fresh test run...'); +async function clearDatabase() { + console.log('Clearing the e2e database for a fresh test run...'); try { - runMongoshScript("use('hyperdx-e2e'); db.dropDatabase();"); - console.log(' ✓ Database cleared successfully (via Docker)'); + await dropE2eDatabase(); + console.log(' ✓ Database cleared successfully'); } catch (error) { console.warn(' ⚠ Warning: Could not clear database'); console.warn(` ${error instanceof Error ? error.message : String(error)}`); @@ -85,7 +85,7 @@ function clearDatabase() { async function globalSetup(_config: FullConfig) { console.log('Setting up full-stack E2E environment'); - console.log(' MongoDB: local (auth, teams, persistence)'); + console.log(' DocumentDB: local (auth, teams, persistence)'); console.log(' ClickHouse: local instance (telemetry data)'); // Set timezone @@ -138,8 +138,8 @@ async function globalSetup(_config: FullConfig) { ); } - // Clear MongoDB database to ensure DEFAULT_SOURCES is applied - clearDatabase(); + // Clear the database to ensure DEFAULT_SOURCES is applied + await clearDatabase(); // Create test user and save auth state console.log('Creating test user and logging in'); @@ -302,7 +302,7 @@ async function globalSetup(_config: FullConfig) { * Seeds an alert with a recorded execution error. The alert is created via the * API (so all referenced documents — saved search, webhook — exist and the * alerts list endpoint populates correctly), then the `errors` array is - * patched in directly via mongosh since it's only ever set by the check-alerts + * patched in directly via the driver since it's only ever set by the check-alerts * background job in normal operation. */ async function seedAlertWithErrors( @@ -379,7 +379,7 @@ async function seedAlertWithErrors( const alert = (await alertRes.json()).data; const alertId: string = alert._id ?? alert.id; - // 4) Patch the `executionErrors` array directly via mongosh. The + // 4) Patch the `executionErrors` array directly in the database. The // check-alerts job is the only code that writes this field in normal // operation, so we write it here to avoid having to run that job during // setup. @@ -390,51 +390,53 @@ async function seedAlertWithErrors( const errorWindowStart = Math.floor(Date.now() / windowMs) * windowMs; const okWindowStart = errorWindowStart - windowMs; - const patchScript = ` -use('hyperdx-e2e'); -db.alerts.updateOne( - { _id: ObjectId(${JSON.stringify(alertId)}) }, - { - $set: { - executionErrors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.errorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.errorMessage)} - } - ], - state: 'OK' - } - } -); -db.alerthistories.deleteMany({ alert: ObjectId(${JSON.stringify(alertId)}) }); -db.alerthistories.insertMany([ - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${okWindowStart}), - state: 'OK', - counts: 0, - lastValues: [{ startTime: new Date(${okWindowStart - windowMs}), count: 0 }] - }, - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${errorWindowStart}), - state: 'ERROR', - counts: 0, - lastValues: [], - errors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorMessage)} - } - ] - } -]); -`; - try { - runMongoshScript(patchScript); + const alertObjectId = new ObjectId(alertId); + await withE2eDb(async db => { + await db.collection('alerts').updateOne( + { _id: alertObjectId }, + { + $set: { + executionErrors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.errorType, + message: SEEDED_ERROR_ALERT.errorMessage, + }, + ], + state: 'OK', + }, + }, + ); + await db + .collection('alerthistories') + .deleteMany({ alert: alertObjectId }); + await db.collection('alerthistories').insertMany([ + { + alert: alertObjectId, + createdAt: new Date(okWindowStart), + state: 'OK', + counts: 0, + lastValues: [ + { startTime: new Date(okWindowStart - windowMs), count: 0 }, + ], + }, + { + alert: alertObjectId, + createdAt: new Date(errorWindowStart), + state: 'ERROR', + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.historyErrorType, + message: SEEDED_ERROR_ALERT.historyErrorMessage, + }, + ], + }, + ]); + }); console.log( ` ✓ Seeded alert "${alert.name}" (${alertId}) with a ${SEEDED_ERROR_ALERT.errorType}`, ); diff --git a/packages/app/tests/e2e/utils/db-helpers.ts b/packages/app/tests/e2e/utils/db-helpers.ts index 3d30f61307..947f4a4974 100644 --- a/packages/app/tests/e2e/utils/db-helpers.ts +++ b/packages/app/tests/e2e/utils/db-helpers.ts @@ -1,55 +1,44 @@ /** - * Direct MongoDB and ClickHouse access helpers for full-stack E2E tests. Only usable in - * full-stack mode (real Mongo via docker-compose) — there is no database in + * Direct metadata-store and ClickHouse access helpers for full-stack E2E tests. Only + * usable in full-stack mode (DocumentDB via docker-compose); there is no database in * local mode, so callers must gate on `{ tag: ['@full-stack'] }`. */ -import { execFileSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; +import { type Db, MongoClient } from 'mongodb'; + +export { ObjectId } from 'mongodb'; + +const E2E_DB_NAME = 'hyperdx-e2e'; + +// Built from the e2e slot port rather than MONGO_URI, so a MONGO_URI exported in a +// developer's shell can never point these destructive helpers at another database. +function e2eMongoUri(): string { + return `mongodb://hyperdx:hyperdx@localhost:${process.env.HDX_E2E_MONGO_PORT || '21100'}/${E2E_DB_NAME}?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; +} + +/** Runs `fn` against the e2e database over the Mongo wire protocol. */ +export async function withE2eDb(fn: (db: Db) => Promise): Promise { + const client = await MongoClient.connect(e2eMongoUri()); + try { + return await fn(client.db(E2E_DB_NAME)); + } finally { + await client.close(); + } +} /** - * Run a mongosh script against the e2e MongoDB container by piping the script - * through stdin. Using stdin (rather than `--eval "<...>"`) avoids having to - * escape quotes in the script body, so callers can pass multi-line JavaScript - * with string literals verbatim. - * - * `execFileSync` with an argument array rather than a shell string: the project - * slug comes from an env var, and with no shell nothing in it can be read as a - * metacharacter. - * - * Throws if the docker-compose file can't be found (meaning we're not running - * in the expected Docker-backed e2e environment). + * Drops the e2e database. DocumentDB intermittently fails this while the API's + * background index builds run (WriteConflict 112, or InternalError 1). */ -export function runMongoshScript(script: string): string { - const dockerComposeFile = path.join(__dirname, '..', 'docker-compose.yml'); - if (!fs.existsSync(dockerComposeFile)) { - throw new Error( - `docker-compose.yml not found at ${dockerComposeFile} — e2e Docker stack unavailable`, - ); +export async function dropE2eDatabase(): Promise { + for (let attempt = 1; ; attempt++) { + try { + await withE2eDb(db => db.dropDatabase()); + return; + } catch (e: any) { + if (attempt >= 5 || ![1, 112].includes(e?.code)) throw e; + await new Promise(resolve => setTimeout(resolve, 200 * attempt)); + } } - - const e2eSlot = process.env.HDX_E2E_SLOT || '0'; - - return execFileSync( - 'docker', - [ - 'compose', - '-p', - `e2e-${e2eSlot}`, - '-f', - dockerComposeFile, - 'exec', - '-T', - 'db', - 'mongosh', - '--quiet', - ], - { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - input: script, - }, - ); } /** @@ -64,11 +53,13 @@ export function runMongoshScript(script: string): string { * racing each other — e.g. via `test.describe.serial(...)` — since * `fullyParallel: true` lets tests in the same file run concurrently. */ -export function setTeamFlag(flagName: string, value: boolean): void { - runMongoshScript(` -use('hyperdx-e2e'); -db.teams.updateOne({}, { $set: { [${JSON.stringify(flagName)}]: ${JSON.stringify(value)} } }); -`); +export async function setTeamFlag( + flagName: string, + value: boolean, +): Promise { + await withE2eDb(db => + db.collection('teams').updateOne({}, { $set: { [flagName]: value } }), + ); } const CLICKHOUSE_HOST = diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 9ef34ae14b..4c01308c26 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -103,7 +103,7 @@ echo "╠═══════════════════════ echo "║ App (Next.js) http://localhost:${HYPERDX_APP_PORT}$(printf '%*s' $((22 - ${#HYPERDX_APP_PORT})) '')║" echo "║ API http://localhost:${HYPERDX_API_PORT}$(printf '%*s' $((22 - ${#HYPERDX_API_PORT})) '')║" echo "║ ClickHouse http://localhost:${HDX_DEV_CH_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_CH_HTTP_PORT})) '')║" -echo "║ MongoDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" +echo "║ DocumentDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" echo "║ OTel HTTP http://localhost:${HDX_DEV_OTEL_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_OTEL_HTTP_PORT})) '')║" echo "║ OTel gRPC localhost:${HDX_DEV_OTEL_GRPC_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_OTEL_GRPC_PORT})) '')║" echo "║ OpAMP localhost:${HYPERDX_OPAMP_PORT}$(printf '%*s' $((29 - ${#HYPERDX_OPAMP_PORT})) '')║" diff --git a/scripts/dev-portal/index.html b/scripts/dev-portal/index.html index 8dc02c85eb..505fe84c97 100644 --- a/scripts/dev-portal/index.html +++ b/scripts/dev-portal/index.html @@ -329,7 +329,7 @@

app: 'App (Next.js)', api: 'API Server', clickhouse: 'ClickHouse', - mongodb: 'MongoDB', + documentdb: 'DocumentDB', 'otel-collector': 'OTel Collector', 'otel-collector-json': 'OTel JSON', alerts: 'Alerts Task', diff --git a/scripts/dev-portal/server.js b/scripts/dev-portal/server.js index 05524a4e0a..ab34e6b096 100755 --- a/scripts/dev-portal/server.js +++ b/scripts/dev-portal/server.js @@ -184,7 +184,7 @@ function parseProject(projectName) { // compose service name (ch-server, db, otel-collector, …). const COMPOSE_SERVICE_ALIASES = { 'ch-server': 'clickhouse', - db: 'mongodb', + db: 'documentdb', }; function canonicalServiceName(labels) { @@ -457,7 +457,7 @@ async function buildDashboardData() { 'alerts', 'common-utils', 'clickhouse', - 'mongodb', + 'documentdb', 'otel-collector', 'otel-collector-json', ]; @@ -517,7 +517,7 @@ const ENV_LOG_DIRS = { // Map canonical service names to Docker Compose service names const DOCKER_SERVICE_NAMES = { clickhouse: 'ch-server', - mongodb: 'db', + documentdb: 'db', 'otel-collector': 'otel-collector', 'otel-collector-json': 'otel-collector-json', }; diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 7b6e060d66..76d3c55ac7 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,7 +1,7 @@ #!/bin/bash # Run E2E tests in full-stack or local mode -# Full-stack mode (default): MongoDB + API + local ClickHouse -# Local mode: Frontend + local ClickHouse (no MongoDB/API) +# Full-stack mode (default): DocumentDB + API + local ClickHouse +# Local mode: Frontend + local ClickHouse (no DocumentDB/API) # # Usage: # ./scripts/test-e2e.sh # Run all tests in fullstack mode @@ -57,11 +57,9 @@ exec > >(tee "$HDX_E2E_LOGS_DIR/e2e.log") 2>&1 # shellcheck source=./ensure-dev-portal.sh source "${REPO_ROOT}/scripts/ensure-dev-portal.sh" -echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} mongo=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" +echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} db=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" # Configuration constants -readonly MAX_MONGODB_WAIT_ATTEMPTS=15 -readonly MONGODB_WAIT_DELAY_SECONDS=1 readonly MAX_CLICKHOUSE_WAIT_ATTEMPTS=30 readonly CLICKHOUSE_WAIT_DELAY_SECONDS=1 @@ -107,22 +105,6 @@ METAEOF rm -rf "$HDX_E2E_LOGS_DIR" 2>/dev/null || true } -check_mongodb_health() { - # Health check script that tests ping, insert, and delete operations - # Note: MongoDB runs on port 27017 inside the container (default) - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db mongosh --quiet --eval " - try { - db.adminCommand('ping'); - db.getSiblingDB('test').test.insertOne({_id: 'healthcheck', ts: new Date()}); - db.getSiblingDB('test').test.deleteOne({_id: 'healthcheck'}); - print('ready'); - } catch(e) { - print('not ready: ' + e); - quit(1); - } - " 2>&1 -} - check_clickhouse_health() { # Health check from HOST perspective (not inside container) # This ensures the port is actually accessible to Playwright @@ -152,40 +134,6 @@ wait_for_clickhouse() { done } -wait_for_mongodb() { - echo "Waiting for MongoDB to be ready..." - local attempt=1 - - # Verify mongosh is available in the container - if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db which mongosh >/dev/null 2>&1; then - echo "ERROR: mongosh not found in MongoDB container" - echo "Container may not be running or using incompatible image" - echo "Try running: docker compose -p $E2E_PROJECT -f $DOCKER_COMPOSE_FILE logs db" - return 1 - fi - - while [ $attempt -le $MAX_MONGODB_WAIT_ATTEMPTS ]; do - local result - result=$(check_mongodb_health) - - if echo "$result" | grep -q "ready"; then - echo "MongoDB is ready and accepting writes" - return 0 - fi - - if [ $attempt -eq $MAX_MONGODB_WAIT_ATTEMPTS ]; then - local total_wait=$((MAX_MONGODB_WAIT_ATTEMPTS * MONGODB_WAIT_DELAY_SECONDS)) - echo "MongoDB failed to become ready after $total_wait seconds" - echo "Last error: $result" - return 1 - fi - - echo "Waiting for MongoDB... ($attempt/$MAX_MONGODB_WAIT_ATTEMPTS)" - attempt=$((attempt + 1)) - sleep $MONGODB_WAIT_DELAY_SECONDS - done -} - # Main execution setup_cleanup_trap() { if [ "$SKIP_CLEANUP" = false ]; then @@ -216,7 +164,7 @@ run_tests() { echo "Running tests in local mode (frontend + ClickHouse)..." yarn test:e2e --local "${PLAYWRIGHT_FLAGS[@]}" else - echo "Running tests in full-stack mode (MongoDB + API + ClickHouse)..." + echo "Running tests in full-stack mode (DocumentDB + API + ClickHouse)..." yarn test:e2e "${PLAYWRIGHT_FLAGS[@]}" fi } @@ -230,12 +178,13 @@ rm -rf "$REPO_ROOT/packages/app/.next-e2e" 2>/dev/null || true # Always start and seed ClickHouse (shared by both modes) setup_clickhouse -# Conditionally start MongoDB for full-stack mode +# Conditionally start DocumentDB for full-stack mode if [ "$LOCAL_MODE" = false ]; then - echo "Starting MongoDB for full-stack mode..." - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d db - - if ! wait_for_mongodb; then + echo "Starting DocumentDB for full-stack mode..." + # --wait blocks on the image healthcheck, which passes once the gateway accepts connections. + if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d --build --wait --wait-timeout 300 db; then + echo "DocumentDB failed to become healthy" + docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" logs db | tail -20 exit 1 fi fi diff --git a/yarn.lock b/yarn.lock index fc99d7bb95..9ffb586e14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4880,6 +4880,7 @@ __metadata: ky: "npm:^0.30.0" ky-universal: "npm:^0.10.1" lodash: "npm:^4.18.1" + mongodb: "npm:^6.15.0" ms: "npm:^2.1.3" msw: "npm:^2.3.0" msw-storybook-addon: "npm:^2.0.2" From a7be35acf04bda6b1c5e892a6bd31f716247d0ab Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 22:20:53 +1100 Subject: [PATCH 2/4] fix(dev): build the e2e DocumentDB image directly and surface Postgres start errors CI e2e setup failed on every shard: `buildx bake` on the e2e compose file resolves its out-of-tree build context as a filesystem entitlement and refuses to run. Build docker/documentdb directly with the same GHA cache. A failed Postgres start now prints the tail of postgresql.log to the container logs, so `docker compose logs db` shows the cause. --- .github/actions/e2e-setup/action.yml | 3 ++- docker/documentdb/entrypoint.sh | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index 3bcbfe1626..bab0316186 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -70,11 +70,12 @@ runs: uses: crazy-max/ghaction-github-runtime@v4 # DocumentDB is built from source; the GHA cache keeps that to the first run. + # The tag must match the e2e compose `db` image so `up` reuses it instead of rebuilding. - name: Build DocumentDB image shell: bash run: | docker buildx create --use --driver=docker-container - docker buildx bake -f packages/app/tests/e2e/docker-compose.yml db --set *.cache-to="type=gha,mode=max,scope=documentdb" --set *.cache-from="type=gha,scope=documentdb" --load + docker buildx build docker/documentdb -t hyperdx/documentdb:local --cache-to "type=gha,mode=max,scope=documentdb" --cache-from "type=gha,scope=documentdb" --load - name: Start E2E Docker Compose shell: bash diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh index 388c264294..02da89a67f 100755 --- a/docker/documentdb/entrypoint.sh +++ b/docker/documentdb/entrypoint.sh @@ -10,6 +10,14 @@ if [ "$(id -u)" = 0 ]; then exec su-exec postgres "$0" "$@" fi +# Postgres logs to a file; print it on a failed start so `docker logs` shows the cause. +start_postgres() { + pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null || { + tail -n 50 "$PGDATA/postgresql.log" >&2 + exit 1 + } +} + # The marker is written last, so a first boot killed part-way is redone, not left broken. if [ ! -f "$PGDATA/.documentdb-initialised" ]; then find "$PGDATA" -mindepth 1 -delete @@ -25,13 +33,13 @@ documentdb.indexBuildsScheduledOnBgWorker = false documentdb.rum_library_load_option = 'require_documentdb_extended_rum' documentdb.alternate_index_handler_name = 'extended_rum' CONF - pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null + start_postgres psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -c "CREATE EXTENSION documentdb CASCADE; CREATE EXTENSION documentdb_extended_rum;" doc=$(printf '{"createUser":"%s","pwd":"%s","roles":[{"role":"readWriteAnyDatabase","db":"admin"},{"role":"clusterAdmin","db":"admin"}]}' "$DOCDB_USER" "$DOCDB_PASSWORD") psql -h /tmp -U postgres -d postgres -qX -v ON_ERROR_STOP=1 -v doc="$doc" <<< "SELECT documentdb_api.create_user(:'doc');" >/dev/null touch "$PGDATA/.documentdb-initialised" else - pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null + start_postgres fi echo "postgresql://postgres@localhost:5432/postgres" > /tmp/pg-url From aecade3cbe3e47caa4f6beff5acf594925fddea6 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 15 Oct 2026 22:38:56 +1100 Subject: [PATCH 3/4] fix(ci): wait only on DocumentDB before integration tests The integration job's `up --wait` failed before any test ran: the OTel collector's healthcheck needs an OpAMP server the job doesn't run, so it never reports healthy. Start every service as before, then wait on db. --- .github/workflows/main.yml | 4 +++- Makefile | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 912d7fe028..8f10361478 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -143,7 +143,9 @@ jobs: # Actions streams stdout. - name: Run integration tests run: | - docker compose -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 + docker compose -f ./docker-compose.ci.yml up -d --quiet-pull + # Wait on DocumentDB only: the collector's healthcheck needs an OpAMP server this job doesn't run. + docker compose -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db status=0 yarn workspace @hyperdx/common-utils ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 yarn workspace @hyperdx/api ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 diff --git a/Makefile b/Makefile index 02caba3ab8..87192ea1e4 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,8 @@ dev-int: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/api:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/api-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -169,7 +170,8 @@ dev-int-common-utils: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/common-utils:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/common-utils-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -178,7 +180,8 @@ dev-int-common-utils: .PHONY: ci-int ci-int: @mkdir -p $(HDX_CI_LOGS_DIR) - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull --wait --wait-timeout 300 + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run-many -t ci:int --parallel=false --output-style=stream 2>&1 | tee $(HDX_CI_LOGS_DIR)/ci-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ From a4a8b4639098ba52495a327a97161c4429870473 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Thu, 5 Nov 2026 16:27:09 +1100 Subject: [PATCH 4/4] fix(ci): give the DocumentDB image its own integration cache scope Integration shards were rebuilding DocumentDB from source and timing out. The bake shared one GHA cache scope between the collector and db images, and a scope holds a single manifest, so each export overwrote the other. db now uses the documentdb scope the e2e job already fills. --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8f10361478..346bc40286 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,7 +121,11 @@ jobs: - name: Spin up docker services run: | docker buildx create --use --driver=docker-container - docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha,mode=max,scope=ci-integration" --set *.cache-from="type=gha,scope=ci-integration" --load + # One cache scope per image (a scope holds one manifest); db reuses the e2e job's documentdb scope. + docker buildx bake -f ./docker-compose.ci.yml \ + --set otel-collector.cache-to="type=gha,mode=max,scope=ci-integration" --set otel-collector.cache-from="type=gha,scope=ci-integration" \ + --set db.cache-to="type=gha,mode=max,scope=documentdb" --set db.cache-from="type=gha,scope=documentdb" \ + --load - name: Build dependencies run: make ci-build # Integration tests boot a MockServer on fixed ports and share a single