diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index f4eaaf1ac9..bab0316186 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -3,7 +3,7 @@ description: | Shared setup for E2E test workflows: sets the HDX_E2E_* port env vars and common Playwright env vars, then runs Node.js setup, dependency install, build, Playwright browser install, and starts the E2E Docker Compose stack - (MongoDB + ClickHouse) with health checks. The caller is responsible for + (DocumentDB + ClickHouse) with health checks. The caller is responsible for `actions/checkout`, running Playwright, uploading artifacts, and tearing down the Docker stack. runs: @@ -22,7 +22,7 @@ runs: echo "HDX_E2E_MONGO_PORT=$MONGO_PORT" >> $GITHUB_ENV echo "HDX_E2E_APP_LOCAL_PORT=21200" >> $GITHUB_ENV echo "HDX_E2E_APP_PORT=21300" >> $GITHUB_ENV - echo "MONGO_URI=mongodb://localhost:$MONGO_PORT/hyperdx-e2e" >> $GITHUB_ENV + echo "MONGO_URI=mongodb://hyperdx:hyperdx@localhost:$MONGO_PORT/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" >> $GITHUB_ENV echo "E2E_FULLSTACK=true" >> $GITHUB_ENV echo "E2E_UNIQUE_USER=true" >> $GITHUB_ENV echo "E2E_API_HEALTH_CHECK_MAX_RETRIES=60" >> $GITHUB_ENV @@ -66,12 +66,23 @@ runs: shell: bash run: cd packages/app && npx playwright install --with-deps chromium + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v4 + + # DocumentDB is built from source; the GHA cache keeps that to the first run. + # The tag must match the e2e compose `db` image so `up` reuses it instead of rebuilding. + - name: Build DocumentDB image + shell: bash + run: | + docker buildx create --use --driver=docker-container + docker buildx build docker/documentdb -t hyperdx/documentdb:local --cache-to "type=gha,mode=max,scope=documentdb" --cache-from "type=gha,scope=documentdb" --load + - name: Start E2E Docker Compose shell: bash run: | # Pre-pull images with retries to handle transient Docker Hub timeouts for attempt in 1 2 3; do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull; then + if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml pull --ignore-buildable; then echo "Docker images pulled successfully on attempt $attempt" break fi @@ -82,20 +93,12 @@ runs: echo "Docker pull failed (attempt $attempt/3), retrying in 10s..." sleep 10 done - docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d - echo "Waiting for MongoDB..." - for i in $(seq 1 30); do - if docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml exec -T db mongosh --quiet --eval "db.adminCommand({ping:1})" >/dev/null 2>&1; then - echo "MongoDB is ready" - break - fi - if [ "$i" -eq 30 ]; then - echo "MongoDB failed to become ready after 30 seconds" - exit 1 - fi - echo "Waiting for MongoDB... ($i/30)" - sleep 1 - done + # --wait blocks on the DocumentDB healthcheck (gateway accepting connections). + if ! docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml up -d --wait --wait-timeout 300; then + echo "E2E services failed to become healthy" + docker compose -p e2e-0 -f packages/app/tests/e2e/docker-compose.yml logs db | tail -30 + exit 1 + fi echo "Waiting for ClickHouse..." for i in $(seq 1 60); do if curl -sf http://localhost:${HDX_E2E_CH_PORT}/ping >/dev/null 2>&1; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b4c3c94a5..346bc40286 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,11 +121,15 @@ jobs: - name: Spin up docker services run: | docker buildx create --use --driver=docker-container - docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha,mode=max,scope=ci-integration" --set *.cache-from="type=gha,scope=ci-integration" --load + # One cache scope per image (a scope holds one manifest); db reuses the e2e job's documentdb scope. + docker buildx bake -f ./docker-compose.ci.yml \ + --set otel-collector.cache-to="type=gha,mode=max,scope=ci-integration" --set otel-collector.cache-from="type=gha,scope=ci-integration" \ + --set db.cache-to="type=gha,mode=max,scope=documentdb" --set db.cache-from="type=gha,scope=documentdb" \ + --load - name: Build dependencies run: make ci-build # Integration tests boot a MockServer on fixed ports and share a single - # MongoDB/ClickHouse database, so they must stay serial (--runInBand) + # DocumentDB/ClickHouse database, so they must stay serial (--runInBand) # within a runner. We shard ACROSS runners instead: each shard is its own # VM with its own Docker stack, and jest's --shard splits the test files # so the shards run in parallel. @@ -144,6 +148,8 @@ jobs: - name: Run integration tests run: | docker compose -f ./docker-compose.ci.yml up -d --quiet-pull + # Wait on DocumentDB only: the collector's healthcheck needs an OpAMP server this job doesn't run. + docker compose -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db status=0 yarn workspace @hyperdx/common-utils ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 yarn workspace @hyperdx/api ci:int --shard=${{ matrix.shard }}/${{ strategy.job-total }} || status=1 diff --git a/AGENTS.md b/AGENTS.md index 9e5653ec04..3590fde95d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,8 @@ yarn dev # Start full stack with worktree-isolated ports ``` The project uses **Yarn 4.13.0** workspaces. Docker Compose manages ClickHouse, -MongoDB, and the OTel Collector. +DocumentDB (the local metadata store, which the API reaches over the Mongo wire +protocol), and the OTel Collector. **This repo is multi-agent friendly.** `yarn dev`, `make dev-int`, and `make dev-e2e` all use slot-based port isolation so multiple worktrees can run @@ -461,7 +462,7 @@ default `/workspace` directory, the slot is **76**, so services are at: - **App**: http://localhost:30276 - **API**: http://localhost:30176 - **ClickHouse**: http://localhost:30576 -- **MongoDB**: localhost:30476 +- **DocumentDB**: localhost:30476 ### Key commands reference @@ -475,7 +476,7 @@ reference. Quick summary: ### First-time registration -When the dev stack starts fresh (empty MongoDB), the app shows a registration +When the dev stack starts fresh (empty DocumentDB), the app shows a registration page. Create any account to get started — no external auth provider is needed. --- diff --git a/Makefile b/Makefile index 91896878ea..87192ea1e4 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ install-tools: # API server : 30100 + slot # App (Next.js) : 30200 + slot # OpAMP : 30300 + slot -# MongoDB : 30400 + slot +# DocumentDB : 30400 + slot # ClickHouse HTTP : 30500 + slot # ClickHouse Native : 30600 + slot # OTel health : 30700 + slot @@ -158,7 +158,8 @@ dev-int: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/api:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/api-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -169,7 +170,8 @@ dev-int-common-utils: @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh - docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --build + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run @hyperdx/common-utils:dev:int $(FILE) 2>&1 | tee $(HDX_CI_LOGS_DIR)/common-utils-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ @@ -179,6 +181,7 @@ dev-int-common-utils: ci-int: @mkdir -p $(HDX_CI_LOGS_DIR) docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --quiet-pull + docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d --wait --wait-timeout 300 db bash -c 'set -o pipefail; npx nx run-many -t ci:int --parallel=false --output-style=stream 2>&1 | tee $(HDX_CI_LOGS_DIR)/ci-int.log'; ret=$$?; \ docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml down -v; \ $(call archive-int-logs); \ diff --git a/agent_docs/development.md b/agent_docs/development.md index 305c0d9380..f3c281c4d2 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -30,7 +30,16 @@ yarn dev # or equivalently: make dev ## Environment Configuration - `.env.development`: Development environment variables -- Docker Compose manages ClickHouse, MongoDB, OTel Collector +- Docker Compose manages ClickHouse, DocumentDB, OTel Collector +- The metadata store is [DocumentDB](https://github.com/documentdb/documentdb) + (Postgres plus a Mongo wire-protocol gateway), built from + `docker/documentdb/Dockerfile`. The API still talks to it through Mongoose. + To browse it with MongoDB Compass or `mongosh`, connect to + `mongodb://hyperdx:hyperdx@localhost:/?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`. + Everything works except Compass's Performance tab (the gateway doesn't + implement `serverStatus` or `top`). + Dev data lives in `.volumes/docdb_dev_`; data from the old MongoDB + volumes (`.volumes/db_dev_`) is not migrated, so the dev stack starts empty. - Hot reload enabled for all services in development ## Worktree Isolation (Multi-Agent / Multi-Developer) @@ -59,7 +68,7 @@ integration tests (14320-40098) and E2E tests (20320-21399). | API server | 30100 | 30100 - 30199 | `HYPERDX_API_PORT` | | App (Next.js) | 30200 | 30200 - 30299 | `HYPERDX_APP_PORT` | | OpAMP | 30300 | 30300 - 30399 | `HYPERDX_OPAMP_PORT` | -| MongoDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | +| DocumentDB | 30400 | 30400 - 30499 | `HDX_DEV_MONGO_PORT` | | ClickHouse HTTP | 30500 | 30500 - 30599 | `HDX_DEV_CH_HTTP_PORT` | | ClickHouse Native | 30600 | 30600 - 30699 | `HDX_DEV_CH_NATIVE_PORT` | | OTel health | 30700 | 30700 - 30799 | `HDX_DEV_OTEL_HEALTH_PORT` | @@ -173,7 +182,7 @@ Port mapping (base + slot): | Service | Default port (slot 0) | Variable | | --------------- | --------------------- | ----------------- | | ClickHouse HTTP | 18123 | HDX_CI_CH_PORT | -| MongoDB | 39999 | HDX_CI_MONGO_PORT | +| DocumentDB | 39999 | HDX_CI_MONGO_PORT | | API test server | 19000 | HDX_CI_API_PORT | | OpAMP | 14320 | HDX_CI_OPAMP_PORT | @@ -181,7 +190,7 @@ Port mapping (base + slot): - Uses separate Docker Compose configuration (`docker-compose.ci.yml`) - Isolated test environment with unique `-p int-` project name -- Includes all necessary services (ClickHouse, MongoDB, OTel Collector) +- Includes all necessary services (ClickHouse, DocumentDB, OTel Collector) - Tests run against real database instances for accurate integration testing ### E2E Testing @@ -198,7 +207,7 @@ E2E port mapping (base + slot): | ClickHouse HTTP | 20500 | 20500 - 20599 | `HDX_E2E_CH_PORT` | | ClickHouse Native | 20600 | 20600 - 20699 | `HDX_E2E_CH_NATIVE_PORT` | | API server | 21000 | 21000 - 21099 | `HDX_E2E_API_PORT` | -| MongoDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | +| DocumentDB | 21100 | 21100 - 21199 | `HDX_E2E_MONGO_PORT` | | App (local) | 21200 | 21200 - 21299 | `HDX_E2E_APP_LOCAL_PORT` | | App (fullstack) | 21300 | 21300 - 21399 | `HDX_E2E_APP_PORT` | diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index bbc07b9799..f425f5d98a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -35,10 +35,13 @@ services: networks: - internal db: - image: mongo:5.0.32-focal - command: --port 29999 + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - '${HDX_CI_MONGO_PORT:-39999}:29999' + - '${HDX_CI_MONGO_PORT:-39999}:10260' networks: - internal networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7f35762648..e06d979ed5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -7,13 +7,17 @@ services: db: labels: <<: *hdx-labels - hdx.dev.service: mongodb + hdx.dev.service: documentdb hdx.dev.port: '${HDX_DEV_MONGO_PORT:-27017}' - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ./docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx volumes: - - .volumes/db_dev_${HDX_DEV_SLOT:-0}:/data/db + - .volumes/docdb_dev_${HDX_DEV_SLOT:-0}:/data/docdb ports: - - '${HDX_DEV_MONGO_PORT:-27017}:27017' + - '${HDX_DEV_MONGO_PORT:-27017}:10260' networks: - internal otel-collector: diff --git a/docker/documentdb/Dockerfile b/docker/documentdb/Dockerfile new file mode 100644 index 0000000000..a17c9c9b0b --- /dev/null +++ b/docker/documentdb/Dockerfile @@ -0,0 +1,48 @@ +# DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used +# in place of MongoDB (SSPL). Built from source because upstream ships glibc +# packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. +ARG DOCUMENTDB_VERSION=v1.0-RC2 +# Tags can move; the build fails unless the tag still resolves to this commit. +ARG DOCUMENTDB_COMMIT=b0cb0dae85e8746f4e0b79c847894179b2f80d64 + +FROM alpine:3.24 AS builder +ARG DOCUMENTDB_VERSION +ARG DOCUMENTDB_COMMIT +RUN apk add --no-cache build-base cmake git curl bash tar pkgconf perl linux-headers python3 \ + icu-dev krb5-dev openssl-dev rust cargo \ + postgresql18 postgresql18-dev postgresql18-contrib postgresql-pg_cron postgresql-pgvector postgis +ENV PG_CONFIG=/usr/libexec/postgresql18/pg_config INSTALL_DEPENDENCIES_ROOT=/deps +RUN git clone --depth 1 --branch ${DOCUMENTDB_VERSION} https://github.com/documentdb/documentdb.git /src \ + && test "$(git -C /src rev-parse HEAD)" = "${DOCUMENTDB_COMMIT}" \ + && mkdir -p /deps +# musl exports _init/_fini from every .so, which the duplicate-export check counts. +RUN sed -i "s#grep -vxE '_PG_init|Pg_magic_func'#grep -vxE '_PG_init|Pg_magic_func|_init|_fini'#g" \ + /src/pg_documentdb_extended_rum/validate_rum_core_exports.sh +RUN cd /src/scripts && MAKE_PROGRAM=cmake bash install_setup_libbson.sh \ + && bash install_setup_pcre2.sh && bash install_setup_intel_decimal_math_lib.sh +# Serial make: with -j the RUM export check can run before the library is linked. +RUN cd /src && make install-documentdb with_llvm=no PG_CONFIG=$PG_CONFIG +RUN cd /src/pg_documentdb_gw && cargo build --release -p documentdb_gateway \ + && install -s target/release/documentdb_gateway /usr/bin/documentdb-gateway +# Keep only what Postgres and DocumentDB link against: no GDAL, JIT or PostGIS raster/topology. +RUN set -e; PG=/usr/libexec/postgresql18; LIB=/usr/lib/postgresql18; SH=/usr/share/postgresql18; \ + rm -f $LIB/llvmjit*.so $LIB/postgis_raster-3.so $LIB/postgis_sfcgal-3.so $LIB/postgis_topology-3.so $LIB/address_standardizer*.so; rm -rf $LIB/bitcode; \ + rm -f $SH/extension/postgis_raster* $SH/extension/postgis_sfcgal* $SH/extension/postgis_topology* $SH/extension/postgis_tiger* $SH/extension/address_standardizer*; rm -rf $SH/contrib/postgis*; \ + for f in $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB/*.so; do echo $f; ldd $f 2>/dev/null | awk '/=> \//{print $3}'; done | sort -u > /tmp/files; \ + mkdir /out; tar -ch -T /tmp/files -f - | tar -x -C /out; \ + tar -ch $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql $LIB $SH /usr/share/proj /usr/share/icu -f - | tar -x -C /out; \ + install -D -m 644 /src/pg_documentdb_gw/SetupConfiguration.json /out/etc/documentdb/gateway/SetupConfiguration.json + +FROM alpine:3.24 AS runtime +RUN apk add --no-cache bash openssl su-exec \ + && addgroup -S postgres && adduser -S -G postgres -h /var/lib/postgresql postgres \ + && install -d -o postgres -g postgres /data/docdb +COPY --from=builder /out/ / +COPY --chmod=755 entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh +ENV PATH=/usr/libexec/postgresql18:$PATH PGDATA=/data/docdb/pgdata +EXPOSE 10260 +# The gateway only listens once Postgres is initialised and the user exists; the +# psql probe catches Postgres dying under a gateway that keeps its port open. +HEALTHCHECK --interval=2s --timeout=3s --start-period=60s --retries=30 \ + CMD nc -z 127.0.0.1 10260 && psql -h /tmp -U postgres -d postgres -qtAX -c 'SELECT 1' >/dev/null +ENTRYPOINT ["bash", "/usr/local/bin/documentdb-entrypoint.sh"] diff --git a/docker/documentdb/entrypoint.sh b/docker/documentdb/entrypoint.sh new file mode 100755 index 0000000000..02da89a67f --- /dev/null +++ b/docker/documentdb/entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Starts Postgres with the DocumentDB extensions, then the gateway on :10260. +# Run as root: fixes data dir ownership, then re-runs itself as postgres. +set -euo pipefail +: "${DOCDB_USER:?DOCDB_USER is required}" "${DOCDB_PASSWORD:?DOCDB_PASSWORD is required}" + +if [ "$(id -u)" = 0 ]; then + # Volumes may be root-owned or hold lost+found, and initdb needs an empty dir it owns. + install -d -o postgres -g postgres -m 700 "$PGDATA" + exec su-exec postgres "$0" "$@" +fi + +# Postgres logs to a file; print it on a failed start so `docker logs` shows the cause. +start_postgres() { + pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" -w start >/dev/null || { + tail -n 50 "$PGDATA/postgresql.log" >&2 + exit 1 + } +} + +# The marker is written last, so a first boot killed part-way is redone, not left broken. +if [ ! -f "$PGDATA/.documentdb-initialised" ]; then + find "$PGDATA" -mindepth 1 -delete + initdb -D "$PGDATA" -U postgres --auth=trust >/dev/null + cat >> "$PGDATA/postgresql.conf" </dev/null + touch "$PGDATA/.documentdb-initialised" +else + start_postgres +fi + +echo "postgresql://postgres@localhost:5432/postgres" > /tmp/pg-url +mkdir -p /tmp/tls +export DOCUMENTDB_PG_URL_FILE=/tmp/pg-url DOCUMENTDB_TLS_STATE_DIR=/tmp/tls + +documentdb-gateway run --config /etc/documentdb/gateway/SetupConfiguration.json & +gateway=$! +# Stop Postgres cleanly on docker stop so persistent volumes skip crash recovery. +trap 'kill "$gateway" 2>/dev/null; pg_ctl -D "$PGDATA" -m fast stop >/dev/null; exit 0' TERM INT +# Keep the gateway's exit code so a crash shows as a failed container, not a clean exit. +rc=0 +wait "$gateway" || rc=$? +pg_ctl -D "$PGDATA" -m fast stop >/dev/null || true +exit "$rc" diff --git a/packages/api/.env.development b/packages/api/.env.development index d4995195fd..9da78e573a 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -6,7 +6,7 @@ FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 HDX_NODE_BETA_MODE=1 HDX_NODE_CONSOLE_CAPTURE=1 -MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT}/hyperdx" +MONGO_URI="mongodb://hyperdx:hyperdx@localhost:${HDX_DEV_MONGO_PORT}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" NODE_ENV=development OTEL_SERVICE_NAME="hdx-oss-dev-api" OTEL_RESOURCE_ATTRIBUTES="service.version=dev" diff --git a/packages/api/.env.e2e b/packages/api/.env.e2e index 9db633c4d6..8b93a929c7 100644 --- a/packages/api/.env.e2e +++ b/packages/api/.env.e2e @@ -10,8 +10,8 @@ CLICKHOUSE_PASSWORD= CLICKHOUSE_USER=default RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://localhost:${HDX_E2E_APP_PORT:-21300} -# MongoDB connection string -MONGO_URI=mongodb://localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e +# DocumentDB (Mongo wire protocol) connection string +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_E2E_MONGO_PORT:-21100}/hyperdx-e2e?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_E2E_API_PORT:-21000} OPAMP_PORT=${HDX_E2E_OPAMP_PORT:-20320} diff --git a/packages/api/.env.test b/packages/api/.env.test index 45ee1e9b43..ddba188320 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -7,7 +7,7 @@ CLICKHOUSE_PASSWORD=api CLICKHOUSE_USER=api RUN_SCHEDULED_TASKS_EXTERNALLY=true FRONTEND_URL=http://app:8080 -MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} diff --git a/packages/api/scripts/seed-alerts.ts b/packages/api/scripts/seed-alerts.ts index 28220ab4ea..fa68ca4e18 100644 --- a/packages/api/scripts/seed-alerts.ts +++ b/packages/api/scripts/seed-alerts.ts @@ -120,7 +120,7 @@ function parseArgs(argv: string[]): Args { } /** - * Mongo URI of this worktree's dev stack. `yarn seed:alerts` runs through + * URI of this worktree's dev DocumentDB. `yarn seed:alerts` runs through * scripts/with-slots.sh, which sets HDX_DEV_MONGO_PORT. */ function devSlotMongoUri(): string { @@ -130,7 +130,7 @@ function devSlotMongoUri(): string { 'HDX_DEV_MONGO_PORT is not set. Run this with `yarn seed:alerts`, or pass --mongo-uri.', ); } - return `mongodb://localhost:${port}/hyperdx`; + return `mongodb://hyperdx:hyperdx@localhost:${port}/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; } function randomInt(minInclusive: number, maxInclusive: number): number { diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 47d2edafd8..58a8678d53 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -22,6 +22,7 @@ import { AlertInterval, AlertSource } from '@/models/alert'; import Server from '@/server'; import logger from '@/utils/logger'; import { MetricModel } from '@/utils/logParser'; +import { withRetry } from '@/utils/retry'; import { ExternalDashboardTile } from './utils/zod'; @@ -141,7 +142,13 @@ export const closeDB = async () => { if (!config.IS_CI) { throw new Error('ONLY execute this in CI env 😈 !!!'); } - await mongooseConnection.dropDatabase(); + // DocumentDB intermittently fails dropDatabase while background index builds + // run (WriteConflict 112, or InternalError 1); a retry clears it. + await withRetry(() => mongooseConnection.dropDatabase(), { + maxRetries: 5, + initialDelayMs: 200, + retryOnlyOnStatus: [1, 112], + }); await mongoose.disconnect(); }; diff --git a/packages/app/package.json b/packages/app/package.json index c6fc18f131..334ab03e62 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -139,6 +139,7 @@ "identity-obj-proxy": "^3.0.0", "jest": "^30.2.0", "jest-environment-jsdom": "^30.2.0", + "mongodb": "^6.15.0", "msw": "^2.3.0", "msw-storybook-addon": "^2.0.2", "postcss": "^8.5.23", diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index b8152ccc6c..a7a2c3a2b1 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -1,9 +1,13 @@ name: hdx-e2e services: db: - image: mongo:5.0.32-focal + image: hyperdx/documentdb:local + build: ../../../../docker/documentdb + environment: + DOCDB_USER: hyperdx + DOCDB_PASSWORD: hyperdx ports: - - ${HDX_E2E_MONGO_PORT:-21100}:27017 + - ${HDX_E2E_MONGO_PORT:-21100}:10260 networks: - internal deploy: diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 459fa92a4b..2d6d864d05 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -12,7 +12,7 @@ import { DEFAULT_METRICS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, } from '../utils/constants'; -import { runMongoshScript } from '../utils/db-helpers'; +import { ObjectId, withE2eDb } from '../utils/db-helpers'; import { getSqlEditor } from '../utils/locators'; test.describe('Dashboard', { tag: ['@dashboard'] }, () => { @@ -1172,17 +1172,16 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written straight to Mongo: the API and the form both refuse this // state now, so the only way to hold it is to predate that rule. const dashboardId = dashboardPage.getCurrentDashboardId(); - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { "filters.0.isVariableEnabled": false } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { 'filters.0.isVariableEnabled': false } }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); await dashboardPage.gotoDashboard(dashboardId); @@ -3450,19 +3449,18 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { await test.step('Mark it provisioned', async () => { // ProvisionDashboardsTask is the only thing that sets this in normal // operation, so write it directly rather than running that job. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - ' { $set: { provisioned: true } }', - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db + .collection('dashboards') + .updateOne( + { _id: new ObjectId(dashboardId) }, + { $set: { provisioned: true } }, + ), ); // A zero-match write would make the assertion below pass for a reason // that has nothing to do with the gate. - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { @@ -3506,19 +3504,33 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { // Written directly rather than built in the chart editor: the editor // flow for a PromQL tile is long and this test is about the gate, not // the editor. - const out = runMongoshScript( - [ - "use('hyperdx-e2e');", - 'print(JSON.stringify(db.dashboards.updateOne(', - ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, - " { $set: { tiles: [{ id: 'promql-1', x: 0, y: 0, w: 4, h: 2,", - " config: { configType: 'promql', promqlExpression: 'up',", - " connection: 'c1', source: 's1', displayType: 'line' } }] } }", - ')));', - ].join('\n'), + const out = await withE2eDb(db => + db.collection('dashboards').updateOne( + { _id: new ObjectId(dashboardId) }, + { + $set: { + tiles: [ + { + id: 'promql-1', + x: 0, + y: 0, + w: 4, + h: 2, + config: { + configType: 'promql', + promqlExpression: 'up', + connection: 'c1', + source: 's1', + displayType: 'line', + }, + }, + ], + }, + }, + ), ); - expect(out).toContain('"matchedCount":1'); - expect(out).toContain('"modifiedCount":1'); + expect(out.matchedCount).toBe(1); + expect(out.modifiedCount).toBe(1); }); await test.step('Verify it is withheld after reload', async () => { diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 0ee087888a..79cac5b20d 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -341,7 +341,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should show series Table field when isMetricsSeriesTableEnabled is on', { tag: ['@full-stack'] }, async () => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); try { await searchPage.goto(); await searchPage.sourceActionsMenu.click(); @@ -358,7 +358,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { await searchPage.page.keyboard.press('Escape'); } finally { - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); @@ -367,7 +367,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { 'should warn when the configured series table does not match the series table schema', { tag: ['@full-stack'] }, async ({ page }) => { - setTeamFlag('isMetricsSeriesTableEnabled', true); + await setTeamFlag('isMetricsSeriesTableEnabled', true); const API_URL = getApiUrl(); const metricSources = await getSources(page, 'metric'); @@ -415,7 +415,7 @@ test.describe('Sources Functionality', { tag: ['@sources'] }, () => { ).toBeVisible({ timeout: 20000 }); } finally { await page.request.delete(`${API_URL}/sources/${createdSourceId}`); - setTeamFlag('isMetricsSeriesTableEnabled', false); + await setTeamFlag('isMetricsSeriesTableEnabled', false); } }, ); diff --git a/packages/app/tests/e2e/global-setup-fullstack.ts b/packages/app/tests/e2e/global-setup-fullstack.ts index 9c196b6900..819d097789 100644 --- a/packages/app/tests/e2e/global-setup-fullstack.ts +++ b/packages/app/tests/e2e/global-setup-fullstack.ts @@ -2,13 +2,13 @@ * Global setup for full-stack E2E tests * * This setup: - * 1. Clears MongoDB database to ensure clean state + * 1. Clears the database to ensure clean state * 2. Creates a test user and team * 3. Applies DEFAULT_SOURCES from .env.e2e * 4. Saves authentication state for tests * * Full-stack mode uses: - * - MongoDB (local) for authentication, teams, users, persistence + * - DocumentDB (local) for authentication, teams, users, persistence * - API server (local) for backend logic * - Demo ClickHouse (remote) for telemetry data (logs, traces, metrics, K8s) */ @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import { chromium, FullConfig } from '@playwright/test'; -import { runMongoshScript } from './utils/db-helpers'; +import { dropE2eDatabase, ObjectId, withE2eDb } from './utils/db-helpers'; import { seedClickHouse } from './seed-clickhouse'; // Configuration constants @@ -63,14 +63,14 @@ export const SEEDED_ERROR_ALERT = { }; /** - * Clears the MongoDB database to ensure a clean slate for tests + * Clears the database to ensure a clean slate for tests */ -function clearDatabase() { - console.log('Clearing MongoDB database for fresh test run...'); +async function clearDatabase() { + console.log('Clearing the e2e database for a fresh test run...'); try { - runMongoshScript("use('hyperdx-e2e'); db.dropDatabase();"); - console.log(' ✓ Database cleared successfully (via Docker)'); + await dropE2eDatabase(); + console.log(' ✓ Database cleared successfully'); } catch (error) { console.warn(' ⚠ Warning: Could not clear database'); console.warn(` ${error instanceof Error ? error.message : String(error)}`); @@ -85,7 +85,7 @@ function clearDatabase() { async function globalSetup(_config: FullConfig) { console.log('Setting up full-stack E2E environment'); - console.log(' MongoDB: local (auth, teams, persistence)'); + console.log(' DocumentDB: local (auth, teams, persistence)'); console.log(' ClickHouse: local instance (telemetry data)'); // Set timezone @@ -138,8 +138,8 @@ async function globalSetup(_config: FullConfig) { ); } - // Clear MongoDB database to ensure DEFAULT_SOURCES is applied - clearDatabase(); + // Clear the database to ensure DEFAULT_SOURCES is applied + await clearDatabase(); // Create test user and save auth state console.log('Creating test user and logging in'); @@ -302,7 +302,7 @@ async function globalSetup(_config: FullConfig) { * Seeds an alert with a recorded execution error. The alert is created via the * API (so all referenced documents — saved search, webhook — exist and the * alerts list endpoint populates correctly), then the `errors` array is - * patched in directly via mongosh since it's only ever set by the check-alerts + * patched in directly via the driver since it's only ever set by the check-alerts * background job in normal operation. */ async function seedAlertWithErrors( @@ -379,7 +379,7 @@ async function seedAlertWithErrors( const alert = (await alertRes.json()).data; const alertId: string = alert._id ?? alert.id; - // 4) Patch the `executionErrors` array directly via mongosh. The + // 4) Patch the `executionErrors` array directly in the database. The // check-alerts job is the only code that writes this field in normal // operation, so we write it here to avoid having to run that job during // setup. @@ -390,51 +390,53 @@ async function seedAlertWithErrors( const errorWindowStart = Math.floor(Date.now() / windowMs) * windowMs; const okWindowStart = errorWindowStart - windowMs; - const patchScript = ` -use('hyperdx-e2e'); -db.alerts.updateOne( - { _id: ObjectId(${JSON.stringify(alertId)}) }, - { - $set: { - executionErrors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.errorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.errorMessage)} - } - ], - state: 'OK' - } - } -); -db.alerthistories.deleteMany({ alert: ObjectId(${JSON.stringify(alertId)}) }); -db.alerthistories.insertMany([ - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${okWindowStart}), - state: 'OK', - counts: 0, - lastValues: [{ startTime: new Date(${okWindowStart - windowMs}), count: 0 }] - }, - { - alert: ObjectId(${JSON.stringify(alertId)}), - createdAt: new Date(${errorWindowStart}), - state: 'ERROR', - counts: 0, - lastValues: [], - errors: [ - { - timestamp: new Date(), - type: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorType)}, - message: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorMessage)} - } - ] - } -]); -`; - try { - runMongoshScript(patchScript); + const alertObjectId = new ObjectId(alertId); + await withE2eDb(async db => { + await db.collection('alerts').updateOne( + { _id: alertObjectId }, + { + $set: { + executionErrors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.errorType, + message: SEEDED_ERROR_ALERT.errorMessage, + }, + ], + state: 'OK', + }, + }, + ); + await db + .collection('alerthistories') + .deleteMany({ alert: alertObjectId }); + await db.collection('alerthistories').insertMany([ + { + alert: alertObjectId, + createdAt: new Date(okWindowStart), + state: 'OK', + counts: 0, + lastValues: [ + { startTime: new Date(okWindowStart - windowMs), count: 0 }, + ], + }, + { + alert: alertObjectId, + createdAt: new Date(errorWindowStart), + state: 'ERROR', + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: SEEDED_ERROR_ALERT.historyErrorType, + message: SEEDED_ERROR_ALERT.historyErrorMessage, + }, + ], + }, + ]); + }); console.log( ` ✓ Seeded alert "${alert.name}" (${alertId}) with a ${SEEDED_ERROR_ALERT.errorType}`, ); diff --git a/packages/app/tests/e2e/utils/db-helpers.ts b/packages/app/tests/e2e/utils/db-helpers.ts index 3d30f61307..947f4a4974 100644 --- a/packages/app/tests/e2e/utils/db-helpers.ts +++ b/packages/app/tests/e2e/utils/db-helpers.ts @@ -1,55 +1,44 @@ /** - * Direct MongoDB and ClickHouse access helpers for full-stack E2E tests. Only usable in - * full-stack mode (real Mongo via docker-compose) — there is no database in + * Direct metadata-store and ClickHouse access helpers for full-stack E2E tests. Only + * usable in full-stack mode (DocumentDB via docker-compose); there is no database in * local mode, so callers must gate on `{ tag: ['@full-stack'] }`. */ -import { execFileSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; +import { type Db, MongoClient } from 'mongodb'; + +export { ObjectId } from 'mongodb'; + +const E2E_DB_NAME = 'hyperdx-e2e'; + +// Built from the e2e slot port rather than MONGO_URI, so a MONGO_URI exported in a +// developer's shell can never point these destructive helpers at another database. +function e2eMongoUri(): string { + return `mongodb://hyperdx:hyperdx@localhost:${process.env.HDX_E2E_MONGO_PORT || '21100'}/${E2E_DB_NAME}?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`; +} + +/** Runs `fn` against the e2e database over the Mongo wire protocol. */ +export async function withE2eDb(fn: (db: Db) => Promise): Promise { + const client = await MongoClient.connect(e2eMongoUri()); + try { + return await fn(client.db(E2E_DB_NAME)); + } finally { + await client.close(); + } +} /** - * Run a mongosh script against the e2e MongoDB container by piping the script - * through stdin. Using stdin (rather than `--eval "<...>"`) avoids having to - * escape quotes in the script body, so callers can pass multi-line JavaScript - * with string literals verbatim. - * - * `execFileSync` with an argument array rather than a shell string: the project - * slug comes from an env var, and with no shell nothing in it can be read as a - * metacharacter. - * - * Throws if the docker-compose file can't be found (meaning we're not running - * in the expected Docker-backed e2e environment). + * Drops the e2e database. DocumentDB intermittently fails this while the API's + * background index builds run (WriteConflict 112, or InternalError 1). */ -export function runMongoshScript(script: string): string { - const dockerComposeFile = path.join(__dirname, '..', 'docker-compose.yml'); - if (!fs.existsSync(dockerComposeFile)) { - throw new Error( - `docker-compose.yml not found at ${dockerComposeFile} — e2e Docker stack unavailable`, - ); +export async function dropE2eDatabase(): Promise { + for (let attempt = 1; ; attempt++) { + try { + await withE2eDb(db => db.dropDatabase()); + return; + } catch (e: any) { + if (attempt >= 5 || ![1, 112].includes(e?.code)) throw e; + await new Promise(resolve => setTimeout(resolve, 200 * attempt)); + } } - - const e2eSlot = process.env.HDX_E2E_SLOT || '0'; - - return execFileSync( - 'docker', - [ - 'compose', - '-p', - `e2e-${e2eSlot}`, - '-f', - dockerComposeFile, - 'exec', - '-T', - 'db', - 'mongosh', - '--quiet', - ], - { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - input: script, - }, - ); } /** @@ -64,11 +53,13 @@ export function runMongoshScript(script: string): string { * racing each other — e.g. via `test.describe.serial(...)` — since * `fullyParallel: true` lets tests in the same file run concurrently. */ -export function setTeamFlag(flagName: string, value: boolean): void { - runMongoshScript(` -use('hyperdx-e2e'); -db.teams.updateOne({}, { $set: { [${JSON.stringify(flagName)}]: ${JSON.stringify(value)} } }); -`); +export async function setTeamFlag( + flagName: string, + value: boolean, +): Promise { + await withE2eDb(db => + db.collection('teams').updateOne({}, { $set: { [flagName]: value } }), + ); } const CLICKHOUSE_HOST = diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 9ef34ae14b..4c01308c26 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -103,7 +103,7 @@ echo "╠═══════════════════════ echo "║ App (Next.js) http://localhost:${HYPERDX_APP_PORT}$(printf '%*s' $((22 - ${#HYPERDX_APP_PORT})) '')║" echo "║ API http://localhost:${HYPERDX_API_PORT}$(printf '%*s' $((22 - ${#HYPERDX_API_PORT})) '')║" echo "║ ClickHouse http://localhost:${HDX_DEV_CH_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_CH_HTTP_PORT})) '')║" -echo "║ MongoDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" +echo "║ DocumentDB localhost:${HDX_DEV_MONGO_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_MONGO_PORT})) '')║" echo "║ OTel HTTP http://localhost:${HDX_DEV_OTEL_HTTP_PORT}$(printf '%*s' $((22 - ${#HDX_DEV_OTEL_HTTP_PORT})) '')║" echo "║ OTel gRPC localhost:${HDX_DEV_OTEL_GRPC_PORT}$(printf '%*s' $((29 - ${#HDX_DEV_OTEL_GRPC_PORT})) '')║" echo "║ OpAMP localhost:${HYPERDX_OPAMP_PORT}$(printf '%*s' $((29 - ${#HYPERDX_OPAMP_PORT})) '')║" diff --git a/scripts/dev-portal/index.html b/scripts/dev-portal/index.html index 8dc02c85eb..505fe84c97 100644 --- a/scripts/dev-portal/index.html +++ b/scripts/dev-portal/index.html @@ -329,7 +329,7 @@

app: 'App (Next.js)', api: 'API Server', clickhouse: 'ClickHouse', - mongodb: 'MongoDB', + documentdb: 'DocumentDB', 'otel-collector': 'OTel Collector', 'otel-collector-json': 'OTel JSON', alerts: 'Alerts Task', diff --git a/scripts/dev-portal/server.js b/scripts/dev-portal/server.js index 05524a4e0a..ab34e6b096 100755 --- a/scripts/dev-portal/server.js +++ b/scripts/dev-portal/server.js @@ -184,7 +184,7 @@ function parseProject(projectName) { // compose service name (ch-server, db, otel-collector, …). const COMPOSE_SERVICE_ALIASES = { 'ch-server': 'clickhouse', - db: 'mongodb', + db: 'documentdb', }; function canonicalServiceName(labels) { @@ -457,7 +457,7 @@ async function buildDashboardData() { 'alerts', 'common-utils', 'clickhouse', - 'mongodb', + 'documentdb', 'otel-collector', 'otel-collector-json', ]; @@ -517,7 +517,7 @@ const ENV_LOG_DIRS = { // Map canonical service names to Docker Compose service names const DOCKER_SERVICE_NAMES = { clickhouse: 'ch-server', - mongodb: 'db', + documentdb: 'db', 'otel-collector': 'otel-collector', 'otel-collector-json': 'otel-collector-json', }; diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 7b6e060d66..76d3c55ac7 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,7 +1,7 @@ #!/bin/bash # Run E2E tests in full-stack or local mode -# Full-stack mode (default): MongoDB + API + local ClickHouse -# Local mode: Frontend + local ClickHouse (no MongoDB/API) +# Full-stack mode (default): DocumentDB + API + local ClickHouse +# Local mode: Frontend + local ClickHouse (no DocumentDB/API) # # Usage: # ./scripts/test-e2e.sh # Run all tests in fullstack mode @@ -57,11 +57,9 @@ exec > >(tee "$HDX_E2E_LOGS_DIR/e2e.log") 2>&1 # shellcheck source=./ensure-dev-portal.sh source "${REPO_ROOT}/scripts/ensure-dev-portal.sh" -echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} mongo=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" +echo "Using E2E slot ${HDX_E2E_SLOT} (project=${E2E_PROJECT} ch=${HDX_E2E_CH_PORT} ch-native=${HDX_E2E_CH_NATIVE_PORT} db=${HDX_E2E_MONGO_PORT} api=${HDX_E2E_API_PORT} app=${HDX_E2E_APP_PORT} app-local=${HDX_E2E_APP_LOCAL_PORT} opamp=${HDX_E2E_OPAMP_PORT})" # Configuration constants -readonly MAX_MONGODB_WAIT_ATTEMPTS=15 -readonly MONGODB_WAIT_DELAY_SECONDS=1 readonly MAX_CLICKHOUSE_WAIT_ATTEMPTS=30 readonly CLICKHOUSE_WAIT_DELAY_SECONDS=1 @@ -107,22 +105,6 @@ METAEOF rm -rf "$HDX_E2E_LOGS_DIR" 2>/dev/null || true } -check_mongodb_health() { - # Health check script that tests ping, insert, and delete operations - # Note: MongoDB runs on port 27017 inside the container (default) - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db mongosh --quiet --eval " - try { - db.adminCommand('ping'); - db.getSiblingDB('test').test.insertOne({_id: 'healthcheck', ts: new Date()}); - db.getSiblingDB('test').test.deleteOne({_id: 'healthcheck'}); - print('ready'); - } catch(e) { - print('not ready: ' + e); - quit(1); - } - " 2>&1 -} - check_clickhouse_health() { # Health check from HOST perspective (not inside container) # This ensures the port is actually accessible to Playwright @@ -152,40 +134,6 @@ wait_for_clickhouse() { done } -wait_for_mongodb() { - echo "Waiting for MongoDB to be ready..." - local attempt=1 - - # Verify mongosh is available in the container - if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" exec -T db which mongosh >/dev/null 2>&1; then - echo "ERROR: mongosh not found in MongoDB container" - echo "Container may not be running or using incompatible image" - echo "Try running: docker compose -p $E2E_PROJECT -f $DOCKER_COMPOSE_FILE logs db" - return 1 - fi - - while [ $attempt -le $MAX_MONGODB_WAIT_ATTEMPTS ]; do - local result - result=$(check_mongodb_health) - - if echo "$result" | grep -q "ready"; then - echo "MongoDB is ready and accepting writes" - return 0 - fi - - if [ $attempt -eq $MAX_MONGODB_WAIT_ATTEMPTS ]; then - local total_wait=$((MAX_MONGODB_WAIT_ATTEMPTS * MONGODB_WAIT_DELAY_SECONDS)) - echo "MongoDB failed to become ready after $total_wait seconds" - echo "Last error: $result" - return 1 - fi - - echo "Waiting for MongoDB... ($attempt/$MAX_MONGODB_WAIT_ATTEMPTS)" - attempt=$((attempt + 1)) - sleep $MONGODB_WAIT_DELAY_SECONDS - done -} - # Main execution setup_cleanup_trap() { if [ "$SKIP_CLEANUP" = false ]; then @@ -216,7 +164,7 @@ run_tests() { echo "Running tests in local mode (frontend + ClickHouse)..." yarn test:e2e --local "${PLAYWRIGHT_FLAGS[@]}" else - echo "Running tests in full-stack mode (MongoDB + API + ClickHouse)..." + echo "Running tests in full-stack mode (DocumentDB + API + ClickHouse)..." yarn test:e2e "${PLAYWRIGHT_FLAGS[@]}" fi } @@ -230,12 +178,13 @@ rm -rf "$REPO_ROOT/packages/app/.next-e2e" 2>/dev/null || true # Always start and seed ClickHouse (shared by both modes) setup_clickhouse -# Conditionally start MongoDB for full-stack mode +# Conditionally start DocumentDB for full-stack mode if [ "$LOCAL_MODE" = false ]; then - echo "Starting MongoDB for full-stack mode..." - docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d db - - if ! wait_for_mongodb; then + echo "Starting DocumentDB for full-stack mode..." + # --wait blocks on the image healthcheck, which passes once the gateway accepts connections. + if ! docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" up -d --build --wait --wait-timeout 300 db; then + echo "DocumentDB failed to become healthy" + docker compose -p "$E2E_PROJECT" -f "$DOCKER_COMPOSE_FILE" logs db | tail -20 exit 1 fi fi diff --git a/yarn.lock b/yarn.lock index dd87de2efb..b643662fa7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4880,6 +4880,7 @@ __metadata: ky: "npm:^0.30.0" ky-universal: "npm:^0.10.1" lodash: "npm:^4.18.1" + mongodb: "npm:^6.15.0" ms: "npm:^2.1.3" msw: "npm:^2.3.0" msw-storybook-addon: "npm:^2.0.2"