diff --git a/.env.dfe-example b/.env.dfe-example index e4e3c142b..5796f4668 100644 --- a/.env.dfe-example +++ b/.env.dfe-example @@ -5,14 +5,14 @@ # docker compose --env-file .env --env-file .env.dfe \ # -f docker-compose.yml -f docker-compose.dfe.yml up -d -# PostgreSQL / FerretDB credentials (used by docker-compose.dfe.yml) +# DocumentDB login (used by docker-compose.dfe.yml), created on its first start POSTGRES_USER=hyperdx POSTGRES_PASSWORD=hyperdx -# FerretDB connection (replaces MongoDB) +# DocumentDB connection (replaces MongoDB), TLS with a SCRAM-SHA-256 login # For the API running inside Docker (app service), this is set in docker-compose.dfe.yml -# For local development (API outside Docker), use: -# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?authMechanism=PLAIN +# For local development (API outside Docker, against docker-compose.dfe.dev.yml), use: +# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 # DFE Auth Mode: 'oidc-proxy' enables OIDC via Envoy, '' or unset = upstream Passport.js DFE_AUTH_MODE=oidc-proxy diff --git a/.env.dfe.example b/.env.dfe.example index a274db5a2..11c530829 100644 --- a/.env.dfe.example +++ b/.env.dfe.example @@ -1,10 +1,12 @@ # DFE dev overrides -- copy to .env.dfe (gitignored) and adjust. # Consumed by `yarn dev:dfe` (compose --env-file + dotenvx -f). -# FerretDB backing store (required by docker-compose.dfe.dev.yml). -# FERRETDB_AUTH is false in dev, so any matching pair works. +# DocumentDB login (required by docker-compose.dfe.dev.yml), created on its +# first start. The gateway requires TLS with a SCRAM-SHA-256 login, on a +# self-signed certificate that tlsAllowInvalidCertificates accepts in dev only. POSTGRES_USER=hyperdx POSTGRES_PASSWORD=hyperdx +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 # DFE identity master switch: empty disables the /dfe/* identity layer, # `oidc-proxy` verifies engine JWTs, `header-dev` trusts dev headers. diff --git a/.github/workflows/dfe-documentdb.yml b/.github/workflows/dfe-documentdb.yml new file mode 100644 index 000000000..465e9f57b --- /dev/null +++ b/.github/workflows/dfe-documentdb.yml @@ -0,0 +1,100 @@ +# Project: dfe-hyperdx +# File: .github/workflows/dfe-documentdb.yml +# Purpose: Run the api integration suite, dfe/ tests included, on DocumentDB. +# +# License: MIT - HYPERI PTY LIMITED +# Copyright: (c) 2026 HYPERI PTY LIMITED +# +# DFE keeps HyperDX's documents in DocumentDB, which speaks the MongoDB wire +# protocol over Postgres and implements a tested subset of MongoDB. Upstream's +# harness runs MongoDB, so this job layers dfe/test/docker-compose.documentdb.yml +# over docker-compose.ci.yml and leaves upstream's files as they ship. + +name: dfe-documentdb + +'on': + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: dfe-documentdb-${{ github.ref }} + cancel-in-progress: true + +env: + # TLS and SCRAM-SHA-256 as deployed. The harness certificate is self-signed, + # hence tlsAllowInvalidCertificates, which no deployment should carry. + DFE_INT_MONGO_URI: mongodb://hyperdx:hyperdx-int@localhost:39999/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 + +jobs: + integration: + name: Integration on DocumentDB (shard ${{ matrix.shard }}) + runs-on: ubuntu-24.04 + # The whole suite took 949s serially on DocumentDB, so it is split like upstream's. + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: '.nvmrc' + cache-dependency-path: 'yarn.lock' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --immutable + + # The bake step's GHA cache needs the runtime token in the shell. + - name: Expose GitHub runtime + uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + + # The collector creates the ClickHouse schema the suite waits for. It is + # built from source, so the layer cache saves the OCB compile on every run + # after the first. The tag is the name compose gives it, so `up` reuses it. + - name: Build the collector + run: | + docker buildx create --use --driver=docker-container + docker buildx bake -f docker-compose.ci.yml \ + --set 'otel-collector.tags=hdx-ci-otel-collector' \ + --set '*.cache-to=type=gha,mode=max,scope=dfe-documentdb' \ + --set '*.cache-from=type=gha,scope=dfe-documentdb' \ + --load + + - name: Build dependencies + run: yarn workspace @hyperdx/common-utils ci:build + + - name: Start ClickHouse, the collector and DocumentDB + run: | + docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --quiet-pull + docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --wait --wait-timeout 300 db + + # SHORTCUT: two upstream MCP assertions are skipped by name. They fail on + # MongoDB too: our lockfile carries @modelcontextprotocol/sdk 1.32.1 where + # upstream's has 1.29.0, and its validation message lacks the text they + # assert. Drop the pattern when either side moves. + - name: Run the api integration suite + env: + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + SKIP: + '^(?!.*(rejects a delta on a non-gauge metric|validate the + displayType allowlist)).*$' + run: + yarn workspace @hyperdx/api ci:int --shard="$SHARD" + --testNamePattern="$SKIP" + + - name: DocumentDB log on failure + if: failure() + run: + docker compose -f docker-compose.ci.yml -f + dfe/test/docker-compose.documentdb.yml logs --tail 200 db diff --git a/dfe/test/docker-compose.documentdb.yml b/dfe/test/docker-compose.documentdb.yml new file mode 100644 index 000000000..02efa1ecc --- /dev/null +++ b/dfe/test/docker-compose.documentdb.yml @@ -0,0 +1,33 @@ +# DocumentDB in place of MongoDB for the integration suite, layered over +# upstream's docker-compose.ci.yml so that file stays as upstream ships it. +# +# docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml \ +# up -d --wait ch-server db +# +# The gateway requires TLS and authenticates with SCRAM-SHA-256, as a deployed +# store does. Its certificate is self-signed, so the suite connects with +# tlsAllowInvalidCertificates=true - acceptable here and nowhere else. The +# connection string is in .github/workflows/dfe-documentdb.yml. + +services: + db: + # DocumentDB 1.0 on Postgres 18, the pairing DFE deploys. + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 + # Upstream's `--port 29999` is a mongod flag the DocumentDB entrypoint refuses. + command: !reset [] + environment: + USERNAME: hyperdx + PASSWORD: hyperdx-int + TLS_MODE: requireTLS + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + DOCUMENTDB_USAGE_TELEMETRY: 'false' + ports: !override + - '${HDX_CI_MONGO_PORT:-39999}:10260' + # The image's own probe, polled faster than its 30s default so `--wait` returns promptly. + healthcheck: + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] + interval: 2s + timeout: 10s + start_period: 120s + retries: 30 diff --git a/docker-compose.dfe.dev.yml b/docker-compose.dfe.dev.yml index 5e8627fe7..f7204b157 100644 --- a/docker-compose.dfe.dev.yml +++ b/docker-compose.dfe.dev.yml @@ -1,4 +1,4 @@ -# DFE Dev Override: FerretDB + PostgreSQL (no app — app runs locally) +# DFE Dev Override: DocumentDB (no app -- app runs locally) # # Usage (requires .env.dfe with POSTGRES_USER, POSTGRES_PASSWORD): # docker compose --env-file .env.dfe -f docker-compose.dev.yml -f docker-compose.dfe.dev.yml up -d @@ -6,6 +6,10 @@ # Use this with docker-compose.dev.yml. The app service is not defined in dev # (it runs locally via yarn dev), so this override omits the app section. # For production, use docker-compose.dfe.yml instead. +# +# The local app reaches the store with this in .env.dfe, the POSTGRES_ pair +# being the DocumentDB login created on its first start: +# MONGO_URI=mongodb://:@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 services: db: @@ -38,38 +42,36 @@ services: deploy: replicas: 0 - postgres: - image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0 + # The app runs on the host, which cannot read a CA held in a volume, so dev + # keeps the gateway's self-signed certificate and the MONGO_URI above sets + # tlsAllowInvalidCertificates. docker-compose.dfe.yml verifies the gateway. + documentdb: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: postgres + USERNAME: ${POSTGRES_USER} + PASSWORD: ${POSTGRES_PASSWORD} + TLS_MODE: requireTLS + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + # On by default upstream: it reports to a third party at start and hourly. + DOCUMENTDB_USAGE_TELEMETRY: 'false' + ports: + - '27017:10260' volumes: - - .volumes/pg_data:/var/lib/postgresql/data + - documentdb-data:/data networks: - internal restart: on-failure healthcheck: - # -d names the real database: without it libpq dials one named after the user. - test: pg_isready -U ${POSTGRES_USER} -d postgres + # The image's own probe, polled faster than its 30s default. + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] interval: 5s - timeout: 3s + timeout: 10s + start_period: 60s retries: 10 - ferretdb: - image: ghcr.io/ferretdb/ferretdb:2.7.0 - environment: - FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres - # Disable auth for local dev — allows createIndexes and other operations - FERRETDB_AUTH: 'false' - ports: - - '27017:27017' - networks: - - internal - depends_on: - postgres: - condition: service_healthy - restart: on-failure +volumes: + documentdb-data: networks: internal: diff --git a/docker-compose.dfe.yml b/docker-compose.dfe.yml index e3fc04f6d..9318a7828 100644 --- a/docker-compose.dfe.yml +++ b/docker-compose.dfe.yml @@ -1,4 +1,4 @@ -# DFE Override: Replace MongoDB with FerretDB + PostgreSQL (DocumentDB) +# DFE Override: Replace MongoDB with DocumentDB # # Usage (production): # docker compose -f docker-compose.yml -f docker-compose.dfe.yml up -d @@ -8,12 +8,14 @@ # # This override: # 1. Removes the upstream 'db' (mongo) service -# 2. Adds 'postgres' (PG 17 + DocumentDB extension) and 'ferretdb' services -# 3. Updates the 'app' service MONGO_URI to point at FerretDB -# 4. FerretDB exposes the MongoDB wire protocol on port 27017 +# 2. Adds 'documentdb' (PostgreSQL 18 with the DocumentDB extensions and its +# Mongo-wire gateway in one container) and 'documentdb-tls', which mints the +# CA and the certificate the gateway serves +# 3. Points the 'app' service MONGO_URI at the gateway, over TLS checked against +# that CA and with a SCRAM-SHA-256 login # -# The HyperDX application code is completely unchanged — Mongoose, connect-mongo, -# and passport-local-mongoose all work through FerretDB's MongoDB protocol proxy. +# The HyperDX application code is completely unchanged -- Mongoose, connect-mongo, +# and passport-local-mongoose all work through the DocumentDB gateway. services: # Disable upstream mongo service (empty profile prevents it from starting) @@ -21,47 +23,99 @@ services: profiles: - disabled - # PostgreSQL 17 with DocumentDB extension (backing store for FerretDB) - postgres: - image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0 + # Mints the CA and the gateway certificate once, into their own volume. The + # DocumentDB image is reused for its openssl. Root, because a fresh named volume + # is root-owned. The CA key is discarded once the certificate is signed. + documentdb-tls: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 + user: '0:0' + entrypoint: ['/bin/sh', '-c'] + command: + - | + set -eu + umask 077 + cd /tls + if [ ! -s ca/ca.crt ] || [ ! -s server/tls.crt ] || [ ! -s server/tls.key ]; then + rm -rf ca server + mkdir ca server + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 -days 3650 \ + -subj /CN=documentdb-ca \ + -addext basicConstraints=critical,CA:TRUE -addext keyUsage=critical,keyCertSign,cRLSign \ + -keyout ca.key -out ca/ca.crt + openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 \ + -subj /CN=documentdb -keyout server/tls.key -out server.csr + printf '%s\n' basicConstraints=critical,CA:FALSE keyUsage=critical,digitalSignature \ + extendedKeyUsage=serverAuth subjectAltName=DNS:documentdb > server.ext + openssl x509 -req -in server.csr -CA ca/ca.crt -CAkey ca.key -CAcreateserial -CAserial ca.srl \ + -days 3650 -sha384 -extfile server.ext -out server/tls.crt + rm -f ca.key ca.srl server.csr server.ext + fi + # The app reads the CA, and documentdb (uid 1000) the pair. + chmod 0755 ca + chmod 0644 ca/ca.crt + chown -R 1000:1000 server + volumes: + - documentdb-tls:/tls + network_mode: none + restart: 'no' + + documentdb: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: postgres + USERNAME: ${POSTGRES_USER} + PASSWORD: ${POSTGRES_PASSWORD} + # The gateway refuses plain connections and serves the minted certificate. + TLS_MODE: requireTLS + CERT_PATH: /etc/documentdb/tls/tls.crt + KEY_FILE: /etc/documentdb/tls/tls.key + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + # On by default upstream: it reports to a third party at start and hourly. + DOCUMENTDB_USAGE_TELEMETRY: 'false' volumes: - - .volumes/pg_data:/var/lib/postgresql/data + - documentdb-data:/data + - type: volume + source: documentdb-tls + target: /etc/documentdb/tls + read_only: true + volume: + subpath: server networks: - internal restart: on-failure + depends_on: + documentdb-tls: + condition: service_completed_successfully healthcheck: - # -d names the real database: without it libpq dials one named after the user. - test: pg_isready -U ${POSTGRES_USER} -d postgres + # The image's own probe, polled faster than its 30s default. + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] interval: 5s - timeout: 3s + timeout: 10s + start_period: 60s retries: 10 - # FerretDB: MongoDB wire protocol proxy → PostgreSQL - ferretdb: - image: ghcr.io/ferretdb/ferretdb:2.7.0 - environment: - FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres - ports: - - '27017:27017' - networks: - - internal - depends_on: - postgres: - condition: service_healthy - restart: on-failure - - # Override app service to use FerretDB instead of mongo + # Override app service to use DocumentDB instead of mongo app: environment: - MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ferretdb:27017/hyperdx?authMechanism=PLAIN' + MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@documentdb:10260/hyperdx?tls=true&tlsCAFile=/etc/dfe/documentdb-ca/ca.crt&authMechanism=SCRAM-SHA-256' + # The CA alone, never the gateway key beside it. + volumes: + - type: volume + source: documentdb-tls + target: /etc/dfe/documentdb-ca + read_only: true + volume: + subpath: ca # Replaces the base list: a merge would keep `db`, which the profile above removes. depends_on: !override - - ch-server - - ferretdb + ch-server: + condition: service_started + documentdb: + condition: service_healthy + +volumes: + documentdb-data: + documentdb-tls: networks: internal: diff --git a/docs/decisions/0001-ferretdb-over-direct-postgres.md b/docs/decisions/0001-ferretdb-over-direct-postgres.md index bb7c89943..f1a8a5d93 100644 --- a/docs/decisions/0001-ferretdb-over-direct-postgres.md +++ b/docs/decisions/0001-ferretdb-over-direct-postgres.md @@ -1,5 +1,7 @@ # FerretDB over a direct PostgreSQL migration +**Status:** superseded by [0006](0006-documentdb-replaces-ferretdb.md). + An assessment of the two approaches to removing the MongoDB dependency: using FerretDB as a transparent proxy vs. migrating the application code to use PostgreSQL directly. diff --git a/docs/decisions/0006-documentdb-replaces-ferretdb.md b/docs/decisions/0006-documentdb-replaces-ferretdb.md new file mode 100644 index 000000000..b96337752 --- /dev/null +++ b/docs/decisions/0006-documentdb-replaces-ferretdb.md @@ -0,0 +1,39 @@ +# 0006 - DocumentDB replaces FerretDB + +**Status:** accepted, 2026-10-22. Supersedes +[0001](0001-ferretdb-over-direct-postgres.md). + +## Context + +HyperDX keeps users, teams, dashboards and saved searches in a document store it +reaches through Mongoose. [0001](0001-ferretdb-over-direct-postgres.md) put +FerretDB in front of PostgreSQL for that. FerretDB is not a long-term option for +us, and MongoDB is licensed under the SSPL, which does not suit a product other +organisations deploy. + +DocumentDB 1.0 shipped on 2026-10-19 under MIT: PostgreSQL extensions plus a +Mongo wire-protocol gateway, in one container. Upstream is moving its own dev, +CI and e2e runs to it in hyperdxio/hyperdx#3375. + +## Decision + +DFE runs HyperDX on DocumentDB 1.0 on PostgreSQL 18. HyperDX connects over TLS +with a SCRAM-SHA-256 login. Application code is unchanged, as it was under +FerretDB. + +## Consequences + +- One container replaces FerretDB and its PostgreSQL. +- `.github/workflows/dfe-documentdb.yml` runs the api integration suite, our + `dfe/` tests included, on DocumentDB for every PR. +- DocumentDB fails the loser of two upserts racing to insert one unique key with + 11000 or 112, where MongoDB retries it. Our upserts go through + `retryUpsertRace` in `packages/api/src/dfe/models/duplicate-key.ts`. +- FerretDB data is not migrated, so a deployment moving over starts fresh. + +## Alternatives considered + +- Staying on FerretDB, which is not a long-term option. +- MongoDB, whose SSPL does not suit. +- A direct PostgreSQL rewrite, rejected in 0001 for its cost on every upstream + sync. That still holds. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f1603139a..9457a165f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -9,11 +9,12 @@ point - a rewritten ADR is worth nothing. | ADR | Decision | Status | | --- | --- | --- | -| [0001](0001-ferretdb-over-direct-postgres.md) | FerretDB over a direct PostgreSQL migration | accepted | +| [0001](0001-ferretdb-over-direct-postgres.md) | FerretDB over a direct PostgreSQL migration | superseded by [0006](0006-documentdb-replaces-ferretdb.md) | | [0002](0002-alerting-disabled-for-dfe-rules.md) | HyperDX alerting disabled in favour of DFE rules and hunts | accepted | | [0003](0003-additive-only-fork-strategy.md) | Additive-only fork strategy | accepted | | [0004](0004-casbin-removed.md) | Casbin RBAC removed as redundant | accepted | | [0005](0005-console-typography.md) | Console typography: Inter + IBM Plex Mono | accepted | +| [0006](0006-documentdb-replaces-ferretdb.md) | DocumentDB replaces FerretDB | accepted | ## Adding one diff --git a/packages/api/jest.dfe.int.config.js b/packages/api/jest.dfe.int.config.js index 8fb400706..4ca8f969f 100644 --- a/packages/api/jest.dfe.int.config.js +++ b/packages/api/jest.dfe.int.config.js @@ -27,4 +27,9 @@ if ( module.exports = { ...base, transformIgnorePatterns, + // After upstream's dotenv loader, so DFE_INT_MONGO_URI can name the store. + setupFiles: [ + ...(base.setupFiles ?? []), + '/dfe/__tests__/int-store.setup.ts', + ], }; diff --git a/packages/api/src/dfe/__tests__/dfe-sources.test.ts b/packages/api/src/dfe/__tests__/dfe-sources.test.ts index 05fb9460c..e946baece 100644 --- a/packages/api/src/dfe/__tests__/dfe-sources.test.ts +++ b/packages/api/src/dfe/__tests__/dfe-sources.test.ts @@ -36,6 +36,11 @@ jest.mock('@/models/team', () => ({ default: { find: jest.fn() }, })); +// The one-name-per-team index, which store-operations.int.test.ts covers for real. +jest.mock('@/models/source', () => ({ + Source: { collection: { createIndex: jest.fn().mockResolvedValue('ok') } }, +})); + jest.mock('@/dfe/models/dfe-source', () => ({ __esModule: true, default: { @@ -172,8 +177,8 @@ describe('upsertEverywhere', () => { await upsertEverywhere('filebeat', SPEC); expect(mockManifestUpsert).toHaveBeenCalledWith( - { name: 'filebeat' }, - { name: 'filebeat', spec: SPEC }, + { name: { $eq: 'filebeat' } }, + { $set: { name: 'filebeat', spec: SPEC } }, expect.objectContaining({ upsert: true }), ); expect(mockManifestUpsert.mock.invocationCallOrder[0]).toBeLessThan( @@ -216,7 +221,9 @@ describe('removeEverywhere', () => { test('drops the registration too, so a later team is not seeded with it', async () => { await removeEverywhere('filebeat'); - expect(mockManifestDelete).toHaveBeenCalledWith({ name: 'filebeat' }); + expect(mockManifestDelete).toHaveBeenCalledWith({ + name: { $eq: 'filebeat' }, + }); }); test('a team that never had it is left alone', async () => { @@ -367,7 +374,9 @@ describe('the /dfe/sources routes', () => { expect(res.status).toBe(200); expect(res.body.removed).toEqual(['dfe-admins', 'customer-acme', 'dfe']); - expect(mockManifestDelete).toHaveBeenCalledWith({ name: 'filebeat' }); + expect(mockManifestDelete).toHaveBeenCalledWith({ + name: { $eq: 'filebeat' }, + }); }); test('DELETE refuses a seeded name and removes nothing', async () => { diff --git a/packages/api/src/dfe/__tests__/duplicate-key.test.ts b/packages/api/src/dfe/__tests__/duplicate-key.test.ts new file mode 100644 index 000000000..30d290c1d --- /dev/null +++ b/packages/api/src/dfe/__tests__/duplicate-key.test.ts @@ -0,0 +1,55 @@ +/** + * retryUpsertRace: which failures it retries, and that it gives up. + */ +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; + +const storeError = (code: number) => + Object.assign(new Error(`store error ${code}`), { code }); + +describe('retryUpsertRace', () => { + test.each([11000, 112])( + 'retries a lost insert race (%i) and returns the next answer', + async code => { + const upsert = jest + .fn() + .mockRejectedValueOnce(storeError(code)) + .mockResolvedValueOnce('upserted'); + + await expect(retryUpsertRace(upsert)).resolves.toBe('upserted'); + expect(upsert).toHaveBeenCalledTimes(2); + }, + ); + + test('rethrows after four attempts that all lose', async () => { + const upsert = jest.fn().mockRejectedValue(storeError(11000)); + + await expect(retryUpsertRace(upsert)).rejects.toMatchObject({ + code: 11000, + }); + expect(upsert).toHaveBeenCalledTimes(4); + }); + + test('rethrows any other store error at once', async () => { + const upsert = jest.fn().mockRejectedValue(storeError(2)); + + await expect(retryUpsertRace(upsert)).rejects.toMatchObject({ code: 2 }); + expect(upsert).toHaveBeenCalledTimes(1); + }); + + test('rethrows an error with no code at once', async () => { + const upsert = jest.fn().mockRejectedValue(new Error('socket closed')); + + await expect(retryUpsertRace(upsert)).rejects.toThrow('socket closed'); + expect(upsert).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/dfe/__tests__/index-once.test.ts b/packages/api/src/dfe/__tests__/index-once.test.ts new file mode 100644 index 000000000..15c15e787 --- /dev/null +++ b/packages/api/src/dfe/__tests__/index-once.test.ts @@ -0,0 +1,56 @@ +/** + * indexOnce: one build for every caller, and a failed build tried again. + */ +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +import { indexOnce } from '@/dfe/models/index-once'; +import logger from '@/utils/logger'; + +describe('indexOnce', () => { + test('concurrent callers wait on one build', async () => { + let finish: () => void = () => {}; + const build = jest.fn( + () => new Promise(resolve => (finish = resolve)), + ); + const ensure = indexOnce(build, 'test index'); + + const settled: number[] = []; + const waiting = [ensure(), ensure(), ensure()].map((call, i) => + call.then(() => settled.push(i)), + ); + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toEqual([]); + + finish(); + await Promise.all(waiting); + await ensure(); + + expect(settled).toHaveLength(3); + expect(build).toHaveBeenCalledTimes(1); + }); + + test('a failed build is logged, resolves, and is tried again next call', async () => { + const build = jest + .fn() + .mockRejectedValueOnce(new Error('index build failed')) + .mockResolvedValueOnce('built'); + const ensure = indexOnce(build, 'test index'); + + await expect(ensure()).resolves.toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: expect.any(Error) }), + 'DFE: failed to ensure test index', + ); + await ensure(); + + expect(build).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/api/src/dfe/__tests__/int-store.setup.ts b/packages/api/src/dfe/__tests__/int-store.setup.ts new file mode 100644 index 000000000..e700f20c3 --- /dev/null +++ b/packages/api/src/dfe/__tests__/int-store.setup.ts @@ -0,0 +1,9 @@ +// Points the integration suite at the store DFE_INT_MONGO_URI names. It runs +// after dotenv loads .env.test with override on, which replaces a MONGO_URI set +// any earlier. +const storeUri = process.env.DFE_INT_MONGO_URI; +if (storeUri) { + process.env.MONGO_URI = storeUri; +} + +export {}; diff --git a/packages/api/src/dfe/__tests__/store-operations.int.test.ts b/packages/api/src/dfe/__tests__/store-operations.int.test.ts new file mode 100644 index 000000000..4a97a3d7b --- /dev/null +++ b/packages/api/src/dfe/__tests__/store-operations.int.test.ts @@ -0,0 +1,228 @@ +/** + * The store operations dfe/ relies on that no other integration test reaches, + * against a real store (`yarn ci:int`): a unique index raced by first logins, an + * upsert on a compound unique index, and a Mixed field replaced whole. CI runs + * this file on DocumentDB as well (.github/workflows/dfe-documentdb.yml). + */ + +jest.mock('@/dfe/config', () => ({ + __esModule: true, + ...jest.requireActual('@/dfe/config'), + DFE_AUTH_MODE: 'oidc-proxy', +})); + +import express from 'express'; +import mongoose from 'mongoose'; +import request from 'supertest'; + +import * as config from '@/config'; +import { + isOwnedName, + listByTeam, + removeEverywhere, + upsertEverywhere, +} from '@/dfe/controllers/dfe-sources'; +import { findOrCreateTeamByName } from '@/dfe/controllers/team-provisioning'; +import DfeDashboardTombstone, { + clearDashboardTombstones, + recordDashboardTombstone, + suppressedDashboardNames, +} from '@/dfe/models/dashboard-tombstone'; +import DfeSource from '@/dfe/models/dfe-source'; +import dfeSourcesRouter from '@/dfe/routers/dfe-sources'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; +import Team from '@/models/team'; + +// These tests lose races on purpose, and a retry would pass a race that was lost. +jest.retryTimes(0); + +// Rounds of two, because one round of a race can win by luck. +const RACE_ROUNDS = 10; + +describe('dfe store operations against a real store', () => { + beforeAll(async () => { + if (!config.IS_CI || config.MONGO_URI == null) { + throw new Error('needs NODE_ENV=test and a disposable MONGO_URI'); + } + await mongoose.connect(config.MONGO_URI); + }); + + afterEach(async () => { + await Promise.all( + Object.values(mongoose.connection.collections).map(collection => + collection.deleteMany({}), + ), + ); + }); + + afterAll(async () => { + await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + test('first logins racing on a new team name all land on one team', async () => { + const logins = await Promise.all( + Array.from({ length: 5 }, () => findOrCreateTeamByName('dfe_org_race')), + ); + + const teamIds = new Set(logins.map(login => String(login.team._id))); + expect(teamIds.size).toBe(1); + expect(logins.filter(login => login.created)).toHaveLength(1); + expect(await Team.countDocuments({ name: 'dfe_org_race' })).toBe(1); + }); + + test('a second team of an existing name is refused with a duplicate key', async () => { + await findOrCreateTeamByName('dfe_org_taken'); + + await expect(Team.create({ name: 'dfe_org_taken' })).rejects.toMatchObject({ + code: 11000, + }); + }); + + test('a tombstone is upserted once per team and name, and cleared per team', async () => { + const teamA = String(new mongoose.Types.ObjectId()); + const teamB = String(new mongoose.Types.ObjectId()); + + await recordDashboardTombstone('Overview', teamA); + const first = await DfeDashboardTombstone.findOne({ + name: 'Overview', + team: teamA, + }).lean(); + await recordDashboardTombstone('Overview', teamA); + await recordDashboardTombstone('Ingest', teamA); + await recordDashboardTombstone('Overview', teamB); + + expect(await DfeDashboardTombstone.countDocuments({ team: teamA })).toBe(2); + const again = await DfeDashboardTombstone.findOne({ + name: 'Overview', + team: teamA, + }).lean(); + expect(String(again?._id)).toBe(String(first?._id)); + expect(again?.deletedAt.getTime()).toBeGreaterThanOrEqual( + first?.deletedAt.getTime() ?? Infinity, + ); + expect(await suppressedDashboardNames(teamA)).toEqual( + new Set(['Overview', 'Ingest']), + ); + + expect(await clearDashboardTombstones(teamA)).toBe(2); + expect(await suppressedDashboardNames(teamA)).toEqual(new Set()); + expect(await suppressedDashboardNames(teamB)).toEqual( + new Set(['Overview']), + ); + }); + + test('the tombstone index refuses a second row for one team and name', async () => { + await DfeDashboardTombstone.init(); + const team = new mongoose.Types.ObjectId(); + await DfeDashboardTombstone.create({ + name: 'Overview', + team, + deletedAt: new Date(), + }); + + await expect( + DfeDashboardTombstone.create({ + name: 'Overview', + team, + deletedAt: new Date(), + }), + ).rejects.toMatchObject({ code: 11000 }); + }); + + // MongoDB retries the losing upsert itself, but DocumentDB fails it with 11000 or 112. + test('racing deletes of one shipped dashboard both succeed and leave one tombstone', async () => { + await DfeDashboardTombstone.init(); + + for (let round = 0; round < RACE_ROUNDS; round++) { + const team = String(new mongoose.Types.ObjectId()); + await Promise.all([ + recordDashboardTombstone('Overview', team), + recordDashboardTombstone('Overview', team), + ]); + expect(await DfeDashboardTombstone.countDocuments({ team })).toBe(1); + } + }); + + const sourcesApp = express(); + sourcesApp.use(express.json()); + sourcesApp.use('/dfe/sources', dfeSourcesRouter); + const putSource = (name: string) => + request(sourcesApp) + .put(`/dfe/sources/${name}`) + .send({ + kind: 'log', + from: { databaseName: 'dfe', tableName: name }, + timestampValueExpression: '_timestamp', + }); + + test('racing PUTs of one source both answer 200 and leave one manifest entry', async () => { + await DfeSource.init(); + + for (let round = 0; round < RACE_ROUNDS; round++) { + const name = `syslog_${round}`; + const answers = await Promise.all([putSource(name), putSource(name)]); + expect(answers.map(answer => answer.status)).toEqual([200, 200]); + expect(await DfeSource.countDocuments({ name })).toBe(1); + } + }); + + test('racing PUTs of one source onto a connected team leave that team one source', async () => { + await DfeSource.init(); + const team = await Team.create({ name: 'acme' }); + await Connection.create({ + team: team._id, + name: 'acme', + host: 'http://ch:8123', + username: 'dfe_org_acme', + password: 'pw', + }); + + for (let round = 0; round < RACE_ROUNDS; round++) { + const name = `firewall_${round}`; + const answers = await Promise.all([putSource(name), putSource(name)]); + expect(answers.map(answer => answer.status)).toEqual([200, 200]); + expect(answers.map(answer => answer.body.written)).toEqual([ + ['acme'], + ['acme'], + ]); + expect(await Source.countDocuments({ team: team._id, name })).toBe(1); + } + }); + + test('the source manifest holds a nested spec and a push replaces it whole', async () => { + const team = await Team.create({ name: 'acme' }); + const first = { + kind: 'log', + from: { databaseName: 'dfe', tableName: 'syslog' }, + timestampValueExpression: '_timestamp', + }; + const second = { + kind: 'log', + from: { databaseName: 'dfe', tableName: 'syslog_v2' }, + }; + + // acme holds no connection, so the source has nothing to hang on there. + expect(await upsertEverywhere('syslog', first)).toEqual({ + written: [], + skipped: ['acme'], + }); + expect((await DfeSource.findOne({ name: 'syslog' }).lean())?.spec).toEqual( + first, + ); + + await upsertEverywhere('syslog', second); + expect(await DfeSource.countDocuments({ name: 'syslog' })).toBe(1); + expect((await DfeSource.findOne({ name: 'syslog' }).lean())?.spec).toEqual( + second, + ); + expect(await isOwnedName('syslog')).toBe(true); + expect(await listByTeam()).toEqual([ + { team: String(team._id), teamName: 'acme', sources: [] }, + ]); + + expect(await removeEverywhere('syslog')).toEqual({ removed: [] }); + expect(await DfeSource.exists({ name: 'syslog' })).toBeNull(); + }); +}); diff --git a/packages/api/src/dfe/controllers/dfe-sources.ts b/packages/api/src/dfe/controllers/dfe-sources.ts index 54dcc97b6..f39784fdb 100644 --- a/packages/api/src/dfe/controllers/dfe-sources.ts +++ b/packages/api/src/dfe/controllers/dfe-sources.ts @@ -25,6 +25,9 @@ import { updateSource, } from '@/controllers/sources'; import DfeSource from '@/dfe/models/dfe-source'; +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; +import { indexOnce } from '@/dfe/models/index-once'; +import { Source } from '@/models/source'; import Team from '@/models/team'; import logger from '@/utils/logger'; @@ -58,6 +61,17 @@ async function sourceNamed(teamId: string, name: string) { return sources.find(source => source.name === name); } +// A team holds at most one source of a name. The index name is ours, so it +// never collides with one upstream adds, and the upstream model stays pristine. +const ensureOneSourceNamePerTeam = indexOnce( + () => + Source.collection.createIndex( + { team: 1, name: 1 }, + { unique: true, name: 'dfe_one_source_name_per_team' }, + ), + 'one source name per team', +); + async function connectionIdFor(teamId: string): Promise { const connections = await getConnectionsByTeam(teamId); const first = connections[0]; @@ -71,7 +85,7 @@ async function connectionIdFor(teamId: string): Promise { * landing view of every team. */ export async function isOwnedName(name: string): Promise { - if (await DfeSource.exists({ name })) { + if (await DfeSource.exists({ name: { $eq: name } })) { return true; } for (const team of await teamIds()) { @@ -99,14 +113,19 @@ export async function upsertOnTeam( return undefined; } - const existing = await sourceNamed(teamId, name); + await ensureOneSourceNamePerTeam(); const body = specForTeam(name, spec, connection); - if (existing) { - const updated = await updateSource(teamId, String(existing._id), body); - return updated ? String(existing._id) : undefined; - } - const created = await createSource(teamId, body); - return created ? String(created._id) : undefined; + // A racing write that creates the source first fails this create on the + // index, and the retry finds that source and replaces it. + return retryUpsertRace(async () => { + const existing = await sourceNamed(teamId, name); + if (existing) { + const updated = await updateSource(teamId, String(existing._id), body); + return updated ? String(existing._id) : undefined; + } + const created = await createSource(teamId, body); + return created ? String(created._id) : undefined; + }); } /** @@ -119,10 +138,12 @@ export async function upsertEverywhere( name: string, spec: DfeSourceSpec, ): Promise<{ written: string[]; skipped: string[] }> { - await DfeSource.findOneAndUpdate( - { name }, - { name, spec }, - { upsert: true, new: true }, + await retryUpsertRace(async () => + DfeSource.findOneAndUpdate( + { name: { $eq: name } }, + { $set: { name, spec } }, + { upsert: true, new: true }, + ), ); const written: string[] = []; @@ -146,7 +167,7 @@ export async function upsertEverywhere( export async function removeEverywhere( name: string, ): Promise<{ removed: string[] }> { - await DfeSource.deleteOne({ name }); + await DfeSource.deleteOne({ name: { $eq: name } }); const removed: string[] = []; for (const team of await teamIds()) { diff --git a/packages/api/src/dfe/controllers/team-provisioning.ts b/packages/api/src/dfe/controllers/team-provisioning.ts index 51c0c3c39..783e971bf 100644 --- a/packages/api/src/dfe/controllers/team-provisioning.ts +++ b/packages/api/src/dfe/controllers/team-provisioning.ts @@ -5,6 +5,7 @@ import * as dfeConfig from '@/dfe/config'; import { isDuplicateKey } from '@/dfe/models/duplicate-key'; +import { indexOnce } from '@/dfe/models/index-once'; import Team from '@/models/team'; import { setupTeamDefaults } from '@/setupDefaults'; import logger from '@/utils/logger'; @@ -13,20 +14,10 @@ import logger from '@/utils/logger'; // constraint is what makes find-or-create race-safe (a concurrent create // collides on it). Ensured from the dfe layer - idempotent createIndex - so the // upstream Team model stays pristine and off the fork conflict surface. -let nameIndexEnsured = false; - -async function ensureUniqueNameIndex(): Promise { - if (nameIndexEnsured) { - return; - } - nameIndexEnsured = true; - try { - await Team.collection.createIndex({ name: 1 }, { unique: true }); - } catch (err) { - nameIndexEnsured = false; - logger.warn({ err }, 'DFE: failed to ensure unique team-name index'); - } -} +const ensureUniqueNameIndex = indexOnce( + () => Team.collection.createIndex({ name: 1 }, { unique: true }), + 'unique team-name index', +); /** * Find an existing team by name, or create a new one. diff --git a/packages/api/src/dfe/models/dashboard-tombstone.ts b/packages/api/src/dfe/models/dashboard-tombstone.ts index 7d25e636b..d45e8c1f9 100644 --- a/packages/api/src/dfe/models/dashboard-tombstone.ts +++ b/packages/api/src/dfe/models/dashboard-tombstone.ts @@ -12,6 +12,7 @@ import mongoose, { Schema } from 'mongoose'; +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; import type { ObjectId } from '@/models'; export interface IDfeDashboardTombstone { @@ -52,10 +53,12 @@ export async function recordDashboardTombstone( name: string, teamId: string, ): Promise { - await DfeDashboardTombstone.findOneAndUpdate( - { name, team: teamId }, - { $set: { deletedAt: new Date() } }, - { upsert: true }, + await retryUpsertRace(async () => + DfeDashboardTombstone.findOneAndUpdate( + { name, team: teamId }, + { $set: { deletedAt: new Date() } }, + { upsert: true }, + ), ); } diff --git a/packages/api/src/dfe/models/duplicate-key.ts b/packages/api/src/dfe/models/duplicate-key.ts index ed0b70dd1..3c1140983 100644 --- a/packages/api/src/dfe/models/duplicate-key.ts +++ b/packages/api/src/dfe/models/duplicate-key.ts @@ -1,10 +1,16 @@ // MongoDB's duplicate-key error, raised when concurrent inserts collide on a -// unique index. FerretDB surfaces the same code. +// unique index. DocumentDB surfaces the same code. // // This is a NEW file - it does not modify any upstream HyperDX files. +import { withRetry } from '@/utils/retry'; + const DUPLICATE_KEY = 11000; +// DocumentDB's WriteConflict. It and 11000 are what the loser of two upserts +// inserting one key gets there, where MongoDB retries the loser itself. +const WRITE_CONFLICT = 112; + export function isDuplicateKey(err: unknown): boolean { return ( typeof err === 'object' && @@ -13,3 +19,17 @@ export function isDuplicateKey(err: unknown): boolean { err.code === DUPLICATE_KEY ); } + +/** + * Run an upsert on a unique key, retrying a lost insert race a few times with + * short jittered backoff before rethrowing. The retry finds the winner's + * document and updates it. + */ +export function retryUpsertRace(upsert: () => Promise): Promise { + return withRetry(upsert, { + maxRetries: 4, + initialDelayMs: 20, + maxDelayMs: 200, + retryOnlyOnStatus: [DUPLICATE_KEY, WRITE_CONFLICT], + }); +} diff --git a/packages/api/src/dfe/models/index-once.ts b/packages/api/src/dfe/models/index-once.ts new file mode 100644 index 000000000..c752644f2 --- /dev/null +++ b/packages/api/src/dfe/models/index-once.ts @@ -0,0 +1,27 @@ +// An index the dfe layer adds to an upstream collection, built once per process. +// +// This is a NEW file - it does not modify any upstream HyperDX files. + +import logger from '@/utils/logger'; + +/** + * Wrap an idempotent createIndex so every caller awaits one build, and none + * reads or writes before the index exists. A failed build is logged and tried + * again by the next call. + */ +export function indexOnce( + build: () => Promise, + what: string, +): () => Promise { + let built: Promise | undefined; + return () => { + built ??= build().then( + () => undefined, + err => { + built = undefined; + logger.warn({ err }, `DFE: failed to ensure ${what}`); + }, + ); + return built; + }; +} diff --git a/renovate.json b/renovate.json index ac9239778..6793e5f96 100644 --- a/renovate.json +++ b/renovate.json @@ -17,8 +17,8 @@ "not planned). includePaths restricts EXTRACTION instead, so there is no", "disabled dependency to mis-filter and no rule to regress.", "", - "WHAT IS ON. The GitHub Actions pinned by SHA in our own five workflows, the", - "two compose files we added, and the node base of the root Dockerfile that", + "WHAT IS ON. The GitHub Actions pinned by SHA in our own six workflows, the", + "three compose files we added, and the node base of the root Dockerfile that", "builds the released image. The Dockerfiles under docker/ and packages/ are", "upstream's and stay out. The org preset supplies the rest: the", "fix(deps): prefix so a bump actually ships, digest pinning, the 7-day", @@ -45,8 +45,10 @@ ".github/workflows/upstream-drift.yml", ".github/workflows/fork-surface.yml", ".github/workflows/fork-security.yml", + ".github/workflows/dfe-documentdb.yml", "docker-compose.dfe.yml", "docker-compose.dfe.dev.yml", + "dfe/test/docker-compose.documentdb.yml", "Dockerfile" ], "enabledManagers": ["github-actions", "docker-compose", "dockerfile"], @@ -62,7 +64,7 @@ } ], "vulnerabilityAlerts": { - "description": "On, but not the primary net. Nothing outside includePaths is extracted, so this only ever covers our own two compose files, five workflows and root Dockerfile. Upstream's tree is covered by Dependabot alerts plus security/overrides.yaml.", + "description": "On, but not the primary net. Nothing outside includePaths is extracted, so this only ever covers our own three compose files, six workflows and root Dockerfile. Upstream's tree is covered by Dependabot alerts plus security/overrides.yaml.", "enabled": true }, "osvVulnerabilityAlerts": true,