From 428d63dabc3181b44a90a35d33e7e72a49b02d33 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 15:35:20 +1100 Subject: [PATCH 01/11] test: run the api integration suite on DocumentDB DFE keeps HyperDX's documents in DocumentDB from 2.2.1, so the api integration suite, our dfe/ tests included, now runs against it on every PR in the new dfe-documentdb workflow. dfe/test/docker-compose.documentdb.yml layers over upstream's docker-compose.ci.yml and swaps its MongoDB for documentdb-local pg17-1.0.0, digest-pinned, requiring TLS and SCRAM-SHA-256. DFE_INT_MONGO_URI points the suite at it through a setup file jest.dfe.int.config.js runs after upstream's dotenv loader. No upstream file changes. store-operations.int.test.ts covers the dfe store calls no other integration test reached: the first-login race on the unique team name, tombstone upserts on their compound unique index and the source manifest's Mixed spec. Two racing upserts on one unique key fail the losing call on DocumentDB with 11000 or 112, where MongoDB retries it, and the test pins the one row either way. Two upstream MCP assertions are skipped by name in the workflow. They fail on MongoDB as well, against the @modelcontextprotocol/sdk 1.32.1 our lockfile carries. --- .github/workflows/dfe-documentdb.yml | 100 ++++++++++ dfe/test/docker-compose.documentdb.yml | 33 ++++ packages/api/jest.dfe.int.config.js | 5 + .../api/src/dfe/__tests__/int-store.setup.ts | 9 + .../__tests__/store-operations.int.test.ts | 176 ++++++++++++++++++ 5 files changed, 323 insertions(+) create mode 100644 .github/workflows/dfe-documentdb.yml create mode 100644 dfe/test/docker-compose.documentdb.yml create mode 100644 packages/api/src/dfe/__tests__/int-store.setup.ts create mode 100644 packages/api/src/dfe/__tests__/store-operations.int.test.ts diff --git a/.github/workflows/dfe-documentdb.yml b/.github/workflows/dfe-documentdb.yml new file mode 100644 index 00000000..465e9f57 --- /dev/null +++ b/.github/workflows/dfe-documentdb.yml @@ -0,0 +1,100 @@ +# Project: dfe-hyperdx +# File: .github/workflows/dfe-documentdb.yml +# Purpose: Run the api integration suite, dfe/ tests included, on DocumentDB. +# +# License: MIT - HYPERI PTY LIMITED +# Copyright: (c) 2026 HYPERI PTY LIMITED +# +# DFE keeps HyperDX's documents in DocumentDB, which speaks the MongoDB wire +# protocol over Postgres and implements a tested subset of MongoDB. Upstream's +# harness runs MongoDB, so this job layers dfe/test/docker-compose.documentdb.yml +# over docker-compose.ci.yml and leaves upstream's files as they ship. + +name: dfe-documentdb + +'on': + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: dfe-documentdb-${{ github.ref }} + cancel-in-progress: true + +env: + # TLS and SCRAM-SHA-256 as deployed. The harness certificate is self-signed, + # hence tlsAllowInvalidCertificates, which no deployment should carry. + DFE_INT_MONGO_URI: mongodb://hyperdx:hyperdx-int@localhost:39999/hyperdx-test?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 + +jobs: + integration: + name: Integration on DocumentDB (shard ${{ matrix.shard }}) + runs-on: ubuntu-24.04 + # The whole suite took 949s serially on DocumentDB, so it is split like upstream's. + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: '.nvmrc' + cache-dependency-path: 'yarn.lock' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --immutable + + # The bake step's GHA cache needs the runtime token in the shell. + - name: Expose GitHub runtime + uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + + # The collector creates the ClickHouse schema the suite waits for. It is + # built from source, so the layer cache saves the OCB compile on every run + # after the first. The tag is the name compose gives it, so `up` reuses it. + - name: Build the collector + run: | + docker buildx create --use --driver=docker-container + docker buildx bake -f docker-compose.ci.yml \ + --set 'otel-collector.tags=hdx-ci-otel-collector' \ + --set '*.cache-to=type=gha,mode=max,scope=dfe-documentdb' \ + --set '*.cache-from=type=gha,scope=dfe-documentdb' \ + --load + + - name: Build dependencies + run: yarn workspace @hyperdx/common-utils ci:build + + - name: Start ClickHouse, the collector and DocumentDB + run: | + docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --quiet-pull + docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml up -d --wait --wait-timeout 300 db + + # SHORTCUT: two upstream MCP assertions are skipped by name. They fail on + # MongoDB too: our lockfile carries @modelcontextprotocol/sdk 1.32.1 where + # upstream's has 1.29.0, and its validation message lacks the text they + # assert. Drop the pattern when either side moves. + - name: Run the api integration suite + env: + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + SKIP: + '^(?!.*(rejects a delta on a non-gauge metric|validate the + displayType allowlist)).*$' + run: + yarn workspace @hyperdx/api ci:int --shard="$SHARD" + --testNamePattern="$SKIP" + + - name: DocumentDB log on failure + if: failure() + run: + docker compose -f docker-compose.ci.yml -f + dfe/test/docker-compose.documentdb.yml logs --tail 200 db diff --git a/dfe/test/docker-compose.documentdb.yml b/dfe/test/docker-compose.documentdb.yml new file mode 100644 index 00000000..f295b633 --- /dev/null +++ b/dfe/test/docker-compose.documentdb.yml @@ -0,0 +1,33 @@ +# DocumentDB in place of MongoDB for the integration suite, layered over +# upstream's docker-compose.ci.yml so that file stays as upstream ships it. +# +# docker compose -f docker-compose.ci.yml -f dfe/test/docker-compose.documentdb.yml \ +# up -d --wait ch-server db +# +# The gateway requires TLS and authenticates with SCRAM-SHA-256, as a deployed +# store does. Its certificate is self-signed, so the suite connects with +# tlsAllowInvalidCertificates=true - acceptable here and nowhere else. The +# connection string is in .github/workflows/dfe-documentdb.yml. + +services: + db: + # pg17-1.0.0 is DocumentDB 1.0, the image the project's `latest` points at. + image: ghcr.io/documentdb/documentdb/documentdb-local:pg17-1.0.0@sha256:4a341daca29b02dabd0d64c357bc9231868b59ef08c8059058b1e5cedbe3899d + # Upstream's `--port 29999` is a mongod flag the DocumentDB entrypoint refuses. + command: !reset [] + environment: + USERNAME: hyperdx + PASSWORD: hyperdx-int + TLS_MODE: requireTLS + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + DOCUMENTDB_USAGE_TELEMETRY: 'false' + ports: !override + - '${HDX_CI_MONGO_PORT:-39999}:10260' + # The image's own probe, polled faster than its 30s default so `--wait` returns promptly. + healthcheck: + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] + interval: 2s + timeout: 10s + start_period: 120s + retries: 30 diff --git a/packages/api/jest.dfe.int.config.js b/packages/api/jest.dfe.int.config.js index 8fb40070..4ca8f969 100644 --- a/packages/api/jest.dfe.int.config.js +++ b/packages/api/jest.dfe.int.config.js @@ -27,4 +27,9 @@ if ( module.exports = { ...base, transformIgnorePatterns, + // After upstream's dotenv loader, so DFE_INT_MONGO_URI can name the store. + setupFiles: [ + ...(base.setupFiles ?? []), + '/dfe/__tests__/int-store.setup.ts', + ], }; diff --git a/packages/api/src/dfe/__tests__/int-store.setup.ts b/packages/api/src/dfe/__tests__/int-store.setup.ts new file mode 100644 index 00000000..e700f20c --- /dev/null +++ b/packages/api/src/dfe/__tests__/int-store.setup.ts @@ -0,0 +1,9 @@ +// Points the integration suite at the store DFE_INT_MONGO_URI names. It runs +// after dotenv loads .env.test with override on, which replaces a MONGO_URI set +// any earlier. +const storeUri = process.env.DFE_INT_MONGO_URI; +if (storeUri) { + process.env.MONGO_URI = storeUri; +} + +export {}; diff --git a/packages/api/src/dfe/__tests__/store-operations.int.test.ts b/packages/api/src/dfe/__tests__/store-operations.int.test.ts new file mode 100644 index 00000000..429bc3b1 --- /dev/null +++ b/packages/api/src/dfe/__tests__/store-operations.int.test.ts @@ -0,0 +1,176 @@ +/** + * The store operations dfe/ relies on that no other integration test reaches, + * against a real store (`yarn ci:int`): a unique index raced by first logins, an + * upsert on a compound unique index, and a Mixed field replaced whole. CI runs + * this file on DocumentDB as well (.github/workflows/dfe-documentdb.yml). + */ + +jest.mock('@/dfe/config', () => ({ + __esModule: true, + ...jest.requireActual('@/dfe/config'), + DFE_AUTH_MODE: 'oidc-proxy', +})); + +import mongoose from 'mongoose'; + +import * as config from '@/config'; +import { + isOwnedName, + listByTeam, + removeEverywhere, + upsertEverywhere, +} from '@/dfe/controllers/dfe-sources'; +import { findOrCreateTeamByName } from '@/dfe/controllers/team-provisioning'; +import DfeDashboardTombstone, { + clearDashboardTombstones, + recordDashboardTombstone, + suppressedDashboardNames, +} from '@/dfe/models/dashboard-tombstone'; +import DfeSource from '@/dfe/models/dfe-source'; +import Team from '@/models/team'; + +describe('dfe store operations against a real store', () => { + beforeAll(async () => { + if (!config.IS_CI || config.MONGO_URI == null) { + throw new Error('needs NODE_ENV=test and a disposable MONGO_URI'); + } + await mongoose.connect(config.MONGO_URI); + }); + + afterEach(async () => { + await Promise.all( + Object.values(mongoose.connection.collections).map(collection => + collection.deleteMany({}), + ), + ); + }); + + afterAll(async () => { + await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + test('first logins racing on a new team name all land on one team', async () => { + const logins = await Promise.all( + Array.from({ length: 5 }, () => findOrCreateTeamByName('dfe_org_race')), + ); + + const teamIds = new Set(logins.map(login => String(login.team._id))); + expect(teamIds.size).toBe(1); + expect(logins.filter(login => login.created)).toHaveLength(1); + expect(await Team.countDocuments({ name: 'dfe_org_race' })).toBe(1); + }); + + test('a second team of an existing name is refused with a duplicate key', async () => { + await findOrCreateTeamByName('dfe_org_taken'); + + await expect(Team.create({ name: 'dfe_org_taken' })).rejects.toMatchObject({ + code: 11000, + }); + }); + + test('a tombstone is upserted once per team and name, and cleared per team', async () => { + const teamA = String(new mongoose.Types.ObjectId()); + const teamB = String(new mongoose.Types.ObjectId()); + + await recordDashboardTombstone('Overview', teamA); + const first = await DfeDashboardTombstone.findOne({ + name: 'Overview', + team: teamA, + }).lean(); + await recordDashboardTombstone('Overview', teamA); + await recordDashboardTombstone('Ingest', teamA); + await recordDashboardTombstone('Overview', teamB); + + expect(await DfeDashboardTombstone.countDocuments({ team: teamA })).toBe(2); + const again = await DfeDashboardTombstone.findOne({ + name: 'Overview', + team: teamA, + }).lean(); + expect(String(again?._id)).toBe(String(first?._id)); + expect(again?.deletedAt.getTime()).toBeGreaterThanOrEqual( + first?.deletedAt.getTime() ?? Infinity, + ); + expect(await suppressedDashboardNames(teamA)).toEqual( + new Set(['Overview', 'Ingest']), + ); + + expect(await clearDashboardTombstones(teamA)).toBe(2); + expect(await suppressedDashboardNames(teamA)).toEqual(new Set()); + expect(await suppressedDashboardNames(teamB)).toEqual( + new Set(['Overview']), + ); + }); + + test('the tombstone index refuses a second row for one team and name', async () => { + await DfeDashboardTombstone.init(); + const team = new mongoose.Types.ObjectId(); + await DfeDashboardTombstone.create({ + name: 'Overview', + team, + deletedAt: new Date(), + }); + + await expect( + DfeDashboardTombstone.create({ + name: 'Overview', + team, + deletedAt: new Date(), + }), + ).rejects.toMatchObject({ code: 11000 }); + }); + + // MongoDB retries the losing upsert, but DocumentDB fails it with 11000 or 112. + test('two deletes of one shipped dashboard racing leave one tombstone', async () => { + await DfeDashboardTombstone.init(); + const team = String(new mongoose.Types.ObjectId()); + + const writes = await Promise.allSettled([ + recordDashboardTombstone('Overview', team), + recordDashboardTombstone('Overview', team), + ]); + + expect(writes.some(write => write.status === 'fulfilled')).toBe(true); + for (const write of writes) { + if (write.status === 'rejected') { + expect([11000, 112]).toContain(write.reason?.code); + } + } + expect(await DfeDashboardTombstone.countDocuments({ team })).toBe(1); + }); + + test('the source manifest holds a nested spec and a push replaces it whole', async () => { + const team = await Team.create({ name: 'acme' }); + const first = { + kind: 'log', + from: { databaseName: 'dfe', tableName: 'syslog' }, + timestampValueExpression: '_timestamp', + }; + const second = { + kind: 'log', + from: { databaseName: 'dfe', tableName: 'syslog_v2' }, + }; + + // acme holds no connection, so the source has nothing to hang on there. + expect(await upsertEverywhere('syslog', first)).toEqual({ + written: [], + skipped: ['acme'], + }); + expect((await DfeSource.findOne({ name: 'syslog' }).lean())?.spec).toEqual( + first, + ); + + await upsertEverywhere('syslog', second); + expect(await DfeSource.countDocuments({ name: 'syslog' })).toBe(1); + expect((await DfeSource.findOne({ name: 'syslog' }).lean())?.spec).toEqual( + second, + ); + expect(await isOwnedName('syslog')).toBe(true); + expect(await listByTeam()).toEqual([ + { team: String(team._id), teamName: 'acme', sources: [] }, + ]); + + expect(await removeEverywhere('syslog')).toEqual({ removed: [] }); + expect(await DfeSource.exists({ name: 'syslog' })).toBeNull(); + }); +}); From fb4a2f0a6a97ba05b161294bb8b54182019da5c5 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 16:37:23 +1100 Subject: [PATCH 02/11] fix: build the team-name index before any first login looks for its team ensureUniqueNameIndex marked the index ensured before createIndex returned, so concurrent first logins skipped it and found-then-created a team each. On a store with no index yet, which is every fresh deploy, five racing logins made four teams on DocumentDB in CI. Every caller now awaits one shared createIndex promise, and a failed build clears it so the next call tries again. --- .../src/dfe/controllers/team-provisioning.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/api/src/dfe/controllers/team-provisioning.ts b/packages/api/src/dfe/controllers/team-provisioning.ts index 51c0c3c3..462c8ba2 100644 --- a/packages/api/src/dfe/controllers/team-provisioning.ts +++ b/packages/api/src/dfe/controllers/team-provisioning.ts @@ -13,19 +13,17 @@ import logger from '@/utils/logger'; // constraint is what makes find-or-create race-safe (a concurrent create // collides on it). Ensured from the dfe layer - idempotent createIndex - so the // upstream Team model stays pristine and off the fork conflict surface. -let nameIndexEnsured = false; +let nameIndex: Promise | undefined; -async function ensureUniqueNameIndex(): Promise { - if (nameIndexEnsured) { - return; - } - nameIndexEnsured = true; - try { - await Team.collection.createIndex({ name: 1 }, { unique: true }); - } catch (err) { - nameIndexEnsured = false; - logger.warn({ err }, 'DFE: failed to ensure unique team-name index'); - } +function ensureUniqueNameIndex(): Promise { + nameIndex ??= Team.collection.createIndex({ name: 1 }, { unique: true }).then( + () => undefined, + err => { + nameIndex = undefined; + logger.warn({ err }, 'DFE: failed to ensure unique team-name index'); + }, + ); + return nameIndex; } /** From 5644dbb5751d0bb57127a4dc62e618f820132db9 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 16:37:38 +1100 Subject: [PATCH 03/11] fix: retry the losing upsert of an insert race on DocumentDB Two upserts racing to insert one unique key fail the loser on DocumentDB with 11000 or 112, where MongoDB retries it itself. retryUpsertRace in dfe/models/duplicate-key.ts retries exactly those two codes up to four attempts with short jittered backoff, then rethrows. recordDashboardTombstone and the DfeSource upsert behind PUT /dfe/sources/:name both go through it, so a concurrent PUT no longer answers 500. store-operations.int.test.ts races each caller ten rounds and runs with jest retries off, so a lost race fails the first attempt. --- .../src/dfe/__tests__/duplicate-key.test.ts | 55 +++++++++++++++++++ .../__tests__/store-operations.int.test.ts | 52 +++++++++++++----- .../api/src/dfe/controllers/dfe-sources.ts | 11 ++-- .../api/src/dfe/models/dashboard-tombstone.ts | 11 ++-- packages/api/src/dfe/models/duplicate-key.ts | 22 +++++++- 5 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 packages/api/src/dfe/__tests__/duplicate-key.test.ts diff --git a/packages/api/src/dfe/__tests__/duplicate-key.test.ts b/packages/api/src/dfe/__tests__/duplicate-key.test.ts new file mode 100644 index 00000000..30d290c1 --- /dev/null +++ b/packages/api/src/dfe/__tests__/duplicate-key.test.ts @@ -0,0 +1,55 @@ +/** + * retryUpsertRace: which failures it retries, and that it gives up. + */ +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; + +const storeError = (code: number) => + Object.assign(new Error(`store error ${code}`), { code }); + +describe('retryUpsertRace', () => { + test.each([11000, 112])( + 'retries a lost insert race (%i) and returns the next answer', + async code => { + const upsert = jest + .fn() + .mockRejectedValueOnce(storeError(code)) + .mockResolvedValueOnce('upserted'); + + await expect(retryUpsertRace(upsert)).resolves.toBe('upserted'); + expect(upsert).toHaveBeenCalledTimes(2); + }, + ); + + test('rethrows after four attempts that all lose', async () => { + const upsert = jest.fn().mockRejectedValue(storeError(11000)); + + await expect(retryUpsertRace(upsert)).rejects.toMatchObject({ + code: 11000, + }); + expect(upsert).toHaveBeenCalledTimes(4); + }); + + test('rethrows any other store error at once', async () => { + const upsert = jest.fn().mockRejectedValue(storeError(2)); + + await expect(retryUpsertRace(upsert)).rejects.toMatchObject({ code: 2 }); + expect(upsert).toHaveBeenCalledTimes(1); + }); + + test('rethrows an error with no code at once', async () => { + const upsert = jest.fn().mockRejectedValue(new Error('socket closed')); + + await expect(retryUpsertRace(upsert)).rejects.toThrow('socket closed'); + expect(upsert).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/dfe/__tests__/store-operations.int.test.ts b/packages/api/src/dfe/__tests__/store-operations.int.test.ts index 429bc3b1..4aee6fe6 100644 --- a/packages/api/src/dfe/__tests__/store-operations.int.test.ts +++ b/packages/api/src/dfe/__tests__/store-operations.int.test.ts @@ -11,7 +11,9 @@ jest.mock('@/dfe/config', () => ({ DFE_AUTH_MODE: 'oidc-proxy', })); +import express from 'express'; import mongoose from 'mongoose'; +import request from 'supertest'; import * as config from '@/config'; import { @@ -27,8 +29,15 @@ import DfeDashboardTombstone, { suppressedDashboardNames, } from '@/dfe/models/dashboard-tombstone'; import DfeSource from '@/dfe/models/dfe-source'; +import dfeSourcesRouter from '@/dfe/routers/dfe-sources'; import Team from '@/models/team'; +// These tests lose races on purpose, and a retry would pass a race that was lost. +jest.retryTimes(0); + +// Rounds of two, because one round of a race can win by luck. +const RACE_ROUNDS = 10; + describe('dfe store operations against a real store', () => { beforeAll(async () => { if (!config.IS_CI || config.MONGO_URI == null) { @@ -120,23 +129,40 @@ describe('dfe store operations against a real store', () => { ).rejects.toMatchObject({ code: 11000 }); }); - // MongoDB retries the losing upsert, but DocumentDB fails it with 11000 or 112. - test('two deletes of one shipped dashboard racing leave one tombstone', async () => { + // MongoDB retries the losing upsert itself, but DocumentDB fails it with 11000 or 112. + test('racing deletes of one shipped dashboard both succeed and leave one tombstone', async () => { await DfeDashboardTombstone.init(); - const team = String(new mongoose.Types.ObjectId()); - const writes = await Promise.allSettled([ - recordDashboardTombstone('Overview', team), - recordDashboardTombstone('Overview', team), - ]); + for (let round = 0; round < RACE_ROUNDS; round++) { + const team = String(new mongoose.Types.ObjectId()); + await Promise.all([ + recordDashboardTombstone('Overview', team), + recordDashboardTombstone('Overview', team), + ]); + expect(await DfeDashboardTombstone.countDocuments({ team })).toBe(1); + } + }); - expect(writes.some(write => write.status === 'fulfilled')).toBe(true); - for (const write of writes) { - if (write.status === 'rejected') { - expect([11000, 112]).toContain(write.reason?.code); - } + test('racing PUTs of one source both answer 200 and leave one manifest entry', async () => { + await DfeSource.init(); + const app = express(); + app.use(express.json()); + app.use('/dfe/sources', dfeSourcesRouter); + const put = (name: string) => + request(app) + .put(`/dfe/sources/${name}`) + .send({ + kind: 'log', + from: { databaseName: 'dfe', tableName: name }, + timestampValueExpression: '_timestamp', + }); + + for (let round = 0; round < RACE_ROUNDS; round++) { + const name = `syslog_${round}`; + const answers = await Promise.all([put(name), put(name)]); + expect(answers.map(answer => answer.status)).toEqual([200, 200]); + expect(await DfeSource.countDocuments({ name })).toBe(1); } - expect(await DfeDashboardTombstone.countDocuments({ team })).toBe(1); }); test('the source manifest holds a nested spec and a push replaces it whole', async () => { diff --git a/packages/api/src/dfe/controllers/dfe-sources.ts b/packages/api/src/dfe/controllers/dfe-sources.ts index 54dcc97b..e740b2ff 100644 --- a/packages/api/src/dfe/controllers/dfe-sources.ts +++ b/packages/api/src/dfe/controllers/dfe-sources.ts @@ -25,6 +25,7 @@ import { updateSource, } from '@/controllers/sources'; import DfeSource from '@/dfe/models/dfe-source'; +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; import Team from '@/models/team'; import logger from '@/utils/logger'; @@ -119,10 +120,12 @@ export async function upsertEverywhere( name: string, spec: DfeSourceSpec, ): Promise<{ written: string[]; skipped: string[] }> { - await DfeSource.findOneAndUpdate( - { name }, - { name, spec }, - { upsert: true, new: true }, + await retryUpsertRace(async () => + DfeSource.findOneAndUpdate( + { name }, + { name, spec }, + { upsert: true, new: true }, + ), ); const written: string[] = []; diff --git a/packages/api/src/dfe/models/dashboard-tombstone.ts b/packages/api/src/dfe/models/dashboard-tombstone.ts index 7d25e636..d45e8c1f 100644 --- a/packages/api/src/dfe/models/dashboard-tombstone.ts +++ b/packages/api/src/dfe/models/dashboard-tombstone.ts @@ -12,6 +12,7 @@ import mongoose, { Schema } from 'mongoose'; +import { retryUpsertRace } from '@/dfe/models/duplicate-key'; import type { ObjectId } from '@/models'; export interface IDfeDashboardTombstone { @@ -52,10 +53,12 @@ export async function recordDashboardTombstone( name: string, teamId: string, ): Promise { - await DfeDashboardTombstone.findOneAndUpdate( - { name, team: teamId }, - { $set: { deletedAt: new Date() } }, - { upsert: true }, + await retryUpsertRace(async () => + DfeDashboardTombstone.findOneAndUpdate( + { name, team: teamId }, + { $set: { deletedAt: new Date() } }, + { upsert: true }, + ), ); } diff --git a/packages/api/src/dfe/models/duplicate-key.ts b/packages/api/src/dfe/models/duplicate-key.ts index ed0b70dd..3c114098 100644 --- a/packages/api/src/dfe/models/duplicate-key.ts +++ b/packages/api/src/dfe/models/duplicate-key.ts @@ -1,10 +1,16 @@ // MongoDB's duplicate-key error, raised when concurrent inserts collide on a -// unique index. FerretDB surfaces the same code. +// unique index. DocumentDB surfaces the same code. // // This is a NEW file - it does not modify any upstream HyperDX files. +import { withRetry } from '@/utils/retry'; + const DUPLICATE_KEY = 11000; +// DocumentDB's WriteConflict. It and 11000 are what the loser of two upserts +// inserting one key gets there, where MongoDB retries the loser itself. +const WRITE_CONFLICT = 112; + export function isDuplicateKey(err: unknown): boolean { return ( typeof err === 'object' && @@ -13,3 +19,17 @@ export function isDuplicateKey(err: unknown): boolean { err.code === DUPLICATE_KEY ); } + +/** + * Run an upsert on a unique key, retrying a lost insert race a few times with + * short jittered backoff before rethrowing. The retry finds the winner's + * document and updates it. + */ +export function retryUpsertRace(upsert: () => Promise): Promise { + return withRetry(upsert, { + maxRetries: 4, + initialDelayMs: 20, + maxDelayMs: 200, + retryOnlyOnStatus: [DUPLICATE_KEY, WRITE_CONFLICT], + }); +} From ef283d5bec96aa7c00054d41f05ae5af454b80f1 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 16:37:48 +1100 Subject: [PATCH 04/11] test: run the DocumentDB harness on Postgres 18 The harness moves to documentdb-local pg18-1.0.0, the same 1.0 build on PostgreSQL 18.6 that dfe-infra and dfe-docker deploy, digest-pinned for amd64 and arm64. Renovate now scopes the harness compose file and the dfe-documentdb workflow, so the digest and the action pins stay current. --- dfe/test/docker-compose.documentdb.yml | 4 ++-- renovate.json | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dfe/test/docker-compose.documentdb.yml b/dfe/test/docker-compose.documentdb.yml index f295b633..02efa1ec 100644 --- a/dfe/test/docker-compose.documentdb.yml +++ b/dfe/test/docker-compose.documentdb.yml @@ -11,8 +11,8 @@ services: db: - # pg17-1.0.0 is DocumentDB 1.0, the image the project's `latest` points at. - image: ghcr.io/documentdb/documentdb/documentdb-local:pg17-1.0.0@sha256:4a341daca29b02dabd0d64c357bc9231868b59ef08c8059058b1e5cedbe3899d + # DocumentDB 1.0 on Postgres 18, the pairing DFE deploys. + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 # Upstream's `--port 29999` is a mongod flag the DocumentDB entrypoint refuses. command: !reset [] environment: diff --git a/renovate.json b/renovate.json index ac923977..6793e5f9 100644 --- a/renovate.json +++ b/renovate.json @@ -17,8 +17,8 @@ "not planned). includePaths restricts EXTRACTION instead, so there is no", "disabled dependency to mis-filter and no rule to regress.", "", - "WHAT IS ON. The GitHub Actions pinned by SHA in our own five workflows, the", - "two compose files we added, and the node base of the root Dockerfile that", + "WHAT IS ON. The GitHub Actions pinned by SHA in our own six workflows, the", + "three compose files we added, and the node base of the root Dockerfile that", "builds the released image. The Dockerfiles under docker/ and packages/ are", "upstream's and stay out. The org preset supplies the rest: the", "fix(deps): prefix so a bump actually ships, digest pinning, the 7-day", @@ -45,8 +45,10 @@ ".github/workflows/upstream-drift.yml", ".github/workflows/fork-surface.yml", ".github/workflows/fork-security.yml", + ".github/workflows/dfe-documentdb.yml", "docker-compose.dfe.yml", "docker-compose.dfe.dev.yml", + "dfe/test/docker-compose.documentdb.yml", "Dockerfile" ], "enabledManagers": ["github-actions", "docker-compose", "dockerfile"], @@ -62,7 +64,7 @@ } ], "vulnerabilityAlerts": { - "description": "On, but not the primary net. Nothing outside includePaths is extracted, so this only ever covers our own two compose files, five workflows and root Dockerfile. Upstream's tree is covered by Dependabot alerts plus security/overrides.yaml.", + "description": "On, but not the primary net. Nothing outside includePaths is extracted, so this only ever covers our own three compose files, six workflows and root Dockerfile. Upstream's tree is covered by Dependabot alerts plus security/overrides.yaml.", "enabled": true }, "osvVulnerabilityAlerts": true, From b3c9ad5d093b02d5a0f6374def1eb334baa3fb48 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 16:38:22 +1100 Subject: [PATCH 05/11] fix: run the production override on DocumentDB with a verified TLS login docker-compose.dfe.yml swaps FerretDB and its Postgres for documentdb-local pg18-1.0.0. A one-shot documentdb-tls mints a CA and the gateway certificate into a volume, the gateway refuses plain connections, and the app gets only the CA, verifying the gateway as documentdb and logging in with SCRAM-SHA-256. Usage telemetry is off. --- docker-compose.dfe.yml | 120 +++++++++++++++++++++++++++++------------ 1 file changed, 87 insertions(+), 33 deletions(-) diff --git a/docker-compose.dfe.yml b/docker-compose.dfe.yml index e3fc04f6..c9f8739a 100644 --- a/docker-compose.dfe.yml +++ b/docker-compose.dfe.yml @@ -1,4 +1,4 @@ -# DFE Override: Replace MongoDB with FerretDB + PostgreSQL (DocumentDB) +# DFE Override: Replace MongoDB with DocumentDB # # Usage (production): # docker compose -f docker-compose.yml -f docker-compose.dfe.yml up -d @@ -8,12 +8,14 @@ # # This override: # 1. Removes the upstream 'db' (mongo) service -# 2. Adds 'postgres' (PG 17 + DocumentDB extension) and 'ferretdb' services -# 3. Updates the 'app' service MONGO_URI to point at FerretDB -# 4. FerretDB exposes the MongoDB wire protocol on port 27017 +# 2. Adds 'documentdb' (PostgreSQL 18 with the DocumentDB extensions and its +# Mongo-wire gateway in one container) and 'documentdb-tls', which mints the +# CA and the certificate the gateway serves +# 3. Points the 'app' service MONGO_URI at the gateway, over TLS checked against +# that CA and with a SCRAM-SHA-256 login # # The HyperDX application code is completely unchanged — Mongoose, connect-mongo, -# and passport-local-mongoose all work through FerretDB's MongoDB protocol proxy. +# and passport-local-mongoose all work through the DocumentDB gateway. services: # Disable upstream mongo service (empty profile prevents it from starting) @@ -21,47 +23,99 @@ services: profiles: - disabled - # PostgreSQL 17 with DocumentDB extension (backing store for FerretDB) - postgres: - image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0 + # Mints the CA and the gateway certificate once, into their own volume. The + # DocumentDB image is reused for its openssl. Root, because a fresh named volume + # is root-owned. The CA key is discarded once the certificate is signed. + documentdb-tls: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 + user: '0:0' + entrypoint: ['/bin/sh', '-c'] + command: + - | + set -eu + umask 077 + cd /tls + if [ ! -s ca/ca.crt ] || [ ! -s server/tls.crt ] || [ ! -s server/tls.key ]; then + rm -rf ca server + mkdir ca server + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 -days 3650 \ + -subj /CN=documentdb-ca \ + -addext basicConstraints=critical,CA:TRUE -addext keyUsage=critical,keyCertSign,cRLSign \ + -keyout ca.key -out ca/ca.crt + openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-384 -nodes -sha384 \ + -subj /CN=documentdb -keyout server/tls.key -out server.csr + printf '%s\n' basicConstraints=critical,CA:FALSE keyUsage=critical,digitalSignature \ + extendedKeyUsage=serverAuth subjectAltName=DNS:documentdb > server.ext + openssl x509 -req -in server.csr -CA ca/ca.crt -CAkey ca.key -CAcreateserial -CAserial ca.srl \ + -days 3650 -sha384 -extfile server.ext -out server/tls.crt + rm -f ca.key ca.srl server.csr server.ext + fi + # The app reads the CA, and documentdb (uid 1000) the pair. + chmod 0755 ca + chmod 0644 ca/ca.crt + chown -R 1000:1000 server + volumes: + - documentdb-tls:/tls + network_mode: none + restart: 'no' + + documentdb: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: postgres + USERNAME: ${POSTGRES_USER} + PASSWORD: ${POSTGRES_PASSWORD} + # The gateway refuses plain connections and serves the minted certificate. + TLS_MODE: requireTLS + CERT_PATH: /etc/documentdb/tls/tls.crt + KEY_FILE: /etc/documentdb/tls/tls.key + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + # On by default upstream: it reports to a third party at start and hourly. + DOCUMENTDB_USAGE_TELEMETRY: 'false' volumes: - - .volumes/pg_data:/var/lib/postgresql/data + - documentdb-data:/data + - type: volume + source: documentdb-tls + target: /etc/documentdb/tls + read_only: true + volume: + subpath: server networks: - internal restart: on-failure + depends_on: + documentdb-tls: + condition: service_completed_successfully healthcheck: - # -d names the real database: without it libpq dials one named after the user. - test: pg_isready -U ${POSTGRES_USER} -d postgres + # The image's own probe, polled faster than its 30s default. + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] interval: 5s - timeout: 3s + timeout: 10s + start_period: 60s retries: 10 - # FerretDB: MongoDB wire protocol proxy → PostgreSQL - ferretdb: - image: ghcr.io/ferretdb/ferretdb:2.7.0 - environment: - FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres - ports: - - '27017:27017' - networks: - - internal - depends_on: - postgres: - condition: service_healthy - restart: on-failure - - # Override app service to use FerretDB instead of mongo + # Override app service to use DocumentDB instead of mongo app: environment: - MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ferretdb:27017/hyperdx?authMechanism=PLAIN' + MONGO_URI: 'mongodb://${POSTGRES_USER}:${POSTGRES_PASSWORD}@documentdb:10260/hyperdx?tls=true&tlsCAFile=/etc/dfe/documentdb-ca/ca.crt&authMechanism=SCRAM-SHA-256' + # The CA alone, never the gateway key beside it. + volumes: + - type: volume + source: documentdb-tls + target: /etc/dfe/documentdb-ca + read_only: true + volume: + subpath: ca # Replaces the base list: a merge would keep `db`, which the profile above removes. depends_on: !override - - ch-server - - ferretdb + ch-server: + condition: service_started + documentdb: + condition: service_healthy + +volumes: + documentdb-data: + documentdb-tls: networks: internal: From ccfa78b0dcc3c7068f2c43bdacd31e5e79b2fc46 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 17:02:37 +1100 Subject: [PATCH 06/11] fix: keep one source of a name per team when engine pushes race upsertOnTeam found and then created, so two concurrent PUTs of one source name each created it on a team with a connection: two sources, on MongoDB and DocumentDB alike. The dfe layer now builds a unique index on the upstream Source collection over {team, name}, named dfe_one_source_name_per_team, and the upstream model stays as it ships. The find-then-write goes through retryUpsertRace, so the loser's create fails on the index and its retry replaces the winner's source. indexOnce in dfe/models/index-once.ts builds such an index once for every caller, and the team-name index now uses it too. --- .../api/src/dfe/__tests__/dfe-sources.test.ts | 5 ++ .../api/src/dfe/__tests__/index-once.test.ts | 56 +++++++++++++++++++ .../__tests__/store-operations.int.test.ts | 50 +++++++++++++---- .../api/src/dfe/controllers/dfe-sources.ts | 32 ++++++++--- .../src/dfe/controllers/team-provisioning.ts | 17 ++---- packages/api/src/dfe/models/index-once.ts | 27 +++++++++ 6 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 packages/api/src/dfe/__tests__/index-once.test.ts create mode 100644 packages/api/src/dfe/models/index-once.ts diff --git a/packages/api/src/dfe/__tests__/dfe-sources.test.ts b/packages/api/src/dfe/__tests__/dfe-sources.test.ts index 05fb9460..0a03d485 100644 --- a/packages/api/src/dfe/__tests__/dfe-sources.test.ts +++ b/packages/api/src/dfe/__tests__/dfe-sources.test.ts @@ -36,6 +36,11 @@ jest.mock('@/models/team', () => ({ default: { find: jest.fn() }, })); +// The one-name-per-team index, which store-operations.int.test.ts covers for real. +jest.mock('@/models/source', () => ({ + Source: { collection: { createIndex: jest.fn().mockResolvedValue('ok') } }, +})); + jest.mock('@/dfe/models/dfe-source', () => ({ __esModule: true, default: { diff --git a/packages/api/src/dfe/__tests__/index-once.test.ts b/packages/api/src/dfe/__tests__/index-once.test.ts new file mode 100644 index 00000000..15c15e78 --- /dev/null +++ b/packages/api/src/dfe/__tests__/index-once.test.ts @@ -0,0 +1,56 @@ +/** + * indexOnce: one build for every caller, and a failed build tried again. + */ +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +import { indexOnce } from '@/dfe/models/index-once'; +import logger from '@/utils/logger'; + +describe('indexOnce', () => { + test('concurrent callers wait on one build', async () => { + let finish: () => void = () => {}; + const build = jest.fn( + () => new Promise(resolve => (finish = resolve)), + ); + const ensure = indexOnce(build, 'test index'); + + const settled: number[] = []; + const waiting = [ensure(), ensure(), ensure()].map((call, i) => + call.then(() => settled.push(i)), + ); + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toEqual([]); + + finish(); + await Promise.all(waiting); + await ensure(); + + expect(settled).toHaveLength(3); + expect(build).toHaveBeenCalledTimes(1); + }); + + test('a failed build is logged, resolves, and is tried again next call', async () => { + const build = jest + .fn() + .mockRejectedValueOnce(new Error('index build failed')) + .mockResolvedValueOnce('built'); + const ensure = indexOnce(build, 'test index'); + + await expect(ensure()).resolves.toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: expect.any(Error) }), + 'DFE: failed to ensure test index', + ); + await ensure(); + + expect(build).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/api/src/dfe/__tests__/store-operations.int.test.ts b/packages/api/src/dfe/__tests__/store-operations.int.test.ts index 4aee6fe6..4a97a3d7 100644 --- a/packages/api/src/dfe/__tests__/store-operations.int.test.ts +++ b/packages/api/src/dfe/__tests__/store-operations.int.test.ts @@ -30,6 +30,8 @@ import DfeDashboardTombstone, { } from '@/dfe/models/dashboard-tombstone'; import DfeSource from '@/dfe/models/dfe-source'; import dfeSourcesRouter from '@/dfe/routers/dfe-sources'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; import Team from '@/models/team'; // These tests lose races on purpose, and a retry would pass a race that was lost. @@ -143,28 +145,52 @@ describe('dfe store operations against a real store', () => { } }); + const sourcesApp = express(); + sourcesApp.use(express.json()); + sourcesApp.use('/dfe/sources', dfeSourcesRouter); + const putSource = (name: string) => + request(sourcesApp) + .put(`/dfe/sources/${name}`) + .send({ + kind: 'log', + from: { databaseName: 'dfe', tableName: name }, + timestampValueExpression: '_timestamp', + }); + test('racing PUTs of one source both answer 200 and leave one manifest entry', async () => { await DfeSource.init(); - const app = express(); - app.use(express.json()); - app.use('/dfe/sources', dfeSourcesRouter); - const put = (name: string) => - request(app) - .put(`/dfe/sources/${name}`) - .send({ - kind: 'log', - from: { databaseName: 'dfe', tableName: name }, - timestampValueExpression: '_timestamp', - }); for (let round = 0; round < RACE_ROUNDS; round++) { const name = `syslog_${round}`; - const answers = await Promise.all([put(name), put(name)]); + const answers = await Promise.all([putSource(name), putSource(name)]); expect(answers.map(answer => answer.status)).toEqual([200, 200]); expect(await DfeSource.countDocuments({ name })).toBe(1); } }); + test('racing PUTs of one source onto a connected team leave that team one source', async () => { + await DfeSource.init(); + const team = await Team.create({ name: 'acme' }); + await Connection.create({ + team: team._id, + name: 'acme', + host: 'http://ch:8123', + username: 'dfe_org_acme', + password: 'pw', + }); + + for (let round = 0; round < RACE_ROUNDS; round++) { + const name = `firewall_${round}`; + const answers = await Promise.all([putSource(name), putSource(name)]); + expect(answers.map(answer => answer.status)).toEqual([200, 200]); + expect(answers.map(answer => answer.body.written)).toEqual([ + ['acme'], + ['acme'], + ]); + expect(await Source.countDocuments({ team: team._id, name })).toBe(1); + } + }); + test('the source manifest holds a nested spec and a push replaces it whole', async () => { const team = await Team.create({ name: 'acme' }); const first = { diff --git a/packages/api/src/dfe/controllers/dfe-sources.ts b/packages/api/src/dfe/controllers/dfe-sources.ts index e740b2ff..e2586c19 100644 --- a/packages/api/src/dfe/controllers/dfe-sources.ts +++ b/packages/api/src/dfe/controllers/dfe-sources.ts @@ -26,6 +26,8 @@ import { } from '@/controllers/sources'; import DfeSource from '@/dfe/models/dfe-source'; import { retryUpsertRace } from '@/dfe/models/duplicate-key'; +import { indexOnce } from '@/dfe/models/index-once'; +import { Source } from '@/models/source'; import Team from '@/models/team'; import logger from '@/utils/logger'; @@ -59,6 +61,17 @@ async function sourceNamed(teamId: string, name: string) { return sources.find(source => source.name === name); } +// A team holds at most one source of a name. The index name is ours, so it +// never collides with one upstream adds, and the upstream model stays pristine. +const ensureOneSourceNamePerTeam = indexOnce( + () => + Source.collection.createIndex( + { team: 1, name: 1 }, + { unique: true, name: 'dfe_one_source_name_per_team' }, + ), + 'one source name per team', +); + async function connectionIdFor(teamId: string): Promise { const connections = await getConnectionsByTeam(teamId); const first = connections[0]; @@ -100,14 +113,19 @@ export async function upsertOnTeam( return undefined; } - const existing = await sourceNamed(teamId, name); + await ensureOneSourceNamePerTeam(); const body = specForTeam(name, spec, connection); - if (existing) { - const updated = await updateSource(teamId, String(existing._id), body); - return updated ? String(existing._id) : undefined; - } - const created = await createSource(teamId, body); - return created ? String(created._id) : undefined; + // A racing write that creates the source first fails this create on the + // index, and the retry finds that source and replaces it. + return retryUpsertRace(async () => { + const existing = await sourceNamed(teamId, name); + if (existing) { + const updated = await updateSource(teamId, String(existing._id), body); + return updated ? String(existing._id) : undefined; + } + const created = await createSource(teamId, body); + return created ? String(created._id) : undefined; + }); } /** diff --git a/packages/api/src/dfe/controllers/team-provisioning.ts b/packages/api/src/dfe/controllers/team-provisioning.ts index 462c8ba2..783e971b 100644 --- a/packages/api/src/dfe/controllers/team-provisioning.ts +++ b/packages/api/src/dfe/controllers/team-provisioning.ts @@ -5,6 +5,7 @@ import * as dfeConfig from '@/dfe/config'; import { isDuplicateKey } from '@/dfe/models/duplicate-key'; +import { indexOnce } from '@/dfe/models/index-once'; import Team from '@/models/team'; import { setupTeamDefaults } from '@/setupDefaults'; import logger from '@/utils/logger'; @@ -13,18 +14,10 @@ import logger from '@/utils/logger'; // constraint is what makes find-or-create race-safe (a concurrent create // collides on it). Ensured from the dfe layer - idempotent createIndex - so the // upstream Team model stays pristine and off the fork conflict surface. -let nameIndex: Promise | undefined; - -function ensureUniqueNameIndex(): Promise { - nameIndex ??= Team.collection.createIndex({ name: 1 }, { unique: true }).then( - () => undefined, - err => { - nameIndex = undefined; - logger.warn({ err }, 'DFE: failed to ensure unique team-name index'); - }, - ); - return nameIndex; -} +const ensureUniqueNameIndex = indexOnce( + () => Team.collection.createIndex({ name: 1 }, { unique: true }), + 'unique team-name index', +); /** * Find an existing team by name, or create a new one. diff --git a/packages/api/src/dfe/models/index-once.ts b/packages/api/src/dfe/models/index-once.ts new file mode 100644 index 00000000..c752644f --- /dev/null +++ b/packages/api/src/dfe/models/index-once.ts @@ -0,0 +1,27 @@ +// An index the dfe layer adds to an upstream collection, built once per process. +// +// This is a NEW file - it does not modify any upstream HyperDX files. + +import logger from '@/utils/logger'; + +/** + * Wrap an idempotent createIndex so every caller awaits one build, and none + * reads or writes before the index exists. A failed build is logged and tried + * again by the next call. + */ +export function indexOnce( + build: () => Promise, + what: string, +): () => Promise { + let built: Promise | undefined; + return () => { + built ??= build().then( + () => undefined, + err => { + built = undefined; + logger.warn({ err }, `DFE: failed to ensure ${what}`); + }, + ); + return built; + }; +} From ac39dcbdca9f21c7514b95e44fae21a2787735f3 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 17:03:24 +1100 Subject: [PATCH 07/11] fix: run the dev override on DocumentDB docker-compose.dfe.dev.yml swaps FerretDB and its Postgres for documentdb-local pg18-1.0.0 on host port 27017, requiring TLS with a SCRAM-SHA-256 login and with usage telemetry off. The app runs on the host, which cannot read a CA held in a volume, so dev keeps the gateway's self-signed certificate, and the file's header gives the dev MONGO_URI with tlsAllowInvalidCertificates. docker-compose.dfe.yml loses one non-ASCII dash. --- docker-compose.dfe.dev.yml | 50 ++++++++++++++++++++------------------ docker-compose.dfe.yml | 2 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docker-compose.dfe.dev.yml b/docker-compose.dfe.dev.yml index 5e8627fe..f7204b15 100644 --- a/docker-compose.dfe.dev.yml +++ b/docker-compose.dfe.dev.yml @@ -1,4 +1,4 @@ -# DFE Dev Override: FerretDB + PostgreSQL (no app — app runs locally) +# DFE Dev Override: DocumentDB (no app -- app runs locally) # # Usage (requires .env.dfe with POSTGRES_USER, POSTGRES_PASSWORD): # docker compose --env-file .env.dfe -f docker-compose.dev.yml -f docker-compose.dfe.dev.yml up -d @@ -6,6 +6,10 @@ # Use this with docker-compose.dev.yml. The app service is not defined in dev # (it runs locally via yarn dev), so this override omits the app section. # For production, use docker-compose.dfe.yml instead. +# +# The local app reaches the store with this in .env.dfe, the POSTGRES_ pair +# being the DocumentDB login created on its first start: +# MONGO_URI=mongodb://:@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 services: db: @@ -38,38 +42,36 @@ services: deploy: replicas: 0 - postgres: - image: ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0 + # The app runs on the host, which cannot read a CA held in a volume, so dev + # keeps the gateway's self-signed certificate and the MONGO_URI above sets + # tlsAllowInvalidCertificates. docker-compose.dfe.yml verifies the gateway. + documentdb: + image: ghcr.io/documentdb/documentdb/documentdb-local:pg18-1.0.0@sha256:03aef53ed72cd3007236f8ce1b4ef7a0c958dbbce1519074b85d8a3e10be89a5 environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: postgres + USERNAME: ${POSTGRES_USER} + PASSWORD: ${POSTGRES_PASSWORD} + TLS_MODE: requireTLS + # Off, the gateway listens on the container's loopback only. + ALLOW_EXTERNAL_CONNECTIONS: 'true' + # On by default upstream: it reports to a third party at start and hourly. + DOCUMENTDB_USAGE_TELEMETRY: 'false' + ports: + - '27017:10260' volumes: - - .volumes/pg_data:/var/lib/postgresql/data + - documentdb-data:/data networks: - internal restart: on-failure healthcheck: - # -d names the real database: without it libpq dials one named after the user. - test: pg_isready -U ${POSTGRES_USER} -d postgres + # The image's own probe, polled faster than its 30s default. + test: ['CMD', '/usr/local/bin/documentdb-healthcheck'] interval: 5s - timeout: 3s + timeout: 10s + start_period: 60s retries: 10 - ferretdb: - image: ghcr.io/ferretdb/ferretdb:2.7.0 - environment: - FERRETDB_POSTGRESQL_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres - # Disable auth for local dev — allows createIndexes and other operations - FERRETDB_AUTH: 'false' - ports: - - '27017:27017' - networks: - - internal - depends_on: - postgres: - condition: service_healthy - restart: on-failure +volumes: + documentdb-data: networks: internal: diff --git a/docker-compose.dfe.yml b/docker-compose.dfe.yml index c9f8739a..9318a782 100644 --- a/docker-compose.dfe.yml +++ b/docker-compose.dfe.yml @@ -14,7 +14,7 @@ # 3. Points the 'app' service MONGO_URI at the gateway, over TLS checked against # that CA and with a SCRAM-SHA-256 login # -# The HyperDX application code is completely unchanged — Mongoose, connect-mongo, +# The HyperDX application code is completely unchanged -- Mongoose, connect-mongo, # and passport-local-mongoose all work through the DocumentDB gateway. services: From 0524ebb42b897e7e55f85512235fcda771dd246d Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 17:03:37 +1100 Subject: [PATCH 08/11] docs: record DocumentDB replacing FerretDB in ADR 0006 0006 records DocumentDB 1.0 replacing FerretDB, following hyperdxio/hyperdx#3375. 0001 keeps its text and gains a status line pointing at it, and the index marks it superseded. --- .../0001-ferretdb-over-direct-postgres.md | 2 + .../0006-documentdb-replaces-ferretdb.md | 39 +++++++++++++++++++ docs/decisions/README.md | 3 +- 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/0006-documentdb-replaces-ferretdb.md diff --git a/docs/decisions/0001-ferretdb-over-direct-postgres.md b/docs/decisions/0001-ferretdb-over-direct-postgres.md index bb7c8994..f1a8a5d9 100644 --- a/docs/decisions/0001-ferretdb-over-direct-postgres.md +++ b/docs/decisions/0001-ferretdb-over-direct-postgres.md @@ -1,5 +1,7 @@ # FerretDB over a direct PostgreSQL migration +**Status:** superseded by [0006](0006-documentdb-replaces-ferretdb.md). + An assessment of the two approaches to removing the MongoDB dependency: using FerretDB as a transparent proxy vs. migrating the application code to use PostgreSQL directly. diff --git a/docs/decisions/0006-documentdb-replaces-ferretdb.md b/docs/decisions/0006-documentdb-replaces-ferretdb.md new file mode 100644 index 00000000..b9633775 --- /dev/null +++ b/docs/decisions/0006-documentdb-replaces-ferretdb.md @@ -0,0 +1,39 @@ +# 0006 - DocumentDB replaces FerretDB + +**Status:** accepted, 2026-10-22. Supersedes +[0001](0001-ferretdb-over-direct-postgres.md). + +## Context + +HyperDX keeps users, teams, dashboards and saved searches in a document store it +reaches through Mongoose. [0001](0001-ferretdb-over-direct-postgres.md) put +FerretDB in front of PostgreSQL for that. FerretDB is not a long-term option for +us, and MongoDB is licensed under the SSPL, which does not suit a product other +organisations deploy. + +DocumentDB 1.0 shipped on 2026-10-19 under MIT: PostgreSQL extensions plus a +Mongo wire-protocol gateway, in one container. Upstream is moving its own dev, +CI and e2e runs to it in hyperdxio/hyperdx#3375. + +## Decision + +DFE runs HyperDX on DocumentDB 1.0 on PostgreSQL 18. HyperDX connects over TLS +with a SCRAM-SHA-256 login. Application code is unchanged, as it was under +FerretDB. + +## Consequences + +- One container replaces FerretDB and its PostgreSQL. +- `.github/workflows/dfe-documentdb.yml` runs the api integration suite, our + `dfe/` tests included, on DocumentDB for every PR. +- DocumentDB fails the loser of two upserts racing to insert one unique key with + 11000 or 112, where MongoDB retries it. Our upserts go through + `retryUpsertRace` in `packages/api/src/dfe/models/duplicate-key.ts`. +- FerretDB data is not migrated, so a deployment moving over starts fresh. + +## Alternatives considered + +- Staying on FerretDB, which is not a long-term option. +- MongoDB, whose SSPL does not suit. +- A direct PostgreSQL rewrite, rejected in 0001 for its cost on every upstream + sync. That still holds. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f1603139..9457a165 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -9,11 +9,12 @@ point - a rewritten ADR is worth nothing. | ADR | Decision | Status | | --- | --- | --- | -| [0001](0001-ferretdb-over-direct-postgres.md) | FerretDB over a direct PostgreSQL migration | accepted | +| [0001](0001-ferretdb-over-direct-postgres.md) | FerretDB over a direct PostgreSQL migration | superseded by [0006](0006-documentdb-replaces-ferretdb.md) | | [0002](0002-alerting-disabled-for-dfe-rules.md) | HyperDX alerting disabled in favour of DFE rules and hunts | accepted | | [0003](0003-additive-only-fork-strategy.md) | Additive-only fork strategy | accepted | | [0004](0004-casbin-removed.md) | Casbin RBAC removed as redundant | accepted | | [0005](0005-console-typography.md) | Console typography: Inter + IBM Plex Mono | accepted | +| [0006](0006-documentdb-replaces-ferretdb.md) | DocumentDB replaces FerretDB | accepted | ## Adding one From a4ed3df2004fb088073e9906dbaca2b3bd837dd4 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 Oct 2026 22:13:38 +1100 Subject: [PATCH 09/11] fix: describe DocumentDB in the env examples --- .env.dfe-example | 8 ++++---- .env.dfe.example | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.env.dfe-example b/.env.dfe-example index e4e3c142..5796f466 100644 --- a/.env.dfe-example +++ b/.env.dfe-example @@ -5,14 +5,14 @@ # docker compose --env-file .env --env-file .env.dfe \ # -f docker-compose.yml -f docker-compose.dfe.yml up -d -# PostgreSQL / FerretDB credentials (used by docker-compose.dfe.yml) +# DocumentDB login (used by docker-compose.dfe.yml), created on its first start POSTGRES_USER=hyperdx POSTGRES_PASSWORD=hyperdx -# FerretDB connection (replaces MongoDB) +# DocumentDB connection (replaces MongoDB), TLS with a SCRAM-SHA-256 login # For the API running inside Docker (app service), this is set in docker-compose.dfe.yml -# For local development (API outside Docker), use: -# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?authMechanism=PLAIN +# For local development (API outside Docker, against docker-compose.dfe.dev.yml), use: +# MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 # DFE Auth Mode: 'oidc-proxy' enables OIDC via Envoy, '' or unset = upstream Passport.js DFE_AUTH_MODE=oidc-proxy diff --git a/.env.dfe.example b/.env.dfe.example index a274db5a..11c53082 100644 --- a/.env.dfe.example +++ b/.env.dfe.example @@ -1,10 +1,12 @@ # DFE dev overrides -- copy to .env.dfe (gitignored) and adjust. # Consumed by `yarn dev:dfe` (compose --env-file + dotenvx -f). -# FerretDB backing store (required by docker-compose.dfe.dev.yml). -# FERRETDB_AUTH is false in dev, so any matching pair works. +# DocumentDB login (required by docker-compose.dfe.dev.yml), created on its +# first start. The gateway requires TLS with a SCRAM-SHA-256 login, on a +# self-signed certificate that tlsAllowInvalidCertificates accepts in dev only. POSTGRES_USER=hyperdx POSTGRES_PASSWORD=hyperdx +MONGO_URI=mongodb://hyperdx:hyperdx@localhost:27017/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256 # DFE identity master switch: empty disables the /dfe/* identity layer, # `oidc-proxy` verifies engine JWTs, `header-dev` trusts dev headers. From 069a79562a9c8b65ff6ea673b0f440937ae180fd Mon Sep 17 00:00:00 2001 From: Derek Date: Fri, 23 Oct 2026 11:08:47 +1100 Subject: [PATCH 10/11] fix(security): match source names by value in the DFE sources store A request that sent an object for the source name, such as {"$ne": null}, made the exists, upsert and delete filters match any source. Each filter now compares with $eq. --- packages/api/src/dfe/controllers/dfe-sources.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/dfe/controllers/dfe-sources.ts b/packages/api/src/dfe/controllers/dfe-sources.ts index e2586c19..efc9d113 100644 --- a/packages/api/src/dfe/controllers/dfe-sources.ts +++ b/packages/api/src/dfe/controllers/dfe-sources.ts @@ -85,7 +85,7 @@ async function connectionIdFor(teamId: string): Promise { * landing view of every team. */ export async function isOwnedName(name: string): Promise { - if (await DfeSource.exists({ name })) { + if (await DfeSource.exists({ name: { $eq: name } })) { return true; } for (const team of await teamIds()) { @@ -140,7 +140,7 @@ export async function upsertEverywhere( ): Promise<{ written: string[]; skipped: string[] }> { await retryUpsertRace(async () => DfeSource.findOneAndUpdate( - { name }, + { name: { $eq: name } }, { name, spec }, { upsert: true, new: true }, ), @@ -167,7 +167,7 @@ export async function upsertEverywhere( export async function removeEverywhere( name: string, ): Promise<{ removed: string[] }> { - await DfeSource.deleteOne({ name }); + await DfeSource.deleteOne({ name: { $eq: name } }); const removed: string[] = []; for (const team of await teamIds()) { From 0ebc511a95c65891847d9116d9352fbb4d296789 Mon Sep 17 00:00:00 2001 From: Derek Date: Fri, 23 Oct 2026 11:24:03 +1100 Subject: [PATCH 11/11] fix(security): write the source manifest through $set The upsert's update document carried the caller's spec at the top level, so CodeQL still read it as operator injection. The update now goes through an explicit $set, and the unit tests expect the $eq filters. --- packages/api/src/dfe/__tests__/dfe-sources.test.ts | 12 ++++++++---- packages/api/src/dfe/controllers/dfe-sources.ts | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/api/src/dfe/__tests__/dfe-sources.test.ts b/packages/api/src/dfe/__tests__/dfe-sources.test.ts index 0a03d485..e946baec 100644 --- a/packages/api/src/dfe/__tests__/dfe-sources.test.ts +++ b/packages/api/src/dfe/__tests__/dfe-sources.test.ts @@ -177,8 +177,8 @@ describe('upsertEverywhere', () => { await upsertEverywhere('filebeat', SPEC); expect(mockManifestUpsert).toHaveBeenCalledWith( - { name: 'filebeat' }, - { name: 'filebeat', spec: SPEC }, + { name: { $eq: 'filebeat' } }, + { $set: { name: 'filebeat', spec: SPEC } }, expect.objectContaining({ upsert: true }), ); expect(mockManifestUpsert.mock.invocationCallOrder[0]).toBeLessThan( @@ -221,7 +221,9 @@ describe('removeEverywhere', () => { test('drops the registration too, so a later team is not seeded with it', async () => { await removeEverywhere('filebeat'); - expect(mockManifestDelete).toHaveBeenCalledWith({ name: 'filebeat' }); + expect(mockManifestDelete).toHaveBeenCalledWith({ + name: { $eq: 'filebeat' }, + }); }); test('a team that never had it is left alone', async () => { @@ -372,7 +374,9 @@ describe('the /dfe/sources routes', () => { expect(res.status).toBe(200); expect(res.body.removed).toEqual(['dfe-admins', 'customer-acme', 'dfe']); - expect(mockManifestDelete).toHaveBeenCalledWith({ name: 'filebeat' }); + expect(mockManifestDelete).toHaveBeenCalledWith({ + name: { $eq: 'filebeat' }, + }); }); test('DELETE refuses a seeded name and removes nothing', async () => { diff --git a/packages/api/src/dfe/controllers/dfe-sources.ts b/packages/api/src/dfe/controllers/dfe-sources.ts index efc9d113..f39784fd 100644 --- a/packages/api/src/dfe/controllers/dfe-sources.ts +++ b/packages/api/src/dfe/controllers/dfe-sources.ts @@ -141,7 +141,7 @@ export async function upsertEverywhere( await retryUpsertRace(async () => DfeSource.findOneAndUpdate( { name: { $eq: name } }, - { name, spec }, + { $set: { name, spec } }, { upsert: true, new: true }, ), );