From 96d4938ecaf673c543d0728c54f183137613ae2c Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 16:06:32 -0700 Subject: [PATCH 01/10] fix(api): recover the session store when MongoDB starts after the API connect-mongo's `mongoUrl` option opens its own MongoClient once at construction and caches the connect promise. If MongoDB is down when the API boots, every session read/write replays that MongoServerSelectionError and every request with a session cookie returns 500 until a restart, even after connectDBWithRetry has connected mongoose (#2966). Back the session store with the mongoose connection instead: build a MongoStore on mongoose's client once it is connected, rebuild it when mongoose replaces the client, and fail session operations fast until then. --- .changeset/session-store-mongo-retry.md | 7 + packages/api/src/api-app.ts | 4 +- .../src/utils/__tests__/sessionStore.test.ts | 154 ++++++++++++++++++ packages/api/src/utils/sessionStore.ts | 101 ++++++++++++ 4 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 .changeset/session-store-mongo-retry.md create mode 100644 packages/api/src/utils/__tests__/sessionStore.test.ts create mode 100644 packages/api/src/utils/sessionStore.ts diff --git a/.changeset/session-store-mongo-retry.md b/.changeset/session-store-mongo-retry.md new file mode 100644 index 0000000000..2b1e4f1467 --- /dev/null +++ b/.changeset/session-store-mongo-retry.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +fix: recover the session store when MongoDB comes up after the API. Sessions now +use the mongoose connection, so they share its startup retry instead of +returning 500 for every logged-in request until the API restarts diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index c2d9028a15..426eb5f751 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -1,5 +1,4 @@ import compression from 'compression'; -import MongoStore from 'connect-mongo'; import express from 'express'; import session from 'express-session'; import onHeaders from 'on-headers'; @@ -22,6 +21,7 @@ import usageStats from './tasks/usageStats'; import { registerAITelemetry } from './utils/aiTelemetry'; import logger, { expressLogger } from './utils/logger'; import passport from './utils/passport'; +import { MongooseSessionStore } from './utils/sessionStore'; registerAITelemetry(); @@ -42,7 +42,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = { maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days }, rolling: true, - store: new MongoStore({ mongoUrl: config.MONGO_URI }), + store: new MongooseSessionStore(), }; app.set('trust proxy', 1); diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts new file mode 100644 index 0000000000..83c8f361b3 --- /dev/null +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -0,0 +1,154 @@ +import session from 'express-session'; +import mongoose from 'mongoose'; + +// --- mocks (hoisted; names must be prefixed with `mock`) --- +type MockStore = { + client: unknown; + collectionP: Promise; + get: jest.Mock; + set: jest.Mock; + destroy: jest.Mock; + touch: jest.Mock; +}; +const mockStores: MockStore[] = []; +let mockCollectionP: () => Promise = () => Promise.resolve({}); +const mockConnection: { readyState: number; client: unknown } = { + readyState: 0, + client: undefined, +}; + +jest.mock('connect-mongo', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(({ client }) => { + const store: MockStore = { + client, + collectionP: mockCollectionP(), + get: jest.fn((_sid, cb) => cb(null, { client })), + set: jest.fn((_sid, _sess, cb) => cb?.()), + destroy: jest.fn((_sid, cb) => cb?.()), + touch: jest.fn((_sid, _sess, cb) => cb?.()), + }; + mockStores.push(store); + return store; + }), +})); +jest.mock('@/models', () => ({ + mongooseConnection: { + get readyState() { + return mockConnection.readyState; + }, + getClient: () => mockConnection.client, + }, +})); + +import { MongooseSessionStore } from '@/utils/sessionStore'; + +const { connected, connecting, disconnected } = mongoose.ConnectionStates; + +const getSession = (store: MongooseSessionStore, sid = 'sid') => + new Promise<{ err: any; session: any }>(resolve => + store.get(sid, (err, session) => resolve({ err, session })), + ); + +const connect = (client: unknown) => { + mockConnection.client = client; + mockConnection.readyState = connected; +}; + +describe('MongooseSessionStore', () => { + beforeEach(() => { + mockStores.length = 0; + mockCollectionP = () => Promise.resolve({}); + mockConnection.readyState = disconnected; + mockConnection.client = undefined; + }); + + it('fails session operations fast until mongoose connects', async () => { + const store = new MongooseSessionStore(); + + const { err } = await getSession(store); + + expect(err).toEqual(new Error('MongoDB is not connected')); + expect(mockStores).toHaveLength(0); + }); + + it('recovers once mongoose connects after a failed start, without a restart', async () => { + const store = new MongooseSessionStore(); + // connectDBWithRetry's first attempt left a client that never connected. + mockConnection.client = { attempt: 1 }; + mockConnection.readyState = connecting; + expect((await getSession(store)).err).toBeDefined(); + + const client = { attempt: 2 }; + connect(client); + + expect(await getSession(store)).toEqual({ + err: null, + session: { client }, + }); + expect(mockStores).toHaveLength(1); + expect(mockStores[0].client).toBe(client); + }); + + it('delegates set, destroy and touch to the store on the connected client', async () => { + connect({}); + const store = new MongooseSessionStore(); + const sess = { cookie: new session.Cookie() }; + const set = jest.fn(); + const destroy = jest.fn(); + const touch = jest.fn(); + + store.set('sid', sess, set); + store.destroy('sid', destroy); + store.touch('sid', sess, touch); + + expect(mockStores).toHaveLength(1); + expect(mockStores[0].set).toHaveBeenCalledWith('sid', sess, set); + expect(mockStores[0].destroy).toHaveBeenCalledWith('sid', destroy); + expect(mockStores[0].touch).toHaveBeenCalledWith('sid', sess, touch); + expect(set).toHaveBeenCalled(); + expect(destroy).toHaveBeenCalled(); + expect(touch).toHaveBeenCalled(); + }); + + it('keeps the store through a transient disconnect of the same client', async () => { + connect({}); + const store = new MongooseSessionStore(); + await getSession(store); + + mockConnection.readyState = disconnected; + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(1); + }); + + it('rebuilds the store when mongoose reconnects with a new client', async () => { + connect({ id: 'a' }); + const store = new MongooseSessionStore(); + await getSession(store); + + // A closed client can't be reused; the reconnect makes a new one. + const client = { id: 'b' }; + mockConnection.client = client; + mockConnection.readyState = connecting; + expect((await getSession(store)).err).toBeDefined(); + + connect(client); + + expect((await getSession(store)).session).toEqual({ client }); + expect(mockStores).toHaveLength(2); + }); + + it('replaces a store whose collection setup failed', async () => { + mockCollectionP = () => Promise.reject(new Error('createIndex failed')); + connect({}); + const store = new MongooseSessionStore(); + await getSession(store); + await new Promise(setImmediate); + + mockCollectionP = () => Promise.resolve({}); + await getSession(store); + + expect(mockStores).toHaveLength(2); + }); +}); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts new file mode 100644 index 0000000000..f3eb7b3a72 --- /dev/null +++ b/packages/api/src/utils/sessionStore.ts @@ -0,0 +1,101 @@ +import MongoStore from 'connect-mongo'; +import session from 'express-session'; +import mongoose from 'mongoose'; + +import { mongooseConnection } from '@/models'; + +type MongoClient = ReturnType; +type ConnectMongoClient = NonNullable< + ConstructorParameters[0]['client'] +>; + +/** + * express-session store that keeps sessions in MongoDB over the app's + * mongoose connection. + * + * `new MongoStore({ mongoUrl })` opens its own MongoClient once, at + * construction, and caches that connect promise. If MongoDB is unreachable + * when the API starts, every session read/write replays the original + * MongoServerSelectionError (so every request with a session cookie 500s) + * until the process restarts, even after connectDBWithRetry has connected + * mongoose (see https://github.com/hyperdxio/hyperdx/issues/2966). + * + * Instead, build a MongoStore on mongoose's client once mongoose is connected, + * and rebuild it whenever mongoose replaces that client: every + * mongoose.connect() creates a new one, and a closed client can't be reused. + * Until mongoose connects, session operations fail fast. + */ +export class MongooseSessionStore extends session.Store { + private store?: MongoStore; + private storeClient?: MongoClient; + + private getStore(): MongoStore | undefined { + const client = mongooseConnection.getClient(); + // Keep using the store through a transient disconnect of the same client; + // the driver reconnects it on its own. + if (this.store != null && this.storeClient === client) { + return this.store; + } + if (mongooseConnection.readyState !== mongoose.ConnectionStates.connected) { + return undefined; + } + // mongoose bundles its own mongodb driver (4.x), so its MongoClient type + // differs from the hoisted 6.x one connect-mongo is typed against. 4.x is + // connect-mongo's declared peer and covers the collection CRUD it uses. + const store = new MongoStore({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + client: client as unknown as ConnectMongoClient, + }); + // A store whose collection setup failed would replay that error forever, + // so drop it and let the next session operation build a fresh one. + store.collectionP.catch(() => { + if (this.store === store) { + this.store = undefined; + } + }); + this.store = store; + this.storeClient = client; + return store; + } + + private withStore( + callback: ((err?: unknown) => void) | undefined, + op: (store: MongoStore) => void, + ) { + const store = this.getStore(); + if (store == null) { + callback?.(new Error('MongoDB is not connected')); + return; + } + op(store); + } + + get( + sid: string, + callback: (err: any, session?: session.SessionData | null) => void, + ) { + this.withStore(callback, store => store.get(sid, callback)); + } + + set( + sid: string, + sess: session.SessionData, + callback?: (err?: unknown) => void, + ) { + this.withStore(callback, store => store.set(sid, sess, callback)); + } + + destroy(sid: string, callback?: (err?: unknown) => void) { + this.withStore(callback, store => store.destroy(sid, callback)); + } + + touch( + sid: string, + sess: session.SessionData, + callback?: (err?: unknown) => void, + ) { + this.withStore(callback, store => + store.touch(sid, sess as Parameters[1], callback), + ); + } +} From 9222f4c0c0b9b4040d22c55a12bae39c9912667d Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 16:40:43 -0700 Subject: [PATCH 02/10] refactor(api): address session store review feedback Use the shared isMongoConnected() readiness check so sessions and /ready agree on what "connected" means, drop a banner comment, and type the test helper's result instead of using any. --- packages/api/src/utils/__tests__/sessionStore.test.ts | 6 ++++-- packages/api/src/utils/sessionStore.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index 83c8f361b3..d2a8d5971b 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -1,7 +1,7 @@ import session from 'express-session'; import mongoose from 'mongoose'; -// --- mocks (hoisted; names must be prefixed with `mock`) --- +// jest.mock factories are hoisted, so the state they use must be named `mock*`. type MockStore = { client: unknown; collectionP: Promise; @@ -45,8 +45,10 @@ import { MongooseSessionStore } from '@/utils/sessionStore'; const { connected, connecting, disconnected } = mongoose.ConnectionStates; +type GetResult = { err: unknown; session?: session.SessionData | null }; + const getSession = (store: MongooseSessionStore, sid = 'sid') => - new Promise<{ err: any; session: any }>(resolve => + new Promise(resolve => store.get(sid, (err, session) => resolve({ err, session })), ); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index f3eb7b3a72..ae1dbd379c 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -1,8 +1,8 @@ import MongoStore from 'connect-mongo'; import session from 'express-session'; -import mongoose from 'mongoose'; import { mongooseConnection } from '@/models'; +import { isMongoConnected } from '@/utils/readiness'; type MongoClient = ReturnType; type ConnectMongoClient = NonNullable< @@ -36,7 +36,7 @@ export class MongooseSessionStore extends session.Store { if (this.store != null && this.storeClient === client) { return this.store; } - if (mongooseConnection.readyState !== mongoose.ConnectionStates.connected) { + if (!isMongoConnected()) { return undefined; } // mongoose bundles its own mongodb driver (4.x), so its MongoClient type From 6cc4d5479e871084f140a36e81644277011d42aa Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 16:50:12 -0700 Subject: [PATCH 03/10] chore(api): account for the session store's driver-type eslint-disable The mongoose (mongodb 4.x) to connect-mongo (mongodb 6.x types) client cast needs one eslint-disable; without it the warning pushes the api package over its --max-warnings ceiling. Raise the api eslint-disable ratchet baseline from 36 to 37 to match. --- scripts/ci/ratchet-baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/ratchet-baseline.json b/scripts/ci/ratchet-baseline.json index 4d7e1f1fba..a243584b2b 100644 --- a/scripts/ci/ratchet-baseline.json +++ b/scripts/ci/ratchet-baseline.json @@ -2,7 +2,7 @@ "api": { "as-any": 89, "ts-ignore": 0, - "eslint-disable": 36, + "eslint-disable": 37, "@source": 0 }, "app": { From 693c5934c61cdf922527d95f30bde4bb0cc91b51 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 16:54:16 -0700 Subject: [PATCH 04/10] refactor(api): type the session store get callback error as unknown --- packages/api/src/utils/sessionStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index ae1dbd379c..e371b721de 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -72,7 +72,7 @@ export class MongooseSessionStore extends session.Store { get( sid: string, - callback: (err: any, session?: session.SessionData | null) => void, + callback: (err: unknown, session?: session.SessionData | null) => void, ) { this.withStore(callback, store => store.get(sid, callback)); } From 440238e7708134b5f5f3b1274f2dea7a66aced5d Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 16:59:48 -0700 Subject: [PATCH 05/10] Revert "chore(api): account for the session store's driver-type eslint-disable" This reverts commit 6cc4d5479e871084f140a36e81644277011d42aa. --- scripts/ci/ratchet-baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/ratchet-baseline.json b/scripts/ci/ratchet-baseline.json index a243584b2b..4d7e1f1fba 100644 --- a/scripts/ci/ratchet-baseline.json +++ b/scripts/ci/ratchet-baseline.json @@ -2,7 +2,7 @@ "api": { "as-any": 89, "ts-ignore": 0, - "eslint-disable": 37, + "eslint-disable": 36, "@source": 0 }, "app": { From 84e498318e39a5e41f1b7455d492497d455decbd Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 17:02:05 -0700 Subject: [PATCH 06/10] refactor(api): create the session store once MongoDB is reachable Handing mongoose's client to connect-mongo needed a cast between mongoose's bundled mongodb 4.x types and the 6.x types connect-mongo is typed against. Instead, keep connect-mongo's own `mongoUrl` client, but only create the store once mongoose is connected and create a new one if its connect or setup fails. This drops the cast and its eslint-disable, and connect-mongo stays on the driver it already used. --- .changeset/session-store-mongo-retry.md | 4 +- packages/api/src/api-app.ts | 4 +- .../src/utils/__tests__/sessionStore.test.ts | 96 ++++++------------- packages/api/src/utils/sessionStore.ts | 48 +++------- 4 files changed, 50 insertions(+), 102 deletions(-) diff --git a/.changeset/session-store-mongo-retry.md b/.changeset/session-store-mongo-retry.md index 2b1e4f1467..a356e7157b 100644 --- a/.changeset/session-store-mongo-retry.md +++ b/.changeset/session-store-mongo-retry.md @@ -2,6 +2,6 @@ '@hyperdx/api': patch --- -fix: recover the session store when MongoDB comes up after the API. Sessions now -use the mongoose connection, so they share its startup retry instead of +fix: recover the session store when MongoDB comes up after the API. The store +now connects once MongoDB is reachable and retries a failed connect, instead of returning 500 for every logged-in request until the API restarts diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index 426eb5f751..d30049717b 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -21,7 +21,7 @@ import usageStats from './tasks/usageStats'; import { registerAITelemetry } from './utils/aiTelemetry'; import logger, { expressLogger } from './utils/logger'; import passport from './utils/passport'; -import { MongooseSessionStore } from './utils/sessionStore'; +import { MongoSessionStore } from './utils/sessionStore'; registerAITelemetry(); @@ -42,7 +42,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = { maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days }, rolling: true, - store: new MongooseSessionStore(), + store: new MongoSessionStore(), }; app.set('trust proxy', 1); diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index d2a8d5971b..94e621ffb2 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -1,9 +1,8 @@ import session from 'express-session'; -import mongoose from 'mongoose'; // jest.mock factories are hoisted, so the state they use must be named `mock*`. type MockStore = { - client: unknown; + mongoUrl: string; collectionP: Promise; get: jest.Mock; set: jest.Mock; @@ -12,18 +11,15 @@ type MockStore = { }; const mockStores: MockStore[] = []; let mockCollectionP: () => Promise = () => Promise.resolve({}); -const mockConnection: { readyState: number; client: unknown } = { - readyState: 0, - client: undefined, -}; +let mockMongoConnected = false; jest.mock('connect-mongo', () => ({ __esModule: true, - default: jest.fn().mockImplementation(({ client }) => { + default: jest.fn().mockImplementation(({ mongoUrl }) => { const store: MockStore = { - client, + mongoUrl, collectionP: mockCollectionP(), - get: jest.fn((_sid, cb) => cb(null, { client })), + get: jest.fn((_sid, cb) => cb(null, { mongoUrl })), set: jest.fn((_sid, _sess, cb) => cb?.()), destroy: jest.fn((_sid, cb) => cb?.()), touch: jest.fn((_sid, _sess, cb) => cb?.()), @@ -32,41 +28,31 @@ jest.mock('connect-mongo', () => ({ return store; }), })); -jest.mock('@/models', () => ({ - mongooseConnection: { - get readyState() { - return mockConnection.readyState; - }, - getClient: () => mockConnection.client, - }, +jest.mock('@/config', () => ({ + MONGO_URI: 'mongodb://localhost:27017/hyperdx-test', +})); +jest.mock('@/utils/readiness', () => ({ + isMongoConnected: () => mockMongoConnected, })); -import { MongooseSessionStore } from '@/utils/sessionStore'; - -const { connected, connecting, disconnected } = mongoose.ConnectionStates; +import { MongoSessionStore } from '@/utils/sessionStore'; type GetResult = { err: unknown; session?: session.SessionData | null }; -const getSession = (store: MongooseSessionStore, sid = 'sid') => +const getSession = (store: MongoSessionStore, sid = 'sid') => new Promise(resolve => store.get(sid, (err, session) => resolve({ err, session })), ); -const connect = (client: unknown) => { - mockConnection.client = client; - mockConnection.readyState = connected; -}; - -describe('MongooseSessionStore', () => { +describe('MongoSessionStore', () => { beforeEach(() => { mockStores.length = 0; mockCollectionP = () => Promise.resolve({}); - mockConnection.readyState = disconnected; - mockConnection.client = undefined; + mockMongoConnected = false; }); it('fails session operations fast until mongoose connects', async () => { - const store = new MongooseSessionStore(); + const store = new MongoSessionStore(); const { err } = await getSession(store); @@ -75,26 +61,21 @@ describe('MongooseSessionStore', () => { }); it('recovers once mongoose connects after a failed start, without a restart', async () => { - const store = new MongooseSessionStore(); - // connectDBWithRetry's first attempt left a client that never connected. - mockConnection.client = { attempt: 1 }; - mockConnection.readyState = connecting; + const store = new MongoSessionStore(); expect((await getSession(store)).err).toBeDefined(); - const client = { attempt: 2 }; - connect(client); + mockMongoConnected = true; expect(await getSession(store)).toEqual({ err: null, - session: { client }, + session: { mongoUrl: 'mongodb://localhost:27017/hyperdx-test' }, }); expect(mockStores).toHaveLength(1); - expect(mockStores[0].client).toBe(client); }); - it('delegates set, destroy and touch to the store on the connected client', async () => { - connect({}); - const store = new MongooseSessionStore(); + it('delegates set, destroy and touch to the store', async () => { + mockMongoConnected = true; + const store = new MongoSessionStore(); const sess = { cookie: new session.Cookie() }; const set = jest.fn(); const destroy = jest.fn(); @@ -113,43 +94,28 @@ describe('MongooseSessionStore', () => { expect(touch).toHaveBeenCalled(); }); - it('keeps the store through a transient disconnect of the same client', async () => { - connect({}); - const store = new MongooseSessionStore(); + it('keeps a connected store through a later MongoDB outage', async () => { + // The store's MongoClient reconnects by itself once it has connected. + mockMongoConnected = true; + const store = new MongoSessionStore(); await getSession(store); - mockConnection.readyState = disconnected; + mockMongoConnected = false; expect((await getSession(store)).err).toBeNull(); expect(mockStores).toHaveLength(1); }); - it('rebuilds the store when mongoose reconnects with a new client', async () => { - connect({ id: 'a' }); - const store = new MongooseSessionStore(); - await getSession(store); - - // A closed client can't be reused; the reconnect makes a new one. - const client = { id: 'b' }; - mockConnection.client = client; - mockConnection.readyState = connecting; - expect((await getSession(store)).err).toBeDefined(); - - connect(client); - - expect((await getSession(store)).session).toEqual({ client }); - expect(mockStores).toHaveLength(2); - }); - - it('replaces a store whose collection setup failed', async () => { - mockCollectionP = () => Promise.reject(new Error('createIndex failed')); - connect({}); - const store = new MongooseSessionStore(); + it('replaces a store whose connect or setup failed', async () => { + mockCollectionP = () => Promise.reject(new Error('connect ECONNREFUSED')); + mockMongoConnected = true; + const store = new MongoSessionStore(); await getSession(store); await new Promise(setImmediate); mockCollectionP = () => Promise.resolve({}); await getSession(store); + await getSession(store); expect(mockStores).toHaveLength(2); }); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index e371b721de..92fcff53f8 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -1,52 +1,35 @@ import MongoStore from 'connect-mongo'; import session from 'express-session'; -import { mongooseConnection } from '@/models'; +import * as config from '@/config'; import { isMongoConnected } from '@/utils/readiness'; -type MongoClient = ReturnType; -type ConnectMongoClient = NonNullable< - ConstructorParameters[0]['client'] ->; - /** - * express-session store that keeps sessions in MongoDB over the app's - * mongoose connection. + * express-session store that keeps sessions in MongoDB and recovers when + * MongoDB starts after the API. * - * `new MongoStore({ mongoUrl })` opens its own MongoClient once, at - * construction, and caches that connect promise. If MongoDB is unreachable - * when the API starts, every session read/write replays the original - * MongoServerSelectionError (so every request with a session cookie 500s) - * until the process restarts, even after connectDBWithRetry has connected - * mongoose (see https://github.com/hyperdxio/hyperdx/issues/2966). + * `new MongoStore({ mongoUrl })` connects once, at construction, and every + * session read/write awaits that one promise. If MongoDB is unreachable when + * the API starts, the store replays the original MongoServerSelectionError + * (so every request with a session cookie 500s) until the process restarts, + * even after connectDBWithRetry has connected mongoose (see + * https://github.com/hyperdxio/hyperdx/issues/2966). * - * Instead, build a MongoStore on mongoose's client once mongoose is connected, - * and rebuild it whenever mongoose replaces that client: every - * mongoose.connect() creates a new one, and a closed client can't be reused. - * Until mongoose connects, session operations fail fast. + * Instead, create the MongoStore once mongoose is connected, and create a new + * one if its connect or setup fails. Until then, session operations fail fast. */ -export class MongooseSessionStore extends session.Store { +export class MongoSessionStore extends session.Store { private store?: MongoStore; - private storeClient?: MongoClient; private getStore(): MongoStore | undefined { - const client = mongooseConnection.getClient(); - // Keep using the store through a transient disconnect of the same client; - // the driver reconnects it on its own. - if (this.store != null && this.storeClient === client) { + if (this.store != null) { return this.store; } if (!isMongoConnected()) { return undefined; } - // mongoose bundles its own mongodb driver (4.x), so its MongoClient type - // differs from the hoisted 6.x one connect-mongo is typed against. 4.x is - // connect-mongo's declared peer and covers the collection CRUD it uses. - const store = new MongoStore({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - client: client as unknown as ConnectMongoClient, - }); - // A store whose collection setup failed would replay that error forever, + const store = new MongoStore({ mongoUrl: config.MONGO_URI }); + // A store whose connect or setup failed would replay that error forever, // so drop it and let the next session operation build a fresh one. store.collectionP.catch(() => { if (this.store === store) { @@ -54,7 +37,6 @@ export class MongooseSessionStore extends session.Store { } }); this.store = store; - this.storeClient = client; return store; } From 3e0d18a42074e880764e29369843969502bd8ec4 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 17:19:52 -0700 Subject: [PATCH 07/10] fix(api): close failed session stores and back off before rebuilding A store whose client connected but whose setup (the TTL index) failed was dropped without closing its MongoClient, leaking a connection pool per failure. Close it, and wait with capped exponential backoff (1s to 30s, as connectDBWithRetry does) before building the next store so a persistent setup failure doesn't open a new client on every session operation. --- .../src/utils/__tests__/sessionStore.test.ts | 45 +++++++++++++++++-- packages/api/src/utils/sessionStore.ts | 33 +++++++++++--- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index 94e621ffb2..776ffdf41b 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -8,6 +8,7 @@ type MockStore = { set: jest.Mock; destroy: jest.Mock; touch: jest.Mock; + close: jest.Mock; }; const mockStores: MockStore[] = []; let mockCollectionP: () => Promise = () => Promise.resolve({}); @@ -23,6 +24,7 @@ jest.mock('connect-mongo', () => ({ set: jest.fn((_sid, _sess, cb) => cb?.()), destroy: jest.fn((_sid, cb) => cb?.()), touch: jest.fn((_sid, _sess, cb) => cb?.()), + close: jest.fn(() => Promise.resolve()), }; mockStores.push(store); return store; @@ -44,13 +46,22 @@ const getSession = (store: MongoSessionStore, sid = 'sid') => store.get(sid, (err, session) => resolve({ err, session })), ); +const flush = () => new Promise(setImmediate); + describe('MongoSessionStore', () => { + let nowSpy: jest.SpyInstance | undefined; + beforeEach(() => { mockStores.length = 0; mockCollectionP = () => Promise.resolve({}); mockMongoConnected = false; }); + afterEach(() => { + nowSpy?.mockRestore(); + nowSpy = undefined; + }); + it('fails session operations fast until mongoose connects', async () => { const store = new MongoSessionStore(); @@ -106,17 +117,43 @@ describe('MongoSessionStore', () => { expect(mockStores).toHaveLength(1); }); - it('replaces a store whose connect or setup failed', async () => { - mockCollectionP = () => Promise.reject(new Error('connect ECONNREFUSED')); + it('closes a store whose connect or setup failed', async () => { + mockCollectionP = () => Promise.reject(new Error('createIndex failed')); mockMongoConnected = true; const store = new MongoSessionStore(); + await getSession(store); - await new Promise(setImmediate); + await flush(); - mockCollectionP = () => Promise.resolve({}); + expect(mockStores[0].close).toHaveBeenCalled(); + }); + + it('backs off exponentially before replacing a failed store', async () => { + const retrying = new Error('Session store setup failed, retrying'); + nowSpy = jest.spyOn(Date, 'now').mockReturnValue(0); + mockCollectionP = () => Promise.reject(new Error('createIndex failed')); + mockMongoConnected = true; + const store = new MongoSessionStore(); await getSession(store); + await flush(); + + nowSpy.mockReturnValue(999); + expect((await getSession(store)).err).toEqual(retrying); + expect(mockStores).toHaveLength(1); + + nowSpy.mockReturnValue(1000); await getSession(store); + await flush(); + expect(mockStores).toHaveLength(2); + nowSpy.mockReturnValue(2999); + expect((await getSession(store)).err).toEqual(retrying); expect(mockStores).toHaveLength(2); + + mockCollectionP = () => Promise.resolve({}); + nowSpy.mockReturnValue(3000); + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(3); + expect(mockStores[2].get).toHaveBeenCalled(); }); }); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index 92fcff53f8..a9831b773e 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -4,6 +4,10 @@ import session from 'express-session'; import * as config from '@/config'; import { isMongoConnected } from '@/utils/readiness'; +// Same schedule as connectDBWithRetry's initial-connect retries. +const REBUILD_BASE_DELAY_MS = 1000; +const REBUILD_MAX_DELAY_MS = 30000; + /** * express-session store that keeps sessions in MongoDB and recovers when * MongoDB starts after the API. @@ -16,25 +20,42 @@ import { isMongoConnected } from '@/utils/readiness'; * https://github.com/hyperdxio/hyperdx/issues/2966). * * Instead, create the MongoStore once mongoose is connected, and create a new - * one if its connect or setup fails. Until then, session operations fail fast. + * one, with backoff, if its connect or setup fails. Until then, session + * operations fail fast. */ export class MongoSessionStore extends session.Store { private store?: MongoStore; + private failures = 0; + private retryAt = 0; - private getStore(): MongoStore | undefined { + private getStore(): MongoStore | Error { if (this.store != null) { return this.store; } if (!isMongoConnected()) { - return undefined; + return new Error('MongoDB is not connected'); + } + // Without a delay, a persistent setup failure (e.g. a missing createIndex + // privilege) would open a new client on every session operation. + if (Date.now() < this.retryAt) { + return new Error('Session store setup failed, retrying'); } const store = new MongoStore({ mongoUrl: config.MONGO_URI }); // A store whose connect or setup failed would replay that error forever, - // so drop it and let the next session operation build a fresh one. + // so drop it and let a later session operation build a fresh one. store.collectionP.catch(() => { if (this.store === store) { this.store = undefined; } + this.failures++; + this.retryAt = + Date.now() + + Math.min( + REBUILD_BASE_DELAY_MS * 2 ** (this.failures - 1), + REBUILD_MAX_DELAY_MS, + ); + // The client may have connected before setup failed. + store.close().catch(() => {}); }); this.store = store; return store; @@ -45,8 +66,8 @@ export class MongoSessionStore extends session.Store { op: (store: MongoStore) => void, ) { const store = this.getStore(); - if (store == null) { - callback?.(new Error('MongoDB is not connected')); + if (store instanceof Error) { + callback?.(store); return; } op(store); From 4bd5295775406c5e37390db7240c5667e4b0d5a2 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 17:29:19 -0700 Subject: [PATCH 08/10] fix(api): close the session store's MongoClient on shutdown The session store opens its own MongoClient, but Server.shutdown only closed mongoose's. Add MongoSessionStore.close(), which closes the current store and forgets it so a restarted server builds a new one, and call it from Server.shutdown. Also drop a guard in the failed-store handler that was always true. --- packages/api/src/api-app.ts | 5 ++++- packages/api/src/server.ts | 18 ++++++++++++++---- .../src/utils/__tests__/sessionStore.test.ts | 17 +++++++++++++++++ packages/api/src/utils/sessionStore.ts | 15 ++++++++++++--- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index d30049717b..c893b87f93 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -27,6 +27,9 @@ registerAITelemetry(); const app: express.Application = express(); +// Exported so Server.shutdown can close the store's MongoClient. +export const sessionStore = new MongoSessionStore(); + const sess: session.SessionOptions & { cookie: session.CookieOptions } = { // Use a slot-specific cookie name in dev so multiple worktrees on localhost // don't overwrite each other's session cookies. @@ -42,7 +45,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = { maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days }, rolling: true, - store: new MongoSessionStore(), + store: sessionStore, }; app.set('trust proxy', 1); diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index a6a1c3d732..ca391730cd 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -2,7 +2,7 @@ import http from 'http'; import gracefulShutdown from 'http-graceful-shutdown'; import { serializeError } from 'serialize-error'; -import app from '@/api-app'; +import app, { sessionStore } from '@/api-app'; import * as config from '@/config'; import { LOCAL_APP_TEAM } from '@/controllers/team'; import { runStartupMigrations } from '@/migrations'; @@ -30,9 +30,11 @@ export default class Server { protected async shutdown(_signal?: string) { let hasError = false; logger.info('Closing all db clients...'); - const [mongoCloseResult] = await Promise.allSettled([ - mongooseConnection.close(false), - ]); + const [mongoCloseResult, sessionStoreCloseResult] = + await Promise.allSettled([ + mongooseConnection.close(false), + sessionStore.close(), + ]); if (mongoCloseResult.status === 'rejected') { hasError = true; @@ -44,6 +46,14 @@ export default class Server { logger.info('MongoDB client closed.'); } + if (sessionStoreCloseResult.status === 'rejected') { + hasError = true; + logger.error( + { err: serializeError(sessionStoreCloseResult.reason) }, + 'Session store MongoDB client close failed', + ); + } + if (hasError) { throw new Error('Failed to close all clients.'); } diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index 776ffdf41b..acbf65ad81 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -156,4 +156,21 @@ describe('MongoSessionStore', () => { expect(mockStores).toHaveLength(3); expect(mockStores[2].get).toHaveBeenCalled(); }); + + it('close() closes the current store and the next operation builds a new one', async () => { + mockMongoConnected = true; + const store = new MongoSessionStore(); + await getSession(store); + + await store.close(); + + expect(mockStores[0].close).toHaveBeenCalled(); + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + expect(mockStores[1].get).toHaveBeenCalled(); + }); + + it('close() resolves when no store was created', async () => { + await expect(new MongoSessionStore().close()).resolves.toBeUndefined(); + }); }); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index a9831b773e..15d21127af 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -44,9 +44,7 @@ export class MongoSessionStore extends session.Store { // A store whose connect or setup failed would replay that error forever, // so drop it and let a later session operation build a fresh one. store.collectionP.catch(() => { - if (this.store === store) { - this.store = undefined; - } + this.store = undefined; this.failures++; this.retryAt = Date.now() + @@ -61,6 +59,17 @@ export class MongoSessionStore extends session.Store { return store; } + /** + * Closes the current store's MongoClient. A closed client can't be reused, + * so the next session operation builds a new store; this lets the server + * start again after a shutdown, as the integration tests do. + */ + async close(): Promise { + const store = this.store; + this.store = undefined; + await store?.close(); + } + private withStore( callback: ((err?: unknown) => void) | undefined, op: (store: MongoStore) => void, From 73c624711e4065d7a0d73200200ae4394e70ea4d Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 17:40:32 -0700 Subject: [PATCH 09/10] fix(api): log and count session store setup failures The failed-store handler dropped the rejection reason, so a persistent setup failure (missing createIndex privilege, auth error on the store's client) only surfaced as the generic retry message during backoff. Log it with the failure count and retry delay, as connectDBWithRetry does, and count it in hyperdx.session_store.setup_failures. --- .../src/utils/__tests__/sessionStore.test.ts | 5 ++++ packages/api/src/utils/sessionStore.ts | 26 ++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index acbf65ad81..e400ccbff5 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -36,6 +36,11 @@ jest.mock('@/config', () => ({ jest.mock('@/utils/readiness', () => ({ isMongoConnected: () => mockMongoConnected, })); +// Keeps the simulated setup failures out of the test output. +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { error: jest.fn() }, +})); import { MongoSessionStore } from '@/utils/sessionStore'; diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index 15d21127af..ecdbb3feec 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -1,13 +1,21 @@ import MongoStore from 'connect-mongo'; import session from 'express-session'; +import { serializeError } from 'serialize-error'; import * as config from '@/config'; +import { getCounter } from '@/utils/instrumentation'; +import logger from '@/utils/logger'; import { isMongoConnected } from '@/utils/readiness'; // Same schedule as connectDBWithRetry's initial-connect retries. const REBUILD_BASE_DELAY_MS = 1000; const REBUILD_MAX_DELAY_MS = 30000; +const setupFailureCounter = getCounter('hyperdx.session_store.setup_failures', { + description: + 'Failed session store connects or setups (TTL index); each schedules a rebuild with backoff.', +}); + /** * express-session store that keeps sessions in MongoDB and recovers when * MongoDB starts after the API. @@ -43,15 +51,19 @@ export class MongoSessionStore extends session.Store { const store = new MongoStore({ mongoUrl: config.MONGO_URI }); // A store whose connect or setup failed would replay that error forever, // so drop it and let a later session operation build a fresh one. - store.collectionP.catch(() => { + store.collectionP.catch((err: unknown) => { this.store = undefined; this.failures++; - this.retryAt = - Date.now() + - Math.min( - REBUILD_BASE_DELAY_MS * 2 ** (this.failures - 1), - REBUILD_MAX_DELAY_MS, - ); + const retryInMs = Math.min( + REBUILD_BASE_DELAY_MS * 2 ** (this.failures - 1), + REBUILD_MAX_DELAY_MS, + ); + this.retryAt = Date.now() + retryInMs; + setupFailureCounter.add(1); + logger.error( + { err: serializeError(err), failures: this.failures, retryInMs }, + 'Session store setup failed, retrying', + ); // The client may have connected before setup failed. store.close().catch(() => {}); }); From 47ff983acaf8480ca0efb3f94f5130b965d55fdc Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 13 Oct 2026 17:43:27 -0700 Subject: [PATCH 10/10] fix(api): ignore rejections from a session store close() already dropped close() replaces the current store, so a store whose setup rejects after close() could clear a newer store (leaking its client) and count the deliberate close as a setup failure, backing off the next start. Skip the handler when the store is no longer current, and reset the backoff in close() so a restarted server builds its store straight away. --- .../src/utils/__tests__/sessionStore.test.ts | 41 +++++++++++++++++++ packages/api/src/utils/sessionStore.ts | 13 +++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts index e400ccbff5..04585f5845 100644 --- a/packages/api/src/utils/__tests__/sessionStore.test.ts +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -178,4 +178,45 @@ describe('MongoSessionStore', () => { it('close() resolves when no store was created', async () => { await expect(new MongoSessionStore().close()).resolves.toBeUndefined(); }); + + describe('when a store closed by close() rejects later', () => { + let rejectSetup: (err: Error) => void = () => {}; + + beforeEach(() => { + mockCollectionP = () => + new Promise((_resolve, reject) => { + rejectSetup = reject; + }); + mockMongoConnected = true; + }); + + it('keeps the newer store built after close()', async () => { + const store = new MongoSessionStore(); + await getSession(store); + await store.close(); + mockCollectionP = () => Promise.resolve({}); + await getSession(store); + + rejectSetup(new Error('client closed')); + await flush(); + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + expect(mockStores[0].close).toHaveBeenCalledTimes(1); + expect(mockStores[1].close).not.toHaveBeenCalled(); + }); + + it('does not back off the next store', async () => { + const store = new MongoSessionStore(); + await getSession(store); + await store.close(); + + rejectSetup(new Error('client closed')); + await flush(); + mockCollectionP = () => Promise.resolve({}); + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + }); + }); }); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts index ecdbb3feec..b549828fbe 100644 --- a/packages/api/src/utils/sessionStore.ts +++ b/packages/api/src/utils/sessionStore.ts @@ -52,6 +52,12 @@ export class MongoSessionStore extends session.Store { // A store whose connect or setup failed would replay that error forever, // so drop it and let a later session operation build a fresh one. store.collectionP.catch((err: unknown) => { + // Only close() replaces the current store, and it has already closed + // this one: its rejection isn't a setup failure, and the current store + // may be a newer one. + if (this.store !== store) { + return; + } this.store = undefined; this.failures++; const retryInMs = Math.min( @@ -73,12 +79,15 @@ export class MongoSessionStore extends session.Store { /** * Closes the current store's MongoClient. A closed client can't be reused, - * so the next session operation builds a new store; this lets the server - * start again after a shutdown, as the integration tests do. + * so the next session operation builds a new store, without waiting out + * any earlier backoff; this lets the server start again after a shutdown, + * as the integration tests do. */ async close(): Promise { const store = this.store; this.store = undefined; + this.failures = 0; + this.retryAt = 0; await store?.close(); }