diff --git a/.changeset/session-store-mongo-retry.md b/.changeset/session-store-mongo-retry.md new file mode 100644 index 0000000000..a356e7157b --- /dev/null +++ b/.changeset/session-store-mongo-retry.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +fix: recover the session store when MongoDB comes up after the API. The store +now connects once MongoDB is reachable and retries a failed connect, instead of +returning 500 for every logged-in request until the API restarts diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index c2d9028a15..c893b87f93 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -1,5 +1,4 @@ import compression from 'compression'; -import MongoStore from 'connect-mongo'; import express from 'express'; import session from 'express-session'; import onHeaders from 'on-headers'; @@ -22,11 +21,15 @@ import usageStats from './tasks/usageStats'; import { registerAITelemetry } from './utils/aiTelemetry'; import logger, { expressLogger } from './utils/logger'; import passport from './utils/passport'; +import { MongoSessionStore } from './utils/sessionStore'; registerAITelemetry(); const app: express.Application = express(); +// Exported so Server.shutdown can close the store's MongoClient. +export const sessionStore = new MongoSessionStore(); + const sess: session.SessionOptions & { cookie: session.CookieOptions } = { // Use a slot-specific cookie name in dev so multiple worktrees on localhost // don't overwrite each other's session cookies. @@ -42,7 +45,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = { maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days }, rolling: true, - store: new MongoStore({ mongoUrl: config.MONGO_URI }), + store: sessionStore, }; app.set('trust proxy', 1); diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index a6a1c3d732..ca391730cd 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -2,7 +2,7 @@ import http from 'http'; import gracefulShutdown from 'http-graceful-shutdown'; import { serializeError } from 'serialize-error'; -import app from '@/api-app'; +import app, { sessionStore } from '@/api-app'; import * as config from '@/config'; import { LOCAL_APP_TEAM } from '@/controllers/team'; import { runStartupMigrations } from '@/migrations'; @@ -30,9 +30,11 @@ export default class Server { protected async shutdown(_signal?: string) { let hasError = false; logger.info('Closing all db clients...'); - const [mongoCloseResult] = await Promise.allSettled([ - mongooseConnection.close(false), - ]); + const [mongoCloseResult, sessionStoreCloseResult] = + await Promise.allSettled([ + mongooseConnection.close(false), + sessionStore.close(), + ]); if (mongoCloseResult.status === 'rejected') { hasError = true; @@ -44,6 +46,14 @@ export default class Server { logger.info('MongoDB client closed.'); } + if (sessionStoreCloseResult.status === 'rejected') { + hasError = true; + logger.error( + { err: serializeError(sessionStoreCloseResult.reason) }, + 'Session store MongoDB client close failed', + ); + } + if (hasError) { throw new Error('Failed to close all clients.'); } diff --git a/packages/api/src/utils/__tests__/sessionStore.test.ts b/packages/api/src/utils/__tests__/sessionStore.test.ts new file mode 100644 index 0000000000..04585f5845 --- /dev/null +++ b/packages/api/src/utils/__tests__/sessionStore.test.ts @@ -0,0 +1,222 @@ +import session from 'express-session'; + +// jest.mock factories are hoisted, so the state they use must be named `mock*`. +type MockStore = { + mongoUrl: string; + collectionP: Promise; + get: jest.Mock; + set: jest.Mock; + destroy: jest.Mock; + touch: jest.Mock; + close: jest.Mock; +}; +const mockStores: MockStore[] = []; +let mockCollectionP: () => Promise = () => Promise.resolve({}); +let mockMongoConnected = false; + +jest.mock('connect-mongo', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(({ mongoUrl }) => { + const store: MockStore = { + mongoUrl, + collectionP: mockCollectionP(), + get: jest.fn((_sid, cb) => cb(null, { mongoUrl })), + set: jest.fn((_sid, _sess, cb) => cb?.()), + destroy: jest.fn((_sid, cb) => cb?.()), + touch: jest.fn((_sid, _sess, cb) => cb?.()), + close: jest.fn(() => Promise.resolve()), + }; + mockStores.push(store); + return store; + }), +})); +jest.mock('@/config', () => ({ + MONGO_URI: 'mongodb://localhost:27017/hyperdx-test', +})); +jest.mock('@/utils/readiness', () => ({ + isMongoConnected: () => mockMongoConnected, +})); +// Keeps the simulated setup failures out of the test output. +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { error: jest.fn() }, +})); + +import { MongoSessionStore } from '@/utils/sessionStore'; + +type GetResult = { err: unknown; session?: session.SessionData | null }; + +const getSession = (store: MongoSessionStore, sid = 'sid') => + new Promise(resolve => + store.get(sid, (err, session) => resolve({ err, session })), + ); + +const flush = () => new Promise(setImmediate); + +describe('MongoSessionStore', () => { + let nowSpy: jest.SpyInstance | undefined; + + beforeEach(() => { + mockStores.length = 0; + mockCollectionP = () => Promise.resolve({}); + mockMongoConnected = false; + }); + + afterEach(() => { + nowSpy?.mockRestore(); + nowSpy = undefined; + }); + + it('fails session operations fast until mongoose connects', async () => { + const store = new MongoSessionStore(); + + const { err } = await getSession(store); + + expect(err).toEqual(new Error('MongoDB is not connected')); + expect(mockStores).toHaveLength(0); + }); + + it('recovers once mongoose connects after a failed start, without a restart', async () => { + const store = new MongoSessionStore(); + expect((await getSession(store)).err).toBeDefined(); + + mockMongoConnected = true; + + expect(await getSession(store)).toEqual({ + err: null, + session: { mongoUrl: 'mongodb://localhost:27017/hyperdx-test' }, + }); + expect(mockStores).toHaveLength(1); + }); + + it('delegates set, destroy and touch to the store', async () => { + mockMongoConnected = true; + const store = new MongoSessionStore(); + const sess = { cookie: new session.Cookie() }; + const set = jest.fn(); + const destroy = jest.fn(); + const touch = jest.fn(); + + store.set('sid', sess, set); + store.destroy('sid', destroy); + store.touch('sid', sess, touch); + + expect(mockStores).toHaveLength(1); + expect(mockStores[0].set).toHaveBeenCalledWith('sid', sess, set); + expect(mockStores[0].destroy).toHaveBeenCalledWith('sid', destroy); + expect(mockStores[0].touch).toHaveBeenCalledWith('sid', sess, touch); + expect(set).toHaveBeenCalled(); + expect(destroy).toHaveBeenCalled(); + expect(touch).toHaveBeenCalled(); + }); + + it('keeps a connected store through a later MongoDB outage', async () => { + // The store's MongoClient reconnects by itself once it has connected. + mockMongoConnected = true; + const store = new MongoSessionStore(); + await getSession(store); + + mockMongoConnected = false; + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(1); + }); + + it('closes a store whose connect or setup failed', async () => { + mockCollectionP = () => Promise.reject(new Error('createIndex failed')); + mockMongoConnected = true; + const store = new MongoSessionStore(); + + await getSession(store); + await flush(); + + expect(mockStores[0].close).toHaveBeenCalled(); + }); + + it('backs off exponentially before replacing a failed store', async () => { + const retrying = new Error('Session store setup failed, retrying'); + nowSpy = jest.spyOn(Date, 'now').mockReturnValue(0); + mockCollectionP = () => Promise.reject(new Error('createIndex failed')); + mockMongoConnected = true; + const store = new MongoSessionStore(); + await getSession(store); + await flush(); + + nowSpy.mockReturnValue(999); + expect((await getSession(store)).err).toEqual(retrying); + expect(mockStores).toHaveLength(1); + + nowSpy.mockReturnValue(1000); + await getSession(store); + await flush(); + expect(mockStores).toHaveLength(2); + + nowSpy.mockReturnValue(2999); + expect((await getSession(store)).err).toEqual(retrying); + expect(mockStores).toHaveLength(2); + + mockCollectionP = () => Promise.resolve({}); + nowSpy.mockReturnValue(3000); + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(3); + expect(mockStores[2].get).toHaveBeenCalled(); + }); + + it('close() closes the current store and the next operation builds a new one', async () => { + mockMongoConnected = true; + const store = new MongoSessionStore(); + await getSession(store); + + await store.close(); + + expect(mockStores[0].close).toHaveBeenCalled(); + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + expect(mockStores[1].get).toHaveBeenCalled(); + }); + + it('close() resolves when no store was created', async () => { + await expect(new MongoSessionStore().close()).resolves.toBeUndefined(); + }); + + describe('when a store closed by close() rejects later', () => { + let rejectSetup: (err: Error) => void = () => {}; + + beforeEach(() => { + mockCollectionP = () => + new Promise((_resolve, reject) => { + rejectSetup = reject; + }); + mockMongoConnected = true; + }); + + it('keeps the newer store built after close()', async () => { + const store = new MongoSessionStore(); + await getSession(store); + await store.close(); + mockCollectionP = () => Promise.resolve({}); + await getSession(store); + + rejectSetup(new Error('client closed')); + await flush(); + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + expect(mockStores[0].close).toHaveBeenCalledTimes(1); + expect(mockStores[1].close).not.toHaveBeenCalled(); + }); + + it('does not back off the next store', async () => { + const store = new MongoSessionStore(); + await getSession(store); + await store.close(); + + rejectSetup(new Error('client closed')); + await flush(); + mockCollectionP = () => Promise.resolve({}); + + expect((await getSession(store)).err).toBeNull(); + expect(mockStores).toHaveLength(2); + }); + }); +}); diff --git a/packages/api/src/utils/sessionStore.ts b/packages/api/src/utils/sessionStore.ts new file mode 100644 index 0000000000..b549828fbe --- /dev/null +++ b/packages/api/src/utils/sessionStore.ts @@ -0,0 +1,134 @@ +import MongoStore from 'connect-mongo'; +import session from 'express-session'; +import { serializeError } from 'serialize-error'; + +import * as config from '@/config'; +import { getCounter } from '@/utils/instrumentation'; +import logger from '@/utils/logger'; +import { isMongoConnected } from '@/utils/readiness'; + +// Same schedule as connectDBWithRetry's initial-connect retries. +const REBUILD_BASE_DELAY_MS = 1000; +const REBUILD_MAX_DELAY_MS = 30000; + +const setupFailureCounter = getCounter('hyperdx.session_store.setup_failures', { + description: + 'Failed session store connects or setups (TTL index); each schedules a rebuild with backoff.', +}); + +/** + * express-session store that keeps sessions in MongoDB and recovers when + * MongoDB starts after the API. + * + * `new MongoStore({ mongoUrl })` connects once, at construction, and every + * session read/write awaits that one promise. If MongoDB is unreachable when + * the API starts, the store replays the original MongoServerSelectionError + * (so every request with a session cookie 500s) until the process restarts, + * even after connectDBWithRetry has connected mongoose (see + * https://github.com/hyperdxio/hyperdx/issues/2966). + * + * Instead, create the MongoStore once mongoose is connected, and create a new + * one, with backoff, if its connect or setup fails. Until then, session + * operations fail fast. + */ +export class MongoSessionStore extends session.Store { + private store?: MongoStore; + private failures = 0; + private retryAt = 0; + + private getStore(): MongoStore | Error { + if (this.store != null) { + return this.store; + } + if (!isMongoConnected()) { + return new Error('MongoDB is not connected'); + } + // Without a delay, a persistent setup failure (e.g. a missing createIndex + // privilege) would open a new client on every session operation. + if (Date.now() < this.retryAt) { + return new Error('Session store setup failed, retrying'); + } + const store = new MongoStore({ mongoUrl: config.MONGO_URI }); + // A store whose connect or setup failed would replay that error forever, + // so drop it and let a later session operation build a fresh one. + store.collectionP.catch((err: unknown) => { + // Only close() replaces the current store, and it has already closed + // this one: its rejection isn't a setup failure, and the current store + // may be a newer one. + if (this.store !== store) { + return; + } + this.store = undefined; + this.failures++; + const retryInMs = Math.min( + REBUILD_BASE_DELAY_MS * 2 ** (this.failures - 1), + REBUILD_MAX_DELAY_MS, + ); + this.retryAt = Date.now() + retryInMs; + setupFailureCounter.add(1); + logger.error( + { err: serializeError(err), failures: this.failures, retryInMs }, + 'Session store setup failed, retrying', + ); + // The client may have connected before setup failed. + store.close().catch(() => {}); + }); + this.store = store; + return store; + } + + /** + * Closes the current store's MongoClient. A closed client can't be reused, + * so the next session operation builds a new store, without waiting out + * any earlier backoff; this lets the server start again after a shutdown, + * as the integration tests do. + */ + async close(): Promise { + const store = this.store; + this.store = undefined; + this.failures = 0; + this.retryAt = 0; + await store?.close(); + } + + private withStore( + callback: ((err?: unknown) => void) | undefined, + op: (store: MongoStore) => void, + ) { + const store = this.getStore(); + if (store instanceof Error) { + callback?.(store); + return; + } + op(store); + } + + get( + sid: string, + callback: (err: unknown, session?: session.SessionData | null) => void, + ) { + this.withStore(callback, store => store.get(sid, callback)); + } + + set( + sid: string, + sess: session.SessionData, + callback?: (err?: unknown) => void, + ) { + this.withStore(callback, store => store.set(sid, sess, callback)); + } + + destroy(sid: string, callback?: (err?: unknown) => void) { + this.withStore(callback, store => store.destroy(sid, callback)); + } + + touch( + sid: string, + sess: session.SessionData, + callback?: (err?: unknown) => void, + ) { + this.withStore(callback, store => + store.touch(sid, sess as Parameters[1], callback), + ); + } +}