Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/session-store-mongo-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/api': patch
---

fix: recover the session store when MongoDB comes up after the API. The store
now connects once MongoDB is reachable and retries a failed connect, instead of
returning 500 for every logged-in request until the API restarts
7 changes: 5 additions & 2 deletions packages/api/src/api-app.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import compression from 'compression';
import MongoStore from 'connect-mongo';
import express from 'express';
import session from 'express-session';
import onHeaders from 'on-headers';
Expand All @@ -22,11 +21,15 @@ import usageStats from './tasks/usageStats';
import { registerAITelemetry } from './utils/aiTelemetry';
import logger, { expressLogger } from './utils/logger';
import passport from './utils/passport';
import { MongoSessionStore } from './utils/sessionStore';

registerAITelemetry();

const app: express.Application = express();

// Exported so Server.shutdown can close the store's MongoClient.
export const sessionStore = new MongoSessionStore();

const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
// Use a slot-specific cookie name in dev so multiple worktrees on localhost
// don't overwrite each other's session cookies.
Expand All @@ -42,7 +45,7 @@ const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days
},
rolling: true,
store: new MongoStore({ mongoUrl: config.MONGO_URI }),
store: sessionStore,
};

app.set('trust proxy', 1);
Expand Down
18 changes: 14 additions & 4 deletions packages/api/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import http from 'http';
import gracefulShutdown from 'http-graceful-shutdown';
import { serializeError } from 'serialize-error';

import app from '@/api-app';
import app, { sessionStore } from '@/api-app';
import * as config from '@/config';
import { LOCAL_APP_TEAM } from '@/controllers/team';
import { runStartupMigrations } from '@/migrations';
Expand Down Expand Up @@ -30,9 +30,11 @@ export default class Server {
protected async shutdown(_signal?: string) {
let hasError = false;
logger.info('Closing all db clients...');
const [mongoCloseResult] = await Promise.allSettled([
mongooseConnection.close(false),
]);
const [mongoCloseResult, sessionStoreCloseResult] =
await Promise.allSettled([
mongooseConnection.close(false),
sessionStore.close(),
]);

if (mongoCloseResult.status === 'rejected') {
hasError = true;
Expand All @@ -44,6 +46,14 @@ export default class Server {
logger.info('MongoDB client closed.');
}

if (sessionStoreCloseResult.status === 'rejected') {
hasError = true;
logger.error(
{ err: serializeError(sessionStoreCloseResult.reason) },
'Session store MongoDB client close failed',
);
}

if (hasError) {
throw new Error('Failed to close all clients.');
}
Expand Down
222 changes: 222 additions & 0 deletions packages/api/src/utils/__tests__/sessionStore.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
import session from 'express-session';

// jest.mock factories are hoisted, so the state they use must be named `mock*`.
type MockStore = {
mongoUrl: string;
collectionP: Promise<unknown>;
get: jest.Mock;
set: jest.Mock;
destroy: jest.Mock;
touch: jest.Mock;
close: jest.Mock;
};
const mockStores: MockStore[] = [];
let mockCollectionP: () => Promise<unknown> = () => Promise.resolve({});
let mockMongoConnected = false;

jest.mock('connect-mongo', () => ({
__esModule: true,
default: jest.fn().mockImplementation(({ mongoUrl }) => {
const store: MockStore = {
mongoUrl,
collectionP: mockCollectionP(),
get: jest.fn((_sid, cb) => cb(null, { mongoUrl })),
set: jest.fn((_sid, _sess, cb) => cb?.()),
destroy: jest.fn((_sid, cb) => cb?.()),
touch: jest.fn((_sid, _sess, cb) => cb?.()),
close: jest.fn(() => Promise.resolve()),
};
mockStores.push(store);
return store;
}),
}));
jest.mock('@/config', () => ({
MONGO_URI: 'mongodb://localhost:27017/hyperdx-test',
}));
jest.mock('@/utils/readiness', () => ({
isMongoConnected: () => mockMongoConnected,
}));
// Keeps the simulated setup failures out of the test output.
jest.mock('@/utils/logger', () => ({
__esModule: true,
default: { error: jest.fn() },
}));

import { MongoSessionStore } from '@/utils/sessionStore';

type GetResult = { err: unknown; session?: session.SessionData | null };

const getSession = (store: MongoSessionStore, sid = 'sid') =>
new Promise<GetResult>(resolve =>
store.get(sid, (err, session) => resolve({ err, session })),
);

const flush = () => new Promise(setImmediate);

describe('MongoSessionStore', () => {
let nowSpy: jest.SpyInstance | undefined;

beforeEach(() => {
mockStores.length = 0;
mockCollectionP = () => Promise.resolve({});
mockMongoConnected = false;
});

afterEach(() => {
nowSpy?.mockRestore();
nowSpy = undefined;
});

it('fails session operations fast until mongoose connects', async () => {
const store = new MongoSessionStore();

const { err } = await getSession(store);

expect(err).toEqual(new Error('MongoDB is not connected'));
expect(mockStores).toHaveLength(0);
});

it('recovers once mongoose connects after a failed start, without a restart', async () => {
const store = new MongoSessionStore();
expect((await getSession(store)).err).toBeDefined();

mockMongoConnected = true;

expect(await getSession(store)).toEqual({
err: null,
session: { mongoUrl: 'mongodb://localhost:27017/hyperdx-test' },
});
expect(mockStores).toHaveLength(1);
});

it('delegates set, destroy and touch to the store', async () => {
mockMongoConnected = true;
const store = new MongoSessionStore();
const sess = { cookie: new session.Cookie() };
const set = jest.fn();
const destroy = jest.fn();
const touch = jest.fn();

store.set('sid', sess, set);
store.destroy('sid', destroy);
store.touch('sid', sess, touch);

expect(mockStores).toHaveLength(1);
expect(mockStores[0].set).toHaveBeenCalledWith('sid', sess, set);
expect(mockStores[0].destroy).toHaveBeenCalledWith('sid', destroy);
expect(mockStores[0].touch).toHaveBeenCalledWith('sid', sess, touch);
expect(set).toHaveBeenCalled();
expect(destroy).toHaveBeenCalled();
expect(touch).toHaveBeenCalled();
});

it('keeps a connected store through a later MongoDB outage', async () => {
// The store's MongoClient reconnects by itself once it has connected.
mockMongoConnected = true;
const store = new MongoSessionStore();
await getSession(store);

mockMongoConnected = false;

expect((await getSession(store)).err).toBeNull();
expect(mockStores).toHaveLength(1);
});

it('closes a store whose connect or setup failed', async () => {
mockCollectionP = () => Promise.reject(new Error('createIndex failed'));
mockMongoConnected = true;
const store = new MongoSessionStore();

await getSession(store);
await flush();

expect(mockStores[0].close).toHaveBeenCalled();
});

it('backs off exponentially before replacing a failed store', async () => {
const retrying = new Error('Session store setup failed, retrying');
nowSpy = jest.spyOn(Date, 'now').mockReturnValue(0);
mockCollectionP = () => Promise.reject(new Error('createIndex failed'));
mockMongoConnected = true;
const store = new MongoSessionStore();
await getSession(store);
await flush();

nowSpy.mockReturnValue(999);
expect((await getSession(store)).err).toEqual(retrying);
expect(mockStores).toHaveLength(1);

nowSpy.mockReturnValue(1000);
await getSession(store);
await flush();
expect(mockStores).toHaveLength(2);

nowSpy.mockReturnValue(2999);
expect((await getSession(store)).err).toEqual(retrying);
expect(mockStores).toHaveLength(2);

mockCollectionP = () => Promise.resolve({});
nowSpy.mockReturnValue(3000);
expect((await getSession(store)).err).toBeNull();
expect(mockStores).toHaveLength(3);
expect(mockStores[2].get).toHaveBeenCalled();
});

it('close() closes the current store and the next operation builds a new one', async () => {
mockMongoConnected = true;
const store = new MongoSessionStore();
await getSession(store);

await store.close();

expect(mockStores[0].close).toHaveBeenCalled();
expect((await getSession(store)).err).toBeNull();
expect(mockStores).toHaveLength(2);
expect(mockStores[1].get).toHaveBeenCalled();
});

it('close() resolves when no store was created', async () => {
await expect(new MongoSessionStore().close()).resolves.toBeUndefined();
});

describe('when a store closed by close() rejects later', () => {
let rejectSetup: (err: Error) => void = () => {};

beforeEach(() => {
mockCollectionP = () =>
new Promise((_resolve, reject) => {
rejectSetup = reject;
});
mockMongoConnected = true;
});

it('keeps the newer store built after close()', async () => {
const store = new MongoSessionStore();
await getSession(store);
await store.close();
mockCollectionP = () => Promise.resolve({});
await getSession(store);

rejectSetup(new Error('client closed'));
await flush();

expect((await getSession(store)).err).toBeNull();
expect(mockStores).toHaveLength(2);
expect(mockStores[0].close).toHaveBeenCalledTimes(1);
expect(mockStores[1].close).not.toHaveBeenCalled();
});

it('does not back off the next store', async () => {
const store = new MongoSessionStore();
await getSession(store);
await store.close();

rejectSetup(new Error('client closed'));
await flush();
mockCollectionP = () => Promise.resolve({});

expect((await getSession(store)).err).toBeNull();
expect(mockStores).toHaveLength(2);
});
});
});
Loading
Loading