From f4a4f71b26ab5015b6eedb122c5a253511af0fc0 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 15 Sep 2026 19:31:58 -0400 Subject: [PATCH 1/2] fix: Exit when startup fails instead of hanging A plugin that failed during startup left the process alive and serving nothing: the startup error surfaced as an unhandled rejection that the backend only logged, readiness stayed at 503, and liveness kept passing, so the pod was never restarted. Production sat in that state for nearly three hours after a momentary database outage. Key changes: - Await the backend start and exit non-zero when it fails, so a failed startup becomes a container restart rather than a pod that serves nothing - Replace the root health service so liveness fails once the backend has never reached ready within a bounded deadline, leaving liveness untouched for a backend that has started - Wrap the database service so a plugin retries an unreachable database for a bounded window before giving up, which absorbs a short blip without costing a restart The retry window closes well before the liveness deadline, so a database that stays away still ends the process rather than holding it open. --- packages/backend/src/index.ts | 10 +- .../backend/src/services/database.test.ts | 99 +++++++++++++++++ packages/backend/src/services/database.ts | 87 +++++++++++++++ .../backend/src/services/rootHealth.test.ts | 104 ++++++++++++++++++ packages/backend/src/services/rootHealth.ts | 76 +++++++++++++ 5 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/services/database.test.ts create mode 100644 packages/backend/src/services/database.ts create mode 100644 packages/backend/src/services/rootHealth.test.ts create mode 100644 packages/backend/src/services/rootHealth.ts diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 03dd229..9a4974d 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -7,9 +7,14 @@ */ import { createBackend } from '@backstage/backend-defaults'; +import { retryingDatabaseServiceFactory } from './services/database'; +import { startupDeadlineRootHealthServiceFactory } from './services/rootHealth'; const backend = createBackend(); +backend.add(retryingDatabaseServiceFactory); +backend.add(startupDeadlineRootHealthServiceFactory); + backend.add(import('@backstage/plugin-app-backend')); backend.add(import('@backstage/plugin-proxy-backend')); @@ -73,4 +78,7 @@ backend.add(import('@backstage/plugin-signals-backend')); // mcp actions plugin backend.add(import('@backstage/plugin-mcp-actions-backend')); -backend.start(); +backend.start().catch(error => { + process.stderr.write(`Backend failed to start: ${error?.stack ?? error}\n`); + process.exit(1); +}); diff --git a/packages/backend/src/services/database.test.ts b/packages/backend/src/services/database.test.ts new file mode 100644 index 0000000..f587855 --- /dev/null +++ b/packages/backend/src/services/database.test.ts @@ -0,0 +1,99 @@ +import type { + DatabaseService, + LoggerService, +} from '@backstage/backend-plugin-api'; +import { + createRetryingDatabaseService, + DATABASE_STARTUP_TIMEOUT_MS, +} from './database'; + +const knexStub = { name: 'knex' } as unknown as Awaited< + ReturnType +>; + +const createLogger = (): LoggerService => { + const logger: LoggerService = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + child: () => logger, + }; + return logger; +}; + +describe('createRetryingDatabaseService', () => { + it('returns the client without waiting when the database is reachable', async () => { + const getClient = jest.fn().mockResolvedValue(knexStub); + const sleep = jest.fn().mockResolvedValue(undefined); + + const service = createRetryingDatabaseService({ + delegate: { getClient }, + logger: createLogger(), + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(getClient).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('rides out a brief outage and then connects', async () => { + let clock = 0; + const getClient = jest + .fn() + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockResolvedValue(knexStub); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + delegate: { getClient }, + logger: createLogger(), + timeoutMs: 30_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(getClient).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it('gives up with the last error once the retry window closes', async () => { + let clock = 0; + const error = new Error('getaddrinfo ENOTFOUND postgres'); + const getClient = jest.fn().mockRejectedValue(error); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + delegate: { getClient }, + logger: createLogger(), + timeoutMs: 5_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).rejects.toThrow(error); + expect(getClient).toHaveBeenCalledTimes(6); + }); + + it('passes the migration settings of the wrapped service through', () => { + const service = createRetryingDatabaseService({ + delegate: { + getClient: jest.fn().mockResolvedValue(knexStub), + migrations: { skip: true }, + }, + logger: createLogger(), + }); + + expect(service.migrations).toEqual({ skip: true }); + expect(DATABASE_STARTUP_TIMEOUT_MS).toBeGreaterThan(0); + }); +}); diff --git a/packages/backend/src/services/database.ts b/packages/backend/src/services/database.ts new file mode 100644 index 0000000..c5fa5cd --- /dev/null +++ b/packages/backend/src/services/database.ts @@ -0,0 +1,87 @@ +import { DatabaseManager } from '@backstage/backend-defaults/database'; +import { + coreServices, + createServiceFactory, + type DatabaseService, + type LoggerService, +} from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; + +export const DATABASE_STARTUP_TIMEOUT_MS = 30_000; +export const DATABASE_RETRY_INTERVAL_MS = 1_000; + +const wait = (ms: number) => + new Promise(resolve => { + setTimeout(resolve, ms); + }); + +export function createRetryingDatabaseService(options: { + delegate: DatabaseService; + logger: LoggerService; + timeoutMs?: number; + intervalMs?: number; + now?: () => number; + sleep?: (ms: number) => Promise; +}): DatabaseService { + const { delegate, logger } = options; + const timeoutMs = options.timeoutMs ?? DATABASE_STARTUP_TIMEOUT_MS; + const intervalMs = options.intervalMs ?? DATABASE_RETRY_INTERVAL_MS; + const now = options.now ?? (() => Date.now()); + const sleep = options.sleep ?? wait; + + return { + get migrations() { + return delegate.migrations; + }, + async getClient() { + const deadline = now() + timeoutMs; + for (let attempt = 1; ; attempt++) { + try { + return await delegate.getClient(); + } catch (error) { + if (now() >= deadline) { + throw error; + } + logger.warn( + `Database is not reachable, retrying in ${intervalMs}ms (attempt ${attempt})`, + error as Error, + ); + await sleep(intervalMs); + } + } + }, + }; +} + +export const retryingDatabaseServiceFactory = createServiceFactory({ + service: coreServices.database, + deps: { + config: coreServices.rootConfig, + lifecycle: coreServices.lifecycle, + logger: coreServices.logger, + pluginMetadata: coreServices.pluginMetadata, + rootLifecycle: coreServices.rootLifecycle, + rootLogger: coreServices.rootLogger, + }, + async createRootContext({ config, rootLifecycle, rootLogger }) { + return config.getOptional('backend.database') + ? DatabaseManager.fromConfig(config, { rootLifecycle, rootLogger }) + : DatabaseManager.fromConfig( + new ConfigReader({ + backend: { + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, + }), + { rootLifecycle, rootLogger }, + ); + }, + async factory({ pluginMetadata, lifecycle, logger }, databaseManager) { + return createRetryingDatabaseService({ + delegate: databaseManager.forPlugin(pluginMetadata.getId(), { + lifecycle, + logger, + }), + logger, + }); + }, +}); diff --git a/packages/backend/src/services/rootHealth.test.ts b/packages/backend/src/services/rootHealth.test.ts new file mode 100644 index 0000000..59ec241 --- /dev/null +++ b/packages/backend/src/services/rootHealth.test.ts @@ -0,0 +1,104 @@ +import type { + LifecycleServiceStartupHook, + RootLifecycleService, +} from '@backstage/backend-plugin-api'; +import { + StartupDeadlineRootHealthService, + STARTUP_LIVENESS_TIMEOUT_MS, +} from './rootHealth'; + +const createLifecycle = () => { + const startupHooks: LifecycleServiceStartupHook[] = []; + const beforeShutdownHooks: Array<() => void | Promise> = []; + const lifecycle: RootLifecycleService = { + addStartupHook: hook => { + startupHooks.push(hook); + }, + addShutdownHook: () => {}, + addBeforeShutdownHook: hook => { + beforeShutdownHooks.push(hook); + }, + }; + return { + lifecycle, + startup: async () => { + for (const hook of startupHooks) { + await hook(); + } + }, + beforeShutdown: async () => { + for (const hook of beforeShutdownHooks) { + await hook(); + } + }, + }; +}; + +describe('StartupDeadlineRootHealthService', () => { + it('serves liveness while the backend is still within its startup deadline', async () => { + let clock = 1000; + const { lifecycle } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + clock += 59_999; + + await expect(health.getLiveness()).resolves.toEqual({ + status: 200, + payload: { status: 'ok' }, + }); + await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 }); + }); + + it('fails liveness once the backend has never become ready in time', async () => { + let clock = 1000; + const { lifecycle } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + clock += 60_000; + + await expect(health.getLiveness()).resolves.toMatchObject({ status: 500 }); + }); + + it('keeps liveness passing indefinitely once the backend has started', async () => { + let clock = 1000; + const { lifecycle, startup } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + await startup(); + clock += 10 * 60_000; + + await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 }); + await expect(health.getReadiness()).resolves.toEqual({ + status: 200, + payload: { status: 'ok' }, + }); + }); + + it('reports not ready while shutting down but stays live', async () => { + const { lifecycle, startup, beforeShutdown } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ lifecycle }); + + await startup(); + await beforeShutdown(); + + await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 }); + await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 }); + }); + + it('defaults to a bounded startup deadline', () => { + expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeGreaterThan(0); + expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeLessThanOrEqual(600_000); + }); +}); diff --git a/packages/backend/src/services/rootHealth.ts b/packages/backend/src/services/rootHealth.ts new file mode 100644 index 0000000..4f84578 --- /dev/null +++ b/packages/backend/src/services/rootHealth.ts @@ -0,0 +1,76 @@ +import { + coreServices, + createServiceFactory, + type RootHealthService, + type RootLifecycleService, +} from '@backstage/backend-plugin-api'; + +export const STARTUP_LIVENESS_TIMEOUT_MS = 300_000; + +type StartupState = 'init' | 'up' | 'down'; + +export class StartupDeadlineRootHealthService implements RootHealthService { + #state: StartupState = 'init'; + readonly #startedAt: number; + readonly #timeoutMs: number; + readonly #now: () => number; + + constructor(options: { + lifecycle: RootLifecycleService; + timeoutMs?: number; + now?: () => number; + }) { + this.#now = options.now ?? (() => Date.now()); + this.#timeoutMs = options.timeoutMs ?? STARTUP_LIVENESS_TIMEOUT_MS; + this.#startedAt = this.#now(); + + options.lifecycle.addStartupHook(() => { + this.#state = 'up'; + }); + options.lifecycle.addBeforeShutdownHook(() => { + this.#state = 'down'; + }); + } + + async getLiveness() { + if ( + this.#state === 'init' && + this.#now() - this.#startedAt >= this.#timeoutMs + ) { + return { + status: 500, + payload: { + status: 'error', + message: 'Backend did not become ready within the startup deadline', + }, + }; + } + return { status: 200, payload: { status: 'ok' } }; + } + + async getReadiness() { + if (this.#state === 'init') { + return { + status: 503, + payload: { status: 'error', message: 'Backend has not started yet' }, + }; + } + if (this.#state === 'down') { + return { + status: 503, + payload: { status: 'error', message: 'Backend is shutting down' }, + }; + } + return { status: 200, payload: { status: 'ok' } }; + } +} + +export const startupDeadlineRootHealthServiceFactory = createServiceFactory({ + service: coreServices.rootHealth, + deps: { + lifecycle: coreServices.rootLifecycle, + }, + async factory({ lifecycle }) { + return new StartupDeadlineRootHealthService({ lifecycle }); + }, +}); From a5de69e3db6e3562a2876263de637cf305313a72 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Thu, 17 Sep 2026 20:02:01 -0400 Subject: [PATCH 2/2] fix: Retry a database outage on a fresh connection The retry wrapper called the same plugin database service on every attempt, and that service memoizes its first connection promise for the lifetime of its manager. A rejected promise stays in that cache, so every later attempt resolved the original failure instantly and no second connection was ever opened. A database that returned after a second still failed the startup it was meant to survive. Key changes: - Build a database manager per attempt, so each retry carries its own cache and opens a real connection - Cover the case with a delegate that memoizes its first failure, which is the behaviour the upstream manager has and the previous mock did not --- .../backend/src/services/database.test.ts | 62 +++++++++++++++--- packages/backend/src/services/database.ts | 64 ++++++++++++------- 2 files changed, 94 insertions(+), 32 deletions(-) diff --git a/packages/backend/src/services/database.test.ts b/packages/backend/src/services/database.test.ts index f587855..16fde95 100644 --- a/packages/backend/src/services/database.test.ts +++ b/packages/backend/src/services/database.test.ts @@ -22,13 +22,31 @@ const createLogger = (): LoggerService => { return logger; }; +const createMemoizingDelegateFactory = ( + connect: () => Promise>>, +) => { + const connects = jest.fn(connect); + const createDelegate = jest.fn((): DatabaseService => { + let cached: ReturnType | undefined; + return { + getClient() { + if (!cached) { + cached = connects(); + } + return cached; + }, + }; + }); + return { createDelegate, connects }; +}; + describe('createRetryingDatabaseService', () => { it('returns the client without waiting when the database is reachable', async () => { const getClient = jest.fn().mockResolvedValue(knexStub); const sleep = jest.fn().mockResolvedValue(undefined); const service = createRetryingDatabaseService({ - delegate: { getClient }, + createDelegate: () => ({ getClient }), logger: createLogger(), sleep, }); @@ -40,7 +58,7 @@ describe('createRetryingDatabaseService', () => { it('rides out a brief outage and then connects', async () => { let clock = 0; - const getClient = jest + const connect = jest .fn() .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) @@ -50,7 +68,7 @@ describe('createRetryingDatabaseService', () => { }); const service = createRetryingDatabaseService({ - delegate: { getClient }, + createDelegate: () => ({ getClient: connect }), logger: createLogger(), timeoutMs: 30_000, intervalMs: 1_000, @@ -59,20 +77,46 @@ describe('createRetryingDatabaseService', () => { }); await expect(service.getClient()).resolves.toBe(knexStub); - expect(getClient).toHaveBeenCalledTimes(3); + expect(connect).toHaveBeenCalledTimes(3); expect(sleep).toHaveBeenCalledTimes(2); }); + it('retries against a delegate that caches its first failure', async () => { + let clock = 0; + const { createDelegate, connects } = createMemoizingDelegateFactory( + jest + .fn() + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockResolvedValue(knexStub), + ); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + createDelegate, + logger: createLogger(), + timeoutMs: 30_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(connects).toHaveBeenCalledTimes(2); + expect(createDelegate).toHaveBeenCalledTimes(2); + }); + it('gives up with the last error once the retry window closes', async () => { let clock = 0; const error = new Error('getaddrinfo ENOTFOUND postgres'); - const getClient = jest.fn().mockRejectedValue(error); + const connect = jest.fn().mockRejectedValue(error); const sleep = jest.fn(async (ms: number) => { clock += ms; }); const service = createRetryingDatabaseService({ - delegate: { getClient }, + createDelegate: () => ({ getClient: connect }), logger: createLogger(), timeoutMs: 5_000, intervalMs: 1_000, @@ -81,15 +125,15 @@ describe('createRetryingDatabaseService', () => { }); await expect(service.getClient()).rejects.toThrow(error); - expect(getClient).toHaveBeenCalledTimes(6); + expect(connect).toHaveBeenCalledTimes(6); }); it('passes the migration settings of the wrapped service through', () => { const service = createRetryingDatabaseService({ - delegate: { + createDelegate: () => ({ getClient: jest.fn().mockResolvedValue(knexStub), migrations: { skip: true }, - }, + }), logger: createLogger(), }); diff --git a/packages/backend/src/services/database.ts b/packages/backend/src/services/database.ts index c5fa5cd..4f5830b 100644 --- a/packages/backend/src/services/database.ts +++ b/packages/backend/src/services/database.ts @@ -4,6 +4,8 @@ import { createServiceFactory, type DatabaseService, type LoggerService, + type RootConfigService, + type RootLifecycleService, } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; @@ -16,25 +18,28 @@ const wait = (ms: number) => }); export function createRetryingDatabaseService(options: { - delegate: DatabaseService; + createDelegate: () => DatabaseService; logger: LoggerService; timeoutMs?: number; intervalMs?: number; now?: () => number; sleep?: (ms: number) => Promise; }): DatabaseService { - const { delegate, logger } = options; + const { createDelegate, logger } = options; const timeoutMs = options.timeoutMs ?? DATABASE_STARTUP_TIMEOUT_MS; const intervalMs = options.intervalMs ?? DATABASE_RETRY_INTERVAL_MS; const now = options.now ?? (() => Date.now()); const sleep = options.sleep ?? wait; + const first = createDelegate(); + return { get migrations() { - return delegate.migrations; + return first.migrations; }, async getClient() { const deadline = now() + timeoutMs; + let delegate = first; for (let attempt = 1; ; attempt++) { try { return await delegate.getClient(); @@ -47,12 +52,30 @@ export function createRetryingDatabaseService(options: { error as Error, ); await sleep(intervalMs); + delegate = createDelegate(); } } }, }; } +export function createDatabaseManager( + config: RootConfigService, + deps: { + rootLifecycle: RootLifecycleService; + rootLogger: LoggerService; + }, +): DatabaseManager { + const resolved = config.getOptional('backend.database') + ? config + : new ConfigReader({ + backend: { + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, + }); + return DatabaseManager.fromConfig(resolved, deps); +} + export const retryingDatabaseServiceFactory = createServiceFactory({ service: coreServices.database, deps: { @@ -63,25 +86,20 @@ export const retryingDatabaseServiceFactory = createServiceFactory({ rootLifecycle: coreServices.rootLifecycle, rootLogger: coreServices.rootLogger, }, - async createRootContext({ config, rootLifecycle, rootLogger }) { - return config.getOptional('backend.database') - ? DatabaseManager.fromConfig(config, { rootLifecycle, rootLogger }) - : DatabaseManager.fromConfig( - new ConfigReader({ - backend: { - database: { client: 'better-sqlite3', connection: ':memory:' }, - }, - }), - { rootLifecycle, rootLogger }, - ); - }, - async factory({ pluginMetadata, lifecycle, logger }, databaseManager) { - return createRetryingDatabaseService({ - delegate: databaseManager.forPlugin(pluginMetadata.getId(), { - lifecycle, - logger, - }), - logger, - }); + async factory({ + config, + lifecycle, + logger, + pluginMetadata, + rootLifecycle, + rootLogger, + }) { + const pluginId = pluginMetadata.getId(); + const createDelegate = () => + createDatabaseManager(config, { rootLifecycle, rootLogger }).forPlugin( + pluginId, + { lifecycle, logger }, + ); + return createRetryingDatabaseService({ createDelegate, logger }); }, });