diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 03dd229..9a4974d 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -7,9 +7,14 @@ */ import { createBackend } from '@backstage/backend-defaults'; +import { retryingDatabaseServiceFactory } from './services/database'; +import { startupDeadlineRootHealthServiceFactory } from './services/rootHealth'; const backend = createBackend(); +backend.add(retryingDatabaseServiceFactory); +backend.add(startupDeadlineRootHealthServiceFactory); + backend.add(import('@backstage/plugin-app-backend')); backend.add(import('@backstage/plugin-proxy-backend')); @@ -73,4 +78,7 @@ backend.add(import('@backstage/plugin-signals-backend')); // mcp actions plugin backend.add(import('@backstage/plugin-mcp-actions-backend')); -backend.start(); +backend.start().catch(error => { + process.stderr.write(`Backend failed to start: ${error?.stack ?? error}\n`); + process.exit(1); +}); diff --git a/packages/backend/src/services/database.test.ts b/packages/backend/src/services/database.test.ts new file mode 100644 index 0000000..16fde95 --- /dev/null +++ b/packages/backend/src/services/database.test.ts @@ -0,0 +1,143 @@ +import type { + DatabaseService, + LoggerService, +} from '@backstage/backend-plugin-api'; +import { + createRetryingDatabaseService, + DATABASE_STARTUP_TIMEOUT_MS, +} from './database'; + +const knexStub = { name: 'knex' } as unknown as Awaited< + ReturnType +>; + +const createLogger = (): LoggerService => { + const logger: LoggerService = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + child: () => logger, + }; + return logger; +}; + +const createMemoizingDelegateFactory = ( + connect: () => Promise>>, +) => { + const connects = jest.fn(connect); + const createDelegate = jest.fn((): DatabaseService => { + let cached: ReturnType | undefined; + return { + getClient() { + if (!cached) { + cached = connects(); + } + return cached; + }, + }; + }); + return { createDelegate, connects }; +}; + +describe('createRetryingDatabaseService', () => { + it('returns the client without waiting when the database is reachable', async () => { + const getClient = jest.fn().mockResolvedValue(knexStub); + const sleep = jest.fn().mockResolvedValue(undefined); + + const service = createRetryingDatabaseService({ + createDelegate: () => ({ getClient }), + logger: createLogger(), + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(getClient).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('rides out a brief outage and then connects', async () => { + let clock = 0; + const connect = jest + .fn() + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockResolvedValue(knexStub); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + createDelegate: () => ({ getClient: connect }), + logger: createLogger(), + timeoutMs: 30_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(connect).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it('retries against a delegate that caches its first failure', async () => { + let clock = 0; + const { createDelegate, connects } = createMemoizingDelegateFactory( + jest + .fn() + .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres')) + .mockResolvedValue(knexStub), + ); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + createDelegate, + logger: createLogger(), + timeoutMs: 30_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).resolves.toBe(knexStub); + expect(connects).toHaveBeenCalledTimes(2); + expect(createDelegate).toHaveBeenCalledTimes(2); + }); + + it('gives up with the last error once the retry window closes', async () => { + let clock = 0; + const error = new Error('getaddrinfo ENOTFOUND postgres'); + const connect = jest.fn().mockRejectedValue(error); + const sleep = jest.fn(async (ms: number) => { + clock += ms; + }); + + const service = createRetryingDatabaseService({ + createDelegate: () => ({ getClient: connect }), + logger: createLogger(), + timeoutMs: 5_000, + intervalMs: 1_000, + now: () => clock, + sleep, + }); + + await expect(service.getClient()).rejects.toThrow(error); + expect(connect).toHaveBeenCalledTimes(6); + }); + + it('passes the migration settings of the wrapped service through', () => { + const service = createRetryingDatabaseService({ + createDelegate: () => ({ + getClient: jest.fn().mockResolvedValue(knexStub), + migrations: { skip: true }, + }), + logger: createLogger(), + }); + + expect(service.migrations).toEqual({ skip: true }); + expect(DATABASE_STARTUP_TIMEOUT_MS).toBeGreaterThan(0); + }); +}); diff --git a/packages/backend/src/services/database.ts b/packages/backend/src/services/database.ts new file mode 100644 index 0000000..4f5830b --- /dev/null +++ b/packages/backend/src/services/database.ts @@ -0,0 +1,105 @@ +import { DatabaseManager } from '@backstage/backend-defaults/database'; +import { + coreServices, + createServiceFactory, + type DatabaseService, + type LoggerService, + type RootConfigService, + type RootLifecycleService, +} from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; + +export const DATABASE_STARTUP_TIMEOUT_MS = 30_000; +export const DATABASE_RETRY_INTERVAL_MS = 1_000; + +const wait = (ms: number) => + new Promise(resolve => { + setTimeout(resolve, ms); + }); + +export function createRetryingDatabaseService(options: { + createDelegate: () => DatabaseService; + logger: LoggerService; + timeoutMs?: number; + intervalMs?: number; + now?: () => number; + sleep?: (ms: number) => Promise; +}): DatabaseService { + const { createDelegate, logger } = options; + const timeoutMs = options.timeoutMs ?? DATABASE_STARTUP_TIMEOUT_MS; + const intervalMs = options.intervalMs ?? DATABASE_RETRY_INTERVAL_MS; + const now = options.now ?? (() => Date.now()); + const sleep = options.sleep ?? wait; + + const first = createDelegate(); + + return { + get migrations() { + return first.migrations; + }, + async getClient() { + const deadline = now() + timeoutMs; + let delegate = first; + for (let attempt = 1; ; attempt++) { + try { + return await delegate.getClient(); + } catch (error) { + if (now() >= deadline) { + throw error; + } + logger.warn( + `Database is not reachable, retrying in ${intervalMs}ms (attempt ${attempt})`, + error as Error, + ); + await sleep(intervalMs); + delegate = createDelegate(); + } + } + }, + }; +} + +export function createDatabaseManager( + config: RootConfigService, + deps: { + rootLifecycle: RootLifecycleService; + rootLogger: LoggerService; + }, +): DatabaseManager { + const resolved = config.getOptional('backend.database') + ? config + : new ConfigReader({ + backend: { + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, + }); + return DatabaseManager.fromConfig(resolved, deps); +} + +export const retryingDatabaseServiceFactory = createServiceFactory({ + service: coreServices.database, + deps: { + config: coreServices.rootConfig, + lifecycle: coreServices.lifecycle, + logger: coreServices.logger, + pluginMetadata: coreServices.pluginMetadata, + rootLifecycle: coreServices.rootLifecycle, + rootLogger: coreServices.rootLogger, + }, + async factory({ + config, + lifecycle, + logger, + pluginMetadata, + rootLifecycle, + rootLogger, + }) { + const pluginId = pluginMetadata.getId(); + const createDelegate = () => + createDatabaseManager(config, { rootLifecycle, rootLogger }).forPlugin( + pluginId, + { lifecycle, logger }, + ); + return createRetryingDatabaseService({ createDelegate, logger }); + }, +}); diff --git a/packages/backend/src/services/rootHealth.test.ts b/packages/backend/src/services/rootHealth.test.ts new file mode 100644 index 0000000..59ec241 --- /dev/null +++ b/packages/backend/src/services/rootHealth.test.ts @@ -0,0 +1,104 @@ +import type { + LifecycleServiceStartupHook, + RootLifecycleService, +} from '@backstage/backend-plugin-api'; +import { + StartupDeadlineRootHealthService, + STARTUP_LIVENESS_TIMEOUT_MS, +} from './rootHealth'; + +const createLifecycle = () => { + const startupHooks: LifecycleServiceStartupHook[] = []; + const beforeShutdownHooks: Array<() => void | Promise> = []; + const lifecycle: RootLifecycleService = { + addStartupHook: hook => { + startupHooks.push(hook); + }, + addShutdownHook: () => {}, + addBeforeShutdownHook: hook => { + beforeShutdownHooks.push(hook); + }, + }; + return { + lifecycle, + startup: async () => { + for (const hook of startupHooks) { + await hook(); + } + }, + beforeShutdown: async () => { + for (const hook of beforeShutdownHooks) { + await hook(); + } + }, + }; +}; + +describe('StartupDeadlineRootHealthService', () => { + it('serves liveness while the backend is still within its startup deadline', async () => { + let clock = 1000; + const { lifecycle } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + clock += 59_999; + + await expect(health.getLiveness()).resolves.toEqual({ + status: 200, + payload: { status: 'ok' }, + }); + await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 }); + }); + + it('fails liveness once the backend has never become ready in time', async () => { + let clock = 1000; + const { lifecycle } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + clock += 60_000; + + await expect(health.getLiveness()).resolves.toMatchObject({ status: 500 }); + }); + + it('keeps liveness passing indefinitely once the backend has started', async () => { + let clock = 1000; + const { lifecycle, startup } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ + lifecycle, + timeoutMs: 60_000, + now: () => clock, + }); + + await startup(); + clock += 10 * 60_000; + + await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 }); + await expect(health.getReadiness()).resolves.toEqual({ + status: 200, + payload: { status: 'ok' }, + }); + }); + + it('reports not ready while shutting down but stays live', async () => { + const { lifecycle, startup, beforeShutdown } = createLifecycle(); + const health = new StartupDeadlineRootHealthService({ lifecycle }); + + await startup(); + await beforeShutdown(); + + await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 }); + await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 }); + }); + + it('defaults to a bounded startup deadline', () => { + expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeGreaterThan(0); + expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeLessThanOrEqual(600_000); + }); +}); diff --git a/packages/backend/src/services/rootHealth.ts b/packages/backend/src/services/rootHealth.ts new file mode 100644 index 0000000..4f84578 --- /dev/null +++ b/packages/backend/src/services/rootHealth.ts @@ -0,0 +1,76 @@ +import { + coreServices, + createServiceFactory, + type RootHealthService, + type RootLifecycleService, +} from '@backstage/backend-plugin-api'; + +export const STARTUP_LIVENESS_TIMEOUT_MS = 300_000; + +type StartupState = 'init' | 'up' | 'down'; + +export class StartupDeadlineRootHealthService implements RootHealthService { + #state: StartupState = 'init'; + readonly #startedAt: number; + readonly #timeoutMs: number; + readonly #now: () => number; + + constructor(options: { + lifecycle: RootLifecycleService; + timeoutMs?: number; + now?: () => number; + }) { + this.#now = options.now ?? (() => Date.now()); + this.#timeoutMs = options.timeoutMs ?? STARTUP_LIVENESS_TIMEOUT_MS; + this.#startedAt = this.#now(); + + options.lifecycle.addStartupHook(() => { + this.#state = 'up'; + }); + options.lifecycle.addBeforeShutdownHook(() => { + this.#state = 'down'; + }); + } + + async getLiveness() { + if ( + this.#state === 'init' && + this.#now() - this.#startedAt >= this.#timeoutMs + ) { + return { + status: 500, + payload: { + status: 'error', + message: 'Backend did not become ready within the startup deadline', + }, + }; + } + return { status: 200, payload: { status: 'ok' } }; + } + + async getReadiness() { + if (this.#state === 'init') { + return { + status: 503, + payload: { status: 'error', message: 'Backend has not started yet' }, + }; + } + if (this.#state === 'down') { + return { + status: 503, + payload: { status: 'error', message: 'Backend is shutting down' }, + }; + } + return { status: 200, payload: { status: 'ok' } }; + } +} + +export const startupDeadlineRootHealthServiceFactory = createServiceFactory({ + service: coreServices.rootHealth, + deps: { + lifecycle: coreServices.rootLifecycle, + }, + async factory({ lifecycle }) { + return new StartupDeadlineRootHealthService({ lifecycle }); + }, +});