fix: Restart Backstage when startup fails to reach ready

packages/backend/src/index.ts

@@ -7,9 +7,14 @@
  */
 
 import { createBackend } from '@backstage/backend-defaults';
+import { retryingDatabaseServiceFactory } from './services/database';
+import { startupDeadlineRootHealthServiceFactory } from './services/rootHealth';
 
 const backend = createBackend();
 
+backend.add(retryingDatabaseServiceFactory);
+backend.add(startupDeadlineRootHealthServiceFactory);
+
 backend.add(import('@backstage/plugin-app-backend'));
 backend.add(import('@backstage/plugin-proxy-backend'));
 
@@ -73,4 +78,7 @@ backend.add(import('@backstage/plugin-signals-backend'));
 // mcp actions plugin
 backend.add(import('@backstage/plugin-mcp-actions-backend'));
 
-backend.start();
+backend.start().catch(error => {
+  process.stderr.write(`Backend failed to start: ${error?.stack ?? error}\n`);
+  process.exit(1);
+});

packages/backend/src/services/database.test.ts

@@ -0,0 +1,143 @@
+import type {
+  DatabaseService,
+  LoggerService,
+} from '@backstage/backend-plugin-api';
+import {
+  createRetryingDatabaseService,
+  DATABASE_STARTUP_TIMEOUT_MS,
+} from './database';
+
+const knexStub = { name: 'knex' } as unknown as Awaited<
+  ReturnType<DatabaseService['getClient']>
+>;
+
+const createLogger = (): LoggerService => {
+  const logger: LoggerService = {
+    error: jest.fn(),
+    warn: jest.fn(),
+    info: jest.fn(),
+    debug: jest.fn(),
+    child: () => logger,
+  };
+  return logger;
+};
+
+const createMemoizingDelegateFactory = (
+  connect: () => Promise<Awaited<ReturnType<DatabaseService['getClient']>>>,
+) => {
+  const connects = jest.fn(connect);
+  const createDelegate = jest.fn((): DatabaseService => {
+    let cached: ReturnType<DatabaseService['getClient']> | undefined;
+    return {
+      getClient() {
+        if (!cached) {
+          cached = connects();
+        }
+        return cached;
+      },
+    };
+  });
+  return { createDelegate, connects };
+};
+
+describe('createRetryingDatabaseService', () => {
+  it('returns the client without waiting when the database is reachable', async () => {
+    const getClient = jest.fn().mockResolvedValue(knexStub);
+    const sleep = jest.fn().mockResolvedValue(undefined);
+
+    const service = createRetryingDatabaseService({
+      createDelegate: () => ({ getClient }),
+      logger: createLogger(),
+      sleep,
+    });
+
+    await expect(service.getClient()).resolves.toBe(knexStub);
+    expect(getClient).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it('rides out a brief outage and then connects', async () => {
+    let clock = 0;
+    const connect = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres'))
+      .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres'))
+      .mockResolvedValue(knexStub);
+    const sleep = jest.fn(async (ms: number) => {
+      clock += ms;
+    });
+
+    const service = createRetryingDatabaseService({
+      createDelegate: () => ({ getClient: connect }),
+      logger: createLogger(),
+      timeoutMs: 30_000,
+      intervalMs: 1_000,
+      now: () => clock,
+      sleep,
+    });
+
+    await expect(service.getClient()).resolves.toBe(knexStub);
+    expect(connect).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries against a delegate that caches its first failure', async () => {
+    let clock = 0;
+    const { createDelegate, connects } = createMemoizingDelegateFactory(
+      jest
+        .fn()
+        .mockRejectedValueOnce(new Error('getaddrinfo ENOTFOUND postgres'))
+        .mockResolvedValue(knexStub),
+    );
+    const sleep = jest.fn(async (ms: number) => {
+      clock += ms;
+    });
+
+    const service = createRetryingDatabaseService({
+      createDelegate,
+      logger: createLogger(),
+      timeoutMs: 30_000,
+      intervalMs: 1_000,
+      now: () => clock,
+      sleep,
+    });
+
+    await expect(service.getClient()).resolves.toBe(knexStub);
+    expect(connects).toHaveBeenCalledTimes(2);
+    expect(createDelegate).toHaveBeenCalledTimes(2);
+  });
+
+  it('gives up with the last error once the retry window closes', async () => {
+    let clock = 0;
+    const error = new Error('getaddrinfo ENOTFOUND postgres');
+    const connect = jest.fn().mockRejectedValue(error);
+    const sleep = jest.fn(async (ms: number) => {
+      clock += ms;
+    });
+
+    const service = createRetryingDatabaseService({
+      createDelegate: () => ({ getClient: connect }),
+      logger: createLogger(),
+      timeoutMs: 5_000,
+      intervalMs: 1_000,
+      now: () => clock,
+      sleep,
+    });
+
+    await expect(service.getClient()).rejects.toThrow(error);
+    expect(connect).toHaveBeenCalledTimes(6);
+  });
+
+  it('passes the migration settings of the wrapped service through', () => {
+    const service = createRetryingDatabaseService({
+      createDelegate: () => ({
+        getClient: jest.fn().mockResolvedValue(knexStub),
+        migrations: { skip: true },
+      }),
+      logger: createLogger(),
+    });
+
+    expect(service.migrations).toEqual({ skip: true });
+    expect(DATABASE_STARTUP_TIMEOUT_MS).toBeGreaterThan(0);
+  });
+});

packages/backend/src/services/database.ts

@@ -0,0 +1,105 @@
+import { DatabaseManager } from '@backstage/backend-defaults/database';
+import {
+  coreServices,
+  createServiceFactory,
+  type DatabaseService,
+  type LoggerService,
+  type RootConfigService,
+  type RootLifecycleService,
+} from '@backstage/backend-plugin-api';
+import { ConfigReader } from '@backstage/config';
+
+export const DATABASE_STARTUP_TIMEOUT_MS = 30_000;
+export const DATABASE_RETRY_INTERVAL_MS = 1_000;
+
+const wait = (ms: number) =>
+  new Promise<void>(resolve => {
+    setTimeout(resolve, ms);
+  });
+
+export function createRetryingDatabaseService(options: {
+  createDelegate: () => DatabaseService;
+  logger: LoggerService;
+  timeoutMs?: number;
+  intervalMs?: number;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}): DatabaseService {
+  const { createDelegate, logger } = options;
+  const timeoutMs = options.timeoutMs ?? DATABASE_STARTUP_TIMEOUT_MS;
+  const intervalMs = options.intervalMs ?? DATABASE_RETRY_INTERVAL_MS;
+  const now = options.now ?? (() => Date.now());
+  const sleep = options.sleep ?? wait;
+
+  const first = createDelegate();
+
+  return {
+    get migrations() {
+      return first.migrations;
+    },
+    async getClient() {
+      const deadline = now() + timeoutMs;
+      let delegate = first;
+      for (let attempt = 1; ; attempt++) {
+        try {
+          return await delegate.getClient();
+        } catch (error) {
+          if (now() >= deadline) {
+            throw error;
+          }
+          logger.warn(
+            `Database is not reachable, retrying in ${intervalMs}ms (attempt ${attempt})`,
+            error as Error,
+          );
+          await sleep(intervalMs);
+          delegate = createDelegate();
+        }
+      }
+    },
+  };
+}
+
+export function createDatabaseManager(
+  config: RootConfigService,
+  deps: {
+    rootLifecycle: RootLifecycleService;
+    rootLogger: LoggerService;
+  },
+): DatabaseManager {
+  const resolved = config.getOptional('backend.database')
+    ? config
+    : new ConfigReader({
+        backend: {
+          database: { client: 'better-sqlite3', connection: ':memory:' },
+        },
+      });
+  return DatabaseManager.fromConfig(resolved, deps);
+}
+
+export const retryingDatabaseServiceFactory = createServiceFactory({
+  service: coreServices.database,
+  deps: {
+    config: coreServices.rootConfig,
+    lifecycle: coreServices.lifecycle,
+    logger: coreServices.logger,
+    pluginMetadata: coreServices.pluginMetadata,
+    rootLifecycle: coreServices.rootLifecycle,
+    rootLogger: coreServices.rootLogger,
+  },
+  async factory({
+    config,
+    lifecycle,
+    logger,
+    pluginMetadata,
+    rootLifecycle,
+    rootLogger,
+  }) {
+    const pluginId = pluginMetadata.getId();
+    const createDelegate = () =>
+      createDatabaseManager(config, { rootLifecycle, rootLogger }).forPlugin(
+        pluginId,
+        { lifecycle, logger },
+      );
+    return createRetryingDatabaseService({ createDelegate, logger });
+  },
+});

packages/backend/src/services/rootHealth.test.ts

@@ -0,0 +1,104 @@
+import type {
+  LifecycleServiceStartupHook,
+  RootLifecycleService,
+} from '@backstage/backend-plugin-api';
+import {
+  StartupDeadlineRootHealthService,
+  STARTUP_LIVENESS_TIMEOUT_MS,
+} from './rootHealth';
+
+const createLifecycle = () => {
+  const startupHooks: LifecycleServiceStartupHook[] = [];
+  const beforeShutdownHooks: Array<() => void | Promise<void>> = [];
+  const lifecycle: RootLifecycleService = {
+    addStartupHook: hook => {
+      startupHooks.push(hook);
+    },
+    addShutdownHook: () => {},
+    addBeforeShutdownHook: hook => {
+      beforeShutdownHooks.push(hook);
+    },
+  };
+  return {
+    lifecycle,
+    startup: async () => {
+      for (const hook of startupHooks) {
+        await hook();
+      }
+    },
+    beforeShutdown: async () => {
+      for (const hook of beforeShutdownHooks) {
+        await hook();
+      }
+    },
+  };
+};
+
+describe('StartupDeadlineRootHealthService', () => {
+  it('serves liveness while the backend is still within its startup deadline', async () => {
+    let clock = 1000;
+    const { lifecycle } = createLifecycle();
+    const health = new StartupDeadlineRootHealthService({
+      lifecycle,
+      timeoutMs: 60_000,
+      now: () => clock,
+    });
+
+    clock += 59_999;
+
+    await expect(health.getLiveness()).resolves.toEqual({
+      status: 200,
+      payload: { status: 'ok' },
+    });
+    await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 });
+  });
+
+  it('fails liveness once the backend has never become ready in time', async () => {
+    let clock = 1000;
+    const { lifecycle } = createLifecycle();
+    const health = new StartupDeadlineRootHealthService({
+      lifecycle,
+      timeoutMs: 60_000,
+      now: () => clock,
+    });
+
+    clock += 60_000;
+
+    await expect(health.getLiveness()).resolves.toMatchObject({ status: 500 });
+  });
+
+  it('keeps liveness passing indefinitely once the backend has started', async () => {
+    let clock = 1000;
+    const { lifecycle, startup } = createLifecycle();
+    const health = new StartupDeadlineRootHealthService({
+      lifecycle,
+      timeoutMs: 60_000,
+      now: () => clock,
+    });
+
+    await startup();
+    clock += 10 * 60_000;
+
+    await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 });
+    await expect(health.getReadiness()).resolves.toEqual({
+      status: 200,
+      payload: { status: 'ok' },
+    });
+  });
+
+  it('reports not ready while shutting down but stays live', async () => {
+    const { lifecycle, startup, beforeShutdown } = createLifecycle();
+    const health = new StartupDeadlineRootHealthService({ lifecycle });
+
+    await startup();
+    await beforeShutdown();
+
+    await expect(health.getReadiness()).resolves.toMatchObject({ status: 503 });
+    await expect(health.getLiveness()).resolves.toMatchObject({ status: 200 });
+  });
+
+  it('defaults to a bounded startup deadline', () => {
+    expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeGreaterThan(0);
+    expect(STARTUP_LIVENESS_TIMEOUT_MS).toBeLessThanOrEqual(600_000);
+  });
+});