From a9c4c3e9def1e943f9da8e8ff756734dde99fe74 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 9 Sep 2026 10:33:38 -0400 Subject: [PATCH 1/3] fix(health): check only wizard dependencies --- README.md | 76 +- .../shared/__tests__/bootstrap-health.test.ts | 267 ++++ src/lib/agent/runner/shared/bootstrap.ts | 88 +- .../health-checks/__tests__/endpoints.test.ts | 276 ++++ .../__tests__/health-checks.test.ts | 1383 ++--------------- src/lib/health-checks/endpoints.ts | 320 ++-- src/lib/health-checks/incidentio.ts | 167 -- src/lib/health-checks/index.ts | 22 +- src/lib/health-checks/readiness.ts | 354 +---- src/lib/health-checks/statuspage.ts | 144 -- src/lib/health-checks/testme.md | 77 +- src/lib/health-checks/types.ts | 29 +- src/lib/programs/shared/health-check-step.ts | 22 +- src/ui/logging-ui.ts | 5 +- src/ui/tui/__tests__/ink-ui-health.test.ts | 85 + src/ui/tui/__tests__/programs.test.ts | 42 +- src/ui/tui/__tests__/router.test.ts | 76 + src/ui/tui/__tests__/store.test.ts | 4 +- src/ui/tui/components/ServiceHealthList.tsx | 59 +- src/ui/tui/ink-ui.ts | 6 +- .../tui/playground/demos/HealthCheckDemo.tsx | 126 +- src/ui/tui/router.ts | 25 +- .../tui/screens/health/HealthCheckScreen.tsx | 134 +- src/ui/tui/store.ts | 22 +- src/utils/anthropic-status.ts | 73 - 25 files changed, 1327 insertions(+), 2555 deletions(-) create mode 100644 src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts create mode 100644 src/lib/health-checks/__tests__/endpoints.test.ts delete mode 100644 src/lib/health-checks/incidentio.ts delete mode 100644 src/lib/health-checks/statuspage.ts create mode 100644 src/ui/tui/__tests__/ink-ui-health.test.ts delete mode 100644 src/utils/anthropic-status.ts diff --git a/README.md b/README.md index 49e39cfeb..432902798 100644 --- a/README.md +++ b/README.md @@ -598,51 +598,37 @@ To make your version of a tool usable with a one-line `npx` command: # Health checks -`src/lib/health-checks/` checks external status pages and PostHog-owned -services before the wizard runs to decide whether it can proceed. The entry -point is `evaluateWizardReadiness()`, which returns one of three values: - -| Decision | Meaning | -| ------------------- | --------------------------------------------------------------- | -| `yes` | All services healthy — proceed normally. | -| `yes_with_warnings` | Some services degraded but no critical dependency is down. | -| `no` | A critical dependency is down or degraded — do not run. | - -### Module layout - -| File | Responsibility | -| --- | --- | -| `types.ts` | Enums, interfaces (`ServiceHealthStatus`, `AllServicesHealth`, etc.) | -| `statuspage.ts` | Statuspage.io v2 API helpers + checks for Anthropic, PostHog, GitHub, npm, Cloudflare | -| `endpoints.ts` | Direct endpoint checks for MCP (`/`) and the skills origins (`skill-menu.json` on GitHub Releases + the AWS mirror) | -| `readiness.ts` | `checkAllExternalServices`, `evaluateWizardReadiness`, readiness config | -| `index.ts` | Barrel re-export | -| `testme.md` | Test running instructions and endpoint reference | - -## What blocks a run - -The `DEFAULT_WIZARD_READINESS_CONFIG` in `readiness.ts` controls this. It has -two arrays: - -- **`downBlocksRun`** — if any of these report status **Down**, readiness is - **No**. -- **`degradedBlocksRun`** — if any of these report **Degraded** (or worse), - readiness is **No**. - -### Current defaults - -```ts -downBlocksRun: ['anthropic', 'npmOverall', 'mcp', 'skillsOrigin'], -degradedBlocksRun: ['anthropic'], -``` - -`skillsOrigin` is one entry covering two origins: skills are published to -GitHub Releases and an AWS mirror under the same filenames, and downloads fail -over between them (`src/lib/fetch-retry.ts`). Both are probed in parallel, so -the key only reports **Down** when neither origin answers — a GitHub Releases -outage on its own doesn't block a run, including a 403 or 404, which is as -often about the origin (expired asset redirect, blocked region, a publish that -reached one origin and not the other) as about the asset. +`src/lib/health-checks/` checks the dependencies the Wizard actually uses: + +- **Skills downloads:** fetch and validate `skill-menu.json` from GitHub + Releases and the AWS mirror. Either source working is healthy. A single-origin + outage does not warn or interrupt the run. Local context-mill runs check their + local server instead. +- **LLM gateway:** after login and token mint, check `/readyz` on the exact + `gateway_url` returned by the backend. This works with regional, custom, and + local gateways without hardcoding the old gateway hostname. + +Anthropic, GitHub, npm, Cloudflare, MCP, and general PostHog status pages are +not queried or displayed. An individual provider outage does not establish a +gateway outage: provider routing and fallback belong to the gateway. + +The shared health screen checks skills before login. Bootstrap reuses that +result and checks the minted gateway before starting the agent. Signup uses the +same policy. Programs without a health screen skip these advisory checks. + +A failed gateway probe or unavailable skills sources interrupts the run. Network +failures are labelled as connection problems, without claiming a confirmed +service outage. Inconclusive checks do not produce warnings. Users can continue +past a gateway warning or download available skills to use with another agent; +when neither skills origin works, the screen offers exit and manual setup docs. +CI reports failures and continues, as before. + +| File | Responsibility | +| -------------- | ---------------------------------------------------------- | +| `types.ts` | Health results for the gateway and skills downloads | +| `endpoints.ts` | Direct probes, bounded retries, and skills mirror fallback | +| `readiness.ts` | Aggregate checks and the two-dependency outage policy | +| `testme.md` | Focused test instructions and endpoint reference | ## Smoke test helper (`scripts/smoke-test-ci.sh`) diff --git a/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts b/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts new file mode 100644 index 000000000..fa25867ce --- /dev/null +++ b/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts @@ -0,0 +1,267 @@ +import { bootstrapProgram } from '../bootstrap'; +import { authenticate, refreshAccessTokenIfNeeded } from '../authenticate'; +import { buildSession, type Credentials } from '@lib/wizard-session'; +import { HostResolution } from '@lib/host-resolution'; +import { gatewayAuth } from '@lib/gateway-session'; +import { createTriageLLMProvider } from '@lib/agent/triage-provider'; +import { + checkLlmGatewayHealth, + checkSkillsOriginHealth, +} from '@lib/health-checks/endpoints'; +import { + WizardReadiness, + type WizardReadinessResult, +} from '@lib/health-checks/readiness'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; +import { wizardAbort } from '@utils/wizard-abort'; +import type { ProgramConfig } from '@lib/programs/program-step'; +import type { ProgramRun } from '../types'; + +const ui = vi.hoisted(() => ({ + showBlockingOutage: vi.fn<() => Promise>(), + waitForAiOptIn: vi.fn<() => Promise>(), + waitForGate: vi.fn<() => Promise>(), +})); + +vi.mock('@ui', () => ({ getUI: () => ui })); +vi.mock('../authenticate', () => ({ + authenticate: vi.fn(), + refreshAccessTokenIfNeeded: vi.fn(), +})); +vi.mock('@lib/gateway-session', () => ({ gatewayAuth: vi.fn() })); +vi.mock('@lib/health-checks/endpoints', () => ({ + checkLlmGatewayHealth: vi.fn(), + checkSkillsOriginHealth: vi.fn(), +})); +vi.mock('@lib/agent/triage-provider', () => ({ + createTriageLLMProvider: vi.fn(), +})); +vi.mock('@lib/programs/posthog-integration/detect', () => ({ + maybeStampAiSdkDetected: vi.fn(), +})); +vi.mock('@lib/agent/runner/switchboard', () => ({ + resolveHarness: () => ({ harness: 'anthropic' }), +})); +vi.mock('@lib/agent/agent-interface', () => ({ buildRunTags: () => ({}) })); +vi.mock('@lib/agent/claude-settings', () => ({ + checkAllSettingsConflicts: () => [], + backupAndFixClaudeSettings: vi.fn(), + classifySettingsConflicts: vi.fn(), +})); +vi.mock('@utils/analytics', () => ({ + analytics: { + build: 'test', + runId: 'test-run', + wizardCapture: vi.fn(), + getAllFlagsForWizard: () => Promise.resolve({}), + getWizardFlagPayloads: () => ({}), + }, +})); +vi.mock('@utils/debug', () => ({ + initLogFile: vi.fn(), + logToFile: vi.fn(), + enableDebugLogs: vi.fn(), +})); +vi.mock('@utils/wizard-abort', () => ({ wizardAbort: vi.fn() })); + +const run: ProgramRun = { + integrationLabel: 'health-test', + spinnerMessage: 'Running', + successMessage: 'Done', + estimatedDurationMinutes: 1, + reportFile: 'report.md', + docsUrl: 'https://example.com/docs', +}; + +function program(hasHealthScreen = true): ProgramConfig { + return { + id: 'health-test', + description: 'Health lifecycle test', + steps: [ + ...(hasHealthScreen + ? [{ id: 'health', label: 'Health', screenId: 'health-check' }] + : []), + { id: 'auth', label: 'Auth', screenId: 'auth' }, + { id: 'run', label: 'Run', screenId: 'run' }, + ], + }; +} + +function credentials(accessToken = 'test-access-token'): Credentials { + return { + accessToken, + projectApiKey: 'test-project-key', + host: HostResolution.fromRegion('us'), + projectId: 1, + }; +} + +function preflight(status: ServiceHealthStatus): WizardReadinessResult { + return { + decision: + status === ServiceHealthStatus.Healthy + ? WizardReadiness.Yes + : WizardReadiness.No, + health: { skillsOrigin: { status } }, + reasons: [], + }; +} + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +describe('bootstrap health lifecycle', () => { + beforeEach(() => { + vi.resetAllMocks(); + ui.showBlockingOutage.mockResolvedValue(); + ui.waitForAiOptIn.mockResolvedValue(); + ui.waitForGate.mockResolvedValue(); + vi.mocked(authenticate).mockImplementation((session) => { + session.credentials = credentials(); + return Promise.resolve(); + }); + vi.mocked(refreshAccessTokenIfNeeded).mockResolvedValue(); + vi.mocked(gatewayAuth).mockResolvedValue({ + gatewayUrl: 'https://gateway.example.com', + token: 'test-scoped-token', + refreshAtMs: Date.now() + 60_000, + }); + vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ + status: ServiceHealthStatus.Healthy, + }); + vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ + status: ServiceHealthStatus.Healthy, + }); + }); + + it.each([ + 'https://gateway.eu.example.com', + 'http://localhost:8766', + 'https://custom.example.com/nested/gateway', + ])( + 'checks the minted URL %s despite a cached healthy preflight', + async (url) => { + const session = buildSession({}); + session.readinessResult = preflight(ServiceHealthStatus.Healthy); + const mint = deferred(); + vi.mocked(gatewayAuth).mockImplementation(async () => { + await mint.promise; + return { + gatewayUrl: url, + token: 'test-scoped-token', + refreshAtMs: 60_000, + }; + }); + + const boot = bootstrapProgram(session, run, program()); + await vi.waitFor(() => expect(gatewayAuth).toHaveBeenCalledOnce()); + expect(authenticate).toHaveBeenCalledOnce(); + expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); + + mint.resolve(); + await boot; + + expect(checkLlmGatewayHealth).toHaveBeenCalledExactlyOnceWith(url); + expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); + expect(ui.showBlockingOutage).not.toHaveBeenCalled(); + }, + ); + + it.each([false, true])( + 'skips advisory checks without a health screen (signup=%s)', + async (signup) => { + await bootstrapProgram(buildSession({ signup }), run, program(false)); + + expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); + expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); + expect(ui.showBlockingOutage).not.toHaveBeenCalled(); + expect(gatewayAuth).toHaveBeenCalledOnce(); + }, + ); + + it.each([false, true])( + 'waits for a skills outage dismissal before auth, then checks the gateway (signup=%s)', + async (signup) => { + const dismissal = deferred(); + ui.showBlockingOutage.mockReturnValueOnce(dismissal.promise); + vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ + status: ServiceHealthStatus.Down, + }); + + const boot = bootstrapProgram(buildSession({ signup }), run, program()); + await vi.waitFor(() => + expect(ui.showBlockingOutage).toHaveBeenCalledOnce(), + ); + expect(authenticate).not.toHaveBeenCalled(); + expect(gatewayAuth).not.toHaveBeenCalled(); + + dismissal.resolve(); + await boot; + + expect(checkSkillsOriginHealth).toHaveBeenCalledOnce(); + expect(checkLlmGatewayHealth).toHaveBeenCalledExactlyOnceWith( + 'https://gateway.example.com', + ); + expect(ui.showBlockingOutage).toHaveBeenCalledOnce(); + expect(wizardAbort).not.toHaveBeenCalled(); + }, + ); + + it('does not repeat a cached, dismissed skills warning when the gateway is healthy', async () => { + const session = buildSession({}); + session.readinessResult = preflight(ServiceHealthStatus.Down); + + await bootstrapProgram(session, run, program()); + + expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); + expect(checkLlmGatewayHealth).toHaveBeenCalledOnce(); + expect(ui.showBlockingOutage).not.toHaveBeenCalled(); + }); + + it('pauses a new gateway outage, then refreshes credentials and resumes after dismissal', async () => { + const session = buildSession({}); + session.readinessResult = preflight(ServiceHealthStatus.Down); + const dismissal = deferred(); + ui.showBlockingOutage.mockReturnValueOnce(dismissal.promise); + vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ + status: ServiceHealthStatus.NoConnection, + }); + vi.mocked(refreshAccessTokenIfNeeded) + .mockResolvedValueOnce() + .mockImplementationOnce((current) => { + current.credentials = credentials('refreshed-access-token'); + return Promise.resolve(); + }); + + const boot = bootstrapProgram(session, run, program()); + await vi.waitFor(() => + expect(ui.showBlockingOutage).toHaveBeenCalledOnce(), + ); + expect(refreshAccessTokenIfNeeded).toHaveBeenCalledOnce(); + expect(createTriageLLMProvider).not.toHaveBeenCalled(); + + dismissal.resolve(); + const result = await boot; + + expect(refreshAccessTokenIfNeeded).toHaveBeenCalledTimes(2); + expect(wizardAbort).not.toHaveBeenCalled(); + expect(result.credentials.accessToken).toBe('refreshed-access-token'); + const resolveTriageAuth = vi.mocked(createTriageLLMProvider).mock + .calls[0]?.[0]; + expect(typeof resolveTriageAuth).toBe('function'); + if (typeof resolveTriageAuth !== 'function') { + throw new Error('Expected a live gateway auth resolver'); + } + await resolveTriageAuth(); + expect(gatewayAuth).toHaveBeenLastCalledWith( + result.credentials.host, + 'refreshed-access-token', + program().id, + ); + }); +}); diff --git a/src/lib/agent/runner/shared/bootstrap.ts b/src/lib/agent/runner/shared/bootstrap.ts index 6c2bd794e..974619e2f 100644 --- a/src/lib/agent/runner/shared/bootstrap.ts +++ b/src/lib/agent/runner/shared/bootstrap.ts @@ -24,9 +24,7 @@ import { import { evaluateWizardReadiness, WizardReadiness, - SIGNUP_WIZARD_READINESS_CONFIG, getBlockingServiceKeys, - SERVICE_LABELS, } from '@lib/health-checks/readiness'; import { enableDebugLogs, logToFile, initLogFile } from '@utils/debug'; import { wizardAbort } from '@utils/wizard-abort'; @@ -112,53 +110,17 @@ export async function bootstrapProgram( `posthog=${session.baseUrl ?? 'region-resolved'}`, ); - // 2. Health check (guarded — skip if TUI already ran it). Only - // programs that declare a health-check screen get pre-flight checks; - // for everything else the checks never fire and never block. + // Pre-auth checks cover skill downloads only. The gateway URL is not known + // until mint; a cached TUI result must not suppress that later check. + // Programs without the health screen skip these advisory checks entirely. const hasHealthCheckScreen = programConfig.steps.some( (s) => s.screenId === 'health-check', ); - if (session.readinessResult) { - logToFile( - `[agent-runner] readiness pre-computed by TUI: decision=${session.readinessResult.decision}` + - `${ - session.outageDismissed ? ' (outage dismissed by user)' : '' - } — skipping re-check`, - ); - } - if (hasHealthCheckScreen && !session.readinessResult) { - logToFile('[agent-runner] evaluating wizard readiness'); - const readinessConfig = session.signup - ? SIGNUP_WIZARD_READINESS_CONFIG - : undefined; - const readiness = await evaluateWizardReadiness(readinessConfig); - logToFile(`[agent-runner] readiness=${readiness.decision}`); - if (readiness.decision === WizardReadiness.No) { - const blockingKeys = getBlockingServiceKeys( - readiness.health, - readinessConfig, - ); - const blockingLabels = blockingKeys.map( - (k) => `${SERVICE_LABELS[k]} (${readiness.health[k].status})`, - ); - logToFile(`[agent-runner] blocked by: ${blockingLabels.join(', ')}`); - - await getUI().showBlockingOutage(readiness); - - // The TUI lets the user continue past an outage; non-interactive runs - // (CI) do the same automatically — the degraded services are reported - // above, but we proceed rather than aborting on a transient upstream blip. - if (!isNonInteractiveEnvironment()) { - await wizardAbort({ - code: ErrorCodes.EnvServiceOutage, - message: - 'Cannot start — external services are down:\n' + - blockingLabels.map((l) => ` - ${l}`).join('\n') + - '\n\nPlease try again later.', - }); - } - } else if (readiness.decision === WizardReadiness.YesWithWarnings) { - getUI().setReadinessWarnings(readiness); + let preflight = session.readinessResult; + if (hasHealthCheckScreen && !preflight) { + preflight = await evaluateWizardReadiness(); + if (preflight.decision === WizardReadiness.No) { + await getUI().showBlockingOutage(preflight); } } @@ -302,17 +264,31 @@ export async function bootstrapProgram( // The agent can't swap tokens mid-run, so freshness is measured after every park above, right before the mint. await refreshAccessTokenIfNeeded(session); - // Credentials (incl. the resolved host family and its MCP url) live on - // `session.credentials`; narrow once at this boundary — `authenticate` above - // set them — so downstream readers get a non-null type without asserting. - const credentials = session.credentials!; + // Read live credentials so a refresh after an outage dismissal is used by + // later mints, too. No agent or skill triage starts before the initial mint. + const currentGatewayAuth = () => { + const credentials = session.credentials!; + return gatewayAuth( + credentials.host, + credentials.accessToken, + programConfig.id, + ); + }; + const auth = await currentGatewayAuth(); - // Mint now so a refusal fails the boot before any agent starts. Later - // readers re-resolve through the cache, which re-mints past the refresh - // point. - const currentGatewayAuth = () => - gatewayAuth(credentials.host, credentials.accessToken, programConfig.id); - await currentGatewayAuth(); + if (hasHealthCheckScreen) { + const readiness = await evaluateWizardReadiness({ + gatewayUrl: auth.gatewayUrl, + skillsHealth: preflight?.health.skillsOrigin, + }); + // A previously dismissed skills result must not hide a new gateway outage + // or cause the same skills warning to be displayed a second time. + if (getBlockingServiceKeys(readiness.health).includes('llmGateway')) { + await getUI().showBlockingOutage(readiness); + await refreshAccessTokenIfNeeded(session); + } + } + const credentials = session.credentials!; return { skillsBaseUrl, diff --git a/src/lib/health-checks/__tests__/endpoints.test.ts b/src/lib/health-checks/__tests__/endpoints.test.ts new file mode 100644 index 000000000..38658629d --- /dev/null +++ b/src/lib/health-checks/__tests__/endpoints.test.ts @@ -0,0 +1,276 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + AWS_SKILLS_BASE_URL, + GITHUB_SKILLS_BASE_URL, + LOCAL_SKILLS_BASE_URL, +} from '@lib/constants'; +import { initLocalDev, resetLocalDev } from '@lib/local-dev'; +import { + checkLlmGatewayHealth, + checkSkillsOriginHealth, + fetchEndpointHealth, +} from '../endpoints'; +import { ServiceHealthStatus } from '../types'; + +vi.mock('@utils/debug', () => ({ logToFile: vi.fn() })); +vi.mock('@utils/analytics', () => ({ analytics: { wizardCapture: vi.fn() } })); + +const fetchMock = vi.fn(); +const primaryMenu = `${GITHUB_SKILLS_BASE_URL}/skill-menu.json`; +const fallbackMenu = `${AWS_SKILLS_BASE_URL}/skill-menu.json`; +const validMenu = { + categories: { + integration: [ + { + id: 'posthog-integration', + name: 'Install PostHog', + downloadUrl: 'posthog-integration.zip', + }, + ], + }, +}; +const menuResponse = () => new Response(JSON.stringify(validMenu)); +const httpResponse = (status: number) => new Response(null, { status }); + +async function finish(pending: Promise): Promise { + await vi.runAllTimersAsync(); + return pending; +} + +beforeEach(() => { + vi.useFakeTimers(); + fetchMock.mockReset(); + vi.stubGlobal('fetch', fetchMock); + resetLocalDev(); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + resetLocalDev(); +}); + +describe('fetchEndpointHealth', () => { + it('retries a transient HTTP error and recovers', async () => { + fetchMock + .mockResolvedValueOnce(httpResponse(503)) + .mockResolvedValueOnce(httpResponse(200)); + + const result = await finish(fetchEndpointHealth('https://example.com')); + + expect(result).toEqual({ + status: ServiceHealthStatus.Healthy, + rawIndicator: 'HTTP 200 (attempts=2)', + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); + + it('retains HTTP failure evidence when later attempts lose connection', async () => { + fetchMock + .mockResolvedValueOnce(httpResponse(503)) + .mockRejectedValue(new Error('Connection reset')); + + expect(await finish(fetchEndpointHealth('https://example.com'))).toEqual({ + status: ServiceHealthStatus.Down, + error: 'HTTP 503', + rawIndicator: 'HTTP 503 (attempts=3)', + }); + }); + + it('keeps network-only failures distinct from confirmed downtime', async () => { + fetchMock.mockRejectedValue(new Error('DNS lookup failed')); + + expect(await finish(fetchEndpointHealth('https://example.com'))).toEqual({ + status: ServiceHealthStatus.NoConnection, + error: 'DNS lookup failed', + rawIndicator: 'attempts=3', + }); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('bounds hung requests and aborts every attempt', async () => { + fetchMock.mockImplementation(() => new Promise(() => undefined)); + + const result = await finish( + fetchEndpointHealth('https://example.com', 100), + ); + + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + expect(result.error).toBe('Request timed out after 100ms'); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect( + fetchMock.mock.calls.every(([, init]) => init?.signal?.aborted), + ).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe('checkLlmGatewayHealth', () => { + it.each([ + [ + 'https://ai-gateway.eu.posthog.com', + 'https://ai-gateway.eu.posthog.com/readyz', + ], + ['http://localhost:8080/', 'http://localhost:8080/readyz'], + ['https://gateway.example.com/v1', 'https://gateway.example.com/readyz'], + ])( + 'probes the origin readiness endpoint for %s without a bearer', + async (base, expected) => { + fetchMock.mockResolvedValue(httpResponse(200)); + + expect((await checkLlmGatewayHealth(base)).status).toBe( + ServiceHealthStatus.Healthy, + ); + expect(fetchMock).toHaveBeenCalledWith(expected, { + signal: expect.any(AbortSignal), + redirect: 'follow', + }); + }, + ); +}); + +describe('checkSkillsOriginHealth', () => { + it.each([primaryMenu, fallbackMenu])( + 'stays healthy when only %s serves a usable menu', + async (healthyUrl) => { + fetchMock.mockImplementation((url) => + Promise.resolve( + url === healthyUrl ? menuResponse() : httpResponse(503), + ), + ); + + const pending = checkSkillsOriginHealth(); + // Both origin requests start before either retry loop waits. + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + primaryMenu, + fallbackMenu, + ]); + const result = await finish(pending); + + expect(result).toEqual({ + status: ServiceHealthStatus.Healthy, + rawIndicator: 'HTTP 200', + }); + expect(result.error).toBeUndefined(); + expect(result.rawIndicator).not.toMatch(/unavailable|degraded/i); + }, + ); + + it('reports downtime only after both sources fail', async () => { + fetchMock.mockImplementation(() => Promise.resolve(httpResponse(503))); + + const result = await finish(checkSkillsOriginHealth()); + + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toBe('Both skill download sources are unavailable'); + expect(result.rawIndicator).toContain('attempts=3'); + expect(fetchMock).toHaveBeenCalledTimes(6); + }); + + it('reports no connection when neither origin can be reached', async () => { + fetchMock.mockRejectedValue(new Error('Offline')); + + expect((await finish(checkSkillsOriginHealth())).status).toBe( + ServiceHealthStatus.NoConnection, + ); + }); + + it('maps pinned releases to the same AWS release', async () => { + fetchMock.mockImplementation(() => Promise.resolve(menuResponse())); + + await checkSkillsOriginHealth( + 'https://github.com/PostHog/context-mill/releases/download/v1.2.3', + ); + + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://github.com/PostHog/context-mill/releases/download/v1.2.3/skill-menu.json', + 'https://context-mill.posthog.com/v1.2.3/skill-menu.json', + ]); + }); + + it.each([ + 'GitHub temporarily unavailable', + JSON.stringify({ status: 'ok' }), + JSON.stringify({ categories: {} }), + JSON.stringify({ categories: { integration: [{ id: 'incomplete' }] } }), + JSON.stringify({ categories: { integration: 'not an array' } }), + ])('rejects unusable HTTP 200 skill menus: %s', async (body) => { + fetchMock.mockImplementation(() => Promise.resolve(new Response(body))); + + expect((await finish(checkSkillsOriginHealth())).status).toBe( + ServiceHealthStatus.Down, + ); + expect(fetchMock).toHaveBeenCalledTimes(6); + }); + + it('uses the mirror when the primary HTTP 200 contains invalid JSON', async () => { + fetchMock.mockImplementation((url) => + Promise.resolve( + url === primaryMenu ? new Response('not JSON') : menuResponse(), + ), + ); + + expect((await finish(checkSkillsOriginHealth())).status).toBe( + ServiceHealthStatus.Healthy, + ); + }); + + it('retries failures while reading a successful response body', async () => { + fetchMock.mockImplementation(() => + Promise.resolve({ + status: 200, + json: () => Promise.reject(new Error('Body stream interrupted')), + } as unknown as Response), + ); + + const result = await finish(checkSkillsOriginHealth(LOCAL_SKILLS_BASE_URL)); + + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toBe('Body stream interrupted'); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('keeps the timeout active until the skill menu body finishes', async () => { + fetchMock.mockImplementation(() => + Promise.resolve({ + status: 200, + json: () => new Promise(() => undefined), + } as unknown as Response), + ); + + const result = await finish(checkSkillsOriginHealth(LOCAL_SKILLS_BASE_URL)); + + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toBe('Request timed out after 5000ms'); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect( + fetchMock.mock.calls.every(([, init]) => init?.signal?.aborted), + ).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }); + + it('honors the local context-mill target without probing production', async () => { + initLocalDev({ localContextMill: true }); + fetchMock.mockImplementation(() => Promise.resolve(httpResponse(503))); + + expect((await finish(checkSkillsOriginHealth())).status).toBe( + ServiceHealthStatus.Down, + ); + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual( + Array(3).fill(`${LOCAL_SKILLS_BASE_URL}/skill-menu.json`), + ); + }); + + it('downloads a menu from the configured custom target', async () => { + fetchMock.mockImplementation(() => Promise.resolve(menuResponse())); + + expect( + (await checkSkillsOriginHealth('http://localhost:9000/custom/')).status, + ).toBe(ServiceHealthStatus.Healthy); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe( + 'http://localhost:9000/custom/skill-menu.json', + ); + }); +}); diff --git a/src/lib/health-checks/__tests__/health-checks.test.ts b/src/lib/health-checks/__tests__/health-checks.test.ts index cbafb2074..2a94e5ef9 100644 --- a/src/lib/health-checks/__tests__/health-checks.test.ts +++ b/src/lib/health-checks/__tests__/health-checks.test.ts @@ -1,1291 +1,148 @@ -/** - * Tests for health-checks.ts - * - * Mock data is modelled on live Statuspage.io v2 API responses. - * Statuspage docs: https://metastatuspage.com/api - * - * status.json – page-level rollup with indicator (none | minor | major | critical) - * summary.json – same rollup plus component list; component statuses: - * operational | degraded_performance | partial_outage | major_outage | under_maintenance - * https://support.atlassian.com/statuspage/docs/show-service-status-with-components - - * - * MCP – Cloudflare Worker, GET / returns an HTML landing page (200) - * Source: posthog/services/mcp/src/index.ts - */ - +import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest'; import { checkAllExternalServices, - checkAnthropicHealth, - checkCloudflareComponentHealth, - checkCloudflareOverallHealth, - checkGithubHealth, - checkSkillsOriginHealth, - checkMcpHealth, - checkNpmComponentHealth, - checkNpmOverallHealth, - checkPosthogComponentHealth, - checkPosthogOverallHealth, - resetPosthogHealthCache, - DEFAULT_WIZARD_READINESS_CONFIG, evaluateWizardReadiness, - ServiceHealthStatus, + getBlockingServiceKeys, WizardReadiness, -} from '@lib/health-checks/index'; -import { fetchEndpointHealth } from '@lib/health-checks/endpoints'; - -// --------------------------------------------------------------------------- -// Real-world Statuspage.io v2 response factories -// https://metastatuspage.com/api -// --------------------------------------------------------------------------- - -function makeStatuspageStatus(opts: { - pageId: string; - pageName: string; - pageUrl: string; - indicator: 'none' | 'minor' | 'major' | 'critical'; - description: string; -}) { - return { - page: { - id: opts.pageId, - name: opts.pageName, - url: opts.pageUrl, - time_zone: 'Etc/UTC', - updated_at: '2026-03-05T16:03:38.861Z', - }, - status: { - indicator: opts.indicator, - description: opts.description, - }, - }; -} - -function makeStatuspageSummary(opts: { - pageId: string; - pageName: string; - pageUrl: string; - indicator: 'none' | 'minor' | 'major' | 'critical'; - description: string; - components: { - id: string; - name: string; - status: string; - position: number; - description: string | null; - }[]; -}) { - return { - page: { - id: opts.pageId, - name: opts.pageName, - url: opts.pageUrl, - time_zone: 'Etc/UTC', - updated_at: '2026-03-05T16:03:38.861Z', - }, - status: { - indicator: opts.indicator, - description: opts.description, - }, - components: opts.components.map((c) => ({ - ...c, - page_id: opts.pageId, - created_at: '2023-07-11T17:52:24.275Z', - updated_at: '2026-03-04T17:01:29.960Z', - showcase: true, - start_date: '2023-07-11', - group_id: null, - group: false, - only_show_if_degraded: false, - })), - incidents: [], - scheduled_maintenances: [], - }; -} - -// Shapes taken from live GET on 2026-03-05 -const ANTHROPIC_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const GITHUB_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'kctbh9vrtdwd', - pageName: 'GitHub', - pageUrl: 'https://www.githubstatus.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const NPM_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'none', - description: 'All Systems Operational', -}); - -const NPM_SUMMARY_HEALTHY = makeStatuspageSummary({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'none', - description: 'All Systems Operational', - components: [ - { - id: 'mvm98gtxvb9b', - name: 'www.npmjs.com website', - status: 'operational', - position: 1, - description: - 'The ability for users to navigate to or interact with the npm website.', - }, - { - id: 'k1wj10x6gmph', - name: 'Package installation', - status: 'operational', - position: 2, - description: - 'The ability for users to read from the registry so that they can install packages.', - }, - ], -}); - -const CLOUDFLARE_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const CLOUDFLARE_SUMMARY_HEALTHY = makeStatuspageSummary({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'none', - description: 'All Systems Operational', - components: [ - { - id: '1km35smx8p41', - name: 'Cloudflare Sites and Services', - status: 'operational', - position: 1, - description: - 'Sites and services that Cloudflare customers use to interact with the Cloudflare Network', - }, - ], -}); - -// PostHog incident.io v1 API mock data -const POSTHOG_INCIDENTIO_HEALTHY = { - page_title: 'PostHog', - page_url: 'https://www.posthogstatus.com/', - ongoing_incidents: [], - in_progress_maintenances: [], - scheduled_maintenances: [], -}; - -// MCP / landing page (from posthog/services/mcp/src/index.ts + src/static/landing.html) -const MCP_LANDING_HTML = - 'PostHog MCP Server'; - -// --------------------------------------------------------------------------- -// URL constants (must match health-checks.ts) -// --------------------------------------------------------------------------- - -const URLS = { - anthropicStatus: 'https://status.claude.com/api/v2/status.json', - posthogIncidentIo: 'https://www.posthogstatus.com/api/v1/summary', - githubStatus: 'https://www.githubstatus.com/api/v2/status.json', - npmStatus: 'https://status.npmjs.org/api/v2/status.json', - npmSummary: 'https://status.npmjs.org/api/v2/summary.json', - cloudflareStatus: 'https://www.cloudflarestatus.com/api/v2/status.json', - cloudflareSummary: 'https://www.cloudflarestatus.com/api/v2/summary.json', - mcpLanding: 'https://mcp.posthog.com/', - githubSkillMenu: - 'https://github.com/PostHog/context-mill/releases/latest/download/skill-menu.json', - awsSkillMenu: 'https://context-mill.posthog.com/latest/skill-menu.json', -} as const; - -// --------------------------------------------------------------------------- -// Helper to build a default "all healthy" fetch mock -// --------------------------------------------------------------------------- - -const HEALTHY_RESPONSES: Record = - { - [URLS.anthropicStatus]: { - body: JSON.stringify(ANTHROPIC_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.posthogIncidentIo]: { - body: JSON.stringify(POSTHOG_INCIDENTIO_HEALTHY), - contentType: 'application/json', - }, - [URLS.githubStatus]: { - body: JSON.stringify(GITHUB_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.npmStatus]: { - body: JSON.stringify(NPM_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.npmSummary]: { - body: JSON.stringify(NPM_SUMMARY_HEALTHY), - contentType: 'application/json', - }, - [URLS.cloudflareStatus]: { - body: JSON.stringify(CLOUDFLARE_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.cloudflareSummary]: { - body: JSON.stringify(CLOUDFLARE_SUMMARY_HEALTHY), - contentType: 'application/json', - }, - [URLS.mcpLanding]: { - body: MCP_LANDING_HTML, - contentType: 'text/html; charset=utf-8', - }, - [URLS.githubSkillMenu]: { - body: JSON.stringify({ categories: { integration: [] } }), - contentType: 'application/json', - }, - [URLS.awsSkillMenu]: { - body: JSON.stringify({ categories: { integration: [] } }), - contentType: 'application/json', - }, - }; - -function allHealthyFetchMock(url: string | URL | Request): Promise { - const urlStr = - typeof url === 'string' - ? url - : url instanceof URL - ? url.toString() - : url.url; - const entry = HEALTHY_RESPONSES[urlStr]; - if (entry) { - return Promise.resolve( - new Response(entry.body, { - status: 200, - headers: { 'Content-Type': entry.contentType }, - }), - ); - } - return Promise.resolve(new Response('Not found', { status: 404 })); -} - -function overrideFetch(overrides: Record Promise>) { - return (url: string | URL | Request): Promise => { - const urlStr = - typeof url === 'string' - ? url - : url instanceof URL - ? url.toString() - : url.url; - if (overrides[urlStr]) return overrides[urlStr](); - return allHealthyFetchMock(urlStr); - }; -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe('health-checks', () => { - const originalFetch = global.fetch; - +} from '../readiness'; +import { checkLlmGatewayHealth, checkSkillsOriginHealth } from '../endpoints'; +import { ServiceHealthStatus, type AllServicesHealth } from '../types'; + +vi.mock('../endpoints', () => ({ + checkLlmGatewayHealth: vi.fn(), + checkSkillsOriginHealth: vi.fn(), +})); +vi.mock('@utils/debug', () => ({ logToFile: vi.fn() })); + +const healthy = { status: ServiceHealthStatus.Healthy }; +const down = { status: ServiceHealthStatus.Down }; +const unreachable = { status: ServiceHealthStatus.NoConnection }; +const gatewayUrl = 'https://ai-gateway.eu.posthog.com'; + +describe('Wizard dependency health', () => { beforeEach(() => { - vi.restoreAllMocks(); - resetPosthogHealthCache(); - (global as any).fetch = vi.fn(allHealthyFetchMock); - }); - - afterAll(() => { - (global as any).fetch = originalFetch; - }); - - // ----------------------------------------------------------------------- - // Statuspage status.json checks (indicator-based) - // ----------------------------------------------------------------------- - - describe('checkAnthropicHealth', () => { - it('returns healthy for indicator=none ("All Systems Operational")', async () => { - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('none'); - }); - - it('returns degraded for indicator=minor ("Minor Service Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.rawIndicator).toBe('minor'); - }); - - it('returns down for indicator=major ("Partial System Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'major', - description: 'Partial System Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns down for indicator=critical ("Major Service Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'critical', - description: 'Major Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns degraded when statuspage returns HTTP 500', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response('Internal Server Error', { status: 500 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.error).toBe('HTTP 500'); - }); - - it('returns degraded when fetch throws (network failure)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.reject( - new Error('getaddrinfo ENOTFOUND status.claude.com'), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.error).toBe('getaddrinfo ENOTFOUND status.claude.com'); - }); + vi.useFakeTimers(); + vi.mocked(checkLlmGatewayHealth).mockReset().mockResolvedValue(healthy); + vi.mocked(checkSkillsOriginHealth).mockReset().mockResolvedValue(healthy); }); - - describe('checkPosthogOverallHealth', () => { - it('returns healthy when no ongoing incidents', async () => { - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('returns down when an incident has full_outage impact', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: '01KA9JH0ZB14TFA8VD4CFC3AYN', - name: 'Major service outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'US Cloud', - current_status: 'full_outage', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns NoConnection when posthogstatus.com fetch fails with a network error', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.reject(new Error('getaddrinfo ENOTFOUND')), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('returns NoConnection when posthogstatus.com fetch times out', async () => { - const abortError = new Error('aborted'); - abortError.name = 'AbortError'; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => Promise.reject(abortError), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('returns Down when posthogstatus.com returns an HTTP error', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns degraded when an incident has partial_outage impact', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: '01KA9JH0ZB14TFA8VD4CFC3AYN', - name: 'Partial outage', - status: 'investigating', - current_worst_impact: 'partial_outage', - affected_components: [], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - }); + afterEach(() => { + vi.useRealTimers(); }); - describe('checkGithubHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkGithubHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); + it('checks skills before auth without guessing a gateway or reporting a warning', async () => { + const result = await evaluateWizardReadiness(); + expect(result).toEqual({ + decision: WizardReadiness.Yes, + health: { skillsOrigin: healthy }, + reasons: [], }); + expect(checkSkillsOriginHealth).toHaveBeenCalledOnce(); + expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); }); - describe('checkNpmOverallHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkNpmOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); + it('uses the supplied gateway and skills targets', async () => { + const health = await checkAllExternalServices({ + gatewayUrl: 'http://localhost:8080', + skillsBaseUrl: 'http://localhost:8765', }); + expect(checkLlmGatewayHealth).toHaveBeenCalledWith('http://localhost:8080'); + expect(checkSkillsOriginHealth).toHaveBeenCalledWith( + 'http://localhost:8765', + ); + expect(health).toEqual({ llmGateway: healthy, skillsOrigin: healthy }); }); - describe('checkCloudflareOverallHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkCloudflareOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('returns degraded for indicator=minor', async () => { - const body = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.cloudflareStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkCloudflareOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); + it('checks the minted gateway even when skills health was cached before auth', async () => { + vi.mocked(checkLlmGatewayHealth).mockResolvedValue(down); + const result = await evaluateWizardReadiness({ + gatewayUrl, + skillsHealth: healthy, }); + expect(checkLlmGatewayHealth).toHaveBeenCalledWith(gatewayUrl); + expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); + expect(result.decision).toBe(WizardReadiness.No); + expect(getBlockingServiceKeys(result.health)).toEqual(['llmGateway']); }); - // ----------------------------------------------------------------------- - // Statuspage summary.json checks (component-based) - // ----------------------------------------------------------------------- - - describe('checkPosthogComponentHealth', () => { - it('reports healthy when no ongoing incidents', async () => { - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.degradedOrDownComponents).toBeUndefined(); - }); - - it('reports affected components from ongoing incidents', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'US Cloud outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'US Cloud 🇺🇸', - current_status: 'full_outage', - }, - { - id: 'c2', - name: 'Event Ingestion', - group_name: 'US Cloud 🇺🇸', - current_status: 'full_outage', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents).toHaveLength(2); - expect(result.degradedOrDownComponents![0].name).toBe( - 'US Cloud 🇺🇸 — App', - ); - expect(result.degradedOrDownComponents![0].status).toBe( - ServiceHealthStatus.Down, - ); - expect(result.degradedOrDownComponents![1].status).toBe( - ServiceHealthStatus.Down, - ); - }); - - it('reports degraded for degraded_performance components', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'Slowness', - status: 'investigating', - current_worst_impact: 'degraded_performance', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'EU Cloud', - current_status: 'degraded_performance', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents![0].rawStatus).toBe( - 'degraded_performance', - ); - expect(result.degradedOrDownComponents![0].status).toBe( - ServiceHealthStatus.Degraded, - ); - }); + it.each([ + [healthy, healthy, []], + [down, healthy, ['llmGateway']], + [unreachable, healthy, ['llmGateway']], + [healthy, down, ['skillsOrigin']], + [healthy, unreachable, ['skillsOrigin']], + [down, down, ['llmGateway', 'skillsOrigin']], + [unreachable, unreachable, ['llmGateway', 'skillsOrigin']], + ])( + 'only interrupts for failed runtime dependencies (%j, %j)', + async (gateway, skills, blocked) => { + vi.mocked(checkLlmGatewayHealth).mockResolvedValue(gateway); + vi.mocked(checkSkillsOriginHealth).mockResolvedValue(skills); + const result = await evaluateWizardReadiness({ gatewayUrl }); + expect(getBlockingServiceKeys(result.health)).toEqual(blocked); + expect(result.decision).toBe( + blocked.length ? WizardReadiness.No : WizardReadiness.Yes, + ); + }, + ); + + it('does not turn a one-origin fallback into warnings or outage reasons', async () => { + vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ + ...healthy, + rawIndicator: 'HTTP 200 (via aws, github unavailable)', + }); + const result = await evaluateWizardReadiness({ gatewayUrl }); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); }); - describe('checkNpmComponentHealth', () => { - it('reports healthy when all npm components operational', async () => { - const result = await checkNpmComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('reports degraded when "Package installation" has partial_outage', async () => { - const body = makeStatuspageSummary({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'major', - description: 'Partial System Outage', - components: [ - { - id: 'mvm98gtxvb9b', - name: 'www.npmjs.com website', - status: 'operational', - position: 1, - description: null, - }, - { - id: 'k1wj10x6gmph', - name: 'Package installation', - status: 'partial_outage', - position: 2, - description: null, - }, - ], - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.npmSummary]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkNpmComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents![0].name).toBe( - 'Package installation', - ); - }); + it('ignores obsolete provider and status-page results even in a stale health object', () => { + const stale: AllServicesHealth & Record = { + skillsOrigin: healthy, + llmGateway: healthy, + anthropic: down, + posthogOverall: down, + posthogComponents: down, + github: down, + npmOverall: down, + npmComponents: down, + cloudflareOverall: down, + cloudflareComponents: down, + mcp: down, + }; + expect(getBlockingServiceKeys(stale)).toEqual([]); }); - describe('checkCloudflareComponentHealth', () => { - it('reports healthy when Cloudflare components operational', async () => { - const result = await checkCloudflareComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); + it('does not include internal gateway diagnostics in outage reasons', async () => { + vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ + ...down, + error: 'private dependency detail', }); + const result = await evaluateWizardReadiness({ gatewayUrl }); + expect(result.reasons).toEqual(['LLM gateway: down']); }); - // ----------------------------------------------------------------------- - // fetchEndpointHealth (retry + status-taxonomy machinery, probed directly - // against a synthetic URL — no production probe uses the strict defaults - // any more, but every endpoint check shares this loop) - // ----------------------------------------------------------------------- - - describe('fetchEndpointHealth', () => { - const PROBE_URL = 'https://probe.posthog.test/_liveness'; - - it('returns healthy on a 200', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => - Promise.resolve(new Response('ok', { status: 200 })), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 200'); - expect(global.fetch).toHaveBeenCalledWith( - PROBE_URL, - expect.objectContaining({ signal: expect.any(AbortSignal) }), - ); - }); - - it('returns down on 302 — the default predicate stays strict, redirects are not OK', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => - Promise.resolve(new Response(null, { status: 302 })), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 302'); - }); - - it('returns down when the endpoint responds 503 (e.g. deploying)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => - Promise.resolve( - new Response('Service Unavailable', { status: 503 }), - ), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 503'); - }); - - it('returns down when the endpoint responds 502 (bad gateway)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 502'); - }); - - it('returns no-connection on DNS resolution failure (no status-page corroboration)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => - Promise.reject( - new Error('getaddrinfo ENOTFOUND probe.posthog.test'), - ), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toBe('getaddrinfo ENOTFOUND probe.posthog.test'); - }); - - it('returns no-connection on timeout (AbortError)', async () => { - const abortError = new Error('The operation was aborted.'); - abortError.name = 'AbortError'; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => Promise.reject(abortError), - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toBe('Request timed out after 5000ms'); - }); - - it('retries on network errors and recovers if a later attempt succeeds', async () => { - let calls = 0; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => { - calls++; - if (calls < 3) { - return Promise.reject(new Error('ECONNRESET')); - } - return Promise.resolve(new Response('ok', { status: 200 })); - }, - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('attempts=3'); - expect(calls).toBe(3); - }); - - it('retries on persistent HTTP errors and stays Down after all attempts fail', async () => { - let calls = 0; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => { - calls++; - return Promise.resolve( - new Response('Service Unavailable', { status: 503 }), - ); - }, - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(calls).toBe(3); - expect(result.error).toContain('HTTP 503'); - expect(result.error).toContain('attempts=3'); - }); - - it('retries on transient 5xx and recovers if a later attempt succeeds', async () => { - let calls = 0; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => { - calls++; - if (calls < 3) { - return Promise.resolve( - new Response('Bad Gateway', { status: 502 }), - ); - } - return Promise.resolve(new Response('ok', { status: 200 })); - }, - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('attempts=3'); - expect(calls).toBe(3); - }); - - it('returns Down (not NoConnection) when last attempt got an HTTP response after earlier network errors', async () => { - let calls = 0; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [PROBE_URL]: () => { - calls++; - if (calls < 3) return Promise.reject(new Error('ECONNRESET')); - return Promise.resolve( - new Response('Bad Gateway', { status: 502 }), - ); - }, - }), - ); - const result = await fetchEndpointHealth(PROBE_URL); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 502'); - }); - }); - - // ----------------------------------------------------------------------- - // MCP (fetchEndpointHealth – / landing) - // ----------------------------------------------------------------------- - - describe('checkMcpHealth', () => { - it('returns healthy when MCP worker responds 200 with landing HTML', async () => { - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 200'); - expect(global.fetch).toHaveBeenCalledWith( - URLS.mcpLanding, - expect.objectContaining({ signal: expect.any(AbortSignal) }), - ); - }); - - it('returns healthy when worker responds 302 (redirect to docs, not followed)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response(null, { status: 302 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 302'); - expect(global.fetch).toHaveBeenCalledWith( - URLS.mcpLanding, - expect.objectContaining({ redirect: 'manual' }), - ); - }); - - it('returns down on 400 — only 2xx-3xx counts as up', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('Bad Request', { status: 400 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 400'); - }); - - it('returns down when worker responds 500', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve( - new Response('Internal Server Error', { status: 500 }), - ), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 500'); - }); - - it('returns down when Cloudflare returns 522 (connection timed out)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('', { status: 522 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 522'); - }); - - it('returns no-connection on network failure', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => Promise.reject(new Error('fetch failed')), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toBe('fetch failed'); - }); - }); - - // ----------------------------------------------------------------------- - // Skills origins (fetchEndpointHealth – skill-menu.json on both origins) - // ----------------------------------------------------------------------- - - describe('checkSkillsOriginHealth', () => { - it('returns healthy on a final 200 and follows redirects (GitHub 302s asset URLs even for missing assets)', async () => { - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 200'); - expect(global.fetch).toHaveBeenCalledWith( - URLS.githubSkillMenu, - expect.objectContaining({ redirect: 'follow' }), - ); - }); - - it('probes both origins', async () => { - await checkSkillsOriginHealth(); - const calledUrls = (global.fetch as Mock).mock.calls.map( - (c: unknown[]) => c[0], - ); - expect(calledUrls).toContain(URLS.githubSkillMenu); - expect(calledUrls).toContain(URLS.awsSkillMenu); - }); - - it('stays healthy when GitHub 5xxs but AWS serves the menu', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('github unavailable'); - }); - - it('stays healthy when GitHub is unreachable but AWS serves the menu', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.reject(new Error('ENOTFOUND github.com')), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('github unavailable'); - }); - - it('stays healthy when GitHub 404s but AWS serves the menu', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.resolve(new Response('Not Found', { status: 404 })), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('github unavailable'); - }); - - it('stays healthy when AWS is unreachable but GitHub serves the menu', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.awsSkillMenu]: () => Promise.reject(new Error('fetch failed')), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toContain('aws unavailable'); - }); - - it('returns down only when both origins 404 (release published without the asset)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.resolve(new Response('Not Found', { status: 404 })), - [URLS.awsSkillMenu]: () => - Promise.resolve(new Response('Not Found', { status: 404 })), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('github: HTTP 404'); - expect(result.error).toContain('aws: HTTP 404'); - }); - - it('returns no-connection when both origins fail at the network layer', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.reject(new Error('ENOTFOUND github.com')), - [URLS.awsSkillMenu]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toContain('ENOTFOUND github.com'); - expect(result.error).toContain('ECONNRESET'); - }); - - it('reports down when GitHub 5xxs and AWS is unreachable', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.githubSkillMenu]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - [URLS.awsSkillMenu]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - const result = await checkSkillsOriginHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - }); - - // ----------------------------------------------------------------------- - // checkAllExternalServices - // ----------------------------------------------------------------------- - - describe('checkAllExternalServices', () => { - it('returns all 10 service keys when everything is healthy', async () => { - const health = await checkAllExternalServices(); - const keys = Object.keys(health); - expect(keys).toEqual( - expect.arrayContaining([ - 'anthropic', - 'posthogOverall', - 'posthogComponents', - 'github', - 'npmOverall', - 'npmComponents', - 'cloudflareOverall', - 'cloudflareComponents', - 'mcp', - 'skillsOrigin', - ]), - ); - expect(keys).toHaveLength(10); - for (const val of Object.values(health)) { - expect(val.status).toBe(ServiceHealthStatus.Healthy); - } - }); - - it('upgrades NoConnection mcp to Down when status page reports an outage', async () => { - const incidentBody = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'Major outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(incidentBody), { status: 200 }), - ), - [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - - const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Down); - expect(health.mcp.status).toBe(ServiceHealthStatus.Down); - expect(health.mcp.error).toContain('corroborated by status page'); - }); - - it('keeps mcp as NoConnection when posthogstatus.com itself is unreachable (the bug-fix scenario)', async () => { - // User on flaky wifi: every PostHog-owned URL fetch fails at the - // network layer, including posthogstatus.com. Previously - // incidentio.ts returned Degraded for fetch failures, which - // tricked reconciliation into upgrading the gateway probe to Down - // and showing the red "Ongoing service disruptions" screen — the - // exact false positive this PR fixes. - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.reject(new Error('ECONNRESET')), - [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - - const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe( - ServiceHealthStatus.NoConnection, - ); - expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('keeps mcp as NoConnection when status page reports no incident', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => Promise.reject(new Error('ETIMEDOUT')), - }), - ); - - const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Healthy); - expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('fires all fetch calls in parallel', async () => { - await checkAllExternalServices(); - const calledUrls = (global.fetch as Mock).mock.calls.map((c: unknown[]) => - typeof c[0] === 'string' ? c[0] : (c[0] as URL).toString(), - ); - // PostHog uses a single incident.io endpoint for both overall + components - expect(calledUrls).toHaveLength(10); - expect(calledUrls).toContain(URLS.posthogIncidentIo); - expect(calledUrls).toContain(URLS.mcpLanding); - expect(calledUrls).toContain(URLS.githubSkillMenu); - expect(calledUrls).toContain(URLS.awsSkillMenu); - }); + it('clears the watchdog after an unexpected failure and proceeds without warnings', async () => { + vi.mocked(checkSkillsOriginHealth).mockRejectedValue( + new Error('unexpected'), + ); + const result = await evaluateWizardReadiness(); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + expect(vi.getTimerCount()).toBe(0); }); - // ----------------------------------------------------------------------- - // evaluateWizardReadiness - // ----------------------------------------------------------------------- - - describe('evaluateWizardReadiness', () => { - it('returns Yes when all services are healthy', async () => { - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.Yes); - }); - - it('returns No when Anthropic is degraded (degradedBlocksRun)', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.No); - expect(result.health.anthropic.status).toBe(ServiceHealthStatus.Degraded); - }); - - it('returns No when MCP is down (downBlocksRun)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.No); - expect(result.health.mcp.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns No when npm overall is down (downBlocksRun)', async () => { - const body = makeStatuspageStatus({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'critical', - description: 'Major Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.npmStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.No); - expect(result.health.npmOverall.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns YesWithWarnings when a non-blocking service is degraded', async () => { - const body = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.cloudflareStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.YesWithWarnings); - }); - - it('includes human-readable reasons for every service', async () => { - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.reasons.length).toBeGreaterThan(0); - expect(result.reasons.some((r) => r.includes('Anthropic'))).toBe(true); - expect(result.reasons.some((r) => r.includes('PostHog'))).toBe(true); - expect(result.reasons.some((r) => r.includes('GitHub'))).toBe(true); - expect(result.reasons.some((r) => r.includes('npm'))).toBe(true); - expect(result.reasons.some((r) => r.includes('Cloudflare'))).toBe(true); - expect(result.reasons.some((r) => r.includes('MCP'))).toBe(true); - }); + it('does not claim an outage when a check cannot finish', async () => { + vi.mocked(checkSkillsOriginHealth).mockReturnValue( + new Promise(() => { + // Deliberately never settles; the readiness watchdog must release the run. + }), + ); + const pending = evaluateWizardReadiness(); + await vi.advanceTimersByTimeAsync(20_000); + const result = await pending; + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + expect(vi.getTimerCount()).toBe(0); }); }); diff --git a/src/lib/health-checks/endpoints.ts b/src/lib/health-checks/endpoints.ts index 2be4c5df9..b5ea17aa3 100644 --- a/src/lib/health-checks/endpoints.ts +++ b/src/lib/health-checks/endpoints.ts @@ -1,84 +1,83 @@ -import { AWS_SKILLS_BASE_URL, GITHUB_SKILLS_BASE_URL } from '@lib/constants'; +import { getSkillsBaseUrl } from '@lib/constants'; +import { awsUrlFor } from '@lib/fetch-retry'; import { logToFile } from '@utils/debug'; import { ServiceHealthStatus, type BaseHealthResult } from './types'; -// --------------------------------------------------------------------------- -// Direct endpoint health checks -// -// These ping PostHog-owned services directly (no Statuspage intermediary). -// Result taxonomy: -// - HTTP 2xx-3xx (per `isExpectedStatus`) → Healthy -// - HTTP 4xx / 5xx → Down (confirmed) -// - Network error / DNS / timeout (after retries) → NoConnection -// NoConnection means we don't know whose fault it is; readiness reconciles -// against the status page before deciding how to surface it to the user. -// -// MCP – Cloudflare Worker -// Source: posthog/services/mcp/src/index.ts -// GET / → 302 to posthog.com docs. The redirect proves the worker is up. -// -// Skills download – context-mill releases -// GET /skill-menu.json on both origins; see checkSkillsOriginHealth. -// --------------------------------------------------------------------------- - -function noConnectionResult(error: string, attempts: number): BaseHealthResult { - return { - status: ServiceHealthStatus.NoConnection, - error, - rawIndicator: attempts > 1 ? `attempts=${attempts}` : undefined, - }; -} - -function downResult(error: string): BaseHealthResult { - return { status: ServiceHealthStatus.Down, error }; -} - -// Backoffs sized to cover typical wifi flakiness — a single dropped -// packet recovers via the 500ms retry; a wifi access point reconnect -// or wifi↔LTE handoff (2-5s) is caught by the 2000ms retry. Tighter -// schedules miss multi-second blips because all retries land in the -// same dead window. +// HTTP failures or unusable downloads confirm the endpoint is unavailable. +// Network errors alone mean we cannot tell whether the service is down. const RETRY_BACKOFFS_MS = [500, 2000]; +type ResponseValidator = (response: Response) => Promise; +type RedirectMode = 'follow' | 'manual' | 'error'; +type FetchOutcome = + | { kind: 'response'; status: number } + | { + kind: 'error'; + error: string; + httpStatus?: number; + }; + async function attemptFetch( url: string, timeoutMs: number, - redirect: 'follow' | 'manual' | 'error', -): Promise< - | { kind: 'response'; res: Response } - | { kind: 'error'; error: Error; timedOut: boolean } -> { + isExpectedStatus: (status: number) => boolean, + redirect: RedirectMode, + validateResponse?: ResponseValidator, +): Promise { const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); + let httpStatus: number | undefined; + let timedOut = false; + let timeout: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + reject(new Error(`Request timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }); + try { - const res = await fetch(url, { signal: controller.signal, redirect }); - clearTimeout(tid); - return { kind: 'response', res }; - } catch (e) { - clearTimeout(tid); - const err = e instanceof Error ? e : new Error('Unknown error'); - return { kind: 'error', error: err, timedOut: err.name === 'AbortError' }; + const request = async (): Promise => { + const response = await fetch(url, { + signal: controller.signal, + redirect, + }); + httpStatus = response.status; + if (isExpectedStatus(response.status) && validateResponse) { + // Keep the deadline active until the body has downloaded and parsed. + await validateResponse(response); + } else { + // Health endpoints only need their status, not their diagnostic body. + void response.body?.cancel().catch(() => undefined); + } + return { kind: 'response', status: response.status }; + }; + return await Promise.race([request(), deadline]); + } catch (error) { + return { + kind: 'error', + httpStatus, + error: timedOut + ? `Request timed out after ${timeoutMs}ms` + : error instanceof Error + ? error.message + : 'Unknown error', + }; + } finally { + clearTimeout(timeout); } } -// Exported so tests can pin the retry/taxonomy machinery directly. +/** Probe an endpoint with bounded retries; downloads may also validate the body. */ export async function fetchEndpointHealth( url: string, timeoutMs = 5000, - isExpectedStatus: (status: number) => boolean = (s) => s === 200, - redirect: 'follow' | 'manual' | 'error' = 'follow', + isExpectedStatus: (status: number) => boolean = (status) => status === 200, + redirect: RedirectMode = 'follow', + validateResponse?: ResponseValidator, ): Promise { - // Total attempts = 1 initial + RETRY_BACKOFFS_MS.length retries. Both - // unexpected HTTP statuses (4xx/5xx) and network errors trigger a retry: - // transient 5xx and Cloudflare edge blips often recover on a retry, and - // even nominally deterministic 4xx can be transient (CDN propagation - // lag after a release, token rotation, rate-limit window resets). GETs - // are idempotent so retrying is safe. - // - // Final status if every attempt fails: - // - At least one HTTP response observed → `Down` (server-side evidence) - // - Only network errors observed → `NoConnection` - let lastHttpStatus: number | null = null; + let lastHttpStatus: number | undefined; + let lastHttpError: string | undefined; let lastError = 'Unknown error'; let attempts = 0; @@ -88,120 +87,139 @@ export async function fetchEndpointHealth( logToFile( `[health-checks] retry ${i}/${RETRY_BACKOFFS_MS.length} for ${url} in ${wait}ms (last: ${lastError})`, ); - await new Promise((r) => setTimeout(r, wait)); + await new Promise((resolve) => setTimeout(resolve, wait)); } attempts++; - - const outcome = await attemptFetch(url, timeoutMs, redirect); + const outcome = await attemptFetch( + url, + timeoutMs, + isExpectedStatus, + redirect, + validateResponse, + ); if (outcome.kind === 'response') { - const res = outcome.res; - if (isExpectedStatus(res.status)) { + if (isExpectedStatus(outcome.status)) { const result: BaseHealthResult = { status: ServiceHealthStatus.Healthy, rawIndicator: attempts > 1 - ? `HTTP ${res.status} (attempts=${attempts})` - : `HTTP ${res.status}`, + ? `HTTP ${outcome.status} (attempts=${attempts})` + : `HTTP ${outcome.status}`, }; logToFile( - `[health-checks] GET ${url} -> ${result.status}` + - ` (${result.rawIndicator})`, + `[health-checks] GET ${url} -> ${result.status} (${ + result.rawIndicator ?? '' + })`, ); return result; } - lastHttpStatus = res.status; - lastError = `HTTP ${res.status}`; - continue; + lastHttpStatus = outcome.status; + lastError = lastHttpError = `HTTP ${outcome.status}`; + } else { + lastError = outcome.error; + if (outcome.httpStatus !== undefined) { + lastHttpStatus = outcome.httpStatus; + lastHttpError = outcome.error; + } } - - lastError = outcome.timedOut - ? `Request timed out after ${timeoutMs}ms` - : outcome.error.message; } - const result = - lastHttpStatus !== null - ? downResult( - `HTTP ${lastHttpStatus} (attempts=${attempts})`, - lastHttpStatus, - ) - : noConnectionResult(lastError, attempts); + const result: BaseHealthResult = { + status: + lastHttpStatus !== undefined + ? ServiceHealthStatus.Down + : ServiceHealthStatus.NoConnection, + error: lastHttpError ?? lastError, + rawIndicator: + lastHttpStatus !== undefined + ? `HTTP ${lastHttpStatus} (attempts=${attempts})` + : `attempts=${attempts}`, + }; logToFile( - `[health-checks] GET ${url} -> ${result.status}` + - ` (attempts=${attempts}, ${result.error})`, + `[health-checks] GET ${url} -> ${result.status} (attempts=${attempts}, ${ + lastHttpError ?? lastError + })`, ); return result; } -export const checkMcpHealth = (): Promise => - fetchEndpointHealth( - 'https://mcp.posthog.com/', - 5000, - // 2xx-3xx counts as up (redirect to docs) - (s) => s >= 200 && s < 400, - 'manual', - ); +/** Readiness checks gateway dependencies, independently of its model providers. */ +export const checkLlmGatewayHealth = ( + gatewayUrl: string, +): Promise => + fetchEndpointHealth(new URL('/readyz', gatewayUrl).href); -/** - * Skills are published to two origins under the same filenames and - * `fetchWithRetry` fails over between them, so the run is only blocked when - * neither answers. Probed in parallel — sequential probes would double the - * worst case past `READINESS_TIMEOUT_MS`. - */ -export const checkSkillsOriginHealth = async (): Promise => { - const [github, aws] = await Promise.all([ - fetchEndpointHealth(`${GITHUB_SKILLS_BASE_URL}/skill-menu.json`), - fetchEndpointHealth(`${AWS_SKILLS_BASE_URL}/skill-menu.json`), - ]); - return combineOriginHealth(github, aws); -}; +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} -/** - * Mirrors `fetchWithRetry`: a download tries GitHub, then AWS, so the run is - * only blocked when neither origin answers. Whichever failure the probes saw, - * one origin serving means skills are reachable. - */ -function combineOriginHealth( - github: BaseHealthResult, - aws: BaseHealthResult, -): BaseHealthResult { - if (github.status === ServiceHealthStatus.Healthy) { - // Naming the dead origin makes a one-sided outage legible in the log and - // in the readiness reasons, where the status alone reads as "fine". - return aws.status === ServiceHealthStatus.Healthy - ? github - : withIndicatorSuffix(github, 'aws unavailable'); +/** Validate fields consumed by fetchSkillMenu, allowing optional newer metadata. */ +async function validateSkillMenu(response: Response): Promise { + const menu: unknown = await response.json(); + if (!isRecord(menu) || !isRecord(menu.categories)) { + throw new Error('Skill menu is missing its categories'); } - - if (aws.status === ServiceHealthStatus.Healthy) { - return withIndicatorSuffix(aws, 'via aws, github unavailable'); + const categories = Object.values(menu.categories); + const valid = categories.every( + (entries) => + Array.isArray(entries) && + entries.every( + (entry: unknown) => + isRecord(entry) && + ['id', 'name', 'downloadUrl'].every( + (key) => typeof entry[key] === 'string' && entry[key].length > 0, + ) && + (entry.variants === undefined || + (Array.isArray(entry.variants) && + entry.variants.every( + (variant: unknown) => + isRecord(variant) && typeof variant.id === 'string', + ))), + ), + ); + if (!valid || !categories.some((entries) => (entries as unknown[]).length)) { + throw new Error('Skill menu has no usable skill entries'); } - - const error = `github: ${github.error ?? 'unknown'} | aws: ${ - aws.error ?? 'unknown' - }`; - const confirmedDown = - github.status === ServiceHealthStatus.Down || - aws.status === ServiceHealthStatus.Down; - return { - status: confirmedDown - ? ServiceHealthStatus.Down - : ServiceHealthStatus.NoConnection, - error, - // Keeps the `attempts=N` the blocked-readiness analytics parses. - rawIndicator: github.rawIndicator ?? aws.rawIndicator, - }; } -function withIndicatorSuffix( - result: BaseHealthResult, - suffix: string, -): BaseHealthResult { +/** Probe the actual skill sources, including the matching release's AWS mirror. */ +export async function checkSkillsOriginHealth( + skillsBaseUrl = getSkillsBaseUrl(), +): Promise { + const primaryUrl = `${skillsBaseUrl.replace(/\/+$/, '')}/skill-menu.json`; + const fallbackUrl = awsUrlFor(primaryUrl); + const probe = (url: string) => + fetchEndpointHealth( + url, + 5000, + (status) => status === 200, + 'follow', + validateSkillMenu, + ); + + // Local and custom sources have no production fallback in fetchWithRetry. + if (!fallbackUrl) return probe(primaryUrl); + + const [primary, fallback] = await Promise.all([ + probe(primaryUrl), + probe(fallbackUrl), + ]); + if (primary.status === ServiceHealthStatus.Healthy) return primary; + if (fallback.status === ServiceHealthStatus.Healthy) return fallback; + + logToFile( + `[health-checks] skill origins unavailable: primary=${ + primary.error ?? 'unknown' + }; fallback=${fallback.error ?? 'unknown'}`, + ); return { - ...result, - rawIndicator: result.rawIndicator - ? `${result.rawIndicator} (${suffix})` - : suffix, + status: + primary.status === ServiceHealthStatus.Down || + fallback.status === ServiceHealthStatus.Down + ? ServiceHealthStatus.Down + : ServiceHealthStatus.NoConnection, + error: 'Both skill download sources are unavailable', + rawIndicator: primary.rawIndicator ?? fallback.rawIndicator, }; } diff --git a/src/lib/health-checks/incidentio.ts b/src/lib/health-checks/incidentio.ts deleted file mode 100644 index 574a4f0f9..000000000 --- a/src/lib/health-checks/incidentio.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { - ServiceHealthStatus, - type BaseHealthResult, - type ComponentHealthResult, - type ComponentStatus, -} from './types'; - -interface IncidentIoAffectedComponent { - id: string; - name: string; - group_name?: string; - current_status: string; -} - -interface IncidentIoIncident { - id: string; - name: string; - status: string; - current_worst_impact: string; - affected_components: IncidentIoAffectedComponent[]; -} - -interface IncidentIoSummary { - ongoing_incidents: IncidentIoIncident[]; - in_progress_maintenances: unknown[]; -} - -function mapIncidentImpact(impact: string): ServiceHealthStatus { - switch (impact) { - case 'full_outage': - return ServiceHealthStatus.Down; - case 'partial_outage': - case 'degraded_performance': - return ServiceHealthStatus.Degraded; - default: - return ServiceHealthStatus.Degraded; - } -} - -function mapComponentStatus(status: string): ServiceHealthStatus { - switch (status) { - case 'operational': - return ServiceHealthStatus.Healthy; - case 'full_outage': - return ServiceHealthStatus.Down; - case 'partial_outage': - case 'degraded_performance': - return ServiceHealthStatus.Degraded; - default: - return ServiceHealthStatus.Degraded; - } -} - -/** - * Build an error result for fetch failures. The kind matters for - * downstream reconciliation: - * - * - 'http' (incident.io returned a bad status code) → `Down`. We - * reached the status page but it told us something is wrong on - * its side. We have a definitive response. - * - 'network' (timeout, DNS failure, TCP/TLS failure) → `NoConnection`. - * We never reached the status page. Treating this as `Degraded` - * (the previous behavior) silently flipped the reconciliation in - * `readiness.ts` from "soft" to "confirmed outage" whenever the - * user's own network was flaky — exactly the false positive this - * module is meant to help diagnose. - */ -function errResult(error: string, kind: 'http' | 'network'): BaseHealthResult { - return { - status: - kind === 'http' - ? ServiceHealthStatus.Down - : ServiceHealthStatus.NoConnection, - error, - }; -} - -const POSTHOG_STATUS_URL = 'https://www.posthogstatus.com/api/v1/summary'; - -async function fetchPosthogStatus( - timeoutMs = 5000, -): Promise<{ overall: BaseHealthResult; components: ComponentHealthResult }> { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(POSTHOG_STATUS_URL, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) { - const err = errResult(`HTTP ${res.status}`, 'http'); - return { overall: err, components: err }; - } - - const data = (await res.json()) as IncidentIoSummary; - const incidents = data.ongoing_incidents ?? []; - - if (incidents.length === 0) { - return { - overall: { status: ServiceHealthStatus.Healthy }, - components: { status: ServiceHealthStatus.Healthy }, - }; - } - - let worstOverall = ServiceHealthStatus.Degraded; - const affected: ComponentStatus[] = []; - - for (const incident of incidents) { - const impact = mapIncidentImpact(incident.current_worst_impact); - if (impact === ServiceHealthStatus.Down) { - worstOverall = ServiceHealthStatus.Down; - } - - for (const comp of incident.affected_components ?? []) { - const compStatus = mapComponentStatus(comp.current_status); - if (compStatus !== ServiceHealthStatus.Healthy) { - affected.push({ - name: comp.group_name - ? `${comp.group_name} — ${comp.name}` - : comp.name, - status: compStatus, - rawStatus: comp.current_status, - }); - } - } - } - - return { - overall: { status: worstOverall }, - components: { - status: - affected.length > 0 ? ServiceHealthStatus.Degraded : worstOverall, - degradedOrDownComponents: affected.length > 0 ? affected : undefined, - }, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') { - const err = errResult('Request timed out', 'network'); - return { overall: err, components: err }; - } - const err = errResult( - e instanceof Error ? e.message : 'Unknown error', - 'network', - ); - return { overall: err, components: err }; - } -} - -let _cache: Promise<{ - overall: BaseHealthResult; - components: ComponentHealthResult; -}> | null = null; - -function getPosthogHealth() { - if (!_cache) _cache = fetchPosthogStatus(); - return _cache; -} - -export function resetPosthogHealthCache(): void { - _cache = null; -} - -export const checkPosthogOverallHealth = async (): Promise => - (await getPosthogHealth()).overall; - -export const checkPosthogComponentHealth = - async (): Promise => - (await getPosthogHealth()).components; diff --git a/src/lib/health-checks/index.ts b/src/lib/health-checks/index.ts index b4c3f6041..679ecba3a 100644 --- a/src/lib/health-checks/index.ts +++ b/src/lib/health-checks/index.ts @@ -1,32 +1,14 @@ export { ServiceHealthStatus, type BaseHealthResult, - type ComponentStatus, - type ComponentHealthResult, type AllServicesHealth, type HealthCheckKey, } from './types'; -export { - checkAnthropicHealth, - checkGithubHealth, - checkNpmOverallHealth, - checkNpmComponentHealth, - checkCloudflareOverallHealth, - checkCloudflareComponentHealth, -} from './statuspage'; - -export { - checkPosthogOverallHealth, - checkPosthogComponentHealth, - resetPosthogHealthCache, -} from './incidentio'; - -export { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; +export { checkLlmGatewayHealth, checkSkillsOriginHealth } from './endpoints'; export { - type WizardReadinessConfig, - DEFAULT_WIZARD_READINESS_CONFIG, + type HealthCheckOptions, checkAllExternalServices, WizardReadiness, type WizardReadinessResult, diff --git a/src/lib/health-checks/readiness.ts b/src/lib/health-checks/readiness.ts index 6cd883678..7996b52bd 100644 --- a/src/lib/health-checks/readiness.ts +++ b/src/lib/health-checks/readiness.ts @@ -2,181 +2,38 @@ import { ServiceHealthStatus, type AllServicesHealth, type BaseHealthResult, - type ComponentHealthResult, type HealthCheckKey, } from './types'; -import { - checkAnthropicHealth, - checkGithubHealth, - checkNpmOverallHealth, - checkNpmComponentHealth, - checkCloudflareOverallHealth, - checkCloudflareComponentHealth, -} from './statuspage'; -import { - checkPosthogOverallHealth, - checkPosthogComponentHealth, -} from './incidentio'; -import { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; +import { checkLlmGatewayHealth, checkSkillsOriginHealth } from './endpoints'; import { logToFile } from '@utils/debug'; -// --------------------------------------------------------------------------- -// Service labels (used in human-readable reason strings) -// --------------------------------------------------------------------------- - export const SERVICE_LABELS: Record = { - anthropic: 'Anthropic', - posthogOverall: 'PostHog', - posthogComponents: 'PostHog (components)', - github: 'GitHub', - npmOverall: 'npm', - npmComponents: 'npm (components)', - cloudflareOverall: 'Cloudflare', - cloudflareComponents: 'Cloudflare (components)', - mcp: 'MCP', + llmGateway: 'LLM gateway', skillsOrigin: 'Skills download', }; -// --------------------------------------------------------------------------- -// Readiness config -// --------------------------------------------------------------------------- +const HEALTH_CHECK_KEYS: HealthCheckKey[] = ['llmGateway', 'skillsOrigin']; -export interface WizardReadinessConfig { - /** Services where status=Down blocks the run (readiness=No). */ - downBlocksRun: HealthCheckKey[]; - /** Services where status=Degraded (or worse) blocks the run (readiness=No). */ - degradedBlocksRun?: HealthCheckKey[]; +export interface HealthCheckOptions { + /** Only the gateway URL returned by this run's token mint; never guessed. */ + gatewayUrl?: string; + /** Defaults to the same release or local server used by skill downloads. */ + skillsBaseUrl?: string; + /** Reuse the pre-auth skills check when checking the gateway after mint. */ + skillsHealth?: BaseHealthResult; } -/** - * See README section "Health checks" for the full rationale. - * Adjust these arrays to change what blocks a wizard run. - * - * The AI gateway is not probed: its URL is only known from the run's token - * mint, and a failed mint already stops the run at bootstrap with the - * server's reason. - */ -export const DEFAULT_WIZARD_READINESS_CONFIG: WizardReadinessConfig = { - downBlocksRun: ['anthropic', 'npmOverall', 'mcp', 'skillsOrigin'], - degradedBlocksRun: ['anthropic'], -}; - -/** - * Reduced readiness config for --signup provisioning flows. - * - * Provisioning only needs PostHog - it doesn't use Anthropic directly, npm, - * the skills origins, or MCP. - */ -export const SIGNUP_WIZARD_READINESS_CONFIG: WizardReadinessConfig = { - downBlocksRun: ['posthogOverall'], -}; - -// --------------------------------------------------------------------------- -// Aggregate check -// --------------------------------------------------------------------------- - -export async function checkAllExternalServices(): Promise { - const [ - anthropic, - posthogOverall, - posthogComponents, - github, - npmOverall, - npmComponents, - cloudflareOverall, - cloudflareComponents, - mcp, - skillsOrigin, - ] = await Promise.all([ - checkAnthropicHealth(), - checkPosthogOverallHealth(), - checkPosthogComponentHealth(), - checkGithubHealth(), - checkNpmOverallHealth(), - checkNpmComponentHealth(), - checkCloudflareOverallHealth(), - checkCloudflareComponentHealth(), - checkMcpHealth(), - checkSkillsOriginHealth(), +/** Direct checks of the dependencies this run uses, without status pages. */ +export async function checkAllExternalServices( + options: HealthCheckOptions = {}, +): Promise { + const [llmGateway, skillsOrigin] = await Promise.all([ + options.gatewayUrl ? checkLlmGatewayHealth(options.gatewayUrl) : undefined, + options.skillsHealth ?? checkSkillsOriginHealth(options.skillsBaseUrl), ]); - - const health: AllServicesHealth = { - anthropic, - posthogOverall, - posthogComponents, - github, - npmOverall, - npmComponents, - cloudflareOverall, - cloudflareComponents, - mcp, - skillsOrigin, - }; - return reconcilePosthogReachability(health); -} - -/** - * When a PostHog-owned endpoint probe returns `NoConnection`, decide - * whether it's a real outage or a likely-local issue by checking the - * official status page (`posthogstatus.com`): - * - * - Status page says PostHog is `Down` / `Degraded` → upgrade - * mcp to `Down`. The status page corroborates. - * - Status page is `Healthy` → keep `NoConnection`. The status page - * contradicts; this is probably the user's network. - * - Status page is also `NoConnection` → keep `NoConnection`. User - * can't reach two independent PostHog properties; almost - * certainly their network. (This case relies on incidentio.ts - * correctly emitting `NoConnection` for fetch failures rather - * than the previous `Degraded`, which used to silently flip the - * reconciliation into a false positive.) - * - * Why `Degraded` corroborates: a `Degraded` reading here only fires - * when incident.io's API parsed successfully and reported a real - * `partial_outage` or `degraded_performance` for some component. That's - * PostHog acknowledging an issue, even if narrower than a full outage. - * If our MCP probe is also failing, those two signals together - * justify pointing at PostHog rather than the user. - * - * A narrower variant — only corroborate when the affected component is - * MCP-related (US/EU Cloud, app) — would be more precise. We - * have the data in `posthogComponents` but don't use it here. If the - * analytics show false positives concentrated in this case, it's a - * cheap follow-up. - * - * Mutates a copy of `health` and returns it. - */ -export function reconcilePosthogReachability( - health: AllServicesHealth, -): AllServicesHealth { - const posthogStatus = health.posthogOverall.status; - const corroboratesOutage = - posthogStatus === ServiceHealthStatus.Down || - posthogStatus === ServiceHealthStatus.Degraded; - - if (!corroboratesOutage) return health; - - const upgrade = (r: BaseHealthResult): BaseHealthResult => - r.status === ServiceHealthStatus.NoConnection - ? { - ...r, - status: ServiceHealthStatus.Down, - error: r.error - ? `${r.error} (corroborated by status page)` - : 'corroborated by status page', - } - : r; - - return { - ...health, - mcp: upgrade(health.mcp), - }; + return { ...(llmGateway ? { llmGateway } : {}), skillsOrigin }; } -// --------------------------------------------------------------------------- -// Wizard readiness evaluation -// --------------------------------------------------------------------------- - export enum WizardReadiness { Yes = 'yes', No = 'no', @@ -189,153 +46,66 @@ export interface WizardReadinessResult { reasons: string[]; } -function describeResult(label: string, h: BaseHealthResult): string { - const parts = [`${label}: ${h.status}`]; - if (h.rawIndicator) parts.push(`indicator=${h.rawIndicator}`); - if (h.error) parts.push(h.error); - return parts.join(' — '); -} - -const MAX_COMPONENT_NAMES = 8; - -function describeComponents(label: string, h: ComponentHealthResult): string { - const affected = h.degradedOrDownComponents; - if (!affected || affected.length === 0) - return `${label} components: all operational`; - const shown = affected - .slice(0, MAX_COMPONENT_NAMES) - .map((c) => `${c.name} (${c.status})`); - const suffix = - affected.length > MAX_COMPONENT_NAMES - ? `, +${affected.length - MAX_COMPONENT_NAMES} more` - : ''; - return `${label} components impacted: ${shown.join(', ')}${suffix}`; +/** + * A gateway failure or the failure of every skills origin interrupts the run. + * An unprobed gateway and an inconclusive check do not produce outage warnings. + */ +export function getBlockingServiceKeys( + health: AllServicesHealth, +): HealthCheckKey[] { + return HEALTH_CHECK_KEYS.filter((key) => { + const status = health[key]?.status; + return ( + status === ServiceHealthStatus.Down || + status === ServiceHealthStatus.NoConnection + ); + }); } -// Each probe can take up to one base timeout + two retries with the -// 500ms / 2000ms backoffs in endpoints.ts (worst case ~17.5s for a -// network failure that exhausts retries). Probes run in parallel so -// the aggregate ceiling is one probe, not the sum. +// Endpoint probes retry within 17.5s; run them in parallel with a final ceiling. const READINESS_TIMEOUT_MS = 20_000; export async function evaluateWizardReadiness( - config: WizardReadinessConfig = DEFAULT_WIZARD_READINESS_CONFIG, + options: HealthCheckOptions = {}, ): Promise { + let timeout: ReturnType | undefined; try { const health = await Promise.race([ - checkAllExternalServices(), - new Promise((resolve) => - setTimeout( - () => resolve(allUnknown('Health check timed out')), + checkAllExternalServices(options), + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error('Health check timed out')), READINESS_TIMEOUT_MS, - ), - ), + ); + }), ]); - - const reasons: string[] = []; - - for (const key of Object.keys(health) as HealthCheckKey[]) { + const blockingKeys = getBlockingServiceKeys(health); + const reasons = blockingKeys.flatMap((key) => { const result = health[key]; - const label = SERVICE_LABELS[key]; - - reasons.push(describeResult(label, result)); - - if ('degradedOrDownComponents' in result) { - reasons.push(describeComponents(label, result)); - } - } - - const blockingKeys = getBlockingServiceKeys(health, config); + return result ? [`${SERVICE_LABELS[key]}: ${result.status}`] : []; + }); if (blockingKeys.length > 0) { - const blockingDetails = blockingKeys.map((key) => { - const h = health[key]; - return `${key} (${h.status}${h.error ? ` — ${h.error}` : ''})`; - }); - logToFile(`[health-checks] blocked by: ${blockingDetails.join(', ')}`); - return { decision: WizardReadiness.No, health, reasons }; + logToFile(`[health-checks] blocked by: ${reasons.join(', ')}`); } - - const hasWarnings = Object.values(health).some( - (h) => h.status !== ServiceHealthStatus.Healthy, - ); - - if (hasWarnings) { - return { decision: WizardReadiness.YesWithWarnings, health, reasons }; - } - - return { decision: WizardReadiness.Yes, health, reasons }; + return { + decision: + blockingKeys.length > 0 ? WizardReadiness.No : WizardReadiness.Yes, + health, + reasons, + }; } catch (err) { - logToFile( - `[health-checks] error: ${err instanceof Error ? err.message : err}`, - ); - // Health checks must never block the wizard run + logToFile('[health-checks] check inconclusive, proceeding:', err); return { decision: WizardReadiness.Yes, - health: allUnknown('Unexpected error'), - reasons: ['Health check failed unexpectedly — proceeding anyway'], + health: { + skillsOrigin: options.skillsHealth ?? { + status: ServiceHealthStatus.Degraded, + error: 'Health check did not complete', + }, + }, + reasons: [], }; + } finally { + clearTimeout(timeout); } } - -// --------------------------------------------------------------------------- -// Blocking service detection -// --------------------------------------------------------------------------- - -/** Keys that are component-level detail, not top-level services. */ -const COMPONENT_KEYS: HealthCheckKey[] = [ - 'posthogComponents', - 'npmComponents', - 'cloudflareComponents', -]; - -/** - * Get the keys of services that would block a wizard run per the given config. - * - * `NoConnection` blocks the same services as `Down` — the wizard genuinely - * can't continue if it can't reach the gateway. The screen shows softer - * framing in that case (HealthCheckScreen) so we don't falsely accuse - * PostHog of an outage when the user's network is the likely cause. - */ -export function getBlockingServiceKeys( - health: AllServicesHealth, - config: WizardReadinessConfig = DEFAULT_WIZARD_READINESS_CONFIG, -): HealthCheckKey[] { - return (Object.keys(health) as HealthCheckKey[]).filter((key) => { - if (COMPONENT_KEYS.includes(key)) return false; - const result = health[key]; - if ( - config.downBlocksRun.includes(key) && - (result.status === ServiceHealthStatus.Down || - result.status === ServiceHealthStatus.NoConnection) - ) { - return true; - } - if ( - (config.degradedBlocksRun ?? []).includes(key) && - result.status !== ServiceHealthStatus.Healthy - ) { - return true; - } - return false; - }); -} - -/** Build an AllServicesHealth where every service is Degraded with the given error. */ -function allUnknown(error: string): AllServicesHealth { - const base: BaseHealthResult = { - status: ServiceHealthStatus.Degraded, - error, - }; - return { - anthropic: base, - posthogOverall: base, - posthogComponents: { ...base }, - github: base, - npmOverall: base, - npmComponents: { ...base }, - cloudflareOverall: base, - cloudflareComponents: { ...base }, - mcp: base, - skillsOrigin: base, - }; -} diff --git a/src/lib/health-checks/statuspage.ts b/src/lib/health-checks/statuspage.ts deleted file mode 100644 index bfcdd6dae..000000000 --- a/src/lib/health-checks/statuspage.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { - ServiceHealthStatus, - type BaseHealthResult, - type ComponentHealthResult, -} from './types'; - -// --------------------------------------------------------------------------- -// Statuspage.io v2 API helpers -// https://metastatuspage.com/api -// -// status.json – page-level rollup; indicator is one of: none | minor | major | critical -// summary.json – same rollup + component list; component status is one of: -// operational | degraded_performance | partial_outage | major_outage | under_maintenance -// https://support.atlassian.com/statuspage/docs/show-service-status-with-components -// --------------------------------------------------------------------------- - -interface StatuspageStatusResponse { - status?: { indicator?: string; description?: string }; -} - -interface StatuspageSummaryResponse extends StatuspageStatusResponse { - components?: { id: string; name: string; status: string }[]; -} - -function mapIndicator(v: string | null | undefined): ServiceHealthStatus { - switch (v) { - case 'none': - return ServiceHealthStatus.Healthy; - case 'minor': - return ServiceHealthStatus.Degraded; - case 'major': - case 'critical': - return ServiceHealthStatus.Down; - default: - return ServiceHealthStatus.Degraded; - } -} - -function mapComponentRaw(v: string | null | undefined): ServiceHealthStatus { - switch (v) { - case 'operational': - return ServiceHealthStatus.Healthy; - case 'degraded_performance': - case 'under_maintenance': - return ServiceHealthStatus.Degraded; - case 'partial_outage': - case 'major_outage': - return ServiceHealthStatus.Down; - default: - return ServiceHealthStatus.Degraded; - } -} - -function errResult(error: string): BaseHealthResult { - return { status: ServiceHealthStatus.Degraded, error }; -} - -async function fetchStatuspageIndicator( - url: string, - timeoutMs = 5000, -): Promise { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(url, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) return errResult(`HTTP ${res.status}`); - - const data = (await res.json()) as StatuspageStatusResponse; - const indicator = data.status?.indicator ?? null; - return { - status: mapIndicator(indicator), - rawIndicator: indicator ?? undefined, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') - return errResult('Request timed out'); - return errResult(e instanceof Error ? e.message : 'Unknown error'); - } -} - -async function fetchStatuspageSummary( - url: string, - timeoutMs = 5000, -): Promise { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(url, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) return errResult(`HTTP ${res.status}`); - - const data = (await res.json()) as StatuspageSummaryResponse; - const indicator = data.status?.indicator ?? null; - const overall = mapIndicator(indicator); - - const affected = (data.components ?? []) - .map((c) => ({ - name: c.name, - status: mapComponentRaw(c.status), - rawStatus: c.status, - })) - .filter((c) => c.status !== ServiceHealthStatus.Healthy); - - return { - status: affected.length > 0 ? ServiceHealthStatus.Degraded : overall, - rawIndicator: indicator ?? undefined, - degradedOrDownComponents: affected.length > 0 ? affected : undefined, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') - return errResult('Request timed out'); - return errResult(e instanceof Error ? e.message : 'Unknown error'); - } -} - -// --------------------------------------------------------------------------- -// Individual statuspage-backed checks -// --------------------------------------------------------------------------- - -export const checkAnthropicHealth = (): Promise => - fetchStatuspageIndicator('https://status.claude.com/api/v2/status.json'); - -export const checkGithubHealth = (): Promise => - fetchStatuspageIndicator('https://www.githubstatus.com/api/v2/status.json'); - -export const checkNpmOverallHealth = (): Promise => - fetchStatuspageIndicator('https://status.npmjs.org/api/v2/status.json'); - -export const checkNpmComponentHealth = (): Promise => - fetchStatuspageSummary('https://status.npmjs.org/api/v2/summary.json'); - -export const checkCloudflareOverallHealth = (): Promise => - fetchStatuspageIndicator( - 'https://www.cloudflarestatus.com/api/v2/status.json', - ); - -export const checkCloudflareComponentHealth = - (): Promise => - fetchStatuspageSummary( - 'https://www.cloudflarestatus.com/api/v2/summary.json', - ); diff --git a/src/lib/health-checks/testme.md b/src/lib/health-checks/testme.md index 9cf6a7202..ea17fc8c4 100644 --- a/src/lib/health-checks/testme.md +++ b/src/lib/health-checks/testme.md @@ -1,61 +1,26 @@ -# Health Checks — Testing Guide +# Health check tests -## Running unit tests +Run the focused suites without building or contacting live services: ```bash -# From the wizard/ root — runs only health-check tests (fast, no build step) -npx jest src/lib/health-checks/__tests__/health-checks.test.ts - -# Watch mode -npx jest src/lib/health-checks/__tests__/health-checks.test.ts --watch - -# With coverage -npx jest src/lib/health-checks/__tests__/health-checks.test.ts --coverage -``` - -## Running health checks live - -To hit all 10 endpoints for real and see the full readiness result: - -```bash -# From the wizard/ root -npx tsx -e "import { evaluateWizardReadiness } from './src/lib/health-checks/index'; evaluateWizardReadiness().then(r => console.log(JSON.stringify(r, null, 2)))" +pnpm exec vitest run src/lib/health-checks/__tests__ src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts src/ui/tui/__tests__/ink-ui-health.test.ts ``` -## How the tests work - -All external HTTP calls are mocked via a global `fetch` override in -`beforeEach`. No network access is required. Mock data is modelled on real -responses captured from production endpoints on 2026-03-05. - -## Endpoints tested - -| Service | URL | Healthy response | -| ----------------------- | ------------------------------------------------------ | ------------------------------------- | -| Anthropic | `https://status.claude.com/api/v2/status.json` | `{"status":{"indicator":"none",...}}` | -| PostHog | `https://www.posthogstatus.com/api/v2/status.json` | Same shape | -| PostHog (components) | `https://www.posthogstatus.com/api/v2/summary.json` | Adds `components[]` array | -| GitHub | `https://www.githubstatus.com/api/v2/status.json` | Same shape | -| npm | `https://status.npmjs.org/api/v2/status.json` | Same shape | -| npm (components) | `https://status.npmjs.org/api/v2/summary.json` | Adds `components[]` array | -| Cloudflare | `https://www.cloudflarestatus.com/api/v2/status.json` | Same shape | -| Cloudflare (components) | `https://www.cloudflarestatus.com/api/v2/summary.json` | Adds `components[]` array | -| MCP | `https://mcp.posthog.com/` | HTML landing page (HTTP 200) | - -### Statuspage.io API v2 reference - -- Docs: -- `status.json` — page-level rollup; `indicator` is one of: `none`, `minor`, - `major`, `critical` -- `summary.json` — same rollup + `components[]`; component `status` is one of: - `operational`, `degraded_performance`, `partial_outage`, `major_outage`, - `under_maintenance` -- Component docs: - - -### MCP - -- Source: `posthog/services/mcp/src/index.ts` -- `GET /` → HTML landing page (200) -- No dedicated `/health` endpoint; 200 on `/` confirms the Cloudflare Worker is - running. +Endpoint tests cover bounded retries, connection failures, malformed skill +menus, GitHub/AWS fallback, local context-mill targets, and the gateway +readiness route. Readiness tests cover the dependency matrix, pre-auth results, +the actual minted gateway target, and absence of unrelated provider warnings. +Bootstrap and UI tests cover cached skills checks and waiting for a fresh outage +dismissal after login. + +| Dependency | Probe | Healthy response | +| ------------------ | ---------------------------------------------------------------------------------- | ------------------------------------ | +| LLM gateway | `/readyz` | HTTP 200; no bearer or model request | +| GitHub skills | `https://github.com/PostHog/context-mill/releases/latest/download/skill-menu.json` | Downloadable, valid skill menu | +| AWS skills mirror | `https://context-mill.posthog.com/latest/skill-menu.json` | Downloadable, valid skill menu | +| Local context-mill | `/skill-menu.json` | Downloadable, valid skill menu | + +The gateway is omitted before mint rather than guessed. Either release origin +working is sufficient; local targets are checked independently of production. +Provider status pages are not queried. No gateway readiness response body is +shown to the user. diff --git a/src/lib/health-checks/types.ts b/src/lib/health-checks/types.ts index f838bbf96..7df9b0b80 100644 --- a/src/lib/health-checks/types.ts +++ b/src/lib/health-checks/types.ts @@ -2,13 +2,7 @@ export enum ServiceHealthStatus { Healthy = 'healthy', Degraded = 'degraded', Down = 'down', - /** - * Probe failed (network error, timeout, DNS failure) AND we have no - * corroborating status-page incident. The service may be fine — the - * user's network is the likely culprit. Distinct from `Down`, which - * is confirmed (HTTP 5xx or status-page incident). User-facing label: - * "No connection". - */ + /** A failed connection does not establish whether the service or local network is at fault. */ NoConnection = 'no-connection', } @@ -18,26 +12,9 @@ export interface BaseHealthResult { error?: string; } -export interface ComponentStatus { - name: string; - status: ServiceHealthStatus; - rawStatus: string; -} - -export interface ComponentHealthResult extends BaseHealthResult { - degradedOrDownComponents?: ComponentStatus[]; -} - export interface AllServicesHealth { - anthropic: BaseHealthResult; - posthogOverall: BaseHealthResult; - posthogComponents: ComponentHealthResult; - github: BaseHealthResult; - npmOverall: BaseHealthResult; - npmComponents: ComponentHealthResult; - cloudflareOverall: BaseHealthResult; - cloudflareComponents: ComponentHealthResult; - mcp: BaseHealthResult; + /** Absent before the token mint tells us this run's actual gateway URL. */ + llmGateway?: BaseHealthResult; skillsOrigin: BaseHealthResult; } diff --git a/src/lib/programs/shared/health-check-step.ts b/src/lib/programs/shared/health-check-step.ts index 1c630a52c..fa41462ca 100644 --- a/src/lib/programs/shared/health-check-step.ts +++ b/src/lib/programs/shared/health-check-step.ts @@ -1,14 +1,12 @@ /** - * Shared health-check step used by every program that runs an agent. + * Shared health-check step for programs that opt into dependency checks. * * Renders the HealthCheckScreen between intro and auth, kicks off the * readiness probe in onInit, and gates the screen on either a clean * readiness result or an explicit user dismissal of the outage. * - * Programs without this step that hit a blocking outage gridlock the - * router: agent-runner calls wizardAbort, which awaits outroDismissed, - * but the router can't advance past the still-incomplete auth step to - * render the OutroScreen. + * Bootstrap checks the minted gateway later for these same programs. + * Programs without this screen skip both advisory checks. */ import type { ProgramStep } from '@lib/programs/program-step'; @@ -16,26 +14,12 @@ import type { WizardSession } from '@lib/wizard-session'; import { evaluateWizardReadiness, WizardReadiness, - SIGNUP_WIZARD_READINESS_CONFIG, - getBlockingServiceKeys, } from '@lib/health-checks/readiness'; import { logToFile } from '@utils/debug'; export function healthCheckReady(session: WizardSession): boolean { if (!session.readinessResult) return false; - if (session.signup) { - const hardBlocking = getBlockingServiceKeys( - session.readinessResult.health, - SIGNUP_WIZARD_READINESS_CONFIG, - ); - const defaultBlocking = getBlockingServiceKeys( - session.readinessResult.health, - ); - if (hardBlocking.length === 0 && defaultBlocking.length === 0) return true; - return session.outageDismissed; - } - if (session.readinessResult.decision === WizardReadiness.No) { return session.outageDismissed; } diff --git a/src/ui/logging-ui.ts b/src/ui/logging-ui.ts index 30ff56dac..2bc064f79 100644 --- a/src/ui/logging-ui.ts +++ b/src/ui/logging-ui.ts @@ -123,8 +123,9 @@ export class LoggingUI implements WizardUI { console.log(`│`); console.log(`│ Blocking services:`); for (const key of blockingKeys) { - const status = result.health[key].status; - const error = result.health[key].error; + const health = result.health[key]; + if (!health) continue; + const { status, error } = health; const label = SERVICE_LABELS[key]; const detail = error ? ` — ${error}` : ''; console.log(`│ ✖ ${label}: ${status}${detail}`); diff --git a/src/ui/tui/__tests__/ink-ui-health.test.ts b/src/ui/tui/__tests__/ink-ui-health.test.ts new file mode 100644 index 000000000..3be86d9b4 --- /dev/null +++ b/src/ui/tui/__tests__/ink-ui-health.test.ts @@ -0,0 +1,85 @@ +import { WizardStore, ScreenId } from '@ui/tui/store'; +import { InkUI } from '@ui/tui/ink-ui'; +import { + WizardReadiness, + type WizardReadinessResult, +} from '@lib/health-checks/readiness'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; +import { analytics } from '@utils/analytics'; + +vi.mock('../../../utils/analytics.js', () => ({ + analytics: { + capture: vi.fn(), + wizardCapture: vi.fn(), + setTag: vi.fn(), + captureException: vi.fn(), + }, + sessionProperties: vi.fn(() => ({})), +})); + +const skillsHealthy = { status: ServiceHealthStatus.Healthy } as const; +const gatewayOutage = (): WizardReadinessResult => ({ + decision: WizardReadiness.No, + health: { + skillsOrigin: skillsHealthy, + llmGateway: { status: ServiceHealthStatus.Down, error: 'HTTP 503' }, + }, + reasons: ['LLM gateway: down'], +}); + +describe('gateway outage after pre-auth health checks', () => { + it('waits for a fresh dismissal after the startup gate already resolved', async () => { + const store = new WizardStore(); + const ui = new InkUI(store); + store.completeSetup(); + store.setReadinessResult({ + decision: WizardReadiness.Yes, + health: { skillsOrigin: skillsHealthy }, + reasons: [], + }); + await store.getGate('health-check'); + store.dismissOutage(); + + let continued = false; + const waiting = ui.showBlockingOutage(gatewayOutage()).then(() => { + continued = true; + }); + await Promise.resolve(); + + expect(store.session.outageDismissed).toBe(false); + expect(store.currentScreen).toBe(ScreenId.HealthCheck); + expect(continued).toBe(false); + + store.dismissOutage(); + await waiting; + expect(continued).toBe(true); + expect(store.currentScreen).not.toBe(ScreenId.HealthCheck); + }); + + it('retains dismissal for the same result and resets it for a new outage', () => { + const store = new WizardStore(); + const first = gatewayOutage(); + store.setReadinessResult(first); + store.dismissOutage(); + store.setReadinessResult(first); + expect(store.session.outageDismissed).toBe(true); + + store.setReadinessResult(gatewayOutage()); + expect(store.session.outageDismissed).toBe(false); + }); + + it('records gateway failure without unrelated status-page claims', () => { + vi.mocked(analytics.wizardCapture).mockClear(); + const store = new WizardStore(); + store.setReadinessResult(gatewayOutage()); + + expect(analytics.wizardCapture).toHaveBeenCalledWith( + 'health check blocked', + { + decision: 'confirmed-outage', + blocking_keys: ['llmGateway'], + retries_used: 0, + }, + ); + }); +}); diff --git a/src/ui/tui/__tests__/programs.test.ts b/src/ui/tui/__tests__/programs.test.ts index 6030b70d2..f959c70ff 100644 --- a/src/ui/tui/__tests__/programs.test.ts +++ b/src/ui/tui/__tests__/programs.test.ts @@ -1,5 +1,7 @@ import { buildSession, McpOutcome, RunPhase } from '@lib/wizard-session'; import { WizardReadiness } from '@lib/health-checks/readiness'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; +import { healthCheckReady } from '@lib/programs/shared/health-check-step'; import { PROGRAM_SEQUENCES, ScreenId } from '@ui/tui/screen-sequences'; import { Program, type ProgramId } from '@lib/programs/program-registry'; @@ -62,6 +64,44 @@ describe('PROGRAM_SEQUENCES', () => { }); describe('Wizard health-check predicate', () => { + it.each([false, true])( + 'applies the same dependency policy with signup=%s', + (signup) => { + const session = buildSession({ signup }); + session.readinessResult = { + decision: WizardReadiness.No, + health: { + skillsOrigin: { status: ServiceHealthStatus.Healthy }, + llmGateway: { status: ServiceHealthStatus.Down }, + }, + reasons: ['LLM gateway: down'], + }; + const check = getEntry( + Program.PostHogIntegration, + ScreenId.HealthCheck, + ); + expect(check.isComplete?.(session)).toBe(false); + session.outageDismissed = true; + expect(check.isComplete?.(session)).toBe(true); + }, + ); + + it('does not release the runnable health gate on a terminal error', () => { + const session = buildSession({}); + session.readinessResult = { + decision: WizardReadiness.No, + health: { skillsOrigin: { status: ServiceHealthStatus.Down } }, + reasons: ['Skills download: down'], + }; + session.runPhase = RunPhase.Error; + expect( + getEntry(Program.PostHogIntegration, ScreenId.HealthCheck).isComplete?.( + session, + ), + ).toBe(false); + expect(session.outageDismissed).toBe(false); + expect(healthCheckReady(session)).toBe(false); + }); it('stays incomplete before readiness exists', () => { const session = buildSession({}); const entry = getEntry(Program.PostHogIntegration, ScreenId.HealthCheck); @@ -76,7 +116,7 @@ describe('PROGRAM_SEQUENCES', () => { session.readinessResult = { decision: WizardReadiness.No, health: {} as never, - reasons: ['Anthropic: down'], + reasons: ['LLM gateway: down'], }; expect(entry.isComplete?.(session)).toBe(false); diff --git a/src/ui/tui/__tests__/router.test.ts b/src/ui/tui/__tests__/router.test.ts index 5cbea8054..dfa37b453 100644 --- a/src/ui/tui/__tests__/router.test.ts +++ b/src/ui/tui/__tests__/router.test.ts @@ -6,6 +6,8 @@ import { } from '@lib/wizard-session'; import { HostResolution } from '@lib/host-resolution'; import { WizardReadiness } from '@lib/health-checks/readiness'; +import { healthCheckReady } from '@lib/programs/shared/health-check-step'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; import { WizardRouter, ScreenId, Overlay, Program } from '@ui/tui/router'; import { Integration } from '@lib/constants'; import { FRAMEWORK_REGISTRY } from '@lib/registry'; @@ -16,6 +18,80 @@ function baseWizardSession() { describe('WizardRouter', () => { describe('resolve', () => { + it('shows the error outro when exiting a pre-auth skills outage without releasing startup', () => { + const router = new WizardRouter(Program.PostHogIntegration); + const session = baseWizardSession(); + session.setupConfirmed = true; + session.readinessResult = { + decision: WizardReadiness.No, + health: { skillsOrigin: { status: ServiceHealthStatus.Down } }, + reasons: [], + }; + expect(router.resolve(session)).toBe(ScreenId.HealthCheck); + session.runPhase = RunPhase.Error; + session.outroData = { + kind: OutroKind.Error, + message: 'Exited due to service outage.', + }; + expect(router.resolve(session)).toBe(ScreenId.Outro); + expect(healthCheckReady(session)).toBe(false); + expect(session.outageDismissed).toBe(false); + }); + + it('shows the error outro when exiting a gateway outage during a composed integration run', () => { + const router = new WizardRouter(Program.SelfDriving); + const session = baseWizardSession(); + session.setupConfirmed = true; + session.integrate = true; + session.integration = Integration.nextjs; + session.credentials = { + accessToken: 'tok', + projectApiKey: 'pk', + host: HostResolution.fromApiHost('https://app.posthog.com'), + projectId: 1, + }; + session.readinessResult = { + decision: WizardReadiness.No, + health: { + skillsOrigin: { status: ServiceHealthStatus.Healthy }, + llmGateway: { status: ServiceHealthStatus.Down }, + }, + reasons: [], + }; + session.runPhase = RunPhase.Error; + session.outroData = { + kind: OutroKind.Error, + message: 'Exited due to service outage.', + }; + expect(router.resolve(session)).toBe(ScreenId.Outro); + expect(session.completedRuns).not.toContain('integrate-run'); + expect(session.outageDismissed).toBe(false); + }); + it('allows a dismissed terminal error to advance past the outro', () => { + const router = new WizardRouter(Program.PostHogIntegration); + const session = baseWizardSession(); + session.setupConfirmed = true; + session.readinessResult = { + decision: WizardReadiness.Yes, + health: { skillsOrigin: { status: ServiceHealthStatus.Healthy } }, + reasons: [], + }; + session.credentials = { + accessToken: 'tok', + projectApiKey: 'pk', + host: HostResolution.fromApiHost('https://app.posthog.com'), + projectId: 1, + }; + session.runPhase = RunPhase.Error; + session.outroData = { + kind: OutroKind.Error, + message: 'A screen crashed.', + }; + expect(router.resolve(session)).toBe(ScreenId.Outro); + session.outroDismissed = true; + expect(router.resolve(session)).toBe(ScreenId.Mcp); + }); + it('returns the first incomplete visible screen for the wizard flow', () => { const router = new WizardRouter(Program.PostHogIntegration); const session = baseWizardSession(); diff --git a/src/ui/tui/__tests__/store.test.ts b/src/ui/tui/__tests__/store.test.ts index 0201a6320..8d6559457 100644 --- a/src/ui/tui/__tests__/store.test.ts +++ b/src/ui/tui/__tests__/store.test.ts @@ -331,7 +331,7 @@ describe('WizardStore', () => { const result = { decision: WizardReadiness.No, health: {} as never, - reasons: ['Anthropic: down'], + reasons: ['LLM gateway: down'], }; store.setReadinessResult(result); expect(store.session.readinessResult).toEqual(result); @@ -1484,7 +1484,7 @@ describe('WizardStore', () => { evaluateWizardReadinessMock.mockResolvedValueOnce({ decision: WizardReadiness.No, health: {} as never, - reasons: ['Anthropic: down'], + reasons: ['LLM gateway: down'], }); const store = createStore(); diff --git a/src/ui/tui/components/ServiceHealthList.tsx b/src/ui/tui/components/ServiceHealthList.tsx index c38e9db4c..ebed560da 100644 --- a/src/ui/tui/components/ServiceHealthList.tsx +++ b/src/ui/tui/components/ServiceHealthList.tsx @@ -1,34 +1,18 @@ /** * ServiceHealthList — Shared component for displaying service health status. * - * Used by HealthCheckScreen (blocking services only) and HealthWarningsTab (all services). + * Used by HealthCheckScreen and its playground demo. */ import { Box, Text } from 'ink'; import { ServiceHealthStatus, type AllServicesHealth, - type ComponentHealthResult, - type ComponentStatus, type HealthCheckKey, } from '@lib/health-checks/types'; import { SERVICE_LABELS } from '@lib/health-checks/readiness'; import { Icons } from '@ui/tui/styles'; -/** Keys that are component-level detail — shown inline under their parent. */ -const COMPONENT_KEYS: HealthCheckKey[] = [ - 'posthogComponents', - 'npmComponents', - 'cloudflareComponents', -]; - -/** Map component key → its parent "overall" key */ -const COMPONENT_PARENT: Partial> = { - posthogComponents: 'posthogOverall', - npmComponents: 'npmOverall', - cloudflareComponents: 'cloudflareOverall', -}; - function statusIcon(status: ServiceHealthStatus): { icon: string; color: string; @@ -58,35 +42,26 @@ export const ServiceHealthList = ({ filterKeys, showHealthy = true, }: ServiceHealthListProps) => { - const topLevelKeys = (Object.keys(health) as HealthCheckKey[]).filter( - (k) => !COMPONENT_KEYS.includes(k), - ); + const serviceKeys = Object.keys(SERVICE_LABELS) as HealthCheckKey[]; const keysToShow = filterKeys - ? topLevelKeys.filter((k) => filterKeys.includes(k)) - : topLevelKeys; + ? serviceKeys.filter((k) => filterKeys.includes(k)) + : serviceKeys; return ( {keysToShow.map((key) => { const result = health[key]; - if (!showHealthy && result.status === ServiceHealthStatus.Healthy) { + if ( + !result || + (!showHealthy && result.status === ServiceHealthStatus.Healthy) + ) { return null; } const { icon, color } = statusIcon(result.status); const label = SERVICE_LABELS[key]; - // Find component-level details if this is a parent key - const componentKey = ( - Object.entries(COMPONENT_PARENT) as [HealthCheckKey, HealthCheckKey][] - ).find(([, parent]) => parent === key)?.[0]; - const componentResult = componentKey - ? (health[componentKey] as ComponentHealthResult) - : undefined; - const affectedComponents: ComponentStatus[] = - componentResult?.degradedOrDownComponents ?? []; - return ( @@ -94,22 +69,10 @@ export const ServiceHealthList = ({ {label} + {result.status === ServiceHealthStatus.NoConnection && ( + — No connection + )} - {affectedComponents.length > 0 && ( - - {affectedComponents.slice(0, 5).map((c) => { - const ci = statusIcon(c.status); - return ( - - {ci.icon} {c.name} - - ); - })} - {affectedComponents.length > 5 && ( - +{affectedComponents.length - 5} more - )} - - )} ); })} diff --git a/src/ui/tui/ink-ui.ts b/src/ui/tui/ink-ui.ts index 60c9f27da..deb7acaeb 100644 --- a/src/ui/tui/ink-ui.ts +++ b/src/ui/tui/ink-ui.ts @@ -134,10 +134,10 @@ export class InkUI implements WizardUI { } showBlockingOutage(result: WizardReadinessResult): Promise { - // In the TUI, the HealthCheckScreen handles outage display. - // This is only called from agent-runner for the CI fallback path. + // A gateway check can fail after the pre-auth health gate resolved. + // Wait for this outage's dismissal, not the already-latched gate. this.store.setReadinessResult(result); - return Promise.resolve(); + return this.store.waitUntil((session) => session.outageDismissed); } setReadinessWarnings(result: WizardReadinessResult): void { diff --git a/src/ui/tui/playground/demos/HealthCheckDemo.tsx b/src/ui/tui/playground/demos/HealthCheckDemo.tsx index 501f6972c..1c6c0eb54 100644 --- a/src/ui/tui/playground/demos/HealthCheckDemo.tsx +++ b/src/ui/tui/playground/demos/HealthCheckDemo.tsx @@ -1,83 +1,48 @@ /** - * HealthCheckDemo — Playground demo for health check UI components. - * - * Cycles through three states (2s checking spinner → 5s confirmed-outage - * red modal → 5s no-connection yellow modal, then loops): - * 1. Checking (spinner) - * 2. Confirmed outage (status page corroborates → red framing) - * 3. No connection only (no status-page incident → yellow "couldn't - * reach PostHog" framing) - * - * Renders components directly (not HealthCheckScreen) to avoid useInput - * conflicts with TabContainer's key handling. + * HealthCheckDemo — checking, gateway outage, and unavailable skill downloads. + * Renders components directly to avoid conflicts with TabContainer input. */ import { useEffect, useState } from 'react'; import { Box, Text } from 'ink'; import { LoadingBox, ModalOverlay } from '@ui/tui/primitives/index'; -import { Icons } from '@ui/tui/styles'; import { ServiceHealthList } from '@ui/tui/components/ServiceHealthList'; import { getBlockingServiceKeys } from '@lib/health-checks/readiness'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; -import type { AllServicesHealth } from '@lib/health-checks/types'; +import { + ServiceHealthStatus, + type AllServicesHealth, +} from '@lib/health-checks/types'; -const HEALTHY = { status: ServiceHealthStatus.Healthy } as const; - -const MOCK_CONFIRMED_OUTAGE: AllServicesHealth = { - anthropic: { status: ServiceHealthStatus.Down, rawIndicator: 'major' }, - posthogOverall: HEALTHY, - posthogComponents: { status: ServiceHealthStatus.Healthy }, - github: HEALTHY, - npmOverall: { - status: ServiceHealthStatus.Degraded, - rawIndicator: 'minor', - }, - npmComponents: { - status: ServiceHealthStatus.Degraded, - degradedOrDownComponents: [ - { - name: 'Registry API', - status: ServiceHealthStatus.Degraded, - rawStatus: 'degraded_performance', - }, - ], - }, - cloudflareOverall: HEALTHY, - cloudflareComponents: { status: ServiceHealthStatus.Healthy }, - mcp: HEALTHY, - skillsOrigin: HEALTHY, +const MOCK_GATEWAY_OUTAGE: AllServicesHealth = { + llmGateway: { status: ServiceHealthStatus.Down, error: 'HTTP 503' }, + skillsOrigin: { status: ServiceHealthStatus.Healthy }, }; -const MOCK_NO_CONNECTION: AllServicesHealth = { - anthropic: HEALTHY, - posthogOverall: HEALTHY, - posthogComponents: { status: ServiceHealthStatus.Healthy }, - github: HEALTHY, - npmOverall: HEALTHY, - npmComponents: { status: ServiceHealthStatus.Healthy }, - cloudflareOverall: HEALTHY, - cloudflareComponents: { status: ServiceHealthStatus.Healthy }, - mcp: { +const MOCK_SKILLS_UNAVAILABLE: AllServicesHealth = { + skillsOrigin: { status: ServiceHealthStatus.NoConnection, - error: 'fetch failed', + error: 'No configured skills source is reachable', }, - skillsOrigin: HEALTHY, }; -type Phase = 'checking' | 'confirmed' | 'no-connection'; +type Phase = 'checking' | 'gateway' | 'skills'; export const HealthCheckDemo = () => { const [phase, setPhase] = useState('checking'); useEffect(() => { - const t1 = setTimeout(() => setPhase('confirmed'), 2000); - const t2 = setTimeout(() => setPhase('no-connection'), 7000); - const t3 = setTimeout(() => setPhase('checking'), 12000); - return () => { - clearTimeout(t1); - clearTimeout(t2); - clearTimeout(t3); - }; + const timer = setTimeout( + () => + setPhase( + phase === 'checking' + ? 'gateway' + : phase === 'gateway' + ? 'skills' + : 'checking', + ), + phase === 'checking' ? 2000 : 5000, + ); + return () => clearTimeout(timer); }, [phase]); if (phase === 'checking') { @@ -88,56 +53,49 @@ export const HealthCheckDemo = () => { alignItems="center" justifyContent="center" > - + ); } - const health = - phase === 'confirmed' ? MOCK_CONFIRMED_OUTAGE : MOCK_NO_CONNECTION; + const skillsUnavailable = phase === 'skills'; + const health = skillsUnavailable + ? MOCK_SKILLS_UNAVAILABLE + : MOCK_GATEWAY_OUTAGE; const blockingKeys = getBlockingServiceKeys(health); - const isNoConnection = phase === 'no-connection'; + const allNoConnection = blockingKeys.every( + (key) => health[key]?.status === ServiceHealthStatus.NoConnection, + ); return ( - Continue [Enter] / Exit [Esc] (disabled in playground) + {skillsUnavailable ? 'Exit [Esc]' : 'Continue [Enter] / Exit [Esc]'}{' '} + (disabled in playground) } > - - - {Icons.squareFilled} - Down - {Icons.squareFilled} - Degraded - {Icons.squareFilled} - No connection - - - - - {isNoConnection - ? "We couldn't reach these services. PostHog's status page shows no incidents, likely a network issue (VPN, firewall, captive portal, or Wi-Fi)." - : 'The wizard may not work reliably while services are affected.'} + {skillsUnavailable + ? 'The Wizard could not download the skills it needs from any configured source. Check your connection and try again.' + : 'The PostHog AI gateway is currently unavailable. You can try continuing, or exit and try again later.'} ); diff --git a/src/ui/tui/router.ts b/src/ui/tui/router.ts index c9b887072..757a64fe8 100644 --- a/src/ui/tui/router.ts +++ b/src/ui/tui/router.ts @@ -64,23 +64,20 @@ export class WizardRouter { return this.overlays[this.overlays.length - 1]; } + // Terminal errors must remain dismissible even when auth, a health check, + // or a composed run never completes. This changes only screen selection; + // the runnable gates stay blocked until the process exits. + if ( + session.runPhase === RunPhase.Error && + session.outroData && + !session.outroDismissed + ) { + return ScreenId.Outro; + } + for (const entry of this.sequence) { if (entry.show && !entry.show(session)) continue; if (entry.isComplete && entry.isComplete(session)) continue; - // A failed login aborts the run: wizardAbort renders the error outro - // and then waits for its dismissal. But the auth step only completes - // on credentials — which an aborted login never set — so the walk - // would park here forever: auth spinner up, outro unreachable, and - // that wait deadlocked. Route to the outro so the error can be read - // and dismissed. Auth only: the run steps already complete on - // RunPhase.Error, so later aborts reach their program's own outro. - if ( - entry.id === ScreenId.Auth && - session.runPhase === RunPhase.Error && - session.outroData - ) { - return ScreenId.Outro; - } return entry.id; } diff --git a/src/ui/tui/screens/health/HealthCheckScreen.tsx b/src/ui/tui/screens/health/HealthCheckScreen.tsx index 8627e3ec7..1f89888e5 100644 --- a/src/ui/tui/screens/health/HealthCheckScreen.tsx +++ b/src/ui/tui/screens/health/HealthCheckScreen.tsx @@ -17,15 +17,12 @@ import { } from '@ui/tui/primitives/index'; import { Colors, Icons } from '@ui/tui/styles'; import { ServiceHealthList } from '@ui/tui/components/ServiceHealthList'; -import { - getBlockingServiceKeys, - SIGNUP_WIZARD_READINESS_CONFIG, -} from '@lib/health-checks/readiness'; +import { getBlockingServiceKeys } from '@lib/health-checks/readiness'; import { ServiceHealthStatus } from '@lib/health-checks/types'; import { wizardAbort } from '@utils/wizard-abort'; import { ErrorCodes } from '@lib/errors'; import { fetchSkillMenu, downloadSkill } from '@lib/wizard-tools'; -import { GITHUB_SKILLS_BASE_URL } from '@lib/constants'; +import { getSkillsBaseUrl } from '@lib/constants'; import { useDismissOnAnyKey } from '@ui/tui/hooks/useDismissOnAnyKey'; interface HealthCheckScreenProps { @@ -68,6 +65,7 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { const [downloaded, setDownloaded] = useState(false); const [downloading, setDownloading] = useState(false); + const [downloadError, setDownloadError] = useState(null); const result = store.session.readinessResult; @@ -84,87 +82,78 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { alignItems="center" justifyContent="center" > - + ); } - const isSignup = store.session.signup; - const blockingKeys = getBlockingServiceKeys( - result.health, - isSignup ? SIGNUP_WIZARD_READINESS_CONFIG : undefined, - ); - - // Signup has a narrower block list (only posthog + llm-gateway), so - // services like Anthropic can be degraded without blocking. Surface - // those as dismissable warnings instead of silently proceeding. - const warningKeys = isSignup - ? getBlockingServiceKeys(result.health).filter( - (k) => !blockingKeys.includes(k), - ) - : []; + const blockingKeys = getBlockingServiceKeys(result.health); + if (blockingKeys.length === 0) return null; - const hasHardBlock = blockingKeys.length > 0; - const displayKeys = hasHardBlock ? blockingKeys : warningKeys; - if (displayKeys.length === 0) return null; - - const isSkillsOriginDown = - hasHardBlock && blockingKeys.includes('skillsOrigin'); + const isSkillsOriginDown = blockingKeys.includes('skillsOrigin'); const canDownloadSkills = result.health.skillsOrigin.status === ServiceHealthStatus.Healthy; const integration = store.session.integration; - - // If every blocking row is `NoConnection` (probe failed, no status-page - // corroboration), reframe the screen to point at the user's network - // instead of accusing PostHog of an outage. Mixed Down + NoConnection - // falls through to the confirmed-outage framing because there's still - // a real incident underneath. - const allBlockingHaveNoConnection = - hasHardBlock && - displayKeys.every( - (k) => result.health[k].status === ServiceHealthStatus.NoConnection, - ); - - const title = isSkillsOriginDown - ? 'Ongoing service disruptions' - : allBlockingHaveNoConnection - ? "Couldn't reach PostHog" - : hasHardBlock - ? 'Ongoing service disruptions' - : 'Service disruption detected'; + const canOfferDownload = canDownloadSkills && Boolean(integration); + const allNoConnection = blockingKeys.every( + (key) => result.health[key]?.status === ServiceHealthStatus.NoConnection, + ); + const title = allNoConnection + ? isSkillsOriginDown + ? 'Could not connect to skill downloads' + : 'Could not connect to the AI gateway' + : isSkillsOriginDown + ? 'Skill downloads unavailable' + : 'AI gateway unavailable'; const docsUrl = store.session.frameworkConfig?.metadata.docsUrl; const description = isSkillsOriginDown - ? "The Wizard can't download the skills it needs — neither GitHub Releases nor PostHog's mirror is reachable right now." - : allBlockingHaveNoConnection - ? "We couldn't reach these services from this machine. PostHog's status page shows no incidents, so this is most likely a network issue — VPN, firewall, captive portal, or flaky Wi-Fi." - : hasHardBlock - ? 'The Wizard cannot start while these services are down.' - : 'Some services are degraded. You can continue, but parts of the wizard may not work reliably.'; + ? 'The Wizard could not download the skills it needs from any configured source. Check your connection and try again.' + : allNoConnection + ? 'The Wizard could not connect to the PostHog AI gateway from this machine. Check your connection and try again.' + : 'The PostHog AI gateway is currently unavailable. You can try continuing, or exit and try again later.'; const handleDownloadAndExit = async () => { - if (downloading) return; + if (downloading || !integration) return; setDownloading(true); - // Primary origin — fetchSkillMenu/downloadSkill fail over to AWS themselves. - const menu = await fetchSkillMenu(GITHUB_SKILLS_BASE_URL); - if (menu) { + setDownloadError(null); + try { + // Use the same source as the run; release downloads fail over themselves. + const menu = await fetchSkillMenu(getSkillsBaseUrl()); + if (!menu) throw new Error('Could not load the integration skills.'); const prefix = `integration-${integration}`; const skills = (menu.categories['integration'] ?? []).filter((s) => s.id.startsWith(prefix), ); + if (skills.length === 0) { + throw new Error('No integration skills were found for this project.'); + } for (const skill of skills) { - // Pre-auth outage cache: no gateway, so a flagged skill fails closed. - await downloadSkill(skill, store.session.installDir, { + // The gateway is unavailable, so a flagged skill must fail closed. + const installed = await downloadSkill(skill, store.session.installDir, { skillsRoot: '.posthog/skills', triage: undefined, }); + if (!installed.success) { + throw new Error( + 'The integration skills could not be downloaded safely.', + ); + } } + setDownloaded(true); + } catch (error) { + setDownloadError( + error instanceof Error + ? error.message + : 'Could not download the integration skills.', + ); + } finally { + setDownloading(false); } - setDownloaded(true); }; const handleCancel = - canDownloadSkills && !isSkillsOriginDown + canOfferDownload && !isSkillsOriginDown && !downloadError ? () => void handleDownloadAndExit() : () => void wizardAbort({ @@ -173,7 +162,7 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { }); const cancelLabel = - canDownloadSkills && !isSkillsOriginDown + canOfferDownload && !isSkillsOriginDown && !downloadError ? downloading ? 'Downloading...' : 'Download skills & Exit [Esc]' @@ -181,9 +170,7 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { return ( { } > - - - {Icons.squareFilled} - Down - {Icons.squareFilled} - Degraded - {Icons.squareFilled} - No connection - - - @@ -245,7 +221,13 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { )} - {canDownloadSkills && !isSkillsOriginDown && ( + {downloadError && ( + + {downloadError} + + )} + + {canOfferDownload && !isSkillsOriginDown && !downloadError && ( You can still download the PostHog integration skills and continue diff --git a/src/ui/tui/store.ts b/src/ui/tui/store.ts index 12cf174ab..47c56f69c 100644 --- a/src/ui/tui/store.ts +++ b/src/ui/tui/store.ts @@ -130,10 +130,7 @@ const MAX_STATUS_MESSAGES = EXPANDED_COUNT; /** * Fired once per blocked readiness result, so we can quantify how often - * the wizard refuses to start and — crucially — split that between - * confirmed PostHog outages and probe-level reachability failures that - * are most likely the user's network. Helps us decide whether the - * health-check UX is over-firing. + * the wizard pauses for its gateway or for unavailable skill downloads. */ function captureHealthCheckBlocked(result: WizardReadinessResult): void { try { @@ -153,10 +150,9 @@ function captureHealthCheckBlocked(result: WizardReadinessResult): void { ? 'no-connection' : 'confirmed-outage'; - const posthogStatus = health.posthogOverall?.status; const retriesUsed = Math.max( 0, - ...(['mcp', 'skillsOrigin'] as const).map((k) => { + ...(['llmGateway', 'skillsOrigin'] as const).map((k) => { const ind = health[k]?.rawIndicator ?? ''; const m = ind.match(/attempts=(\d+)/); return m ? Number(m[1]) - 1 : 0; @@ -166,11 +162,6 @@ function captureHealthCheckBlocked(result: WizardReadinessResult): void { analytics.wizardCapture('health check blocked', { decision, blocking_keys: blockingKeys, - posthog_status_reachable: - posthogStatus !== ServiceHealthStatus.NoConnection, - posthog_status_reports_incident: - posthogStatus === ServiceHealthStatus.Down || - posthogStatus === ServiceHealthStatus.Degraded, retries_used: retriesUsed, }); } catch (err) { @@ -558,10 +549,15 @@ export class WizardStore { } setReadinessResult(result: WizardReadinessResult | null): void { - this.$session.setKey('readinessResult', result); - if (result && result.decision === WizardReadiness.No) { + const newBlockedResult = + result && + result !== this.session.readinessResult && + result.decision === WizardReadiness.No; + if (newBlockedResult) { + this.$session.setKey('outageDismissed', false); captureHealthCheckBlocked(result); } + this.$session.setKey('readinessResult', result); this.emitChange(); } diff --git a/src/utils/anthropic-status.ts b/src/utils/anthropic-status.ts deleted file mode 100644 index 5f05fbfa5..000000000 --- a/src/utils/anthropic-status.ts +++ /dev/null @@ -1,73 +0,0 @@ -const CLAUDE_STATUS_URL = 'https://status.claude.com/api/v2/status.json'; - -type StatusIndicator = 'none' | 'minor' | 'major' | 'critical'; - -interface ClaudeStatusResponse { - page: { - id: string; - name: string; - url: string; - time_zone: string; - updated_at: string; - }; - status: { - indicator: StatusIndicator; - description: string; - }; -} - -export type StatusCheckResult = - | { status: 'operational' } - | { status: 'degraded'; description: string } - | { status: 'down'; description: string } - | { status: 'unknown'; error: string }; - -/** - * Check the Anthropic/Claude status page for service health. - * Pure function — no UI calls. - */ -export async function checkAnthropicStatus(): Promise { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 5000); - - const response = await fetch(CLAUDE_STATUS_URL, { - signal: controller.signal, - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - return { - status: 'unknown', - error: `Status page returned ${response.status}`, - }; - } - - const data = (await response.json()) as ClaudeStatusResponse; - const indicator = data.status.indicator; - const rawDesc = data.status.description; - const description = - rawDesc.charAt(0).toUpperCase() + rawDesc.slice(1).toLowerCase(); - - switch (indicator) { - case 'none': - return { status: 'operational' }; - case 'minor': - return { status: 'degraded', description }; - case 'major': - case 'critical': - return { status: 'down', description }; - default: - return { status: 'unknown', error: `Unknown indicator: ${indicator}` }; - } - } catch (error) { - if (error instanceof Error && error.name === 'AbortError') { - return { status: 'unknown', error: 'Request timed out' }; - } - return { - status: 'unknown', - error: error instanceof Error ? error.message : 'Unknown error', - }; - } -} From 97695376f7e2fde09b1a4f66a1088179a3df4672 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 9 Sep 2026 10:48:11 -0400 Subject: [PATCH 2/3] refactor(health): narrow readiness changes --- README.md | 78 +- src/lib/__tests__/gateway-session.test.ts | 42 + .../shared/__tests__/bootstrap-health.test.ts | 267 ---- src/lib/agent/runner/shared/bootstrap.ts | 88 +- src/lib/gateway-session.ts | 10 + .../health-checks/__tests__/endpoints.test.ts | 276 ---- .../__tests__/health-checks.test.ts | 1418 +++++++++++++++-- src/lib/health-checks/endpoints.ts | 314 ++-- src/lib/health-checks/incidentio.ts | 167 ++ src/lib/health-checks/index.ts | 22 +- src/lib/health-checks/readiness.ts | 303 +++- src/lib/health-checks/statuspage.ts | 144 ++ src/lib/health-checks/testme.md | 77 +- src/lib/health-checks/types.ts | 29 +- src/lib/programs/shared/health-check-step.ts | 22 +- src/ui/logging-ui.ts | 5 +- src/ui/tui/__tests__/ink-ui-health.test.ts | 85 - src/ui/tui/__tests__/programs.test.ts | 42 +- src/ui/tui/__tests__/router.test.ts | 76 - src/ui/tui/__tests__/store.test.ts | 4 +- src/ui/tui/components/ServiceHealthList.tsx | 59 +- src/ui/tui/ink-ui.ts | 6 +- .../tui/playground/demos/HealthCheckDemo.tsx | 126 +- src/ui/tui/router.ts | 25 +- .../tui/screens/health/HealthCheckScreen.tsx | 134 +- src/ui/tui/store.ts | 22 +- src/utils/anthropic-status.ts | 73 + 27 files changed, 2598 insertions(+), 1316 deletions(-) delete mode 100644 src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts delete mode 100644 src/lib/health-checks/__tests__/endpoints.test.ts create mode 100644 src/lib/health-checks/incidentio.ts create mode 100644 src/lib/health-checks/statuspage.ts delete mode 100644 src/ui/tui/__tests__/ink-ui-health.test.ts create mode 100644 src/utils/anthropic-status.ts diff --git a/README.md b/README.md index 432902798..ad120f6ff 100644 --- a/README.md +++ b/README.md @@ -598,37 +598,53 @@ To make your version of a tool usable with a one-line `npx` command: # Health checks -`src/lib/health-checks/` checks the dependencies the Wizard actually uses: - -- **Skills downloads:** fetch and validate `skill-menu.json` from GitHub - Releases and the AWS mirror. Either source working is healthy. A single-origin - outage does not warn or interrupt the run. Local context-mill runs check their - local server instead. -- **LLM gateway:** after login and token mint, check `/readyz` on the exact - `gateway_url` returned by the backend. This works with regional, custom, and - local gateways without hardcoding the old gateway hostname. - -Anthropic, GitHub, npm, Cloudflare, MCP, and general PostHog status pages are -not queried or displayed. An individual provider outage does not establish a -gateway outage: provider routing and fallback belong to the gateway. - -The shared health screen checks skills before login. Bootstrap reuses that -result and checks the minted gateway before starting the agent. Signup uses the -same policy. Programs without a health screen skip these advisory checks. - -A failed gateway probe or unavailable skills sources interrupts the run. Network -failures are labelled as connection problems, without claiming a confirmed -service outage. Inconclusive checks do not produce warnings. Users can continue -past a gateway warning or download available skills to use with another agent; -when neither skills origin works, the screen offers exit and manual setup docs. -CI reports failures and continues, as before. - -| File | Responsibility | -| -------------- | ---------------------------------------------------------- | -| `types.ts` | Health results for the gateway and skills downloads | -| `endpoints.ts` | Direct probes, bounded retries, and skills mirror fallback | -| `readiness.ts` | Aggregate checks and the two-dependency outage policy | -| `testme.md` | Focused test instructions and endpoint reference | +`src/lib/health-checks/` checks external status pages and PostHog-owned +services before the wizard runs to decide whether it can proceed. The entry +point is `evaluateWizardReadiness()`, which only blocks on skill downloads: + +| Decision | Meaning | +| ------------------- | --------------------------------------------------------------- | +| `yes` | Skills are reachable — proceed without outage warnings. | +| `no` | Neither skills origin is reachable — do not run. | + +### Module layout + +| File | Responsibility | +| --- | --- | +| `types.ts` | Enums, interfaces (`ServiceHealthStatus`, `AllServicesHealth`, etc.) | +| `statuspage.ts` | Statuspage.io v2 API helpers + checks for Anthropic, PostHog, GitHub, npm, Cloudflare | +| `endpoints.ts` | Direct endpoint checks for MCP (`/`) and the skills origins (`skill-menu.json` on GitHub Releases + the AWS mirror) | +| `readiness.ts` | `checkAllExternalServices`, `evaluateWizardReadiness`, readiness config | +| `index.ts` | Barrel re-export | +| `testme.md` | Test running instructions and endpoint reference | + +## What blocks a run + +The `DEFAULT_WIZARD_READINESS_CONFIG` in `readiness.ts` controls this. It has +two arrays: + +- **`downBlocksRun`** — if any of these report status **Down**, readiness is + **No**. +- **`degradedBlocksRun`** — if any of these report **Degraded** (or worse), + readiness is **No**. + +### Current defaults + +```ts +downBlocksRun: ['skillsOrigin'], +``` + +The same policy applies during signup. Other status-page results do not warn or +block. After minting a token, `gateway-session.ts` checks `/readyz` on the returned +gateway URL and reports an unavailable gateway through the existing error path. + +`skillsOrigin` is one entry covering two origins: skills are published to +GitHub Releases and an AWS mirror under the same filenames, and downloads fail +over between them (`src/lib/fetch-retry.ts`). Both are probed in parallel, so +the key only reports **Down** when neither origin answers — a GitHub Releases +outage on its own doesn't block a run, including a 403 or 404, which is as +often about the origin (expired asset redirect, blocked region, a publish that +reached one origin and not the other) as about the asset. ## Smoke test helper (`scripts/smoke-test-ci.sh`) diff --git a/src/lib/__tests__/gateway-session.test.ts b/src/lib/__tests__/gateway-session.test.ts index 5e5213b7f..8ac4fdacf 100644 --- a/src/lib/__tests__/gateway-session.test.ts +++ b/src/lib/__tests__/gateway-session.test.ts @@ -13,6 +13,12 @@ import { ErrorCodes } from '@lib/errors'; import { WizardError } from '@utils/wizard-abort'; import { analytics } from '@utils/analytics'; import { logToFile } from '@utils/debug'; +import { checkLlmGatewayHealth } from '@lib/health-checks/endpoints'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; + +vi.mock('@lib/health-checks/endpoints', () => ({ + checkLlmGatewayHealth: vi.fn(), +})); vi.mock('@utils/analytics', () => ({ analytics: { wizardCapture: vi.fn(), captureException: vi.fn() }, @@ -45,6 +51,9 @@ describe('gatewayAuth', () => { beforeEach(() => { resetGatewaySession(); fetchMock.mockReset(); + vi.mocked(checkLlmGatewayHealth) + .mockReset() + .mockResolvedValue({ status: ServiceHealthStatus.Healthy }); vi.mocked(analytics.wizardCapture).mockClear(); vi.mocked(logToFile).mockClear(); vi.stubGlobal('fetch', fetchMock); @@ -86,8 +95,41 @@ describe('gatewayAuth', () => { // Second resolve inside the TTL reuses the cache, so no second mint. await gatewayAuth(host, 'pha_oauth', 'integration'); expect(fetchMock).toHaveBeenCalledTimes(1); + expect(checkLlmGatewayHealth).toHaveBeenCalledExactlyOnceWith( + 'https://gateway.us.posthog.com', + ); }); + it.each([ServiceHealthStatus.Down, ServiceHealthStatus.NoConnection])( + 'reports gateway %s without exposing diagnostics or caching auth', + async (status) => { + fetchMock.mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + token: 'phe_minted', + expires_at: new Date(Date.now() + 3600_000).toISOString(), + gateway_url: 'https://ai-gateway.us.posthog.com', + }), + }); + vi.mocked(checkLlmGatewayHealth).mockResolvedValueOnce({ + status, + error: 'private dependency details', + }); + await expect( + gatewayAuth(host, 'pha_oauth', 'integration'), + ).rejects.toMatchObject({ + name: 'WizardError', + code: ErrorCodes.EnvServiceOutage, + message: + 'The PostHog AI gateway is unavailable. Please try again later.', + }); + await gatewayAuth(host, 'pha_oauth', 'integration'); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(checkLlmGatewayHealth).toHaveBeenCalledTimes(2); + }, + ); + it('records a successful mint without ever logging the token', async () => { fetchMock.mockResolvedValue({ ok: true, diff --git a/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts b/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts deleted file mode 100644 index fa25867ce..000000000 --- a/src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { bootstrapProgram } from '../bootstrap'; -import { authenticate, refreshAccessTokenIfNeeded } from '../authenticate'; -import { buildSession, type Credentials } from '@lib/wizard-session'; -import { HostResolution } from '@lib/host-resolution'; -import { gatewayAuth } from '@lib/gateway-session'; -import { createTriageLLMProvider } from '@lib/agent/triage-provider'; -import { - checkLlmGatewayHealth, - checkSkillsOriginHealth, -} from '@lib/health-checks/endpoints'; -import { - WizardReadiness, - type WizardReadinessResult, -} from '@lib/health-checks/readiness'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; -import { wizardAbort } from '@utils/wizard-abort'; -import type { ProgramConfig } from '@lib/programs/program-step'; -import type { ProgramRun } from '../types'; - -const ui = vi.hoisted(() => ({ - showBlockingOutage: vi.fn<() => Promise>(), - waitForAiOptIn: vi.fn<() => Promise>(), - waitForGate: vi.fn<() => Promise>(), -})); - -vi.mock('@ui', () => ({ getUI: () => ui })); -vi.mock('../authenticate', () => ({ - authenticate: vi.fn(), - refreshAccessTokenIfNeeded: vi.fn(), -})); -vi.mock('@lib/gateway-session', () => ({ gatewayAuth: vi.fn() })); -vi.mock('@lib/health-checks/endpoints', () => ({ - checkLlmGatewayHealth: vi.fn(), - checkSkillsOriginHealth: vi.fn(), -})); -vi.mock('@lib/agent/triage-provider', () => ({ - createTriageLLMProvider: vi.fn(), -})); -vi.mock('@lib/programs/posthog-integration/detect', () => ({ - maybeStampAiSdkDetected: vi.fn(), -})); -vi.mock('@lib/agent/runner/switchboard', () => ({ - resolveHarness: () => ({ harness: 'anthropic' }), -})); -vi.mock('@lib/agent/agent-interface', () => ({ buildRunTags: () => ({}) })); -vi.mock('@lib/agent/claude-settings', () => ({ - checkAllSettingsConflicts: () => [], - backupAndFixClaudeSettings: vi.fn(), - classifySettingsConflicts: vi.fn(), -})); -vi.mock('@utils/analytics', () => ({ - analytics: { - build: 'test', - runId: 'test-run', - wizardCapture: vi.fn(), - getAllFlagsForWizard: () => Promise.resolve({}), - getWizardFlagPayloads: () => ({}), - }, -})); -vi.mock('@utils/debug', () => ({ - initLogFile: vi.fn(), - logToFile: vi.fn(), - enableDebugLogs: vi.fn(), -})); -vi.mock('@utils/wizard-abort', () => ({ wizardAbort: vi.fn() })); - -const run: ProgramRun = { - integrationLabel: 'health-test', - spinnerMessage: 'Running', - successMessage: 'Done', - estimatedDurationMinutes: 1, - reportFile: 'report.md', - docsUrl: 'https://example.com/docs', -}; - -function program(hasHealthScreen = true): ProgramConfig { - return { - id: 'health-test', - description: 'Health lifecycle test', - steps: [ - ...(hasHealthScreen - ? [{ id: 'health', label: 'Health', screenId: 'health-check' }] - : []), - { id: 'auth', label: 'Auth', screenId: 'auth' }, - { id: 'run', label: 'Run', screenId: 'run' }, - ], - }; -} - -function credentials(accessToken = 'test-access-token'): Credentials { - return { - accessToken, - projectApiKey: 'test-project-key', - host: HostResolution.fromRegion('us'), - projectId: 1, - }; -} - -function preflight(status: ServiceHealthStatus): WizardReadinessResult { - return { - decision: - status === ServiceHealthStatus.Healthy - ? WizardReadiness.Yes - : WizardReadiness.No, - health: { skillsOrigin: { status } }, - reasons: [], - }; -} - -function deferred() { - let resolve!: () => void; - const promise = new Promise((done) => { - resolve = done; - }); - return { promise, resolve }; -} - -describe('bootstrap health lifecycle', () => { - beforeEach(() => { - vi.resetAllMocks(); - ui.showBlockingOutage.mockResolvedValue(); - ui.waitForAiOptIn.mockResolvedValue(); - ui.waitForGate.mockResolvedValue(); - vi.mocked(authenticate).mockImplementation((session) => { - session.credentials = credentials(); - return Promise.resolve(); - }); - vi.mocked(refreshAccessTokenIfNeeded).mockResolvedValue(); - vi.mocked(gatewayAuth).mockResolvedValue({ - gatewayUrl: 'https://gateway.example.com', - token: 'test-scoped-token', - refreshAtMs: Date.now() + 60_000, - }); - vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ - status: ServiceHealthStatus.Healthy, - }); - vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ - status: ServiceHealthStatus.Healthy, - }); - }); - - it.each([ - 'https://gateway.eu.example.com', - 'http://localhost:8766', - 'https://custom.example.com/nested/gateway', - ])( - 'checks the minted URL %s despite a cached healthy preflight', - async (url) => { - const session = buildSession({}); - session.readinessResult = preflight(ServiceHealthStatus.Healthy); - const mint = deferred(); - vi.mocked(gatewayAuth).mockImplementation(async () => { - await mint.promise; - return { - gatewayUrl: url, - token: 'test-scoped-token', - refreshAtMs: 60_000, - }; - }); - - const boot = bootstrapProgram(session, run, program()); - await vi.waitFor(() => expect(gatewayAuth).toHaveBeenCalledOnce()); - expect(authenticate).toHaveBeenCalledOnce(); - expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); - - mint.resolve(); - await boot; - - expect(checkLlmGatewayHealth).toHaveBeenCalledExactlyOnceWith(url); - expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); - expect(ui.showBlockingOutage).not.toHaveBeenCalled(); - }, - ); - - it.each([false, true])( - 'skips advisory checks without a health screen (signup=%s)', - async (signup) => { - await bootstrapProgram(buildSession({ signup }), run, program(false)); - - expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); - expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); - expect(ui.showBlockingOutage).not.toHaveBeenCalled(); - expect(gatewayAuth).toHaveBeenCalledOnce(); - }, - ); - - it.each([false, true])( - 'waits for a skills outage dismissal before auth, then checks the gateway (signup=%s)', - async (signup) => { - const dismissal = deferred(); - ui.showBlockingOutage.mockReturnValueOnce(dismissal.promise); - vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ - status: ServiceHealthStatus.Down, - }); - - const boot = bootstrapProgram(buildSession({ signup }), run, program()); - await vi.waitFor(() => - expect(ui.showBlockingOutage).toHaveBeenCalledOnce(), - ); - expect(authenticate).not.toHaveBeenCalled(); - expect(gatewayAuth).not.toHaveBeenCalled(); - - dismissal.resolve(); - await boot; - - expect(checkSkillsOriginHealth).toHaveBeenCalledOnce(); - expect(checkLlmGatewayHealth).toHaveBeenCalledExactlyOnceWith( - 'https://gateway.example.com', - ); - expect(ui.showBlockingOutage).toHaveBeenCalledOnce(); - expect(wizardAbort).not.toHaveBeenCalled(); - }, - ); - - it('does not repeat a cached, dismissed skills warning when the gateway is healthy', async () => { - const session = buildSession({}); - session.readinessResult = preflight(ServiceHealthStatus.Down); - - await bootstrapProgram(session, run, program()); - - expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); - expect(checkLlmGatewayHealth).toHaveBeenCalledOnce(); - expect(ui.showBlockingOutage).not.toHaveBeenCalled(); - }); - - it('pauses a new gateway outage, then refreshes credentials and resumes after dismissal', async () => { - const session = buildSession({}); - session.readinessResult = preflight(ServiceHealthStatus.Down); - const dismissal = deferred(); - ui.showBlockingOutage.mockReturnValueOnce(dismissal.promise); - vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ - status: ServiceHealthStatus.NoConnection, - }); - vi.mocked(refreshAccessTokenIfNeeded) - .mockResolvedValueOnce() - .mockImplementationOnce((current) => { - current.credentials = credentials('refreshed-access-token'); - return Promise.resolve(); - }); - - const boot = bootstrapProgram(session, run, program()); - await vi.waitFor(() => - expect(ui.showBlockingOutage).toHaveBeenCalledOnce(), - ); - expect(refreshAccessTokenIfNeeded).toHaveBeenCalledOnce(); - expect(createTriageLLMProvider).not.toHaveBeenCalled(); - - dismissal.resolve(); - const result = await boot; - - expect(refreshAccessTokenIfNeeded).toHaveBeenCalledTimes(2); - expect(wizardAbort).not.toHaveBeenCalled(); - expect(result.credentials.accessToken).toBe('refreshed-access-token'); - const resolveTriageAuth = vi.mocked(createTriageLLMProvider).mock - .calls[0]?.[0]; - expect(typeof resolveTriageAuth).toBe('function'); - if (typeof resolveTriageAuth !== 'function') { - throw new Error('Expected a live gateway auth resolver'); - } - await resolveTriageAuth(); - expect(gatewayAuth).toHaveBeenLastCalledWith( - result.credentials.host, - 'refreshed-access-token', - program().id, - ); - }); -}); diff --git a/src/lib/agent/runner/shared/bootstrap.ts b/src/lib/agent/runner/shared/bootstrap.ts index 974619e2f..6c2bd794e 100644 --- a/src/lib/agent/runner/shared/bootstrap.ts +++ b/src/lib/agent/runner/shared/bootstrap.ts @@ -24,7 +24,9 @@ import { import { evaluateWizardReadiness, WizardReadiness, + SIGNUP_WIZARD_READINESS_CONFIG, getBlockingServiceKeys, + SERVICE_LABELS, } from '@lib/health-checks/readiness'; import { enableDebugLogs, logToFile, initLogFile } from '@utils/debug'; import { wizardAbort } from '@utils/wizard-abort'; @@ -110,17 +112,53 @@ export async function bootstrapProgram( `posthog=${session.baseUrl ?? 'region-resolved'}`, ); - // Pre-auth checks cover skill downloads only. The gateway URL is not known - // until mint; a cached TUI result must not suppress that later check. - // Programs without the health screen skip these advisory checks entirely. + // 2. Health check (guarded — skip if TUI already ran it). Only + // programs that declare a health-check screen get pre-flight checks; + // for everything else the checks never fire and never block. const hasHealthCheckScreen = programConfig.steps.some( (s) => s.screenId === 'health-check', ); - let preflight = session.readinessResult; - if (hasHealthCheckScreen && !preflight) { - preflight = await evaluateWizardReadiness(); - if (preflight.decision === WizardReadiness.No) { - await getUI().showBlockingOutage(preflight); + if (session.readinessResult) { + logToFile( + `[agent-runner] readiness pre-computed by TUI: decision=${session.readinessResult.decision}` + + `${ + session.outageDismissed ? ' (outage dismissed by user)' : '' + } — skipping re-check`, + ); + } + if (hasHealthCheckScreen && !session.readinessResult) { + logToFile('[agent-runner] evaluating wizard readiness'); + const readinessConfig = session.signup + ? SIGNUP_WIZARD_READINESS_CONFIG + : undefined; + const readiness = await evaluateWizardReadiness(readinessConfig); + logToFile(`[agent-runner] readiness=${readiness.decision}`); + if (readiness.decision === WizardReadiness.No) { + const blockingKeys = getBlockingServiceKeys( + readiness.health, + readinessConfig, + ); + const blockingLabels = blockingKeys.map( + (k) => `${SERVICE_LABELS[k]} (${readiness.health[k].status})`, + ); + logToFile(`[agent-runner] blocked by: ${blockingLabels.join(', ')}`); + + await getUI().showBlockingOutage(readiness); + + // The TUI lets the user continue past an outage; non-interactive runs + // (CI) do the same automatically — the degraded services are reported + // above, but we proceed rather than aborting on a transient upstream blip. + if (!isNonInteractiveEnvironment()) { + await wizardAbort({ + code: ErrorCodes.EnvServiceOutage, + message: + 'Cannot start — external services are down:\n' + + blockingLabels.map((l) => ` - ${l}`).join('\n') + + '\n\nPlease try again later.', + }); + } + } else if (readiness.decision === WizardReadiness.YesWithWarnings) { + getUI().setReadinessWarnings(readiness); } } @@ -264,32 +302,18 @@ export async function bootstrapProgram( // The agent can't swap tokens mid-run, so freshness is measured after every park above, right before the mint. await refreshAccessTokenIfNeeded(session); - // Read live credentials so a refresh after an outage dismissal is used by - // later mints, too. No agent or skill triage starts before the initial mint. - const currentGatewayAuth = () => { - const credentials = session.credentials!; - return gatewayAuth( - credentials.host, - credentials.accessToken, - programConfig.id, - ); - }; - const auth = await currentGatewayAuth(); - - if (hasHealthCheckScreen) { - const readiness = await evaluateWizardReadiness({ - gatewayUrl: auth.gatewayUrl, - skillsHealth: preflight?.health.skillsOrigin, - }); - // A previously dismissed skills result must not hide a new gateway outage - // or cause the same skills warning to be displayed a second time. - if (getBlockingServiceKeys(readiness.health).includes('llmGateway')) { - await getUI().showBlockingOutage(readiness); - await refreshAccessTokenIfNeeded(session); - } - } + // Credentials (incl. the resolved host family and its MCP url) live on + // `session.credentials`; narrow once at this boundary — `authenticate` above + // set them — so downstream readers get a non-null type without asserting. const credentials = session.credentials!; + // Mint now so a refusal fails the boot before any agent starts. Later + // readers re-resolve through the cache, which re-mints past the refresh + // point. + const currentGatewayAuth = () => + gatewayAuth(credentials.host, credentials.accessToken, programConfig.id); + await currentGatewayAuth(); + return { skillsBaseUrl, credentials, diff --git a/src/lib/gateway-session.ts b/src/lib/gateway-session.ts index b26da0238..4df8db4fe 100644 --- a/src/lib/gateway-session.ts +++ b/src/lib/gateway-session.ts @@ -11,6 +11,8 @@ import { analytics } from '@utils/analytics'; import { WizardError } from '@utils/wizard-abort'; import { ErrorCodes } from '@lib/errors'; import type { HostResolution } from '@lib/host-resolution'; +import { checkLlmGatewayHealth } from '@lib/health-checks/endpoints'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; export interface GatewayAuth { /** Base URL for model calls (no `/v1`; transports append their route). */ @@ -93,6 +95,14 @@ async function resolveGatewayAuth( ); } const minted = await mintGatewayToken(host, accessToken, program); + const health = await checkLlmGatewayHealth(minted.gatewayUrl); + if (health.status !== ServiceHealthStatus.Healthy) { + throw new WizardError( + 'The PostHog AI gateway is unavailable. Please try again later.', + undefined, + ErrorCodes.EnvServiceOutage, + ); + } const expiresAtMs = Date.parse(minted.expiresAt); const ttlMs = expiresAtMs - Date.now(); if (!Number.isFinite(expiresAtMs) || ttlMs < MIN_USABLE_TTL_MS) { diff --git a/src/lib/health-checks/__tests__/endpoints.test.ts b/src/lib/health-checks/__tests__/endpoints.test.ts deleted file mode 100644 index 38658629d..000000000 --- a/src/lib/health-checks/__tests__/endpoints.test.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - AWS_SKILLS_BASE_URL, - GITHUB_SKILLS_BASE_URL, - LOCAL_SKILLS_BASE_URL, -} from '@lib/constants'; -import { initLocalDev, resetLocalDev } from '@lib/local-dev'; -import { - checkLlmGatewayHealth, - checkSkillsOriginHealth, - fetchEndpointHealth, -} from '../endpoints'; -import { ServiceHealthStatus } from '../types'; - -vi.mock('@utils/debug', () => ({ logToFile: vi.fn() })); -vi.mock('@utils/analytics', () => ({ analytics: { wizardCapture: vi.fn() } })); - -const fetchMock = vi.fn(); -const primaryMenu = `${GITHUB_SKILLS_BASE_URL}/skill-menu.json`; -const fallbackMenu = `${AWS_SKILLS_BASE_URL}/skill-menu.json`; -const validMenu = { - categories: { - integration: [ - { - id: 'posthog-integration', - name: 'Install PostHog', - downloadUrl: 'posthog-integration.zip', - }, - ], - }, -}; -const menuResponse = () => new Response(JSON.stringify(validMenu)); -const httpResponse = (status: number) => new Response(null, { status }); - -async function finish(pending: Promise): Promise { - await vi.runAllTimersAsync(); - return pending; -} - -beforeEach(() => { - vi.useFakeTimers(); - fetchMock.mockReset(); - vi.stubGlobal('fetch', fetchMock); - resetLocalDev(); -}); - -afterEach(() => { - vi.unstubAllGlobals(); - vi.useRealTimers(); - resetLocalDev(); -}); - -describe('fetchEndpointHealth', () => { - it('retries a transient HTTP error and recovers', async () => { - fetchMock - .mockResolvedValueOnce(httpResponse(503)) - .mockResolvedValueOnce(httpResponse(200)); - - const result = await finish(fetchEndpointHealth('https://example.com')); - - expect(result).toEqual({ - status: ServiceHealthStatus.Healthy, - rawIndicator: 'HTTP 200 (attempts=2)', - }); - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(vi.getTimerCount()).toBe(0); - }); - - it('retains HTTP failure evidence when later attempts lose connection', async () => { - fetchMock - .mockResolvedValueOnce(httpResponse(503)) - .mockRejectedValue(new Error('Connection reset')); - - expect(await finish(fetchEndpointHealth('https://example.com'))).toEqual({ - status: ServiceHealthStatus.Down, - error: 'HTTP 503', - rawIndicator: 'HTTP 503 (attempts=3)', - }); - }); - - it('keeps network-only failures distinct from confirmed downtime', async () => { - fetchMock.mockRejectedValue(new Error('DNS lookup failed')); - - expect(await finish(fetchEndpointHealth('https://example.com'))).toEqual({ - status: ServiceHealthStatus.NoConnection, - error: 'DNS lookup failed', - rawIndicator: 'attempts=3', - }); - expect(fetchMock).toHaveBeenCalledTimes(3); - }); - - it('bounds hung requests and aborts every attempt', async () => { - fetchMock.mockImplementation(() => new Promise(() => undefined)); - - const result = await finish( - fetchEndpointHealth('https://example.com', 100), - ); - - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toBe('Request timed out after 100ms'); - expect(fetchMock).toHaveBeenCalledTimes(3); - expect( - fetchMock.mock.calls.every(([, init]) => init?.signal?.aborted), - ).toBe(true); - expect(vi.getTimerCount()).toBe(0); - }); -}); - -describe('checkLlmGatewayHealth', () => { - it.each([ - [ - 'https://ai-gateway.eu.posthog.com', - 'https://ai-gateway.eu.posthog.com/readyz', - ], - ['http://localhost:8080/', 'http://localhost:8080/readyz'], - ['https://gateway.example.com/v1', 'https://gateway.example.com/readyz'], - ])( - 'probes the origin readiness endpoint for %s without a bearer', - async (base, expected) => { - fetchMock.mockResolvedValue(httpResponse(200)); - - expect((await checkLlmGatewayHealth(base)).status).toBe( - ServiceHealthStatus.Healthy, - ); - expect(fetchMock).toHaveBeenCalledWith(expected, { - signal: expect.any(AbortSignal), - redirect: 'follow', - }); - }, - ); -}); - -describe('checkSkillsOriginHealth', () => { - it.each([primaryMenu, fallbackMenu])( - 'stays healthy when only %s serves a usable menu', - async (healthyUrl) => { - fetchMock.mockImplementation((url) => - Promise.resolve( - url === healthyUrl ? menuResponse() : httpResponse(503), - ), - ); - - const pending = checkSkillsOriginHealth(); - // Both origin requests start before either retry loop waits. - expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ - primaryMenu, - fallbackMenu, - ]); - const result = await finish(pending); - - expect(result).toEqual({ - status: ServiceHealthStatus.Healthy, - rawIndicator: 'HTTP 200', - }); - expect(result.error).toBeUndefined(); - expect(result.rawIndicator).not.toMatch(/unavailable|degraded/i); - }, - ); - - it('reports downtime only after both sources fail', async () => { - fetchMock.mockImplementation(() => Promise.resolve(httpResponse(503))); - - const result = await finish(checkSkillsOriginHealth()); - - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toBe('Both skill download sources are unavailable'); - expect(result.rawIndicator).toContain('attempts=3'); - expect(fetchMock).toHaveBeenCalledTimes(6); - }); - - it('reports no connection when neither origin can be reached', async () => { - fetchMock.mockRejectedValue(new Error('Offline')); - - expect((await finish(checkSkillsOriginHealth())).status).toBe( - ServiceHealthStatus.NoConnection, - ); - }); - - it('maps pinned releases to the same AWS release', async () => { - fetchMock.mockImplementation(() => Promise.resolve(menuResponse())); - - await checkSkillsOriginHealth( - 'https://github.com/PostHog/context-mill/releases/download/v1.2.3', - ); - - expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ - 'https://github.com/PostHog/context-mill/releases/download/v1.2.3/skill-menu.json', - 'https://context-mill.posthog.com/v1.2.3/skill-menu.json', - ]); - }); - - it.each([ - 'GitHub temporarily unavailable', - JSON.stringify({ status: 'ok' }), - JSON.stringify({ categories: {} }), - JSON.stringify({ categories: { integration: [{ id: 'incomplete' }] } }), - JSON.stringify({ categories: { integration: 'not an array' } }), - ])('rejects unusable HTTP 200 skill menus: %s', async (body) => { - fetchMock.mockImplementation(() => Promise.resolve(new Response(body))); - - expect((await finish(checkSkillsOriginHealth())).status).toBe( - ServiceHealthStatus.Down, - ); - expect(fetchMock).toHaveBeenCalledTimes(6); - }); - - it('uses the mirror when the primary HTTP 200 contains invalid JSON', async () => { - fetchMock.mockImplementation((url) => - Promise.resolve( - url === primaryMenu ? new Response('not JSON') : menuResponse(), - ), - ); - - expect((await finish(checkSkillsOriginHealth())).status).toBe( - ServiceHealthStatus.Healthy, - ); - }); - - it('retries failures while reading a successful response body', async () => { - fetchMock.mockImplementation(() => - Promise.resolve({ - status: 200, - json: () => Promise.reject(new Error('Body stream interrupted')), - } as unknown as Response), - ); - - const result = await finish(checkSkillsOriginHealth(LOCAL_SKILLS_BASE_URL)); - - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toBe('Body stream interrupted'); - expect(fetchMock).toHaveBeenCalledTimes(3); - }); - - it('keeps the timeout active until the skill menu body finishes', async () => { - fetchMock.mockImplementation(() => - Promise.resolve({ - status: 200, - json: () => new Promise(() => undefined), - } as unknown as Response), - ); - - const result = await finish(checkSkillsOriginHealth(LOCAL_SKILLS_BASE_URL)); - - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toBe('Request timed out after 5000ms'); - expect(fetchMock).toHaveBeenCalledTimes(3); - expect( - fetchMock.mock.calls.every(([, init]) => init?.signal?.aborted), - ).toBe(true); - expect(vi.getTimerCount()).toBe(0); - }); - - it('honors the local context-mill target without probing production', async () => { - initLocalDev({ localContextMill: true }); - fetchMock.mockImplementation(() => Promise.resolve(httpResponse(503))); - - expect((await finish(checkSkillsOriginHealth())).status).toBe( - ServiceHealthStatus.Down, - ); - expect(fetchMock.mock.calls.map(([url]) => url)).toEqual( - Array(3).fill(`${LOCAL_SKILLS_BASE_URL}/skill-menu.json`), - ); - }); - - it('downloads a menu from the configured custom target', async () => { - fetchMock.mockImplementation(() => Promise.resolve(menuResponse())); - - expect( - (await checkSkillsOriginHealth('http://localhost:9000/custom/')).status, - ).toBe(ServiceHealthStatus.Healthy); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(fetchMock.mock.calls[0][0]).toBe( - 'http://localhost:9000/custom/skill-menu.json', - ); - }); -}); diff --git a/src/lib/health-checks/__tests__/health-checks.test.ts b/src/lib/health-checks/__tests__/health-checks.test.ts index 2a94e5ef9..12ea8afb8 100644 --- a/src/lib/health-checks/__tests__/health-checks.test.ts +++ b/src/lib/health-checks/__tests__/health-checks.test.ts @@ -1,148 +1,1340 @@ -import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest'; +/** + * Tests for health-checks.ts + * + * Mock data is modelled on live Statuspage.io v2 API responses. + * Statuspage docs: https://metastatuspage.com/api + * + * status.json – page-level rollup with indicator (none | minor | major | critical) + * summary.json – same rollup plus component list; component statuses: + * operational | degraded_performance | partial_outage | major_outage | under_maintenance + * https://support.atlassian.com/statuspage/docs/show-service-status-with-components + + * + * MCP – Cloudflare Worker, GET / returns an HTML landing page (200) + * Source: posthog/services/mcp/src/index.ts + */ + import { checkAllExternalServices, + checkAnthropicHealth, + checkCloudflareComponentHealth, + checkCloudflareOverallHealth, + checkGithubHealth, + checkSkillsOriginHealth, + checkMcpHealth, + checkNpmComponentHealth, + checkNpmOverallHealth, + checkPosthogComponentHealth, + checkPosthogOverallHealth, + resetPosthogHealthCache, + DEFAULT_WIZARD_READINESS_CONFIG, evaluateWizardReadiness, - getBlockingServiceKeys, + ServiceHealthStatus, WizardReadiness, -} from '../readiness'; -import { checkLlmGatewayHealth, checkSkillsOriginHealth } from '../endpoints'; -import { ServiceHealthStatus, type AllServicesHealth } from '../types'; - -vi.mock('../endpoints', () => ({ - checkLlmGatewayHealth: vi.fn(), - checkSkillsOriginHealth: vi.fn(), -})); -vi.mock('@utils/debug', () => ({ logToFile: vi.fn() })); - -const healthy = { status: ServiceHealthStatus.Healthy }; -const down = { status: ServiceHealthStatus.Down }; -const unreachable = { status: ServiceHealthStatus.NoConnection }; -const gatewayUrl = 'https://ai-gateway.eu.posthog.com'; - -describe('Wizard dependency health', () => { +} from '@lib/health-checks/index'; +import { + checkLlmGatewayHealth, + fetchEndpointHealth, +} from '@lib/health-checks/endpoints'; +import { SIGNUP_WIZARD_READINESS_CONFIG } from '@lib/health-checks/readiness'; + +// --------------------------------------------------------------------------- +// Real-world Statuspage.io v2 response factories +// https://metastatuspage.com/api +// --------------------------------------------------------------------------- + +function makeStatuspageStatus(opts: { + pageId: string; + pageName: string; + pageUrl: string; + indicator: 'none' | 'minor' | 'major' | 'critical'; + description: string; +}) { + return { + page: { + id: opts.pageId, + name: opts.pageName, + url: opts.pageUrl, + time_zone: 'Etc/UTC', + updated_at: '2026-03-05T16:03:38.861Z', + }, + status: { + indicator: opts.indicator, + description: opts.description, + }, + }; +} + +function makeStatuspageSummary(opts: { + pageId: string; + pageName: string; + pageUrl: string; + indicator: 'none' | 'minor' | 'major' | 'critical'; + description: string; + components: { + id: string; + name: string; + status: string; + position: number; + description: string | null; + }[]; +}) { + return { + page: { + id: opts.pageId, + name: opts.pageName, + url: opts.pageUrl, + time_zone: 'Etc/UTC', + updated_at: '2026-03-05T16:03:38.861Z', + }, + status: { + indicator: opts.indicator, + description: opts.description, + }, + components: opts.components.map((c) => ({ + ...c, + page_id: opts.pageId, + created_at: '2023-07-11T17:52:24.275Z', + updated_at: '2026-03-04T17:01:29.960Z', + showcase: true, + start_date: '2023-07-11', + group_id: null, + group: false, + only_show_if_degraded: false, + })), + incidents: [], + scheduled_maintenances: [], + }; +} + +// Shapes taken from live GET on 2026-03-05 +const ANTHROPIC_STATUS_HEALTHY = makeStatuspageStatus({ + pageId: 'tymt9n04zgry', + pageName: 'Claude', + pageUrl: 'https://status.claude.com', + indicator: 'none', + description: 'All Systems Operational', +}); + +const GITHUB_STATUS_HEALTHY = makeStatuspageStatus({ + pageId: 'kctbh9vrtdwd', + pageName: 'GitHub', + pageUrl: 'https://www.githubstatus.com', + indicator: 'none', + description: 'All Systems Operational', +}); + +const NPM_STATUS_HEALTHY = makeStatuspageStatus({ + pageId: 'wyvgptkd90hm', + pageName: 'npm', + pageUrl: 'https://status.npmjs.org', + indicator: 'none', + description: 'All Systems Operational', +}); + +const NPM_SUMMARY_HEALTHY = makeStatuspageSummary({ + pageId: 'wyvgptkd90hm', + pageName: 'npm', + pageUrl: 'https://status.npmjs.org', + indicator: 'none', + description: 'All Systems Operational', + components: [ + { + id: 'mvm98gtxvb9b', + name: 'www.npmjs.com website', + status: 'operational', + position: 1, + description: + 'The ability for users to navigate to or interact with the npm website.', + }, + { + id: 'k1wj10x6gmph', + name: 'Package installation', + status: 'operational', + position: 2, + description: + 'The ability for users to read from the registry so that they can install packages.', + }, + ], +}); + +const CLOUDFLARE_STATUS_HEALTHY = makeStatuspageStatus({ + pageId: 'yh6f0r4529hb', + pageName: 'Cloudflare', + pageUrl: 'https://www.cloudflarestatus.com', + indicator: 'none', + description: 'All Systems Operational', +}); + +const CLOUDFLARE_SUMMARY_HEALTHY = makeStatuspageSummary({ + pageId: 'yh6f0r4529hb', + pageName: 'Cloudflare', + pageUrl: 'https://www.cloudflarestatus.com', + indicator: 'none', + description: 'All Systems Operational', + components: [ + { + id: '1km35smx8p41', + name: 'Cloudflare Sites and Services', + status: 'operational', + position: 1, + description: + 'Sites and services that Cloudflare customers use to interact with the Cloudflare Network', + }, + ], +}); + +// PostHog incident.io v1 API mock data +const POSTHOG_INCIDENTIO_HEALTHY = { + page_title: 'PostHog', + page_url: 'https://www.posthogstatus.com/', + ongoing_incidents: [], + in_progress_maintenances: [], + scheduled_maintenances: [], +}; + +// MCP / landing page (from posthog/services/mcp/src/index.ts + src/static/landing.html) +const MCP_LANDING_HTML = + 'PostHog MCP Server'; + +// --------------------------------------------------------------------------- +// URL constants (must match health-checks.ts) +// --------------------------------------------------------------------------- + +const URLS = { + anthropicStatus: 'https://status.claude.com/api/v2/status.json', + posthogIncidentIo: 'https://www.posthogstatus.com/api/v1/summary', + githubStatus: 'https://www.githubstatus.com/api/v2/status.json', + npmStatus: 'https://status.npmjs.org/api/v2/status.json', + npmSummary: 'https://status.npmjs.org/api/v2/summary.json', + cloudflareStatus: 'https://www.cloudflarestatus.com/api/v2/status.json', + cloudflareSummary: 'https://www.cloudflarestatus.com/api/v2/summary.json', + mcpLanding: 'https://mcp.posthog.com/', + githubSkillMenu: + 'https://github.com/PostHog/context-mill/releases/latest/download/skill-menu.json', + awsSkillMenu: 'https://context-mill.posthog.com/latest/skill-menu.json', +} as const; + +// --------------------------------------------------------------------------- +// Helper to build a default "all healthy" fetch mock +// --------------------------------------------------------------------------- + +const HEALTHY_RESPONSES: Record = + { + [URLS.anthropicStatus]: { + body: JSON.stringify(ANTHROPIC_STATUS_HEALTHY), + contentType: 'application/json', + }, + [URLS.posthogIncidentIo]: { + body: JSON.stringify(POSTHOG_INCIDENTIO_HEALTHY), + contentType: 'application/json', + }, + [URLS.githubStatus]: { + body: JSON.stringify(GITHUB_STATUS_HEALTHY), + contentType: 'application/json', + }, + [URLS.npmStatus]: { + body: JSON.stringify(NPM_STATUS_HEALTHY), + contentType: 'application/json', + }, + [URLS.npmSummary]: { + body: JSON.stringify(NPM_SUMMARY_HEALTHY), + contentType: 'application/json', + }, + [URLS.cloudflareStatus]: { + body: JSON.stringify(CLOUDFLARE_STATUS_HEALTHY), + contentType: 'application/json', + }, + [URLS.cloudflareSummary]: { + body: JSON.stringify(CLOUDFLARE_SUMMARY_HEALTHY), + contentType: 'application/json', + }, + [URLS.mcpLanding]: { + body: MCP_LANDING_HTML, + contentType: 'text/html; charset=utf-8', + }, + [URLS.githubSkillMenu]: { + body: JSON.stringify({ categories: { integration: [] } }), + contentType: 'application/json', + }, + [URLS.awsSkillMenu]: { + body: JSON.stringify({ categories: { integration: [] } }), + contentType: 'application/json', + }, + }; + +function allHealthyFetchMock(url: string | URL | Request): Promise { + const urlStr = + typeof url === 'string' + ? url + : url instanceof URL + ? url.toString() + : url.url; + const entry = HEALTHY_RESPONSES[urlStr]; + if (entry) { + return Promise.resolve( + new Response(entry.body, { + status: 200, + headers: { 'Content-Type': entry.contentType }, + }), + ); + } + return Promise.resolve(new Response('Not found', { status: 404 })); +} + +function overrideFetch(overrides: Record Promise>) { + return (url: string | URL | Request): Promise => { + const urlStr = + typeof url === 'string' + ? url + : url instanceof URL + ? url.toString() + : url.url; + if (overrides[urlStr]) return overrides[urlStr](); + return allHealthyFetchMock(urlStr); + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('health-checks', () => { + const originalFetch = global.fetch; + beforeEach(() => { - vi.useFakeTimers(); - vi.mocked(checkLlmGatewayHealth).mockReset().mockResolvedValue(healthy); - vi.mocked(checkSkillsOriginHealth).mockReset().mockResolvedValue(healthy); + vi.restoreAllMocks(); + resetPosthogHealthCache(); + (global as any).fetch = vi.fn(allHealthyFetchMock); + }); + + afterAll(() => { + (global as any).fetch = originalFetch; + }); + + // ----------------------------------------------------------------------- + // Statuspage status.json checks (indicator-based) + // ----------------------------------------------------------------------- + + describe('checkAnthropicHealth', () => { + it('returns healthy for indicator=none ("All Systems Operational")', async () => { + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toBe('none'); + }); + + it('returns degraded for indicator=minor ("Minor Service Outage")', async () => { + const body = makeStatuspageStatus({ + pageId: 'tymt9n04zgry', + pageName: 'Claude', + pageUrl: 'https://status.claude.com', + indicator: 'minor', + description: 'Minor Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.rawIndicator).toBe('minor'); + }); + + it('returns down for indicator=major ("Partial System Outage")', async () => { + const body = makeStatuspageStatus({ + pageId: 'tymt9n04zgry', + pageName: 'Claude', + pageUrl: 'https://status.claude.com', + indicator: 'major', + description: 'Partial System Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + }); + + it('returns down for indicator=critical ("Major Service Outage")', async () => { + const body = makeStatuspageStatus({ + pageId: 'tymt9n04zgry', + pageName: 'Claude', + pageUrl: 'https://status.claude.com', + indicator: 'critical', + description: 'Major Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + }); + + it('returns degraded when statuspage returns HTTP 500', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.resolve( + new Response('Internal Server Error', { status: 500 }), + ), + }), + ); + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.error).toBe('HTTP 500'); + }); + + it('returns degraded when fetch throws (network failure)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.reject( + new Error('getaddrinfo ENOTFOUND status.claude.com'), + ), + }), + ); + const result = await checkAnthropicHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.error).toBe('getaddrinfo ENOTFOUND status.claude.com'); + }); }); - afterEach(() => { - vi.useRealTimers(); + + describe('checkPosthogOverallHealth', () => { + it('returns healthy when no ongoing incidents', async () => { + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + }); + + it('returns down when an incident has full_outage impact', async () => { + const body = { + ...POSTHOG_INCIDENTIO_HEALTHY, + ongoing_incidents: [ + { + id: '01KA9JH0ZB14TFA8VD4CFC3AYN', + name: 'Major service outage', + status: 'identified', + current_worst_impact: 'full_outage', + affected_components: [ + { + id: 'c1', + name: 'App', + group_name: 'US Cloud', + current_status: 'full_outage', + }, + ], + url: 'https://www.posthogstatus.com/incidents/test', + last_update_at: '2026-04-22T00:00:00Z', + last_update_message: 'Investigating', + }, + ], + }; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + }); + + it('returns NoConnection when posthogstatus.com fetch fails with a network error', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.reject(new Error('getaddrinfo ENOTFOUND')), + }), + ); + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + }); + + it('returns NoConnection when posthogstatus.com fetch times out', async () => { + const abortError = new Error('aborted'); + abortError.name = 'AbortError'; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => Promise.reject(abortError), + }), + ); + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + }); + + it('returns Down when posthogstatus.com returns an HTTP error', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve(new Response('Bad Gateway', { status: 502 })), + }), + ); + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + }); + + it('returns degraded when an incident has partial_outage impact', async () => { + const body = { + ...POSTHOG_INCIDENTIO_HEALTHY, + ongoing_incidents: [ + { + id: '01KA9JH0ZB14TFA8VD4CFC3AYN', + name: 'Partial outage', + status: 'investigating', + current_worst_impact: 'partial_outage', + affected_components: [], + url: 'https://www.posthogstatus.com/incidents/test', + last_update_at: '2026-04-22T00:00:00Z', + last_update_message: 'Investigating', + }, + ], + }; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkPosthogOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + }); }); - it('checks skills before auth without guessing a gateway or reporting a warning', async () => { - const result = await evaluateWizardReadiness(); - expect(result).toEqual({ - decision: WizardReadiness.Yes, - health: { skillsOrigin: healthy }, - reasons: [], + describe('checkGithubHealth', () => { + it('returns healthy for indicator=none', async () => { + const result = await checkGithubHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); }); - expect(checkSkillsOriginHealth).toHaveBeenCalledOnce(); - expect(checkLlmGatewayHealth).not.toHaveBeenCalled(); - expect(vi.getTimerCount()).toBe(0); }); - it('uses the supplied gateway and skills targets', async () => { - const health = await checkAllExternalServices({ - gatewayUrl: 'http://localhost:8080', - skillsBaseUrl: 'http://localhost:8765', + describe('checkNpmOverallHealth', () => { + it('returns healthy for indicator=none', async () => { + const result = await checkNpmOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); }); - expect(checkLlmGatewayHealth).toHaveBeenCalledWith('http://localhost:8080'); - expect(checkSkillsOriginHealth).toHaveBeenCalledWith( - 'http://localhost:8765', - ); - expect(health).toEqual({ llmGateway: healthy, skillsOrigin: healthy }); }); - it('checks the minted gateway even when skills health was cached before auth', async () => { - vi.mocked(checkLlmGatewayHealth).mockResolvedValue(down); - const result = await evaluateWizardReadiness({ - gatewayUrl, - skillsHealth: healthy, + describe('checkCloudflareOverallHealth', () => { + it('returns healthy for indicator=none', async () => { + const result = await checkCloudflareOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + }); + + it('returns degraded for indicator=minor', async () => { + const body = makeStatuspageStatus({ + pageId: 'yh6f0r4529hb', + pageName: 'Cloudflare', + pageUrl: 'https://www.cloudflarestatus.com', + indicator: 'minor', + description: 'Minor Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.cloudflareStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkCloudflareOverallHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); }); - expect(checkLlmGatewayHealth).toHaveBeenCalledWith(gatewayUrl); - expect(checkSkillsOriginHealth).not.toHaveBeenCalled(); - expect(result.decision).toBe(WizardReadiness.No); - expect(getBlockingServiceKeys(result.health)).toEqual(['llmGateway']); }); + // ----------------------------------------------------------------------- + // Statuspage summary.json checks (component-based) + // ----------------------------------------------------------------------- + + describe('checkPosthogComponentHealth', () => { + it('reports healthy when no ongoing incidents', async () => { + const result = await checkPosthogComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.degradedOrDownComponents).toBeUndefined(); + }); + + it('reports affected components from ongoing incidents', async () => { + const body = { + ...POSTHOG_INCIDENTIO_HEALTHY, + ongoing_incidents: [ + { + id: 'inc1', + name: 'US Cloud outage', + status: 'identified', + current_worst_impact: 'full_outage', + affected_components: [ + { + id: 'c1', + name: 'App', + group_name: 'US Cloud 🇺🇸', + current_status: 'full_outage', + }, + { + id: 'c2', + name: 'Event Ingestion', + group_name: 'US Cloud 🇺🇸', + current_status: 'full_outage', + }, + ], + url: 'https://www.posthogstatus.com/incidents/test', + last_update_at: '2026-04-22T00:00:00Z', + last_update_message: 'Investigating', + }, + ], + }; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkPosthogComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.degradedOrDownComponents).toHaveLength(2); + expect(result.degradedOrDownComponents![0].name).toBe( + 'US Cloud 🇺🇸 — App', + ); + expect(result.degradedOrDownComponents![0].status).toBe( + ServiceHealthStatus.Down, + ); + expect(result.degradedOrDownComponents![1].status).toBe( + ServiceHealthStatus.Down, + ); + }); + + it('reports degraded for degraded_performance components', async () => { + const body = { + ...POSTHOG_INCIDENTIO_HEALTHY, + ongoing_incidents: [ + { + id: 'inc1', + name: 'Slowness', + status: 'investigating', + current_worst_impact: 'degraded_performance', + affected_components: [ + { + id: 'c1', + name: 'App', + group_name: 'EU Cloud', + current_status: 'degraded_performance', + }, + ], + url: 'https://www.posthogstatus.com/incidents/test', + last_update_at: '2026-04-22T00:00:00Z', + last_update_message: 'Investigating', + }, + ], + }; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkPosthogComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.degradedOrDownComponents![0].rawStatus).toBe( + 'degraded_performance', + ); + expect(result.degradedOrDownComponents![0].status).toBe( + ServiceHealthStatus.Degraded, + ); + }); + }); + + describe('checkNpmComponentHealth', () => { + it('reports healthy when all npm components operational', async () => { + const result = await checkNpmComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + }); + + it('reports degraded when "Package installation" has partial_outage', async () => { + const body = makeStatuspageSummary({ + pageId: 'wyvgptkd90hm', + pageName: 'npm', + pageUrl: 'https://status.npmjs.org', + indicator: 'major', + description: 'Partial System Outage', + components: [ + { + id: 'mvm98gtxvb9b', + name: 'www.npmjs.com website', + status: 'operational', + position: 1, + description: null, + }, + { + id: 'k1wj10x6gmph', + name: 'Package installation', + status: 'partial_outage', + position: 2, + description: null, + }, + ], + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.npmSummary]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await checkNpmComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Degraded); + expect(result.degradedOrDownComponents![0].name).toBe( + 'Package installation', + ); + }); + }); + + describe('checkCloudflareComponentHealth', () => { + it('reports healthy when Cloudflare components operational', async () => { + const result = await checkCloudflareComponentHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + }); + }); + + // ----------------------------------------------------------------------- + // fetchEndpointHealth (retry + status-taxonomy machinery, probed directly + // against a synthetic URL — no production probe uses the strict defaults + // any more, but every endpoint check shares this loop) + // ----------------------------------------------------------------------- + + describe('fetchEndpointHealth', () => { + const PROBE_URL = 'https://probe.posthog.test/_liveness'; + + it('returns healthy on a 200', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => + Promise.resolve(new Response('ok', { status: 200 })), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toBe('HTTP 200'); + expect(global.fetch).toHaveBeenCalledWith( + PROBE_URL, + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + }); + + it('returns down on 302 — the default predicate stays strict, redirects are not OK', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => + Promise.resolve(new Response(null, { status: 302 })), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 302'); + }); + + it('returns down when the endpoint responds 503 (e.g. deploying)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => + Promise.resolve( + new Response('Service Unavailable', { status: 503 }), + ), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 503'); + }); + + it('returns down when the endpoint responds 502 (bad gateway)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => + Promise.resolve(new Response('Bad Gateway', { status: 502 })), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 502'); + }); + + it('returns no-connection on DNS resolution failure (no status-page corroboration)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => + Promise.reject( + new Error('getaddrinfo ENOTFOUND probe.posthog.test'), + ), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + expect(result.error).toBe('getaddrinfo ENOTFOUND probe.posthog.test'); + }); + + it('returns no-connection on timeout (AbortError)', async () => { + const abortError = new Error('The operation was aborted.'); + abortError.name = 'AbortError'; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => Promise.reject(abortError), + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + expect(result.error).toBe('Request timed out after 5000ms'); + }); + + it('retries on network errors and recovers if a later attempt succeeds', async () => { + let calls = 0; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => { + calls++; + if (calls < 3) { + return Promise.reject(new Error('ECONNRESET')); + } + return Promise.resolve(new Response('ok', { status: 200 })); + }, + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('attempts=3'); + expect(calls).toBe(3); + }); + + it('retries on persistent HTTP errors and stays Down after all attempts fail', async () => { + let calls = 0; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => { + calls++; + return Promise.resolve( + new Response('Service Unavailable', { status: 503 }), + ); + }, + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(calls).toBe(3); + expect(result.error).toContain('HTTP 503'); + expect(result.error).toContain('attempts=3'); + }); + + it('retries on transient 5xx and recovers if a later attempt succeeds', async () => { + let calls = 0; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => { + calls++; + if (calls < 3) { + return Promise.resolve( + new Response('Bad Gateway', { status: 502 }), + ); + } + return Promise.resolve(new Response('ok', { status: 200 })); + }, + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('attempts=3'); + expect(calls).toBe(3); + }); + + it('returns Down (not NoConnection) when last attempt got an HTTP response after earlier network errors', async () => { + let calls = 0; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [PROBE_URL]: () => { + calls++; + if (calls < 3) return Promise.reject(new Error('ECONNRESET')); + return Promise.resolve( + new Response('Bad Gateway', { status: 502 }), + ); + }, + }), + ); + const result = await fetchEndpointHealth(PROBE_URL); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 502'); + }); + }); + + // ----------------------------------------------------------------------- + // MCP (fetchEndpointHealth – / landing) + // ----------------------------------------------------------------------- + it.each([ - [healthy, healthy, []], - [down, healthy, ['llmGateway']], - [unreachable, healthy, ['llmGateway']], - [healthy, down, ['skillsOrigin']], - [healthy, unreachable, ['skillsOrigin']], - [down, down, ['llmGateway', 'skillsOrigin']], - [unreachable, unreachable, ['llmGateway', 'skillsOrigin']], + 'https://ai-gateway.us.posthog.com', + 'https://ai-gateway.eu.posthog.com/', + 'http://localhost:8789/v1', ])( - 'only interrupts for failed runtime dependencies (%j, %j)', - async (gateway, skills, blocked) => { - vi.mocked(checkLlmGatewayHealth).mockResolvedValue(gateway); - vi.mocked(checkSkillsOriginHealth).mockResolvedValue(skills); - const result = await evaluateWizardReadiness({ gatewayUrl }); - expect(getBlockingServiceKeys(result.health)).toEqual(blocked); - expect(result.decision).toBe( - blocked.length ? WizardReadiness.No : WizardReadiness.Yes, + 'checks readiness on the minted gateway %s without credentials', + async (gatewayUrl) => { + (global.fetch as Mock).mockResolvedValue( + new Response(null, { status: 200 }), + ); + const result = await checkLlmGatewayHealth(gatewayUrl); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(global.fetch).toHaveBeenCalledWith( + new URL('/readyz', gatewayUrl).href, + { + signal: expect.any(AbortSignal), + redirect: 'follow', + }, ); }, ); - it('does not turn a one-origin fallback into warnings or outage reasons', async () => { - vi.mocked(checkSkillsOriginHealth).mockResolvedValue({ - ...healthy, - rawIndicator: 'HTTP 200 (via aws, github unavailable)', + describe('checkMcpHealth', () => { + it('returns healthy when MCP worker responds 200 with landing HTML', async () => { + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toBe('HTTP 200'); + expect(global.fetch).toHaveBeenCalledWith( + URLS.mcpLanding, + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + }); + + it('returns healthy when worker responds 302 (redirect to docs, not followed)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => + Promise.resolve(new Response(null, { status: 302 })), + }), + ); + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toBe('HTTP 302'); + expect(global.fetch).toHaveBeenCalledWith( + URLS.mcpLanding, + expect.objectContaining({ redirect: 'manual' }), + ); + }); + + it('returns down on 400 — only 2xx-3xx counts as up', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => + Promise.resolve(new Response('Bad Request', { status: 400 })), + }), + ); + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 400'); + }); + + it('returns down when worker responds 500', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => + Promise.resolve( + new Response('Internal Server Error', { status: 500 }), + ), + }), + ); + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 500'); }); - const result = await evaluateWizardReadiness({ gatewayUrl }); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - }); - it('ignores obsolete provider and status-page results even in a stale health object', () => { - const stale: AllServicesHealth & Record = { - skillsOrigin: healthy, - llmGateway: healthy, - anthropic: down, - posthogOverall: down, - posthogComponents: down, - github: down, - npmOverall: down, - npmComponents: down, - cloudflareOverall: down, - cloudflareComponents: down, - mcp: down, - }; - expect(getBlockingServiceKeys(stale)).toEqual([]); + it('returns down when Cloudflare returns 522 (connection timed out)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => + Promise.resolve(new Response('', { status: 522 })), + }), + ); + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('HTTP 522'); + }); + + it('returns no-connection on network failure', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => Promise.reject(new Error('fetch failed')), + }), + ); + const result = await checkMcpHealth(); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + expect(result.error).toBe('fetch failed'); + }); }); - it('does not include internal gateway diagnostics in outage reasons', async () => { - vi.mocked(checkLlmGatewayHealth).mockResolvedValue({ - ...down, - error: 'private dependency detail', + // ----------------------------------------------------------------------- + // Skills origins (fetchEndpointHealth – skill-menu.json on both origins) + // ----------------------------------------------------------------------- + + describe('checkSkillsOriginHealth', () => { + it('returns healthy on a final 200 and follows redirects (GitHub 302s asset URLs even for missing assets)', async () => { + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toBe('HTTP 200'); + expect(global.fetch).toHaveBeenCalledWith( + URLS.githubSkillMenu, + expect.objectContaining({ redirect: 'follow' }), + ); + }); + + it('probes both origins', async () => { + await checkSkillsOriginHealth(); + const calledUrls = (global.fetch as Mock).mock.calls.map( + (c: unknown[]) => c[0], + ); + expect(calledUrls).toContain(URLS.githubSkillMenu); + expect(calledUrls).toContain(URLS.awsSkillMenu); + }); + + it('stays healthy when GitHub 5xxs but AWS serves the menu', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.resolve(new Response('Bad Gateway', { status: 502 })), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('github unavailable'); + }); + + it('stays healthy when GitHub is unreachable but AWS serves the menu', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.reject(new Error('ENOTFOUND github.com')), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('github unavailable'); + }); + + it('stays healthy when GitHub 404s but AWS serves the menu', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.resolve(new Response('Not Found', { status: 404 })), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('github unavailable'); + }); + + it('stays healthy when AWS is unreachable but GitHub serves the menu', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.awsSkillMenu]: () => Promise.reject(new Error('fetch failed')), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Healthy); + expect(result.rawIndicator).toContain('aws unavailable'); + }); + + it('returns down only when both origins 404 (release published without the asset)', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.resolve(new Response('Not Found', { status: 404 })), + [URLS.awsSkillMenu]: () => + Promise.resolve(new Response('Not Found', { status: 404 })), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); + expect(result.error).toContain('github: HTTP 404'); + expect(result.error).toContain('aws: HTTP 404'); + }); + + it('returns no-connection when both origins fail at the network layer', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.reject(new Error('ENOTFOUND github.com')), + [URLS.awsSkillMenu]: () => Promise.reject(new Error('ECONNRESET')), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.NoConnection); + expect(result.error).toContain('ENOTFOUND github.com'); + expect(result.error).toContain('ECONNRESET'); + }); + + it('reports down when GitHub 5xxs and AWS is unreachable', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.githubSkillMenu]: () => + Promise.resolve(new Response('Bad Gateway', { status: 502 })), + [URLS.awsSkillMenu]: () => Promise.reject(new Error('ECONNRESET')), + }), + ); + const result = await checkSkillsOriginHealth(); + expect(result.status).toBe(ServiceHealthStatus.Down); }); - const result = await evaluateWizardReadiness({ gatewayUrl }); - expect(result.reasons).toEqual(['LLM gateway: down']); }); - it('clears the watchdog after an unexpected failure and proceeds without warnings', async () => { - vi.mocked(checkSkillsOriginHealth).mockRejectedValue( - new Error('unexpected'), - ); - const result = await evaluateWizardReadiness(); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - expect(vi.getTimerCount()).toBe(0); + // ----------------------------------------------------------------------- + // checkAllExternalServices + // ----------------------------------------------------------------------- + + describe('checkAllExternalServices', () => { + it('returns all 10 service keys when everything is healthy', async () => { + const health = await checkAllExternalServices(); + const keys = Object.keys(health); + expect(keys).toEqual( + expect.arrayContaining([ + 'anthropic', + 'posthogOverall', + 'posthogComponents', + 'github', + 'npmOverall', + 'npmComponents', + 'cloudflareOverall', + 'cloudflareComponents', + 'mcp', + 'skillsOrigin', + ]), + ); + expect(keys).toHaveLength(10); + for (const val of Object.values(health)) { + expect(val.status).toBe(ServiceHealthStatus.Healthy); + } + }); + + it('upgrades NoConnection mcp to Down when status page reports an outage', async () => { + const incidentBody = { + ...POSTHOG_INCIDENTIO_HEALTHY, + ongoing_incidents: [ + { + id: 'inc1', + name: 'Major outage', + status: 'identified', + current_worst_impact: 'full_outage', + affected_components: [], + url: 'https://www.posthogstatus.com/incidents/test', + last_update_at: '2026-04-22T00:00:00Z', + last_update_message: 'Investigating', + }, + ], + }; + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.resolve( + new Response(JSON.stringify(incidentBody), { status: 200 }), + ), + [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), + }), + ); + + const health = await checkAllExternalServices(); + expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Down); + expect(health.mcp.status).toBe(ServiceHealthStatus.Down); + expect(health.mcp.error).toContain('corroborated by status page'); + }); + + it('keeps mcp as NoConnection when posthogstatus.com itself is unreachable (the bug-fix scenario)', async () => { + // User on flaky wifi: every PostHog-owned URL fetch fails at the + // network layer, including posthogstatus.com. Previously + // incidentio.ts returned Degraded for fetch failures, which + // tricked reconciliation into upgrading the gateway probe to Down + // and showing the red "Ongoing service disruptions" screen — the + // exact false positive this PR fixes. + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.posthogIncidentIo]: () => + Promise.reject(new Error('ECONNRESET')), + [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), + }), + ); + + const health = await checkAllExternalServices(); + expect(health.posthogOverall.status).toBe( + ServiceHealthStatus.NoConnection, + ); + expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); + }); + + it('keeps mcp as NoConnection when status page reports no incident', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => Promise.reject(new Error('ETIMEDOUT')), + }), + ); + + const health = await checkAllExternalServices(); + expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Healthy); + expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); + }); + + it('fires all fetch calls in parallel', async () => { + await checkAllExternalServices(); + const calledUrls = (global.fetch as Mock).mock.calls.map((c: unknown[]) => + typeof c[0] === 'string' ? c[0] : (c[0] as URL).toString(), + ); + // PostHog uses a single incident.io endpoint for both overall + components + expect(calledUrls).toHaveLength(10); + expect(calledUrls).toContain(URLS.posthogIncidentIo); + expect(calledUrls).toContain(URLS.mcpLanding); + expect(calledUrls).toContain(URLS.githubSkillMenu); + expect(calledUrls).toContain(URLS.awsSkillMenu); + }); }); - it('does not claim an outage when a check cannot finish', async () => { - vi.mocked(checkSkillsOriginHealth).mockReturnValue( - new Promise(() => { - // Deliberately never settles; the readiness watchdog must release the run. - }), + // ----------------------------------------------------------------------- + // evaluateWizardReadiness + // ----------------------------------------------------------------------- + + describe('evaluateWizardReadiness', () => { + it('returns Yes when all services are healthy', async () => { + const result = await evaluateWizardReadiness( + DEFAULT_WIZARD_READINESS_CONFIG, + ); + expect(result.decision).toBe(WizardReadiness.Yes); + }); + + it('proceeds without warnings when Anthropic is degraded', async () => { + const body = makeStatuspageStatus({ + pageId: 'tymt9n04zgry', + pageName: 'Claude', + pageUrl: 'https://status.claude.com', + indicator: 'minor', + description: 'Minor Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.anthropicStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await evaluateWizardReadiness( + DEFAULT_WIZARD_READINESS_CONFIG, + ); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + expect(result.health.anthropic.status).toBe(ServiceHealthStatus.Degraded); + }); + + it('proceeds without warnings when MCP is down', async () => { + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.mcpLanding]: () => + Promise.resolve(new Response('Bad Gateway', { status: 502 })), + }), + ); + const result = await evaluateWizardReadiness( + DEFAULT_WIZARD_READINESS_CONFIG, + ); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + expect(result.health.mcp.status).toBe(ServiceHealthStatus.Down); + }); + + it('proceeds without warnings when npm overall is down', async () => { + const body = makeStatuspageStatus({ + pageId: 'wyvgptkd90hm', + pageName: 'npm', + pageUrl: 'https://status.npmjs.org', + indicator: 'critical', + description: 'Major Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.npmStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await evaluateWizardReadiness( + DEFAULT_WIZARD_READINESS_CONFIG, + ); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + expect(result.health.npmOverall.status).toBe(ServiceHealthStatus.Down); + }); + + it('proceeds without warnings when a non-blocking service is degraded', async () => { + const body = makeStatuspageStatus({ + pageId: 'yh6f0r4529hb', + pageName: 'Cloudflare', + pageUrl: 'https://www.cloudflarestatus.com', + indicator: 'minor', + description: 'Minor Service Outage', + }); + (global.fetch as Mock).mockImplementation( + overrideFetch({ + [URLS.cloudflareStatus]: () => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + }), + ); + const result = await evaluateWizardReadiness( + DEFAULT_WIZARD_READINESS_CONFIG, + ); + expect(result.decision).toBe(WizardReadiness.Yes); + expect(result.reasons).toEqual([]); + }); + + it.each([ + [true, true, WizardReadiness.Yes], + [false, true, WizardReadiness.Yes], + [true, false, WizardReadiness.Yes], + [false, false, WizardReadiness.No], + ])( + 'skills availability: GitHub=%s AWS=%s yields %s', + async (github, aws, decision) => { + (global.fetch as Mock).mockImplementation((url: string | URL) => { + const healthy = + (url.toString() === URLS.githubSkillMenu && github) || + (url.toString() === URLS.awsSkillMenu && aws); + return Promise.resolve( + new Response(null, { status: healthy ? 200 : 503 }), + ); + }); + const results = await Promise.all([ + evaluateWizardReadiness(), + evaluateWizardReadiness(SIGNUP_WIZARD_READINESS_CONFIG), + ]); + for (const result of results) { + expect(result.decision).toBe(decision); + expect(result.reasons).toHaveLength( + decision === WizardReadiness.No ? 1 : 0, + ); + if (decision === WizardReadiness.No) { + expect(result.reasons[0]).toContain('Skills download'); + } + } + }, ); - const pending = evaluateWizardReadiness(); - await vi.advanceTimersByTimeAsync(20_000); - const result = await pending; - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - expect(vi.getTimerCount()).toBe(0); }); }); diff --git a/src/lib/health-checks/endpoints.ts b/src/lib/health-checks/endpoints.ts index b5ea17aa3..9450cb8bd 100644 --- a/src/lib/health-checks/endpoints.ts +++ b/src/lib/health-checks/endpoints.ts @@ -1,83 +1,84 @@ -import { getSkillsBaseUrl } from '@lib/constants'; -import { awsUrlFor } from '@lib/fetch-retry'; +import { AWS_SKILLS_BASE_URL, GITHUB_SKILLS_BASE_URL } from '@lib/constants'; import { logToFile } from '@utils/debug'; import { ServiceHealthStatus, type BaseHealthResult } from './types'; -// HTTP failures or unusable downloads confirm the endpoint is unavailable. -// Network errors alone mean we cannot tell whether the service is down. -const RETRY_BACKOFFS_MS = [500, 2000]; +// --------------------------------------------------------------------------- +// Direct endpoint health checks +// +// These ping PostHog-owned services directly (no Statuspage intermediary). +// Result taxonomy: +// - HTTP 2xx-3xx (per `isExpectedStatus`) → Healthy +// - HTTP 4xx / 5xx → Down (confirmed) +// - Network error / DNS / timeout (after retries) → NoConnection +// NoConnection means we don't know whose fault it is; readiness reconciles +// against the status page before deciding how to surface it to the user. +// +// MCP – Cloudflare Worker +// Source: posthog/services/mcp/src/index.ts +// GET / → 302 to posthog.com docs. The redirect proves the worker is up. +// +// Skills download – context-mill releases +// GET /skill-menu.json on both origins; see checkSkillsOriginHealth. +// --------------------------------------------------------------------------- + +function noConnectionResult(error: string, attempts: number): BaseHealthResult { + return { + status: ServiceHealthStatus.NoConnection, + error, + rawIndicator: attempts > 1 ? `attempts=${attempts}` : undefined, + }; +} + +function downResult(error: string): BaseHealthResult { + return { status: ServiceHealthStatus.Down, error }; +} -type ResponseValidator = (response: Response) => Promise; -type RedirectMode = 'follow' | 'manual' | 'error'; -type FetchOutcome = - | { kind: 'response'; status: number } - | { - kind: 'error'; - error: string; - httpStatus?: number; - }; +// Backoffs sized to cover typical wifi flakiness — a single dropped +// packet recovers via the 500ms retry; a wifi access point reconnect +// or wifi↔LTE handoff (2-5s) is caught by the 2000ms retry. Tighter +// schedules miss multi-second blips because all retries land in the +// same dead window. +const RETRY_BACKOFFS_MS = [500, 2000]; async function attemptFetch( url: string, timeoutMs: number, - isExpectedStatus: (status: number) => boolean, - redirect: RedirectMode, - validateResponse?: ResponseValidator, -): Promise { + redirect: 'follow' | 'manual' | 'error', +): Promise< + | { kind: 'response'; res: Response } + | { kind: 'error'; error: Error; timedOut: boolean } +> { const controller = new AbortController(); - let httpStatus: number | undefined; - let timedOut = false; - let timeout: ReturnType | undefined; - const deadline = new Promise((_, reject) => { - timeout = setTimeout(() => { - timedOut = true; - controller.abort(); - reject(new Error(`Request timed out after ${timeoutMs}ms`)); - }, timeoutMs); - }); - + const tid = setTimeout(() => controller.abort(), timeoutMs); try { - const request = async (): Promise => { - const response = await fetch(url, { - signal: controller.signal, - redirect, - }); - httpStatus = response.status; - if (isExpectedStatus(response.status) && validateResponse) { - // Keep the deadline active until the body has downloaded and parsed. - await validateResponse(response); - } else { - // Health endpoints only need their status, not their diagnostic body. - void response.body?.cancel().catch(() => undefined); - } - return { kind: 'response', status: response.status }; - }; - return await Promise.race([request(), deadline]); - } catch (error) { - return { - kind: 'error', - httpStatus, - error: timedOut - ? `Request timed out after ${timeoutMs}ms` - : error instanceof Error - ? error.message - : 'Unknown error', - }; - } finally { - clearTimeout(timeout); + const res = await fetch(url, { signal: controller.signal, redirect }); + clearTimeout(tid); + return { kind: 'response', res }; + } catch (e) { + clearTimeout(tid); + const err = e instanceof Error ? e : new Error('Unknown error'); + return { kind: 'error', error: err, timedOut: err.name === 'AbortError' }; } } -/** Probe an endpoint with bounded retries; downloads may also validate the body. */ +// Exported so tests can pin the retry/taxonomy machinery directly. export async function fetchEndpointHealth( url: string, timeoutMs = 5000, - isExpectedStatus: (status: number) => boolean = (status) => status === 200, - redirect: RedirectMode = 'follow', - validateResponse?: ResponseValidator, + isExpectedStatus: (status: number) => boolean = (s) => s === 200, + redirect: 'follow' | 'manual' | 'error' = 'follow', ): Promise { - let lastHttpStatus: number | undefined; - let lastHttpError: string | undefined; + // Total attempts = 1 initial + RETRY_BACKOFFS_MS.length retries. Both + // unexpected HTTP statuses (4xx/5xx) and network errors trigger a retry: + // transient 5xx and Cloudflare edge blips often recover on a retry, and + // even nominally deterministic 4xx can be transient (CDN propagation + // lag after a release, token rotation, rate-limit window resets). GETs + // are idempotent so retrying is safe. + // + // Final status if every attempt fails: + // - At least one HTTP response observed → `Down` (server-side evidence) + // - Only network errors observed → `NoConnection` + let lastHttpStatus: number | null = null; let lastError = 'Unknown error'; let attempts = 0; @@ -87,139 +88,122 @@ export async function fetchEndpointHealth( logToFile( `[health-checks] retry ${i}/${RETRY_BACKOFFS_MS.length} for ${url} in ${wait}ms (last: ${lastError})`, ); - await new Promise((resolve) => setTimeout(resolve, wait)); + await new Promise((r) => setTimeout(r, wait)); } attempts++; - const outcome = await attemptFetch( - url, - timeoutMs, - isExpectedStatus, - redirect, - validateResponse, - ); + + const outcome = await attemptFetch(url, timeoutMs, redirect); if (outcome.kind === 'response') { - if (isExpectedStatus(outcome.status)) { + const res = outcome.res; + if (isExpectedStatus(res.status)) { const result: BaseHealthResult = { status: ServiceHealthStatus.Healthy, rawIndicator: attempts > 1 - ? `HTTP ${outcome.status} (attempts=${attempts})` - : `HTTP ${outcome.status}`, + ? `HTTP ${res.status} (attempts=${attempts})` + : `HTTP ${res.status}`, }; logToFile( - `[health-checks] GET ${url} -> ${result.status} (${ - result.rawIndicator ?? '' - })`, + `[health-checks] GET ${url} -> ${result.status}` + + ` (${result.rawIndicator})`, ); return result; } - lastHttpStatus = outcome.status; - lastError = lastHttpError = `HTTP ${outcome.status}`; - } else { - lastError = outcome.error; - if (outcome.httpStatus !== undefined) { - lastHttpStatus = outcome.httpStatus; - lastHttpError = outcome.error; - } + lastHttpStatus = res.status; + lastError = `HTTP ${res.status}`; + continue; } + + lastError = outcome.timedOut + ? `Request timed out after ${timeoutMs}ms` + : outcome.error.message; } - const result: BaseHealthResult = { - status: - lastHttpStatus !== undefined - ? ServiceHealthStatus.Down - : ServiceHealthStatus.NoConnection, - error: lastHttpError ?? lastError, - rawIndicator: - lastHttpStatus !== undefined - ? `HTTP ${lastHttpStatus} (attempts=${attempts})` - : `attempts=${attempts}`, - }; + const result = + lastHttpStatus !== null + ? downResult(`HTTP ${lastHttpStatus} (attempts=${attempts})`) + : noConnectionResult(lastError, attempts); logToFile( - `[health-checks] GET ${url} -> ${result.status} (attempts=${attempts}, ${ - lastHttpError ?? lastError - })`, + `[health-checks] GET ${url} -> ${result.status}` + + ` (attempts=${attempts}, ${result.error})`, ); return result; } -/** Readiness checks gateway dependencies, independently of its model providers. */ export const checkLlmGatewayHealth = ( gatewayUrl: string, ): Promise => fetchEndpointHealth(new URL('/readyz', gatewayUrl).href); -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} +export const checkMcpHealth = (): Promise => + fetchEndpointHealth( + 'https://mcp.posthog.com/', + 5000, + // 2xx-3xx counts as up (redirect to docs) + (s) => s >= 200 && s < 400, + 'manual', + ); + +/** + * Skills are published to two origins under the same filenames and + * `fetchWithRetry` fails over between them, so the run is only blocked when + * neither answers. Probed in parallel — sequential probes would double the + * worst case past `READINESS_TIMEOUT_MS`. + */ +export const checkSkillsOriginHealth = async (): Promise => { + const [github, aws] = await Promise.all([ + fetchEndpointHealth(`${GITHUB_SKILLS_BASE_URL}/skill-menu.json`), + fetchEndpointHealth(`${AWS_SKILLS_BASE_URL}/skill-menu.json`), + ]); + return combineOriginHealth(github, aws); +}; -/** Validate fields consumed by fetchSkillMenu, allowing optional newer metadata. */ -async function validateSkillMenu(response: Response): Promise { - const menu: unknown = await response.json(); - if (!isRecord(menu) || !isRecord(menu.categories)) { - throw new Error('Skill menu is missing its categories'); +/** + * Mirrors `fetchWithRetry`: a download tries GitHub, then AWS, so the run is + * only blocked when neither origin answers. Whichever failure the probes saw, + * one origin serving means skills are reachable. + */ +function combineOriginHealth( + github: BaseHealthResult, + aws: BaseHealthResult, +): BaseHealthResult { + if (github.status === ServiceHealthStatus.Healthy) { + // Naming the dead origin makes a one-sided outage legible in the log and + // in the readiness reasons, where the status alone reads as "fine". + return aws.status === ServiceHealthStatus.Healthy + ? github + : withIndicatorSuffix(github, 'aws unavailable'); } - const categories = Object.values(menu.categories); - const valid = categories.every( - (entries) => - Array.isArray(entries) && - entries.every( - (entry: unknown) => - isRecord(entry) && - ['id', 'name', 'downloadUrl'].every( - (key) => typeof entry[key] === 'string' && entry[key].length > 0, - ) && - (entry.variants === undefined || - (Array.isArray(entry.variants) && - entry.variants.every( - (variant: unknown) => - isRecord(variant) && typeof variant.id === 'string', - ))), - ), - ); - if (!valid || !categories.some((entries) => (entries as unknown[]).length)) { - throw new Error('Skill menu has no usable skill entries'); + + if (aws.status === ServiceHealthStatus.Healthy) { + return withIndicatorSuffix(aws, 'via aws, github unavailable'); } -} -/** Probe the actual skill sources, including the matching release's AWS mirror. */ -export async function checkSkillsOriginHealth( - skillsBaseUrl = getSkillsBaseUrl(), -): Promise { - const primaryUrl = `${skillsBaseUrl.replace(/\/+$/, '')}/skill-menu.json`; - const fallbackUrl = awsUrlFor(primaryUrl); - const probe = (url: string) => - fetchEndpointHealth( - url, - 5000, - (status) => status === 200, - 'follow', - validateSkillMenu, - ); - - // Local and custom sources have no production fallback in fetchWithRetry. - if (!fallbackUrl) return probe(primaryUrl); - - const [primary, fallback] = await Promise.all([ - probe(primaryUrl), - probe(fallbackUrl), - ]); - if (primary.status === ServiceHealthStatus.Healthy) return primary; - if (fallback.status === ServiceHealthStatus.Healthy) return fallback; + const error = `github: ${github.error ?? 'unknown'} | aws: ${ + aws.error ?? 'unknown' + }`; + const confirmedDown = + github.status === ServiceHealthStatus.Down || + aws.status === ServiceHealthStatus.Down; + return { + status: confirmedDown + ? ServiceHealthStatus.Down + : ServiceHealthStatus.NoConnection, + error, + // Keeps the `attempts=N` the blocked-readiness analytics parses. + rawIndicator: github.rawIndicator ?? aws.rawIndicator, + }; +} - logToFile( - `[health-checks] skill origins unavailable: primary=${ - primary.error ?? 'unknown' - }; fallback=${fallback.error ?? 'unknown'}`, - ); +function withIndicatorSuffix( + result: BaseHealthResult, + suffix: string, +): BaseHealthResult { return { - status: - primary.status === ServiceHealthStatus.Down || - fallback.status === ServiceHealthStatus.Down - ? ServiceHealthStatus.Down - : ServiceHealthStatus.NoConnection, - error: 'Both skill download sources are unavailable', - rawIndicator: primary.rawIndicator ?? fallback.rawIndicator, + ...result, + rawIndicator: result.rawIndicator + ? `${result.rawIndicator} (${suffix})` + : suffix, }; } diff --git a/src/lib/health-checks/incidentio.ts b/src/lib/health-checks/incidentio.ts new file mode 100644 index 000000000..574a4f0f9 --- /dev/null +++ b/src/lib/health-checks/incidentio.ts @@ -0,0 +1,167 @@ +import { + ServiceHealthStatus, + type BaseHealthResult, + type ComponentHealthResult, + type ComponentStatus, +} from './types'; + +interface IncidentIoAffectedComponent { + id: string; + name: string; + group_name?: string; + current_status: string; +} + +interface IncidentIoIncident { + id: string; + name: string; + status: string; + current_worst_impact: string; + affected_components: IncidentIoAffectedComponent[]; +} + +interface IncidentIoSummary { + ongoing_incidents: IncidentIoIncident[]; + in_progress_maintenances: unknown[]; +} + +function mapIncidentImpact(impact: string): ServiceHealthStatus { + switch (impact) { + case 'full_outage': + return ServiceHealthStatus.Down; + case 'partial_outage': + case 'degraded_performance': + return ServiceHealthStatus.Degraded; + default: + return ServiceHealthStatus.Degraded; + } +} + +function mapComponentStatus(status: string): ServiceHealthStatus { + switch (status) { + case 'operational': + return ServiceHealthStatus.Healthy; + case 'full_outage': + return ServiceHealthStatus.Down; + case 'partial_outage': + case 'degraded_performance': + return ServiceHealthStatus.Degraded; + default: + return ServiceHealthStatus.Degraded; + } +} + +/** + * Build an error result for fetch failures. The kind matters for + * downstream reconciliation: + * + * - 'http' (incident.io returned a bad status code) → `Down`. We + * reached the status page but it told us something is wrong on + * its side. We have a definitive response. + * - 'network' (timeout, DNS failure, TCP/TLS failure) → `NoConnection`. + * We never reached the status page. Treating this as `Degraded` + * (the previous behavior) silently flipped the reconciliation in + * `readiness.ts` from "soft" to "confirmed outage" whenever the + * user's own network was flaky — exactly the false positive this + * module is meant to help diagnose. + */ +function errResult(error: string, kind: 'http' | 'network'): BaseHealthResult { + return { + status: + kind === 'http' + ? ServiceHealthStatus.Down + : ServiceHealthStatus.NoConnection, + error, + }; +} + +const POSTHOG_STATUS_URL = 'https://www.posthogstatus.com/api/v1/summary'; + +async function fetchPosthogStatus( + timeoutMs = 5000, +): Promise<{ overall: BaseHealthResult; components: ComponentHealthResult }> { + try { + const controller = new AbortController(); + const tid = setTimeout(() => controller.abort(), timeoutMs); + const res = await fetch(POSTHOG_STATUS_URL, { signal: controller.signal }); + clearTimeout(tid); + + if (!res.ok) { + const err = errResult(`HTTP ${res.status}`, 'http'); + return { overall: err, components: err }; + } + + const data = (await res.json()) as IncidentIoSummary; + const incidents = data.ongoing_incidents ?? []; + + if (incidents.length === 0) { + return { + overall: { status: ServiceHealthStatus.Healthy }, + components: { status: ServiceHealthStatus.Healthy }, + }; + } + + let worstOverall = ServiceHealthStatus.Degraded; + const affected: ComponentStatus[] = []; + + for (const incident of incidents) { + const impact = mapIncidentImpact(incident.current_worst_impact); + if (impact === ServiceHealthStatus.Down) { + worstOverall = ServiceHealthStatus.Down; + } + + for (const comp of incident.affected_components ?? []) { + const compStatus = mapComponentStatus(comp.current_status); + if (compStatus !== ServiceHealthStatus.Healthy) { + affected.push({ + name: comp.group_name + ? `${comp.group_name} — ${comp.name}` + : comp.name, + status: compStatus, + rawStatus: comp.current_status, + }); + } + } + } + + return { + overall: { status: worstOverall }, + components: { + status: + affected.length > 0 ? ServiceHealthStatus.Degraded : worstOverall, + degradedOrDownComponents: affected.length > 0 ? affected : undefined, + }, + }; + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') { + const err = errResult('Request timed out', 'network'); + return { overall: err, components: err }; + } + const err = errResult( + e instanceof Error ? e.message : 'Unknown error', + 'network', + ); + return { overall: err, components: err }; + } +} + +let _cache: Promise<{ + overall: BaseHealthResult; + components: ComponentHealthResult; +}> | null = null; + +function getPosthogHealth() { + if (!_cache) _cache = fetchPosthogStatus(); + return _cache; +} + +export function resetPosthogHealthCache(): void { + _cache = null; +} + +export const checkPosthogOverallHealth = async (): Promise => + (await getPosthogHealth()).overall; + +export const checkPosthogComponentHealth = + async (): Promise => + (await getPosthogHealth()).components; diff --git a/src/lib/health-checks/index.ts b/src/lib/health-checks/index.ts index 679ecba3a..b4c3f6041 100644 --- a/src/lib/health-checks/index.ts +++ b/src/lib/health-checks/index.ts @@ -1,14 +1,32 @@ export { ServiceHealthStatus, type BaseHealthResult, + type ComponentStatus, + type ComponentHealthResult, type AllServicesHealth, type HealthCheckKey, } from './types'; -export { checkLlmGatewayHealth, checkSkillsOriginHealth } from './endpoints'; +export { + checkAnthropicHealth, + checkGithubHealth, + checkNpmOverallHealth, + checkNpmComponentHealth, + checkCloudflareOverallHealth, + checkCloudflareComponentHealth, +} from './statuspage'; + +export { + checkPosthogOverallHealth, + checkPosthogComponentHealth, + resetPosthogHealthCache, +} from './incidentio'; + +export { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; export { - type HealthCheckOptions, + type WizardReadinessConfig, + DEFAULT_WIZARD_READINESS_CONFIG, checkAllExternalServices, WizardReadiness, type WizardReadinessResult, diff --git a/src/lib/health-checks/readiness.ts b/src/lib/health-checks/readiness.ts index 7996b52bd..fc6e12aa8 100644 --- a/src/lib/health-checks/readiness.ts +++ b/src/lib/health-checks/readiness.ts @@ -4,36 +4,162 @@ import { type BaseHealthResult, type HealthCheckKey, } from './types'; -import { checkLlmGatewayHealth, checkSkillsOriginHealth } from './endpoints'; +import { + checkAnthropicHealth, + checkGithubHealth, + checkNpmOverallHealth, + checkNpmComponentHealth, + checkCloudflareOverallHealth, + checkCloudflareComponentHealth, +} from './statuspage'; +import { + checkPosthogOverallHealth, + checkPosthogComponentHealth, +} from './incidentio'; +import { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; import { logToFile } from '@utils/debug'; +// --------------------------------------------------------------------------- +// Service labels (used in human-readable reason strings) +// --------------------------------------------------------------------------- + export const SERVICE_LABELS: Record = { - llmGateway: 'LLM gateway', + anthropic: 'Anthropic', + posthogOverall: 'PostHog', + posthogComponents: 'PostHog (components)', + github: 'GitHub', + npmOverall: 'npm', + npmComponents: 'npm (components)', + cloudflareOverall: 'Cloudflare', + cloudflareComponents: 'Cloudflare (components)', + mcp: 'MCP', skillsOrigin: 'Skills download', }; -const HEALTH_CHECK_KEYS: HealthCheckKey[] = ['llmGateway', 'skillsOrigin']; +// --------------------------------------------------------------------------- +// Readiness config +// --------------------------------------------------------------------------- -export interface HealthCheckOptions { - /** Only the gateway URL returned by this run's token mint; never guessed. */ - gatewayUrl?: string; - /** Defaults to the same release or local server used by skill downloads. */ - skillsBaseUrl?: string; - /** Reuse the pre-auth skills check when checking the gateway after mint. */ - skillsHealth?: BaseHealthResult; +export interface WizardReadinessConfig { + /** Services where status=Down blocks the run (readiness=No). */ + downBlocksRun: HealthCheckKey[]; + /** Services where status=Degraded (or worse) blocks the run (readiness=No). */ + degradedBlocksRun?: HealthCheckKey[]; } -/** Direct checks of the dependencies this run uses, without status pages. */ -export async function checkAllExternalServices( - options: HealthCheckOptions = {}, -): Promise { - const [llmGateway, skillsOrigin] = await Promise.all([ - options.gatewayUrl ? checkLlmGatewayHealth(options.gatewayUrl) : undefined, - options.skillsHealth ?? checkSkillsOriginHealth(options.skillsBaseUrl), +// Skills gate startup; gateway readiness is checked against the minted URL. +export const DEFAULT_WIZARD_READINESS_CONFIG: WizardReadinessConfig = { + downBlocksRun: ['skillsOrigin'], +}; + +export const SIGNUP_WIZARD_READINESS_CONFIG = DEFAULT_WIZARD_READINESS_CONFIG; + +// --------------------------------------------------------------------------- +// Aggregate check +// --------------------------------------------------------------------------- + +export async function checkAllExternalServices(): Promise { + const [ + anthropic, + posthogOverall, + posthogComponents, + github, + npmOverall, + npmComponents, + cloudflareOverall, + cloudflareComponents, + mcp, + skillsOrigin, + ] = await Promise.all([ + checkAnthropicHealth(), + checkPosthogOverallHealth(), + checkPosthogComponentHealth(), + checkGithubHealth(), + checkNpmOverallHealth(), + checkNpmComponentHealth(), + checkCloudflareOverallHealth(), + checkCloudflareComponentHealth(), + checkMcpHealth(), + checkSkillsOriginHealth(), ]); - return { ...(llmGateway ? { llmGateway } : {}), skillsOrigin }; + + const health: AllServicesHealth = { + anthropic, + posthogOverall, + posthogComponents, + github, + npmOverall, + npmComponents, + cloudflareOverall, + cloudflareComponents, + mcp, + skillsOrigin, + }; + return reconcilePosthogReachability(health); +} + +/** + * When a PostHog-owned endpoint probe returns `NoConnection`, decide + * whether it's a real outage or a likely-local issue by checking the + * official status page (`posthogstatus.com`): + * + * - Status page says PostHog is `Down` / `Degraded` → upgrade + * mcp to `Down`. The status page corroborates. + * - Status page is `Healthy` → keep `NoConnection`. The status page + * contradicts; this is probably the user's network. + * - Status page is also `NoConnection` → keep `NoConnection`. User + * can't reach two independent PostHog properties; almost + * certainly their network. (This case relies on incidentio.ts + * correctly emitting `NoConnection` for fetch failures rather + * than the previous `Degraded`, which used to silently flip the + * reconciliation into a false positive.) + * + * Why `Degraded` corroborates: a `Degraded` reading here only fires + * when incident.io's API parsed successfully and reported a real + * `partial_outage` or `degraded_performance` for some component. That's + * PostHog acknowledging an issue, even if narrower than a full outage. + * If our MCP probe is also failing, those two signals together + * justify pointing at PostHog rather than the user. + * + * A narrower variant — only corroborate when the affected component is + * MCP-related (US/EU Cloud, app) — would be more precise. We + * have the data in `posthogComponents` but don't use it here. If the + * analytics show false positives concentrated in this case, it's a + * cheap follow-up. + * + * Mutates a copy of `health` and returns it. + */ +export function reconcilePosthogReachability( + health: AllServicesHealth, +): AllServicesHealth { + const posthogStatus = health.posthogOverall.status; + const corroboratesOutage = + posthogStatus === ServiceHealthStatus.Down || + posthogStatus === ServiceHealthStatus.Degraded; + + if (!corroboratesOutage) return health; + + const upgrade = (r: BaseHealthResult): BaseHealthResult => + r.status === ServiceHealthStatus.NoConnection + ? { + ...r, + status: ServiceHealthStatus.Down, + error: r.error + ? `${r.error} (corroborated by status page)` + : 'corroborated by status page', + } + : r; + + return { + ...health, + mcp: upgrade(health.mcp), + }; } +// --------------------------------------------------------------------------- +// Wizard readiness evaluation +// --------------------------------------------------------------------------- + export enum WizardReadiness { Yes = 'yes', No = 'no', @@ -46,66 +172,119 @@ export interface WizardReadinessResult { reasons: string[]; } -/** - * A gateway failure or the failure of every skills origin interrupts the run. - * An unprobed gateway and an inconclusive check do not produce outage warnings. - */ -export function getBlockingServiceKeys( - health: AllServicesHealth, -): HealthCheckKey[] { - return HEALTH_CHECK_KEYS.filter((key) => { - const status = health[key]?.status; - return ( - status === ServiceHealthStatus.Down || - status === ServiceHealthStatus.NoConnection - ); - }); +function describeResult(label: string, h: BaseHealthResult): string { + const parts = [`${label}: ${h.status}`]; + if (h.rawIndicator) parts.push(`indicator=${h.rawIndicator}`); + if (h.error) parts.push(h.error); + return parts.join(' — '); } -// Endpoint probes retry within 17.5s; run them in parallel with a final ceiling. +// Each probe can take up to one base timeout + two retries with the +// 500ms / 2000ms backoffs in endpoints.ts (worst case ~17.5s for a +// network failure that exhausts retries). Probes run in parallel so +// the aggregate ceiling is one probe, not the sum. const READINESS_TIMEOUT_MS = 20_000; export async function evaluateWizardReadiness( - options: HealthCheckOptions = {}, + config: WizardReadinessConfig = DEFAULT_WIZARD_READINESS_CONFIG, ): Promise { - let timeout: ReturnType | undefined; try { const health = await Promise.race([ - checkAllExternalServices(options), - new Promise((_, reject) => { - timeout = setTimeout( - () => reject(new Error('Health check timed out')), + checkAllExternalServices(), + new Promise((resolve) => + setTimeout( + () => resolve(allUnknown('Health check timed out')), READINESS_TIMEOUT_MS, - ); - }), + ), + ), ]); - const blockingKeys = getBlockingServiceKeys(health); - const reasons = blockingKeys.flatMap((key) => { - const result = health[key]; - return result ? [`${SERVICE_LABELS[key]}: ${result.status}`] : []; - }); + + const blockingKeys = getBlockingServiceKeys(health, config); + const reasons = blockingKeys.map((key) => + describeResult(SERVICE_LABELS[key], health[key]), + ); if (blockingKeys.length > 0) { - logToFile(`[health-checks] blocked by: ${reasons.join(', ')}`); + const blockingDetails = blockingKeys.map((key) => { + const h = health[key]; + return `${key} (${h.status}${h.error ? ` — ${h.error}` : ''})`; + }); + logToFile(`[health-checks] blocked by: ${blockingDetails.join(', ')}`); + return { decision: WizardReadiness.No, health, reasons }; } - return { - decision: - blockingKeys.length > 0 ? WizardReadiness.No : WizardReadiness.Yes, - health, - reasons, - }; + + return { decision: WizardReadiness.Yes, health, reasons }; } catch (err) { - logToFile('[health-checks] check inconclusive, proceeding:', err); + logToFile( + `[health-checks] error: ${err instanceof Error ? err.message : err}`, + ); + // Health checks must never block the wizard run return { decision: WizardReadiness.Yes, - health: { - skillsOrigin: options.skillsHealth ?? { - status: ServiceHealthStatus.Degraded, - error: 'Health check did not complete', - }, - }, + health: allUnknown('Unexpected error'), reasons: [], }; - } finally { - clearTimeout(timeout); } } + +// --------------------------------------------------------------------------- +// Blocking service detection +// --------------------------------------------------------------------------- + +/** Keys that are component-level detail, not top-level services. */ +const COMPONENT_KEYS: HealthCheckKey[] = [ + 'posthogComponents', + 'npmComponents', + 'cloudflareComponents', +]; + +/** + * Get the keys of services that would block a wizard run per the given config. + * + * `NoConnection` blocks the same services as `Down` — the wizard genuinely + * can't continue if it can't reach the gateway. The screen shows softer + * framing in that case (HealthCheckScreen) so we don't falsely accuse + * PostHog of an outage when the user's network is the likely cause. + */ +export function getBlockingServiceKeys( + health: AllServicesHealth, + config: WizardReadinessConfig = DEFAULT_WIZARD_READINESS_CONFIG, +): HealthCheckKey[] { + return (Object.keys(health) as HealthCheckKey[]).filter((key) => { + if (COMPONENT_KEYS.includes(key)) return false; + const result = health[key]; + if ( + config.downBlocksRun.includes(key) && + (result.status === ServiceHealthStatus.Down || + result.status === ServiceHealthStatus.NoConnection) + ) { + return true; + } + if ( + (config.degradedBlocksRun ?? []).includes(key) && + result.status !== ServiceHealthStatus.Healthy + ) { + return true; + } + return false; + }); +} + +/** Build an AllServicesHealth where every service is Degraded with the given error. */ +function allUnknown(error: string): AllServicesHealth { + const base: BaseHealthResult = { + status: ServiceHealthStatus.Degraded, + error, + }; + return { + anthropic: base, + posthogOverall: base, + posthogComponents: { ...base }, + github: base, + npmOverall: base, + npmComponents: { ...base }, + cloudflareOverall: base, + cloudflareComponents: { ...base }, + mcp: base, + skillsOrigin: base, + }; +} diff --git a/src/lib/health-checks/statuspage.ts b/src/lib/health-checks/statuspage.ts new file mode 100644 index 000000000..bfcdd6dae --- /dev/null +++ b/src/lib/health-checks/statuspage.ts @@ -0,0 +1,144 @@ +import { + ServiceHealthStatus, + type BaseHealthResult, + type ComponentHealthResult, +} from './types'; + +// --------------------------------------------------------------------------- +// Statuspage.io v2 API helpers +// https://metastatuspage.com/api +// +// status.json – page-level rollup; indicator is one of: none | minor | major | critical +// summary.json – same rollup + component list; component status is one of: +// operational | degraded_performance | partial_outage | major_outage | under_maintenance +// https://support.atlassian.com/statuspage/docs/show-service-status-with-components +// --------------------------------------------------------------------------- + +interface StatuspageStatusResponse { + status?: { indicator?: string; description?: string }; +} + +interface StatuspageSummaryResponse extends StatuspageStatusResponse { + components?: { id: string; name: string; status: string }[]; +} + +function mapIndicator(v: string | null | undefined): ServiceHealthStatus { + switch (v) { + case 'none': + return ServiceHealthStatus.Healthy; + case 'minor': + return ServiceHealthStatus.Degraded; + case 'major': + case 'critical': + return ServiceHealthStatus.Down; + default: + return ServiceHealthStatus.Degraded; + } +} + +function mapComponentRaw(v: string | null | undefined): ServiceHealthStatus { + switch (v) { + case 'operational': + return ServiceHealthStatus.Healthy; + case 'degraded_performance': + case 'under_maintenance': + return ServiceHealthStatus.Degraded; + case 'partial_outage': + case 'major_outage': + return ServiceHealthStatus.Down; + default: + return ServiceHealthStatus.Degraded; + } +} + +function errResult(error: string): BaseHealthResult { + return { status: ServiceHealthStatus.Degraded, error }; +} + +async function fetchStatuspageIndicator( + url: string, + timeoutMs = 5000, +): Promise { + try { + const controller = new AbortController(); + const tid = setTimeout(() => controller.abort(), timeoutMs); + const res = await fetch(url, { signal: controller.signal }); + clearTimeout(tid); + + if (!res.ok) return errResult(`HTTP ${res.status}`); + + const data = (await res.json()) as StatuspageStatusResponse; + const indicator = data.status?.indicator ?? null; + return { + status: mapIndicator(indicator), + rawIndicator: indicator ?? undefined, + }; + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') + return errResult('Request timed out'); + return errResult(e instanceof Error ? e.message : 'Unknown error'); + } +} + +async function fetchStatuspageSummary( + url: string, + timeoutMs = 5000, +): Promise { + try { + const controller = new AbortController(); + const tid = setTimeout(() => controller.abort(), timeoutMs); + const res = await fetch(url, { signal: controller.signal }); + clearTimeout(tid); + + if (!res.ok) return errResult(`HTTP ${res.status}`); + + const data = (await res.json()) as StatuspageSummaryResponse; + const indicator = data.status?.indicator ?? null; + const overall = mapIndicator(indicator); + + const affected = (data.components ?? []) + .map((c) => ({ + name: c.name, + status: mapComponentRaw(c.status), + rawStatus: c.status, + })) + .filter((c) => c.status !== ServiceHealthStatus.Healthy); + + return { + status: affected.length > 0 ? ServiceHealthStatus.Degraded : overall, + rawIndicator: indicator ?? undefined, + degradedOrDownComponents: affected.length > 0 ? affected : undefined, + }; + } catch (e) { + if (e instanceof Error && e.name === 'AbortError') + return errResult('Request timed out'); + return errResult(e instanceof Error ? e.message : 'Unknown error'); + } +} + +// --------------------------------------------------------------------------- +// Individual statuspage-backed checks +// --------------------------------------------------------------------------- + +export const checkAnthropicHealth = (): Promise => + fetchStatuspageIndicator('https://status.claude.com/api/v2/status.json'); + +export const checkGithubHealth = (): Promise => + fetchStatuspageIndicator('https://www.githubstatus.com/api/v2/status.json'); + +export const checkNpmOverallHealth = (): Promise => + fetchStatuspageIndicator('https://status.npmjs.org/api/v2/status.json'); + +export const checkNpmComponentHealth = (): Promise => + fetchStatuspageSummary('https://status.npmjs.org/api/v2/summary.json'); + +export const checkCloudflareOverallHealth = (): Promise => + fetchStatuspageIndicator( + 'https://www.cloudflarestatus.com/api/v2/status.json', + ); + +export const checkCloudflareComponentHealth = + (): Promise => + fetchStatuspageSummary( + 'https://www.cloudflarestatus.com/api/v2/summary.json', + ); diff --git a/src/lib/health-checks/testme.md b/src/lib/health-checks/testme.md index ea17fc8c4..9cf6a7202 100644 --- a/src/lib/health-checks/testme.md +++ b/src/lib/health-checks/testme.md @@ -1,26 +1,61 @@ -# Health check tests +# Health Checks — Testing Guide -Run the focused suites without building or contacting live services: +## Running unit tests ```bash -pnpm exec vitest run src/lib/health-checks/__tests__ src/lib/agent/runner/shared/__tests__/bootstrap-health.test.ts src/ui/tui/__tests__/ink-ui-health.test.ts +# From the wizard/ root — runs only health-check tests (fast, no build step) +npx jest src/lib/health-checks/__tests__/health-checks.test.ts + +# Watch mode +npx jest src/lib/health-checks/__tests__/health-checks.test.ts --watch + +# With coverage +npx jest src/lib/health-checks/__tests__/health-checks.test.ts --coverage +``` + +## Running health checks live + +To hit all 10 endpoints for real and see the full readiness result: + +```bash +# From the wizard/ root +npx tsx -e "import { evaluateWizardReadiness } from './src/lib/health-checks/index'; evaluateWizardReadiness().then(r => console.log(JSON.stringify(r, null, 2)))" ``` -Endpoint tests cover bounded retries, connection failures, malformed skill -menus, GitHub/AWS fallback, local context-mill targets, and the gateway -readiness route. Readiness tests cover the dependency matrix, pre-auth results, -the actual minted gateway target, and absence of unrelated provider warnings. -Bootstrap and UI tests cover cached skills checks and waiting for a fresh outage -dismissal after login. - -| Dependency | Probe | Healthy response | -| ------------------ | ---------------------------------------------------------------------------------- | ------------------------------------ | -| LLM gateway | `/readyz` | HTTP 200; no bearer or model request | -| GitHub skills | `https://github.com/PostHog/context-mill/releases/latest/download/skill-menu.json` | Downloadable, valid skill menu | -| AWS skills mirror | `https://context-mill.posthog.com/latest/skill-menu.json` | Downloadable, valid skill menu | -| Local context-mill | `/skill-menu.json` | Downloadable, valid skill menu | - -The gateway is omitted before mint rather than guessed. Either release origin -working is sufficient; local targets are checked independently of production. -Provider status pages are not queried. No gateway readiness response body is -shown to the user. +## How the tests work + +All external HTTP calls are mocked via a global `fetch` override in +`beforeEach`. No network access is required. Mock data is modelled on real +responses captured from production endpoints on 2026-03-05. + +## Endpoints tested + +| Service | URL | Healthy response | +| ----------------------- | ------------------------------------------------------ | ------------------------------------- | +| Anthropic | `https://status.claude.com/api/v2/status.json` | `{"status":{"indicator":"none",...}}` | +| PostHog | `https://www.posthogstatus.com/api/v2/status.json` | Same shape | +| PostHog (components) | `https://www.posthogstatus.com/api/v2/summary.json` | Adds `components[]` array | +| GitHub | `https://www.githubstatus.com/api/v2/status.json` | Same shape | +| npm | `https://status.npmjs.org/api/v2/status.json` | Same shape | +| npm (components) | `https://status.npmjs.org/api/v2/summary.json` | Adds `components[]` array | +| Cloudflare | `https://www.cloudflarestatus.com/api/v2/status.json` | Same shape | +| Cloudflare (components) | `https://www.cloudflarestatus.com/api/v2/summary.json` | Adds `components[]` array | +| MCP | `https://mcp.posthog.com/` | HTML landing page (HTTP 200) | + +### Statuspage.io API v2 reference + +- Docs: +- `status.json` — page-level rollup; `indicator` is one of: `none`, `minor`, + `major`, `critical` +- `summary.json` — same rollup + `components[]`; component `status` is one of: + `operational`, `degraded_performance`, `partial_outage`, `major_outage`, + `under_maintenance` +- Component docs: + + +### MCP + +- Source: `posthog/services/mcp/src/index.ts` +- `GET /` → HTML landing page (200) +- No dedicated `/health` endpoint; 200 on `/` confirms the Cloudflare Worker is + running. diff --git a/src/lib/health-checks/types.ts b/src/lib/health-checks/types.ts index 7df9b0b80..f838bbf96 100644 --- a/src/lib/health-checks/types.ts +++ b/src/lib/health-checks/types.ts @@ -2,7 +2,13 @@ export enum ServiceHealthStatus { Healthy = 'healthy', Degraded = 'degraded', Down = 'down', - /** A failed connection does not establish whether the service or local network is at fault. */ + /** + * Probe failed (network error, timeout, DNS failure) AND we have no + * corroborating status-page incident. The service may be fine — the + * user's network is the likely culprit. Distinct from `Down`, which + * is confirmed (HTTP 5xx or status-page incident). User-facing label: + * "No connection". + */ NoConnection = 'no-connection', } @@ -12,9 +18,26 @@ export interface BaseHealthResult { error?: string; } +export interface ComponentStatus { + name: string; + status: ServiceHealthStatus; + rawStatus: string; +} + +export interface ComponentHealthResult extends BaseHealthResult { + degradedOrDownComponents?: ComponentStatus[]; +} + export interface AllServicesHealth { - /** Absent before the token mint tells us this run's actual gateway URL. */ - llmGateway?: BaseHealthResult; + anthropic: BaseHealthResult; + posthogOverall: BaseHealthResult; + posthogComponents: ComponentHealthResult; + github: BaseHealthResult; + npmOverall: BaseHealthResult; + npmComponents: ComponentHealthResult; + cloudflareOverall: BaseHealthResult; + cloudflareComponents: ComponentHealthResult; + mcp: BaseHealthResult; skillsOrigin: BaseHealthResult; } diff --git a/src/lib/programs/shared/health-check-step.ts b/src/lib/programs/shared/health-check-step.ts index fa41462ca..1c630a52c 100644 --- a/src/lib/programs/shared/health-check-step.ts +++ b/src/lib/programs/shared/health-check-step.ts @@ -1,12 +1,14 @@ /** - * Shared health-check step for programs that opt into dependency checks. + * Shared health-check step used by every program that runs an agent. * * Renders the HealthCheckScreen between intro and auth, kicks off the * readiness probe in onInit, and gates the screen on either a clean * readiness result or an explicit user dismissal of the outage. * - * Bootstrap checks the minted gateway later for these same programs. - * Programs without this screen skip both advisory checks. + * Programs without this step that hit a blocking outage gridlock the + * router: agent-runner calls wizardAbort, which awaits outroDismissed, + * but the router can't advance past the still-incomplete auth step to + * render the OutroScreen. */ import type { ProgramStep } from '@lib/programs/program-step'; @@ -14,12 +16,26 @@ import type { WizardSession } from '@lib/wizard-session'; import { evaluateWizardReadiness, WizardReadiness, + SIGNUP_WIZARD_READINESS_CONFIG, + getBlockingServiceKeys, } from '@lib/health-checks/readiness'; import { logToFile } from '@utils/debug'; export function healthCheckReady(session: WizardSession): boolean { if (!session.readinessResult) return false; + if (session.signup) { + const hardBlocking = getBlockingServiceKeys( + session.readinessResult.health, + SIGNUP_WIZARD_READINESS_CONFIG, + ); + const defaultBlocking = getBlockingServiceKeys( + session.readinessResult.health, + ); + if (hardBlocking.length === 0 && defaultBlocking.length === 0) return true; + return session.outageDismissed; + } + if (session.readinessResult.decision === WizardReadiness.No) { return session.outageDismissed; } diff --git a/src/ui/logging-ui.ts b/src/ui/logging-ui.ts index 2bc064f79..30ff56dac 100644 --- a/src/ui/logging-ui.ts +++ b/src/ui/logging-ui.ts @@ -123,9 +123,8 @@ export class LoggingUI implements WizardUI { console.log(`│`); console.log(`│ Blocking services:`); for (const key of blockingKeys) { - const health = result.health[key]; - if (!health) continue; - const { status, error } = health; + const status = result.health[key].status; + const error = result.health[key].error; const label = SERVICE_LABELS[key]; const detail = error ? ` — ${error}` : ''; console.log(`│ ✖ ${label}: ${status}${detail}`); diff --git a/src/ui/tui/__tests__/ink-ui-health.test.ts b/src/ui/tui/__tests__/ink-ui-health.test.ts deleted file mode 100644 index 3be86d9b4..000000000 --- a/src/ui/tui/__tests__/ink-ui-health.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { WizardStore, ScreenId } from '@ui/tui/store'; -import { InkUI } from '@ui/tui/ink-ui'; -import { - WizardReadiness, - type WizardReadinessResult, -} from '@lib/health-checks/readiness'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; -import { analytics } from '@utils/analytics'; - -vi.mock('../../../utils/analytics.js', () => ({ - analytics: { - capture: vi.fn(), - wizardCapture: vi.fn(), - setTag: vi.fn(), - captureException: vi.fn(), - }, - sessionProperties: vi.fn(() => ({})), -})); - -const skillsHealthy = { status: ServiceHealthStatus.Healthy } as const; -const gatewayOutage = (): WizardReadinessResult => ({ - decision: WizardReadiness.No, - health: { - skillsOrigin: skillsHealthy, - llmGateway: { status: ServiceHealthStatus.Down, error: 'HTTP 503' }, - }, - reasons: ['LLM gateway: down'], -}); - -describe('gateway outage after pre-auth health checks', () => { - it('waits for a fresh dismissal after the startup gate already resolved', async () => { - const store = new WizardStore(); - const ui = new InkUI(store); - store.completeSetup(); - store.setReadinessResult({ - decision: WizardReadiness.Yes, - health: { skillsOrigin: skillsHealthy }, - reasons: [], - }); - await store.getGate('health-check'); - store.dismissOutage(); - - let continued = false; - const waiting = ui.showBlockingOutage(gatewayOutage()).then(() => { - continued = true; - }); - await Promise.resolve(); - - expect(store.session.outageDismissed).toBe(false); - expect(store.currentScreen).toBe(ScreenId.HealthCheck); - expect(continued).toBe(false); - - store.dismissOutage(); - await waiting; - expect(continued).toBe(true); - expect(store.currentScreen).not.toBe(ScreenId.HealthCheck); - }); - - it('retains dismissal for the same result and resets it for a new outage', () => { - const store = new WizardStore(); - const first = gatewayOutage(); - store.setReadinessResult(first); - store.dismissOutage(); - store.setReadinessResult(first); - expect(store.session.outageDismissed).toBe(true); - - store.setReadinessResult(gatewayOutage()); - expect(store.session.outageDismissed).toBe(false); - }); - - it('records gateway failure without unrelated status-page claims', () => { - vi.mocked(analytics.wizardCapture).mockClear(); - const store = new WizardStore(); - store.setReadinessResult(gatewayOutage()); - - expect(analytics.wizardCapture).toHaveBeenCalledWith( - 'health check blocked', - { - decision: 'confirmed-outage', - blocking_keys: ['llmGateway'], - retries_used: 0, - }, - ); - }); -}); diff --git a/src/ui/tui/__tests__/programs.test.ts b/src/ui/tui/__tests__/programs.test.ts index f959c70ff..6030b70d2 100644 --- a/src/ui/tui/__tests__/programs.test.ts +++ b/src/ui/tui/__tests__/programs.test.ts @@ -1,7 +1,5 @@ import { buildSession, McpOutcome, RunPhase } from '@lib/wizard-session'; import { WizardReadiness } from '@lib/health-checks/readiness'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; -import { healthCheckReady } from '@lib/programs/shared/health-check-step'; import { PROGRAM_SEQUENCES, ScreenId } from '@ui/tui/screen-sequences'; import { Program, type ProgramId } from '@lib/programs/program-registry'; @@ -64,44 +62,6 @@ describe('PROGRAM_SEQUENCES', () => { }); describe('Wizard health-check predicate', () => { - it.each([false, true])( - 'applies the same dependency policy with signup=%s', - (signup) => { - const session = buildSession({ signup }); - session.readinessResult = { - decision: WizardReadiness.No, - health: { - skillsOrigin: { status: ServiceHealthStatus.Healthy }, - llmGateway: { status: ServiceHealthStatus.Down }, - }, - reasons: ['LLM gateway: down'], - }; - const check = getEntry( - Program.PostHogIntegration, - ScreenId.HealthCheck, - ); - expect(check.isComplete?.(session)).toBe(false); - session.outageDismissed = true; - expect(check.isComplete?.(session)).toBe(true); - }, - ); - - it('does not release the runnable health gate on a terminal error', () => { - const session = buildSession({}); - session.readinessResult = { - decision: WizardReadiness.No, - health: { skillsOrigin: { status: ServiceHealthStatus.Down } }, - reasons: ['Skills download: down'], - }; - session.runPhase = RunPhase.Error; - expect( - getEntry(Program.PostHogIntegration, ScreenId.HealthCheck).isComplete?.( - session, - ), - ).toBe(false); - expect(session.outageDismissed).toBe(false); - expect(healthCheckReady(session)).toBe(false); - }); it('stays incomplete before readiness exists', () => { const session = buildSession({}); const entry = getEntry(Program.PostHogIntegration, ScreenId.HealthCheck); @@ -116,7 +76,7 @@ describe('PROGRAM_SEQUENCES', () => { session.readinessResult = { decision: WizardReadiness.No, health: {} as never, - reasons: ['LLM gateway: down'], + reasons: ['Anthropic: down'], }; expect(entry.isComplete?.(session)).toBe(false); diff --git a/src/ui/tui/__tests__/router.test.ts b/src/ui/tui/__tests__/router.test.ts index dfa37b453..5cbea8054 100644 --- a/src/ui/tui/__tests__/router.test.ts +++ b/src/ui/tui/__tests__/router.test.ts @@ -6,8 +6,6 @@ import { } from '@lib/wizard-session'; import { HostResolution } from '@lib/host-resolution'; import { WizardReadiness } from '@lib/health-checks/readiness'; -import { healthCheckReady } from '@lib/programs/shared/health-check-step'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; import { WizardRouter, ScreenId, Overlay, Program } from '@ui/tui/router'; import { Integration } from '@lib/constants'; import { FRAMEWORK_REGISTRY } from '@lib/registry'; @@ -18,80 +16,6 @@ function baseWizardSession() { describe('WizardRouter', () => { describe('resolve', () => { - it('shows the error outro when exiting a pre-auth skills outage without releasing startup', () => { - const router = new WizardRouter(Program.PostHogIntegration); - const session = baseWizardSession(); - session.setupConfirmed = true; - session.readinessResult = { - decision: WizardReadiness.No, - health: { skillsOrigin: { status: ServiceHealthStatus.Down } }, - reasons: [], - }; - expect(router.resolve(session)).toBe(ScreenId.HealthCheck); - session.runPhase = RunPhase.Error; - session.outroData = { - kind: OutroKind.Error, - message: 'Exited due to service outage.', - }; - expect(router.resolve(session)).toBe(ScreenId.Outro); - expect(healthCheckReady(session)).toBe(false); - expect(session.outageDismissed).toBe(false); - }); - - it('shows the error outro when exiting a gateway outage during a composed integration run', () => { - const router = new WizardRouter(Program.SelfDriving); - const session = baseWizardSession(); - session.setupConfirmed = true; - session.integrate = true; - session.integration = Integration.nextjs; - session.credentials = { - accessToken: 'tok', - projectApiKey: 'pk', - host: HostResolution.fromApiHost('https://app.posthog.com'), - projectId: 1, - }; - session.readinessResult = { - decision: WizardReadiness.No, - health: { - skillsOrigin: { status: ServiceHealthStatus.Healthy }, - llmGateway: { status: ServiceHealthStatus.Down }, - }, - reasons: [], - }; - session.runPhase = RunPhase.Error; - session.outroData = { - kind: OutroKind.Error, - message: 'Exited due to service outage.', - }; - expect(router.resolve(session)).toBe(ScreenId.Outro); - expect(session.completedRuns).not.toContain('integrate-run'); - expect(session.outageDismissed).toBe(false); - }); - it('allows a dismissed terminal error to advance past the outro', () => { - const router = new WizardRouter(Program.PostHogIntegration); - const session = baseWizardSession(); - session.setupConfirmed = true; - session.readinessResult = { - decision: WizardReadiness.Yes, - health: { skillsOrigin: { status: ServiceHealthStatus.Healthy } }, - reasons: [], - }; - session.credentials = { - accessToken: 'tok', - projectApiKey: 'pk', - host: HostResolution.fromApiHost('https://app.posthog.com'), - projectId: 1, - }; - session.runPhase = RunPhase.Error; - session.outroData = { - kind: OutroKind.Error, - message: 'A screen crashed.', - }; - expect(router.resolve(session)).toBe(ScreenId.Outro); - session.outroDismissed = true; - expect(router.resolve(session)).toBe(ScreenId.Mcp); - }); - it('returns the first incomplete visible screen for the wizard flow', () => { const router = new WizardRouter(Program.PostHogIntegration); const session = baseWizardSession(); diff --git a/src/ui/tui/__tests__/store.test.ts b/src/ui/tui/__tests__/store.test.ts index 8d6559457..0201a6320 100644 --- a/src/ui/tui/__tests__/store.test.ts +++ b/src/ui/tui/__tests__/store.test.ts @@ -331,7 +331,7 @@ describe('WizardStore', () => { const result = { decision: WizardReadiness.No, health: {} as never, - reasons: ['LLM gateway: down'], + reasons: ['Anthropic: down'], }; store.setReadinessResult(result); expect(store.session.readinessResult).toEqual(result); @@ -1484,7 +1484,7 @@ describe('WizardStore', () => { evaluateWizardReadinessMock.mockResolvedValueOnce({ decision: WizardReadiness.No, health: {} as never, - reasons: ['LLM gateway: down'], + reasons: ['Anthropic: down'], }); const store = createStore(); diff --git a/src/ui/tui/components/ServiceHealthList.tsx b/src/ui/tui/components/ServiceHealthList.tsx index ebed560da..c38e9db4c 100644 --- a/src/ui/tui/components/ServiceHealthList.tsx +++ b/src/ui/tui/components/ServiceHealthList.tsx @@ -1,18 +1,34 @@ /** * ServiceHealthList — Shared component for displaying service health status. * - * Used by HealthCheckScreen and its playground demo. + * Used by HealthCheckScreen (blocking services only) and HealthWarningsTab (all services). */ import { Box, Text } from 'ink'; import { ServiceHealthStatus, type AllServicesHealth, + type ComponentHealthResult, + type ComponentStatus, type HealthCheckKey, } from '@lib/health-checks/types'; import { SERVICE_LABELS } from '@lib/health-checks/readiness'; import { Icons } from '@ui/tui/styles'; +/** Keys that are component-level detail — shown inline under their parent. */ +const COMPONENT_KEYS: HealthCheckKey[] = [ + 'posthogComponents', + 'npmComponents', + 'cloudflareComponents', +]; + +/** Map component key → its parent "overall" key */ +const COMPONENT_PARENT: Partial> = { + posthogComponents: 'posthogOverall', + npmComponents: 'npmOverall', + cloudflareComponents: 'cloudflareOverall', +}; + function statusIcon(status: ServiceHealthStatus): { icon: string; color: string; @@ -42,26 +58,35 @@ export const ServiceHealthList = ({ filterKeys, showHealthy = true, }: ServiceHealthListProps) => { - const serviceKeys = Object.keys(SERVICE_LABELS) as HealthCheckKey[]; + const topLevelKeys = (Object.keys(health) as HealthCheckKey[]).filter( + (k) => !COMPONENT_KEYS.includes(k), + ); const keysToShow = filterKeys - ? serviceKeys.filter((k) => filterKeys.includes(k)) - : serviceKeys; + ? topLevelKeys.filter((k) => filterKeys.includes(k)) + : topLevelKeys; return ( {keysToShow.map((key) => { const result = health[key]; - if ( - !result || - (!showHealthy && result.status === ServiceHealthStatus.Healthy) - ) { + if (!showHealthy && result.status === ServiceHealthStatus.Healthy) { return null; } const { icon, color } = statusIcon(result.status); const label = SERVICE_LABELS[key]; + // Find component-level details if this is a parent key + const componentKey = ( + Object.entries(COMPONENT_PARENT) as [HealthCheckKey, HealthCheckKey][] + ).find(([, parent]) => parent === key)?.[0]; + const componentResult = componentKey + ? (health[componentKey] as ComponentHealthResult) + : undefined; + const affectedComponents: ComponentStatus[] = + componentResult?.degradedOrDownComponents ?? []; + return ( @@ -69,10 +94,22 @@ export const ServiceHealthList = ({ {label} - {result.status === ServiceHealthStatus.NoConnection && ( - — No connection - )} + {affectedComponents.length > 0 && ( + + {affectedComponents.slice(0, 5).map((c) => { + const ci = statusIcon(c.status); + return ( + + {ci.icon} {c.name} + + ); + })} + {affectedComponents.length > 5 && ( + +{affectedComponents.length - 5} more + )} + + )} ); })} diff --git a/src/ui/tui/ink-ui.ts b/src/ui/tui/ink-ui.ts index deb7acaeb..60c9f27da 100644 --- a/src/ui/tui/ink-ui.ts +++ b/src/ui/tui/ink-ui.ts @@ -134,10 +134,10 @@ export class InkUI implements WizardUI { } showBlockingOutage(result: WizardReadinessResult): Promise { - // A gateway check can fail after the pre-auth health gate resolved. - // Wait for this outage's dismissal, not the already-latched gate. + // In the TUI, the HealthCheckScreen handles outage display. + // This is only called from agent-runner for the CI fallback path. this.store.setReadinessResult(result); - return this.store.waitUntil((session) => session.outageDismissed); + return Promise.resolve(); } setReadinessWarnings(result: WizardReadinessResult): void { diff --git a/src/ui/tui/playground/demos/HealthCheckDemo.tsx b/src/ui/tui/playground/demos/HealthCheckDemo.tsx index 1c6c0eb54..501f6972c 100644 --- a/src/ui/tui/playground/demos/HealthCheckDemo.tsx +++ b/src/ui/tui/playground/demos/HealthCheckDemo.tsx @@ -1,48 +1,83 @@ /** - * HealthCheckDemo — checking, gateway outage, and unavailable skill downloads. - * Renders components directly to avoid conflicts with TabContainer input. + * HealthCheckDemo — Playground demo for health check UI components. + * + * Cycles through three states (2s checking spinner → 5s confirmed-outage + * red modal → 5s no-connection yellow modal, then loops): + * 1. Checking (spinner) + * 2. Confirmed outage (status page corroborates → red framing) + * 3. No connection only (no status-page incident → yellow "couldn't + * reach PostHog" framing) + * + * Renders components directly (not HealthCheckScreen) to avoid useInput + * conflicts with TabContainer's key handling. */ import { useEffect, useState } from 'react'; import { Box, Text } from 'ink'; import { LoadingBox, ModalOverlay } from '@ui/tui/primitives/index'; +import { Icons } from '@ui/tui/styles'; import { ServiceHealthList } from '@ui/tui/components/ServiceHealthList'; import { getBlockingServiceKeys } from '@lib/health-checks/readiness'; -import { - ServiceHealthStatus, - type AllServicesHealth, -} from '@lib/health-checks/types'; +import { ServiceHealthStatus } from '@lib/health-checks/types'; +import type { AllServicesHealth } from '@lib/health-checks/types'; -const MOCK_GATEWAY_OUTAGE: AllServicesHealth = { - llmGateway: { status: ServiceHealthStatus.Down, error: 'HTTP 503' }, - skillsOrigin: { status: ServiceHealthStatus.Healthy }, +const HEALTHY = { status: ServiceHealthStatus.Healthy } as const; + +const MOCK_CONFIRMED_OUTAGE: AllServicesHealth = { + anthropic: { status: ServiceHealthStatus.Down, rawIndicator: 'major' }, + posthogOverall: HEALTHY, + posthogComponents: { status: ServiceHealthStatus.Healthy }, + github: HEALTHY, + npmOverall: { + status: ServiceHealthStatus.Degraded, + rawIndicator: 'minor', + }, + npmComponents: { + status: ServiceHealthStatus.Degraded, + degradedOrDownComponents: [ + { + name: 'Registry API', + status: ServiceHealthStatus.Degraded, + rawStatus: 'degraded_performance', + }, + ], + }, + cloudflareOverall: HEALTHY, + cloudflareComponents: { status: ServiceHealthStatus.Healthy }, + mcp: HEALTHY, + skillsOrigin: HEALTHY, }; -const MOCK_SKILLS_UNAVAILABLE: AllServicesHealth = { - skillsOrigin: { +const MOCK_NO_CONNECTION: AllServicesHealth = { + anthropic: HEALTHY, + posthogOverall: HEALTHY, + posthogComponents: { status: ServiceHealthStatus.Healthy }, + github: HEALTHY, + npmOverall: HEALTHY, + npmComponents: { status: ServiceHealthStatus.Healthy }, + cloudflareOverall: HEALTHY, + cloudflareComponents: { status: ServiceHealthStatus.Healthy }, + mcp: { status: ServiceHealthStatus.NoConnection, - error: 'No configured skills source is reachable', + error: 'fetch failed', }, + skillsOrigin: HEALTHY, }; -type Phase = 'checking' | 'gateway' | 'skills'; +type Phase = 'checking' | 'confirmed' | 'no-connection'; export const HealthCheckDemo = () => { const [phase, setPhase] = useState('checking'); useEffect(() => { - const timer = setTimeout( - () => - setPhase( - phase === 'checking' - ? 'gateway' - : phase === 'gateway' - ? 'skills' - : 'checking', - ), - phase === 'checking' ? 2000 : 5000, - ); - return () => clearTimeout(timer); + const t1 = setTimeout(() => setPhase('confirmed'), 2000); + const t2 = setTimeout(() => setPhase('no-connection'), 7000); + const t3 = setTimeout(() => setPhase('checking'), 12000); + return () => { + clearTimeout(t1); + clearTimeout(t2); + clearTimeout(t3); + }; }, [phase]); if (phase === 'checking') { @@ -53,49 +88,56 @@ export const HealthCheckDemo = () => { alignItems="center" justifyContent="center" > - + ); } - const skillsUnavailable = phase === 'skills'; - const health = skillsUnavailable - ? MOCK_SKILLS_UNAVAILABLE - : MOCK_GATEWAY_OUTAGE; + const health = + phase === 'confirmed' ? MOCK_CONFIRMED_OUTAGE : MOCK_NO_CONNECTION; const blockingKeys = getBlockingServiceKeys(health); - const allNoConnection = blockingKeys.every( - (key) => health[key]?.status === ServiceHealthStatus.NoConnection, - ); + const isNoConnection = phase === 'no-connection'; return ( - {skillsUnavailable ? 'Exit [Esc]' : 'Continue [Enter] / Exit [Esc]'}{' '} - (disabled in playground) + Continue [Enter] / Exit [Esc] (disabled in playground) } > + + + {Icons.squareFilled} + Down + {Icons.squareFilled} + Degraded + {Icons.squareFilled} + No connection + + + + - {skillsUnavailable - ? 'The Wizard could not download the skills it needs from any configured source. Check your connection and try again.' - : 'The PostHog AI gateway is currently unavailable. You can try continuing, or exit and try again later.'} + {isNoConnection + ? "We couldn't reach these services. PostHog's status page shows no incidents, likely a network issue (VPN, firewall, captive portal, or Wi-Fi)." + : 'The wizard may not work reliably while services are affected.'} ); diff --git a/src/ui/tui/router.ts b/src/ui/tui/router.ts index 757a64fe8..c9b887072 100644 --- a/src/ui/tui/router.ts +++ b/src/ui/tui/router.ts @@ -64,20 +64,23 @@ export class WizardRouter { return this.overlays[this.overlays.length - 1]; } - // Terminal errors must remain dismissible even when auth, a health check, - // or a composed run never completes. This changes only screen selection; - // the runnable gates stay blocked until the process exits. - if ( - session.runPhase === RunPhase.Error && - session.outroData && - !session.outroDismissed - ) { - return ScreenId.Outro; - } - for (const entry of this.sequence) { if (entry.show && !entry.show(session)) continue; if (entry.isComplete && entry.isComplete(session)) continue; + // A failed login aborts the run: wizardAbort renders the error outro + // and then waits for its dismissal. But the auth step only completes + // on credentials — which an aborted login never set — so the walk + // would park here forever: auth spinner up, outro unreachable, and + // that wait deadlocked. Route to the outro so the error can be read + // and dismissed. Auth only: the run steps already complete on + // RunPhase.Error, so later aborts reach their program's own outro. + if ( + entry.id === ScreenId.Auth && + session.runPhase === RunPhase.Error && + session.outroData + ) { + return ScreenId.Outro; + } return entry.id; } diff --git a/src/ui/tui/screens/health/HealthCheckScreen.tsx b/src/ui/tui/screens/health/HealthCheckScreen.tsx index 1f89888e5..8627e3ec7 100644 --- a/src/ui/tui/screens/health/HealthCheckScreen.tsx +++ b/src/ui/tui/screens/health/HealthCheckScreen.tsx @@ -17,12 +17,15 @@ import { } from '@ui/tui/primitives/index'; import { Colors, Icons } from '@ui/tui/styles'; import { ServiceHealthList } from '@ui/tui/components/ServiceHealthList'; -import { getBlockingServiceKeys } from '@lib/health-checks/readiness'; +import { + getBlockingServiceKeys, + SIGNUP_WIZARD_READINESS_CONFIG, +} from '@lib/health-checks/readiness'; import { ServiceHealthStatus } from '@lib/health-checks/types'; import { wizardAbort } from '@utils/wizard-abort'; import { ErrorCodes } from '@lib/errors'; import { fetchSkillMenu, downloadSkill } from '@lib/wizard-tools'; -import { getSkillsBaseUrl } from '@lib/constants'; +import { GITHUB_SKILLS_BASE_URL } from '@lib/constants'; import { useDismissOnAnyKey } from '@ui/tui/hooks/useDismissOnAnyKey'; interface HealthCheckScreenProps { @@ -65,7 +68,6 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { const [downloaded, setDownloaded] = useState(false); const [downloading, setDownloading] = useState(false); - const [downloadError, setDownloadError] = useState(null); const result = store.session.readinessResult; @@ -82,78 +84,87 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { alignItems="center" justifyContent="center" > - + ); } - const blockingKeys = getBlockingServiceKeys(result.health); - if (blockingKeys.length === 0) return null; + const isSignup = store.session.signup; + const blockingKeys = getBlockingServiceKeys( + result.health, + isSignup ? SIGNUP_WIZARD_READINESS_CONFIG : undefined, + ); + + // Signup has a narrower block list (only posthog + llm-gateway), so + // services like Anthropic can be degraded without blocking. Surface + // those as dismissable warnings instead of silently proceeding. + const warningKeys = isSignup + ? getBlockingServiceKeys(result.health).filter( + (k) => !blockingKeys.includes(k), + ) + : []; - const isSkillsOriginDown = blockingKeys.includes('skillsOrigin'); + const hasHardBlock = blockingKeys.length > 0; + const displayKeys = hasHardBlock ? blockingKeys : warningKeys; + if (displayKeys.length === 0) return null; + + const isSkillsOriginDown = + hasHardBlock && blockingKeys.includes('skillsOrigin'); const canDownloadSkills = result.health.skillsOrigin.status === ServiceHealthStatus.Healthy; const integration = store.session.integration; - const canOfferDownload = canDownloadSkills && Boolean(integration); - const allNoConnection = blockingKeys.every( - (key) => result.health[key]?.status === ServiceHealthStatus.NoConnection, - ); - const title = allNoConnection - ? isSkillsOriginDown - ? 'Could not connect to skill downloads' - : 'Could not connect to the AI gateway' - : isSkillsOriginDown - ? 'Skill downloads unavailable' - : 'AI gateway unavailable'; + + // If every blocking row is `NoConnection` (probe failed, no status-page + // corroboration), reframe the screen to point at the user's network + // instead of accusing PostHog of an outage. Mixed Down + NoConnection + // falls through to the confirmed-outage framing because there's still + // a real incident underneath. + const allBlockingHaveNoConnection = + hasHardBlock && + displayKeys.every( + (k) => result.health[k].status === ServiceHealthStatus.NoConnection, + ); + + const title = isSkillsOriginDown + ? 'Ongoing service disruptions' + : allBlockingHaveNoConnection + ? "Couldn't reach PostHog" + : hasHardBlock + ? 'Ongoing service disruptions' + : 'Service disruption detected'; const docsUrl = store.session.frameworkConfig?.metadata.docsUrl; const description = isSkillsOriginDown - ? 'The Wizard could not download the skills it needs from any configured source. Check your connection and try again.' - : allNoConnection - ? 'The Wizard could not connect to the PostHog AI gateway from this machine. Check your connection and try again.' - : 'The PostHog AI gateway is currently unavailable. You can try continuing, or exit and try again later.'; + ? "The Wizard can't download the skills it needs — neither GitHub Releases nor PostHog's mirror is reachable right now." + : allBlockingHaveNoConnection + ? "We couldn't reach these services from this machine. PostHog's status page shows no incidents, so this is most likely a network issue — VPN, firewall, captive portal, or flaky Wi-Fi." + : hasHardBlock + ? 'The Wizard cannot start while these services are down.' + : 'Some services are degraded. You can continue, but parts of the wizard may not work reliably.'; const handleDownloadAndExit = async () => { - if (downloading || !integration) return; + if (downloading) return; setDownloading(true); - setDownloadError(null); - try { - // Use the same source as the run; release downloads fail over themselves. - const menu = await fetchSkillMenu(getSkillsBaseUrl()); - if (!menu) throw new Error('Could not load the integration skills.'); + // Primary origin — fetchSkillMenu/downloadSkill fail over to AWS themselves. + const menu = await fetchSkillMenu(GITHUB_SKILLS_BASE_URL); + if (menu) { const prefix = `integration-${integration}`; const skills = (menu.categories['integration'] ?? []).filter((s) => s.id.startsWith(prefix), ); - if (skills.length === 0) { - throw new Error('No integration skills were found for this project.'); - } for (const skill of skills) { - // The gateway is unavailable, so a flagged skill must fail closed. - const installed = await downloadSkill(skill, store.session.installDir, { + // Pre-auth outage cache: no gateway, so a flagged skill fails closed. + await downloadSkill(skill, store.session.installDir, { skillsRoot: '.posthog/skills', triage: undefined, }); - if (!installed.success) { - throw new Error( - 'The integration skills could not be downloaded safely.', - ); - } } - setDownloaded(true); - } catch (error) { - setDownloadError( - error instanceof Error - ? error.message - : 'Could not download the integration skills.', - ); - } finally { - setDownloading(false); } + setDownloaded(true); }; const handleCancel = - canOfferDownload && !isSkillsOriginDown && !downloadError + canDownloadSkills && !isSkillsOriginDown ? () => void handleDownloadAndExit() : () => void wizardAbort({ @@ -162,7 +173,7 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { }); const cancelLabel = - canOfferDownload && !isSkillsOriginDown && !downloadError + canDownloadSkills && !isSkillsOriginDown ? downloading ? 'Downloading...' : 'Download skills & Exit [Esc]' @@ -170,7 +181,9 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { return ( { } > + + + {Icons.squareFilled} + Down + {Icons.squareFilled} + Degraded + {Icons.squareFilled} + No connection + + + @@ -221,13 +245,7 @@ export const HealthCheckScreen = ({ store }: HealthCheckScreenProps) => { )} - {downloadError && ( - - {downloadError} - - )} - - {canOfferDownload && !isSkillsOriginDown && !downloadError && ( + {canDownloadSkills && !isSkillsOriginDown && ( You can still download the PostHog integration skills and continue diff --git a/src/ui/tui/store.ts b/src/ui/tui/store.ts index 47c56f69c..12cf174ab 100644 --- a/src/ui/tui/store.ts +++ b/src/ui/tui/store.ts @@ -130,7 +130,10 @@ const MAX_STATUS_MESSAGES = EXPANDED_COUNT; /** * Fired once per blocked readiness result, so we can quantify how often - * the wizard pauses for its gateway or for unavailable skill downloads. + * the wizard refuses to start and — crucially — split that between + * confirmed PostHog outages and probe-level reachability failures that + * are most likely the user's network. Helps us decide whether the + * health-check UX is over-firing. */ function captureHealthCheckBlocked(result: WizardReadinessResult): void { try { @@ -150,9 +153,10 @@ function captureHealthCheckBlocked(result: WizardReadinessResult): void { ? 'no-connection' : 'confirmed-outage'; + const posthogStatus = health.posthogOverall?.status; const retriesUsed = Math.max( 0, - ...(['llmGateway', 'skillsOrigin'] as const).map((k) => { + ...(['mcp', 'skillsOrigin'] as const).map((k) => { const ind = health[k]?.rawIndicator ?? ''; const m = ind.match(/attempts=(\d+)/); return m ? Number(m[1]) - 1 : 0; @@ -162,6 +166,11 @@ function captureHealthCheckBlocked(result: WizardReadinessResult): void { analytics.wizardCapture('health check blocked', { decision, blocking_keys: blockingKeys, + posthog_status_reachable: + posthogStatus !== ServiceHealthStatus.NoConnection, + posthog_status_reports_incident: + posthogStatus === ServiceHealthStatus.Down || + posthogStatus === ServiceHealthStatus.Degraded, retries_used: retriesUsed, }); } catch (err) { @@ -549,15 +558,10 @@ export class WizardStore { } setReadinessResult(result: WizardReadinessResult | null): void { - const newBlockedResult = - result && - result !== this.session.readinessResult && - result.decision === WizardReadiness.No; - if (newBlockedResult) { - this.$session.setKey('outageDismissed', false); + this.$session.setKey('readinessResult', result); + if (result && result.decision === WizardReadiness.No) { captureHealthCheckBlocked(result); } - this.$session.setKey('readinessResult', result); this.emitChange(); } diff --git a/src/utils/anthropic-status.ts b/src/utils/anthropic-status.ts new file mode 100644 index 000000000..5f05fbfa5 --- /dev/null +++ b/src/utils/anthropic-status.ts @@ -0,0 +1,73 @@ +const CLAUDE_STATUS_URL = 'https://status.claude.com/api/v2/status.json'; + +type StatusIndicator = 'none' | 'minor' | 'major' | 'critical'; + +interface ClaudeStatusResponse { + page: { + id: string; + name: string; + url: string; + time_zone: string; + updated_at: string; + }; + status: { + indicator: StatusIndicator; + description: string; + }; +} + +export type StatusCheckResult = + | { status: 'operational' } + | { status: 'degraded'; description: string } + | { status: 'down'; description: string } + | { status: 'unknown'; error: string }; + +/** + * Check the Anthropic/Claude status page for service health. + * Pure function — no UI calls. + */ +export async function checkAnthropicStatus(): Promise { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 5000); + + const response = await fetch(CLAUDE_STATUS_URL, { + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + return { + status: 'unknown', + error: `Status page returned ${response.status}`, + }; + } + + const data = (await response.json()) as ClaudeStatusResponse; + const indicator = data.status.indicator; + const rawDesc = data.status.description; + const description = + rawDesc.charAt(0).toUpperCase() + rawDesc.slice(1).toLowerCase(); + + switch (indicator) { + case 'none': + return { status: 'operational' }; + case 'minor': + return { status: 'degraded', description }; + case 'major': + case 'critical': + return { status: 'down', description }; + default: + return { status: 'unknown', error: `Unknown indicator: ${indicator}` }; + } + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return { status: 'unknown', error: 'Request timed out' }; + } + return { + status: 'unknown', + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} From ca479b7935f108182c7d677b44b3b0fff8e39ac9 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 9 Sep 2026 10:54:34 -0400 Subject: [PATCH 3/3] refactor(health): remove unused service checks --- README.md | 12 +- .../__tests__/health-checks.test.ts | 934 +----------------- src/lib/health-checks/endpoints.ts | 28 +- src/lib/health-checks/incidentio.ts | 167 ---- src/lib/health-checks/index.ts | 19 +- src/lib/health-checks/readiness.ts | 149 +-- src/lib/health-checks/statuspage.ts | 144 --- src/lib/health-checks/testme.md | 66 +- src/lib/health-checks/types.ts | 27 +- src/ui/tui/components/ServiceHealthList.tsx | 51 +- .../tui/playground/demos/HealthCheckDemo.tsx | 65 +- src/ui/tui/store.ts | 41 +- src/utils/anthropic-status.ts | 73 -- 13 files changed, 47 insertions(+), 1729 deletions(-) delete mode 100644 src/lib/health-checks/incidentio.ts delete mode 100644 src/lib/health-checks/statuspage.ts delete mode 100644 src/utils/anthropic-status.ts diff --git a/README.md b/README.md index ad120f6ff..6f9d1d056 100644 --- a/README.md +++ b/README.md @@ -598,9 +598,8 @@ To make your version of a tool usable with a one-line `npx` command: # Health checks -`src/lib/health-checks/` checks external status pages and PostHog-owned -services before the wizard runs to decide whether it can proceed. The entry -point is `evaluateWizardReadiness()`, which only blocks on skill downloads: +`src/lib/health-checks/` checks skills download origins before the wizard runs. +The entry point is `evaluateWizardReadiness()`, which only blocks on skill downloads: | Decision | Meaning | | ------------------- | --------------------------------------------------------------- | @@ -612,8 +611,7 @@ point is `evaluateWizardReadiness()`, which only blocks on skill downloads: | File | Responsibility | | --- | --- | | `types.ts` | Enums, interfaces (`ServiceHealthStatus`, `AllServicesHealth`, etc.) | -| `statuspage.ts` | Statuspage.io v2 API helpers + checks for Anthropic, PostHog, GitHub, npm, Cloudflare | -| `endpoints.ts` | Direct endpoint checks for MCP (`/`) and the skills origins (`skill-menu.json` on GitHub Releases + the AWS mirror) | +| `endpoints.ts` | Direct gateway (`/readyz`) and skills origin (`skill-menu.json`) checks | | `readiness.ts` | `checkAllExternalServices`, `evaluateWizardReadiness`, readiness config | | `index.ts` | Barrel re-export | | `testme.md` | Test running instructions and endpoint reference | @@ -634,8 +632,8 @@ two arrays: downBlocksRun: ['skillsOrigin'], ``` -The same policy applies during signup. Other status-page results do not warn or -block. After minting a token, `gateway-session.ts` checks `/readyz` on the returned +The same policy applies during signup. Third-party status pages are not queried. +After minting a token, `gateway-session.ts` checks `/readyz` on the returned gateway URL and reports an unavailable gateway through the existing error path. `skillsOrigin` is one entry covering two origins: skills are published to diff --git a/src/lib/health-checks/__tests__/health-checks.test.ts b/src/lib/health-checks/__tests__/health-checks.test.ts index 12ea8afb8..b23b7e2c4 100644 --- a/src/lib/health-checks/__tests__/health-checks.test.ts +++ b/src/lib/health-checks/__tests__/health-checks.test.ts @@ -1,32 +1,6 @@ -/** - * Tests for health-checks.ts - * - * Mock data is modelled on live Statuspage.io v2 API responses. - * Statuspage docs: https://metastatuspage.com/api - * - * status.json – page-level rollup with indicator (none | minor | major | critical) - * summary.json – same rollup plus component list; component statuses: - * operational | degraded_performance | partial_outage | major_outage | under_maintenance - * https://support.atlassian.com/statuspage/docs/show-service-status-with-components - - * - * MCP – Cloudflare Worker, GET / returns an HTML landing page (200) - * Source: posthog/services/mcp/src/index.ts - */ - import { checkAllExternalServices, - checkAnthropicHealth, - checkCloudflareComponentHealth, - checkCloudflareOverallHealth, - checkGithubHealth, checkSkillsOriginHealth, - checkMcpHealth, - checkNpmComponentHealth, - checkNpmOverallHealth, - checkPosthogComponentHealth, - checkPosthogOverallHealth, - resetPosthogHealthCache, DEFAULT_WIZARD_READINESS_CONFIG, evaluateWizardReadiness, ServiceHealthStatus, @@ -38,221 +12,14 @@ import { } from '@lib/health-checks/endpoints'; import { SIGNUP_WIZARD_READINESS_CONFIG } from '@lib/health-checks/readiness'; -// --------------------------------------------------------------------------- -// Real-world Statuspage.io v2 response factories -// https://metastatuspage.com/api -// --------------------------------------------------------------------------- - -function makeStatuspageStatus(opts: { - pageId: string; - pageName: string; - pageUrl: string; - indicator: 'none' | 'minor' | 'major' | 'critical'; - description: string; -}) { - return { - page: { - id: opts.pageId, - name: opts.pageName, - url: opts.pageUrl, - time_zone: 'Etc/UTC', - updated_at: '2026-03-05T16:03:38.861Z', - }, - status: { - indicator: opts.indicator, - description: opts.description, - }, - }; -} - -function makeStatuspageSummary(opts: { - pageId: string; - pageName: string; - pageUrl: string; - indicator: 'none' | 'minor' | 'major' | 'critical'; - description: string; - components: { - id: string; - name: string; - status: string; - position: number; - description: string | null; - }[]; -}) { - return { - page: { - id: opts.pageId, - name: opts.pageName, - url: opts.pageUrl, - time_zone: 'Etc/UTC', - updated_at: '2026-03-05T16:03:38.861Z', - }, - status: { - indicator: opts.indicator, - description: opts.description, - }, - components: opts.components.map((c) => ({ - ...c, - page_id: opts.pageId, - created_at: '2023-07-11T17:52:24.275Z', - updated_at: '2026-03-04T17:01:29.960Z', - showcase: true, - start_date: '2023-07-11', - group_id: null, - group: false, - only_show_if_degraded: false, - })), - incidents: [], - scheduled_maintenances: [], - }; -} - -// Shapes taken from live GET on 2026-03-05 -const ANTHROPIC_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const GITHUB_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'kctbh9vrtdwd', - pageName: 'GitHub', - pageUrl: 'https://www.githubstatus.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const NPM_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'none', - description: 'All Systems Operational', -}); - -const NPM_SUMMARY_HEALTHY = makeStatuspageSummary({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'none', - description: 'All Systems Operational', - components: [ - { - id: 'mvm98gtxvb9b', - name: 'www.npmjs.com website', - status: 'operational', - position: 1, - description: - 'The ability for users to navigate to or interact with the npm website.', - }, - { - id: 'k1wj10x6gmph', - name: 'Package installation', - status: 'operational', - position: 2, - description: - 'The ability for users to read from the registry so that they can install packages.', - }, - ], -}); - -const CLOUDFLARE_STATUS_HEALTHY = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'none', - description: 'All Systems Operational', -}); - -const CLOUDFLARE_SUMMARY_HEALTHY = makeStatuspageSummary({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'none', - description: 'All Systems Operational', - components: [ - { - id: '1km35smx8p41', - name: 'Cloudflare Sites and Services', - status: 'operational', - position: 1, - description: - 'Sites and services that Cloudflare customers use to interact with the Cloudflare Network', - }, - ], -}); - -// PostHog incident.io v1 API mock data -const POSTHOG_INCIDENTIO_HEALTHY = { - page_title: 'PostHog', - page_url: 'https://www.posthogstatus.com/', - ongoing_incidents: [], - in_progress_maintenances: [], - scheduled_maintenances: [], -}; - -// MCP / landing page (from posthog/services/mcp/src/index.ts + src/static/landing.html) -const MCP_LANDING_HTML = - 'PostHog MCP Server'; - -// --------------------------------------------------------------------------- -// URL constants (must match health-checks.ts) -// --------------------------------------------------------------------------- - const URLS = { - anthropicStatus: 'https://status.claude.com/api/v2/status.json', - posthogIncidentIo: 'https://www.posthogstatus.com/api/v1/summary', - githubStatus: 'https://www.githubstatus.com/api/v2/status.json', - npmStatus: 'https://status.npmjs.org/api/v2/status.json', - npmSummary: 'https://status.npmjs.org/api/v2/summary.json', - cloudflareStatus: 'https://www.cloudflarestatus.com/api/v2/status.json', - cloudflareSummary: 'https://www.cloudflarestatus.com/api/v2/summary.json', - mcpLanding: 'https://mcp.posthog.com/', githubSkillMenu: 'https://github.com/PostHog/context-mill/releases/latest/download/skill-menu.json', awsSkillMenu: 'https://context-mill.posthog.com/latest/skill-menu.json', } as const; -// --------------------------------------------------------------------------- -// Helper to build a default "all healthy" fetch mock -// --------------------------------------------------------------------------- - const HEALTHY_RESPONSES: Record = { - [URLS.anthropicStatus]: { - body: JSON.stringify(ANTHROPIC_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.posthogIncidentIo]: { - body: JSON.stringify(POSTHOG_INCIDENTIO_HEALTHY), - contentType: 'application/json', - }, - [URLS.githubStatus]: { - body: JSON.stringify(GITHUB_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.npmStatus]: { - body: JSON.stringify(NPM_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.npmSummary]: { - body: JSON.stringify(NPM_SUMMARY_HEALTHY), - contentType: 'application/json', - }, - [URLS.cloudflareStatus]: { - body: JSON.stringify(CLOUDFLARE_STATUS_HEALTHY), - contentType: 'application/json', - }, - [URLS.cloudflareSummary]: { - body: JSON.stringify(CLOUDFLARE_SUMMARY_HEALTHY), - contentType: 'application/json', - }, - [URLS.mcpLanding]: { - body: MCP_LANDING_HTML, - contentType: 'text/html; charset=utf-8', - }, [URLS.githubSkillMenu]: { body: JSON.stringify({ categories: { integration: [] } }), contentType: 'application/json', @@ -295,16 +62,11 @@ function overrideFetch(overrides: Record Promise>) { }; } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - describe('health-checks', () => { const originalFetch = global.fetch; beforeEach(() => { vi.restoreAllMocks(); - resetPosthogHealthCache(); (global as any).fetch = vi.fn(allHealthyFetchMock); }); @@ -312,415 +74,6 @@ describe('health-checks', () => { (global as any).fetch = originalFetch; }); - // ----------------------------------------------------------------------- - // Statuspage status.json checks (indicator-based) - // ----------------------------------------------------------------------- - - describe('checkAnthropicHealth', () => { - it('returns healthy for indicator=none ("All Systems Operational")', async () => { - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('none'); - }); - - it('returns degraded for indicator=minor ("Minor Service Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.rawIndicator).toBe('minor'); - }); - - it('returns down for indicator=major ("Partial System Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'major', - description: 'Partial System Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns down for indicator=critical ("Major Service Outage")', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'critical', - description: 'Major Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns degraded when statuspage returns HTTP 500', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response('Internal Server Error', { status: 500 }), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.error).toBe('HTTP 500'); - }); - - it('returns degraded when fetch throws (network failure)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.reject( - new Error('getaddrinfo ENOTFOUND status.claude.com'), - ), - }), - ); - const result = await checkAnthropicHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.error).toBe('getaddrinfo ENOTFOUND status.claude.com'); - }); - }); - - describe('checkPosthogOverallHealth', () => { - it('returns healthy when no ongoing incidents', async () => { - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('returns down when an incident has full_outage impact', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: '01KA9JH0ZB14TFA8VD4CFC3AYN', - name: 'Major service outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'US Cloud', - current_status: 'full_outage', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns NoConnection when posthogstatus.com fetch fails with a network error', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.reject(new Error('getaddrinfo ENOTFOUND')), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('returns NoConnection when posthogstatus.com fetch times out', async () => { - const abortError = new Error('aborted'); - abortError.name = 'AbortError'; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => Promise.reject(abortError), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('returns Down when posthogstatus.com returns an HTTP error', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - }); - - it('returns degraded when an incident has partial_outage impact', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: '01KA9JH0ZB14TFA8VD4CFC3AYN', - name: 'Partial outage', - status: 'investigating', - current_worst_impact: 'partial_outage', - affected_components: [], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - }); - }); - - describe('checkGithubHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkGithubHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - }); - - describe('checkNpmOverallHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkNpmOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - }); - - describe('checkCloudflareOverallHealth', () => { - it('returns healthy for indicator=none', async () => { - const result = await checkCloudflareOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('returns degraded for indicator=minor', async () => { - const body = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.cloudflareStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkCloudflareOverallHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - }); - }); - - // ----------------------------------------------------------------------- - // Statuspage summary.json checks (component-based) - // ----------------------------------------------------------------------- - - describe('checkPosthogComponentHealth', () => { - it('reports healthy when no ongoing incidents', async () => { - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.degradedOrDownComponents).toBeUndefined(); - }); - - it('reports affected components from ongoing incidents', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'US Cloud outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'US Cloud 🇺🇸', - current_status: 'full_outage', - }, - { - id: 'c2', - name: 'Event Ingestion', - group_name: 'US Cloud 🇺🇸', - current_status: 'full_outage', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents).toHaveLength(2); - expect(result.degradedOrDownComponents![0].name).toBe( - 'US Cloud 🇺🇸 — App', - ); - expect(result.degradedOrDownComponents![0].status).toBe( - ServiceHealthStatus.Down, - ); - expect(result.degradedOrDownComponents![1].status).toBe( - ServiceHealthStatus.Down, - ); - }); - - it('reports degraded for degraded_performance components', async () => { - const body = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'Slowness', - status: 'investigating', - current_worst_impact: 'degraded_performance', - affected_components: [ - { - id: 'c1', - name: 'App', - group_name: 'EU Cloud', - current_status: 'degraded_performance', - }, - ], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkPosthogComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents![0].rawStatus).toBe( - 'degraded_performance', - ); - expect(result.degradedOrDownComponents![0].status).toBe( - ServiceHealthStatus.Degraded, - ); - }); - }); - - describe('checkNpmComponentHealth', () => { - it('reports healthy when all npm components operational', async () => { - const result = await checkNpmComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - - it('reports degraded when "Package installation" has partial_outage', async () => { - const body = makeStatuspageSummary({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'major', - description: 'Partial System Outage', - components: [ - { - id: 'mvm98gtxvb9b', - name: 'www.npmjs.com website', - status: 'operational', - position: 1, - description: null, - }, - { - id: 'k1wj10x6gmph', - name: 'Package installation', - status: 'partial_outage', - position: 2, - description: null, - }, - ], - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.npmSummary]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await checkNpmComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Degraded); - expect(result.degradedOrDownComponents![0].name).toBe( - 'Package installation', - ); - }); - }); - - describe('checkCloudflareComponentHealth', () => { - it('reports healthy when Cloudflare components operational', async () => { - const result = await checkCloudflareComponentHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - }); - }); - - // ----------------------------------------------------------------------- - // fetchEndpointHealth (retry + status-taxonomy machinery, probed directly - // against a synthetic URL — no production probe uses the strict defaults - // any more, but every endpoint check shares this loop) - // ----------------------------------------------------------------------- - describe('fetchEndpointHealth', () => { const PROBE_URL = 'https://probe.posthog.test/_liveness'; @@ -883,10 +236,6 @@ describe('health-checks', () => { }); }); - // ----------------------------------------------------------------------- - // MCP (fetchEndpointHealth – / landing) - // ----------------------------------------------------------------------- - it.each([ 'https://ai-gateway.us.posthog.com', 'https://ai-gateway.eu.posthog.com/', @@ -909,87 +258,6 @@ describe('health-checks', () => { }, ); - describe('checkMcpHealth', () => { - it('returns healthy when MCP worker responds 200 with landing HTML', async () => { - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 200'); - expect(global.fetch).toHaveBeenCalledWith( - URLS.mcpLanding, - expect.objectContaining({ signal: expect.any(AbortSignal) }), - ); - }); - - it('returns healthy when worker responds 302 (redirect to docs, not followed)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response(null, { status: 302 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Healthy); - expect(result.rawIndicator).toBe('HTTP 302'); - expect(global.fetch).toHaveBeenCalledWith( - URLS.mcpLanding, - expect.objectContaining({ redirect: 'manual' }), - ); - }); - - it('returns down on 400 — only 2xx-3xx counts as up', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('Bad Request', { status: 400 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 400'); - }); - - it('returns down when worker responds 500', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve( - new Response('Internal Server Error', { status: 500 }), - ), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 500'); - }); - - it('returns down when Cloudflare returns 522 (connection timed out)', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('', { status: 522 })), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.Down); - expect(result.error).toContain('HTTP 522'); - }); - - it('returns no-connection on network failure', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => Promise.reject(new Error('fetch failed')), - }), - ); - const result = await checkMcpHealth(); - expect(result.status).toBe(ServiceHealthStatus.NoConnection); - expect(result.error).toBe('fetch failed'); - }); - }); - - // ----------------------------------------------------------------------- - // Skills origins (fetchEndpointHealth – skill-menu.json on both origins) - // ----------------------------------------------------------------------- - describe('checkSkillsOriginHealth', () => { it('returns healthy on a final 200 and follows redirects (GitHub 302s asset URLs even for missing assets)', async () => { const result = await checkSkillsOriginHealth(); @@ -1099,118 +367,22 @@ describe('health-checks', () => { }); }); - // ----------------------------------------------------------------------- - // checkAllExternalServices - // ----------------------------------------------------------------------- - describe('checkAllExternalServices', () => { - it('returns all 10 service keys when everything is healthy', async () => { - const health = await checkAllExternalServices(); - const keys = Object.keys(health); - expect(keys).toEqual( - expect.arrayContaining([ - 'anthropic', - 'posthogOverall', - 'posthogComponents', - 'github', - 'npmOverall', - 'npmComponents', - 'cloudflareOverall', - 'cloudflareComponents', - 'mcp', - 'skillsOrigin', - ]), - ); - expect(keys).toHaveLength(10); - for (const val of Object.values(health)) { - expect(val.status).toBe(ServiceHealthStatus.Healthy); - } - }); - - it('upgrades NoConnection mcp to Down when status page reports an outage', async () => { - const incidentBody = { - ...POSTHOG_INCIDENTIO_HEALTHY, - ongoing_incidents: [ - { - id: 'inc1', - name: 'Major outage', - status: 'identified', - current_worst_impact: 'full_outage', - affected_components: [], - url: 'https://www.posthogstatus.com/incidents/test', - last_update_at: '2026-04-22T00:00:00Z', - last_update_message: 'Investigating', - }, - ], - }; - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.resolve( - new Response(JSON.stringify(incidentBody), { status: 200 }), - ), - [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - - const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Down); - expect(health.mcp.status).toBe(ServiceHealthStatus.Down); - expect(health.mcp.error).toContain('corroborated by status page'); - }); - - it('keeps mcp as NoConnection when posthogstatus.com itself is unreachable (the bug-fix scenario)', async () => { - // User on flaky wifi: every PostHog-owned URL fetch fails at the - // network layer, including posthogstatus.com. Previously - // incidentio.ts returned Degraded for fetch failures, which - // tricked reconciliation into upgrading the gateway probe to Down - // and showing the red "Ongoing service disruptions" screen — the - // exact false positive this PR fixes. - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.posthogIncidentIo]: () => - Promise.reject(new Error('ECONNRESET')), - [URLS.mcpLanding]: () => Promise.reject(new Error('ECONNRESET')), - }), - ); - - const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe( - ServiceHealthStatus.NoConnection, - ); - expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('keeps mcp as NoConnection when status page reports no incident', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => Promise.reject(new Error('ETIMEDOUT')), - }), - ); - + it('returns skills health and only probes the two download origins', async () => { const health = await checkAllExternalServices(); - expect(health.posthogOverall.status).toBe(ServiceHealthStatus.Healthy); - expect(health.mcp.status).toBe(ServiceHealthStatus.NoConnection); - }); - - it('fires all fetch calls in parallel', async () => { - await checkAllExternalServices(); - const calledUrls = (global.fetch as Mock).mock.calls.map((c: unknown[]) => - typeof c[0] === 'string' ? c[0] : (c[0] as URL).toString(), + expect(health).toEqual({ + skillsOrigin: { + status: ServiceHealthStatus.Healthy, + rawIndicator: 'HTTP 200', + }, + }); + const calledUrls = (global.fetch as Mock).mock.calls.map( + (call: unknown[]) => call[0], ); - // PostHog uses a single incident.io endpoint for both overall + components - expect(calledUrls).toHaveLength(10); - expect(calledUrls).toContain(URLS.posthogIncidentIo); - expect(calledUrls).toContain(URLS.mcpLanding); - expect(calledUrls).toContain(URLS.githubSkillMenu); - expect(calledUrls).toContain(URLS.awsSkillMenu); + expect(calledUrls).toEqual([URLS.githubSkillMenu, URLS.awsSkillMenu]); }); }); - // ----------------------------------------------------------------------- - // evaluateWizardReadiness - // ----------------------------------------------------------------------- - describe('evaluateWizardReadiness', () => { it('returns Yes when all services are healthy', async () => { const result = await evaluateWizardReadiness( @@ -1219,92 +391,6 @@ describe('health-checks', () => { expect(result.decision).toBe(WizardReadiness.Yes); }); - it('proceeds without warnings when Anthropic is degraded', async () => { - const body = makeStatuspageStatus({ - pageId: 'tymt9n04zgry', - pageName: 'Claude', - pageUrl: 'https://status.claude.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.anthropicStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - expect(result.health.anthropic.status).toBe(ServiceHealthStatus.Degraded); - }); - - it('proceeds without warnings when MCP is down', async () => { - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.mcpLanding]: () => - Promise.resolve(new Response('Bad Gateway', { status: 502 })), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - expect(result.health.mcp.status).toBe(ServiceHealthStatus.Down); - }); - - it('proceeds without warnings when npm overall is down', async () => { - const body = makeStatuspageStatus({ - pageId: 'wyvgptkd90hm', - pageName: 'npm', - pageUrl: 'https://status.npmjs.org', - indicator: 'critical', - description: 'Major Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.npmStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - expect(result.health.npmOverall.status).toBe(ServiceHealthStatus.Down); - }); - - it('proceeds without warnings when a non-blocking service is degraded', async () => { - const body = makeStatuspageStatus({ - pageId: 'yh6f0r4529hb', - pageName: 'Cloudflare', - pageUrl: 'https://www.cloudflarestatus.com', - indicator: 'minor', - description: 'Minor Service Outage', - }); - (global.fetch as Mock).mockImplementation( - overrideFetch({ - [URLS.cloudflareStatus]: () => - Promise.resolve( - new Response(JSON.stringify(body), { status: 200 }), - ), - }), - ); - const result = await evaluateWizardReadiness( - DEFAULT_WIZARD_READINESS_CONFIG, - ); - expect(result.decision).toBe(WizardReadiness.Yes); - expect(result.reasons).toEqual([]); - }); - it.each([ [true, true, WizardReadiness.Yes], [false, true, WizardReadiness.Yes], diff --git a/src/lib/health-checks/endpoints.ts b/src/lib/health-checks/endpoints.ts index 9450cb8bd..f98a9eb7b 100644 --- a/src/lib/health-checks/endpoints.ts +++ b/src/lib/health-checks/endpoints.ts @@ -2,24 +2,7 @@ import { AWS_SKILLS_BASE_URL, GITHUB_SKILLS_BASE_URL } from '@lib/constants'; import { logToFile } from '@utils/debug'; import { ServiceHealthStatus, type BaseHealthResult } from './types'; -// --------------------------------------------------------------------------- -// Direct endpoint health checks -// -// These ping PostHog-owned services directly (no Statuspage intermediary). -// Result taxonomy: -// - HTTP 2xx-3xx (per `isExpectedStatus`) → Healthy -// - HTTP 4xx / 5xx → Down (confirmed) -// - Network error / DNS / timeout (after retries) → NoConnection -// NoConnection means we don't know whose fault it is; readiness reconciles -// against the status page before deciding how to surface it to the user. -// -// MCP – Cloudflare Worker -// Source: posthog/services/mcp/src/index.ts -// GET / → 302 to posthog.com docs. The redirect proves the worker is up. -// -// Skills download – context-mill releases -// GET /skill-menu.json on both origins; see checkSkillsOriginHealth. -// --------------------------------------------------------------------------- +// Direct gateway and skill-origin checks distinguish HTTP failures from connection failures. function noConnectionResult(error: string, attempts: number): BaseHealthResult { return { @@ -136,15 +119,6 @@ export const checkLlmGatewayHealth = ( ): Promise => fetchEndpointHealth(new URL('/readyz', gatewayUrl).href); -export const checkMcpHealth = (): Promise => - fetchEndpointHealth( - 'https://mcp.posthog.com/', - 5000, - // 2xx-3xx counts as up (redirect to docs) - (s) => s >= 200 && s < 400, - 'manual', - ); - /** * Skills are published to two origins under the same filenames and * `fetchWithRetry` fails over between them, so the run is only blocked when diff --git a/src/lib/health-checks/incidentio.ts b/src/lib/health-checks/incidentio.ts deleted file mode 100644 index 574a4f0f9..000000000 --- a/src/lib/health-checks/incidentio.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { - ServiceHealthStatus, - type BaseHealthResult, - type ComponentHealthResult, - type ComponentStatus, -} from './types'; - -interface IncidentIoAffectedComponent { - id: string; - name: string; - group_name?: string; - current_status: string; -} - -interface IncidentIoIncident { - id: string; - name: string; - status: string; - current_worst_impact: string; - affected_components: IncidentIoAffectedComponent[]; -} - -interface IncidentIoSummary { - ongoing_incidents: IncidentIoIncident[]; - in_progress_maintenances: unknown[]; -} - -function mapIncidentImpact(impact: string): ServiceHealthStatus { - switch (impact) { - case 'full_outage': - return ServiceHealthStatus.Down; - case 'partial_outage': - case 'degraded_performance': - return ServiceHealthStatus.Degraded; - default: - return ServiceHealthStatus.Degraded; - } -} - -function mapComponentStatus(status: string): ServiceHealthStatus { - switch (status) { - case 'operational': - return ServiceHealthStatus.Healthy; - case 'full_outage': - return ServiceHealthStatus.Down; - case 'partial_outage': - case 'degraded_performance': - return ServiceHealthStatus.Degraded; - default: - return ServiceHealthStatus.Degraded; - } -} - -/** - * Build an error result for fetch failures. The kind matters for - * downstream reconciliation: - * - * - 'http' (incident.io returned a bad status code) → `Down`. We - * reached the status page but it told us something is wrong on - * its side. We have a definitive response. - * - 'network' (timeout, DNS failure, TCP/TLS failure) → `NoConnection`. - * We never reached the status page. Treating this as `Degraded` - * (the previous behavior) silently flipped the reconciliation in - * `readiness.ts` from "soft" to "confirmed outage" whenever the - * user's own network was flaky — exactly the false positive this - * module is meant to help diagnose. - */ -function errResult(error: string, kind: 'http' | 'network'): BaseHealthResult { - return { - status: - kind === 'http' - ? ServiceHealthStatus.Down - : ServiceHealthStatus.NoConnection, - error, - }; -} - -const POSTHOG_STATUS_URL = 'https://www.posthogstatus.com/api/v1/summary'; - -async function fetchPosthogStatus( - timeoutMs = 5000, -): Promise<{ overall: BaseHealthResult; components: ComponentHealthResult }> { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(POSTHOG_STATUS_URL, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) { - const err = errResult(`HTTP ${res.status}`, 'http'); - return { overall: err, components: err }; - } - - const data = (await res.json()) as IncidentIoSummary; - const incidents = data.ongoing_incidents ?? []; - - if (incidents.length === 0) { - return { - overall: { status: ServiceHealthStatus.Healthy }, - components: { status: ServiceHealthStatus.Healthy }, - }; - } - - let worstOverall = ServiceHealthStatus.Degraded; - const affected: ComponentStatus[] = []; - - for (const incident of incidents) { - const impact = mapIncidentImpact(incident.current_worst_impact); - if (impact === ServiceHealthStatus.Down) { - worstOverall = ServiceHealthStatus.Down; - } - - for (const comp of incident.affected_components ?? []) { - const compStatus = mapComponentStatus(comp.current_status); - if (compStatus !== ServiceHealthStatus.Healthy) { - affected.push({ - name: comp.group_name - ? `${comp.group_name} — ${comp.name}` - : comp.name, - status: compStatus, - rawStatus: comp.current_status, - }); - } - } - } - - return { - overall: { status: worstOverall }, - components: { - status: - affected.length > 0 ? ServiceHealthStatus.Degraded : worstOverall, - degradedOrDownComponents: affected.length > 0 ? affected : undefined, - }, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') { - const err = errResult('Request timed out', 'network'); - return { overall: err, components: err }; - } - const err = errResult( - e instanceof Error ? e.message : 'Unknown error', - 'network', - ); - return { overall: err, components: err }; - } -} - -let _cache: Promise<{ - overall: BaseHealthResult; - components: ComponentHealthResult; -}> | null = null; - -function getPosthogHealth() { - if (!_cache) _cache = fetchPosthogStatus(); - return _cache; -} - -export function resetPosthogHealthCache(): void { - _cache = null; -} - -export const checkPosthogOverallHealth = async (): Promise => - (await getPosthogHealth()).overall; - -export const checkPosthogComponentHealth = - async (): Promise => - (await getPosthogHealth()).components; diff --git a/src/lib/health-checks/index.ts b/src/lib/health-checks/index.ts index b4c3f6041..a6af0fb7c 100644 --- a/src/lib/health-checks/index.ts +++ b/src/lib/health-checks/index.ts @@ -1,28 +1,11 @@ export { ServiceHealthStatus, type BaseHealthResult, - type ComponentStatus, - type ComponentHealthResult, type AllServicesHealth, type HealthCheckKey, } from './types'; -export { - checkAnthropicHealth, - checkGithubHealth, - checkNpmOverallHealth, - checkNpmComponentHealth, - checkCloudflareOverallHealth, - checkCloudflareComponentHealth, -} from './statuspage'; - -export { - checkPosthogOverallHealth, - checkPosthogComponentHealth, - resetPosthogHealthCache, -} from './incidentio'; - -export { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; +export { checkLlmGatewayHealth, checkSkillsOriginHealth } from './endpoints'; export { type WizardReadinessConfig, diff --git a/src/lib/health-checks/readiness.ts b/src/lib/health-checks/readiness.ts index fc6e12aa8..0eba5a3b4 100644 --- a/src/lib/health-checks/readiness.ts +++ b/src/lib/health-checks/readiness.ts @@ -4,19 +4,7 @@ import { type BaseHealthResult, type HealthCheckKey, } from './types'; -import { - checkAnthropicHealth, - checkGithubHealth, - checkNpmOverallHealth, - checkNpmComponentHealth, - checkCloudflareOverallHealth, - checkCloudflareComponentHealth, -} from './statuspage'; -import { - checkPosthogOverallHealth, - checkPosthogComponentHealth, -} from './incidentio'; -import { checkMcpHealth, checkSkillsOriginHealth } from './endpoints'; +import { checkSkillsOriginHealth } from './endpoints'; import { logToFile } from '@utils/debug'; // --------------------------------------------------------------------------- @@ -24,15 +12,6 @@ import { logToFile } from '@utils/debug'; // --------------------------------------------------------------------------- export const SERVICE_LABELS: Record = { - anthropic: 'Anthropic', - posthogOverall: 'PostHog', - posthogComponents: 'PostHog (components)', - github: 'GitHub', - npmOverall: 'npm', - npmComponents: 'npm (components)', - cloudflareOverall: 'Cloudflare', - cloudflareComponents: 'Cloudflare (components)', - mcp: 'MCP', skillsOrigin: 'Skills download', }; @@ -59,101 +38,7 @@ export const SIGNUP_WIZARD_READINESS_CONFIG = DEFAULT_WIZARD_READINESS_CONFIG; // --------------------------------------------------------------------------- export async function checkAllExternalServices(): Promise { - const [ - anthropic, - posthogOverall, - posthogComponents, - github, - npmOverall, - npmComponents, - cloudflareOverall, - cloudflareComponents, - mcp, - skillsOrigin, - ] = await Promise.all([ - checkAnthropicHealth(), - checkPosthogOverallHealth(), - checkPosthogComponentHealth(), - checkGithubHealth(), - checkNpmOverallHealth(), - checkNpmComponentHealth(), - checkCloudflareOverallHealth(), - checkCloudflareComponentHealth(), - checkMcpHealth(), - checkSkillsOriginHealth(), - ]); - - const health: AllServicesHealth = { - anthropic, - posthogOverall, - posthogComponents, - github, - npmOverall, - npmComponents, - cloudflareOverall, - cloudflareComponents, - mcp, - skillsOrigin, - }; - return reconcilePosthogReachability(health); -} - -/** - * When a PostHog-owned endpoint probe returns `NoConnection`, decide - * whether it's a real outage or a likely-local issue by checking the - * official status page (`posthogstatus.com`): - * - * - Status page says PostHog is `Down` / `Degraded` → upgrade - * mcp to `Down`. The status page corroborates. - * - Status page is `Healthy` → keep `NoConnection`. The status page - * contradicts; this is probably the user's network. - * - Status page is also `NoConnection` → keep `NoConnection`. User - * can't reach two independent PostHog properties; almost - * certainly their network. (This case relies on incidentio.ts - * correctly emitting `NoConnection` for fetch failures rather - * than the previous `Degraded`, which used to silently flip the - * reconciliation into a false positive.) - * - * Why `Degraded` corroborates: a `Degraded` reading here only fires - * when incident.io's API parsed successfully and reported a real - * `partial_outage` or `degraded_performance` for some component. That's - * PostHog acknowledging an issue, even if narrower than a full outage. - * If our MCP probe is also failing, those two signals together - * justify pointing at PostHog rather than the user. - * - * A narrower variant — only corroborate when the affected component is - * MCP-related (US/EU Cloud, app) — would be more precise. We - * have the data in `posthogComponents` but don't use it here. If the - * analytics show false positives concentrated in this case, it's a - * cheap follow-up. - * - * Mutates a copy of `health` and returns it. - */ -export function reconcilePosthogReachability( - health: AllServicesHealth, -): AllServicesHealth { - const posthogStatus = health.posthogOverall.status; - const corroboratesOutage = - posthogStatus === ServiceHealthStatus.Down || - posthogStatus === ServiceHealthStatus.Degraded; - - if (!corroboratesOutage) return health; - - const upgrade = (r: BaseHealthResult): BaseHealthResult => - r.status === ServiceHealthStatus.NoConnection - ? { - ...r, - status: ServiceHealthStatus.Down, - error: r.error - ? `${r.error} (corroborated by status page)` - : 'corroborated by status page', - } - : r; - - return { - ...health, - mcp: upgrade(health.mcp), - }; + return { skillsOrigin: await checkSkillsOriginHealth() }; } // --------------------------------------------------------------------------- @@ -230,27 +115,12 @@ export async function evaluateWizardReadiness( // Blocking service detection // --------------------------------------------------------------------------- -/** Keys that are component-level detail, not top-level services. */ -const COMPONENT_KEYS: HealthCheckKey[] = [ - 'posthogComponents', - 'npmComponents', - 'cloudflareComponents', -]; - -/** - * Get the keys of services that would block a wizard run per the given config. - * - * `NoConnection` blocks the same services as `Down` — the wizard genuinely - * can't continue if it can't reach the gateway. The screen shows softer - * framing in that case (HealthCheckScreen) so we don't falsely accuse - * PostHog of an outage when the user's network is the likely cause. - */ +// Report only dependencies that prevent this run from starting. export function getBlockingServiceKeys( health: AllServicesHealth, config: WizardReadinessConfig = DEFAULT_WIZARD_READINESS_CONFIG, ): HealthCheckKey[] { return (Object.keys(health) as HealthCheckKey[]).filter((key) => { - if (COMPONENT_KEYS.includes(key)) return false; const result = health[key]; if ( config.downBlocksRun.includes(key) && @@ -275,16 +145,5 @@ function allUnknown(error: string): AllServicesHealth { status: ServiceHealthStatus.Degraded, error, }; - return { - anthropic: base, - posthogOverall: base, - posthogComponents: { ...base }, - github: base, - npmOverall: base, - npmComponents: { ...base }, - cloudflareOverall: base, - cloudflareComponents: { ...base }, - mcp: base, - skillsOrigin: base, - }; + return { skillsOrigin: base }; } diff --git a/src/lib/health-checks/statuspage.ts b/src/lib/health-checks/statuspage.ts deleted file mode 100644 index bfcdd6dae..000000000 --- a/src/lib/health-checks/statuspage.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { - ServiceHealthStatus, - type BaseHealthResult, - type ComponentHealthResult, -} from './types'; - -// --------------------------------------------------------------------------- -// Statuspage.io v2 API helpers -// https://metastatuspage.com/api -// -// status.json – page-level rollup; indicator is one of: none | minor | major | critical -// summary.json – same rollup + component list; component status is one of: -// operational | degraded_performance | partial_outage | major_outage | under_maintenance -// https://support.atlassian.com/statuspage/docs/show-service-status-with-components -// --------------------------------------------------------------------------- - -interface StatuspageStatusResponse { - status?: { indicator?: string; description?: string }; -} - -interface StatuspageSummaryResponse extends StatuspageStatusResponse { - components?: { id: string; name: string; status: string }[]; -} - -function mapIndicator(v: string | null | undefined): ServiceHealthStatus { - switch (v) { - case 'none': - return ServiceHealthStatus.Healthy; - case 'minor': - return ServiceHealthStatus.Degraded; - case 'major': - case 'critical': - return ServiceHealthStatus.Down; - default: - return ServiceHealthStatus.Degraded; - } -} - -function mapComponentRaw(v: string | null | undefined): ServiceHealthStatus { - switch (v) { - case 'operational': - return ServiceHealthStatus.Healthy; - case 'degraded_performance': - case 'under_maintenance': - return ServiceHealthStatus.Degraded; - case 'partial_outage': - case 'major_outage': - return ServiceHealthStatus.Down; - default: - return ServiceHealthStatus.Degraded; - } -} - -function errResult(error: string): BaseHealthResult { - return { status: ServiceHealthStatus.Degraded, error }; -} - -async function fetchStatuspageIndicator( - url: string, - timeoutMs = 5000, -): Promise { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(url, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) return errResult(`HTTP ${res.status}`); - - const data = (await res.json()) as StatuspageStatusResponse; - const indicator = data.status?.indicator ?? null; - return { - status: mapIndicator(indicator), - rawIndicator: indicator ?? undefined, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') - return errResult('Request timed out'); - return errResult(e instanceof Error ? e.message : 'Unknown error'); - } -} - -async function fetchStatuspageSummary( - url: string, - timeoutMs = 5000, -): Promise { - try { - const controller = new AbortController(); - const tid = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(url, { signal: controller.signal }); - clearTimeout(tid); - - if (!res.ok) return errResult(`HTTP ${res.status}`); - - const data = (await res.json()) as StatuspageSummaryResponse; - const indicator = data.status?.indicator ?? null; - const overall = mapIndicator(indicator); - - const affected = (data.components ?? []) - .map((c) => ({ - name: c.name, - status: mapComponentRaw(c.status), - rawStatus: c.status, - })) - .filter((c) => c.status !== ServiceHealthStatus.Healthy); - - return { - status: affected.length > 0 ? ServiceHealthStatus.Degraded : overall, - rawIndicator: indicator ?? undefined, - degradedOrDownComponents: affected.length > 0 ? affected : undefined, - }; - } catch (e) { - if (e instanceof Error && e.name === 'AbortError') - return errResult('Request timed out'); - return errResult(e instanceof Error ? e.message : 'Unknown error'); - } -} - -// --------------------------------------------------------------------------- -// Individual statuspage-backed checks -// --------------------------------------------------------------------------- - -export const checkAnthropicHealth = (): Promise => - fetchStatuspageIndicator('https://status.claude.com/api/v2/status.json'); - -export const checkGithubHealth = (): Promise => - fetchStatuspageIndicator('https://www.githubstatus.com/api/v2/status.json'); - -export const checkNpmOverallHealth = (): Promise => - fetchStatuspageIndicator('https://status.npmjs.org/api/v2/status.json'); - -export const checkNpmComponentHealth = (): Promise => - fetchStatuspageSummary('https://status.npmjs.org/api/v2/summary.json'); - -export const checkCloudflareOverallHealth = (): Promise => - fetchStatuspageIndicator( - 'https://www.cloudflarestatus.com/api/v2/status.json', - ); - -export const checkCloudflareComponentHealth = - (): Promise => - fetchStatuspageSummary( - 'https://www.cloudflarestatus.com/api/v2/summary.json', - ); diff --git a/src/lib/health-checks/testme.md b/src/lib/health-checks/testme.md index 9cf6a7202..ba5b04187 100644 --- a/src/lib/health-checks/testme.md +++ b/src/lib/health-checks/testme.md @@ -1,61 +1,17 @@ -# Health Checks — Testing Guide +# Health checks -## Running unit tests +Run the existing health and gateway tests with mocked HTTP requests: ```bash -# From the wizard/ root — runs only health-check tests (fast, no build step) -npx jest src/lib/health-checks/__tests__/health-checks.test.ts - -# Watch mode -npx jest src/lib/health-checks/__tests__/health-checks.test.ts --watch - -# With coverage -npx jest src/lib/health-checks/__tests__/health-checks.test.ts --coverage -``` - -## Running health checks live - -To hit all 10 endpoints for real and see the full readiness result: - -```bash -# From the wizard/ root -npx tsx -e "import { evaluateWizardReadiness } from './src/lib/health-checks/index'; evaluateWizardReadiness().then(r => console.log(JSON.stringify(r, null, 2)))" +pnpm exec vitest run src/lib/health-checks/__tests__/health-checks.test.ts src/lib/__tests__/gateway-session.test.ts ``` -## How the tests work - -All external HTTP calls are mocked via a global `fetch` override in -`beforeEach`. No network access is required. Mock data is modelled on real -responses captured from production endpoints on 2026-03-05. - -## Endpoints tested - -| Service | URL | Healthy response | -| ----------------------- | ------------------------------------------------------ | ------------------------------------- | -| Anthropic | `https://status.claude.com/api/v2/status.json` | `{"status":{"indicator":"none",...}}` | -| PostHog | `https://www.posthogstatus.com/api/v2/status.json` | Same shape | -| PostHog (components) | `https://www.posthogstatus.com/api/v2/summary.json` | Adds `components[]` array | -| GitHub | `https://www.githubstatus.com/api/v2/status.json` | Same shape | -| npm | `https://status.npmjs.org/api/v2/status.json` | Same shape | -| npm (components) | `https://status.npmjs.org/api/v2/summary.json` | Adds `components[]` array | -| Cloudflare | `https://www.cloudflarestatus.com/api/v2/status.json` | Same shape | -| Cloudflare (components) | `https://www.cloudflarestatus.com/api/v2/summary.json` | Adds `components[]` array | -| MCP | `https://mcp.posthog.com/` | HTML landing page (HTTP 200) | - -### Statuspage.io API v2 reference - -- Docs: -- `status.json` — page-level rollup; `indicator` is one of: `none`, `minor`, - `major`, `critical` -- `summary.json` — same rollup + `components[]`; component `status` is one of: - `operational`, `degraded_performance`, `partial_outage`, `major_outage`, - `under_maintenance` -- Component docs: - - -### MCP +The checks cover gateway readiness, endpoint retries, and skills downloads from +GitHub Releases and AWS. Either working skills origin is sufficient; both must +fail before startup is blocked. Third-party status pages are not queried. -- Source: `posthog/services/mcp/src/index.ts` -- `GET /` → HTML landing page (200) -- No dedicated `/health` endpoint; 200 on `/` confirms the Cloudflare Worker is - running. +| Check | Endpoint | Healthy response | +| ------------- | ------------------------------------------ | ------------------------ | +| Gateway | `/readyz` on the minted gateway URL | HTTP 200 | +| GitHub skills | `/skill-menu.json` | HTTP 200 after redirects | +| AWS skills | `/skill-menu.json` | HTTP 200 after redirects | diff --git a/src/lib/health-checks/types.ts b/src/lib/health-checks/types.ts index f838bbf96..2b59f9d14 100644 --- a/src/lib/health-checks/types.ts +++ b/src/lib/health-checks/types.ts @@ -2,13 +2,7 @@ export enum ServiceHealthStatus { Healthy = 'healthy', Degraded = 'degraded', Down = 'down', - /** - * Probe failed (network error, timeout, DNS failure) AND we have no - * corroborating status-page incident. The service may be fine — the - * user's network is the likely culprit. Distinct from `Down`, which - * is confirmed (HTTP 5xx or status-page incident). User-facing label: - * "No connection". - */ + // A network failure does not establish a service outage. NoConnection = 'no-connection', } @@ -18,26 +12,7 @@ export interface BaseHealthResult { error?: string; } -export interface ComponentStatus { - name: string; - status: ServiceHealthStatus; - rawStatus: string; -} - -export interface ComponentHealthResult extends BaseHealthResult { - degradedOrDownComponents?: ComponentStatus[]; -} - export interface AllServicesHealth { - anthropic: BaseHealthResult; - posthogOverall: BaseHealthResult; - posthogComponents: ComponentHealthResult; - github: BaseHealthResult; - npmOverall: BaseHealthResult; - npmComponents: ComponentHealthResult; - cloudflareOverall: BaseHealthResult; - cloudflareComponents: ComponentHealthResult; - mcp: BaseHealthResult; skillsOrigin: BaseHealthResult; } diff --git a/src/ui/tui/components/ServiceHealthList.tsx b/src/ui/tui/components/ServiceHealthList.tsx index c38e9db4c..99ac8c7da 100644 --- a/src/ui/tui/components/ServiceHealthList.tsx +++ b/src/ui/tui/components/ServiceHealthList.tsx @@ -1,34 +1,14 @@ -/** - * ServiceHealthList — Shared component for displaying service health status. - * - * Used by HealthCheckScreen (blocking services only) and HealthWarningsTab (all services). - */ +// Display the dependencies reported by the health check. import { Box, Text } from 'ink'; import { ServiceHealthStatus, type AllServicesHealth, - type ComponentHealthResult, - type ComponentStatus, type HealthCheckKey, } from '@lib/health-checks/types'; import { SERVICE_LABELS } from '@lib/health-checks/readiness'; import { Icons } from '@ui/tui/styles'; -/** Keys that are component-level detail — shown inline under their parent. */ -const COMPONENT_KEYS: HealthCheckKey[] = [ - 'posthogComponents', - 'npmComponents', - 'cloudflareComponents', -]; - -/** Map component key → its parent "overall" key */ -const COMPONENT_PARENT: Partial> = { - posthogComponents: 'posthogOverall', - npmComponents: 'npmOverall', - cloudflareComponents: 'cloudflareOverall', -}; - function statusIcon(status: ServiceHealthStatus): { icon: string; color: string; @@ -58,9 +38,7 @@ export const ServiceHealthList = ({ filterKeys, showHealthy = true, }: ServiceHealthListProps) => { - const topLevelKeys = (Object.keys(health) as HealthCheckKey[]).filter( - (k) => !COMPONENT_KEYS.includes(k), - ); + const topLevelKeys = Object.keys(health) as HealthCheckKey[]; const keysToShow = filterKeys ? topLevelKeys.filter((k) => filterKeys.includes(k)) @@ -77,16 +55,6 @@ export const ServiceHealthList = ({ const { icon, color } = statusIcon(result.status); const label = SERVICE_LABELS[key]; - // Find component-level details if this is a parent key - const componentKey = ( - Object.entries(COMPONENT_PARENT) as [HealthCheckKey, HealthCheckKey][] - ).find(([, parent]) => parent === key)?.[0]; - const componentResult = componentKey - ? (health[componentKey] as ComponentHealthResult) - : undefined; - const affectedComponents: ComponentStatus[] = - componentResult?.degradedOrDownComponents ?? []; - return ( @@ -95,21 +63,6 @@ export const ServiceHealthList = ({ {label} - {affectedComponents.length > 0 && ( - - {affectedComponents.slice(0, 5).map((c) => { - const ci = statusIcon(c.status); - return ( - - {ci.icon} {c.name} - - ); - })} - {affectedComponents.length > 5 && ( - +{affectedComponents.length - 5} more - )} - - )} ); })} diff --git a/src/ui/tui/playground/demos/HealthCheckDemo.tsx b/src/ui/tui/playground/demos/HealthCheckDemo.tsx index 501f6972c..a4e12b936 100644 --- a/src/ui/tui/playground/demos/HealthCheckDemo.tsx +++ b/src/ui/tui/playground/demos/HealthCheckDemo.tsx @@ -1,16 +1,4 @@ -/** - * HealthCheckDemo — Playground demo for health check UI components. - * - * Cycles through three states (2s checking spinner → 5s confirmed-outage - * red modal → 5s no-connection yellow modal, then loops): - * 1. Checking (spinner) - * 2. Confirmed outage (status page corroborates → red framing) - * 3. No connection only (no status-page incident → yellow "couldn't - * reach PostHog" framing) - * - * Renders components directly (not HealthCheckScreen) to avoid useInput - * conflicts with TabContainer's key handling. - */ +// Preview the skills health-check states without handling input. import { useEffect, useState } from 'react'; import { Box, Text } from 'ink'; @@ -21,47 +9,12 @@ import { getBlockingServiceKeys } from '@lib/health-checks/readiness'; import { ServiceHealthStatus } from '@lib/health-checks/types'; import type { AllServicesHealth } from '@lib/health-checks/types'; -const HEALTHY = { status: ServiceHealthStatus.Healthy } as const; - const MOCK_CONFIRMED_OUTAGE: AllServicesHealth = { - anthropic: { status: ServiceHealthStatus.Down, rawIndicator: 'major' }, - posthogOverall: HEALTHY, - posthogComponents: { status: ServiceHealthStatus.Healthy }, - github: HEALTHY, - npmOverall: { - status: ServiceHealthStatus.Degraded, - rawIndicator: 'minor', - }, - npmComponents: { - status: ServiceHealthStatus.Degraded, - degradedOrDownComponents: [ - { - name: 'Registry API', - status: ServiceHealthStatus.Degraded, - rawStatus: 'degraded_performance', - }, - ], - }, - cloudflareOverall: HEALTHY, - cloudflareComponents: { status: ServiceHealthStatus.Healthy }, - mcp: HEALTHY, - skillsOrigin: HEALTHY, + skillsOrigin: { status: ServiceHealthStatus.Down }, }; const MOCK_NO_CONNECTION: AllServicesHealth = { - anthropic: HEALTHY, - posthogOverall: HEALTHY, - posthogComponents: { status: ServiceHealthStatus.Healthy }, - github: HEALTHY, - npmOverall: HEALTHY, - npmComponents: { status: ServiceHealthStatus.Healthy }, - cloudflareOverall: HEALTHY, - cloudflareComponents: { status: ServiceHealthStatus.Healthy }, - mcp: { - status: ServiceHealthStatus.NoConnection, - error: 'fetch failed', - }, - skillsOrigin: HEALTHY, + skillsOrigin: { status: ServiceHealthStatus.NoConnection }, }; type Phase = 'checking' | 'confirmed' | 'no-connection'; @@ -103,15 +56,13 @@ export const HealthCheckDemo = () => { borderColor={isNoConnection ? 'yellow' : 'red'} title={ isNoConnection - ? "Couldn't reach PostHog" - : `${Icons.warning} Ongoing service disruptions` + ? "Couldn't reach skill downloads" + : `${Icons.warning} Skill downloads unavailable` } width={72} footer={ - - Continue [Enter] / Exit [Esc] (disabled in playground) - + Exit [Esc] (disabled in playground) } > @@ -136,8 +87,8 @@ export const HealthCheckDemo = () => { {isNoConnection - ? "We couldn't reach these services. PostHog's status page shows no incidents, likely a network issue (VPN, firewall, captive portal, or Wi-Fi)." - : 'The wizard may not work reliably while services are affected.'} + ? "We couldn't reach either skills source. Check your connection and try again." + : 'Neither GitHub Releases nor the AWS mirror is available.'} ); diff --git a/src/ui/tui/store.ts b/src/ui/tui/store.ts index 12cf174ab..165dacf33 100644 --- a/src/ui/tui/store.ts +++ b/src/ui/tui/store.ts @@ -41,7 +41,6 @@ import { getBlockingServiceKeys, type WizardReadinessResult, } from '@lib/health-checks/readiness'; -import { ServiceHealthStatus } from '@lib/health-checks/types'; import { WizardRouter, type ScreenName, @@ -128,49 +127,17 @@ interface GateEntry { */ const MAX_STATUS_MESSAGES = EXPANDED_COUNT; -/** - * Fired once per blocked readiness result, so we can quantify how often - * the wizard refuses to start and — crucially — split that between - * confirmed PostHog outages and probe-level reachability failures that - * are most likely the user's network. Helps us decide whether the - * health-check UX is over-firing. - */ +// Capture blocked skill downloads once per readiness result. function captureHealthCheckBlocked(result: WizardReadinessResult): void { try { const health = result.health; const blockingKeys = getBlockingServiceKeys(health); - const blockingStatuses = blockingKeys.map((k) => health[k]?.status); - - const allNoConnection = - blockingStatuses.length > 0 && - blockingStatuses.every((s) => s === ServiceHealthStatus.NoConnection); - const onlySkillsOrigin = - blockingKeys.length === 1 && blockingKeys[0] === 'skillsOrigin'; - - const decision = onlySkillsOrigin - ? 'skills-origin-down' - : allNoConnection - ? 'no-connection' - : 'confirmed-outage'; - - const posthogStatus = health.posthogOverall?.status; - const retriesUsed = Math.max( - 0, - ...(['mcp', 'skillsOrigin'] as const).map((k) => { - const ind = health[k]?.rawIndicator ?? ''; - const m = ind.match(/attempts=(\d+)/); - return m ? Number(m[1]) - 1 : 0; - }), - ); + const attempts = health.skillsOrigin.rawIndicator?.match(/attempts=(\d+)/); + const retriesUsed = Math.max(0, attempts ? Number(attempts[1]) - 1 : 0); analytics.wizardCapture('health check blocked', { - decision, + decision: 'skills-origin-down', blocking_keys: blockingKeys, - posthog_status_reachable: - posthogStatus !== ServiceHealthStatus.NoConnection, - posthog_status_reports_incident: - posthogStatus === ServiceHealthStatus.Down || - posthogStatus === ServiceHealthStatus.Degraded, retries_used: retriesUsed, }); } catch (err) { diff --git a/src/utils/anthropic-status.ts b/src/utils/anthropic-status.ts deleted file mode 100644 index 5f05fbfa5..000000000 --- a/src/utils/anthropic-status.ts +++ /dev/null @@ -1,73 +0,0 @@ -const CLAUDE_STATUS_URL = 'https://status.claude.com/api/v2/status.json'; - -type StatusIndicator = 'none' | 'minor' | 'major' | 'critical'; - -interface ClaudeStatusResponse { - page: { - id: string; - name: string; - url: string; - time_zone: string; - updated_at: string; - }; - status: { - indicator: StatusIndicator; - description: string; - }; -} - -export type StatusCheckResult = - | { status: 'operational' } - | { status: 'degraded'; description: string } - | { status: 'down'; description: string } - | { status: 'unknown'; error: string }; - -/** - * Check the Anthropic/Claude status page for service health. - * Pure function — no UI calls. - */ -export async function checkAnthropicStatus(): Promise { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 5000); - - const response = await fetch(CLAUDE_STATUS_URL, { - signal: controller.signal, - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - return { - status: 'unknown', - error: `Status page returned ${response.status}`, - }; - } - - const data = (await response.json()) as ClaudeStatusResponse; - const indicator = data.status.indicator; - const rawDesc = data.status.description; - const description = - rawDesc.charAt(0).toUpperCase() + rawDesc.slice(1).toLowerCase(); - - switch (indicator) { - case 'none': - return { status: 'operational' }; - case 'minor': - return { status: 'degraded', description }; - case 'major': - case 'critical': - return { status: 'down', description }; - default: - return { status: 'unknown', error: `Unknown indicator: ${indicator}` }; - } - } catch (error) { - if (error instanceof Error && error.name === 'AbortError') { - return { status: 'unknown', error: 'Request timed out' }; - } - return { - status: 'unknown', - error: error instanceof Error ? error.message : 'Unknown error', - }; - } -}