From 3b10c3dd3dd855ae0a7825f0d0d1c6c764e2df19 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Thu, 19 Nov 2026 23:58:49 +0300 Subject: [PATCH] fix(strata): retry session health checks before unloading --- ...1-19-retry-strata-session-health-checks.md | 14 ++++ docs/decisions/INDEX.md | 1 + package.json | 2 +- src/core/create.ts | 1 + src/core/strata.ts | 3 + src/runtime/strata/strata-http.test.ts | 67 ++++++++++++++++ src/runtime/strata/strata-http.ts | 43 ++++++---- src/runtime/strata/strata-runtime.test.ts | 78 +++++++++++++++++-- src/runtime/strata/strata-runtime.ts | 70 ++++++++++++----- src/version.ts | 2 +- test/contract/strata-http.test.ts | 6 +- 11 files changed, 244 insertions(+), 43 deletions(-) create mode 100644 docs/decisions/2026-11-19-retry-strata-session-health-checks.md create mode 100644 src/runtime/strata/strata-http.test.ts diff --git a/docs/decisions/2026-11-19-retry-strata-session-health-checks.md b/docs/decisions/2026-11-19-retry-strata-session-health-checks.md new file mode 100644 index 00000000..49781ed5 --- /dev/null +++ b/docs/decisions/2026-11-19-retry-strata-session-health-checks.md @@ -0,0 +1,14 @@ +--- +date: 2026-11-19 +title: "Retry Strata session health checks before unloading" +--- + +# Retry Strata session health checks before unloading + +The Strata runtime verifies its owned Control instance and its authenticated HTTP telemetry once every five seconds after loading. A slow or failed check used to close the local gateway immediately, interrupting chat streams even when the engine could recover. + +Keep the loaded session and gateway for the first two failed checks. Close the gateway and stop the engine after three consecutive failures. A check that verifies both Control and telemetry resets the count. Initial loading still requires both checks to pass before the gateway opens. Explicit unload and shutdown still cancel the monitor immediately. GPU ownership still depends on confirmed process exit. + +Log the attempt and cause of every failure in the core log, along with any recovery. Report the final cause in the session death event. Diagnostics use status codes and summaries so credentials and response bodies stay out of the log. + +Owner: team. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 9cd64c2b..77a55ce8 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -290,3 +290,4 @@ add a new one that says which record it supersedes. - [2026-11-19 — Reuse Strata engine profiles from the model catalog](2026-11-19-reuse-strata-profiles-from-the-model-catalog.md) - [Select Strata releases compatible with the running core](2026-11-19-skip-incompatible-strata-releases.md) - [Start Windows DeepSeek from existing Strata packs](2026-11-19-start-windows-deepseek-from-existing-strata-packs.md) +- [Retry Strata session health checks before unloading](2026-11-19-retry-strata-session-health-checks.md) diff --git a/package.json b/package.json index c5e160af..df8f9d90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@atomic-chat/core", - "version": "0.24.7", + "version": "0.24.8", "repository": { "type": "git", "url": "git+https://github.com/AtomicBot-ai/atomic-chat-core.git" diff --git a/src/core/create.ts b/src/core/create.ts index d947f3aa..4887dc47 100644 --- a/src/core/create.ts +++ b/src/core/create.ts @@ -880,6 +880,7 @@ export async function createAtomicCore( trustedHosts: managedTrustedHosts, claimGpu: gpuResidency.hook('strata'), emit: (name, payload) => emitter.emit(name, payload), + log, }) runtimes.set('strata', strata.runtime) registries.set('strata', new ModelRegistry(layout, 'strata')) diff --git a/src/core/strata.ts b/src/core/strata.ts index 2b104c05..44ac1787 100644 --- a/src/core/strata.ts +++ b/src/core/strata.ts @@ -45,6 +45,7 @@ import type { StrataHostFacts, StrataLaunchIntent } from '../runtime/strata/inde import type { StrataBackend, StrataCheckResult, StrataModelDescriptor } from '../contracts/index.js' import type { StrataModelEntry } from '../models/index.js' import { CORE_VERSION } from '../version.js' +import type { CoreLogger } from './types.js' interface Options { layout: DataLayout @@ -58,6 +59,7 @@ interface Options { trustedHosts: string[] claimGpu: GpuClaimHook emit: EmitFn + log?: CoreLogger } const insufficientStrataVram = (requiredBytes: number) => @@ -312,6 +314,7 @@ export async function wireStrata(options: Options) { const runtime: StrataRuntime = new StrataRuntime({ claimGpu: options.claimGpu, emit: options.emit, + ...(options.log ? { log: options.log } : {}), guardLoad: (id, work) => maintenance.withLoad(id, work), stopPreparations: () => preparations.shutdown(), retained: () => journal.list().filter((entry) => entry.core.instanceId !== instanceId), diff --git a/src/runtime/strata/strata-http.test.ts b/src/runtime/strata/strata-http.test.ts new file mode 100644 index 00000000..4effb026 --- /dev/null +++ b/src/runtime/strata/strata-http.test.ts @@ -0,0 +1,67 @@ +import { expect, it, vi } from 'vitest' +import { strataHttp } from './strata-http.js' + +const status = { instanceId: 'expected-instance' } +const telemetry = { instance: { id: status.instanceId }, model: { id: 'model', loaded: true } } + +function probe(fetcher: typeof fetch) { + return strataHttp( + 12345, + 'key', + 'model', + { chat: true, tools: true, structured_output: true, vision: false, reasoning: true }, + 8192, + [], + fetcher + ) +} + +it.each([ + [() => Response.json(telemetry), undefined], + [() => new Response(null, { status: 503 }), 'Strata telemetry returned HTTP 503.'], + [() => new Response('not json'), 'Could not read the Strata telemetry JSON.'], + [() => Response.json(null), 'Strata telemetry instance ID did not match.'], + [ + () => Response.json({ ...telemetry, model: { id: 'other', loaded: true } }), + 'Strata telemetry model ID did not match.', + ], + [ + () => Response.json({ ...telemetry, model: { id: 'model', loaded: false } }), + 'Strata telemetry reported that the model was not loaded.', + ], +] as const)('reports the HTTP health result as %s → %s', async (reply, reason) => { + const fetcher = vi.fn(async () => reply()) + await expect(probe(fetcher).httpHealthFailure(status, new AbortController().signal)).resolves.toBe(reason) + expect(fetcher).toHaveBeenCalledWith( + 'http://127.0.0.1:12345/v1/telemetry', + expect.objectContaining({ + headers: { authorization: 'Bearer key' }, + redirect: 'error', + }) + ) +}) + +it('distinguishes an HTTP request timeout from an immediate connection failure', async () => { + const lifecycle = new AbortController() + const missing = probe(vi.fn().mockRejectedValue(new TypeError('fetch failed'))) + await expect(missing.httpHealthFailure(status, lifecycle.signal)).resolves.toBe( + 'Could not reach Strata telemetry.' + ) + + const timeout = new AbortController() + const timeoutSpy = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(timeout.signal) + try { + const stalled = probe( + async (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }) + }) + ) + const result = stalled.httpHealthFailure(status, lifecycle.signal) + timeout.abort(new DOMException('timed out', 'TimeoutError')) + await expect(result).resolves.toBe('Strata telemetry timed out after 2000 ms.') + expect(timeoutSpy).toHaveBeenCalledWith(2000) + } finally { + timeoutSpy.mockRestore() + } +}) diff --git a/src/runtime/strata/strata-http.ts b/src/runtime/strata/strata-http.ts index d965cf97..84ecaa4b 100644 --- a/src/runtime/strata/strata-http.ts +++ b/src/runtime/strata/strata-http.ts @@ -6,6 +6,7 @@ import { strataRequest } from './strata-policy.js' const chat = { method: 'POST', path: '/v1/chat/completions' } const routes: readonly ManagedRoute[] = [chat, { method: 'GET', path: '/v1/models' }] +const telemetryTimeoutMs = 2000 export function strataHttp( upstreamPort: number, @@ -25,27 +26,39 @@ export function strataHttp( } return { policy, - async proveHttp(status: { instanceId?: string }, signal: AbortSignal): Promise { + async httpHealthFailure( + status: { instanceId?: string }, + signal: AbortSignal + ): Promise { + if (!status.instanceId) return 'Strata Control did not report an instance ID.' + const timeout = AbortSignal.timeout(telemetryTimeoutMs) + let response: Response try { - if (!status.instanceId) return false - const response = await fetcher('http://127.0.0.1:' + upstreamPort + '/v1/telemetry', { + response = await fetcher('http://127.0.0.1:' + upstreamPort + '/v1/telemetry', { headers: { authorization: 'Bearer ' + upstreamKey }, - signal: AbortSignal.any([signal, AbortSignal.timeout(2000)]), + signal: AbortSignal.any([signal, timeout]), redirect: 'error', }) - if (!response.ok) return false - const telemetry = (await response.json()) as { - instance?: { id?: unknown } - model?: { id?: unknown; loaded?: unknown } - } - return ( - telemetry.instance?.id === status.instanceId && - telemetry.model?.id === modelId && - telemetry.model.loaded === true - ) } catch { - return false + return timeout.aborted + ? `Strata telemetry timed out after ${telemetryTimeoutMs} ms.` + : 'Could not reach Strata telemetry.' } + if (!response.ok) return `Strata telemetry returned HTTP ${response.status}.` + let telemetry: { + instance?: { id?: unknown } + model?: { id?: unknown; loaded?: unknown } + } | null + try { + telemetry = (await response.json()) as typeof telemetry + } catch { + return timeout.aborted + ? `Strata telemetry timed out after ${telemetryTimeoutMs} ms.` + : 'Could not read the Strata telemetry JSON.' + } + if (telemetry?.instance?.id !== status.instanceId) return 'Strata telemetry instance ID did not match.' + if (telemetry.model?.id !== modelId) return 'Strata telemetry model ID did not match.' + if (telemetry.model.loaded !== true) return 'Strata telemetry reported that the model was not loaded.' }, async expose() { const apiKey = generateGatewayKey() diff --git a/src/runtime/strata/strata-runtime.test.ts b/src/runtime/strata/strata-runtime.test.ts index 0965797a..d5520a79 100644 --- a/src/runtime/strata/strata-runtime.test.ts +++ b/src/runtime/strata/strata-runtime.test.ts @@ -2,14 +2,17 @@ import { expect, it, vi } from 'vitest' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { AtomicCoreError } from '../../contracts/index.js' import { StrataRuntime } from './strata-runtime.js' import type { StrataControlStatus } from './strata-control-plan.js' import type { GpuClaimHook } from '../shared/index.js' const ready = { state: 'ready', hostPid: 111, ownerId: 'a'.repeat(32), instanceId: 'b'.repeat(32) } as const -function fixture(logPath?: string) { +function fixture(logPath?: string, monitorPollMs?: number) { const status = vi.fn(async (): Promise => ready) const stop = vi.fn(async () => true) - const proof = vi.fn(async () => true) + const httpHealth = vi.fn(async (): Promise => undefined) + const emit = vi.fn() + const log = vi.fn() const close = vi.fn(async () => {}) const expose = vi.fn(async () => ({ port: 10300, apiKey: 'gateway-key', close })) const start = vi.fn(async () => ({ state: 'starting' as const })) @@ -22,13 +25,16 @@ function fixture(logPath?: string) { timeoutMs: 1_000, owner: { start, status, stop }, ...(logPath ? { logPath } : {}), - proveHttp: proof, + httpHealthFailure: httpHealth, expose, }), pollMs: 1, + ...(monitorPollMs === undefined ? {} : { monitorPollMs }), claimGpu, + emit, + log, }) - return { runtime, status, stop, proof, expose, start, close, claimGpu } + return { runtime, status, stop, httpHealth, expose, start, close, claimGpu, emit, log } } it.each([ { @@ -68,7 +74,7 @@ it.each([ it('publishes the owner PID and gateway only after Control and HTTP independently report readiness', async () => { const f = fixture() f.status.mockResolvedValueOnce({ state: 'starting' }) - f.proof.mockResolvedValueOnce(false) + f.httpHealth.mockResolvedValueOnce('Strata telemetry returned HTTP 503.') const a = f.runtime.load('model') const b = f.runtime.load('model') expect(await a).toMatchObject({ pid: 111, port: 10300, api_key: 'gateway-key', model_id: 'model' }) @@ -80,10 +86,70 @@ it('publishes the owner PID and gateway only after Control and HTTP independentl expect.any(Function) ) expect(f.status).toHaveBeenCalledTimes(3) - expect(f.proof).toHaveBeenCalledTimes(2) + expect(f.httpHealth).toHaveBeenCalledTimes(2) expect(f.expose).toHaveBeenCalledTimes(1) await f.runtime.shutdown() }) + +it('keeps the gateway open during health failures and resets the count after recovery', async () => { + const f = fixture(undefined, 1) + const throttle = 'Strata telemetry returned HTTP 503.' + const timeout = 'Strata telemetry timed out after 2000 ms.' + const session = await f.runtime.load('model') + try { + f.httpHealth + .mockResolvedValueOnce(throttle) + .mockResolvedValueOnce(timeout) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(throttle) + await vi.waitFor(() => expect(f.log).toHaveBeenCalledTimes(5)) + expect(f.log.mock.calls).toEqual([ + ['warn', `Strata model health check 1/3 failed. ${throttle}`], + ['warn', `Strata model health check 2/3 failed. ${timeout}`], + ['info', 'Strata model health check recovered after 2 failure(s).'], + ['warn', `Strata model health check 1/3 failed. ${throttle}`], + ['info', 'Strata model health check recovered after 1 failure(s).'], + ]) + expect(f.runtime.findSession('model')).toEqual(session) + expect(f.emit).toHaveBeenCalledOnce() + expect(f.close).not.toHaveBeenCalled() + expect(f.stop).not.toHaveBeenCalled() + } finally { + await f.runtime.shutdown() + } +}) + +it('withdraws the session after three consecutive failures and reports their causes', async () => { + const f = fixture(undefined, 1) + await f.runtime.load('model') + try { + f.status + .mockRejectedValueOnce( + new AtomicCoreError('MODEL_LOAD_FAILED', 'Strata Control command failed.', 'unavailable') + ) + .mockResolvedValueOnce({ ...ready, instanceId: 'c'.repeat(32) }) + f.httpHealth.mockResolvedValueOnce('Strata telemetry returned HTTP 503.') + await vi.waitFor(() => expect(f.stop).toHaveBeenCalledOnce()) + expect(f.runtime.findSession('model')).toBeUndefined() + expect(f.runtime.gpuOccupancy()).toEqual([]) + expect(f.close).toHaveBeenCalledOnce() + expect(f.log.mock.calls).toEqual([ + ['warn', 'Strata model health check 1/3 failed. Strata Control command failed (unavailable).'], + ['warn', 'Strata model health check 2/3 failed. Strata Control instance ID did not match.'], + ['warn', 'Strata model health check 3/3 failed. Strata telemetry returned HTTP 503.'], + ]) + expect(f.emit).toHaveBeenLastCalledWith('session:died', { + provider: 'strata', + model_id: 'model', + pid: 111, + exit_code: null, + signal: null, + message: 'Strata model health check 3/3 failed. Strata telemetry returned HTTP 503.', + }) + } finally { + await f.runtime.shutdown() + } +}) it('cancels a queued Strata reload while a different model holds the runtime turn', async () => { const f = fixture() let notifyAcquired!: () => void diff --git a/src/runtime/strata/strata-runtime.ts b/src/runtime/strata/strata-runtime.ts index 204b0be5..8cf9068a 100644 --- a/src/runtime/strata/strata-runtime.ts +++ b/src/runtime/strata/strata-runtime.ts @@ -29,7 +29,7 @@ export interface StrataRuntimeLaunch { logPath?: string timeoutMs: number owner: Pick - proveHttp(status: StrataControlStatus, signal: AbortSignal): Promise + httpHealthFailure(status: StrataControlStatus, signal: AbortSignal): Promise expose(): Promise policy?: SessionRoutePolicy } @@ -47,7 +47,9 @@ interface Options { ): Promise claimGpu?: GpuClaimHook emit?: EmitFn + log?: (level: 'info' | 'warn', message: string) => void pollMs?: number + monitorPollMs?: number } interface Held { launch: StrataRuntimeLaunch @@ -63,6 +65,14 @@ interface Pending { } const busy = () => new AtomicCoreError('GPU_BUSY', 'Strata process exit is unconfirmed. The selected GPU is still held.') +const healthFailureLimit = 3 + +function controlFailure(reason: string, detail?: unknown): string { + // Control may include native diagnostics. Only log short status codes, never raw process output. + return typeof detail === 'string' && /^[a-z][a-z0-9_]{0,63}$/.test(detail) + ? `${reason.replace(/\.$/, '')} (${detail}).` + : reason +} async function startupMemoryError(logPath: string): Promise { const file = await open(logPath, constants.O_RDONLY | constants.O_NONBLOCK | (constants.O_NOFOLLOW ?? 0)) @@ -217,7 +227,11 @@ export class StrataRuntime implements LocalRuntime { await launch.owner.start(launch.timeoutMs, startup) while (!startup.aborted) { const status = await launch.owner.status() - if (status.state === 'ready' && status.hostPid && (await launch.proveHttp(status, startup))) { + if ( + status.state === 'ready' && + status.hostPid && + !(await launch.httpHealthFailure(status, startup)) + ) { startup.throwIfAborted() active.endpoint = await launch.expose() startup.throwIfAborted() @@ -269,23 +283,42 @@ export class StrataRuntime implements LocalRuntime { } } - private async monitor(modelId: string, row: Held, instanceId: string | undefined, signal: AbortSignal) { + private async healthFailure(row: Held, instanceId: string | undefined, signal: AbortSignal) { + let status: StrataControlStatus try { - while (!signal.aborted) { - await sleep(5000, undefined, { signal, ref: false }) - const status = await row.launch.owner.status() - if ( - status.state === 'ready' && - status.instanceId === instanceId && - (await row.launch.proveHttp(status, signal)) - ) - continue - break - } - } catch { - // Treat a lost identity as unusable. Stop confirmation remains a separate check. + status = await row.launch.owner.status() + } catch (error) { + return error instanceof AtomicCoreError + ? controlFailure(error.message, error.details) + : 'Strata Control status check failed.' } - if (!signal.aborted && row.session) { + if (signal.aborted) return + if (status.state !== 'ready') + return controlFailure(`Strata Control reported ${status.state}.`, status.error) + if (status.instanceId !== instanceId) return 'Strata Control instance ID did not match.' + return row.launch.httpHealthFailure(status, signal) + } + + private async monitor(modelId: string, row: Held, instanceId: string | undefined, signal: AbortSignal) { + let failures = 0 + while (!signal.aborted && row.session) { + try { + await sleep(this.options.monitorPollMs ?? 5000, undefined, { signal, ref: false }) + } catch { + return // Explicit unload or shutdown cancelled the timer. + } + const failure = await this.healthFailure(row, instanceId, signal) + if (signal.aborted || !row.session) return + if (!failure) { + if (failures) + this.options.log?.('info', `Strata ${modelId} health check recovered after ${failures} failure(s).`) + failures = 0 + continue + } + failures++ + const message = `Strata ${modelId} health check ${failures}/${healthFailureLimit} failed. ${failure}` + this.options.log?.('warn', message) + if (failures < healthFailureLimit) continue const pid = row.session.pid delete row.session this.options.emit?.('session:died', { @@ -294,9 +327,10 @@ export class StrataRuntime implements LocalRuntime { pid, exit_code: null, signal: null, - message: 'The Strata runtime no longer reports the loaded instance.', + message, }) await this.stop(modelId, row) + return } } diff --git a/src/version.ts b/src/version.ts index 8ec703e7..f3b22f95 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ /** Kept in sync with package.json by `src/version.test.ts`. Bump both together. */ -export const CORE_VERSION = '0.24.7' +export const CORE_VERSION = '0.24.8' diff --git a/test/contract/strata-http.test.ts b/test/contract/strata-http.test.ts index f0634428..51c31272 100644 --- a/test/contract/strata-http.test.ts +++ b/test/contract/strata-http.test.ts @@ -49,8 +49,10 @@ it('proves HTTP instance and model, isolates credentials, and forwards another r ) try { const signal = new AbortController().signal - await expect(http.proveHttp({ instanceId: 'wrong' }, signal)).resolves.toBe(false) - await expect(http.proveHttp({ instanceId: 'instance' }, signal)).resolves.toBe(true) + await expect(http.httpHealthFailure({ instanceId: 'wrong' }, signal)).resolves.toBe( + 'Strata telemetry instance ID did not match.' + ) + await expect(http.httpHealthFailure({ instanceId: 'instance' }, signal)).resolves.toBeUndefined() const endpoint = await http.expose() try { const url = 'http://127.0.0.1:' + endpoint.port