Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/decisions/2026-11-19-retry-strata-session-health-checks.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
date: 2026-11-19
title: "Retry Strata session health checks before unloading"
---

# Retry Strata session health checks before unloading

The Strata runtime verifies its owned Control instance and its authenticated HTTP telemetry once every five seconds after loading. A slow or failed check used to close the local gateway immediately, interrupting chat streams even when the engine could recover.

Keep the loaded session and gateway for the first two failed checks. Close the gateway and stop the engine after three consecutive failures. A check that verifies both Control and telemetry resets the count. Initial loading still requires both checks to pass before the gateway opens. Explicit unload and shutdown still cancel the monitor immediately. GPU ownership still depends on confirmed process exit.

Log the attempt and cause of every failure in the core log, along with any recovery. Report the final cause in the session death event. Diagnostics use status codes and summaries so credentials and response bodies stay out of the log.

Owner: team.
1 change: 1 addition & 0 deletions docs/decisions/INDEX.md
Original file line number Diff line number Diff line change
Expand Up @@ -290,3 +290,4 @@ add a new one that says which record it supersedes.
- [2026-11-19 — Reuse Strata engine profiles from the model catalog](2026-11-19-reuse-strata-profiles-from-the-model-catalog.md)
- [Select Strata releases compatible with the running core](2026-11-19-skip-incompatible-strata-releases.md)
- [Start Windows DeepSeek from existing Strata packs](2026-11-19-start-windows-deepseek-from-existing-strata-packs.md)
- [Retry Strata session health checks before unloading](2026-11-19-retry-strata-session-health-checks.md)
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@atomic-chat/core",
"version": "0.24.7",
"version": "0.24.8",
"repository": {
"type": "git",
"url": "git+https://github.com/AtomicBot-ai/atomic-chat-core.git"
Expand Down
1 change: 1 addition & 0 deletions src/core/create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,7 @@ export async function createAtomicCore(
trustedHosts: managedTrustedHosts,
claimGpu: gpuResidency.hook('strata'),
emit: (name, payload) => emitter.emit(name, payload),
log,
})
runtimes.set('strata', strata.runtime)
registries.set('strata', new ModelRegistry(layout, 'strata'))
Expand Down
3 changes: 3 additions & 0 deletions src/core/strata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import type { StrataHostFacts, StrataLaunchIntent } from '../runtime/strata/inde
import type { StrataBackend, StrataCheckResult, StrataModelDescriptor } from '../contracts/index.js'
import type { StrataModelEntry } from '../models/index.js'
import { CORE_VERSION } from '../version.js'
import type { CoreLogger } from './types.js'

interface Options {
layout: DataLayout
Expand All @@ -58,6 +59,7 @@ interface Options {
trustedHosts: string[]
claimGpu: GpuClaimHook
emit: EmitFn
log?: CoreLogger
}

const insufficientStrataVram = (requiredBytes: number) =>
Expand Down Expand Up @@ -312,6 +314,7 @@ export async function wireStrata(options: Options) {
const runtime: StrataRuntime = new StrataRuntime({
claimGpu: options.claimGpu,
emit: options.emit,
...(options.log ? { log: options.log } : {}),
guardLoad: (id, work) => maintenance.withLoad(id, work),
stopPreparations: () => preparations.shutdown(),
retained: () => journal.list().filter((entry) => entry.core.instanceId !== instanceId),
Expand Down
67 changes: 67 additions & 0 deletions src/runtime/strata/strata-http.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { expect, it, vi } from 'vitest'
import { strataHttp } from './strata-http.js'

const status = { instanceId: 'expected-instance' }
const telemetry = { instance: { id: status.instanceId }, model: { id: 'model', loaded: true } }

function probe(fetcher: typeof fetch) {
return strataHttp(
12345,
'key',
'model',
{ chat: true, tools: true, structured_output: true, vision: false, reasoning: true },
8192,
[],
fetcher
)
}

it.each([
[() => Response.json(telemetry), undefined],
[() => new Response(null, { status: 503 }), 'Strata telemetry returned HTTP 503.'],
[() => new Response('not json'), 'Could not read the Strata telemetry JSON.'],
[() => Response.json(null), 'Strata telemetry instance ID did not match.'],
[
() => Response.json({ ...telemetry, model: { id: 'other', loaded: true } }),
'Strata telemetry model ID did not match.',
],
[
() => Response.json({ ...telemetry, model: { id: 'model', loaded: false } }),
'Strata telemetry reported that the model was not loaded.',
],
] as const)('reports the HTTP health result as %s → %s', async (reply, reason) => {
const fetcher = vi.fn(async () => reply())
await expect(probe(fetcher).httpHealthFailure(status, new AbortController().signal)).resolves.toBe(reason)
expect(fetcher).toHaveBeenCalledWith(
'http://127.0.0.1:12345/v1/telemetry',
expect.objectContaining({
headers: { authorization: 'Bearer key' },
redirect: 'error',
})
)
})

it('distinguishes an HTTP request timeout from an immediate connection failure', async () => {
const lifecycle = new AbortController()
const missing = probe(vi.fn().mockRejectedValue(new TypeError('fetch failed')))
await expect(missing.httpHealthFailure(status, lifecycle.signal)).resolves.toBe(
'Could not reach Strata telemetry.'
)

const timeout = new AbortController()
const timeoutSpy = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(timeout.signal)
try {
const stalled = probe(
async (_url, init) =>
new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true })
})
)
const result = stalled.httpHealthFailure(status, lifecycle.signal)
timeout.abort(new DOMException('timed out', 'TimeoutError'))
await expect(result).resolves.toBe('Strata telemetry timed out after 2000 ms.')
expect(timeoutSpy).toHaveBeenCalledWith(2000)
} finally {
timeoutSpy.mockRestore()
}
})
43 changes: 28 additions & 15 deletions src/runtime/strata/strata-http.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { strataRequest } from './strata-policy.js'

const chat = { method: 'POST', path: '/v1/chat/completions' }
const routes: readonly ManagedRoute[] = [chat, { method: 'GET', path: '/v1/models' }]
const telemetryTimeoutMs = 2000

export function strataHttp(
upstreamPort: number,
Expand All @@ -25,27 +26,39 @@ export function strataHttp(
}
return {
policy,
async proveHttp(status: { instanceId?: string }, signal: AbortSignal): Promise<boolean> {
async httpHealthFailure(
status: { instanceId?: string },
signal: AbortSignal
): Promise<string | undefined> {
if (!status.instanceId) return 'Strata Control did not report an instance ID.'
const timeout = AbortSignal.timeout(telemetryTimeoutMs)
let response: Response
try {
if (!status.instanceId) return false
const response = await fetcher('http://127.0.0.1:' + upstreamPort + '/v1/telemetry', {
response = await fetcher('http://127.0.0.1:' + upstreamPort + '/v1/telemetry', {
headers: { authorization: 'Bearer ' + upstreamKey },
signal: AbortSignal.any([signal, AbortSignal.timeout(2000)]),
signal: AbortSignal.any([signal, timeout]),
redirect: 'error',
})
if (!response.ok) return false
const telemetry = (await response.json()) as {
instance?: { id?: unknown }
model?: { id?: unknown; loaded?: unknown }
}
return (
telemetry.instance?.id === status.instanceId &&
telemetry.model?.id === modelId &&
telemetry.model.loaded === true
)
} catch {
return false
return timeout.aborted
? `Strata telemetry timed out after ${telemetryTimeoutMs} ms.`
: 'Could not reach Strata telemetry.'
}
if (!response.ok) return `Strata telemetry returned HTTP ${response.status}.`
let telemetry: {
instance?: { id?: unknown }
model?: { id?: unknown; loaded?: unknown }
} | null
try {
telemetry = (await response.json()) as typeof telemetry
} catch {
return timeout.aborted
? `Strata telemetry timed out after ${telemetryTimeoutMs} ms.`
: 'Could not read the Strata telemetry JSON.'
}
if (telemetry?.instance?.id !== status.instanceId) return 'Strata telemetry instance ID did not match.'
if (telemetry.model?.id !== modelId) return 'Strata telemetry model ID did not match.'
if (telemetry.model.loaded !== true) return 'Strata telemetry reported that the model was not loaded.'
},
async expose() {
const apiKey = generateGatewayKey()
Expand Down
78 changes: 72 additions & 6 deletions src/runtime/strata/strata-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ import { expect, it, vi } from 'vitest'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AtomicCoreError } from '../../contracts/index.js'
import { StrataRuntime } from './strata-runtime.js'
import type { StrataControlStatus } from './strata-control-plan.js'
import type { GpuClaimHook } from '../shared/index.js'
const ready = { state: 'ready', hostPid: 111, ownerId: 'a'.repeat(32), instanceId: 'b'.repeat(32) } as const
function fixture(logPath?: string) {
function fixture(logPath?: string, monitorPollMs?: number) {
const status = vi.fn(async (): Promise<StrataControlStatus> => ready)
const stop = vi.fn(async () => true)
const proof = vi.fn(async () => true)
const httpHealth = vi.fn(async (): Promise<string | undefined> => undefined)
const emit = vi.fn()
const log = vi.fn()
const close = vi.fn(async () => {})
const expose = vi.fn(async () => ({ port: 10300, apiKey: 'gateway-key', close }))
const start = vi.fn(async () => ({ state: 'starting' as const }))
Expand All @@ -22,13 +25,16 @@ function fixture(logPath?: string) {
timeoutMs: 1_000,
owner: { start, status, stop },
...(logPath ? { logPath } : {}),
proveHttp: proof,
httpHealthFailure: httpHealth,
expose,
}),
pollMs: 1,
...(monitorPollMs === undefined ? {} : { monitorPollMs }),
claimGpu,
emit,
log,
})
return { runtime, status, stop, proof, expose, start, close, claimGpu }
return { runtime, status, stop, httpHealth, expose, start, close, claimGpu, emit, log }
}
it.each([
{
Expand Down Expand Up @@ -68,7 +74,7 @@ it.each([
it('publishes the owner PID and gateway only after Control and HTTP independently report readiness', async () => {
const f = fixture()
f.status.mockResolvedValueOnce({ state: 'starting' })
f.proof.mockResolvedValueOnce(false)
f.httpHealth.mockResolvedValueOnce('Strata telemetry returned HTTP 503.')
const a = f.runtime.load('model')
const b = f.runtime.load('model')
expect(await a).toMatchObject({ pid: 111, port: 10300, api_key: 'gateway-key', model_id: 'model' })
Expand All @@ -80,10 +86,70 @@ it('publishes the owner PID and gateway only after Control and HTTP independentl
expect.any(Function)
)
expect(f.status).toHaveBeenCalledTimes(3)
expect(f.proof).toHaveBeenCalledTimes(2)
expect(f.httpHealth).toHaveBeenCalledTimes(2)
expect(f.expose).toHaveBeenCalledTimes(1)
await f.runtime.shutdown()
})

it('keeps the gateway open during health failures and resets the count after recovery', async () => {
const f = fixture(undefined, 1)
const throttle = 'Strata telemetry returned HTTP 503.'
const timeout = 'Strata telemetry timed out after 2000 ms.'
const session = await f.runtime.load('model')
try {
f.httpHealth
.mockResolvedValueOnce(throttle)
.mockResolvedValueOnce(timeout)
.mockResolvedValueOnce(undefined)
.mockResolvedValueOnce(throttle)
await vi.waitFor(() => expect(f.log).toHaveBeenCalledTimes(5))
expect(f.log.mock.calls).toEqual([
['warn', `Strata model health check 1/3 failed. ${throttle}`],
['warn', `Strata model health check 2/3 failed. ${timeout}`],
['info', 'Strata model health check recovered after 2 failure(s).'],
['warn', `Strata model health check 1/3 failed. ${throttle}`],
['info', 'Strata model health check recovered after 1 failure(s).'],
])
expect(f.runtime.findSession('model')).toEqual(session)
expect(f.emit).toHaveBeenCalledOnce()
expect(f.close).not.toHaveBeenCalled()
expect(f.stop).not.toHaveBeenCalled()
} finally {
await f.runtime.shutdown()
}
})

it('withdraws the session after three consecutive failures and reports their causes', async () => {
const f = fixture(undefined, 1)
await f.runtime.load('model')
try {
f.status
.mockRejectedValueOnce(
new AtomicCoreError('MODEL_LOAD_FAILED', 'Strata Control command failed.', 'unavailable')
)
.mockResolvedValueOnce({ ...ready, instanceId: 'c'.repeat(32) })
f.httpHealth.mockResolvedValueOnce('Strata telemetry returned HTTP 503.')
await vi.waitFor(() => expect(f.stop).toHaveBeenCalledOnce())
expect(f.runtime.findSession('model')).toBeUndefined()
expect(f.runtime.gpuOccupancy()).toEqual([])
expect(f.close).toHaveBeenCalledOnce()
expect(f.log.mock.calls).toEqual([
['warn', 'Strata model health check 1/3 failed. Strata Control command failed (unavailable).'],
['warn', 'Strata model health check 2/3 failed. Strata Control instance ID did not match.'],
['warn', 'Strata model health check 3/3 failed. Strata telemetry returned HTTP 503.'],
])
expect(f.emit).toHaveBeenLastCalledWith('session:died', {
provider: 'strata',
model_id: 'model',
pid: 111,
exit_code: null,
signal: null,
message: 'Strata model health check 3/3 failed. Strata telemetry returned HTTP 503.',
})
} finally {
await f.runtime.shutdown()
}
})
it('cancels a queued Strata reload while a different model holds the runtime turn', async () => {
const f = fixture()
let notifyAcquired!: () => void
Expand Down
Loading
Loading