diff --git a/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md new file mode 100644 index 00000000..e9546267 --- /dev/null +++ b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md @@ -0,0 +1,32 @@ +--- +date: 2026-11-17 +title: "Tag Foundry Local host load failures at their source" +--- + +# 2026-11-17 — Tag Foundry Local host load failures at their source + +## Context + +Foundry Local emits `FOUNDRY_LOCAL_HOST_ERROR` for several host exits before model readiness. +An exit with code 3 means library loading failed or the native API was incompatible. Other +early exits include failures before the library loads, so the generic host error cannot establish +which startup stage the process reached. The install-time probe uses the same library message +when it returns an unreadable response, even with an exit code of zero. + +## Decision + +In the runtime's exit handler, use the existing `operation_detail` tag to distinguish +`foundry_local_library_not_loaded` for exit 3 and `foundry_local_host_stopped_before_ready` +for the remaining `FOUNDRY_LOCAL_HOST_ERROR` exits. Exit 5 keeps its accelerator refusal. +The install-time probe does not use either load detail. + +## Consequences + +Sentry can split existing Foundry model-load failures by `operation_detail`, while clients keep +the same error code, message and details. The tag contains a fixed value from the diagnostic +contract. Grouping and severity retain their existing rules. + +Owner: team + +References: `src/runtime/foundry-local/runtime.ts`, `src/runtime/foundry-local/host-process.ts`, +Sentry `ATOMIC-CHAT-CORE-CM`. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 94fc3707..2049a940 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,6 +258,7 @@ add a new one that says which record it supersedes. - [Preserve error cause on staged engine-build install failures](2026-11-06-preserve-cause-on-staged-engine-install-errors.md) - [Tag MLX install probe failures with finite diagnostic codes](2026-11-12-extend-bounded-diagnostics-to-mlx-install-probe.md) - [Wrap model-claim filesystem failures as coded IO errors](2026-11-12-wrap-model-claim-filesystem-failures-as-coded-io-errors.md) +- [Tag Foundry Local host load failures at their source](2026-11-17-foundry-local-host-load-failures-get-finite-codes.md) - [Re-establish a model claim whose directory vanished](2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md) - [Accept the app's client keys next to the server key](2026-11-12-accept-app-client-keys-next-to-the-server-key.md) — `PUT /atomic/v1/server/client-keys`: in-memory extra keys next to a non-empty `api_key`; keyring wins, keyless stays open - [Downloads report the disk reason they already tag](2026-11-11-downloads-report-their-own-disk-reason.md) diff --git a/src/contracts/diagnostics.ts b/src/contracts/diagnostics.ts index b7c0cf43..90ce65ab 100644 --- a/src/contracts/diagnostics.ts +++ b/src/contracts/diagnostics.ts @@ -40,6 +40,8 @@ export const FAILURE_OPERATION_DETAILS = [ 'mlx_explicit_binary_missing', 'mlx_resolved_binary_missing', 'mlx_resources_binary_missing', + 'foundry_local_library_not_loaded', + 'foundry_local_host_stopped_before_ready', ] as const export const FAILURE_REASONS = [ 'cancelled', diff --git a/src/runtime/foundry-local/host-process.test.ts b/src/runtime/foundry-local/host-process.test.ts index f5a59cc4..019e0b93 100644 --- a/src/runtime/foundry-local/host-process.test.ts +++ b/src/runtime/foundry-local/host-process.test.ts @@ -2,6 +2,7 @@ import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { classifyFailure, diagnosticTags } from '../../telemetry/index.js' import { foundryHostCommand, runFoundryHostProbe } from './host-process.js' const POSIX = process.platform !== 'win32' @@ -58,11 +59,17 @@ describe.skipIf(!POSIX)('runFoundryHostProbe', () => { it('fails when the host prints no probe answer', async () => { const exe = await fakeHost(`echo 'not json'`) - await expect( - runFoundryHostProbe({ exe, args: [] }, { libDir: '/l', appDataDir: '/a', logsDir: '/g' }) - ).rejects.toMatchObject({ + const error: unknown = await runFoundryHostProbe( + { exe, args: [] }, + { libDir: '/l', appDataDir: '/a', logsDir: '/g' } + ).catch((cause: unknown) => cause) + expect(error).toMatchObject({ code: 'ENGINE_INSTALL_FAILED', + message: 'The Foundry Local library does not load on this computer.', }) + const tags = diagnosticTags(error, classifyFailure(error, { component: 'backend', operation: 'install' })) + expect(tags).not.toHaveProperty('operation_detail') + expect(tags).not.toHaveProperty('diagnostic_code') }) it('stops a host that does not answer in time', async () => { diff --git a/src/runtime/foundry-local/runtime.test.ts b/src/runtime/foundry-local/runtime.test.ts index bd31afdb..ca08733e 100644 --- a/src/runtime/foundry-local/runtime.test.ts +++ b/src/runtime/foundry-local/runtime.test.ts @@ -8,6 +8,7 @@ import { makeTmpDataFolder } from '../../../test/helpers/tmp-data-folder.js' import type { TmpDataFolder } from '../../../test/helpers/tmp-data-folder.js' import type { CoreEvents } from '../../contracts/index.js' import { ModelRegistry, serializeModelYml } from '../../models/index.js' +import { diagnosticTags, loadFailureReport } from '../../telemetry/index.js' import { spawnAndAwaitReady } from '../shared/index.js' import type { GpuClaimHook } from '../shared/index.js' import { FoundryLocalRuntime } from './runtime.js' @@ -361,6 +362,27 @@ describe.skipIf(!POSIX)('FoundryLocalRuntime (task 6.3)', () => { }) }) + it.each([ + ['exit-3', 'foundry_local_library_not_loaded'], + ['exit-4', 'foundry_local_host_stopped_before_ready'], + ])('reports the Foundry host load detail for %s', async (mode, detail) => { + const modelId = 'foundry-diagnostic' + await variant(modelId, { execution_provider: 'webgpu', device: 'gpu' }) + const error: unknown = await runtime({ mode }) + .rt.load(modelId) + .catch((cause: unknown) => cause) + expect(error).toMatchObject({ code: 'FOUNDRY_LOCAL_HOST_ERROR' }) + const report = loadFailureReport({ provider: 'foundry-local', modelId, error }) + if (!report?.diagnostic) throw new Error('Expected a Foundry host load report') + expect(report.level).toBe('error') + expect(diagnosticTags(report.error, report.diagnostic)).toMatchObject({ + component: 'runtime', + operation: 'load', + failure_reason: 'unknown', + operation_detail: detail, + }) + }) + it('holds every card for a GPU variant and none for a CPU or NPU one, claiming the GPU only for the former', async () => { await variant('webgpu-variant', { execution_provider: 'webgpu', device: 'gpu' }) await variant('cpu-variant', { execution_provider: 'cpu', device: 'cpu' }) diff --git a/src/runtime/foundry-local/runtime.ts b/src/runtime/foundry-local/runtime.ts index 0862d89f..eebd8bb5 100644 --- a/src/runtime/foundry-local/runtime.ts +++ b/src/runtime/foundry-local/runtime.ts @@ -132,12 +132,22 @@ function hostError(exitCode: number | null, stderr: string): AtomicCoreError { 'The accelerator of this model is not prepared; prepare it on the Foundry Local page first.', said ) + const libraryDidNotLoad = exitCode === 3 return new AtomicCoreError( 'FOUNDRY_LOCAL_HOST_ERROR', - exitCode === 3 + libraryDidNotLoad ? 'The Foundry Local library does not load on this computer.' : 'The Foundry Local host stopped before the model was ready.', - said || `exit code ${String(exitCode)}` + said || `exit code ${String(exitCode)}`, + { + diagnostic: { + component: 'runtime', + operation: 'load', + operationDetail: libraryDidNotLoad + ? 'foundry_local_library_not_loaded' + : 'foundry_local_host_stopped_before_ready', + }, + } ) } diff --git a/test/helpers/fake-foundry-host.mjs b/test/helpers/fake-foundry-host.mjs index ab6dde82..b943ef05 100644 --- a/test/helpers/fake-foundry-host.mjs +++ b/test/helpers/fake-foundry-host.mjs @@ -4,7 +4,7 @@ * arguments, readiness line, `/catalogs/local/v1/*` routes, registration-id model matching and stop * on SIGTERM or stdin end as `src/runtime/foundry-local/host/run.ts` with a real library. * - * FAKE_FOUNDRY_MODE ready (default) | echo-chat-request | crash-after-ready | hang | exit-3 | exit-5 + * FAKE_FOUNDRY_MODE ready (default) | echo-chat-request | crash-after-ready | hang | exit-3 | exit-4 | exit-5 * FAKE_FOUNDRY_ARGV a file each start appends {argv, ort_telemetry_disabled, * ort_enable_cudnn_flash_attention, ci} to */ @@ -90,6 +90,10 @@ if (mode === 'exit-3') { console.error('foundry_local does not load from /b: libatomic.so.1: cannot open shared object file') process.exit(3) } +if (mode === 'exit-4') { + console.error('Foundry library call failed before readiness') + process.exit(4) +} const port = Number(arg('--port')) const id = arg('--id')