From 5923ddbd7fbc8d1754e753758010126ca54db522 Mon Sep 17 00:00:00 2001 From: atomic-sentry-bot Date: Tue, 17 Nov 2026 04:19:35 +0000 Subject: [PATCH 1/2] diagnostic: collect evidence for Sentry issue 7851254696 --- ...cal-host-load-failures-get-finite-codes.md | 12 +++ docs/decisions/INDEX.md | 1 + .../sentry-agent-diagnostics.test.ts | 96 +++++++++++++++++++ src/telemetry/sentry-agent-diagnostics.ts | 4 +- 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md diff --git a/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md new file mode 100644 index 00000000..ccc9314e --- /dev/null +++ b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md @@ -0,0 +1,12 @@ +--- +date: 2026-11-17 +title: "Tag Foundry Local host load failures with finite diagnostic codes" +--- + +# 2026-11-17 — Tag Foundry Local host load failures with finite diagnostic codes + +- **Context:** `FOUNDRY_LOCAL_HOST_ERROR` events from `atomic-chat-core@0.23.4` carried the generic exception value `runtime load: unknown`, `failure_category=unknown`, `failure_reason=unknown` and no exit code or signal. `hostError` in `src/runtime/foundry-local/runtime.ts` collapses four distinct host outcomes into one error code: exit 5 becomes `FOUNDRY_LOCAL_ACCELERATOR_NOT_PREPARED`, exit 3 becomes "The Foundry Local library does not load on this computer.", and every other exit or signal death becomes "The Foundry Local host stopped before the model was ready.". `loadFailureReport` does not tag the host exit code or stderr, so an event cannot separate a native library that fails to load on Windows from a library call that failed after loading. The GenieX host already had the identical pair of codes; Foundry Local had none. +- **Decision:** Add two exact-message rules to the bounded diagnostic observer (`src/telemetry/sentry-agent-diagnostics.ts`, generated): "The Foundry Local library does not load on this computer." → `FOUNDRY_LOCAL_LIBRARY_NOT_LOADED` and "The Foundry Local host stopped before the model was ready." → `FOUNDRY_LOCAL_HOST_STOPPED_EARLY`, sourced from `src/runtime/foundry-local/runtime.ts`. Cause codes stay enabled. Nothing else changes: no new report, no severity, consent, throttling, grouping or exit-behavior change, and the error the caller receives is byte-identical. +- **Consequences:** The next released events from this path separate "the native library cannot load on this machine" from "the library loaded and the host then stopped before readiness". The observer still cannot say whether the host died from a signal or exited with a code, because the report tags never carried them; adding those facts is a separate reporting decision. Both rules are removable once the split is established, or once the exit code and signal reach the report directly. +- **Owner:** team +- **Links:** Sentry `ATOMIC-CHAT-CORE-CM` (event `ced119d9c8434e5e9a41fa0e30013202`), `src/runtime/foundry-local/runtime.ts`, `src/runtime/foundry-local/host/run.ts`, `src/telemetry/diagnostics.ts`, `src/telemetry/sentry-agent-diagnostics.test.ts` diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index a8af0842..ab4103a7 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,3 +258,4 @@ add a new one that says which record it supersedes. - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) - [Check MLX builds with the engine candidate workflow](2026-11-12-check-mlx-builds-with-the-engine-candidate.md) — `engine=mlx`: the fork's `vX.Y.Z`, conf's `mlxvlm-macos-arm64-` mirror; everything on macos-15, watched folders, folder live models - [Spell the sd.cpp argv for the installed build](2026-11-13-spell-the-sd-cpp-argv-for-the-installed-build.md) — `--backend te=cpu,vae=cpu` and `--llm` everywhere; `--flash-attn`/`--conv-direct` from build 982, the deprecated pair before; no `-M` +- [Tag Foundry Local host load failures with finite diagnostic codes](2026-11-17-foundry-local-host-load-failures-get-finite-codes.md) diff --git a/src/telemetry/sentry-agent-diagnostics.test.ts b/src/telemetry/sentry-agent-diagnostics.test.ts index 401f2501..908e256f 100644 --- a/src/telemetry/sentry-agent-diagnostics.test.ts +++ b/src/telemetry/sentry-agent-diagnostics.test.ts @@ -231,3 +231,99 @@ describe('MLX install probe bounded diagnostics', () => { expect(error.message).toBe('The downloaded mlx-server does not run on this Mac.') }) }) + +describe('Foundry Local host load bounded diagnostics', () => { + const load = { component: 'runtime', operation: 'load' } as const + + it('tags Foundry Local host load failures with finite diagnostic codes', () => { + const libraryLoadError = new AtomicCoreError( + 'FOUNDRY_LOCAL_HOST_ERROR', + 'The Foundry Local library does not load on this computer.', + '/private/fixture/foundry-host: exit code 3' + ) + const libraryTags = diagnosticTags(libraryLoadError, classifyFailure(libraryLoadError, load)) + expect(libraryTags).toMatchObject({ + diagnostic_version: 'core-v1', + component: 'runtime', + operation: 'load', + failure_category: 'unknown', + failure_reason: 'unknown', + cause_code: 'FOUNDRY_LOCAL_HOST_ERROR', + diagnostic_code: 'FOUNDRY_LOCAL_LIBRARY_NOT_LOADED', + }) + expect(libraryTags['diagnostic_cause_code']).toBeUndefined() + expect(JSON.stringify(libraryTags)).not.toContain('/private') + expect(libraryLoadError.toJSON()).toEqual({ + code: 'FOUNDRY_LOCAL_HOST_ERROR', + message: 'The Foundry Local library does not load on this computer.', + details: '/private/fixture/foundry-host: exit code 3', + }) + + const cause = Object.assign(new Error('/private/fixture/onnxruntime-genai-cuda.dll missing'), { + code: 'ENOENT', + }) + const stoppedError = new AtomicCoreError( + 'FOUNDRY_LOCAL_HOST_ERROR', + 'The Foundry Local host stopped before the model was ready.', + '/private/fixture/model: exit code 4', + { cause } + ) + const stoppedTags = diagnosticTags(stoppedError, classifyFailure(stoppedError, load)) + expect(stoppedTags).toMatchObject({ + diagnostic_version: 'core-v1', + component: 'runtime', + operation: 'load', + failure_category: 'unknown', + failure_reason: 'unknown', + cause_code: 'ENOENT', + diagnostic_code: 'FOUNDRY_LOCAL_HOST_STOPPED_EARLY', + diagnostic_cause_code: 'ENOENT', + }) + expect(JSON.stringify(stoppedTags)).not.toContain('/private') + expect(JSON.stringify(stoppedTags)).not.toContain('onnxruntime-genai-cuda') + }) + + it('leaves other Foundry Local refusals untagged and keeps cyclic causes private', () => { + const accelerator = new AtomicCoreError( + 'FOUNDRY_LOCAL_ACCELERATOR_NOT_PREPARED', + 'The accelerator of this model is not prepared; prepare it on the Foundry Local page first.' + ) + const acceleratorTags = diagnosticTags(accelerator, classifyFailure(accelerator, load)) + expect(acceleratorTags['diagnostic_code']).toBeUndefined() + expect(acceleratorTags['diagnostic_cause_code']).toBeUndefined() + expect(JSON.stringify(acceleratorTags)).not.toContain('prepare it on the Foundry Local page') + + const cycleA = Object.assign(new Error('/private/fixture/cycle-a'), { code: 'PRIVATE_CODE' }) + const cycleB = new Error('/private/fixture/cycle-b', { cause: cycleA }) + cycleA.cause = cycleB + const cycleTags = diagnosticTags(cycleA, classifyFailure(cycleA, load)) + expect(cycleTags['diagnostic_code']).toBeUndefined() + expect(cycleTags['diagnostic_cause_code']).toBeUndefined() + expect(JSON.stringify(cycleTags)).not.toContain('/private') + expect(JSON.stringify(cycleTags)).not.toContain('PRIVATE_CODE') + }) + + it('reports the host refusal as before, only adding finite tags', () => { + const capture = vi.fn() + const observe = failureObserver({ capture }) + const error = new AtomicCoreError( + 'FOUNDRY_LOCAL_HOST_ERROR', + 'The Foundry Local host stopped before the model was ready.', + '/private/fixture/model: exit code 4' + ) + observe(error, load) + expect(capture).toHaveBeenCalledTimes(1) + expect(capture).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'operation', + error, + level: 'error', + }) + ) + expect(error.toJSON()).toEqual({ + code: 'FOUNDRY_LOCAL_HOST_ERROR', + message: 'The Foundry Local host stopped before the model was ready.', + details: '/private/fixture/model: exit code 4', + }) + }) +}) diff --git a/src/telemetry/sentry-agent-diagnostics.ts b/src/telemetry/sentry-agent-diagnostics.ts index a9714747..18ec1513 100644 --- a/src/telemetry/sentry-agent-diagnostics.ts +++ b/src/telemetry/sentry-agent-diagnostics.ts @@ -1,4 +1,4 @@ -// SENTRY_AGENT_DIAGNOSTIC_CONFIG {"rules":[{"source":"src/diffusion/install.ts","message":"The image engine could not load a library it needs.","code":"SDCPP_PROBE_LIBRARY_LOAD_FAILED"},{"source":"src/diffusion/install.ts","message":"The image engine exited without printing anything.","code":"SDCPP_PROBE_SILENT_EXIT"},{"source":"src/diffusion/install.ts","message":"The downloaded binary is not stable-diffusion.cpp.","code":"SDCPP_PROBE_UNEXPECTED_OUTPUT"},{"source":"src/diffusion/install.ts","message":"The image engine did not respond to --help.","code":"SDCPP_PROBE_HELP_TIMEOUT"},{"source":"src/diffusion/install.ts","message":"The image engine could not be started.","code":"SDCPP_PROBE_START_FAILED"},{"source":"src/diffusion/install.ts","message":"The archive did not contain sd-server.","code":"SDCPP_ARCHIVE_MISSING_SERVER"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX library does not load on this computer.","code":"GENIEX_HOST_LIBRARY_LOAD_FAILED"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX host stopped before the model was ready.","code":"GENIEX_HOST_STOPPED_BEFORE_READY"},{"source":"src/engine-builds/builds.ts","message":"The downloaded mlx-server does not run on this Mac.","code":"MLX_PROBE_DOES_NOT_RUN"},{"source":"src/engine-builds/builds.ts","message":"The archive did not contain mlx-server.","code":"MLX_ARCHIVE_MISSING_SERVER"},{"source":"src/engine-builds/builds.ts","message":"mlx-server did not respond to --help.","code":"MLX_PROBE_HELP_TIMEOUT"},{"source":"src/engine-builds/builds.ts","message":"mlx-server could not be started.","code":"MLX_PROBE_START_FAILED"}],"include_cause_codes":true} +// SENTRY_AGENT_DIAGNOSTIC_CONFIG {"rules":[{"source":"src/diffusion/install.ts","message":"The image engine could not load a library it needs.","code":"SDCPP_PROBE_LIBRARY_LOAD_FAILED"},{"source":"src/diffusion/install.ts","message":"The image engine exited without printing anything.","code":"SDCPP_PROBE_SILENT_EXIT"},{"source":"src/diffusion/install.ts","message":"The downloaded binary is not stable-diffusion.cpp.","code":"SDCPP_PROBE_UNEXPECTED_OUTPUT"},{"source":"src/diffusion/install.ts","message":"The image engine did not respond to --help.","code":"SDCPP_PROBE_HELP_TIMEOUT"},{"source":"src/diffusion/install.ts","message":"The image engine could not be started.","code":"SDCPP_PROBE_START_FAILED"},{"source":"src/diffusion/install.ts","message":"The archive did not contain sd-server.","code":"SDCPP_ARCHIVE_MISSING_SERVER"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX library does not load on this computer.","code":"GENIEX_HOST_LIBRARY_LOAD_FAILED"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX host stopped before the model was ready.","code":"GENIEX_HOST_STOPPED_BEFORE_READY"},{"source":"src/engine-builds/builds.ts","message":"The downloaded mlx-server does not run on this Mac.","code":"MLX_PROBE_DOES_NOT_RUN"},{"source":"src/engine-builds/builds.ts","message":"The archive did not contain mlx-server.","code":"MLX_ARCHIVE_MISSING_SERVER"},{"source":"src/engine-builds/builds.ts","message":"mlx-server did not respond to --help.","code":"MLX_PROBE_HELP_TIMEOUT"},{"source":"src/engine-builds/builds.ts","message":"mlx-server could not be started.","code":"MLX_PROBE_START_FAILED"},{"source":"src/runtime/foundry-local/runtime.ts","message":"The Foundry Local library does not load on this computer.","code":"FOUNDRY_LOCAL_LIBRARY_NOT_LOADED"},{"source":"src/runtime/foundry-local/runtime.ts","message":"The Foundry Local host stopped before the model was ready.","code":"FOUNDRY_LOCAL_HOST_STOPPED_EARLY"}],"include_cause_codes":true} // This generated observer only enriches an existing error event. It never sends or logs. const SENTRY_AGENT_MESSAGES = new Map([ ['The image engine could not load a library it needs.', 'SDCPP_PROBE_LIBRARY_LOAD_FAILED'], @@ -13,6 +13,8 @@ const SENTRY_AGENT_MESSAGES = new Map([ ['The archive did not contain mlx-server.', 'MLX_ARCHIVE_MISSING_SERVER'], ['mlx-server did not respond to --help.', 'MLX_PROBE_HELP_TIMEOUT'], ['mlx-server could not be started.', 'MLX_PROBE_START_FAILED'], + ['The Foundry Local library does not load on this computer.', 'FOUNDRY_LOCAL_LIBRARY_NOT_LOADED'], + ['The Foundry Local host stopped before the model was ready.', 'FOUNDRY_LOCAL_HOST_STOPPED_EARLY'], ]) const SENTRY_AGENT_CAUSES = new Set([ 'ENOENT', From f528895d598313ad2adfa80e0ba787fb7bccd8bc Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 18 Nov 2026 14:32:52 +0300 Subject: [PATCH 2/2] fix(telemetry): tag Foundry host load failures at their source --- ...cal-host-load-failures-get-finite-codes.md | 34 +++++-- src/contracts/diagnostics.ts | 2 + .../foundry-local/host-process.test.ts | 13 ++- src/runtime/foundry-local/runtime.test.ts | 22 +++++ src/runtime/foundry-local/runtime.ts | 14 ++- .../sentry-agent-diagnostics.test.ts | 96 ------------------- src/telemetry/sentry-agent-diagnostics.ts | 4 +- test/helpers/fake-foundry-host.mjs | 6 +- 8 files changed, 79 insertions(+), 112 deletions(-) diff --git a/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md index ccc9314e..e9546267 100644 --- a/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md +++ b/docs/decisions/2026-11-17-foundry-local-host-load-failures-get-finite-codes.md @@ -1,12 +1,32 @@ --- date: 2026-11-17 -title: "Tag Foundry Local host load failures with finite diagnostic codes" +title: "Tag Foundry Local host load failures at their source" --- -# 2026-11-17 — Tag Foundry Local host load failures with finite diagnostic codes +# 2026-11-17 — Tag Foundry Local host load failures at their source -- **Context:** `FOUNDRY_LOCAL_HOST_ERROR` events from `atomic-chat-core@0.23.4` carried the generic exception value `runtime load: unknown`, `failure_category=unknown`, `failure_reason=unknown` and no exit code or signal. `hostError` in `src/runtime/foundry-local/runtime.ts` collapses four distinct host outcomes into one error code: exit 5 becomes `FOUNDRY_LOCAL_ACCELERATOR_NOT_PREPARED`, exit 3 becomes "The Foundry Local library does not load on this computer.", and every other exit or signal death becomes "The Foundry Local host stopped before the model was ready.". `loadFailureReport` does not tag the host exit code or stderr, so an event cannot separate a native library that fails to load on Windows from a library call that failed after loading. The GenieX host already had the identical pair of codes; Foundry Local had none. -- **Decision:** Add two exact-message rules to the bounded diagnostic observer (`src/telemetry/sentry-agent-diagnostics.ts`, generated): "The Foundry Local library does not load on this computer." → `FOUNDRY_LOCAL_LIBRARY_NOT_LOADED` and "The Foundry Local host stopped before the model was ready." → `FOUNDRY_LOCAL_HOST_STOPPED_EARLY`, sourced from `src/runtime/foundry-local/runtime.ts`. Cause codes stay enabled. Nothing else changes: no new report, no severity, consent, throttling, grouping or exit-behavior change, and the error the caller receives is byte-identical. -- **Consequences:** The next released events from this path separate "the native library cannot load on this machine" from "the library loaded and the host then stopped before readiness". The observer still cannot say whether the host died from a signal or exited with a code, because the report tags never carried them; adding those facts is a separate reporting decision. Both rules are removable once the split is established, or once the exit code and signal reach the report directly. -- **Owner:** team -- **Links:** Sentry `ATOMIC-CHAT-CORE-CM` (event `ced119d9c8434e5e9a41fa0e30013202`), `src/runtime/foundry-local/runtime.ts`, `src/runtime/foundry-local/host/run.ts`, `src/telemetry/diagnostics.ts`, `src/telemetry/sentry-agent-diagnostics.test.ts` +## Context + +Foundry Local emits `FOUNDRY_LOCAL_HOST_ERROR` for several host exits before model readiness. +An exit with code 3 means library loading failed or the native API was incompatible. Other +early exits include failures before the library loads, so the generic host error cannot establish +which startup stage the process reached. The install-time probe uses the same library message +when it returns an unreadable response, even with an exit code of zero. + +## Decision + +In the runtime's exit handler, use the existing `operation_detail` tag to distinguish +`foundry_local_library_not_loaded` for exit 3 and `foundry_local_host_stopped_before_ready` +for the remaining `FOUNDRY_LOCAL_HOST_ERROR` exits. Exit 5 keeps its accelerator refusal. +The install-time probe does not use either load detail. + +## Consequences + +Sentry can split existing Foundry model-load failures by `operation_detail`, while clients keep +the same error code, message and details. The tag contains a fixed value from the diagnostic +contract. Grouping and severity retain their existing rules. + +Owner: team + +References: `src/runtime/foundry-local/runtime.ts`, `src/runtime/foundry-local/host-process.ts`, +Sentry `ATOMIC-CHAT-CORE-CM`. diff --git a/src/contracts/diagnostics.ts b/src/contracts/diagnostics.ts index b7c0cf43..90ce65ab 100644 --- a/src/contracts/diagnostics.ts +++ b/src/contracts/diagnostics.ts @@ -40,6 +40,8 @@ export const FAILURE_OPERATION_DETAILS = [ 'mlx_explicit_binary_missing', 'mlx_resolved_binary_missing', 'mlx_resources_binary_missing', + 'foundry_local_library_not_loaded', + 'foundry_local_host_stopped_before_ready', ] as const export const FAILURE_REASONS = [ 'cancelled', diff --git a/src/runtime/foundry-local/host-process.test.ts b/src/runtime/foundry-local/host-process.test.ts index f5a59cc4..019e0b93 100644 --- a/src/runtime/foundry-local/host-process.test.ts +++ b/src/runtime/foundry-local/host-process.test.ts @@ -2,6 +2,7 @@ import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { classifyFailure, diagnosticTags } from '../../telemetry/index.js' import { foundryHostCommand, runFoundryHostProbe } from './host-process.js' const POSIX = process.platform !== 'win32' @@ -58,11 +59,17 @@ describe.skipIf(!POSIX)('runFoundryHostProbe', () => { it('fails when the host prints no probe answer', async () => { const exe = await fakeHost(`echo 'not json'`) - await expect( - runFoundryHostProbe({ exe, args: [] }, { libDir: '/l', appDataDir: '/a', logsDir: '/g' }) - ).rejects.toMatchObject({ + const error: unknown = await runFoundryHostProbe( + { exe, args: [] }, + { libDir: '/l', appDataDir: '/a', logsDir: '/g' } + ).catch((cause: unknown) => cause) + expect(error).toMatchObject({ code: 'ENGINE_INSTALL_FAILED', + message: 'The Foundry Local library does not load on this computer.', }) + const tags = diagnosticTags(error, classifyFailure(error, { component: 'backend', operation: 'install' })) + expect(tags).not.toHaveProperty('operation_detail') + expect(tags).not.toHaveProperty('diagnostic_code') }) it('stops a host that does not answer in time', async () => { diff --git a/src/runtime/foundry-local/runtime.test.ts b/src/runtime/foundry-local/runtime.test.ts index bd31afdb..ca08733e 100644 --- a/src/runtime/foundry-local/runtime.test.ts +++ b/src/runtime/foundry-local/runtime.test.ts @@ -8,6 +8,7 @@ import { makeTmpDataFolder } from '../../../test/helpers/tmp-data-folder.js' import type { TmpDataFolder } from '../../../test/helpers/tmp-data-folder.js' import type { CoreEvents } from '../../contracts/index.js' import { ModelRegistry, serializeModelYml } from '../../models/index.js' +import { diagnosticTags, loadFailureReport } from '../../telemetry/index.js' import { spawnAndAwaitReady } from '../shared/index.js' import type { GpuClaimHook } from '../shared/index.js' import { FoundryLocalRuntime } from './runtime.js' @@ -361,6 +362,27 @@ describe.skipIf(!POSIX)('FoundryLocalRuntime (task 6.3)', () => { }) }) + it.each([ + ['exit-3', 'foundry_local_library_not_loaded'], + ['exit-4', 'foundry_local_host_stopped_before_ready'], + ])('reports the Foundry host load detail for %s', async (mode, detail) => { + const modelId = 'foundry-diagnostic' + await variant(modelId, { execution_provider: 'webgpu', device: 'gpu' }) + const error: unknown = await runtime({ mode }) + .rt.load(modelId) + .catch((cause: unknown) => cause) + expect(error).toMatchObject({ code: 'FOUNDRY_LOCAL_HOST_ERROR' }) + const report = loadFailureReport({ provider: 'foundry-local', modelId, error }) + if (!report?.diagnostic) throw new Error('Expected a Foundry host load report') + expect(report.level).toBe('error') + expect(diagnosticTags(report.error, report.diagnostic)).toMatchObject({ + component: 'runtime', + operation: 'load', + failure_reason: 'unknown', + operation_detail: detail, + }) + }) + it('holds every card for a GPU variant and none for a CPU or NPU one, claiming the GPU only for the former', async () => { await variant('webgpu-variant', { execution_provider: 'webgpu', device: 'gpu' }) await variant('cpu-variant', { execution_provider: 'cpu', device: 'cpu' }) diff --git a/src/runtime/foundry-local/runtime.ts b/src/runtime/foundry-local/runtime.ts index 0862d89f..eebd8bb5 100644 --- a/src/runtime/foundry-local/runtime.ts +++ b/src/runtime/foundry-local/runtime.ts @@ -132,12 +132,22 @@ function hostError(exitCode: number | null, stderr: string): AtomicCoreError { 'The accelerator of this model is not prepared; prepare it on the Foundry Local page first.', said ) + const libraryDidNotLoad = exitCode === 3 return new AtomicCoreError( 'FOUNDRY_LOCAL_HOST_ERROR', - exitCode === 3 + libraryDidNotLoad ? 'The Foundry Local library does not load on this computer.' : 'The Foundry Local host stopped before the model was ready.', - said || `exit code ${String(exitCode)}` + said || `exit code ${String(exitCode)}`, + { + diagnostic: { + component: 'runtime', + operation: 'load', + operationDetail: libraryDidNotLoad + ? 'foundry_local_library_not_loaded' + : 'foundry_local_host_stopped_before_ready', + }, + } ) } diff --git a/src/telemetry/sentry-agent-diagnostics.test.ts b/src/telemetry/sentry-agent-diagnostics.test.ts index 908e256f..401f2501 100644 --- a/src/telemetry/sentry-agent-diagnostics.test.ts +++ b/src/telemetry/sentry-agent-diagnostics.test.ts @@ -231,99 +231,3 @@ describe('MLX install probe bounded diagnostics', () => { expect(error.message).toBe('The downloaded mlx-server does not run on this Mac.') }) }) - -describe('Foundry Local host load bounded diagnostics', () => { - const load = { component: 'runtime', operation: 'load' } as const - - it('tags Foundry Local host load failures with finite diagnostic codes', () => { - const libraryLoadError = new AtomicCoreError( - 'FOUNDRY_LOCAL_HOST_ERROR', - 'The Foundry Local library does not load on this computer.', - '/private/fixture/foundry-host: exit code 3' - ) - const libraryTags = diagnosticTags(libraryLoadError, classifyFailure(libraryLoadError, load)) - expect(libraryTags).toMatchObject({ - diagnostic_version: 'core-v1', - component: 'runtime', - operation: 'load', - failure_category: 'unknown', - failure_reason: 'unknown', - cause_code: 'FOUNDRY_LOCAL_HOST_ERROR', - diagnostic_code: 'FOUNDRY_LOCAL_LIBRARY_NOT_LOADED', - }) - expect(libraryTags['diagnostic_cause_code']).toBeUndefined() - expect(JSON.stringify(libraryTags)).not.toContain('/private') - expect(libraryLoadError.toJSON()).toEqual({ - code: 'FOUNDRY_LOCAL_HOST_ERROR', - message: 'The Foundry Local library does not load on this computer.', - details: '/private/fixture/foundry-host: exit code 3', - }) - - const cause = Object.assign(new Error('/private/fixture/onnxruntime-genai-cuda.dll missing'), { - code: 'ENOENT', - }) - const stoppedError = new AtomicCoreError( - 'FOUNDRY_LOCAL_HOST_ERROR', - 'The Foundry Local host stopped before the model was ready.', - '/private/fixture/model: exit code 4', - { cause } - ) - const stoppedTags = diagnosticTags(stoppedError, classifyFailure(stoppedError, load)) - expect(stoppedTags).toMatchObject({ - diagnostic_version: 'core-v1', - component: 'runtime', - operation: 'load', - failure_category: 'unknown', - failure_reason: 'unknown', - cause_code: 'ENOENT', - diagnostic_code: 'FOUNDRY_LOCAL_HOST_STOPPED_EARLY', - diagnostic_cause_code: 'ENOENT', - }) - expect(JSON.stringify(stoppedTags)).not.toContain('/private') - expect(JSON.stringify(stoppedTags)).not.toContain('onnxruntime-genai-cuda') - }) - - it('leaves other Foundry Local refusals untagged and keeps cyclic causes private', () => { - const accelerator = new AtomicCoreError( - 'FOUNDRY_LOCAL_ACCELERATOR_NOT_PREPARED', - 'The accelerator of this model is not prepared; prepare it on the Foundry Local page first.' - ) - const acceleratorTags = diagnosticTags(accelerator, classifyFailure(accelerator, load)) - expect(acceleratorTags['diagnostic_code']).toBeUndefined() - expect(acceleratorTags['diagnostic_cause_code']).toBeUndefined() - expect(JSON.stringify(acceleratorTags)).not.toContain('prepare it on the Foundry Local page') - - const cycleA = Object.assign(new Error('/private/fixture/cycle-a'), { code: 'PRIVATE_CODE' }) - const cycleB = new Error('/private/fixture/cycle-b', { cause: cycleA }) - cycleA.cause = cycleB - const cycleTags = diagnosticTags(cycleA, classifyFailure(cycleA, load)) - expect(cycleTags['diagnostic_code']).toBeUndefined() - expect(cycleTags['diagnostic_cause_code']).toBeUndefined() - expect(JSON.stringify(cycleTags)).not.toContain('/private') - expect(JSON.stringify(cycleTags)).not.toContain('PRIVATE_CODE') - }) - - it('reports the host refusal as before, only adding finite tags', () => { - const capture = vi.fn() - const observe = failureObserver({ capture }) - const error = new AtomicCoreError( - 'FOUNDRY_LOCAL_HOST_ERROR', - 'The Foundry Local host stopped before the model was ready.', - '/private/fixture/model: exit code 4' - ) - observe(error, load) - expect(capture).toHaveBeenCalledTimes(1) - expect(capture).toHaveBeenCalledWith( - expect.objectContaining({ - source: 'operation', - error, - level: 'error', - }) - ) - expect(error.toJSON()).toEqual({ - code: 'FOUNDRY_LOCAL_HOST_ERROR', - message: 'The Foundry Local host stopped before the model was ready.', - details: '/private/fixture/model: exit code 4', - }) - }) -}) diff --git a/src/telemetry/sentry-agent-diagnostics.ts b/src/telemetry/sentry-agent-diagnostics.ts index 18ec1513..a9714747 100644 --- a/src/telemetry/sentry-agent-diagnostics.ts +++ b/src/telemetry/sentry-agent-diagnostics.ts @@ -1,4 +1,4 @@ -// SENTRY_AGENT_DIAGNOSTIC_CONFIG {"rules":[{"source":"src/diffusion/install.ts","message":"The image engine could not load a library it needs.","code":"SDCPP_PROBE_LIBRARY_LOAD_FAILED"},{"source":"src/diffusion/install.ts","message":"The image engine exited without printing anything.","code":"SDCPP_PROBE_SILENT_EXIT"},{"source":"src/diffusion/install.ts","message":"The downloaded binary is not stable-diffusion.cpp.","code":"SDCPP_PROBE_UNEXPECTED_OUTPUT"},{"source":"src/diffusion/install.ts","message":"The image engine did not respond to --help.","code":"SDCPP_PROBE_HELP_TIMEOUT"},{"source":"src/diffusion/install.ts","message":"The image engine could not be started.","code":"SDCPP_PROBE_START_FAILED"},{"source":"src/diffusion/install.ts","message":"The archive did not contain sd-server.","code":"SDCPP_ARCHIVE_MISSING_SERVER"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX library does not load on this computer.","code":"GENIEX_HOST_LIBRARY_LOAD_FAILED"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX host stopped before the model was ready.","code":"GENIEX_HOST_STOPPED_BEFORE_READY"},{"source":"src/engine-builds/builds.ts","message":"The downloaded mlx-server does not run on this Mac.","code":"MLX_PROBE_DOES_NOT_RUN"},{"source":"src/engine-builds/builds.ts","message":"The archive did not contain mlx-server.","code":"MLX_ARCHIVE_MISSING_SERVER"},{"source":"src/engine-builds/builds.ts","message":"mlx-server did not respond to --help.","code":"MLX_PROBE_HELP_TIMEOUT"},{"source":"src/engine-builds/builds.ts","message":"mlx-server could not be started.","code":"MLX_PROBE_START_FAILED"},{"source":"src/runtime/foundry-local/runtime.ts","message":"The Foundry Local library does not load on this computer.","code":"FOUNDRY_LOCAL_LIBRARY_NOT_LOADED"},{"source":"src/runtime/foundry-local/runtime.ts","message":"The Foundry Local host stopped before the model was ready.","code":"FOUNDRY_LOCAL_HOST_STOPPED_EARLY"}],"include_cause_codes":true} +// SENTRY_AGENT_DIAGNOSTIC_CONFIG {"rules":[{"source":"src/diffusion/install.ts","message":"The image engine could not load a library it needs.","code":"SDCPP_PROBE_LIBRARY_LOAD_FAILED"},{"source":"src/diffusion/install.ts","message":"The image engine exited without printing anything.","code":"SDCPP_PROBE_SILENT_EXIT"},{"source":"src/diffusion/install.ts","message":"The downloaded binary is not stable-diffusion.cpp.","code":"SDCPP_PROBE_UNEXPECTED_OUTPUT"},{"source":"src/diffusion/install.ts","message":"The image engine did not respond to --help.","code":"SDCPP_PROBE_HELP_TIMEOUT"},{"source":"src/diffusion/install.ts","message":"The image engine could not be started.","code":"SDCPP_PROBE_START_FAILED"},{"source":"src/diffusion/install.ts","message":"The archive did not contain sd-server.","code":"SDCPP_ARCHIVE_MISSING_SERVER"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX library does not load on this computer.","code":"GENIEX_HOST_LIBRARY_LOAD_FAILED"},{"source":"src/runtime/geniex/runtime.ts","message":"The GenieX host stopped before the model was ready.","code":"GENIEX_HOST_STOPPED_BEFORE_READY"},{"source":"src/engine-builds/builds.ts","message":"The downloaded mlx-server does not run on this Mac.","code":"MLX_PROBE_DOES_NOT_RUN"},{"source":"src/engine-builds/builds.ts","message":"The archive did not contain mlx-server.","code":"MLX_ARCHIVE_MISSING_SERVER"},{"source":"src/engine-builds/builds.ts","message":"mlx-server did not respond to --help.","code":"MLX_PROBE_HELP_TIMEOUT"},{"source":"src/engine-builds/builds.ts","message":"mlx-server could not be started.","code":"MLX_PROBE_START_FAILED"}],"include_cause_codes":true} // This generated observer only enriches an existing error event. It never sends or logs. const SENTRY_AGENT_MESSAGES = new Map([ ['The image engine could not load a library it needs.', 'SDCPP_PROBE_LIBRARY_LOAD_FAILED'], @@ -13,8 +13,6 @@ const SENTRY_AGENT_MESSAGES = new Map([ ['The archive did not contain mlx-server.', 'MLX_ARCHIVE_MISSING_SERVER'], ['mlx-server did not respond to --help.', 'MLX_PROBE_HELP_TIMEOUT'], ['mlx-server could not be started.', 'MLX_PROBE_START_FAILED'], - ['The Foundry Local library does not load on this computer.', 'FOUNDRY_LOCAL_LIBRARY_NOT_LOADED'], - ['The Foundry Local host stopped before the model was ready.', 'FOUNDRY_LOCAL_HOST_STOPPED_EARLY'], ]) const SENTRY_AGENT_CAUSES = new Set([ 'ENOENT', diff --git a/test/helpers/fake-foundry-host.mjs b/test/helpers/fake-foundry-host.mjs index ab6dde82..b943ef05 100644 --- a/test/helpers/fake-foundry-host.mjs +++ b/test/helpers/fake-foundry-host.mjs @@ -4,7 +4,7 @@ * arguments, readiness line, `/catalogs/local/v1/*` routes, registration-id model matching and stop * on SIGTERM or stdin end as `src/runtime/foundry-local/host/run.ts` with a real library. * - * FAKE_FOUNDRY_MODE ready (default) | echo-chat-request | crash-after-ready | hang | exit-3 | exit-5 + * FAKE_FOUNDRY_MODE ready (default) | echo-chat-request | crash-after-ready | hang | exit-3 | exit-4 | exit-5 * FAKE_FOUNDRY_ARGV a file each start appends {argv, ort_telemetry_disabled, * ort_enable_cudnn_flash_attention, ci} to */ @@ -90,6 +90,10 @@ if (mode === 'exit-3') { console.error('foundry_local does not load from /b: libatomic.so.1: cannot open shared object file') process.exit(3) } +if (mode === 'exit-4') { + console.error('Foundry library call failed before readiness') + process.exit(4) +} const port = Number(arg('--port')) const id = arg('--id')