From 39810b6d6a4b2ee06384216c55d1413c16e58f03 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 2 Oct 2026 08:01:39 +0000 Subject: [PATCH] fix(desktop): recover when a wsl-only backend never becomes ready After three consecutive pre-ready crashes or readiness timeouts, stop the restart loop and use the Windows backend for this launch. Restarts stay suppressed while that recovery is pending, and the dialog only reports a bounded failure category. Co-authored-by: maco --- .../src/backend/DesktopBackendManager.test.ts | 345 ++++++++++++++++++ .../src/backend/DesktopBackendManager.ts | 344 ++++++++++++++++- .../src/backend/DesktopBackendPool.test.ts | 186 ++++++++++ .../desktop/src/backend/DesktopBackendPool.ts | 57 ++- 4 files changed, 920 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 901d9f4a2708..cf70f9e179fa 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -126,6 +126,11 @@ interface MakeInstanceInput { readonly onPreflightFailed?: ( failure: DesktopBackendManager.PreflightFailure, ) => Effect.Effect; + readonly onStartupFailed?: ( + failure: DesktopBackendManager.StartupFailure, + runningDistro: string | undefined, + ) => Effect.Effect>>; + readonly readinessTimeout?: Duration.Duration; readonly config?: DesktopBackendManager.DesktopBackendStartConfig; readonly configResolve?: Effect.Effect< DesktopBackendManager.DesktopBackendStartConfig, @@ -185,6 +190,8 @@ function makeTestInstance(input: MakeInstanceInput) { ...(input.onReady ? { onReady: () => input.onReady! } : {}), ...(input.onShutdown ? { onShutdown: () => input.onShutdown! } : {}), ...(input.onPreflightFailed ? { onPreflightFailed: input.onPreflightFailed } : {}), + ...(input.onStartupFailed ? { onStartupFailed: input.onStartupFailed } : {}), + ...(input.readinessTimeout === undefined ? {} : { readinessTimeout: input.readinessTimeout }), }); return instance.pipe(Effect.provide(servicesLayer)); @@ -777,6 +784,7 @@ describe("DesktopBackendManager", () => { onReadinessFailure: () => Effect.sync(() => { readinessTimeoutCount += 1; + return false; }), }).pipe(Effect.provide(Layer.merge(spawnerLayer, httpLayer)), Effect.forkChild); @@ -1575,4 +1583,341 @@ describe("DesktopBackendManager", () => { }).pipe(Effect.provide(TestClock.layer())), ), ); + + it.effect( + "uses the startup failure hook after three pre-ready exits and does not restart while it is pending", + () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const failures = yield* Queue.unbounded(); + const hooks = yield* Queue.unbounded(); + const gate = yield* Deferred.make(); + const mode = yield* Ref.make<"wsl" | "windows">("wsl"); + const wslConfig: DesktopBackendManager.DesktopBackendStartConfig = { + ...baseConfig, + runningDistro: "Ubuntu-22.04", + httpBaseUrl: new URL("http://10.0.0.8:3773"), + }; + const windowsConfig: DesktopBackendManager.DesktopBackendStartConfig = { + ...baseConfig, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const kind = yield* Ref.get(mode); + return makeProcess({ + exitCode: Queue.offer(starts, kind).pipe( + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + httpClientLayer: httpClientLayer(() => Effect.never), + configResolve: Ref.get(mode).pipe( + Effect.map((kind) => (kind === "wsl" ? wslConfig : windowsConfig)), + ), + backendOutputLog: { + persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid), + }, + onStartupFailed: (failure) => + Effect.gen(function* () { + yield* Queue.offer(hooks, failure); + yield* Deferred.await(gate); + yield* Ref.set(mode, "windows"); + return Option.some(Effect.void); + }), + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(starts), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(starts), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + + const failure = yield* Queue.take(hooks); + assert.deepEqual(failure, { kind: "exited", exitCode: 1 }); + yield* TestClock.adjust(Duration.seconds(30)); + assert.equal(yield* Queue.size(starts), 0); + + yield* Deferred.succeed(gate, undefined); + assert.equal(yield* Queue.take(starts), "windows"); + const running = yield* instance.snapshot; + assert.equal(running.desiredRunning, true); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("keeps restarting when the startup failure hook declines recovery", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const failures = yield* Queue.unbounded(); + const hooks = yield* Queue.unbounded(); + let startCount = 0; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + return makeProcess({ + exitCode: Queue.offer(starts, startCount).pipe( + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + httpClientLayer: httpClientLayer(() => Effect.never), + backendOutputLog: { + persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid), + }, + onStartupFailed: () => + Effect.sync(() => { + return Option.none>(); + }).pipe(Effect.tap(() => Queue.offer(hooks, startCount))), + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(starts), 2); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(starts), 3); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.equal(yield* Queue.take(hooks), 3); + yield* TestClock.adjust(Duration.seconds(2)); + assert.equal(yield* Queue.take(starts), 4); + assert.equal(yield* Queue.size(hooks), 0); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("does not run the startup failure hook for exits after the backend was ready", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const readies = yield* Queue.unbounded(); + const exited = yield* Queue.unbounded(); + const release = yield* Queue.unbounded(); + let startCount = 0; + let hookCalls = 0; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + const pid = startCount; + return makeProcess({ + exitCode: Queue.offer(starts, pid).pipe( + Effect.andThen(Queue.take(release)), + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + onReady: Effect.sync(() => startCount).pipe( + Effect.flatMap((pid) => Queue.offer(readies, pid)), + Effect.asVoid, + ), + backendOutputLog: { + persistFailure: () => Queue.offer(exited, startCount).pipe(Effect.asVoid), + }, + onStartupFailed: () => + Effect.sync(() => { + hookCalls += 1; + return Option.none>(); + }), + }); + + yield* instance.start; + for (const delay of [ + Duration.zero, + Duration.millis(500), + Duration.seconds(1), + Duration.seconds(2), + Duration.seconds(4), + ]) { + if (Duration.toMillis(delay) > 0) { + yield* TestClock.adjust(delay); + } + const pid = yield* Queue.take(starts); + assert.equal(yield* Queue.take(readies), pid); + yield* Queue.offer(release, undefined); + assert.equal(yield* Queue.take(exited), pid); + } + assert.equal(hookCalls, 0); + assert.equal(startCount, 5); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("does not apply startup recovery when stop interrupts the pending hook", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const failures = yield* Queue.unbounded(); + const hooks = yield* Queue.unbounded(); + const gate = yield* Deferred.make(); + let startCount = 0; + let applied = false; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + return makeProcess({ + exitCode: Queue.offer(starts, startCount).pipe( + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + httpClientLayer: httpClientLayer(() => Effect.never), + backendOutputLog: { + persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid), + }, + onStartupFailed: () => + Effect.gen(function* () { + yield* Queue.offer(hooks, "entered"); + yield* Deferred.await(gate); + return Option.some( + Effect.sync(() => { + applied = true; + }), + ); + }), + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(starts), 2); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(starts), 3); + assert.equal(yield* Queue.take(hooks), "entered"); + + yield* instance.stop(); + yield* Deferred.succeed(gate, undefined); + assert.equal(applied, false); + yield* TestClock.adjust(Duration.seconds(30)); + assert.equal(yield* Queue.size(starts), 0); + const stopped = yield* instance.snapshot; + assert.equal(stopped.desiredRunning, false); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect( + "falls back when a live backend exhausts three readiness budgets and does not restart early", + () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const timeouts = yield* Queue.unbounded(); + const hooks = yield* Queue.unbounded(); + const gate = yield* Deferred.make(); + const mode = yield* Ref.make<"wsl" | "windows">("wsl"); + const exit = yield* Deferred.make(); + const wslConfig: DesktopBackendManager.DesktopBackendStartConfig = { + ...baseConfig, + runningDistro: "Ubuntu-22.04", + }; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const kind = yield* Ref.get(mode); + if (kind === "windows") { + return makeProcess({ + exitCode: Queue.offer(starts, kind).pipe(Effect.andThen(Effect.never)), + }); + } + return makeProcess({ + exitCode: Queue.offer(starts, kind).pipe(Effect.andThen(Deferred.await(exit))), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(1)).pipe(Effect.asVoid), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + readinessTimeout: Duration.millis(50), + httpClientLayer: httpClientLayer((request) => + Effect.succeed(responseForRequest(request, 503)), + ), + configResolve: Ref.get(mode).pipe( + Effect.map((kind) => + kind === "wsl" + ? wslConfig + : { + ...baseConfig, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }, + ), + ), + backendOutputLog: { + persistFailureSnapshot: ({ details }) => + Queue.offer(timeouts, details).pipe(Effect.asVoid), + }, + onStartupFailed: (failure) => + Effect.gen(function* () { + yield* Queue.offer(hooks, failure); + yield* Deferred.await(gate); + yield* Ref.set(mode, "windows"); + return Option.some(Effect.void); + }), + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), "wsl"); + yield* TestClock.adjust(Duration.millis(50)); + assert.equal(yield* Queue.take(timeouts), "readiness-timeout"); + yield* TestClock.adjust(Duration.millis(50)); + assert.equal(yield* Queue.take(timeouts), "readiness-timeout"); + yield* TestClock.adjust(Duration.millis(50)); + assert.equal(yield* Queue.take(timeouts), "readiness-timeout"); + + const failure = yield* Queue.take(hooks); + assert.deepEqual(failure, { kind: "unreachable" }); + yield* TestClock.adjust(Duration.seconds(30)); + assert.equal(yield* Queue.size(starts), 0); + + yield* Deferred.succeed(gate, undefined); + assert.equal(yield* Queue.take(starts), "windows"); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 6f1ea139bdca..9a72f47a9f79 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -61,6 +61,9 @@ const MAX_RESTART_DELAY = Duration.seconds(10); // failures may instead provide their own larger retryLimit when they should // self-heal for a while but must not leave the app connecting forever. const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5; +// Preflight can pass and the child can still die or never answer. Cap those +// consecutive pre-ready failures, then let onStartupFailed recover once. +const MAX_STARTUP_FAILURE_ATTEMPTS = 3; const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1); const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100); const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1); @@ -117,6 +120,27 @@ export interface PreflightFailure { readonly retryLimit?: number; } +// A startup failure is categorized before it reaches dialogs or logs. `exited` +// is a child that ended before it was ever ready. `unreachable` is a child that +// stayed up while a readiness budget ran out. `exitCode` is only set for a +// numeric status in 0..255; process output and readiness URLs are not included. +export interface StartupFailure { + readonly kind: "exited" | "unreachable"; + readonly exitCode?: number; +} + +const exitedFailure = (reason: string): StartupFailure => { + const match = /^code=(\d+)$/.exec(reason); + if (match === null) { + return { kind: "exited" }; + } + const exitCode = Number(match[1]); + if (!Number.isInteger(exitCode) || exitCode < 0 || exitCode > 255) { + return { kind: "exited" }; + } + return { kind: "exited", exitCode }; +}; + interface BackendProcessExit { readonly code: Option.Option; readonly reason: string; @@ -230,7 +254,9 @@ interface RunBackendProcessOptions extends DesktopBackendStartConfig { readonly onStarted?: (pid: number) => Effect.Effect; readonly onExitObserved?: () => Effect.Effect; readonly onReady?: () => Effect.Effect; - readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect; + // True asks the supervisor to stop probing and kill the child. The exit then + // owns recovery, so a live-but-unreachable backend uses the same path as a crash. + readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect; readonly onOutput?: ( streamName: BackendProcessOutputStream, chunk: Uint8Array, @@ -299,6 +325,16 @@ export interface BackendInstanceSpec { // retries. Returns true when the callback changed configuration and the // manager should resolve once more; false stops the failed instance. readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect; + // Fired once MAX_STARTUP_FAILURE_ATTEMPTS consecutive pre-ready failures + // have accumulated. `Some(recovery)` runs recovery and then starts again so + // the next configResolve sees it. `None` keeps the normal restart loop. + // Exits after a successful ready do not count. + readonly onStartupFailed?: ( + failure: StartupFailure, + runningDistro: string | undefined, + ) => Effect.Effect>>; + // Overrides the per-round readiness budget. Production uses the default. + readonly readinessTimeout?: Duration.Duration; } interface ActiveBackendRun { @@ -319,6 +355,16 @@ interface BackendManagerState { // Consecutive bounded/fatal preflight failures, reset on a clean or // unbounded-transient preflight. restartAttempt counts all restarts. readonly preflightFailureAttempt: number; + // Consecutive pre-ready exits and readiness timeouts. Reset once ready. + readonly startupFailureAttempt: number; + // True from the moment the cap is claimed until the hook finishes. Restarts + // are suppressed for the whole window so a later exit cannot spawn ahead of + // the hook and leave its replacement pointing at a stale run. + readonly startupFailurePending: boolean; + readonly startupFailureNotice: Option.Option; + readonly startupFailureFiber: Option.Option>; + // Bumped by every stop(), so a recovery start can tell a quit landed first. + readonly stopGeneration: number; readonly restartFiber: Option.Option>; readonly nextRunId: number; } @@ -330,10 +376,27 @@ const initialState: BackendManagerState = { active: Option.none(), restartAttempt: 0, preflightFailureAttempt: 0, + startupFailureAttempt: 0, + startupFailurePending: false, + startupFailureNotice: Option.none(), + startupFailureFiber: Option.none(), + stopGeneration: 0, restartFiber: Option.none(), nextRunId: 1, }; +const withoutStartupFailure = (latest: BackendManagerState): BackendManagerState => + latest.startupFailurePending || + Option.isSome(latest.startupFailureNotice) || + Option.isSome(latest.startupFailureFiber) + ? { + ...latest, + startupFailurePending: false, + startupFailureNotice: Option.none(), + startupFailureFiber: Option.none(), + } + : latest; + const activePid = (active: Option.Option): Option.Option => Option.flatMap(active, (run) => run.pid); @@ -588,7 +651,16 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( Effect.as(true), Effect.catchTags({ BackendReadinessTimeoutError: (error) => - (options.onReadinessFailure?.(error) ?? Effect.void).pipe(Effect.as(false)), + Effect.gen(function* () { + const giveUp = yield* options.onReadinessFailure?.(error) ?? Effect.succeed(false); + if (!giveUp) { + return false; + } + // End this run so finalizeRun can recover. Killing from the probe + // avoids stopping the instance from inside its own fiber. + yield* handle.kill().pipe(Effect.ignore); + return true; + }), }), ), ); @@ -655,8 +727,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const httpClient = yield* HttpClient.HttpClient; const state = yield* Ref.make(initialState); const mutex = yield* Semaphore.make(1); + // Set by the startup-failure hook immediately before its recovery start. + // start() consumes it under the mutex and no-ops when stop() has moved on. + const replacementStopGenerationRef = yield* Ref.make(Option.none()); - const { logWarning: logInstanceWarning, logError: logInstanceError } = + const { annotate: annotateInstanceLog, logError: logInstanceError } = DesktopObservability.makeComponentLogger(`desktop-backend-instance:${spec.id}`); const updateActiveRun = (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) => @@ -692,6 +767,21 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( mutex.withPermits(1)( Effect.gen(function* () { const current = yield* Ref.get(state); + const replacementStopGeneration = yield* Ref.getAndSet( + replacementStopGenerationRef, + Option.none(), + ); + if (Option.isSome(replacementStopGeneration)) { + if (current.stopGeneration !== replacementStopGeneration.value) { + yield* Ref.update(state, withoutStartupFailure); + return; + } + yield* Ref.update(state, withoutStartupFailure); + } else if (current.startupFailurePending) { + // The hook owns the next spawn. Starting here would resolve the old + // config and make the hook's later start a second process. + return; + } if (Option.isSome(current.active)) { if (!current.desiredRunning) { yield* Ref.update(state, (latest) => ({ @@ -735,6 +825,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( ready: false, config: Option.some(config.value), preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt, + // A user-driven start gets a fresh startup budget. Restart-loop + // starts keep the streak so repeated crashes can reach the cap. + startupFailureAttempt: current.desiredRunning ? latest.startupFailureAttempt : 0, })); const preflightFailure = config.value.preflightFailure; @@ -829,6 +922,14 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* ( reason: string, ) { + // Armed after this mutex is released. Recovery calls start(), which + // takes the mutex, so the hook cannot be forked while it is held. + let startupFailureToArm: + | { + readonly failure: StartupFailure; + readonly runningDistro: string | undefined; + } + | undefined; yield* mutex.withPermits(1)( Effect.gen(function* () { const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } = @@ -897,15 +998,55 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( } } - if (isCurrentRun && nextState.desiredRunning) { + // A pending startup-failure hook owns the next spawn. Scheduling + // here would start another run before that hook applies its + // fallback. + let suppressRestartForStartupFailure = false; + if (isCurrentRun && !stopRequested && !wasReady) { + const pending = yield* Ref.get(state); + if (pending.startupFailurePending && Option.isSome(pending.startupFailureNotice)) { + suppressRestartForStartupFailure = true; + startupFailureToArm = { + failure: pending.startupFailureNotice.value, + runningDistro: config.value.runningDistro, + }; + } else { + const failure = exitedFailure(reason); + if (yield* claimStartupFailure(failure)) { + suppressRestartForStartupFailure = true; + startupFailureToArm = { + failure, + runningDistro: config.value.runningDistro, + }; + } + } + } else if (isCurrentRun && (yield* Ref.get(state)).startupFailurePending) { + suppressRestartForStartupFailure = true; + } + + if ( + isCurrentRun && + nextState.desiredRunning && + !suppressRestartForStartupFailure && + !(yield* Ref.get(state)).startupFailurePending + ) { yield* scheduleRestart(reason); } }), ); + if (startupFailureToArm !== undefined) { + yield* armStartupFailure( + startupFailureToArm.failure, + startupFailureToArm.runningDistro, + ); + } }); const program = runBackendProcess({ ...config.value, + ...(spec.readinessTimeout === undefined + ? {} + : { readinessTimeout: spec.readinessTimeout }), desktopTelemetryStream: desktopTelemetryPublisher.encoded, onDesktopTelemetryControl: (message) => desktopTelemetryPublisher.handleControlForSource(spec.id, message), @@ -935,6 +1076,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( { ...latest, restartAttempt: 0, + startupFailureAttempt: 0, ready: true, }, ] as const; @@ -956,12 +1098,31 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( }), onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")( function* (error) { - yield* logInstanceWarning("backend readiness check failed during bootstrap", { - error: error.message, - }); + // Keep the timeout error on the log cause. The annotation stays a + // bounded category so readiness URLs and probe text are not copied + // into structured fields. + yield* annotateInstanceLog( + Effect.logWarning( + "backend readiness check failed during bootstrap", + Cause.fail(error), + ), + { failure: "unreachable" }, + ); yield* backendOutputLog.persistFailureSnapshot({ - details: error.message, + details: "readiness-timeout", }); + return yield* mutex.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(state); + if (Option.getOrUndefined(current.active)?.id !== runId) { + return false; + } + if (!current.desiredRunning || current.ready) { + return false; + } + return yield* claimStartupFailure({ kind: "unreachable" }); + }), + ); }, ), onOutput: (streamName, chunk) => backendOutputLog.writeOutputChunk(streamName, chunk), @@ -989,7 +1150,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( reason: string, ) { const scheduled = yield* Ref.modify(state, (latest) => { - if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) { + if ( + !latest.desiredRunning || + Option.isSome(latest.restartFiber) || + latest.startupFailurePending + ) { return [Option.none(), latest] as const; } @@ -1014,7 +1179,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( Effect.sleep(delay).pipe( Effect.andThen( Ref.modify(state, (latest) => { - const shouldRestart = latest.desiredRunning; + const shouldRestart = latest.desiredRunning && !latest.startupFailurePending; return [ shouldRestart, { @@ -1045,10 +1210,158 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( }); }); + // True only when this pre-ready failure reaches the cap and no hook is running. + const claimStartupFailure = (failure: StartupFailure): Effect.Effect => + Ref.modify(state, (latest) => { + if ( + spec.onStartupFailed === undefined || + latest.ready || + latest.startupFailurePending || + Option.isSome(latest.startupFailureFiber) + ) { + return [false, latest] as const; + } + const next = latest.startupFailureAttempt + 1; + if (next < MAX_STARTUP_FAILURE_ATTEMPTS) { + return [false, { ...latest, startupFailureAttempt: next }] as const; + } + return [ + true, + { + ...latest, + startupFailureAttempt: 0, + startupFailurePending: true, + startupFailureNotice: Option.some(failure), + }, + ] as const; + }); + + const clearTrackedStartupFailure = (tracked: Fiber.Fiber) => + Ref.update(state, (latest) => + Option.isSome(latest.startupFailureFiber) && latest.startupFailureFiber.value === tracked + ? withoutStartupFailure(latest) + : latest, + ); + + // The failed run has already exited. If it is still stopped, continue the + // restart loop now that the hook has declined recovery. + const resumeAfterDeclinedStartupFailure: Effect.Effect = Effect.gen(function* () { + const current = yield* Ref.modify(state, (latest) => { + const next = withoutStartupFailure(latest); + return [next, next] as const; + }); + if ( + current.desiredRunning && + !current.ready && + Option.isNone(current.active) && + Option.isNone(current.restartFiber) + ) { + yield* scheduleRestart("startup-failure"); + } + }); + + const commitStartupRecovery = (recovery: Effect.Effect): Effect.Effect => + Effect.gen(function* () { + const proceed = yield* Ref.modify(state, (latest) => { + if (!latest.startupFailurePending || !latest.desiredRunning || latest.ready) { + return [false, withoutStartupFailure(latest)] as const; + } + return [true, latest] as const; + }); + if (!proceed) { + return; + } + + const applied = yield* recovery.pipe(Effect.exit); + if (Exit.isFailure(applied)) { + if (Cause.hasInterruptsOnly(applied.cause)) { + return; + } + // Keep the raw cause on the log cause. The annotation is only a + // bounded category, so process output cannot land in structured fields. + yield* annotateInstanceLog( + Effect.logError("desktop backend startup fallback failed", applied.cause), + { failure: "fallback" }, + ); + yield* resumeAfterDeclinedStartupFailure; + return; + } + + const current = yield* Ref.get(state); + if (!current.startupFailurePending || !current.desiredRunning || current.ready) { + yield* Ref.update(state, withoutStartupFailure); + return; + } + // Drop the hook fiber before start() so stop() cannot interrupt this + // fiber while it holds the start mutex. A quit that landed since the + // generation was read makes start() no-op. + const generation = current.stopGeneration; + yield* Ref.update(state, (latest) => + Option.isSome(latest.startupFailureFiber) + ? { ...latest, startupFailureFiber: Option.none() } + : latest, + ); + yield* Ref.set(replacementStopGenerationRef, Option.some(generation)); + yield* start; + }); + + // The start mutex is already released. The hook runs in the parent scope so + // a dialog does not hold that mutex, and stop() can interrupt it. + const armStartupFailure = ( + failure: StartupFailure, + runningDistro: string | undefined, + ): Effect.Effect => + Effect.gen(function* () { + const onStartupFailed = spec.onStartupFailed; + if (onStartupFailed === undefined) { + yield* Ref.update(state, withoutStartupFailure); + return; + } + + const tracked: { fiber?: Fiber.Fiber } = {}; + const fiber = yield* Effect.forkIn( + onStartupFailed(failure, runningDistro).pipe( + Effect.exit, + Effect.flatMap((decision) => { + if (Exit.isFailure(decision)) { + if (Cause.hasInterruptsOnly(decision.cause)) { + return Effect.void; + } + return annotateInstanceLog( + Effect.logError("desktop backend startup failure hook failed", decision.cause), + { failure: failure.kind }, + ).pipe(Effect.andThen(resumeAfterDeclinedStartupFailure)); + } + return Option.match(decision.value, { + onNone: () => resumeAfterDeclinedStartupFailure, + onSome: (recovery) => commitStartupRecovery(recovery), + }); + }), + Effect.ensuring( + Effect.suspend(() => + tracked.fiber === undefined ? Effect.void : clearTrackedStartupFailure(tracked.fiber), + ), + ), + ), + parentScope, + ); + tracked.fiber = fiber; + yield* Ref.update(state, (latest) => + latest.startupFailurePending + ? { + ...latest, + startupFailureFiber: Option.some(fiber), + } + : latest, + ); + }); + const stop = Effect.fn("desktop.backendInstance.stop")(function* (options?: { readonly timeout?: Duration.Duration; }) { - const { active, restartFiber, notifyShutdown } = yield* mutex.withPermits(1)( + const { active, restartFiber, startupFailureFiber, notifyShutdown } = yield* mutex.withPermits( + 1, + )( Effect.gen(function* () { const result = yield* Ref.modify(state, (latest) => { const active = Option.map(latest.active, (run) => @@ -1058,6 +1371,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( { active, restartFiber: latest.restartFiber, + startupFailureFiber: latest.startupFailureFiber, notifyShutdown: latest.ready, }, { @@ -1066,6 +1380,10 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( ready: false, active, restartFiber: Option.none>(), + startupFailureFiber: Option.none>(), + startupFailurePending: false, + startupFailureNotice: Option.none(), + stopGeneration: latest.stopGeneration + 1, }, ] as const; }); @@ -1080,6 +1398,10 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( onNone: () => Effect.void, onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid), }); + yield* Option.match(startupFailureFiber, { + onNone: () => Effect.void, + onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid), + }); yield* Option.match(active, { onNone: () => Effect.void, onSome: (run) => diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index 8373437ae312..7152fff282e2 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -4,8 +4,11 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; @@ -154,4 +157,187 @@ describe("DesktopBackendPool", () => { }), ), ); + + it.effect( + "shows a bounded error and uses Windows in memory when a wsl-only primary keeps exiting", + () => + Effect.scoped( + Effect.gen(function* () { + const launches = yield* Queue.unbounded<"wsl" | "windows">(); + const failures = yield* Queue.unbounded(); + const dialogs = yield* Queue.unbounded<{ + readonly title: string; + readonly content: string; + }>(); + const launchKind = yield* Ref.make<"wsl" | "windows">("wsl"); + const wslConfig: DesktopBackendStartConfig = { + ...backendConfig, + runningDistro: "Ubuntu-22.04", + httpBaseUrl: new URL("http://10.0.0.8:3773"), + }; + const windowsConfig: DesktopBackendStartConfig = { + ...backendConfig, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }; + const settingsLayer = DesktopAppSettings.layerTest({ + ...DesktopAppSettings.DEFAULT_DESKTOP_SETTINGS, + wslBackendEnabled: true, + wslOnly: true, + wslDistro: "Ubuntu-22.04", + }); + + const context = yield* Layer.build( + DesktopBackendPool.layer.pipe( + Layer.provide( + Layer.mergeAll( + FileSystem.layerNoop({ exists: () => Effect.succeed(true) }), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const kind = yield* Ref.get(launchKind); + yield* Queue.offer(launches, kind); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ), + ), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.never), + ), + Layer.succeed(DesktopObservability.DesktopBackendOutputLogFactory, { + forInstance: () => + Effect.succeed({ + beginSession: () => Effect.void, + writeOutputChunk: () => Effect.void, + persistFailureSnapshot: () => Effect.void, + persistFailure: ({ details }) => + Queue.offer(failures, details).pipe(Effect.asVoid), + discardSession: Effect.void, + } satisfies DesktopObservability.DesktopBackendOutputLogShape), + } satisfies DesktopObservability.DesktopBackendOutputLogFactory["Service"]), + Layer.succeed(DesktopTelemetryPublisher.DesktopTelemetryPublisher, { + latest: Effect.succeedNone, + changes: Stream.empty, + encoded: Stream.empty, + handleControlForSource: () => Effect.void, + removeControlSource: () => Effect.void, + publishUpdateReport: () => Effect.void, + updateRequests: Stream.empty, + updateCommits: Stream.empty, + updateCancellations: Stream.empty, + }), + Layer.effect( + DesktopBackendConfiguration.DesktopBackendConfiguration, + Effect.gen(function* () { + const appSettings = yield* DesktopAppSettings.DesktopAppSettings; + return { + resolvePrimary: appSettings.get.pipe( + Effect.tap((settings) => + Ref.set(launchKind, settings.wslOnly ? "wsl" : "windows"), + ), + Effect.map((settings) => (settings.wslOnly ? wslConfig : windowsConfig)), + ), + resolvePrimaryLabel: Effect.succeed("WSL (Ubuntu-22.04)"), + resolveWsl: () => Effect.die("unexpected WSL config resolve"), + } satisfies DesktopBackendConfiguration.DesktopBackendConfiguration["Service"]; + }), + ), + DesktopWslEnvironment.layerTest(), + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.die("unexpected folder picker"), + pickFiles: () => Effect.die("unexpected file picker"), + showMessageBox: () => Effect.die("unexpected message box"), + showErrorBox: (title, content) => + Queue.offer(dialogs, { title, content }).pipe(Effect.asVoid), + } satisfies ElectronDialog.ElectronDialog["Service"]), + Layer.succeed(DesktopWindow.DesktopWindow, { + createMain: Effect.die("unexpected window create"), + ensureMain: Effect.die("unexpected window ensure"), + revealOrCreateMain: Effect.die("unexpected window reveal"), + activate: Effect.die("unexpected window activate"), + createMainIfBackendReady: Effect.die("unexpected window create"), + showConnectingSplash: Effect.void, + handleBackendReady: () => Effect.void, + handleBackendNotReady: Effect.void, + flushMainWindowBounds: Effect.void, + prepareCaptureReveal: Effect.void, + dispatchMenuAction: () => Effect.die("unexpected menu action"), + dispatchSnapShotEvent: () => Effect.void, + zoomMain: () => Effect.die("unexpected zoom"), + syncAppearance: Effect.void, + } satisfies DesktopWindow.DesktopWindow["Service"]), + ), + ), + Layer.provideMerge(settingsLayer), + ), + ); + const pool = yield* DesktopBackendPool.DesktopBackendPool.pipe(Effect.provide(context)); + const settings = yield* DesktopAppSettings.DesktopAppSettings.pipe( + Effect.provide(context), + ); + + const primary = yield* pool.primary; + yield* primary.start; + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(launches), "wsl"); + + const dialog = yield* Queue.take(dialogs); + assert.equal(dialog.title, "WSL backend isn't responding"); + assert.equal( + dialog.content, + "The WSL backend (Ubuntu-22.04) exited before it was ready (exit code 1).\n\nT3 Code will use the Windows backend for this launch and retry WSL the next time the app starts.", + ); + assert.isFalse(dialog.content.includes("http")); + assert.isFalse(dialog.content.includes("MODULE_NOT_FOUND")); + assert.equal(yield* Queue.take(launches), "windows"); + + const stored = yield* settings.get; + assert.equal(stored.wslOnly, false); + assert.equal(stored.wslBackendEnabled, false); + assert.equal(stored.wslDistro, "Ubuntu-22.04"); + yield* primary.stop(); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); }); + +const backendConfig: DesktopBackendStartConfig = { + executablePath: "/electron", + args: ["/server/bin.mjs"], + entryPath: "/server/bin.mjs", + cwd: "/server", + env: { ELECTRON_RUN_AS_NODE: "1" }, + bootstrap: { + mode: "desktop", + noBrowser: true, + port: 3773, + t3Home: "/tmp/t3", + host: "127.0.0.1", + desktopBootstrapToken: "token", + tailscaleServeEnabled: false, + tailscaleServePort: 443, + }, + bootstrapDelivery: "fd3", + extendEnv: true, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + captureOutput: false, + preflightFailure: Option.none(), +}; diff --git a/apps/desktop/src/backend/DesktopBackendPool.ts b/apps/desktop/src/backend/DesktopBackendPool.ts index 2930780d4102..c2a16e359835 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.ts @@ -79,6 +79,7 @@ // 10. CommandPalette enables file-manager picker for desktop-local // envs, routes pickFolder by env id. (38e8477a) +import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -102,9 +103,25 @@ import * as DesktopWindow from "../window/DesktopWindow.ts"; import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts"; import * as ElectronDialog from "../electron/ElectronDialog.ts"; -const { logWarning: logBackendPoolWarning } = +const { annotate: annotatePoolLog, logWarning: logBackendPoolWarning } = DesktopObservability.makeComponentLogger("desktop-backend-pool"); +const startupFailureDialogBody = ( + failure: DesktopBackendManager.StartupFailure, + distro: string, +): string => { + const subject = `The WSL backend (${distro})`; + const next = + "T3 Code will use the Windows backend for this launch and retry WSL the next time the app starts."; + if (failure.kind === "exited" && failure.exitCode !== undefined) { + return `${subject} exited before it was ready (exit code ${failure.exitCode}).\n\n${next}`; + } + if (failure.kind === "exited") { + return `${subject} exited before it was ready.\n\n${next}`; + } + return `${subject} did not become ready.\n\n${next}`; +}; + export type BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const PRIMARY_INSTANCE_ID = DesktopBackendManager.PRIMARY_INSTANCE_ID; @@ -277,6 +294,43 @@ export const layer = Layer.effect( }, ); + // Preflight passed, but the primary still never became ready. The connecting + // splash has no controls, so wsl-only mode would sit there until the process + // is killed. Use Windows for this launch only — the same in-memory fallback + // as a bounded preflight failure — and try WSL again on the next launch. + // A run that already resolved to Windows has no distro and keeps restarting. + const handlePrimaryStartupFailure = Effect.fn("desktop.backendPool.primaryStartupFailed")( + function* (failure: DesktopBackendManager.StartupFailure, runningDistro: string | undefined) { + if (runningDistro === undefined) { + return Option.none>(); + } + yield* logBackendPoolWarning( + "primary WSL backend did not become ready; using Windows for this launch", + { + failure: failure.kind, + ...(failure.exitCode === undefined ? {} : { exitCode: failure.exitCode }), + distro: runningDistro, + }, + ); + const dialog = yield* electronDialog + .showErrorBox( + "WSL backend isn't responding", + startupFailureDialogBody(failure, runningDistro), + ) + .pipe(Effect.exit); + if (Exit.isFailure(dialog)) { + if (Cause.hasInterruptsOnly(dialog.cause)) { + return yield* Effect.interrupt; + } + yield* annotatePoolLog( + Effect.logError("desktop backend startup failure dialog failed", dialog.cause), + { failure: failure.kind }, + ); + } + return Option.some(appSettings.applyWslWindowsFallbackInMemory.pipe(Effect.asVoid)); + }, + ); + const primary = yield* DesktopBackendManager.makeBackendInstance({ id: DesktopBackendManager.PRIMARY_INSTANCE_ID, // Keep this lazy. The pool layer is initialized before startup loads @@ -301,6 +355,7 @@ export const layer = Layer.effect( ), onShutdown: () => desktopWindow.handleBackendNotReady, onPreflightFailed: handlePrimaryPreflightFailure, + onStartupFailed: handlePrimaryStartupFailure, }); const instancesRef = yield* SynchronizedRef.make<