diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 901d9f4a2708..6f437a16b894 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -126,6 +126,11 @@ interface MakeInstanceInput { readonly onPreflightFailed?: ( failure: DesktopBackendManager.PreflightFailure, ) => Effect.Effect; + readonly onStartupFailed?: ( + failure: DesktopBackendManager.StartupFailure, + runningDistro: string | undefined, + ) => Effect.Effect; + readonly readinessTimeout?: Duration.Duration; readonly config?: DesktopBackendManager.DesktopBackendStartConfig; readonly configResolve?: Effect.Effect< DesktopBackendManager.DesktopBackendStartConfig, @@ -185,6 +190,8 @@ function makeTestInstance(input: MakeInstanceInput) { ...(input.onReady ? { onReady: () => input.onReady! } : {}), ...(input.onShutdown ? { onShutdown: () => input.onShutdown! } : {}), ...(input.onPreflightFailed ? { onPreflightFailed: input.onPreflightFailed } : {}), + ...(input.onStartupFailed ? { onStartupFailed: input.onStartupFailed } : {}), + ...(input.readinessTimeout === undefined ? {} : { readinessTimeout: input.readinessTimeout }), }); return instance.pipe(Effect.provide(servicesLayer)); @@ -1575,4 +1582,313 @@ describe("DesktopBackendManager", () => { }).pipe(Effect.provide(TestClock.layer())), ), ); + + it.effect("recovers after three pre-ready exits instead of restarting without a cap", () => + Effect.scoped( + Effect.gen(function* () { + const launches = yield* Queue.unbounded<"wsl" | "windows">(); + const failures = yield* Queue.unbounded(); + const startupFailures: Array = []; + const useWindows = yield* Ref.make(false); + const ready = yield* Deferred.make(); + const wslConfig: DesktopBackendManager.DesktopBackendStartConfig = { + ...baseConfig, + runningDistro: "Ubuntu-22.04", + httpBaseUrl: new URL("http://10.0.0.8:3773"), + }; + const windowsConfig: DesktopBackendManager.DesktopBackendStartConfig = { + ...baseConfig, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const windows = yield* Ref.get(useWindows); + yield* Queue.offer(launches, windows ? "windows" : "wsl"); + if (!windows) { + return makeProcess({ + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + }); + } + const closed = yield* Deferred.make(); + return makeProcess({ + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + configResolve: Ref.get(useWindows).pipe( + Effect.map((windows) => (windows ? windowsConfig : wslConfig)), + ), + httpClientLayer: httpClientLayer((request) => + request.url.startsWith("http://127.0.0.1:3773") + ? Effect.succeed(responseForRequest(request, 200)) + : Effect.never, + ), + onReady: Deferred.succeed(ready, void 0).pipe(Effect.asVoid), + onStartupFailed: (failure) => + Effect.sync(() => { + startupFailures.push(failure); + }).pipe(Effect.andThen(Ref.set(useWindows, true)), Effect.as(true)), + backendOutputLog: { + persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid), + }, + }); + + yield* instance.start; + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.deepEqual(startupFailures, []); + + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.deepEqual(startupFailures, []); + + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.equal(yield* Queue.take(launches), "windows"); + yield* Deferred.await(ready).pipe(Effect.timeout("1 second")); + + assert.deepEqual(startupFailures, [{ kind: "exited", exitCode: 1 }]); + // The recovered Windows backend stays up. Advancing well past the + // restart cap must not spawn another WSL process. + yield* TestClock.adjust(Duration.seconds(30)); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* instance.snapshot).ready, true); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("gives a user-driven start during teardown a fresh startup budget", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const failures = yield* Queue.unbounded(); + const startupFailures: Array = []; + const teardownStarted = yield* Deferred.make(); + const finishTeardown = yield* Deferred.make(); + let startCount = 0; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + startCount += 1; + const id = startCount; + yield* Queue.offer(starts, id); + if (id === 3) { + const closed = yield* Deferred.make(); + yield* Scope.addFinalizer( + scope, + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishTeardown)), + Effect.andThen(Deferred.succeed(closed, undefined)), + Effect.asVoid, + ), + ); + return makeProcess({ + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + } + return makeProcess({ + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + config: { + ...baseConfig, + runningDistro: "Ubuntu-22.04", + }, + httpClientLayer: httpClientLayer(() => Effect.never), + onStartupFailed: (failure) => + Effect.sync(() => { + startupFailures.push(failure); + }).pipe(Effect.as(false)), + backendOutputLog: { + persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid), + }, + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(starts), 2); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(starts), 3); + assert.deepEqual(startupFailures, []); + + // Two pre-ready crashes are already on the budget. start() while that + // run is closing must not leave them there for the replacement. + const stopFiber = yield* instance.stop().pipe(Effect.forkChild); + yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); + yield* instance.start; + yield* Deferred.succeed(finishTeardown, undefined); + yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); + + // Closing the stopped run schedules the replacement. Two earlier + // crashes leave the backoff at 2s (500ms * 2^2). + yield* TestClock.adjust(Duration.seconds(2)); + assert.equal(yield* Queue.take(starts), 4); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.deepEqual(startupFailures, []); + + yield* TestClock.adjust(Duration.seconds(4)); + assert.equal(yield* Queue.take(starts), 5); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.deepEqual(startupFailures, []); + + yield* TestClock.adjust(Duration.seconds(8)); + assert.equal(yield* Queue.take(starts), 6); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + assert.deepEqual(startupFailures, [{ kind: "exited", exitCode: 1 }]); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("recovers when a live backend exhausts three readiness budgets", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const startupFailures: Array = []; + const ready = yield* Deferred.make(); + const childExit = yield* Deferred.make(); + const healthy = yield* Ref.make(false); + let startCount = 0; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + startCount += 1; + const id = startCount; + yield* Queue.offer(starts, id); + if (id === 1) { + return makeProcess({ + exitCode: Deferred.await(childExit).pipe( + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + kill: () => Deferred.succeed(childExit, undefined), + }); + } + const closed = yield* Deferred.make(); + return makeProcess({ + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + readinessTimeout: Duration.millis(50), + config: { + ...baseConfig, + runningDistro: "Ubuntu-22.04", + }, + httpClientLayer: httpClientLayer((request) => + Ref.get(healthy).pipe( + Effect.map((isHealthy) => responseForRequest(request, isHealthy ? 200 : 503)), + ), + ), + onReady: Deferred.succeed(ready, void 0).pipe(Effect.asVoid), + onStartupFailed: (failure) => + Effect.sync(() => { + startupFailures.push(failure); + }).pipe(Effect.andThen(Ref.set(healthy, true)), Effect.as(true)), + }); + + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); + assert.deepEqual(startupFailures, []); + + yield* TestClock.adjust(Duration.millis(50)); + assert.deepEqual(startupFailures, []); + yield* TestClock.adjust(Duration.millis(50)); + assert.deepEqual(startupFailures, []); + yield* TestClock.adjust(Duration.millis(50)); + + assert.equal(yield* Queue.take(starts).pipe(Effect.timeout("1 second")), 2); + yield* Deferred.await(ready).pipe(Effect.timeout("1 second")); + assert.deepEqual(startupFailures, [{ kind: "unreachable" }]); + assert.equal((yield* instance.snapshot).ready, true); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + + it.effect("does not spend the startup budget on exits after the backend was ready", () => + Effect.scoped( + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const readies = yield* Queue.unbounded(); + const exited = yield* Queue.unbounded(); + const release = yield* Queue.unbounded(); + let startCount = 0; + const startupFailures: Array = []; + + const spawnerLayer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + const pid = startCount; + return makeProcess({ + exitCode: Queue.offer(starts, pid).pipe( + Effect.andThen(Queue.take(release)), + Effect.as(ChildProcessSpawner.ExitCode(1)), + ), + }); + }), + ), + ); + + const instance = yield* makeTestInstance({ + spawnerLayer, + onReady: Effect.sync(() => startCount).pipe( + Effect.flatMap((pid) => Queue.offer(readies, pid)), + Effect.asVoid, + ), + onStartupFailed: (failure) => + Effect.sync(() => { + startupFailures.push(failure); + }).pipe(Effect.as(false)), + backendOutputLog: { + persistFailure: () => Queue.offer(exited, startCount).pipe(Effect.asVoid), + }, + }); + + yield* instance.start; + for (const delay of [ + Duration.zero, + Duration.millis(500), + Duration.seconds(1), + Duration.seconds(2), + Duration.seconds(4), + ]) { + if (Duration.toMillis(delay) > 0) { + yield* TestClock.adjust(delay); + } + const pid = yield* Queue.take(starts); + assert.equal(yield* Queue.take(readies), pid); + yield* Queue.offer(release, undefined); + assert.equal(yield* Queue.take(exited), pid); + } + assert.deepEqual(startupFailures, []); + assert.equal(startCount, 5); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 6f1ea139bdca..4540e963fa51 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -61,6 +61,10 @@ const MAX_RESTART_DELAY = Duration.seconds(10); // failures may instead provide their own larger retryLimit when they should // self-heal for a while but must not leave the app connecting forever. const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5; +// Preflight can pass and the child can still exit, or stay up and never +// answer. Cap those consecutive pre-ready failures, then let onStartupFailed +// recover. A backend that has already been ready does not count. +const MAX_STARTUP_FAILURE_ATTEMPTS = 3; const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1); const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100); const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1); @@ -117,6 +121,27 @@ export interface PreflightFailure { readonly retryLimit?: number; } +// Categorized before it reaches a dialog or a log annotation. `exited` is a +// child that ended before it was ever ready. `unreachable` is a child that +// stayed up while its readiness budget ran out. `exitCode` is only a numeric +// status in 0..255; process output and readiness URLs are not included. +export interface StartupFailure { + readonly kind: "exited" | "unreachable"; + readonly exitCode?: number; +} + +const startupFailureFromExitReason = (reason: string): StartupFailure => { + const match = /^code=(\d{1,3})$/.exec(reason); + if (match === null) { + return { kind: "exited" }; + } + const exitCode = Number(match[1]); + if (!Number.isInteger(exitCode) || exitCode > 255) { + return { kind: "exited" }; + } + return { kind: "exited", exitCode }; +}; + interface BackendProcessExit { readonly code: Option.Option; readonly reason: string; @@ -230,7 +255,11 @@ interface RunBackendProcessOptions extends DesktopBackendStartConfig { readonly onStarted?: (pid: number) => Effect.Effect; readonly onExitObserved?: () => Effect.Effect; readonly onReady?: () => Effect.Effect; - readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect; + // `true` stops probing and kills the child so the exit path can recover. + // `void` / `false` keeps the current probe loop. + readonly onReadinessFailure?: ( + error: BackendReadinessTimeoutError, + ) => Effect.Effect; readonly onOutput?: ( streamName: BackendProcessOutputStream, chunk: Uint8Array, @@ -299,6 +328,16 @@ export interface BackendInstanceSpec { // retries. Returns true when the callback changed configuration and the // manager should resolve once more; false stops the failed instance. readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect; + // Fired once MAX_STARTUP_FAILURE_ATTEMPTS consecutive pre-ready failures + // have accumulated. Returns true when configuration changed and the manager + // should start again immediately; false keeps the normal restart loop. + // Exits after a successful ready do not count. + readonly onStartupFailed?: ( + failure: StartupFailure, + runningDistro: string | undefined, + ) => Effect.Effect; + // Overrides the per-round readiness budget. Production uses the default. + readonly readinessTimeout?: Duration.Duration; } interface ActiveBackendRun { @@ -319,6 +358,12 @@ interface BackendManagerState { // Consecutive bounded/fatal preflight failures, reset on a clean or // unbounded-transient preflight. restartAttempt counts all restarts. readonly preflightFailureAttempt: number; + // Consecutive pre-ready exits and readiness timeouts. Reset once ready, + // and when a user-driven start() asks for a new process. + readonly startupFailureAttempt: number; + // The readiness probe hit the cap and killed the child. The following exit + // is that unreachable failure, not a separate crash. + readonly startupFailureUnreachable: boolean; readonly restartFiber: Option.Option>; readonly nextRunId: number; } @@ -330,10 +375,21 @@ const initialState: BackendManagerState = { active: Option.none(), restartAttempt: 0, preflightFailureAttempt: 0, + startupFailureAttempt: 0, + startupFailureUnreachable: false, restartFiber: Option.none(), nextRunId: 1, }; +const withFreshStartupBudget = (state: BackendManagerState): BackendManagerState => + state.startupFailureAttempt === 0 && !state.startupFailureUnreachable + ? state + : { + ...state, + startupFailureAttempt: 0, + startupFailureUnreachable: false, + }; + const activePid = (active: Option.Option): Option.Option => Option.flatMap(active, (run) => run.pid); @@ -588,7 +644,21 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( Effect.as(true), Effect.catchTags({ BackendReadinessTimeoutError: (error) => - (options.onReadinessFailure?.(error) ?? Effect.void).pipe(Effect.as(false)), + Effect.gen(function* () { + const notified = options.onReadinessFailure?.(error); + const giveUp = + notified === undefined + ? false + : yield* Effect.map(notified, (value) => value === true); + if (!giveUp) { + return false; + } + // The probe loop is forked beside the exit wait. Killing here lets + // that wait finish and own recovery, instead of stopping the + // instance from inside its own fiber. + yield* handle.kill().pipe(Effect.ignore); + return true; + }), }), ), ); @@ -692,10 +762,16 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( mutex.withPermits(1)( Effect.gen(function* () { const current = yield* Ref.get(state); + const userDrivenStart = !current.desiredRunning; if (Option.isSome(current.active)) { - if (!current.desiredRunning) { + if (userDrivenStart) { + // The replacement is spawned only after this run closes, and by + // then desiredRunning is already true. The later start sees a + // restart, not a user start, so the fresh budget has to be taken + // here — otherwise one pre-ready failure on the replacement can + // spend a budget the previous run already used up. yield* Ref.update(state, (latest) => ({ - ...latest, + ...withFreshStartupBudget(latest), desiredRunning: true, })); } @@ -726,16 +802,18 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( .exists(config.value.entryPath) .pipe(Effect.orElseSucceed(() => false)); - const resetFatalPreflightCounter = - !current.desiredRunning && current.preflightFailureAttempt > 0; + const resetFatalPreflightCounter = userDrivenStart && current.preflightFailureAttempt > 0; yield* cancelRestart; - yield* Ref.update(state, (latest) => ({ - ...latest, - desiredRunning: true, - ready: false, - config: Option.some(config.value), - preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt, - })); + yield* Ref.update(state, (latest) => { + const next = userDrivenStart ? withFreshStartupBudget(latest) : latest; + return { + ...next, + desiredRunning: true, + ready: false, + config: Option.some(config.value), + preflightFailureAttempt: resetFatalPreflightCounter ? 0 : next.preflightFailureAttempt, + }; + }); const preflightFailure = config.value.preflightFailure; if (Option.isSome(preflightFailure)) { @@ -829,57 +907,72 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* ( reason: string, ) { - yield* mutex.withPermits(1)( + const recovery = yield* mutex.withPermits(1)( Effect.gen(function* () { - const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } = - yield* Ref.modify( - state, - ( - latest, - ): readonly [ - { - readonly isCurrentRun: boolean; - readonly nextState: BackendManagerState; - readonly pid: Option.Option; - readonly exitObserved: boolean; - readonly stopRequested: boolean; - readonly wasReady: boolean; - }, - BackendManagerState, - ] => { - const currentRun = Option.getOrUndefined(latest.active); - if (currentRun?.id !== runId) { - return [ - { - isCurrentRun: false, - nextState: latest, - pid: Option.none(), - exitObserved: false, - stopRequested: false, - wasReady: false, - }, - latest, - ] as const; - } - - const next = { - ...latest, - active: Option.none(), - ready: false, - }; + const { + isCurrentRun, + nextState, + pid, + exitObserved, + stopRequested, + wasReady, + startupFailureAttempt, + startupFailureUnreachable, + } = yield* Ref.modify( + state, + ( + latest, + ): readonly [ + { + readonly isCurrentRun: boolean; + readonly nextState: BackendManagerState; + readonly pid: Option.Option; + readonly exitObserved: boolean; + readonly stopRequested: boolean; + readonly wasReady: boolean; + readonly startupFailureAttempt: number; + readonly startupFailureUnreachable: boolean; + }, + BackendManagerState, + ] => { + const currentRun = Option.getOrUndefined(latest.active); + if (currentRun?.id !== runId) { return [ { - isCurrentRun: true, - nextState: next, - pid: currentRun.pid, - exitObserved: currentRun.exitObserved, - stopRequested: currentRun.stopRequested, - wasReady: latest.ready, + isCurrentRun: false, + nextState: latest, + pid: Option.none(), + exitObserved: false, + stopRequested: false, + wasReady: false, + startupFailureAttempt: latest.startupFailureAttempt, + startupFailureUnreachable: false, }, - next, + latest, ] as const; - }, - ); + } + + const next = { + ...latest, + active: Option.none(), + ready: false, + startupFailureUnreachable: false, + }; + return [ + { + isCurrentRun: true, + nextState: next, + pid: currentRun.pid, + exitObserved: currentRun.exitObserved, + stopRequested: currentRun.stopRequested, + wasReady: latest.ready, + startupFailureAttempt: latest.startupFailureAttempt, + startupFailureUnreachable: latest.startupFailureUnreachable, + }, + next, + ] as const; + }, + ); if (isCurrentRun) { yield* desktopTelemetryPublisher.removeControlSource(spec.id); @@ -897,15 +990,71 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( } } - if (isCurrentRun && nextState.desiredRunning) { + const countsTowardStartupBudget = + isCurrentRun && !stopRequested && !wasReady && spec.onStartupFailed !== undefined; + let startupFailure: StartupFailure | undefined; + if (countsTowardStartupBudget) { + const nextAttempt = startupFailureAttempt + 1; + if (nextAttempt < MAX_STARTUP_FAILURE_ATTEMPTS) { + yield* Ref.update(state, (latest) => ({ + ...latest, + startupFailureAttempt: nextAttempt, + })); + } else { + startupFailure = startupFailureUnreachable + ? { kind: "unreachable" } + : startupFailureFromExitReason(reason); + yield* Ref.update(state, withFreshStartupBudget); + } + } else if (isCurrentRun && wasReady) { + yield* Ref.update(state, withFreshStartupBudget); + } + + if (startupFailure === undefined && isCurrentRun && nextState.desiredRunning) { yield* scheduleRestart(reason); } + return startupFailure; }), ); + + // The hook runs after the mutex is released so a dialog cannot sit + // on it. stop() sets desiredRunning false before it waits for this + // fiber, so a quit during the hook does not start a replacement. + if (recovery === undefined || spec.onStartupFailed === undefined) { + return; + } + const decision = yield* spec + .onStartupFailed(recovery, config.value.runningDistro) + .pipe(Effect.exit); + if (Exit.isFailure(decision)) { + if (Cause.hasInterruptsOnly(decision.cause)) { + return; + } + yield* logInstanceWarning("desktop backend startup recovery failed", { + failure: recovery.kind, + }); + } + const current = yield* Ref.get(state); + if ( + !current.desiredRunning || + current.ready || + Option.isSome(current.active) || + Option.isSome(current.restartFiber) + ) { + return; + } + if (Exit.isSuccess(decision) && decision.value) { + yield* start; + return; + } + yield* scheduleRestart(reason); }); const program = runBackendProcess({ ...config.value, + ...(spec.readinessTimeout === undefined + ? {} + : { readinessTimeout: spec.readinessTimeout }), desktopTelemetryStream: desktopTelemetryPublisher.encoded, onDesktopTelemetryControl: (message) => desktopTelemetryPublisher.handleControlForSource(spec.id, message), @@ -932,11 +1081,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( return [ true, - { + withFreshStartupBudget({ ...latest, restartAttempt: 0, ready: true, - }, + }), ] as const; }); if (!isCurrentRun) { @@ -960,8 +1109,29 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( error: error.message, }); yield* backendOutputLog.persistFailureSnapshot({ - details: error.message, + details: "readiness-timeout", }); + if (spec.onStartupFailed === undefined) { + return false; + } + return yield* mutex.withPermits(1)( + Ref.modify(state, (latest) => { + const active = Option.getOrUndefined(latest.active); + if ( + active?.id !== runId || + active.stopRequested || + !latest.desiredRunning || + latest.ready + ) { + return [false, latest] as const; + } + const nextAttempt = latest.startupFailureAttempt + 1; + if (nextAttempt < MAX_STARTUP_FAILURE_ATTEMPTS) { + return [false, { ...latest, startupFailureAttempt: nextAttempt }] as const; + } + return [true, { ...latest, startupFailureUnreachable: true }] as const; + }), + ); }, ), onOutput: (streamName, chunk) => backendOutputLog.writeOutputChunk(streamName, chunk), diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index 8373437ae312..042834650da5 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -1,12 +1,17 @@ import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; +import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; -import { HttpClient } from "effect/unstable/http"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import * as DesktopObservability from "../app/DesktopObservability.ts"; @@ -19,6 +24,29 @@ import * as DesktopBackendConfiguration from "./DesktopBackendConfiguration.ts"; import * as DesktopBackendPool from "./DesktopBackendPool.ts"; import type { DesktopBackendSnapshot, DesktopBackendStartConfig } from "./DesktopBackendManager.ts"; +const backendConfig: DesktopBackendStartConfig = { + executablePath: "/electron", + args: ["/server/bin.mjs"], + entryPath: "/server/bin.mjs", + cwd: "/server", + env: { ELECTRON_RUN_AS_NODE: "1" }, + bootstrap: { + mode: "desktop", + noBrowser: true, + port: 3773, + t3Home: "/tmp/t3", + host: "127.0.0.1", + desktopBootstrapToken: "token", + tailscaleServeEnabled: false, + tailscaleServePort: 443, + }, + bootstrapDelivery: "fd3", + extendEnv: true, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + captureOutput: false, + preflightFailure: Option.none(), +}; + function makeStubInstance( id: DesktopBackendPool.BackendInstanceId, label: string, @@ -154,4 +182,182 @@ describe("DesktopBackendPool", () => { }), ), ); + + it.effect("uses Windows for this launch when a wsl-only primary never becomes ready", () => + Effect.scoped( + Effect.gen(function* () { + const launches = yield* Queue.unbounded<"wsl" | "windows">(); + const failures = yield* Queue.unbounded(); + const dialogs = yield* Queue.unbounded<{ + readonly title: string; + readonly content: string; + }>(); + const ready = yield* Queue.unbounded(); + const launchKind = yield* Ref.make<"wsl" | "windows">("wsl"); + const windowsExit = yield* Deferred.make(); + const wslConfig: DesktopBackendStartConfig = { + ...backendConfig, + runningDistro: "Ubuntu-22.04", + httpBaseUrl: new URL("http://10.0.0.8:3773"), + }; + const windowsConfig: DesktopBackendStartConfig = { + ...backendConfig, + httpBaseUrl: new URL("http://127.0.0.1:3773"), + }; + const settingsLayer = DesktopAppSettings.layerTest({ + ...DesktopAppSettings.DEFAULT_DESKTOP_SETTINGS, + wslBackendEnabled: true, + wslOnly: true, + wslDistro: "Ubuntu-22.04", + }); + + const context = yield* Layer.build( + DesktopBackendPool.layer.pipe( + Layer.provide( + Layer.mergeAll( + FileSystem.layerNoop({ exists: () => Effect.succeed(true) }), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + const kind = yield* Ref.get(launchKind); + yield* Queue.offer(launches, kind); + const releaseWindows = Deferred.succeed(windowsExit, undefined).pipe( + Effect.ignore, + ); + if (kind === "windows") { + yield* Scope.addFinalizer(scope, releaseWindows); + } + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: + kind === "windows" + ? Deferred.await(windowsExit).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ) + : Effect.succeed(ChildProcessSpawner.ExitCode(1)), + isRunning: Effect.succeed(false), + kill: () => (kind === "windows" ? releaseWindows : Effect.void), + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ), + ), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + request.url.startsWith("http://127.0.0.1:3773") + ? Effect.succeed( + HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), + ) + : Effect.never, + ), + ), + Layer.succeed(DesktopObservability.DesktopBackendOutputLogFactory, { + forInstance: () => + Effect.succeed({ + beginSession: () => Effect.void, + writeOutputChunk: () => Effect.void, + persistFailureSnapshot: () => Effect.void, + persistFailure: ({ details }) => + Queue.offer(failures, details).pipe(Effect.asVoid), + discardSession: Effect.void, + } satisfies DesktopObservability.DesktopBackendOutputLogShape), + } satisfies DesktopObservability.DesktopBackendOutputLogFactory["Service"]), + Layer.succeed(DesktopTelemetryPublisher.DesktopTelemetryPublisher, { + latest: Effect.succeedNone, + changes: Stream.empty, + encoded: Stream.empty, + handleControlForSource: () => Effect.void, + removeControlSource: () => Effect.void, + publishUpdateReport: () => Effect.void, + updateRequests: Stream.empty, + updateCommits: Stream.empty, + updateCancellations: Stream.empty, + }), + Layer.effect( + DesktopBackendConfiguration.DesktopBackendConfiguration, + Effect.gen(function* () { + const appSettings = yield* DesktopAppSettings.DesktopAppSettings; + return { + resolvePrimary: appSettings.get.pipe( + Effect.tap((settings) => + Ref.set(launchKind, settings.wslOnly ? "wsl" : "windows"), + ), + Effect.map((settings) => (settings.wslOnly ? wslConfig : windowsConfig)), + ), + resolvePrimaryLabel: Effect.succeed("WSL (Ubuntu-22.04)"), + resolveWsl: () => Effect.die("unexpected WSL config resolve"), + } satisfies DesktopBackendConfiguration.DesktopBackendConfiguration["Service"]; + }), + ), + DesktopWslEnvironment.layerTest(), + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.die("unexpected folder picker"), + pickFiles: () => Effect.die("unexpected file picker"), + showMessageBox: () => Effect.die("unexpected message box"), + showErrorBox: (title, content) => + Queue.offer(dialogs, { title, content }).pipe(Effect.asVoid), + } satisfies ElectronDialog.ElectronDialog["Service"]), + Layer.succeed(DesktopWindow.DesktopWindow, { + createMain: Effect.die("unexpected window create"), + ensureMain: Effect.die("unexpected window ensure"), + revealOrCreateMain: Effect.die("unexpected window reveal"), + activate: Effect.die("unexpected window activate"), + createMainIfBackendReady: Effect.die("unexpected window create"), + showConnectingSplash: Effect.void, + handleBackendReady: () => Queue.offer(ready, "ready").pipe(Effect.asVoid), + handleBackendNotReady: Effect.void, + flushMainWindowBounds: Effect.void, + prepareCaptureReveal: Effect.void, + dispatchMenuAction: () => Effect.die("unexpected menu action"), + dispatchSnapShotEvent: () => Effect.void, + zoomMain: () => Effect.die("unexpected zoom"), + syncAppearance: Effect.void, + } satisfies DesktopWindow.DesktopWindow["Service"]), + ), + ), + Layer.provideMerge(settingsLayer), + ), + ); + const pool = yield* DesktopBackendPool.DesktopBackendPool.pipe(Effect.provide(context)); + const settings = yield* DesktopAppSettings.DesktopAppSettings.pipe(Effect.provide(context)); + + const primary = yield* pool.primary; + yield* primary.start; + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.millis(500)); + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + yield* TestClock.adjust(Duration.seconds(1)); + assert.equal(yield* Queue.take(launches), "wsl"); + assert.equal(yield* Queue.take(failures), "pid=123 code=1"); + + const dialog = yield* Queue.take(dialogs); + assert.equal(dialog.title, "WSL backend isn't responding"); + assert.equal( + dialog.content, + "The WSL backend (Ubuntu-22.04) exited before it was ready (exit code 1).\n\nT3 Code will use the Windows backend for this launch and retry WSL the next time the app starts.", + ); + assert.isFalse(dialog.content.includes("http")); + assert.isFalse(dialog.content.includes("MODULE_NOT_FOUND")); + assert.equal(yield* Queue.take(launches), "windows"); + assert.equal(yield* Queue.take(ready), "ready"); + + const stored = yield* settings.get; + assert.equal(stored.wslOnly, false); + assert.equal(stored.wslBackendEnabled, false); + assert.equal(stored.wslDistro, "Ubuntu-22.04"); + yield* primary.stop(); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); }); diff --git a/apps/desktop/src/backend/DesktopBackendPool.ts b/apps/desktop/src/backend/DesktopBackendPool.ts index 2930780d4102..17a81f7a2fb9 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.ts @@ -105,6 +105,22 @@ import * as ElectronDialog from "../electron/ElectronDialog.ts"; const { logWarning: logBackendPoolWarning } = DesktopObservability.makeComponentLogger("desktop-backend-pool"); +const startupFailureDialogBody = ( + failure: DesktopBackendManager.StartupFailure, + distro: string, +): string => { + const subject = `The WSL backend (${distro})`; + const next = + "T3 Code will use the Windows backend for this launch and retry WSL the next time the app starts."; + if (failure.kind === "exited" && failure.exitCode !== undefined) { + return `${subject} exited before it was ready (exit code ${failure.exitCode}).\n\n${next}`; + } + if (failure.kind === "exited") { + return `${subject} exited before it was ready.\n\n${next}`; + } + return `${subject} did not become ready.\n\n${next}`; +}; + export type BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const PRIMARY_INSTANCE_ID = DesktopBackendManager.PRIMARY_INSTANCE_ID; @@ -277,6 +293,33 @@ export const layer = Layer.effect( }, ); + // Preflight passed, but the primary still never became ready. The connecting + // splash has no controls, so wsl-only mode would sit there until the process + // is killed. Use Windows for this launch only — the same in-memory fallback + // as a bounded preflight failure — and try WSL again on the next launch. + // A run that already resolved to Windows has no distro and keeps restarting. + const handlePrimaryStartupFailure = Effect.fn("desktop.backendPool.primaryStartupFailed")( + function* (failure: DesktopBackendManager.StartupFailure, runningDistro: string | undefined) { + if (runningDistro === undefined) { + return false; + } + yield* logBackendPoolWarning( + "primary WSL backend did not become ready; using Windows for this launch", + { + failure: failure.kind, + ...(failure.exitCode === undefined ? {} : { exitCode: failure.exitCode }), + distro: runningDistro, + }, + ); + yield* electronDialog.showErrorBox( + "WSL backend isn't responding", + startupFailureDialogBody(failure, runningDistro), + ); + yield* appSettings.applyWslWindowsFallbackInMemory; + return true; + }, + ); + const primary = yield* DesktopBackendManager.makeBackendInstance({ id: DesktopBackendManager.PRIMARY_INSTANCE_ID, // Keep this lazy. The pool layer is initialized before startup loads @@ -301,6 +344,7 @@ export const layer = Layer.effect( ), onShutdown: () => desktopWindow.handleBackendNotReady, onPreflightFailed: handlePrimaryPreflightFailure, + onStartupFailed: handlePrimaryStartupFailure, }); const instancesRef = yield* SynchronizedRef.make<