Skip to content
119 changes: 90 additions & 29 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import {
canAutoFix,
doctorPromptsAllowed,
dryRunPlan,
failureReport,
parseEnvFile,
type Diagnostic,
type DiagnosticFixResult,
Expand Down Expand Up @@ -104,6 +105,7 @@ import { bearerHeaders, resolveClientSecret } from "./secret-store.js";
import { assertComposePortsFree, composeCall, composeRegistryFailureMessage, composeStateRegistryFailure, composeWorkerFailureSummary, ownedComposeProject, readComposeLogFromOffset, startComposeDaemon, writeComposeState, type ComposeStatus } from "./cli/compose-control.js";
import { COMPOSE_WORKER_VERSIONS, composeDownTimeoutMs, composeNamespace } from "./cli/compose-project.js";
import { legacyDockerStateMessage, type LegacyDockerEngineState } from "./cli/legacy-docker-state.js";
import { DEFAULT_START_COMMAND, engineOnlyLines, probeWorkerConnection, shellArg, tcpPortOpen, unrecognizedServiceLines, type WorkerConnection } from "./cli/worker-connection.js";
import { NO_HOOK_CAPTURE, hookCaptureMissing } from "./functions/status.js";
import type { CaptureStatus } from "./functions/capture.js";

Expand Down Expand Up @@ -1842,6 +1844,61 @@ async function apiFetch<T = unknown>(base: string, path: string, timeoutMs = 500
}
}

function recoveryTargetArgs(): string {
const parts: string[] = [];
if (hasInstanceArgument) parts.push(`--instance ${selectedInstance}`);
if (portValue || (!hasInstanceArgument && getRestPort() !== 3111)) parts.push(`--port ${getRestPort()}`);
const dataDir = cliArgValue("--data-dir");
if (dataDir) parts.push(`--data-dir ${shellArg(dataDir)}`);
return parts.map((part) => ` ${part}`).join("");
}

function recoveryStartCommand(): string {
return `${DEFAULT_START_COMMAND}${recoveryTargetArgs()}`;
}

function workerEnginePort(): number {
const workerUrl = process.env["III_URL"];
if (!process.env["III_ENGINE_URL"] && workerUrl) {
try {
const parsed = new URL(workerUrl).port;
if (parsed) return parseInt(parsed, 10);
} catch {}
}
return getEnginePort();
}

function probeTargetWorker(base: string): Promise<WorkerConnection> {
return probeWorkerConnection(base, {
headers: bearerHeaders(base),
engineBusReachable: async () => tcpPortOpen(new URL(base).hostname.replace(/^\[|\]$/g, ""), workerEnginePort()),
});
}

async function savedComposeFailure(state: ComposeEngineState): Promise<string | null> {
if (!state.composePid || !pidAlive(state.composePid)) return null;
try {
const composeStatus = await composeCall<ComposeStatus>({
engineUrl: state.engineUrl,
namespace: state.namespace,
functionId: "compose::status",
file: state.file,
});
return composeWorkerFailureSummary(ownedComposeProject(composeStatus, state.file));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
p.log.warn(`Worker Compose status unavailable: ${message}`);
return null;
}
}

function exitWithComposeFailure(state: ComposeEngineState, failure: string, unavailable: string): never {
p.log.error(`Worker Compose failed after exhausting restart attempts; ${unavailable}.`);
p.log.error(failure);
p.log.info(`Inspect ${state.logFile}, then run agentmemory stop before retrying.`);
process.exit(1);
}

async function runStatus() {
const base = getBaseUrl();
p.intro("agentmemory status");
Expand All @@ -1854,32 +1911,29 @@ async function runStatus() {

const up = await isEngineRunning();
if (!up) {
if (savedState?.kind === "compose" && savedState.composePid && pidAlive(savedState.composePid)) {
try {
const composeStatus = await composeCall<ComposeStatus>({
engineUrl: savedState.engineUrl,
namespace: savedState.namespace,
functionId: "compose::status",
file: savedState.file,
});
const project = ownedComposeProject(composeStatus, savedState.file);
const failure = composeWorkerFailureSummary(project);
if (failure) {
p.log.error(`Worker Compose failed after exhausting restart attempts; REST is unavailable at ${base}.`);
p.log.error(failure);
p.log.info(`Inspect ${savedState.logFile}, then run agentmemory stop before retrying.`);
process.exit(1);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
p.log.warn(`Worker Compose status unavailable: ${message}`);
}
if (savedState?.kind === "compose") {
const failure = await savedComposeFailure(savedState);
if (failure) exitWithComposeFailure(savedState, failure, `REST is unavailable at ${base}`);
}
p.log.error(`Not running — no response at ${base}`);
p.log.info("Start with: npx @agentmemory/agentmemory");
process.exit(1);
}

const connection = await probeTargetWorker(base);
const workerMissing = connection !== "connected";
const composeExplainsMissingRoutes = workerMissing
&& savedState?.kind === "compose"
&& (await savedComposeFailure(savedState)) !== null;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (workerMissing && !composeExplainsMissingRoutes) {
let lines: string[];
if (connection === "engine-only") lines = engineOnlyLines(base, recoveryStartCommand());
else if (connection === "unrecognized") lines = unrecognizedServiceLines(base);
else lines = [`A server answers at ${base}, but /agentmemory/livez and /agentmemory/health did not respond.`, "Check the agentmemory logs, then run status again."];
p.log.error(lines.join("\n"));
process.exit(1);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

try {
const [healthRes, sessionsRes, graphRes, memoriesRes, flagsRes, followupRes] = await Promise.all([
apiFetch<any>(base, "health"),
Expand Down Expand Up @@ -2072,9 +2126,17 @@ function buildDoctorContext(): DoctorContext {
pidfilePath: readEngineState()?.kind === "compose" ? engineStatePath() : enginePidfilePath(),
enginePath: engineStatePath(),
pinnedVersion: IIPINNED_VERSION,
startCommand: recoveryStartCommand(),
targetArgs: recoveryTargetArgs(),
};
}

function doctorTargetsManagedCompose(): boolean {
const state = readEngineState();
const remoteTarget = shouldUseAgentmemoryUrl() && process.env["AGENTMEMORY_URL"];
return state?.kind === "compose" && state.restPort === getRestPort() && !remoteTarget;
}

function buildDoctorEffects(): DoctorEffects {
return {
envFileExists: () => existsSync(doctorEnvPath()),
Expand Down Expand Up @@ -2109,11 +2171,11 @@ function buildDoctorEffects(): DoctorEffects {
return state?.kind === "compose" ? state.binPath : privateIiiPath();
},
iiiBinaryVersion: (binPath: string) => iiiBinVersion(binPath),
externalEngineRunning: async () => {
const state = readEngineState();
const remoteTarget = shouldUseAgentmemoryUrl() && process.env["AGENTMEMORY_URL"];
const managed = state?.kind === "compose" && state.restPort === getRestPort() && !remoteTarget;
return !managed && await isEngineRunning();
externalEngineRunning: async () => !doctorTargetsManagedCompose() && await isEngineRunning(),
workerConnection: async () => {
const connection = await probeTargetWorker(getBaseUrl());
if (connection === "connected" || connection === "unreachable") return connection;
return doctorTargetsManagedCompose() ? "managed-worker-down" : connection;
},
viewerReachable: async (timeoutMs = 2000) => {
try {
Expand Down Expand Up @@ -2421,16 +2483,15 @@ async function runDoctor() {
continue;
}
failed++;
p.log.warn(`${d.id} ✗ ${status.detail ?? ""}`.trim());
const report = failureReport(d, status);
p.log.warn(report.problem);
p.log.info(report.fix);

if (!canAutoFix(d, status)) {
p.log.info(`manual fix: ${status.manualFix ?? d.fixPreview}`);
skipped++;
continue;
}

p.log.info(`why: ${d.fixPreview}`);

if (applyAll) {
const r = await applyFixWithReport(d, ctx, false);
if (r.ok) fixed++;
Expand Down
79 changes: 73 additions & 6 deletions src/cli/doctor-diagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
llmOptInHint,
parseEnvContent,
} from "../config.js";
import { DEFAULT_START_COMMAND, ENGINE_ONLY_DOCKER_GUIDE, engineOnlyFix, UNRECOGNIZED_DOCKER_NOTE, UNRECOGNIZED_SERVICE_FIX, type WorkerConnection } from "./worker-connection.js";

export type DiagnosticStatus = {
ok: boolean;
Expand All @@ -50,8 +51,12 @@ export type DoctorContext = {
enginePath: string;
/** Pinned engine version. */
pinnedVersion: string;
startCommand?: string;
targetArgs?: string;
};

export type DoctorWorkerConnection = WorkerConnection | "managed-worker-down";

export type Diagnostic = {
/** Stable id. Used in --json and tests. */
id: string;
Expand All @@ -73,6 +78,13 @@ export function canAutoFix(d: Diagnostic, status?: DiagnosticStatus): boolean {
return !d.manualOnly && !status?.manualFix;
}

export function failureReport(d: Diagnostic, status: DiagnosticStatus): { problem: string; fix: string } {
return {
problem: `${d.id} ✗ ${d.message}${status.detail ? ` (${status.detail})` : ""}`,
fix: `${canAutoFix(d, status) ? "fix" : "manual fix"}: ${status.manualFix ?? d.fixPreview}`,
};
}

export function doctorPromptsAllowed(
env: NodeJS.ProcessEnv,
stdinIsTTY: boolean | undefined,
Expand All @@ -87,6 +99,7 @@ export const DIAGNOSTIC_IDS = [
"no-llm-provider-key",
"embedding-provider-not-chosen",
"engine-version-mismatch",
"worker-not-connected",
"viewer-unreachable",
"stale-pidfile",
"env-placeholder-keys",
Expand Down Expand Up @@ -143,6 +156,7 @@ export type DoctorEffects = {
/** Run `iii --version`; null if it fails. */
iiiBinaryVersion: (binPath: string) => string | null;
externalEngineRunning: () => Promise<boolean>;
workerConnection: () => Promise<DoctorWorkerConnection>;
/** Probe the viewer URL; true if it returns OK within timeoutMs. */
viewerReachable: (timeoutMs?: number) => Promise<boolean>;
/** Run init logic (copies .env.example). */
Expand Down Expand Up @@ -289,6 +303,47 @@ export function buildDiagnostics(effects: DoctorEffects): Diagnostic[] {
return effects.runStart();
},
},
{
id: "worker-not-connected",
message: "The server answers, but the agentmemory worker routes return 404.",
fixPreview: engineOnlyFix(),
moreInfo:
"The REST port answers, but /agentmemory/livez and /agentmemory/health return 404. " +
"Only the agentmemory worker registers those routes, so the engine runs alone: status shows " +
"no version and no health, and the viewer is not served. Old Docker images from the legacy " +
"CLI Docker mode ran only the engine. Migration guide: " +
ENGINE_ONLY_DOCKER_GUIDE,
manualOnly: true,
check: async (ctx) => {
const connection = await effects.workerConnection();
const startCommand = ctx.startCommand ?? DEFAULT_START_COMMAND;
const targetArgs = ctx.targetArgs ?? "";
if (connection === "engine-only") {
return {
ok: false,
detail: `engine at ${ctx.baseUrl} has no agentmemory routes (404)`,
manualFix: engineOnlyFix(startCommand),
};
}
if (connection === "managed-worker-down") {
return {
ok: false,
detail: `managed Worker Compose engine at ${ctx.baseUrl} has no agentmemory routes (404)`,
manualFix: `Run agentmemory status${targetArgs} to see the Worker Compose failure report, then run agentmemory stop${targetArgs} and start again with: ${startCommand}.`,
};
}
if (connection === "unrecognized") {
return {
ok: false,
detail: `server at ${ctx.baseUrl} returns 404 for the agentmemory routes and the iii engine port does not answer`,
manualFix: `${UNRECOGNIZED_SERVICE_FIX} ${UNRECOGNIZED_DOCKER_NOTE}`,
};
}
if (connection === "unreachable") return { ok: true, detail: "server not reachable (see Server reachable)" };
return { ok: true };
},
fix: async (ctx) => ({ ok: false, message: engineOnlyFix(ctx.startCommand) }),
},
{
id: "viewer-unreachable",
message: "Viewer port not reachable.",
Expand Down Expand Up @@ -357,14 +412,16 @@ export function buildDiagnostics(effects: DoctorEffects): Diagnostic[] {
{
id: "iii-on-path-not-local-bin",
message:
"iii is on PATH but agentmemory's private install is absent or mismatched.",
"agentmemory's own iii at ~/.agentmemory/bin/iii is missing or a different version, and another iii is on your PATH.",
fixPreview:
"Install the pinned version to ~/.agentmemory/bin — won't touch your PATH.",
"Run npx -y @agentmemory/agentmemory@latest. It installs the pinned iii to ~/.agentmemory/bin/iii and does not change your PATH or the other iii.",
moreInfo:
"agentmemory installs its pinned engine to ~/.agentmemory/bin/iii so a " +
"agentmemory keeps its pinned engine at ~/.agentmemory/bin/iii so a " +
"user-managed iii on PATH (homebrew, cargo, manual install) stays untouched. " +
"When agentmemory needs the pin and PATH doesn't have it, it falls back to the " +
"private install. If neither exists, run the installer.",
"When that copy is missing or has a different version, startup uses the iii on PATH " +
"if it matches the pin. If the PATH iii has a different version, startup installs the " +
"pinned copy into ~/.agentmemory/bin first. The engine-version-mismatch check shows " +
"which version each binary reports.",
manualOnly: true,
check: async (ctx) => {
const bin = effects.findIiiBinary();
Expand All @@ -378,7 +435,17 @@ export function buildDiagnostics(effects: DoctorEffects): Diagnostic[] {
detail: `private install pinned; PATH iii at ${bin} stays untouched`,
};
}
return { ok: false, detail: `iii at: ${bin}` };
const pathVersion = effects.iiiBinaryVersion(bin);
const detail = `PATH iii at ${bin} is ${pathVersion ?? "an unknown version"} (pinned ${ctx.pinnedVersion})`;
if (pathVersion === ctx.pinnedVersion) {
return {
ok: false,
detail,
manualFix:
"The PATH iii is the pinned version, so startup can use it and no action is needed. To give agentmemory its own copy in ~/.agentmemory/bin, run npx -y @agentmemory/agentmemory@latest upgrade and accept the iii-engine install step.",
};
}
return { ok: false, detail };
},
fix: async () =>
effects.runIiiInstaller().then((r) => ({
Expand Down
Loading
Loading