diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs index 84955ed..1fe898d 100644 --- a/.dependency-cruiser.cjs +++ b/.dependency-cruiser.cjs @@ -25,6 +25,18 @@ module.exports = { from: { path: "^apps/api-ts/src/domains/providers/services/" }, to: { path: "^apps/api-ts/src/app/" }, }, + { + name: "providers-must-not-import-threads-domain", + severity: "error", + from: { + path: "^apps/api-ts/src/domains/providers/", + pathNot: "\\.test\\.ts$", + }, + to: { + path: "^apps/api-ts/src/domains/threads/", + pathNot: "^apps/api-ts/src/domains/threads/query\\.ts$", + }, + }, ], options: { doNotFollow: { diff --git a/apps/api-ts/src/domains/providers/actions.test.ts b/apps/api-ts/src/domains/providers/actions.test.ts index 351ea66..13ad676 100644 --- a/apps/api-ts/src/domains/providers/actions.test.ts +++ b/apps/api-ts/src/domains/providers/actions.test.ts @@ -7,9 +7,9 @@ import { CODEX_HOME, COPILOT_VSCODE_GLOBAL, GEMINI_TMP_DIR, - BACKUP_ROOT, HOME_DIR, -} from "../../lib/constants.js"; +} from "./constants.js"; +import { BACKUP_ROOT } from "../recovery/constants.js"; import { deriveProviderBackupRelativePath, runProviderSessionAction } from "./actions.js"; import type { ProviderId, ProviderSessionAction } from "./types.js"; @@ -171,7 +171,7 @@ describe("runProviderSessionAction", () => { process.env.THREADLENS_STATE_DIR = path.join(root, "state"); vi.resetModules(); - const constants = await import("../../lib/constants.js"); + const constants = await import("./constants.js"); const { runProviderSessionAction: runAction } = await import("./actions.js"); const sourceDir = path.join(constants.CODEX_HOME, "sessions", "backup-root-smoke"); const sourcePath = path.join(sourceDir, "session.jsonl"); @@ -225,7 +225,7 @@ describe("runProviderSessionAction", () => { process.env.THREADLENS_STATE_DIR = path.join(root, "state"); vi.resetModules(); - const constants = await import("../../lib/constants.js"); + const constants = await import("./constants.js"); const { runProviderSessionAction: runAction } = await import("./actions.js"); const sourceDir = path.join(constants.CODEX_HOME, "sessions", "backup-dest-smoke"); const sourcePath = path.join(sourceDir, "session.jsonl"); @@ -333,7 +333,7 @@ describe("runProviderSessionAction", () => { nowIsoUtc: () => fixedNow, })); - const constants = await import("../../lib/constants.js"); + const constants = await import("./constants.js"); const { runProviderSessionAction: runAction } = await import("./actions.js"); const sourceDir = path.join(constants.CODEX_HOME, "sessions", "manifest-smoke"); const sourcePath = path.join(sourceDir, "session.jsonl"); @@ -507,6 +507,342 @@ describe("runProviderSessionAction", () => { } }); + it("does not delete Codex sessions when the archive root is a symlink outside the provider tree", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-archive-root-")); + + try { + const homeDir = path.join(root, "home"); + const outsideDir = path.join(root, "outside"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + + const constants = await import("./constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const sourceDir = path.join(constants.CODEX_HOME, "sessions", "archive-symlink-smoke"); + const sourcePath = path.join(sourceDir, "session.jsonl"); + const archivedRootLink = path.join(constants.CODEX_HOME, "archived_sessions"); + const outsideTargetPath = path.join(outsideDir, "archive-symlink-smoke", "session.jsonl"); + const payload = "{\"type\":\"session\",\"id\":\"archive-symlink-smoke\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => sourcePath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(sourceDir, { recursive: true }); + await mkdir(outsideDir, { recursive: true }); + await writeFile(sourcePath, payload, "utf8"); + await symlink(outsideDir, archivedRootLink, "dir"); + + const preview = await runAction( + deps, + "codex", + "archive_local", + [sourcePath], + true, + "", + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "archive_local", + [sourcePath], + false, + preview.confirm_token_expected, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + step: "archive_local", + error: "unsafe-archived-session-target", + }), + ]), + ); + await expect(readFile(sourcePath, "utf8")).resolves.toBe(payload); + await expect(stat(outsideTargetPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + + it("does not restore archived Codex sessions through a symlinked sessions subdirectory", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-restore-root-")); + + try { + const homeDir = path.join(root, "home"); + const outsideDir = path.join(root, "outside"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + + const constants = await import("./constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const archivedDir = path.join(constants.CODEX_HOME, "archived_sessions", "restore-symlink-smoke"); + const archivedPath = path.join(archivedDir, "session.jsonl"); + const sessionsRoot = path.join(constants.CODEX_HOME, "sessions"); + const restoreLink = path.join(sessionsRoot, "restore-symlink-smoke"); + const outsideTargetPath = path.join(outsideDir, "session.jsonl"); + const payload = "{\"type\":\"session\",\"id\":\"restore-symlink-smoke\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => archivedPath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(archivedDir, { recursive: true }); + await mkdir(sessionsRoot, { recursive: true }); + await mkdir(outsideDir, { recursive: true }); + await writeFile(archivedPath, payload, "utf8"); + await symlink(outsideDir, restoreLink, "dir"); + + const preview = await runAction( + deps, + "codex", + "unarchive_local", + [archivedPath], + true, + "", + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "unarchive_local", + [archivedPath], + false, + preview.confirm_token_expected, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + step: "unarchive_local", + error: "unsafe-archived-session-target", + }), + ]), + ); + await expect(readFile(archivedPath, "utf8")).resolves.toBe(payload); + await expect(stat(outsideTargetPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + + it("does not archive Codex sessions when backup-before-delete copy fails", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-archive-backup-")); + const fixedNow = "2026-05-14T00:00:00.000Z"; + const folderName = "2026-05-14T00-00-00-000Z-archive_local"; + + try { + const homeDir = path.join(root, "home"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + vi.doMock("../../lib/utils.js", async (importOriginal) => ({ + ...(await importOriginal()), + nowIsoUtc: () => fixedNow, + })); + + const constants = await import("./constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const sourceDir = path.join(constants.CODEX_HOME, "sessions", "archive-backup-failure"); + const sourcePath = path.join(sourceDir, "session.jsonl"); + const archivedPath = path.join(constants.CODEX_HOME, "archived_sessions", "archive-backup-failure", "session.jsonl"); + const backupRoot = path.join(constants.HOME_DIR, "ThreadLens Test Backups"); + const conflictingBackupTarget = path.join( + backupRoot, + "provider_actions", + "codex", + folderName, + "sessions", + "archive-backup-failure", + "session.jsonl", + ); + const payload = "{\"type\":\"session\",\"id\":\"archive-backup-failure\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => sourcePath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(sourceDir, { recursive: true }); + await mkdir(conflictingBackupTarget, { recursive: true }); + await writeFile(sourcePath, payload, "utf8"); + + const preview = await runAction( + deps, + "codex", + "archive_local", + [sourcePath], + true, + "", + { backup_before_delete: true, backup_root: backupRoot }, + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "archive_local", + [sourcePath], + false, + preview.confirm_token_expected, + { backup_before_delete: true, backup_root: backupRoot }, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ step: "archive_local:backup_copy" }), + ]), + ); + await expect(readFile(sourcePath, "utf8")).resolves.toBe(payload); + await expect(stat(archivedPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.doUnmock("../../lib/utils.js"); + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + + it("does not restore archived Codex sessions when backup-before-delete copy fails", async () => { + const previousHome = process.env.HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousStateDir = process.env.THREADLENS_STATE_DIR; + const root = await mkdtemp(path.join(os.tmpdir(), "threadlens-provider-unarchive-backup-")); + const fixedNow = "2026-05-14T00:00:00.000Z"; + const folderName = "2026-05-14T00-00-00-000Z-unarchive_local"; + + try { + const homeDir = path.join(root, "home"); + process.env.HOME = homeDir; + process.env.CODEX_HOME = path.join(homeDir, ".codex"); + process.env.THREADLENS_STATE_DIR = path.join(root, "state"); + vi.resetModules(); + vi.doMock("../../lib/utils.js", async (importOriginal) => ({ + ...(await importOriginal()), + nowIsoUtc: () => fixedNow, + })); + + const constants = await import("./constants.js"); + const { runProviderSessionAction: runAction } = await import("./actions.js"); + const archivedDir = path.join(constants.CODEX_HOME, "archived_sessions", "unarchive-backup-failure"); + const archivedPath = path.join(archivedDir, "session.jsonl"); + const restoredPath = path.join(constants.CODEX_HOME, "sessions", "unarchive-backup-failure", "session.jsonl"); + const backupRoot = path.join(constants.HOME_DIR, "ThreadLens Test Backups"); + const conflictingBackupTarget = path.join( + backupRoot, + "provider_actions", + "codex", + folderName, + "archived_sessions", + "unarchive-backup-failure", + "session.jsonl", + ); + const payload = "{\"type\":\"session\",\"id\":\"unarchive-backup-failure\"}\n"; + const deps = { + resolveAllowedProviderFilePath: async () => archivedPath, + supportsProviderAction: () => true, + invalidateProviderCaches: () => undefined, + }; + + await mkdir(archivedDir, { recursive: true }); + await mkdir(conflictingBackupTarget, { recursive: true }); + await writeFile(archivedPath, payload, "utf8"); + + const preview = await runAction( + deps, + "codex", + "unarchive_local", + [archivedPath], + true, + "", + { backup_before_delete: true, backup_root: backupRoot }, + ); + expect(preview.ok).toBe(true); + expect(preview.confirm_token_expected).toMatch(/^PROVIDER-/); + + const result = await runAction( + deps, + "codex", + "unarchive_local", + [archivedPath], + false, + preview.confirm_token_expected, + { backup_before_delete: true, backup_root: backupRoot }, + ); + + expect(result.ok).toBe(false); + expect(result.mode).toBe("failed"); + expect(result.applied_count).toBe(0); + expect(result.failed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ step: "unarchive_local:backup_copy" }), + ]), + ); + await expect(readFile(archivedPath, "utf8")).resolves.toBe(payload); + await expect(stat(restoredPath)).rejects.toThrow(); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousStateDir === undefined) delete process.env.THREADLENS_STATE_DIR; + else process.env.THREADLENS_STATE_DIR = previousStateDir; + vi.doUnmock("../../lib/utils.js"); + vi.resetModules(); + await rm(root, { recursive: true, force: true }); + } + }); + it("archives and restores Codex sessions when safe paths are realpath-normalized", async () => { const previousHome = process.env.HOME; const previousCodexHome = process.env.CODEX_HOME; @@ -520,7 +856,7 @@ describe("runProviderSessionAction", () => { process.env.THREADLENS_STATE_DIR = path.join(root, "state"); vi.resetModules(); - const constants = await import("../../lib/constants.js"); + const constants = await import("./constants.js"); const { runProviderSessionAction: runAction } = await import("./actions.js"); const sourceDir = path.join(constants.CODEX_HOME, "sessions", "realpath-smoke"); diff --git a/apps/api-ts/src/domains/providers/adapters.test.ts b/apps/api-ts/src/domains/providers/adapters.test.ts index f826e7e..612fb74 100644 --- a/apps/api-ts/src/domains/providers/adapters.test.ts +++ b/apps/api-ts/src/domains/providers/adapters.test.ts @@ -51,6 +51,15 @@ describe("provider adapters", () => { expect("capabilities" in (getProviderAdapter("gemini") ?? {})).toBe(false); }); + it("keeps session scanning as an optional adapter extension point", () => { + for (const provider of IMPLEMENTED_PROVIDER_IDS) { + const adapter = getProviderAdapter(provider); + expect(adapter).toBeDefined(); + expect(adapter?.scanRoots).toEqual(expect.any(Function)); + expect(adapter?.scanSessions).toBeUndefined(); + } + }); + it("defines session locators without forcing DB-backed providers into fake paths", () => { const fileLocator: ProviderSessionLocator = { kind: "file", diff --git a/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts b/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts index ac766ba..3e230ea 100644 --- a/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts +++ b/apps/api-ts/src/domains/providers/adapters/chatgpt/roots.ts @@ -1,6 +1,6 @@ import { readdir } from "node:fs/promises"; import path from "node:path"; -import { CHAT_DIR } from "../../../../lib/constants.js"; +import { CHAT_DIR } from "../../constants.js"; import { pathExists } from "../../../../lib/utils.js"; import type { ProviderRootSpec, diff --git a/apps/api-ts/src/domains/providers/adapters/claude/title.ts b/apps/api-ts/src/domains/providers/adapters/claude/title.ts index 9a9a10b..5167106 100644 --- a/apps/api-ts/src/domains/providers/adapters/claude/title.ts +++ b/apps/api-ts/src/domains/providers/adapters/claude/title.ts @@ -1,4 +1,4 @@ -import { CLAUDE_PROJECTS_DIR } from "../../../../lib/constants.js"; +import { CLAUDE_PROJECTS_DIR } from "../../constants.js"; import { isRecord, readFileTail, safeJsonParse } from "../../../../lib/utils.js"; import { isPathInsideRoot } from "../../path-safety.js"; import { normalizeDetectedTitle } from "../../title-normalization.js"; diff --git a/apps/api-ts/src/domains/providers/adapters/codex/roots.ts b/apps/api-ts/src/domains/providers/adapters/codex/roots.ts index 9142b8a..27a93ce 100644 --- a/apps/api-ts/src/domains/providers/adapters/codex/roots.ts +++ b/apps/api-ts/src/domains/providers/adapters/codex/roots.ts @@ -1,10 +1,10 @@ import { readdir } from "node:fs/promises"; import path from "node:path"; import { - BACKUP_ROOT, CODEX_HOME, HOME_DIR, -} from "../../../../lib/constants.js"; +} from "../../constants.js"; +import { BACKUP_ROOT } from "../../../recovery/constants.js"; import type { ProviderRootSpec, } from "../../types.js"; diff --git a/apps/api-ts/src/domains/providers/adapters/codex/title-map.ts b/apps/api-ts/src/domains/providers/adapters/codex/title-map.ts index 4559f58..62570c3 100644 --- a/apps/api-ts/src/domains/providers/adapters/codex/title-map.ts +++ b/apps/api-ts/src/domains/providers/adapters/codex/title-map.ts @@ -1,6 +1,6 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; -import { CODEX_HOME } from "../../../../lib/constants.js"; +import { CODEX_HOME } from "../../constants.js"; import { isRecord, safeJsonParse } from "../../../../lib/utils.js"; import { normalizeDetectedTitle } from "../../title-normalization.js"; diff --git a/apps/api-ts/src/domains/providers/adapters/gemini/health.ts b/apps/api-ts/src/domains/providers/adapters/gemini/health.ts index a1478fc..4197d9c 100644 --- a/apps/api-ts/src/domains/providers/adapters/gemini/health.ts +++ b/apps/api-ts/src/domains/providers/adapters/gemini/health.ts @@ -3,7 +3,7 @@ import { GEMINI_HISTORY_DIR, GEMINI_HOME, GEMINI_TMP_DIR, -} from "../../../../lib/constants.js"; +} from "../../constants.js"; import { countFilesRecursiveByExt, pathExists } from "../../../../lib/utils.js"; import type { ProviderHealthEvidence } from "../../types.js"; diff --git a/apps/api-ts/src/domains/providers/capabilities.ts b/apps/api-ts/src/domains/providers/capabilities.ts index 892fca8..334b141 100644 --- a/apps/api-ts/src/domains/providers/capabilities.ts +++ b/apps/api-ts/src/domains/providers/capabilities.ts @@ -79,7 +79,7 @@ export function listProviderActionProviderIds(): ProviderId[] { return implementedCapabilities() .filter( (capability) => - capability.read_sessions || capability.safe_cleanup || capability.hard_delete, + capability.safe_cleanup || capability.hard_delete, ) .map((capability) => capability.id); } @@ -91,7 +91,7 @@ export function supportsProviderAction( const capability = findProviderCapability(provider); if (!capability) return false; if (action === "backup_local") { - return capability.read_sessions === true; + return capability.read_sessions === true && capability.safe_cleanup === true; } if (action === "delete_local") { return capability.safe_cleanup === true && capability.hard_delete === true; diff --git a/apps/api-ts/src/domains/providers/constants.ts b/apps/api-ts/src/domains/providers/constants.ts new file mode 100644 index 0000000..20b038f --- /dev/null +++ b/apps/api-ts/src/domains/providers/constants.ts @@ -0,0 +1,82 @@ +import path from "node:path"; +import { + APP_DATA_DIR, + HOME_DIR, + resolvePlatformAppDataDir, + resolvePlatformHomeDir, +} from "../../platform/paths.js"; + +export { + APP_DATA_DIR, + HOME_DIR, + resolvePlatformAppDataDir, + resolvePlatformHomeDir, +}; + +export const PROJECTS_DIR = String( + process.env.THREADLENS_PROJECTS_DIR ?? process.env.PROJECTS_DIR ?? "", +).trim(); + +export function resolvePlatformChatDir( + platform = process.platform, + env: Record = process.env, +) { + if (platform === "darwin") { + return path.join( + resolvePlatformHomeDir(platform, env), + "Library", + "Application Support", + "com.openai.chat", + ); + } + return path.join(resolvePlatformAppDataDir(platform, env), "com.openai.chat"); +} + +export const CHAT_DIR = resolvePlatformChatDir(); + +export const CODEX_HOME = + process.env.CODEX_HOME ?? path.join(HOME_DIR, ".codex"); +export const CODEX_GLOBAL_STATE_FILE = path.join( + CODEX_HOME, + ".codex-global-state.json", +); + +export const CLAUDE_HOME = path.join(HOME_DIR, ".claude"); +export const CLAUDE_PROJECTS_DIR = path.join(CLAUDE_HOME, "projects"); +export const CLAUDE_TRANSCRIPTS_DIR = path.join(CLAUDE_HOME, "transcripts"); + +export const GEMINI_HOME = path.join(HOME_DIR, ".gemini"); +export const GEMINI_HISTORY_DIR = path.join(GEMINI_HOME, "history"); +export const GEMINI_TMP_DIR = path.join(GEMINI_HOME, "tmp"); +export const GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR = path.join( + GEMINI_HOME, + "antigravity", + "conversations", +); + +export const COPILOT_VSCODE_GLOBAL = path.join( + APP_DATA_DIR, + "Code", + "User", + "globalStorage", + "github.copilot-chat", +); +export const COPILOT_VSCODE_WORKSPACE_STORAGE = path.join( + APP_DATA_DIR, + "Code", + "User", + "workspaceStorage", +); +export const COPILOT_CURSOR_GLOBAL = path.join( + APP_DATA_DIR, + "Cursor", + "User", + "globalStorage", + "github.copilot-chat", +); +export const COPILOT_CURSOR_WORKSPACE_STORAGE = path.join( + APP_DATA_DIR, + "Cursor", + "User", + "workspaceStorage", +); diff --git a/apps/api-ts/src/domains/providers/index.test.ts b/apps/api-ts/src/domains/providers/index.test.ts index 2253691..201b334 100644 --- a/apps/api-ts/src/domains/providers/index.test.ts +++ b/apps/api-ts/src/domains/providers/index.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import type { ProviderId } from "@threadlens/shared-contracts"; -import { APP_DATA_DIR, CHAT_DIR } from "../../lib/constants.js"; +import { APP_DATA_DIR, CHAT_DIR } from "./constants.js"; import { buildProviderActionToken, codexTranscriptSearchRoots, diff --git a/apps/api-ts/src/domains/providers/matrix.test.ts b/apps/api-ts/src/domains/providers/matrix.test.ts index 89284b9..3e94e0d 100644 --- a/apps/api-ts/src/domains/providers/matrix.test.ts +++ b/apps/api-ts/src/domains/providers/matrix.test.ts @@ -9,8 +9,7 @@ vi.mock("@threadlens/shared-contracts", async (importOriginal) => { }; }); -vi.mock("../../lib/constants.js", () => ({ - BACKUP_ROOT: "/mock/backups", +vi.mock("./constants.js", () => ({ CHAT_DIR: "/mock/chat", CODEX_HOME: "/mock/codex", CLAUDE_HOME: "/mock/claude", @@ -26,6 +25,10 @@ vi.mock("../../lib/constants.js", () => ({ GEMINI_TMP_DIR: "/mock/gemini/tmp", })); +vi.mock("../recovery/constants.js", () => ({ + BACKUP_ROOT: "/mock/backups", +})); + vi.mock("../../lib/utils.js", () => ({ countFilesRecursiveByExt: vi.fn(async () => 1), countJsonlFilesRecursive: vi.fn(async () => 1), @@ -56,6 +59,7 @@ vi.mock("./probe.js", () => ({ import { listProviderAdapters } from "./registry.js"; import { getProviderMatrixTs, invalidateProviderMatrixCache } from "./matrix.js"; +import { buildProviderMatrixProviders } from "./services/matrix/provider-entries.js"; describe("provider matrix notes", () => { beforeEach(() => { @@ -139,4 +143,35 @@ describe("provider matrix notes", () => { }, }); }); + + it("does not report Codex cleanup readiness when its runtime roots are missing", () => { + const providers = buildProviderMatrixProviders({ + codexHomes: ["/mock/codex"], + codexRootExists: false, + codexSessionLogs: 0, + chatGptRootExists: false, + chatGptSessionLogs: 0, + claudeRootExists: false, + claudeSessionLogs: 0, + geminiRootExists: false, + geminiSessionLogs: 0, + geminiRoots: [], + geminiNotes: "History, tmp, and checkpoint files.", + copilotProviderRoots: [], + copilotRootExists: false, + copilotSignalFiles: 0, + }); + const codex = providers.find((provider) => provider.provider === "codex"); + + expect(codex).toMatchObject({ + status: "missing", + capability_level: "unavailable", + capabilities: { + read_sessions: false, + analyze_context: false, + safe_cleanup: false, + hard_delete: false, + }, + }); + }); }); diff --git a/apps/api-ts/src/domains/providers/parser-fixtures.test.ts b/apps/api-ts/src/domains/providers/parser-fixtures.test.ts index bc28d8a..308076a 100644 --- a/apps/api-ts/src/domains/providers/parser-fixtures.test.ts +++ b/apps/api-ts/src/domains/providers/parser-fixtures.test.ts @@ -23,7 +23,7 @@ describe("provider parser fixtures", () => { process.env.CODEX_HOME = path.join(tempHome, ".codex"); vi.resetModules(); const appDataDir = path.join(tempHome, ".config"); - vi.doMock("../../lib/constants.js", () => ({ + vi.doMock("./constants.js", () => ({ HOME_DIR: tempHome, APP_DATA_DIR: appDataDir, CODEX_HOME: path.join(tempHome, ".codex"), @@ -87,7 +87,7 @@ describe("provider parser fixtures", () => { process.env.CODEX_HOME = originalCodexHome; } vi.resetModules(); - vi.doUnmock("../../lib/constants.js"); + vi.doUnmock("./constants.js"); if (tempHome) { await rm(tempHome, { recursive: true, force: true }); } diff --git a/apps/api-ts/src/domains/providers/path-safety.test.ts b/apps/api-ts/src/domains/providers/path-safety.test.ts index d6b3562..1e48078 100644 --- a/apps/api-ts/src/domains/providers/path-safety.test.ts +++ b/apps/api-ts/src/domains/providers/path-safety.test.ts @@ -6,7 +6,7 @@ import { getProviderCapability, type ProviderId, } from "@threadlens/shared-contracts"; -import { APP_DATA_DIR, CHAT_DIR, CODEX_HOME } from "../../lib/constants.js"; +import { APP_DATA_DIR, CHAT_DIR, CODEX_HOME } from "./constants.js"; import { IMPLEMENTED_PROVIDER_IDS, } from "./capabilities.js"; diff --git a/apps/api-ts/src/domains/providers/registry.ts b/apps/api-ts/src/domains/providers/registry.ts index 844c965..0008540 100644 --- a/apps/api-ts/src/domains/providers/registry.ts +++ b/apps/api-ts/src/domains/providers/registry.ts @@ -7,7 +7,12 @@ import { } from "./capabilities.js"; import { getGeminiProviderHealth } from "./adapters/gemini/health.js"; import { providerRootSpecs, providerScanRootSpecs } from "./provider-roots.js"; -import type { ProviderHealthEvidence, ProviderRootSpec } from "./types.js"; +import type { + ProviderHealthEvidence, + ProviderRootSpec, + ProviderSessionCandidate, + ProviderSessionRow, +} from "./types.js"; export type ProviderSessionLocator = | { kind: "file"; file_path: string } @@ -18,6 +23,7 @@ export type ProviderAdapter = { label: string; roots(): ProviderRootSpec[]; scanRoots?(): Promise; + scanSessions?(): Promise; health?(): Promise; }; diff --git a/apps/api-ts/src/domains/providers/search-policy.test.ts b/apps/api-ts/src/domains/providers/search-policy.test.ts new file mode 100644 index 0000000..03b2df0 --- /dev/null +++ b/apps/api-ts/src/domains/providers/search-policy.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { + listProviderActionProviderIds, + listSearchableProviderIds, + supportsProviderAction, +} from "./capabilities.js"; +import { + buildConversationSearchProviderBudgets, + defaultConversationSearchProviders, + isMetadataOnlyConversationQuery, + resolveConversationSearchLimits, +} from "./search.js"; + +describe("resolveConversationSearchLimits", () => { + it("keeps scan coverage wider than the visible result limit", () => { + expect(resolveConversationSearchLimits({ limit: 20 })).toEqual({ + resultLimit: 20, + scanLimit: 160, + }); + expect(resolveConversationSearchLimits({ limit: 120 })).toEqual({ + resultLimit: 120, + scanLimit: 480, + }); + }); +}); + +describe("buildConversationSearchProviderBudgets", () => { + it("splits the shared scan budget across providers", () => { + const budgets = buildConversationSearchProviderBudgets( + ["codex", "chatgpt", "claude", "gemini", "copilot"], + 160, + ); + + expect(budgets.map((entry) => entry.provider)).toEqual([ + "codex", + "chatgpt", + "claude", + "gemini", + "copilot", + ]); + expect(budgets.reduce((sum, entry) => sum + entry.limit, 0)).toBe(160); + expect( + Object.fromEntries(budgets.map((entry) => [entry.provider, entry.limit])), + ).toEqual({ + codex: 42, + chatgpt: 19, + claude: 42, + gemini: 31, + copilot: 26, + }); + }); + + it("gives a single provider the full scan budget", () => { + expect(buildConversationSearchProviderBudgets(["chatgpt"], 160)).toEqual([ + { provider: "chatgpt", limit: 160 }, + ]); + }); +}); + +describe("isMetadataOnlyConversationQuery", () => { + it("detects file/session/path oriented queries", () => { + expect(isMetadataOnlyConversationQuery("rollout-2026-03-25")).toBe(true); + expect(isMetadataOnlyConversationQuery("agent-session.jsonl")).toBe(true); + expect(isMetadataOnlyConversationQuery("/workspace/.codex/sessions")).toBe(true); + expect(isMetadataOnlyConversationQuery("69ab83eb-72a0-8320-8853-72ca88526762")).toBe(true); + }); + + it("keeps normal phrase searches transcript-eligible", () => { + expect(isMetadataOnlyConversationQuery("open the transcript")).toBe(false); + expect(isMetadataOnlyConversationQuery("cleanup preview token")).toBe(false); + }); +}); + +describe("defaultConversationSearchProviders", () => { + it("tracks the implemented api-ts searchable provider policy", () => { + expect(defaultConversationSearchProviders()).toEqual([ + ...listSearchableProviderIds(), + ]); + }); +}); + +describe("provider action policy", () => { + it("keeps internal read-only ChatGPT outside provider session actions", () => { + expect(listProviderActionProviderIds()).not.toContain("chatgpt"); + expect(supportsProviderAction("chatgpt", "backup_local")).toBe(false); + expect(supportsProviderAction("chatgpt", "archive_local")).toBe(false); + expect(supportsProviderAction("chatgpt", "delete_local")).toBe(false); + }); +}); diff --git a/apps/api-ts/src/domains/providers/search.test.ts b/apps/api-ts/src/domains/providers/search.test.ts deleted file mode 100644 index e87212c..0000000 --- a/apps/api-ts/src/domains/providers/search.test.ts +++ /dev/null @@ -1,2047 +0,0 @@ -import { createHash } from "node:crypto"; -import path from "node:path"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; -import { SEARCHABLE_PROVIDER_IDS } from "@threadlens/shared-contracts"; -import { describe, expect, it, vi } from "vitest"; -import type { - ProviderSessionRow, - TranscriptPayload, -} from "./index.js"; -import { - buildConversationSearchProviderBudgets, - createCachedConversationTranscriptLoader, - defaultConversationSearchProviders, - isMetadataOnlyConversationQuery, - resolveConversationSearchLimits, - searchConversationSessionHits, - searchConversationSessions, - searchConversationRows, - selectConversationSessionHitsRow, -} from "./search.js"; - -function makeRow(overrides: Partial = {}): ProviderSessionRow { - return { - provider: "codex", - source: "sessions", - session_id: "rollout-2026-03-25T10-00-00-019d-search-test", - display_title: "Search fixture", - file_path: "/tmp/search-fixture.jsonl", - size_bytes: 128, - mtime: "2026-03-25T10:00:00.000Z", - probe: { - ok: true, - format: "jsonl", - error: null, - detected_title: "Search fixture", - title_source: "fixture", - }, - ...overrides, - }; -} - -function makeTranscript( - row: ProviderSessionRow, - messages: TranscriptPayload["messages"], -): TranscriptPayload { - return { - provider: row.provider, - thread_id: "019d-search-test-thread", - file_path: row.file_path, - scanned_lines: messages.length, - message_count: messages.length, - truncated: false, - messages, - }; -} - -function deferred() { - let resolve!: (value: T | PromiseLike) => void; - let reject!: (reason?: unknown) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} - -async function waitForCondition( - predicate: () => boolean, - label: string, - attempts = 100, -) { - for (let index = 0; index < attempts; index += 1) { - if (predicate()) return; - await new Promise((resolve) => setImmediate(resolve)); - } - throw new Error(`timed out waiting for ${label}`); -} - -describe("searchConversationRows", () => { - it("returns metadata matches without loading transcripts when the limit is satisfied", async () => { - const row = makeRow({ - display_title: "Unrelated title", - session_id: "rollout-2026-03-25T10-00-00-019d-chatgpt-scope", - file_path: "/tmp/chatgpt-scope.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "chatgpt only in transcript", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - - const result = await searchConversationRows([row], "chatgpt", { - limit: 1, - transcriptLoader, - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - match_kind: "title", - session_id: "chatgpt-scope", - }); - expect(transcriptLoader).not.toHaveBeenCalled(); - }); - - it("stops before loading later transcripts once metadata hits fill the result limit", async () => { - const metadataRow = makeRow({ - display_title: "Unrelated title", - session_id: "rollout-2026-03-25T10-00-00-019d-obsidian-review", - file_path: "/tmp/obsidian-review.jsonl", - mtime: "2026-03-25T10:05:00.000Z", - }); - const transcriptRow = makeRow({ - display_title: "Another unrelated title", - session_id: "rollout-2026-03-25T10-00-00-019d-transcript-only", - file_path: "/tmp/transcript-only.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const transcriptLoader = vi.fn(async (_provider, filePath) => - makeTranscript( - filePath === metadataRow.file_path ? metadataRow : transcriptRow, - [ - { - idx: 0, - role: "assistant", - text: "obsidian transcript fallback", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ], - ), - ); - - const result = await searchConversationRows( - [metadataRow, transcriptRow], - "obsidian", - { - limit: 1, - transcriptLoader, - }, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - match_kind: "title", - session_id: "obsidian-review", - }); - expect(transcriptLoader).not.toHaveBeenCalled(); - }); - - it("suppresses exact duplicate message hits within the same session", async () => { - const row = makeRow({ - display_title: "Unrelated title", - file_path: "/tmp/search-dedup-case.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "user", - text: "agent duplicate result", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - { - idx: 1, - role: "user", - text: "agent duplicate result", - ts: "2026-03-25T10:00:01.000Z", - source_type: "event_msg.user_message", - }, - ]), - ); - - const result = await searchConversationRows([row], "agent", { - limit: 40, - transcriptLoader, - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - match_kind: "message", - role: "user", - }); - }); - - it("falls back to transcript scanning when metadata matches are insufficient", async () => { - const row = makeRow({ - display_title: "Unrelated title", - file_path: "/tmp/search-transcript-case.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "search fallback hit", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - - const result = await searchConversationRows([row], "fallback", { - limit: 5, - transcriptLoader, - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - match_kind: "message", - role: "assistant", - }); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - }); - - it("skips transcript scanning entirely for metadata-only queries", async () => { - const row = makeRow({ - display_title: "Unrelated title", - session_id: "rollout-2026-03-25T10-00-00-019d-rollout-query", - file_path: "/tmp/rollout-2026-03-25T10-00-00-019d-rollout-query.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "rollout token only in transcript", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - - const result = await searchConversationRows([row], "rollout-2026-03-25", { - limit: 10, - transcriptLoader, - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - match_kind: "title", - session_id: "rollout-2026-03-25T10-00-00-019d-rollout-query", - }); - expect(transcriptLoader).not.toHaveBeenCalled(); - }); - - it("omits cleanup thread ids that are not openable in the current thread read model", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41", - file_path: - "/tmp/rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41.jsonl", - display_title: "ThreadLens handoff", - }); - - const result = await searchConversationRows([row], "handoff", { - limit: 10, - openableThreadIds: new Set(), - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - provider: "codex", - session_id: "rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41", - match_kind: "title", - }); - expect(result.results[0].thread_id).toBeUndefined(); - }); - - it("keeps cleanup thread ids when they are openable in the current thread read model", async () => { - const threadId = "019d355d-51c3-7753-b2f2-8db585337e41"; - const row = makeRow({ - session_id: `rollout-2026-03-29T01-53-21-${threadId}`, - file_path: `/tmp/rollout-2026-03-29T01-53-21-${threadId}.jsonl`, - display_title: "ThreadLens handoff", - }); - - const result = await searchConversationRows([row], "handoff", { - limit: 10, - openableThreadIds: new Set([threadId]), - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toMatchObject({ - provider: "codex", - thread_id: threadId, - match_kind: "title", - }); - }); -}); - -describe("searchConversationSessions", () => { - it("collapses duplicate logical sessions across backup copies", async () => { - const liveRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-duplicate", - display_title: "Duplicate token session", - file_path: "/tmp/search-duplicate-live.jsonl", - source: "sessions", - mtime: "2026-03-25T10:10:00.000Z", - }); - const backupRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-duplicate", - display_title: "Duplicate token session", - file_path: "/tmp/search-duplicate-backup.jsonl", - source: "cleanup_backups", - mtime: "2026-03-25T10:11:00.000Z", - }); - - const result = await searchConversationSessions([liveRow, backupRow], "token", { - pageSize: 10, - previewHitsPerSession: 2, - }); - - expect(result.sessions).toHaveLength(1); - expect(result.sessions[0]).toMatchObject({ - session_id: "search-duplicate-live", - source: "sessions", - }); - }); - - it("keeps distinct matching sessions even when one session has many hits", async () => { - const dominantRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-dominant", - display_title: "Dominant token session", - file_path: "/tmp/search-dominant.jsonl", - mtime: "2026-03-25T10:10:00.000Z", - }); - const secondaryRow = makeRow({ - session_id: "rollout-2026-03-25T09-00-00-019d-secondary", - display_title: "Secondary token session", - file_path: "/tmp/search-secondary.jsonl", - mtime: "2026-03-25T09:00:00.000Z", - }); - const transcriptLoader = vi.fn(async (_provider, filePath) => { - if (filePath === dominantRow.file_path) { - return makeTranscript(dominantRow, [ - { - idx: 0, - role: "assistant", - text: "token dominant match one", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - { - idx: 1, - role: "assistant", - text: "token dominant match two", - ts: "2026-03-25T10:00:01.000Z", - source_type: "response_item.message", - }, - { - idx: 2, - role: "assistant", - text: "token dominant match three", - ts: "2026-03-25T10:00:02.000Z", - source_type: "response_item.message", - }, - ]); - } - return makeTranscript(secondaryRow, [ - { - idx: 0, - role: "user", - text: "token secondary match", - ts: "2026-03-25T09:00:00.000Z", - source_type: "response_item.message", - }, - ]); - }); - - const result = await searchConversationSessions( - [dominantRow, secondaryRow], - "token", - { - pageSize: 10, - previewHitsPerSession: 2, - transcriptLoader, - }, - ); - - expect(result.total_matching_sessions).toBe(2); - expect(result.total_matching_hits).toBeNull(); - expect(result.sessions).toHaveLength(2); - expect(result.sessions[0]).toMatchObject({ - session_id: "search-dominant", - match_count: 3, - has_more_hits: true, - }); - expect(result.sessions[0]?.preview_matches).toHaveLength(2); - expect(result.sessions[1]).toMatchObject({ - session_id: "search-secondary", - match_count: 2, - has_more_hits: false, - }); - expect(result.results).toHaveLength(4); - }); - - it("keeps the full session id as the fallback title when no detected title exists", async () => { - const sessionId = "rollout-2026-04-16T23-47-06-019d96c2-9123-7481-9127-224fad716008"; - const row = makeRow({ - session_id: sessionId, - display_title: "", - file_path: `/tmp/${sessionId}.jsonl`, - probe: { - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - }, - }); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - row.file_path, - { - snippets: ["token preview one"], - match_count: 1, - has_more_hits: false, - exact_phrase_count: 1, - }, - ], - ]), - ); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token from parsed transcript", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - - const result = await searchConversationSessions([row], "token", { - pageSize: 10, - previewHitsPerSession: 3, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(result.sessions[0]?.title).toBe(sessionId); - }); - - it("paginates session results with a stable next cursor", async () => { - const rows = [ - makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-alpha", - display_title: "alpha token", - file_path: "/tmp/search-alpha.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }), - makeRow({ - session_id: "rollout-2026-03-25T09-00-00-019d-bravo", - display_title: "bravo token", - file_path: "/tmp/search-bravo.jsonl", - mtime: "2026-03-25T09:00:00.000Z", - }), - makeRow({ - session_id: "rollout-2026-03-25T08-00-00-019d-charlie", - display_title: "charlie token", - file_path: "/tmp/search-charlie.jsonl", - mtime: "2026-03-25T08:00:00.000Z", - }), - ]; - - const page1 = await searchConversationSessions(rows, "token", { - pageSize: 2, - previewHitsPerSession: 1, - }); - const page2 = await searchConversationSessions(rows, "token", { - pageSize: 2, - cursor: page1.next_cursor ?? undefined, - previewHitsPerSession: 1, - }); - - expect(page1.sessions).toHaveLength(2); - expect(page1.has_more).toBe(true); - expect(page1.truncated).toBe(false); - expect(page1.next_cursor).toBe("2"); - expect(page1.total_matching_sessions).toBe(3); - expect(page1.total_matching_hits).toBeNull(); - expect(page2.sessions).toHaveLength(1); - expect(page2.has_more).toBe(false); - expect(page2.truncated).toBe(false); - expect(page2.next_cursor).toBeNull(); - expect(page2.total_matching_sessions).toBe(3); - expect(page2.total_matching_hits).toBeNull(); - }); - - it("orders pages by global session quality before applying the cursor", async () => { - const newerWeakRow = makeRow({ - session_id: "rollout-2026-03-25T11-00-00-019d-newer-weak", - display_title: "recent chat", - file_path: "/tmp/search-newer-weak.jsonl", - mtime: "2026-03-25T11:00:00.000Z", - }); - const middleWeakRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-middle-weak", - display_title: "middle chat", - file_path: "/tmp/search-middle-weak.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const olderStrongRow = makeRow({ - session_id: "rollout-2026-03-25T09-00-00-019d-older-strong", - display_title: "token in title", - file_path: "/tmp/search-older-strong.jsonl", - mtime: "2026-03-25T09:00:00.000Z", - }); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - newerWeakRow.file_path, - { - snippets: ["token weak recent"], - match_count: 1, - has_more_hits: false, - exact_phrase_count: 0, - }, - ], - [ - middleWeakRow.file_path, - { - snippets: ["token weak middle"], - match_count: 1, - has_more_hits: false, - exact_phrase_count: 0, - }, - ], - ]), - ); - const transcriptLoader = vi.fn(async (_provider, filePath) => { - const row = - [newerWeakRow, middleWeakRow, olderStrongRow].find((item) => item.file_path === filePath) ?? - newerWeakRow; - return makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: `token transcript hit for ${row.session_id}`, - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]); - }); - - const page1 = await searchConversationSessions( - [newerWeakRow, middleWeakRow, olderStrongRow], - "token", - { - pageSize: 1, - previewHitsPerSession: 1, - transcriptLoader, - rawFileSearchLoader, - }, - ); - const page2 = await searchConversationSessions( - [newerWeakRow, middleWeakRow, olderStrongRow], - "token", - { - pageSize: 1, - cursor: page1.next_cursor ?? undefined, - previewHitsPerSession: 1, - transcriptLoader, - rawFileSearchLoader, - }, - ); - - expect(page1.sessions[0]?.session_id).toBe("search-older-strong"); - expect(page1.total_matching_sessions).toBe(3); - expect(page2.sessions[0]?.session_id).not.toBe("search-older-strong"); - }); - - it("uses raw file hits as a prefilter but builds previews from parsed transcripts", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-raw-preview", - display_title: "Raw preview token session", - file_path: "/tmp/search-raw-preview.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token from parsed transcript", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - row.file_path, - { - snippets: [ - "token preview one", - "token preview two", - "token preview three", - ], - match_count: 4, - has_more_hits: true, - exact_phrase_count: 2, - }, - ], - ]), - ); - - const result = await searchConversationSessions([row], "token", { - pageSize: 10, - previewHitsPerSession: 3, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(result.sessions).toHaveLength(1); - expect(result.sessions[0]).toMatchObject({ - session_id: "search-raw-preview", - match_count: 2, - has_more_hits: false, - }); - expect(result.sessions[0]?.preview_matches.map((match) => match.snippet)).toEqual([ - "Raw preview token session", - "token from parsed transcript", - ]); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); - }); - - it("does not expose raw matches from system or policy transcript messages", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-policy-only", - display_title: "Policy only session", - file_path: "/tmp/search-policy-only.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "system", - text: "token from system instructions", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - { - idx: 1, - role: "assistant", - text: "# AGENTS.md instructions for token handling", - ts: "2026-03-25T10:00:01.000Z", - source_type: "response_item.message", - }, - ]), - ); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - row.file_path, - { - snippets: ["token raw system preview"], - match_count: 1, - has_more_hits: false, - exact_phrase_count: 0, - }, - ], - ]), - ); - - const result = await searchConversationSessions([row], "token", { - pageSize: 10, - previewHitsPerSession: 3, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(result.sessions).toHaveLength(0); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); - }); - - it("skips transcript parsing for raw-search-eligible rows when ripgrep found no match", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-no-match", - display_title: "No raw match session", - file_path: "/tmp/search-no-match.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token would only appear if transcript parsing ran", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - const rawFileSearchLoader = vi.fn(async () => new Map()); - - const result = await searchConversationSessions([row], "token", { - pageSize: 10, - previewHitsPerSession: 3, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(result.sessions).toHaveLength(0); - expect(transcriptLoader).not.toHaveBeenCalled(); - expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); - }); - - it("falls back to transcript parsing when raw file search fails unexpectedly", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-raw-failure", - display_title: "Raw loader failure session", - file_path: "/tmp/search-raw-failure.jsonl", - mtime: "2026-03-25T10:00:00.000Z", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token fallback transcript hit", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - const rawFileSearchLoader = vi.fn(async () => { - throw new Error("rg blew up"); - }); - - const result = await searchConversationSessions([row], "token", { - pageSize: 10, - previewHitsPerSession: 3, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(result.sessions).toHaveLength(1); - expect(result.sessions[0]?.session_id).toBe("search-raw-failure"); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); - }); -}); - -describe("searchConversationSessionHits", () => { - it("prefers the explicit file path when duplicate logical session ids exist", () => { - const liveRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-duplicate-hits", - file_path: "/tmp/search-hits-live.jsonl", - source: "sessions", - mtime: "2026-03-25T10:00:00.000Z", - }); - const backupRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-duplicate-hits", - file_path: "/tmp/search-hits-backup.jsonl", - source: "cleanup_backups", - mtime: "2026-03-25T10:05:00.000Z", - }); - - const targetRow = selectConversationSessionHitsRow([backupRow, liveRow], { - sessionId: liveRow.session_id, - filePath: liveRow.file_path, - }); - - expect(targetRow?.file_path).toBe(liveRow.file_path); - expect(targetRow?.source).toBe("sessions"); - }); - - it("returns paginated hits for a single matching session", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-session-hits", - display_title: "token session detail", - file_path: "/tmp/search-session-hits.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token first hit", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - { - idx: 1, - role: "assistant", - text: "token second hit", - ts: "2026-03-25T10:00:01.000Z", - source_type: "response_item.message", - }, - { - idx: 2, - role: "assistant", - text: "token third hit", - ts: "2026-03-25T10:00:02.000Z", - source_type: "response_item.message", - }, - ]), - ); - - const page1 = await searchConversationSessionHits(row, "token", { - pageSize: 2, - transcriptLoader, - }); - const page2 = await searchConversationSessionHits(row, "token", { - pageSize: 2, - cursor: page1.next_cursor ?? undefined, - transcriptLoader, - }); - - expect(page1.total_hits).toBe(4); - expect(page1.hits).toHaveLength(2); - expect(page1.has_more).toBe(true); - expect(page2.hits).toHaveLength(2); - expect(page2.has_more).toBe(false); - }); - - it("uses raw file hits as a prefilter for session detail expansion", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-session-hits-raw", - display_title: "Unrelated title", - file_path: "/tmp/search-session-hits-raw.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "token transcript one", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - { - idx: 1, - role: "assistant", - text: "token transcript two", - ts: "2026-03-25T10:00:01.000Z", - source_type: "response_item.message", - }, - { - idx: 2, - role: "assistant", - text: "token transcript three", - ts: "2026-03-25T10:00:02.000Z", - source_type: "response_item.message", - }, - { - idx: 3, - role: "assistant", - text: "token transcript four", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - row.file_path, - { - snippets: [ - "token preview one", - "token preview two", - "token preview three", - "token preview four", - ], - match_count: 4, - has_more_hits: false, - exact_phrase_count: 4, - }, - ], - ]), - ); - - const page1 = await searchConversationSessionHits(row, "token", { - pageSize: 2, - transcriptLoader, - rawFileSearchLoader, - }); - const page2 = await searchConversationSessionHits(row, "token", { - pageSize: 2, - cursor: page1.next_cursor ?? undefined, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(page1.total_hits).toBe(4); - expect(page1.hits.map((hit) => hit.snippet)).toEqual([ - "token transcript one", - "token transcript two", - ]); - expect(page1.has_more).toBe(true); - expect(page2.hits.map((hit) => hit.snippet)).toEqual([ - "token transcript three", - "token transcript four", - ]); - expect(page2.has_more).toBe(false); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - expect(rawFileSearchLoader).toHaveBeenCalledTimes(2); - }); - - it("falls back to transcript hits when a later cursor exceeds the raw snippet window", async () => { - const row = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-session-hits-fallback", - display_title: "Token overflow session", - file_path: "/tmp/search-session-hits-fallback.jsonl", - }); - const transcriptLoader = vi.fn(async () => - makeTranscript( - row, - Array.from({ length: 45 }, (_, index) => ({ - idx: index, - role: "assistant" as const, - text: `token transcript hit ${index + 1}`, - ts: `2026-03-25T10:00:${String(index).padStart(2, "0")}.000Z`, - source_type: "response_item.message" as const, - })), - ), - ); - const rawFileSearchLoader = vi.fn(async () => - new Map([ - [ - row.file_path, - { - snippets: Array.from({ length: 20 }, (_, index) => `token preview ${index + 1}`), - match_count: 120, - has_more_hits: true, - exact_phrase_count: 20, - }, - ], - ]), - ); - - const page1 = await searchConversationSessionHits(row, "token", { - pageSize: 40, - transcriptLoader, - rawFileSearchLoader, - }); - const page2 = await searchConversationSessionHits(row, "token", { - pageSize: 40, - cursor: page1.next_cursor ?? undefined, - transcriptLoader, - rawFileSearchLoader, - }); - - expect(page1.hits).toHaveLength(40); - expect(page1.has_more).toBe(true); - expect(page2.hits.length).toBeGreaterThan(0); - expect(page2.hits[0]?.snippet).toContain("token transcript hit"); - expect(transcriptLoader).toHaveBeenCalledTimes(1); - }); -}); - -describe("createCachedConversationTranscriptLoader", () => { - it("reuses transcripts while file mtime is unchanged", async () => { - const row = makeRow({ - file_path: "/tmp/search-cache-fixture.jsonl", - mtime: "2026-03-25T10:10:00.000Z", - }); - const baseLoader = vi.fn(async () => - makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "agent cache check", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]), - ); - const cachedLoader = createCachedConversationTranscriptLoader(baseLoader); - - await cachedLoader(row); - await cachedLoader(row); - await cachedLoader({ ...row, mtime: "2026-03-25T10:00:05.000Z" }); - - expect(baseLoader).toHaveBeenCalledTimes(2); - }); - - it("retries after a transient transcript load failure when mtime is unchanged", async () => { - const row = makeRow({ - file_path: "/tmp/search-cache-transient-error.jsonl", - mtime: "2026-03-25T10:10:00.000Z", - }); - let shouldFail = true; - const baseLoader = vi.fn(async () => { - if (shouldFail) { - shouldFail = false; - throw new Error("transient transcript failure"); - } - return makeTranscript(row, [ - { - idx: 0, - role: "assistant", - text: "recovered transcript", - ts: "2026-03-25T10:00:00.000Z", - source_type: "response_item.message", - }, - ]); - }); - const cachedLoader = createCachedConversationTranscriptLoader(baseLoader); - - const first = await cachedLoader(row); - const second = await cachedLoader(row); - - expect(first).toBeNull(); - expect(second?.messages[0]?.text).toBe("recovered transcript"); - expect(baseLoader).toHaveBeenCalledTimes(2); - }); -}); - -describe("resolveConversationSearchLimits", () => { - it("keeps scan coverage wider than the visible result limit", () => { - expect(resolveConversationSearchLimits({ limit: 20 })).toEqual({ - resultLimit: 20, - scanLimit: 160, - }); - expect(resolveConversationSearchLimits({ limit: 120 })).toEqual({ - resultLimit: 120, - scanLimit: 480, - }); - }); -}); - -describe("buildConversationSearchProviderBudgets", () => { - it("splits the shared scan budget across providers", () => { - const budgets = buildConversationSearchProviderBudgets( - ["codex", "chatgpt", "claude", "gemini", "copilot"], - 160, - ); - - expect(budgets.map((entry) => entry.provider)).toEqual([ - "codex", - "chatgpt", - "claude", - "gemini", - "copilot", - ]); - expect(budgets.reduce((sum, entry) => sum + entry.limit, 0)).toBe(160); - expect( - Object.fromEntries(budgets.map((entry) => [entry.provider, entry.limit])), - ).toEqual({ - codex: 42, - chatgpt: 19, - claude: 42, - gemini: 31, - copilot: 26, - }); - }); - - it("gives a single provider the full scan budget", () => { - expect(buildConversationSearchProviderBudgets(["chatgpt"], 160)).toEqual([ - { provider: "chatgpt", limit: 160 }, - ]); - }); -}); - -describe("selectConversationSessionHitsRow", () => { - it("does not trust file_path when it points at a different logical session", () => { - const requestedSessionId = "rollout-2026-03-25T10-00-00-019d-target"; - const requestedRow = makeRow({ - session_id: requestedSessionId, - file_path: "/tmp/search-target.jsonl", - }); - const mismatchedRow = makeRow({ - session_id: "rollout-2026-03-25T10-00-00-019d-other", - file_path: "/tmp/search-other.jsonl", - }); - - const selected = selectConversationSessionHitsRow( - [mismatchedRow, requestedRow], - { - sessionId: requestedSessionId, - filePath: mismatchedRow.file_path, - }, - ); - - expect(selected).toEqual(requestedRow); - }); -}); - -describe("isMetadataOnlyConversationQuery", () => { - it("detects file/session/path oriented queries", () => { - expect(isMetadataOnlyConversationQuery("rollout-2026-03-25")).toBe(true); - expect(isMetadataOnlyConversationQuery("agent-session.jsonl")).toBe(true); - expect(isMetadataOnlyConversationQuery("/workspace/.codex/sessions")).toBe(true); - expect(isMetadataOnlyConversationQuery("69ab83eb-72a0-8320-8853-72ca88526762")).toBe(true); - }); - - it("keeps normal phrase searches transcript-eligible", () => { - expect(isMetadataOnlyConversationQuery("open the transcript")).toBe(false); - expect(isMetadataOnlyConversationQuery("cleanup preview token")).toBe(false); - }); -}); - -describe("defaultConversationSearchProviders", () => { - it("tracks the shared searchable provider contract", () => { - expect(defaultConversationSearchProviders()).toEqual([ - ...SEARCHABLE_PROVIDER_IDS, - ]); - }); -}); - -describe("provider manifest cache behavior", () => { - it("reuses the provider manifest after the scan cache TTL until invalidated", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - const realpathMock = vi.fn(async (target: string) => path.resolve(target)); - const probeSessionFile = vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - realpath: realpathMock, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile, - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - - await mod.getProviderSessionScan("claude", 1); - expect(walkFilesByExt).toHaveBeenCalledTimes(1); - - vi.advanceTimersByTime(61_000); - - await mod.getProviderSessionScan("claude", 1); - expect(walkFilesByExt).toHaveBeenCalledTimes(1); - expect(probeSessionFile).toHaveBeenCalledTimes(2); - - mod.invalidateProviderSearchCaches("claude"); - await mod.getProviderSessionScan("claude", 1); - expect(walkFilesByExt).toHaveBeenCalledTimes(2); - } finally { - vi.useRealTimers(); - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("reuses a persisted provider manifest across module reloads", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const installMocks = (walkFilesByExt: ReturnType) => { - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - const realpathMock = vi.fn(async (target: string) => path.resolve(target)); - const probeSessionFile = vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - })); - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - realpath: realpathMock, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile, - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - }; - - try { - const firstWalk = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); - installMocks(firstWalk); - const firstMod = await import("./search.js"); - await firstMod.getProviderSessionScan("claude", 1); - expect(firstWalk).toHaveBeenCalledTimes(1); - - vi.resetModules(); - - const secondWalk = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); - installMocks(secondWalk); - const secondMod = await import("./search.js"); - const secondScan = await secondMod.getProviderSessionScan("claude", 1); - expect(secondWalk).not.toHaveBeenCalled(); - expect(secondScan.rows[0]?.source).toBe("projects"); - } finally { - vi.useRealTimers(); - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("reclassifies persisted manifest candidate source from the current matching root", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); - const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); - const sessionPath = "/virtual/claude/session-a.jsonl"; - await mkdir(path.dirname(cacheFile), { recursive: true }); - await writeFile( - cacheFile, - JSON.stringify({ - expires_at: Date.now() + 60_000, - manifest: { - provider: "claude", - name: "Claude", - root_exists: true, - candidates: [ - { - source: "stale-cache-source", - file_path: sessionPath, - size_bytes: 128, - mtime: "2026-03-25T10:00:00.000Z", - mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), - }, - ], - total_bytes: 128, - }, - }), - "utf8", - ); - - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - realpath: vi.fn(async (target: string) => path.resolve(target)), - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, - ], - })); - - try { - const { readPersistedProviderManifest } = await import( - "./services/search/manifest-store.js" - ); - const persisted = await readPersistedProviderManifest("claude"); - - expect(persisted?.manifest.candidates).toHaveLength(1); - expect(persisted?.manifest.candidates[0]).toMatchObject({ - file_path: path.resolve(sessionPath), - source: "projects", - }); - } finally { - vi.useRealTimers(); - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("discards persisted manifest candidates outside current provider roots", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); - const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); - await mkdir(path.dirname(cacheFile), { recursive: true }); - await writeFile( - cacheFile, - JSON.stringify({ - expires_at: Date.now() + 60_000, - manifest: { - provider: "claude", - name: "Claude", - root_exists: true, - candidates: [ - { - source: "projects", - file_path: "/outside/session-a.jsonl", - size_bytes: 128, - mtime: "2026-03-25T10:00:00.000Z", - mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), - }, - ], - total_bytes: 128, - }, - }), - "utf8", - ); - - const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); - const statMock = vi.fn(async () => ({ - size: 256, - mtimeMs: Date.parse("2026-03-25T10:05:00.000Z"), - })); - const realpathMock = vi.fn(async (target: string) => path.resolve(target)); - const probeSessionFile = vi.fn(async (filePath: string) => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: path.basename(filePath, ".jsonl"), - title_source: "fixture", - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - realpath: realpathMock, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile, - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - const scan = await mod.getProviderSessionScan("claude", 1); - - expect(walkFilesByExt).toHaveBeenCalledTimes(1); - expect(scan.rows[0]?.file_path).toBe("/virtual/claude/session-b.jsonl"); - expect(probeSessionFile).toHaveBeenCalledWith("/virtual/claude/session-b.jsonl"); - } finally { - vi.useRealTimers(); - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("does not let a stale inflight rebuild overwrite a newer manifest after invalidation", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const firstWalk = deferred(); - const walkFilesByExt = vi - .fn() - .mockImplementationOnce(async () => firstWalk.promise) - .mockImplementationOnce(async () => ["/virtual/claude/session-b.jsonl"]); - const statMock = vi.fn(async (filePath: string) => ({ - size: 128, - mtimeMs: filePath.includes("session-b") - ? Date.parse("2026-03-25T10:05:00.000Z") - : Date.parse("2026-03-25T10:00:00.000Z"), - })); - const probeSessionFile = vi.fn(async (filePath: string) => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: path.basename(filePath, ".jsonl"), - title_source: "fixture", - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile, - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - const stalePromise = mod.getProviderSessionScan("claude", 1); - await waitForCondition( - () => walkFilesByExt.mock.calls.length === 1, - "the stale manifest rebuild to start", - ); - - mod.invalidateProviderSearchCaches("claude"); - const freshPromise = mod.getProviderSessionScan("claude", 1); - const freshScan = await freshPromise; - - expect(freshScan.rows[0]?.file_path).toContain("session-b.jsonl"); - - firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); - const staleScan = await stalePromise; - expect(staleScan.rows[0]?.file_path).toContain("session-a.jsonl"); - - const cachedScan = await mod.getProviderSessionScan("claude", 1); - expect(cachedScan.rows[0]?.file_path).toContain("session-b.jsonl"); - expect(walkFilesByExt).toHaveBeenCalledTimes(2); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - -}); - -describe("searchLocalConversationsTs metadata-only fast path", () => { - it("avoids probing session files for metadata-only queries", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - - const walkFilesByExt = vi.fn(async () => [ - "/virtual/claude/rollout-2026-03-25T10-00-00-019d-meta-only.jsonl", - ]); - const statMock = vi.fn(async (target: string) => { - if (target === "/virtual/claude") { - return { - size: 0, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - }; - } - return { - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - }; - }); - const probeSessionFile = vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "Should not load", - title_source: "fixture", - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile, - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - vi.doMock("../threads/query.js", () => ({ - getThreadsTs: vi.fn(async () => ({ rows: [] })), - })); - - try { - const mod = await import("./search.js"); - const result = await mod.searchLocalConversationsTs( - "rollout-2026-03-25", - { - providers: ["claude"], - pageSize: 10, - sessionLimitPerProvider: 1, - }, - ); - - expect(result.sessions).toHaveLength(1); - expect(result.sessions[0]?.session_id).toContain("rollout-2026-03-25"); - expect(probeSessionFile).not.toHaveBeenCalled(); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - vi.doUnmock("../threads/query.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); -}); - -describe("abort propagation", () => { - it("propagates abort signals into conversation search loaders", async () => { - const controller = new AbortController(); - controller.abort(); - const rawFileSearchLoader = vi.fn(async (_rows, _q, options) => { - if (options?.signal?.aborted) { - const error = new Error("aborted"); - error.name = "AbortError"; - throw error; - } - return new Map(); - }); - - await expect( - searchConversationSessions([makeRow()], "token", { - signal: controller.signal, - rawFileSearchLoader, - }), - ).rejects.toMatchObject({ name: "AbortError" }); - expect(rawFileSearchLoader).toHaveBeenCalled(); - }); - - it("aborts before rebuilding provider scans when the signal is already cancelled", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile: vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - })), - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - const controller = new AbortController(); - controller.abort(); - - try { - const mod = await import("./search.js"); - await expect( - mod.getProviderSessionScan("claude", 1, { signal: controller.signal }), - ).rejects.toMatchObject({ name: "AbortError" }); - expect(walkFilesByExt).not.toHaveBeenCalled(); - expect(statMock).not.toHaveBeenCalled(); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("lets callers abort while waiting on a shared manifest inflight promise", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - const firstWalk = deferred(); - const walkFilesByExt = vi - .fn() - .mockImplementationOnce(async () => firstWalk.promise); - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile: vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - })), - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - const firstRequest = mod.getProviderSessionScan("claude", 1); - await waitForCondition( - () => walkFilesByExt.mock.calls.length === 1, - "the shared manifest rebuild to start", - ); - - const controller = new AbortController(); - const secondRequest = mod.getProviderSessionScan("claude", 2, { - signal: controller.signal, - }); - controller.abort(); - - await expect(secondRequest).rejects.toMatchObject({ name: "AbortError" }); - firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); - const firstScan = await firstRequest; - expect(firstScan.rows[0]?.file_path).toContain("session-a.jsonl"); - expect(walkFilesByExt).toHaveBeenCalledTimes(1); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); - - it("keeps shared manifest work alive when the first caller aborts", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - const firstWalk = deferred(); - const walkFilesByExt = vi - .fn() - .mockImplementationOnce(async () => firstWalk.promise); - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - realpath: vi.fn(async (target: string) => path.resolve(target)), - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile: vi.fn(async () => ({ - ok: true, - format: "jsonl", - error: null, - detected_title: "", - title_source: null, - })), - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - const controller = new AbortController(); - const firstRequest = mod.getProviderSessionScan("claude", 1, { - signal: controller.signal, - }); - await waitForCondition( - () => walkFilesByExt.mock.calls.length === 1, - "the shared manifest rebuild to start", - ); - const secondRequest = mod.getProviderSessionScan("claude", 1); - - controller.abort(); - await expect(firstRequest).rejects.toMatchObject({ name: "AbortError" }); - - firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); - const secondScan = await secondRequest; - expect(secondScan.rows[0]?.file_path).toContain("session-a.jsonl"); - expect(walkFilesByExt).toHaveBeenCalledTimes(1); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); -}); - -describe("searchConversationSessionHitsTs", () => { - it("returns null when a manifest candidate points to a deleted session file", async () => { - vi.resetModules(); - const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); - vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); - const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); - const statMock = vi.fn(async () => ({ - size: 128, - mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), - })); - - vi.doMock("../../lib/utils.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - walkFilesByExt, - }; - }); - vi.doMock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - stat: statMock, - }; - }); - vi.doMock("./path-safety.js", () => ({ - providerScanRootSpecs: async () => [ - { - root: "/virtual/claude", - source: "projects", - exts: [".jsonl"], - }, - ], - providerName: () => "Claude", - codexTranscriptSearchRoots: async () => [], - })); - vi.doMock("./title-detection.js", () => ({ - getCodexThreadTitleMap: vi.fn(async () => new Map()), - invalidateCodexThreadTitleMapCache: vi.fn(), - extractCodexThreadIdFromSessionName: vi.fn(() => ""), - })); - vi.doMock("./probe.js", () => ({ - inferSessionId: vi.fn((filePath: string) => - filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, - ), - isCopilotGlobalSessionLikeFile: vi.fn(() => false), - isWorkspaceChatSessionPath: vi.fn(() => false), - probeSessionFile: vi.fn(async () => ({ - ok: false, - format: null, - error: "missing", - detected_title: "", - title_source: null, - })), - })); - vi.doMock("./matrix.js", () => ({ - providerStatus: vi.fn(() => "ready"), - })); - - try { - const mod = await import("./search.js"); - const result = await mod.searchConversationSessionHitsTs("token", { - provider: "claude", - sessionId: "session-a", - filePath: "/virtual/claude/session-a.jsonl", - }); - - expect(result).toBeNull(); - } finally { - vi.unstubAllEnvs(); - vi.resetModules(); - vi.doUnmock("../../lib/utils.js"); - vi.doUnmock("node:fs/promises"); - vi.doUnmock("./path-safety.js"); - vi.doUnmock("./title-detection.js"); - vi.doUnmock("./probe.js"); - vi.doUnmock("./matrix.js"); - await rm(cacheDir, { recursive: true, force: true }); - } - }); -}); diff --git a/apps/api-ts/src/domains/providers/services/actions/archives.ts b/apps/api-ts/src/domains/providers/services/actions/archives.ts index 75ff416..f5ad1d1 100644 --- a/apps/api-ts/src/domains/providers/services/actions/archives.ts +++ b/apps/api-ts/src/domains/providers/services/actions/archives.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { realpath } from "node:fs/promises"; +import { lstat, realpath } from "node:fs/promises"; import { isPathInsideRoot, providerRootSpecs } from "../../path-safety.js"; import type { ProviderId, ProviderRootSpec } from "../../types.js"; @@ -37,6 +37,44 @@ async function relativePathWithinProviderRoot( return null; } +async function hasSymlinkAncestorFromRoot( + rootPath: string, + targetDir: string, +): Promise { + const root = path.resolve(rootPath); + const target = path.resolve(targetDir); + if (!isPathInsideRoot(target, root)) return true; + + let current = root; + while (true) { + try { + const st = await lstat(current); + if (st.isSymbolicLink()) return true; + if (!st.isDirectory()) return true; + } catch { + return false; + } + if (current === target) return false; + const relative = path.relative(current, target); + const [nextSegment] = relative.split(path.sep); + if (!nextSegment || nextSegment === "..") return true; + current = path.join(current, nextSegment); + } +} + +async function isSafeActionDestinationWithinRoot( + filePath: string, + rootPath: string, +): Promise { + const resolvedPath = path.resolve(filePath); + const resolvedRoot = path.resolve(rootPath); + if (!isPathInsideRoot(resolvedPath, resolvedRoot)) return false; + return !(await hasSymlinkAncestorFromRoot( + resolvedRoot, + path.dirname(resolvedPath), + )); +} + export async function resolveArchivedSessionRestoreTarget( provider: ProviderId, filePath: string, @@ -111,3 +149,36 @@ export function resolveArchivedSessionRoot(provider: ProviderId): string | null return providerRootSpecs(provider).find((spec) => spec.source === "archived_sessions") ?.root ?? null; } + +export async function isSafeArchivedSessionStoreTarget( + provider: ProviderId, + targetPath: string, +): Promise { + const ext = path.extname(targetPath).toLowerCase(); + const archivedSpec = providerRootSpecs(provider).find( + (spec) => spec.source === "archived_sessions" && spec.exts.includes(ext), + ); + if (!archivedSpec) return false; + return isSafeActionDestinationWithinRoot(targetPath, archivedSpec.root); +} + +export async function isSafeArchivedSessionRestoreTarget( + provider: ProviderId, + targetPath: string, +): Promise { + const ext = path.extname(targetPath).toLowerCase(); + const sourceSpecs = providerRootSpecs(provider) + .filter( + (spec) => + spec.source !== "cleanup_backups" && + spec.source !== "archived_sessions" && + spec.exts.includes(ext), + ) + .sort((left, right) => right.root.length - left.root.length); + for (const spec of sourceSpecs) { + if (await isSafeActionDestinationWithinRoot(targetPath, spec.root)) { + return true; + } + } + return false; +} diff --git a/apps/api-ts/src/domains/providers/services/actions/backups.ts b/apps/api-ts/src/domains/providers/services/actions/backups.ts index cfce5b9..100f7d8 100644 --- a/apps/api-ts/src/domains/providers/services/actions/backups.ts +++ b/apps/api-ts/src/domains/providers/services/actions/backups.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { copyFile, mkdir, realpath, writeFile } from "node:fs/promises"; -import { BACKUP_ROOT, HOME_DIR } from "../../../../lib/constants.js"; +import { HOME_DIR } from "../../constants.js"; +import { BACKUP_ROOT } from "../../../recovery/constants.js"; import { nowIsoUtc } from "../../../../lib/utils.js"; import { isPathInsideRoot, diff --git a/apps/api-ts/src/domains/providers/services/actions/execution.ts b/apps/api-ts/src/domains/providers/services/actions/execution.ts index 5f475c3..94d30b2 100644 --- a/apps/api-ts/src/domains/providers/services/actions/execution.ts +++ b/apps/api-ts/src/domains/providers/services/actions/execution.ts @@ -1,7 +1,9 @@ import path from "node:path"; -import { copyFile, mkdir, stat, unlink } from "node:fs/promises"; +import { copyFile, lstat, mkdir, unlink } from "node:fs/promises"; import { + isSafeArchivedSessionRestoreTarget, + isSafeArchivedSessionStoreTarget, resolveArchivedSessionRestoreTarget, resolveArchivedSessionRoot, resolveArchivedSessionStoreTarget, @@ -38,7 +40,7 @@ export async function runArchiveProviderAction( continue; } try { - await stat(targetPath); + await lstat(targetPath); failed.push({ file_path: sourcePath, step: "archive_local", @@ -48,6 +50,14 @@ export async function runArchiveProviderAction( } catch { // Missing target is expected before archive. } + if (!(await isSafeArchivedSessionStoreTarget(provider, targetPath))) { + failed.push({ + file_path: sourcePath, + step: "archive_local", + error: "unsafe-archived-session-target", + }); + continue; + } try { await mkdir(path.dirname(targetPath), { recursive: true }); await copyFile(sourcePath, targetPath); @@ -84,7 +94,7 @@ export async function runUnarchiveProviderAction( continue; } try { - await stat(targetPath); + await lstat(targetPath); failed.push({ file_path: sourcePath, step: "unarchive_local", @@ -94,6 +104,14 @@ export async function runUnarchiveProviderAction( } catch { // Missing target is expected before restore. } + if (!(await isSafeArchivedSessionRestoreTarget(provider, targetPath))) { + failed.push({ + file_path: sourcePath, + step: "unarchive_local", + error: "unsafe-archived-session-target", + }); + continue; + } try { await mkdir(path.dirname(targetPath), { recursive: true }); await copyFile(sourcePath, targetPath); diff --git a/apps/api-ts/src/domains/providers/services/actions/index.ts b/apps/api-ts/src/domains/providers/services/actions/index.ts index c4a8ba5..99dd883 100644 --- a/apps/api-ts/src/domains/providers/services/actions/index.ts +++ b/apps/api-ts/src/domains/providers/services/actions/index.ts @@ -1,4 +1,4 @@ -import { BACKUP_ROOT } from "../../../../lib/constants.js"; +import { BACKUP_ROOT } from "../../../recovery/constants.js"; import { deriveProviderBackupId, resolveProviderActionBackupRoot, @@ -182,9 +182,8 @@ export async function runProviderSessionAction( backupManifestPath = backupStage.manifest_path; backedUpCount = backupStage.items.length; } - const backupManifestWriteFailed = - backupStage?.failed.some((failure) => failure.step === "manifest_write") ?? false; - if (backupManifestWriteFailed && action !== "backup_local") { + const backupStageFailed = (backupStage?.failed.length ?? 0) > 0; + if (backupStageFailed && action !== "backup_local") { return buildProviderActionAppliedResponse({ provider, action, diff --git a/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts b/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts index 8701106..b39fe1a 100644 --- a/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/provider-entries.ts @@ -8,7 +8,7 @@ import { COPILOT_CURSOR_WORKSPACE_STORAGE, COPILOT_VSCODE_GLOBAL, COPILOT_VSCODE_WORKSPACE_STORAGE, -} from "../../../../lib/constants.js"; +} from "../../constants.js"; import { providerLabel, supportsProviderCleanup, @@ -33,18 +33,21 @@ export function buildProviderMatrixProviders( const claudeStatus = providerStatus(signals.claudeRootExists, signals.claudeSessionLogs); const geminiStatus = providerStatus(signals.geminiRootExists, signals.geminiSessionLogs); const copilotStatus = providerStatus(signals.copilotRootExists, signals.copilotSignalFiles); + const codexCleanupReady = supportsProviderCleanup("codex") && codexStatus !== "missing"; + const codexHardDeleteReady = + supportsProviderHardDelete("codex") && codexStatus !== "missing"; return [ { provider: "codex" as ProviderId, name: providerLabel("codex"), status: codexStatus, - capability_level: capabilityLevel(codexStatus, supportsProviderCleanup("codex")), + capability_level: capabilityLevel(codexStatus, codexCleanupReady), capabilities: { - read_sessions: true, - analyze_context: true, - safe_cleanup: supportsProviderCleanup("codex"), - hard_delete: supportsProviderHardDelete("codex"), + read_sessions: signals.codexRootExists, + analyze_context: signals.codexSessionLogs > 0, + safe_cleanup: codexCleanupReady, + hard_delete: codexHardDeleteReady, }, evidence: { roots: signals.codexHomes, diff --git a/apps/api-ts/src/domains/providers/services/matrix/signals.ts b/apps/api-ts/src/domains/providers/services/matrix/signals.ts index c2a6ce8..3de7e88 100644 --- a/apps/api-ts/src/domains/providers/services/matrix/signals.ts +++ b/apps/api-ts/src/domains/providers/services/matrix/signals.ts @@ -9,7 +9,7 @@ import { COPILOT_CURSOR_WORKSPACE_STORAGE, COPILOT_VSCODE_GLOBAL, COPILOT_VSCODE_WORKSPACE_STORAGE, -} from "../../../../lib/constants.js"; +} from "../../constants.js"; import { countFilesRecursiveByExt, countJsonlFilesRecursive, diff --git a/apps/api-ts/src/domains/providers/services/search/abort.test.ts b/apps/api-ts/src/domains/providers/services/search/abort.test.ts new file mode 100644 index 0000000..96dac3f --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/abort.test.ts @@ -0,0 +1,297 @@ +import path from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import { describe, expect, it, vi } from "vitest"; +import { searchConversationSessionHits, searchConversationSessions } from "./index.js"; +import { deferred, makeRow, makeTranscript, waitForCondition } from "./test-fixtures.js"; + +describe("abort propagation", () => { + it("propagates abort signals into conversation search loaders", async () => { + const controller = new AbortController(); + controller.abort(); + const rawFileSearchLoader = vi.fn(async (_rows, _q, options) => { + if (options?.signal?.aborted) { + const error = new Error("aborted"); + error.name = "AbortError"; + throw error; + } + return new Map(); + }); + + await expect( + searchConversationSessions([makeRow()], "token", { + signal: controller.signal, + rawFileSearchLoader, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(rawFileSearchLoader).toHaveBeenCalled(); + }); + + it("aborts before rebuilding provider scans when the signal is already cancelled", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile: vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })), + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + const controller = new AbortController(); + controller.abort(); + + try { + const mod = await import("./index.js"); + await expect( + mod.getProviderSessionScan("claude", 1, { signal: controller.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(walkFilesByExt).not.toHaveBeenCalled(); + expect(statMock).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("lets callers abort while waiting on a shared manifest inflight promise", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + const firstWalk = deferred(); + const walkFilesByExt = vi + .fn() + .mockImplementationOnce(async () => firstWalk.promise); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile: vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })), + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + const firstRequest = mod.getProviderSessionScan("claude", 1); + await waitForCondition( + () => walkFilesByExt.mock.calls.length === 1, + "the shared manifest rebuild to start", + ); + + const controller = new AbortController(); + const secondRequest = mod.getProviderSessionScan("claude", 2, { + signal: controller.signal, + }); + controller.abort(); + + await expect(secondRequest).rejects.toMatchObject({ name: "AbortError" }); + firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); + const firstScan = await firstRequest; + expect(firstScan.rows[0]?.file_path).toContain("session-a.jsonl"); + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("keeps shared manifest work alive when the first caller aborts", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + const firstWalk = deferred(); + const walkFilesByExt = vi + .fn() + .mockImplementationOnce(async () => firstWalk.promise); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: vi.fn(async (target: string) => path.resolve(target)), + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile: vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })), + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + const controller = new AbortController(); + const firstRequest = mod.getProviderSessionScan("claude", 1, { + signal: controller.signal, + }); + await waitForCondition( + () => walkFilesByExt.mock.calls.length === 1, + "the shared manifest rebuild to start", + ); + const secondRequest = mod.getProviderSessionScan("claude", 1); + + controller.abort(); + await expect(firstRequest).rejects.toMatchObject({ name: "AbortError" }); + + firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); + const secondScan = await secondRequest; + expect(secondScan.rows[0]?.file_path).toContain("session-a.jsonl"); + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/local-search.test.ts b/apps/api-ts/src/domains/providers/services/search/local-search.test.ts new file mode 100644 index 0000000..7c49c64 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/local-search.test.ts @@ -0,0 +1,107 @@ +import path from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import { describe, expect, it, vi } from "vitest"; + +describe("searchLocalConversationsTs metadata-only fast path", () => { + it("avoids probing session files for metadata-only queries", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const walkFilesByExt = vi.fn(async () => [ + "/virtual/claude/rollout-2026-03-25T10-00-00-019d-meta-only.jsonl", + ]); + const statMock = vi.fn(async (target: string) => { + if (target === "/virtual/claude") { + return { + size: 0, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + }; + } + return { + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + }; + }); + const probeSessionFile = vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "Should not load", + title_source: "fixture", + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + vi.doMock("../../../threads/query.js", () => ({ + getThreadsTs: vi.fn(async () => ({ rows: [] })), + })); + + try { + const mod = await import("./index.js"); + const result = await mod.searchLocalConversationsTs( + "rollout-2026-03-25", + { + providers: ["claude"], + pageSize: 10, + sessionLimitPerProvider: 1, + }, + ); + + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0]?.session_id).toContain("rollout-2026-03-25"); + expect(probeSessionFile).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + vi.doUnmock("../../../threads/query.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/manifest-store.test.ts b/apps/api-ts/src/domains/providers/services/search/manifest-store.test.ts new file mode 100644 index 0000000..905b4bf --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/manifest-store.test.ts @@ -0,0 +1,462 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import { describe, expect, it, vi } from "vitest"; +import { deferred, waitForCondition } from "./test-fixtures.js"; + +describe("provider manifest cache behavior", () => { + it("reuses the provider manifest after the scan cache TTL until invalidated", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); + const probeSessionFile = vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: realpathMock, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + + await mod.getProviderSessionScan("claude", 1); + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(61_000); + + await mod.getProviderSessionScan("claude", 1); + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + expect(probeSessionFile).toHaveBeenCalledTimes(2); + + mod.invalidateProviderSearchCaches("claude"); + await mod.getProviderSessionScan("claude", 1); + expect(walkFilesByExt).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("reuses a persisted provider manifest across module reloads", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const installMocks = (walkFilesByExt: ReturnType) => { + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); + const probeSessionFile = vi.fn(async () => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + })); + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: realpathMock, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + }; + + try { + const firstWalk = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); + installMocks(firstWalk); + const firstMod = await import("./index.js"); + await firstMod.getProviderSessionScan("claude", 1); + expect(firstWalk).toHaveBeenCalledTimes(1); + + vi.resetModules(); + + const secondWalk = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); + installMocks(secondWalk); + const secondMod = await import("./index.js"); + const secondScan = await secondMod.getProviderSessionScan("claude", 1); + expect(secondWalk).not.toHaveBeenCalled(); + expect(secondScan.rows[0]?.source).toBe("projects"); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("reclassifies persisted manifest candidate source from the current matching root", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); + const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); + const sessionPath = "/virtual/claude/session-a.jsonl"; + await mkdir(path.dirname(cacheFile), { recursive: true }); + await writeFile( + cacheFile, + JSON.stringify({ + expires_at: Date.now() + 60_000, + manifest: { + provider: "claude", + name: "Claude", + root_exists: true, + candidates: [ + { + source: "stale-cache-source", + file_path: sessionPath, + size_bytes: 128, + mtime: "2026-03-25T10:00:00.000Z", + mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), + }, + ], + total_bytes: 128, + }, + }), + "utf8", + ); + + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: vi.fn(async (target: string) => path.resolve(target)), + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + })); + + try { + const { readPersistedProviderManifest } = await import( + "./manifest-store.js" + ); + const persisted = await readPersistedProviderManifest("claude"); + + expect(persisted?.manifest.candidates).toHaveLength(1); + expect(persisted?.manifest.candidates[0]).toMatchObject({ + file_path: path.resolve(sessionPath), + source: "projects", + }); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("discards persisted manifest candidates outside current provider roots", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-03-25T10:00:00.000Z")); + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const version = createHash("sha1").update("claude").digest("hex").slice(0, 8); + const cacheFile = path.join(cacheDir, `manifest-claude-${version}.json`); + await mkdir(path.dirname(cacheFile), { recursive: true }); + await writeFile( + cacheFile, + JSON.stringify({ + expires_at: Date.now() + 60_000, + manifest: { + provider: "claude", + name: "Claude", + root_exists: true, + candidates: [ + { + source: "projects", + file_path: "/outside/session-a.jsonl", + size_bytes: 128, + mtime: "2026-03-25T10:00:00.000Z", + mtime_ms: Date.parse("2026-03-25T10:00:00.000Z"), + }, + ], + total_bytes: 128, + }, + }), + "utf8", + ); + + const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-b.jsonl"]); + const statMock = vi.fn(async () => ({ + size: 256, + mtimeMs: Date.parse("2026-03-25T10:05:00.000Z"), + })); + const realpathMock = vi.fn(async (target: string) => path.resolve(target)); + const probeSessionFile = vi.fn(async (filePath: string) => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: path.basename(filePath, ".jsonl"), + title_source: "fixture", + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + realpath: realpathMock, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + const scan = await mod.getProviderSessionScan("claude", 1); + + expect(walkFilesByExt).toHaveBeenCalledTimes(1); + expect(scan.rows[0]?.file_path).toBe("/virtual/claude/session-b.jsonl"); + expect(probeSessionFile).toHaveBeenCalledWith("/virtual/claude/session-b.jsonl"); + } finally { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + + it("does not let a stale inflight rebuild overwrite a newer manifest after invalidation", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + + const firstWalk = deferred(); + const walkFilesByExt = vi + .fn() + .mockImplementationOnce(async () => firstWalk.promise) + .mockImplementationOnce(async () => ["/virtual/claude/session-b.jsonl"]); + const statMock = vi.fn(async (filePath: string) => ({ + size: 128, + mtimeMs: filePath.includes("session-b") + ? Date.parse("2026-03-25T10:05:00.000Z") + : Date.parse("2026-03-25T10:00:00.000Z"), + })); + const probeSessionFile = vi.fn(async (filePath: string) => ({ + ok: true, + format: "jsonl", + error: null, + detected_title: path.basename(filePath, ".jsonl"), + title_source: "fixture", + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { root: "/virtual/claude", source: "projects", exts: [".jsonl"] }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile, + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + const stalePromise = mod.getProviderSessionScan("claude", 1); + await waitForCondition( + () => walkFilesByExt.mock.calls.length === 1, + "the stale manifest rebuild to start", + ); + + mod.invalidateProviderSearchCaches("claude"); + const freshPromise = mod.getProviderSessionScan("claude", 1); + const freshScan = await freshPromise; + + expect(freshScan.rows[0]?.file_path).toContain("session-b.jsonl"); + + firstWalk.resolve(["/virtual/claude/session-a.jsonl"]); + const staleScan = await stalePromise; + expect(staleScan.rows[0]?.file_path).toContain("session-a.jsonl"); + + const cachedScan = await mod.getProviderSessionScan("claude", 1); + expect(cachedScan.rows[0]?.file_path).toContain("session-b.jsonl"); + expect(walkFilesByExt).toHaveBeenCalledTimes(2); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); + +}); diff --git a/apps/api-ts/src/domains/providers/services/search/row-search.test.ts b/apps/api-ts/src/domains/providers/services/search/row-search.test.ts new file mode 100644 index 0000000..543c377 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/row-search.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it, vi } from "vitest"; +import { searchConversationRows } from "./index.js"; +import { makeRow, makeTranscript } from "./test-fixtures.js"; + +describe("searchConversationRows", () => { + it("returns metadata matches without loading transcripts when the limit is satisfied", async () => { + const row = makeRow({ + display_title: "Unrelated title", + session_id: "rollout-2026-03-25T10-00-00-019d-chatgpt-scope", + file_path: "/tmp/chatgpt-scope.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "chatgpt only in transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + + const result = await searchConversationRows([row], "chatgpt", { + limit: 1, + transcriptLoader, + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + match_kind: "title", + session_id: "chatgpt-scope", + }); + expect(transcriptLoader).not.toHaveBeenCalled(); + }); + + it("stops before loading later transcripts once metadata hits fill the result limit", async () => { + const metadataRow = makeRow({ + display_title: "Unrelated title", + session_id: "rollout-2026-03-25T10-00-00-019d-obsidian-review", + file_path: "/tmp/obsidian-review.jsonl", + mtime: "2026-03-25T10:05:00.000Z", + }); + const transcriptRow = makeRow({ + display_title: "Another unrelated title", + session_id: "rollout-2026-03-25T10-00-00-019d-transcript-only", + file_path: "/tmp/transcript-only.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async (_provider, filePath) => + makeTranscript( + filePath === metadataRow.file_path ? metadataRow : transcriptRow, + [ + { + idx: 0, + role: "assistant", + text: "obsidian transcript fallback", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ], + ), + ); + + const result = await searchConversationRows( + [metadataRow, transcriptRow], + "obsidian", + { + limit: 1, + transcriptLoader, + }, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + match_kind: "title", + session_id: "obsidian-review", + }); + expect(transcriptLoader).not.toHaveBeenCalled(); + }); + + it("suppresses exact duplicate message hits within the same session", async () => { + const row = makeRow({ + display_title: "Unrelated title", + file_path: "/tmp/search-dedup-case.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "user", + text: "agent duplicate result", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "user", + text: "agent duplicate result", + ts: "2026-03-25T10:00:01.000Z", + source_type: "event_msg.user_message", + }, + ]), + ); + + const result = await searchConversationRows([row], "agent", { + limit: 40, + transcriptLoader, + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + match_kind: "message", + role: "user", + }); + }); + + it("falls back to transcript scanning when metadata matches are insufficient", async () => { + const row = makeRow({ + display_title: "Unrelated title", + file_path: "/tmp/search-transcript-case.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "search fallback hit", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + + const result = await searchConversationRows([row], "fallback", { + limit: 5, + transcriptLoader, + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + match_kind: "message", + role: "assistant", + }); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + }); + + it("skips transcript scanning entirely for metadata-only queries", async () => { + const row = makeRow({ + display_title: "Unrelated title", + session_id: "rollout-2026-03-25T10-00-00-019d-rollout-query", + file_path: "/tmp/rollout-2026-03-25T10-00-00-019d-rollout-query.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "rollout token only in transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + + const result = await searchConversationRows([row], "rollout-2026-03-25", { + limit: 10, + transcriptLoader, + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + match_kind: "title", + session_id: "rollout-2026-03-25T10-00-00-019d-rollout-query", + }); + expect(transcriptLoader).not.toHaveBeenCalled(); + }); + + it("omits cleanup thread ids that are not openable in the current thread read model", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41", + file_path: + "/tmp/rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41.jsonl", + display_title: "ThreadLens handoff", + }); + + const result = await searchConversationRows([row], "handoff", { + limit: 10, + openableThreadIds: new Set(), + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + provider: "codex", + session_id: "rollout-2026-03-29T01-53-21-019d355d-51c3-7753-b2f2-8db585337e41", + match_kind: "title", + }); + expect(result.results[0].thread_id).toBeUndefined(); + }); + + it("keeps cleanup thread ids when they are openable in the current thread read model", async () => { + const threadId = "019d355d-51c3-7753-b2f2-8db585337e41"; + const row = makeRow({ + session_id: `rollout-2026-03-29T01-53-21-${threadId}`, + file_path: `/tmp/rollout-2026-03-29T01-53-21-${threadId}.jsonl`, + display_title: "ThreadLens handoff", + }); + + const result = await searchConversationRows([row], "handoff", { + limit: 10, + openableThreadIds: new Set([threadId]), + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + provider: "codex", + thread_id: threadId, + match_kind: "title", + }); + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/session-hit-resolver.test.ts b/apps/api-ts/src/domains/providers/services/search/session-hit-resolver.test.ts new file mode 100644 index 0000000..2a475c3 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/session-hit-resolver.test.ts @@ -0,0 +1,86 @@ +import path from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import { describe, expect, it, vi } from "vitest"; + +describe("searchConversationSessionHitsTs", () => { + it("returns null when a manifest candidate points to a deleted session file", async () => { + vi.resetModules(); + const cacheDir = await mkdtemp(path.join(os.tmpdir(), "threadlens-search-test-")); + vi.stubEnv("THREADLENS_SEARCH_CACHE_DIR", cacheDir); + const walkFilesByExt = vi.fn(async () => ["/virtual/claude/session-a.jsonl"]); + const statMock = vi.fn(async () => ({ + size: 128, + mtimeMs: Date.parse("2026-03-25T10:00:00.000Z"), + })); + + vi.doMock("../../../../lib/utils.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + walkFilesByExt, + }; + }); + vi.doMock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stat: statMock, + }; + }); + vi.doMock("../../path-safety.js", () => ({ + providerScanRootSpecs: async () => [ + { + root: "/virtual/claude", + source: "projects", + exts: [".jsonl"], + }, + ], + providerName: () => "Claude", + codexTranscriptSearchRoots: async () => [], + })); + vi.doMock("../../title-detection.js", () => ({ + getCodexThreadTitleMap: vi.fn(async () => new Map()), + invalidateCodexThreadTitleMapCache: vi.fn(), + extractCodexThreadIdFromSessionName: vi.fn(() => ""), + })); + vi.doMock("../../probe.js", () => ({ + inferSessionId: vi.fn((filePath: string) => + filePath.split("/").at(-1)?.replace(/\.jsonl$/i, "") ?? filePath, + ), + isCopilotGlobalSessionLikeFile: vi.fn(() => false), + isWorkspaceChatSessionPath: vi.fn(() => false), + probeSessionFile: vi.fn(async () => ({ + ok: false, + format: null, + error: "missing", + detected_title: "", + title_source: null, + })), + })); + vi.doMock("../../matrix.js", () => ({ + providerStatus: vi.fn(() => "ready"), + })); + + try { + const mod = await import("./index.js"); + const result = await mod.searchConversationSessionHitsTs("token", { + provider: "claude", + sessionId: "session-a", + filePath: "/virtual/claude/session-a.jsonl", + }); + + expect(result).toBeNull(); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + vi.doUnmock("../../../../lib/utils.js"); + vi.doUnmock("node:fs/promises"); + vi.doUnmock("../../path-safety.js"); + vi.doUnmock("../../title-detection.js"); + vi.doUnmock("../../probe.js"); + vi.doUnmock("../../matrix.js"); + await rm(cacheDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/session-hit-search.test.ts b/apps/api-ts/src/domains/providers/services/search/session-hit-search.test.ts new file mode 100644 index 0000000..2314a91 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/session-hit-search.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it, vi } from "vitest"; +import { searchConversationSessionHits, selectConversationSessionHitsRow } from "./index.js"; +import { makeRow, makeTranscript } from "./test-fixtures.js"; + +describe("searchConversationSessionHits", () => { + it("prefers the explicit file path when duplicate logical session ids exist", () => { + const liveRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-duplicate-hits", + file_path: "/tmp/search-hits-live.jsonl", + source: "sessions", + mtime: "2026-03-25T10:00:00.000Z", + }); + const backupRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-duplicate-hits", + file_path: "/tmp/search-hits-backup.jsonl", + source: "cleanup_backups", + mtime: "2026-03-25T10:05:00.000Z", + }); + + const targetRow = selectConversationSessionHitsRow([backupRow, liveRow], { + sessionId: liveRow.session_id, + filePath: liveRow.file_path, + }); + + expect(targetRow?.file_path).toBe(liveRow.file_path); + expect(targetRow?.source).toBe("sessions"); + }); + + it("returns paginated hits for a single matching session", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-session-hits", + display_title: "token session detail", + file_path: "/tmp/search-session-hits.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token first hit", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "token second hit", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + { + idx: 2, + role: "assistant", + text: "token third hit", + ts: "2026-03-25T10:00:02.000Z", + source_type: "response_item.message", + }, + ]), + ); + + const page1 = await searchConversationSessionHits(row, "token", { + pageSize: 2, + transcriptLoader, + }); + const page2 = await searchConversationSessionHits(row, "token", { + pageSize: 2, + cursor: page1.next_cursor ?? undefined, + transcriptLoader, + }); + + expect(page1.total_hits).toBe(4); + expect(page1.hits).toHaveLength(2); + expect(page1.has_more).toBe(true); + expect(page2.hits).toHaveLength(2); + expect(page2.has_more).toBe(false); + }); + + it("uses raw file hits as a prefilter for session detail expansion", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-session-hits-raw", + display_title: "Unrelated title", + file_path: "/tmp/search-session-hits-raw.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token transcript one", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "token transcript two", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + { + idx: 2, + role: "assistant", + text: "token transcript three", + ts: "2026-03-25T10:00:02.000Z", + source_type: "response_item.message", + }, + { + idx: 3, + role: "assistant", + text: "token transcript four", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: [ + "token preview one", + "token preview two", + "token preview three", + "token preview four", + ], + match_count: 4, + has_more_hits: false, + exact_phrase_count: 4, + }, + ], + ]), + ); + + const page1 = await searchConversationSessionHits(row, "token", { + pageSize: 2, + transcriptLoader, + rawFileSearchLoader, + }); + const page2 = await searchConversationSessionHits(row, "token", { + pageSize: 2, + cursor: page1.next_cursor ?? undefined, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(page1.total_hits).toBe(4); + expect(page1.hits.map((hit) => hit.snippet)).toEqual([ + "token transcript one", + "token transcript two", + ]); + expect(page1.has_more).toBe(true); + expect(page2.hits.map((hit) => hit.snippet)).toEqual([ + "token transcript three", + "token transcript four", + ]); + expect(page2.has_more).toBe(false); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(2); + }); + + it("falls back to transcript hits when a later cursor exceeds the raw snippet window", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-session-hits-fallback", + display_title: "Token overflow session", + file_path: "/tmp/search-session-hits-fallback.jsonl", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript( + row, + Array.from({ length: 45 }, (_, index) => ({ + idx: index, + role: "assistant" as const, + text: `token transcript hit ${index + 1}`, + ts: `2026-03-25T10:00:${String(index).padStart(2, "0")}.000Z`, + source_type: "response_item.message" as const, + })), + ), + ); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: Array.from({ length: 20 }, (_, index) => `token preview ${index + 1}`), + match_count: 120, + has_more_hits: true, + exact_phrase_count: 20, + }, + ], + ]), + ); + + const page1 = await searchConversationSessionHits(row, "token", { + pageSize: 40, + transcriptLoader, + rawFileSearchLoader, + }); + const page2 = await searchConversationSessionHits(row, "token", { + pageSize: 40, + cursor: page1.next_cursor ?? undefined, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(page1.hits).toHaveLength(40); + expect(page1.has_more).toBe(true); + expect(page2.hits.length).toBeGreaterThan(0); + expect(page2.hits[0]?.snippet).toContain("token transcript hit"); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/session-hit-selection.test.ts b/apps/api-ts/src/domains/providers/services/search/session-hit-selection.test.ts new file mode 100644 index 0000000..2e0ad80 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/session-hit-selection.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { selectConversationSessionHitsRow } from "./index.js"; +import { makeRow } from "./test-fixtures.js"; + +describe("selectConversationSessionHitsRow", () => { + it("does not trust file_path when it points at a different logical session", () => { + const requestedSessionId = "rollout-2026-03-25T10-00-00-019d-target"; + const requestedRow = makeRow({ + session_id: requestedSessionId, + file_path: "/tmp/search-target.jsonl", + }); + const mismatchedRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-other", + file_path: "/tmp/search-other.jsonl", + }); + + const selected = selectConversationSessionHitsRow( + [mismatchedRow, requestedRow], + { + sessionId: requestedSessionId, + filePath: mismatchedRow.file_path, + }, + ); + + expect(selected).toEqual(requestedRow); + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/session-search.test.ts b/apps/api-ts/src/domains/providers/services/search/session-search.test.ts new file mode 100644 index 0000000..417e37f --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/session-search.test.ts @@ -0,0 +1,463 @@ +import { describe, expect, it, vi } from "vitest"; +import { searchConversationSessions } from "./index.js"; +import { makeRow, makeTranscript } from "./test-fixtures.js"; + +describe("searchConversationSessions", () => { + it("collapses duplicate logical sessions across backup copies", async () => { + const liveRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-duplicate", + display_title: "Duplicate token session", + file_path: "/tmp/search-duplicate-live.jsonl", + source: "sessions", + mtime: "2026-03-25T10:10:00.000Z", + }); + const backupRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-duplicate", + display_title: "Duplicate token session", + file_path: "/tmp/search-duplicate-backup.jsonl", + source: "cleanup_backups", + mtime: "2026-03-25T10:11:00.000Z", + }); + + const result = await searchConversationSessions([liveRow, backupRow], "token", { + pageSize: 10, + previewHitsPerSession: 2, + }); + + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0]).toMatchObject({ + session_id: "search-duplicate-live", + source: "sessions", + }); + }); + + it("keeps distinct matching sessions even when one session has many hits", async () => { + const dominantRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-dominant", + display_title: "Dominant token session", + file_path: "/tmp/search-dominant.jsonl", + mtime: "2026-03-25T10:10:00.000Z", + }); + const secondaryRow = makeRow({ + session_id: "rollout-2026-03-25T09-00-00-019d-secondary", + display_title: "Secondary token session", + file_path: "/tmp/search-secondary.jsonl", + mtime: "2026-03-25T09:00:00.000Z", + }); + const transcriptLoader = vi.fn(async (_provider, filePath) => { + if (filePath === dominantRow.file_path) { + return makeTranscript(dominantRow, [ + { + idx: 0, + role: "assistant", + text: "token dominant match one", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "token dominant match two", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + { + idx: 2, + role: "assistant", + text: "token dominant match three", + ts: "2026-03-25T10:00:02.000Z", + source_type: "response_item.message", + }, + ]); + } + return makeTranscript(secondaryRow, [ + { + idx: 0, + role: "user", + text: "token secondary match", + ts: "2026-03-25T09:00:00.000Z", + source_type: "response_item.message", + }, + ]); + }); + + const result = await searchConversationSessions( + [dominantRow, secondaryRow], + "token", + { + pageSize: 10, + previewHitsPerSession: 2, + transcriptLoader, + }, + ); + + expect(result.total_matching_sessions).toBe(2); + expect(result.total_matching_hits).toBeNull(); + expect(result.sessions).toHaveLength(2); + expect(result.sessions[0]).toMatchObject({ + session_id: "search-dominant", + match_count: 3, + has_more_hits: true, + }); + expect(result.sessions[0]?.preview_matches).toHaveLength(2); + expect(result.sessions[1]).toMatchObject({ + session_id: "search-secondary", + match_count: 2, + has_more_hits: false, + }); + expect(result.results).toHaveLength(4); + }); + + it("keeps the full session id as the fallback title when no detected title exists", async () => { + const sessionId = "rollout-2026-04-16T23-47-06-019d96c2-9123-7481-9127-224fad716008"; + const row = makeRow({ + session_id: sessionId, + display_title: "", + file_path: `/tmp/${sessionId}.jsonl`, + probe: { + ok: true, + format: "jsonl", + error: null, + detected_title: "", + title_source: null, + }, + }); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: ["token preview one"], + match_count: 1, + has_more_hits: false, + exact_phrase_count: 1, + }, + ], + ]), + ); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token from parsed transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions[0]?.title).toBe(sessionId); + }); + + it("paginates session results with a stable next cursor", async () => { + const rows = [ + makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-alpha", + display_title: "alpha token", + file_path: "/tmp/search-alpha.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }), + makeRow({ + session_id: "rollout-2026-03-25T09-00-00-019d-bravo", + display_title: "bravo token", + file_path: "/tmp/search-bravo.jsonl", + mtime: "2026-03-25T09:00:00.000Z", + }), + makeRow({ + session_id: "rollout-2026-03-25T08-00-00-019d-charlie", + display_title: "charlie token", + file_path: "/tmp/search-charlie.jsonl", + mtime: "2026-03-25T08:00:00.000Z", + }), + ]; + + const page1 = await searchConversationSessions(rows, "token", { + pageSize: 2, + previewHitsPerSession: 1, + }); + const page2 = await searchConversationSessions(rows, "token", { + pageSize: 2, + cursor: page1.next_cursor ?? undefined, + previewHitsPerSession: 1, + }); + + expect(page1.sessions).toHaveLength(2); + expect(page1.has_more).toBe(true); + expect(page1.truncated).toBe(false); + expect(page1.next_cursor).toBe("2"); + expect(page1.total_matching_sessions).toBe(3); + expect(page1.total_matching_hits).toBeNull(); + expect(page2.sessions).toHaveLength(1); + expect(page2.has_more).toBe(false); + expect(page2.truncated).toBe(false); + expect(page2.next_cursor).toBeNull(); + expect(page2.total_matching_sessions).toBe(3); + expect(page2.total_matching_hits).toBeNull(); + }); + + it("orders pages by global session quality before applying the cursor", async () => { + const newerWeakRow = makeRow({ + session_id: "rollout-2026-03-25T11-00-00-019d-newer-weak", + display_title: "recent chat", + file_path: "/tmp/search-newer-weak.jsonl", + mtime: "2026-03-25T11:00:00.000Z", + }); + const middleWeakRow = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-middle-weak", + display_title: "middle chat", + file_path: "/tmp/search-middle-weak.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const olderStrongRow = makeRow({ + session_id: "rollout-2026-03-25T09-00-00-019d-older-strong", + display_title: "token in title", + file_path: "/tmp/search-older-strong.jsonl", + mtime: "2026-03-25T09:00:00.000Z", + }); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + newerWeakRow.file_path, + { + snippets: ["token weak recent"], + match_count: 1, + has_more_hits: false, + exact_phrase_count: 0, + }, + ], + [ + middleWeakRow.file_path, + { + snippets: ["token weak middle"], + match_count: 1, + has_more_hits: false, + exact_phrase_count: 0, + }, + ], + ]), + ); + const transcriptLoader = vi.fn(async (_provider, filePath) => { + const row = + [newerWeakRow, middleWeakRow, olderStrongRow].find((item) => item.file_path === filePath) ?? + newerWeakRow; + return makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: `token transcript hit for ${row.session_id}`, + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]); + }); + + const page1 = await searchConversationSessions( + [newerWeakRow, middleWeakRow, olderStrongRow], + "token", + { + pageSize: 1, + previewHitsPerSession: 1, + transcriptLoader, + rawFileSearchLoader, + }, + ); + const page2 = await searchConversationSessions( + [newerWeakRow, middleWeakRow, olderStrongRow], + "token", + { + pageSize: 1, + cursor: page1.next_cursor ?? undefined, + previewHitsPerSession: 1, + transcriptLoader, + rawFileSearchLoader, + }, + ); + + expect(page1.sessions[0]?.session_id).toBe("search-older-strong"); + expect(page1.total_matching_sessions).toBe(3); + expect(page2.sessions[0]?.session_id).not.toBe("search-older-strong"); + }); + + it("uses raw file hits as a prefilter but builds previews from parsed transcripts", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-raw-preview", + display_title: "Raw preview token session", + file_path: "/tmp/search-raw-preview.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token from parsed transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: [ + "token preview one", + "token preview two", + "token preview three", + ], + match_count: 4, + has_more_hits: true, + exact_phrase_count: 2, + }, + ], + ]), + ); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0]).toMatchObject({ + session_id: "search-raw-preview", + match_count: 2, + has_more_hits: false, + }); + expect(result.sessions[0]?.preview_matches.map((match) => match.snippet)).toEqual([ + "Raw preview token session", + "token from parsed transcript", + ]); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); + }); + + it("does not expose raw matches from system or policy transcript messages", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-policy-only", + display_title: "Policy only session", + file_path: "/tmp/search-policy-only.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "system", + text: "token from system instructions", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + { + idx: 1, + role: "assistant", + text: "# AGENTS.md instructions for token handling", + ts: "2026-03-25T10:00:01.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => + new Map([ + [ + row.file_path, + { + snippets: ["token raw system preview"], + match_count: 1, + has_more_hits: false, + exact_phrase_count: 0, + }, + ], + ]), + ); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions).toHaveLength(0); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); + }); + + it("skips transcript parsing for raw-search-eligible rows when ripgrep found no match", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-no-match", + display_title: "No raw match session", + file_path: "/tmp/search-no-match.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token would only appear if transcript parsing ran", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => new Map()); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions).toHaveLength(0); + expect(transcriptLoader).not.toHaveBeenCalled(); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); + }); + + it("falls back to transcript parsing when raw file search fails unexpectedly", async () => { + const row = makeRow({ + session_id: "rollout-2026-03-25T10-00-00-019d-raw-failure", + display_title: "Raw loader failure session", + file_path: "/tmp/search-raw-failure.jsonl", + mtime: "2026-03-25T10:00:00.000Z", + }); + const transcriptLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "token fallback transcript hit", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + const rawFileSearchLoader = vi.fn(async () => { + throw new Error("rg blew up"); + }); + + const result = await searchConversationSessions([row], "token", { + pageSize: 10, + previewHitsPerSession: 3, + transcriptLoader, + rawFileSearchLoader, + }); + + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0]?.session_id).toBe("search-raw-failure"); + expect(transcriptLoader).toHaveBeenCalledTimes(1); + expect(rawFileSearchLoader).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/test-fixtures.ts b/apps/api-ts/src/domains/providers/services/search/test-fixtures.ts new file mode 100644 index 0000000..7189211 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/test-fixtures.ts @@ -0,0 +1,58 @@ +import type { ProviderSessionRow, TranscriptPayload } from "../../types.js"; + +export function makeRow(overrides: Partial = {}): ProviderSessionRow { + return { + provider: "codex", + source: "sessions", + session_id: "rollout-2026-03-25T10-00-00-019d-search-test", + display_title: "Search fixture", + file_path: "/tmp/search-fixture.jsonl", + size_bytes: 128, + mtime: "2026-03-25T10:00:00.000Z", + probe: { + ok: true, + format: "jsonl", + error: null, + detected_title: "Search fixture", + title_source: "fixture", + }, + ...overrides, + }; +} + +export function makeTranscript( + row: ProviderSessionRow, + messages: TranscriptPayload["messages"], +): TranscriptPayload { + return { + provider: row.provider, + thread_id: "019d-search-test-thread", + file_path: row.file_path, + scanned_lines: messages.length, + message_count: messages.length, + truncated: false, + messages, + }; +} + +export function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +export async function waitForCondition( + predicate: () => boolean, + label: string, + attempts = 100, +) { + for (let index = 0; index < attempts; index += 1) { + if (predicate()) return; + await new Promise((resolve) => setImmediate(resolve)); + } + throw new Error(`timed out waiting for ${label}`); +} diff --git a/apps/api-ts/src/domains/providers/services/search/transcript-loader-cache.test.ts b/apps/api-ts/src/domains/providers/services/search/transcript-loader-cache.test.ts new file mode 100644 index 0000000..96ccf90 --- /dev/null +++ b/apps/api-ts/src/domains/providers/services/search/transcript-loader-cache.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { createCachedConversationTranscriptLoader } from "./index.js"; +import { makeRow, makeTranscript } from "./test-fixtures.js"; + +describe("createCachedConversationTranscriptLoader", () => { + it("reuses transcripts while file mtime is unchanged", async () => { + const row = makeRow({ + file_path: "/tmp/search-cache-fixture.jsonl", + mtime: "2026-03-25T10:10:00.000Z", + }); + const baseLoader = vi.fn(async () => + makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "agent cache check", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]), + ); + const cachedLoader = createCachedConversationTranscriptLoader(baseLoader); + + await cachedLoader(row); + await cachedLoader(row); + await cachedLoader({ ...row, mtime: "2026-03-25T10:00:05.000Z" }); + + expect(baseLoader).toHaveBeenCalledTimes(2); + }); + + it("retries after a transient transcript load failure when mtime is unchanged", async () => { + const row = makeRow({ + file_path: "/tmp/search-cache-transient-error.jsonl", + mtime: "2026-03-25T10:10:00.000Z", + }); + let shouldFail = true; + const baseLoader = vi.fn(async () => { + if (shouldFail) { + shouldFail = false; + throw new Error("transient transcript failure"); + } + return makeTranscript(row, [ + { + idx: 0, + role: "assistant", + text: "recovered transcript", + ts: "2026-03-25T10:00:00.000Z", + source_type: "response_item.message", + }, + ]); + }); + const cachedLoader = createCachedConversationTranscriptLoader(baseLoader); + + const first = await cachedLoader(row); + const second = await cachedLoader(row); + + expect(first).toBeNull(); + expect(second?.messages[0]?.text).toBe("recovered transcript"); + expect(baseLoader).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/api-ts/src/domains/providers/services/search/types.ts b/apps/api-ts/src/domains/providers/services/search/types.ts index 0bf6ee3..33145a7 100644 --- a/apps/api-ts/src/domains/providers/services/search/types.ts +++ b/apps/api-ts/src/domains/providers/services/search/types.ts @@ -1,24 +1,19 @@ import type { ConversationSearchPayload, ProviderId, + ProviderSessionCandidate, ProviderSessionRow, ProviderSessionScan, TranscriptPayload, } from "../../types.js"; +export type { ProviderSessionCandidate } from "../../types.js"; + export type ProviderScanCacheEntry = { expires_at: number; scan: ProviderSessionScan; }; -export type ProviderSessionCandidate = { - source: string; - file_path: string; - size_bytes: number; - mtime: string; - mtime_ms: number; -}; - export type ProviderSessionManifest = { provider: ProviderId; name: string; diff --git a/apps/api-ts/src/domains/providers/shared/backup-roots.ts b/apps/api-ts/src/domains/providers/shared/backup-roots.ts index eb0c3fc..47191e6 100644 --- a/apps/api-ts/src/domains/providers/shared/backup-roots.ts +++ b/apps/api-ts/src/domains/providers/shared/backup-roots.ts @@ -1,10 +1,8 @@ import path from "node:path"; import type { ProviderId } from "@threadlens/shared-contracts"; -import { - BACKUP_ROOT, - CODEX_HOME, -} from "../../../lib/constants.js"; +import { BACKUP_ROOT } from "../../recovery/constants.js"; +import { CODEX_HOME } from "../constants.js"; import type { ProviderRootSpec } from "../types.js"; function providerActionBackupRoots(provider: ProviderId): string[] { diff --git a/apps/api-ts/src/domains/providers/shared/file-roots.ts b/apps/api-ts/src/domains/providers/shared/file-roots.ts index 2ec37db..2b21623 100644 --- a/apps/api-ts/src/domains/providers/shared/file-roots.ts +++ b/apps/api-ts/src/domains/providers/shared/file-roots.ts @@ -15,7 +15,7 @@ import { GEMINI_ANTIGRAVITY_CONVERSATIONS_DIR, GEMINI_HISTORY_DIR, GEMINI_TMP_DIR, -} from "../../../lib/constants.js"; +} from "../constants.js"; import { discoverChatGptConversationRoots } from "../adapters/chatgpt/roots.js"; import { discoverCodexCwdBackupRoots } from "../adapters/codex/roots.js"; import { diff --git a/apps/api-ts/src/domains/providers/shared/path-safety.ts b/apps/api-ts/src/domains/providers/shared/path-safety.ts index a9a0339..2c4b9be 100644 --- a/apps/api-ts/src/domains/providers/shared/path-safety.ts +++ b/apps/api-ts/src/domains/providers/shared/path-safety.ts @@ -7,7 +7,7 @@ import { import { CHAT_DIR, CODEX_HOME, -} from "../../../lib/constants.js"; +} from "../constants.js"; import { providerRootSpecs, providerScanRootSpecs, diff --git a/apps/api-ts/src/domains/providers/types.ts b/apps/api-ts/src/domains/providers/types.ts index 842874a..97f2204 100644 --- a/apps/api-ts/src/domains/providers/types.ts +++ b/apps/api-ts/src/domains/providers/types.ts @@ -33,6 +33,14 @@ export type ProviderSessionRow = { probe: ProviderSessionProbe; }; +export type ProviderSessionCandidate = { + source: string; + file_path: string; + size_bytes: number; + mtime: string; + mtime_ms: number; +}; + export type ProviderSessionScan = { provider: ProviderId; name: string; diff --git a/apps/api-ts/src/domains/recovery/backups.ts b/apps/api-ts/src/domains/recovery/backups.ts index e9e0383..f02e37e 100644 --- a/apps/api-ts/src/domains/recovery/backups.ts +++ b/apps/api-ts/src/domains/recovery/backups.ts @@ -1,6 +1,6 @@ import { readdir, stat } from "node:fs/promises"; import path from "node:path"; -import { BACKUP_ROOT } from "../../lib/constants.js"; +import { BACKUP_ROOT } from "./constants.js"; import { walkFiles } from "../../lib/utils.js"; export type RecoveryBackupSet = { diff --git a/apps/api-ts/src/domains/recovery/checklist.ts b/apps/api-ts/src/domains/recovery/checklist.ts index 1aef8d9..1e92a8d 100644 --- a/apps/api-ts/src/domains/recovery/checklist.ts +++ b/apps/api-ts/src/domains/recovery/checklist.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { RECOVERY_CHECKLIST_FILE } from "../../lib/constants.js"; +import { RECOVERY_CHECKLIST_FILE } from "./constants.js"; import { isRecord, readJsonFile } from "../../lib/utils.js"; export type RecoveryChecklistItem = { diff --git a/apps/api-ts/src/domains/recovery/constants.ts b/apps/api-ts/src/domains/recovery/constants.ts new file mode 100644 index 0000000..bc04dad --- /dev/null +++ b/apps/api-ts/src/domains/recovery/constants.ts @@ -0,0 +1,37 @@ +import path from "node:path"; +import { + DOCUMENTS_DIR, + DOWNLOADS_DIR, + PROJECT_ROOT, + STATE_DIR, + START_TS, +} from "../../platform/paths.js"; +import { + CHAT_DIR, + CODEX_HOME, +} from "../providers/constants.js"; + +export { + CHAT_DIR, + CODEX_HOME, + PROJECT_ROOT, + START_TS, +}; + +export const ROADMAP_STATE_FILE = path.join(STATE_DIR, "roadmap_state.json"); +export const ROADMAP_LOG_FILE = path.join( + STATE_DIR, + "roadmap_checkins.jsonl", +); +export const RECOVERY_CHECKLIST_FILE = path.join( + STATE_DIR, + "w4_checklist.json", +); +export const RECOVERY_PLAN_DIR = path.join(STATE_DIR, "recovery_plans"); + +export const BACKUP_ROOT = path.join(DOCUMENTS_DIR, "ThreadLens", "backups"); +export const RECOVERY_EXPORT_ROOT = path.join( + DOWNLOADS_DIR, + "ThreadLens", + "recovery-exports", +); diff --git a/apps/api-ts/src/domains/recovery/export.ts b/apps/api-ts/src/domains/recovery/export.ts index 01a1a8a..12ced6f 100644 --- a/apps/api-ts/src/domains/recovery/export.ts +++ b/apps/api-ts/src/domains/recovery/export.ts @@ -1,7 +1,7 @@ import { createReadStream } from "node:fs"; import { cp, mkdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; -import { BACKUP_ROOT, RECOVERY_EXPORT_ROOT } from "../../lib/constants.js"; +import { BACKUP_ROOT, RECOVERY_EXPORT_ROOT } from "./constants.js"; import { scanBackupSets } from "./backups.js"; import { writePortableZipArchive } from "./portable-zip.js"; import { nowIsoUtc } from "../../lib/utils.js"; diff --git a/apps/api-ts/src/domains/recovery/inventory.ts b/apps/api-ts/src/domains/recovery/inventory.ts index f8ad3eb..0607a08 100644 --- a/apps/api-ts/src/domains/recovery/inventory.ts +++ b/apps/api-ts/src/domains/recovery/inventory.ts @@ -9,7 +9,7 @@ import { COPILOT_VSCODE_GLOBAL, GEMINI_HOME, GEMINI_TMP_DIR, -} from "../../lib/constants.js"; +} from "../providers/constants.js"; import { nowIsoUtc, pathExists, diff --git a/apps/api-ts/src/domains/recovery/roadmap.ts b/apps/api-ts/src/domains/recovery/roadmap.ts index 56d7f75..a0034b3 100644 --- a/apps/api-ts/src/domains/recovery/roadmap.ts +++ b/apps/api-ts/src/domains/recovery/roadmap.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { ROADMAP_LOG_FILE, ROADMAP_STATE_FILE, -} from "../../lib/constants.js"; +} from "./constants.js"; import { cleanTitleText, isRecord, diff --git a/apps/api-ts/src/domains/recovery/runtime.ts b/apps/api-ts/src/domains/recovery/runtime.ts index 38564a3..b3cf378 100644 --- a/apps/api-ts/src/domains/recovery/runtime.ts +++ b/apps/api-ts/src/domains/recovery/runtime.ts @@ -18,7 +18,7 @@ import { RECOVERY_EXPORT_ROOT, RECOVERY_PLAN_DIR, CHAT_DIR, -} from "../../lib/constants.js"; +} from "./constants.js"; import { pathExists, walkFiles, diff --git a/apps/api-ts/src/domains/recovery/smoke.ts b/apps/api-ts/src/domains/recovery/smoke.ts index 6fc6b46..9686d9c 100644 --- a/apps/api-ts/src/domains/recovery/smoke.ts +++ b/apps/api-ts/src/domains/recovery/smoke.ts @@ -1,6 +1,6 @@ import { readFile, readdir } from "node:fs/promises"; import path from "node:path"; -import { PROJECT_ROOT } from "../../lib/constants.js"; +import { PROJECT_ROOT } from "./constants.js"; import { isRecord, nowIsoUtc, diff --git a/apps/api-ts/src/lib/constants.ts b/apps/api-ts/src/lib/constants.ts index 1b8bc37..b92f1ea 100644 --- a/apps/api-ts/src/lib/constants.ts +++ b/apps/api-ts/src/lib/constants.ts @@ -6,45 +6,41 @@ * dependency graph acyclic. */ -import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; - -/* ── Build-time paths ─────────────────────────────────────────────── */ - -const THIS_DIR = process.env.THREADLENS_PROJECT_ROOT - ? path.join(process.env.THREADLENS_PROJECT_ROOT, ".api-root") - : path.dirname(fileURLToPath(import.meta.url)); -// lib/ → src/ → api-ts/ → apps/ → project root -export const PROJECT_ROOT = - process.env.THREADLENS_PROJECT_ROOT ?? - path.resolve(THIS_DIR, "../../../.."); - -function resolveAppVersion(projectRoot = PROJECT_ROOT) { - const envVersion = process.env.APP_VERSION?.trim(); - if (envVersion) return envVersion; - - try { - const packageJsonPath = path.join(projectRoot, "package.json"); - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as { - version?: unknown; - }; - const packageVersion = typeof packageJson.version === "string" - ? packageJson.version.trim() - : ""; - if (packageVersion) return packageVersion; - } catch { - // Fall back to a stable placeholder when package metadata is unavailable. - } - - return "0.1.0"; -} - -/* ── Server config ────────────────────────────────────────────────── */ +import { + APP_DATA_DIR, + APP_VERSION, + DEFAULT_PORT, + DOCUMENTS_DIR, + DOWNLOADS_DIR, + HOME_DIR, + PROJECT_ROOT, + START_TS, + STATE_DIR, + resolveAppVersion, + resolvePlatformAppDataDir, + resolvePlatformDocumentsDir, + resolvePlatformDownloadsDir, + resolvePlatformHomeDir, +} from "../platform/paths.js"; + +export { + APP_DATA_DIR, + APP_VERSION, + DEFAULT_PORT, + DOCUMENTS_DIR, + DOWNLOADS_DIR, + HOME_DIR, + PROJECT_ROOT, + START_TS, + STATE_DIR, + resolveAppVersion, + resolvePlatformAppDataDir, + resolvePlatformDocumentsDir, + resolvePlatformDownloadsDir, + resolvePlatformHomeDir, +}; -export const DEFAULT_PORT = Number(process.env.API_TS_PORT ?? 8788); -export const APP_VERSION = resolveAppVersion(); -export const START_TS = Date.now(); export const THREADLENS_RELEASE_REPO = String( process.env.THREADLENS_RELEASE_REPO ?? "hanityx/threadlens", ).trim(); @@ -53,16 +49,7 @@ export const THREADLENS_LATEST_RELEASE_URL = `${THREADLENS_RELEASES_URL}/latest` export const THREADLENS_GITHUB_RELEASE_API_URL = `https://api.github.com/repos/${THREADLENS_RELEASE_REPO}/releases/latest`; -const STATE_DIR_OVERRIDE = String( - process.env.THREADLENS_STATE_DIR ?? "", -).trim(); - /* ── Project-level files ──────────────────────────────────────────── */ - -export const STATE_DIR = path.resolve( - PROJECT_ROOT, - STATE_DIR_OVERRIDE || path.join(".run", "state"), -); export const ROADMAP_STATE_FILE = path.join(STATE_DIR, "roadmap_state.json"); export const ROADMAP_LOG_FILE = path.join( STATE_DIR, @@ -79,54 +66,10 @@ export const ALERT_EVENTS_FILE = path.join(STATE_DIR, "alert_events.jsonl"); export const UPDATE_CHECK_CACHE_FILE = path.join(STATE_DIR, "update_check.json"); /* ── Provider storage roots ───────────────────────────────────────── */ - -export function resolvePlatformHomeDir( - platform = process.platform, - env: Record = process.env, -) { - if (platform === "win32") { - const userProfile = env.USERPROFILE?.trim(); - if (userProfile) return userProfile; - const homeDrive = env.HOMEDRIVE?.trim() ?? ""; - const homePath = env.HOMEPATH?.trim() ?? ""; - if (homeDrive && homePath) return `${homeDrive}${homePath}`; - } - return env.HOME ?? ""; -} - -export const HOME_DIR = resolvePlatformHomeDir(); export const PROJECTS_DIR = String( process.env.THREADLENS_PROJECTS_DIR ?? process.env.PROJECTS_DIR ?? "", ).trim(); -export function resolvePlatformAppDataDir( - platform = process.platform, - env: Record = process.env, -) { - const homeDir = resolvePlatformHomeDir(platform, env); - if (platform === "darwin") { - return path.join(homeDir, "Library", "Application Support"); - } - if (platform === "win32") { - return env.APPDATA ?? path.join(homeDir, "AppData", "Roaming"); - } - return env.XDG_CONFIG_HOME ?? path.join(homeDir, ".config"); -} - -export function resolvePlatformDocumentsDir( - platform = process.platform, - env: Record = process.env, -) { - return path.join(resolvePlatformHomeDir(platform, env), "Documents"); -} - -export function resolvePlatformDownloadsDir( - platform = process.platform, - env: Record = process.env, -) { - return path.join(resolvePlatformHomeDir(platform, env), "Downloads"); -} - export function resolvePlatformChatDir( platform = process.platform, env: Record = process.env, @@ -150,8 +93,6 @@ export const CODEX_GLOBAL_STATE_FILE = path.join( CODEX_HOME, ".codex-global-state.json", ); -export const DOCUMENTS_DIR = resolvePlatformDocumentsDir(); -export const DOWNLOADS_DIR = resolvePlatformDownloadsDir(); export const BACKUP_ROOT = path.join(DOCUMENTS_DIR, "ThreadLens", "backups"); export const RECOVERY_EXPORT_ROOT = path.join( DOWNLOADS_DIR, @@ -164,7 +105,6 @@ export const THREADS_BOOT_CACHE_FILE = path.join( "threads_boot_cache.json", ); -export const APP_DATA_DIR = resolvePlatformAppDataDir(); export const CHAT_DIR = resolvePlatformChatDir(); export const CLAUDE_HOME = path.join(HOME_DIR, ".claude"); export const CLAUDE_PROJECTS_DIR = path.join(CLAUDE_HOME, "projects"); diff --git a/apps/api-ts/src/lib/envelope.test.ts b/apps/api-ts/src/lib/envelope.test.ts new file mode 100644 index 0000000..e482e42 --- /dev/null +++ b/apps/api-ts/src/lib/envelope.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { SCHEMA_VERSION } from "@threadlens/shared-contracts"; +import { envelope, withSchemaVersion } from "./envelope.js"; + +describe("envelope", () => { + it("wraps data with the current schema version", () => { + expect(envelope({ ok: true })).toEqual({ + ok: true, + schema_version: SCHEMA_VERSION, + data: { ok: true }, + error: null, + }); + }); + + it("marks error envelopes as not ok", () => { + expect(envelope(null, "boom")).toEqual({ + ok: false, + schema_version: SCHEMA_VERSION, + data: null, + error: "boom", + }); + }); +}); + +describe("withSchemaVersion", () => { + it("adds schema_version to object payloads without one", () => { + expect(withSchemaVersion({ ok: true })).toEqual({ + ok: true, + schema_version: SCHEMA_VERSION, + }); + }); + + it("keeps object payloads that already have a schema_version unchanged", () => { + const payload = { ok: true, schema_version: "custom" }; + expect(withSchemaVersion(payload)).toBe(payload); + }); + + it("wraps primitive payloads in an envelope", () => { + expect(withSchemaVersion("value")).toEqual({ + ok: true, + schema_version: SCHEMA_VERSION, + data: "value", + error: null, + }); + }); +}); diff --git a/apps/api-ts/src/lib/envelope.ts b/apps/api-ts/src/lib/envelope.ts new file mode 100644 index 0000000..27445c1 --- /dev/null +++ b/apps/api-ts/src/lib/envelope.ts @@ -0,0 +1,27 @@ +import { + ApiEnvelope, + SCHEMA_VERSION, +} from "@threadlens/shared-contracts"; + +export function envelope( + data: T | null, + error: string | null = null, +): ApiEnvelope { + return { + ok: !error, + schema_version: SCHEMA_VERSION, + data, + error, + }; +} + +export function withSchemaVersion(payload: unknown): unknown { + if (payload && typeof payload === "object" && !Array.isArray(payload)) { + const record = payload as Record; + if (!record.schema_version) { + return { ...record, schema_version: SCHEMA_VERSION }; + } + return payload; + } + return envelope(payload, null); +} diff --git a/apps/api-ts/src/lib/fs.test.ts b/apps/api-ts/src/lib/fs.test.ts new file mode 100644 index 0000000..1ac85d8 --- /dev/null +++ b/apps/api-ts/src/lib/fs.test.ts @@ -0,0 +1,85 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + matchesPattern, + pathExists, + readFileHead, + readFileTail, + readHeadLines, + scanPathStatsTs, + walkFiles, + walkFilesByExt, +} from "./fs.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })), + ); +}); + +async function makeTempDir() { + const dir = await mkdtemp(path.join(os.tmpdir(), "threadlens-fs-test-")); + tempDirs.push(dir); + return dir; +} + +describe("filesystem helpers", () => { + it("checks paths using stat semantics", async () => { + const dir = await makeTempDir(); + const filePath = path.join(dir, "file.txt"); + await writeFile(filePath, "hello", "utf8"); + + await expect(pathExists(filePath)).resolves.toBe(true); + await expect(pathExists(path.join(dir, "missing.txt"))).resolves.toBe(false); + }); + + it("reads file heads, tails, and head lines", async () => { + const dir = await makeTempDir(); + const filePath = path.join(dir, "file.txt"); + await writeFile(filePath, "one\ntwo\nthree\n", "utf8"); + + await expect(readFileHead(filePath, 3)).resolves.toBe("one"); + await expect(readHeadLines(filePath, 2)).resolves.toEqual(["one", "two"]); + await expect(readFileTail(filePath, 5)).resolves.toEqual({ + text: "hree\n", + truncated: true, + }); + }); + + it("walks files and filters by extension", async () => { + const dir = await makeTempDir(); + await mkdir(path.join(dir, "nested")); + await writeFile(path.join(dir, "a.jsonl"), "{}", "utf8"); + await writeFile(path.join(dir, "nested", "b.txt"), "text", "utf8"); + + const allFiles = await walkFiles(dir); + expect(allFiles.map((filePath) => path.basename(filePath)).sort()).toEqual([ + "a.jsonl", + "b.txt", + ]); + await expect(walkFilesByExt(dir, [".jsonl"])).resolves.toEqual([ + path.join(dir, "a.jsonl"), + ]); + }); + + it("scans path stats with simple glob patterns", async () => { + const dir = await makeTempDir(); + await mkdir(path.join(dir, "nested")); + await writeFile(path.join(dir, "a.jsonl"), "1234", "utf8"); + await writeFile(path.join(dir, "nested", "b.txt"), "text", "utf8"); + + expect(matchesPattern("a.jsonl", "*.jsonl")).toBe(true); + const stats = await scanPathStatsTs(dir, true, "*.jsonl"); + expect(stats).toMatchObject({ + exists: true, + file_count: 1, + dir_count: 1, + total_bytes: 4, + }); + expect(stats.latest_mtime).toBeTruthy(); + }); +}); diff --git a/apps/api-ts/src/lib/fs.ts b/apps/api-ts/src/lib/fs.ts new file mode 100644 index 0000000..7c59651 --- /dev/null +++ b/apps/api-ts/src/lib/fs.ts @@ -0,0 +1,264 @@ +import { + open, + readdir, + stat, +} from "node:fs/promises"; +import path from "node:path"; + +export async function pathExists(p: string): Promise { + try { + await stat(p); + return true; + } catch { + return false; + } +} + +export async function readFileHead( + filePath: string, + maxBytes = 8192, +): Promise { + let fh: Awaited> | null = null; + try { + fh = await open(filePath, "r"); + const buf = Buffer.alloc(maxBytes); + const { bytesRead } = await fh.read(buf, 0, maxBytes, 0); + return buf.subarray(0, bytesRead).toString("utf-8"); + } catch { + return ""; + } finally { + if (fh) await fh.close(); + } +} + +export async function readHeadLines( + filePath: string, + maxLines = 5, +): Promise { + const text = await readFileHead(filePath, 8192); + if (!text) return []; + return text + .split(/\r?\n/) + .map((line) => line.trimEnd()) + .filter(Boolean) + .slice(0, Math.max(0, maxLines)); +} + +export async function readFileTail( + filePath: string, + maxBytes = 2_097_152, +): Promise<{ text: string; truncated: boolean }> { + let fh: Awaited> | null = null; + try { + const st = await stat(filePath); + const size = Number(st.size); + if (!Number.isFinite(size) || size <= 0) + return { text: "", truncated: false }; + const readBytes = Math.max(1, Math.min(maxBytes, size)); + const start = Math.max(0, size - readBytes); + fh = await open(filePath, "r"); + const buf = Buffer.alloc(readBytes); + const { bytesRead } = await fh.read(buf, 0, readBytes, start); + return { + text: buf.subarray(0, bytesRead).toString("utf-8"), + truncated: start > 0, + }; + } catch { + return { text: "", truncated: false }; + } finally { + if (fh) await fh.close(); + } +} + +export async function walkFiles( + root: string, + maxItems = Number.MAX_SAFE_INTEGER, +): Promise { + const out: string[] = []; + let stop = false; + async function walk(dir: string): Promise { + if (stop) return; + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (stop) return; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile()) { + out.push(full); + if (out.length >= maxItems) { + stop = true; + return; + } + } + } + } + await walk(root); + return out; +} + +export async function walkFilesByExt( + root: string, + exts: string[], + maxItems = 1000, +): Promise { + const out: string[] = []; + const extSet = new Set(exts.map((x) => x.toLowerCase())); + async function walk(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile()) { + if (extSet.has(path.extname(entry.name).toLowerCase())) { + out.push(full); + if (out.length >= maxItems) return; + } + } + if (out.length >= maxItems) return; + } + } + try { + await walk(root); + } catch { + return out; + } + return out; +} + +export async function countDirsWithPrefix( + root: string, + prefix: string, +): Promise { + try { + const entries = await readdir(root, { withFileTypes: true }); + return entries.filter( + (entry) => entry.isDirectory() && entry.name.startsWith(prefix), + ).length; + } catch { + return 0; + } +} + +export async function quickFileCount(root: string): Promise { + try { + const entries = await readdir(root, { withFileTypes: true }); + return entries.length; + } catch { + return 0; + } +} + +export async function countJsonlFilesRecursive( + root: string, + limit = 5000, +): Promise { + let count = 0; + async function walk(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + count += 1; + } + if (count >= limit) return; + } + } + try { + await walk(root); + } catch { + return count; + } + return count; +} + +export async function countFilesRecursiveByExt( + root: string, + exts: string[], + limit = 5000, +): Promise { + let count = 0; + const extSet = new Set(exts.map((ext) => String(ext || "").toLowerCase())); + async function walk(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile()) { + const ext = path.extname(entry.name).toLowerCase(); + if (extSet.has(ext)) count += 1; + } + if (count >= limit) return; + } + } + try { + await walk(root); + } catch { + return count; + } + return count; +} + +export function matchesPattern(fileName: string, pattern: string): boolean { + if (!pattern || pattern === "*") return true; + if (pattern.startsWith("*.")) { + return fileName.endsWith(pattern.slice(1)); + } + return fileName === pattern; +} + +export async function scanPathStatsTs( + targetPath: string, + recursive = true, + filePattern = "*", +) { + const out = { + path: targetPath, + exists: false, + file_count: 0, + dir_count: 0, + total_bytes: 0, + latest_mtime: "", + }; + + if (!(await pathExists(targetPath))) return out; + out.exists = true; + + const st = await stat(targetPath); + if (st.isFile()) { + out.file_count = 1; + out.total_bytes = Number(st.size); + out.latest_mtime = new Date(Number(st.mtimeMs)).toISOString(); + return out; + } + + let latestMtime = 0; + async function walk(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.dir_count += 1; + if (recursive) await walk(full); + continue; + } + if (!entry.isFile()) continue; + if (!matchesPattern(entry.name, filePattern)) continue; + try { + const fs = await stat(full); + out.file_count += 1; + out.total_bytes += Number(fs.size); + latestMtime = Math.max(latestMtime, Number(fs.mtimeMs)); + } catch { + // no-op + } + } + } + await walk(targetPath); + out.latest_mtime = latestMtime ? new Date(latestMtime).toISOString() : ""; + return out; +} diff --git a/apps/api-ts/src/lib/guards.ts b/apps/api-ts/src/lib/guards.ts new file mode 100644 index 0000000..60a91e2 --- /dev/null +++ b/apps/api-ts/src/lib/guards.ts @@ -0,0 +1,3 @@ +export function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/apps/api-ts/src/lib/http.ts b/apps/api-ts/src/lib/http.ts new file mode 100644 index 0000000..d01e053 --- /dev/null +++ b/apps/api-ts/src/lib/http.ts @@ -0,0 +1,13 @@ +export async function fetchWithTimeout( + url: string, + init: RequestInit = {}, + timeoutMs = 2500, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { ...init, signal: controller.signal }); + } finally { + clearTimeout(timer); + } +} diff --git a/apps/api-ts/src/lib/json.test.ts b/apps/api-ts/src/lib/json.test.ts new file mode 100644 index 0000000..3027a94 --- /dev/null +++ b/apps/api-ts/src/lib/json.test.ts @@ -0,0 +1,30 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { readJsonFile, safeJsonParse } from "./json.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })), + ); +}); + +describe("json helpers", () => { + it("returns null for invalid JSON strings", () => { + expect(safeJsonParse("{")).toBeNull(); + expect(safeJsonParse("{\"ok\":true}")).toEqual({ ok: true }); + }); + + it("reads JSON files and falls back to an empty object on failure", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "threadlens-json-test-")); + tempDirs.push(dir); + const filePath = path.join(dir, "data.json"); + await writeFile(filePath, JSON.stringify({ ok: true }), "utf8"); + + await expect(readJsonFile(filePath)).resolves.toEqual({ ok: true }); + await expect(readJsonFile(path.join(dir, "missing.json"))).resolves.toEqual({}); + }); +}); diff --git a/apps/api-ts/src/lib/json.ts b/apps/api-ts/src/lib/json.ts new file mode 100644 index 0000000..478e768 --- /dev/null +++ b/apps/api-ts/src/lib/json.ts @@ -0,0 +1,18 @@ +import { readFile } from "node:fs/promises"; + +export function safeJsonParse(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + return null; + } +} + +export async function readJsonFile(filePath: string): Promise { + try { + const raw = await readFile(filePath, "utf-8"); + return JSON.parse(raw); + } catch { + return {}; + } +} diff --git a/apps/api-ts/src/lib/process.ts b/apps/api-ts/src/lib/process.ts new file mode 100644 index 0000000..809e19e --- /dev/null +++ b/apps/api-ts/src/lib/process.ts @@ -0,0 +1,24 @@ +import { execSync } from "node:child_process"; + +export function runCmdText(command: string, timeout = 4000): string { + try { + const out = execSync(command, { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + timeout, + shell: "/bin/zsh", + }); + return String(out || "").trim(); + } catch { + return ""; + } +} + +export function getTmuxSessions(): string[] { + const out = runCmdText("tmux ls -F '#S'", 700); + if (!out) return []; + return out + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} diff --git a/apps/api-ts/src/lib/query.test.ts b/apps/api-ts/src/lib/query.test.ts new file mode 100644 index 0000000..6337e28 --- /dev/null +++ b/apps/api-ts/src/lib/query.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { + canonicalizeQuery, + parseNumber, + parseQueryNumber, + parseQueryString, +} from "./query.js"; + +describe("query helpers", () => { + it("parses scalar and array query strings", () => { + expect(parseQueryString("alpha")).toBe("alpha"); + expect(parseQueryString(["alpha", "beta"])).toBe("alpha"); + expect(parseQueryString(undefined)).toBe(""); + }); + + it("falls back when numeric query values are invalid", () => { + expect(parseQueryNumber("42", 1)).toBe(42); + expect(parseQueryNumber("nope", 7)).toBe(7); + expect(parseNumber("5", 0)).toBe(5); + expect(parseNumber("nan", 9)).toBe(9); + }); + + it("canonicalizes query keys and array values", () => { + expect( + canonicalizeQuery({ + b: "two words", + a: ["z", "x"], + skip: undefined, + }), + ).toBe("a=x&a=z&b=two%20words"); + }); +}); diff --git a/apps/api-ts/src/lib/query.ts b/apps/api-ts/src/lib/query.ts new file mode 100644 index 0000000..ed01a26 --- /dev/null +++ b/apps/api-ts/src/lib/query.ts @@ -0,0 +1,39 @@ +export type QueryMap = Record; + +export function parseNumber(value: unknown, fallback = 0): number { + const n = Number(value); + return Number.isFinite(n) ? n : fallback; +} + +export function parseQueryString( + value: string | string[] | undefined, +): string { + if (Array.isArray(value)) return String(value[0] ?? ""); + return String(value ?? ""); +} + +export function parseQueryNumber( + value: string | string[] | undefined, + fallback: number, +): number { + const n = Number(parseQueryString(value)); + return Number.isFinite(n) ? n : fallback; +} + +export function canonicalizeQuery(query?: QueryMap): string { + if (!query) return ""; + const keys = Object.keys(query).sort(); + const parts: string[] = []; + for (const key of keys) { + const value = query[key]; + if (value === undefined) continue; + if (Array.isArray(value)) { + [...value].sort().forEach((item) => { + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(item)}`); + }); + continue; + } + parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`); + } + return parts.join("&"); +} diff --git a/apps/api-ts/src/lib/schemas.ts b/apps/api-ts/src/lib/schemas.ts new file mode 100644 index 0000000..3aacaaa --- /dev/null +++ b/apps/api-ts/src/lib/schemas.ts @@ -0,0 +1,6 @@ +import { z } from "zod"; + +export const bulkRequestSchema = z.object({ + action: z.enum(["pin", "unpin", "archive_local", "unarchive_local", "resume_command"]), + thread_ids: z.array(z.string().min(1)).min(1).max(500), +}); diff --git a/apps/api-ts/src/lib/text.ts b/apps/api-ts/src/lib/text.ts new file mode 100644 index 0000000..6c1a89f --- /dev/null +++ b/apps/api-ts/src/lib/text.ts @@ -0,0 +1,9 @@ +export function cleanTitleText(text: string, maxLen = 280): string { + const t = String(text || "") + .replace(/\n/g, " ") + .replace(/\s+/g, " ") + .trim(); + if (!t) return ""; + if (t.length <= maxLen) return t; + return `${t.slice(0, maxLen - 1).trim()}…`; +} diff --git a/apps/api-ts/src/lib/time.ts b/apps/api-ts/src/lib/time.ts new file mode 100644 index 0000000..b0b3d70 --- /dev/null +++ b/apps/api-ts/src/lib/time.ts @@ -0,0 +1,3 @@ +export function nowIsoUtc(): string { + return new Date().toISOString(); +} diff --git a/apps/api-ts/src/lib/utils.ts b/apps/api-ts/src/lib/utils.ts index a085b64..665fbe2 100644 --- a/apps/api-ts/src/lib/utils.ts +++ b/apps/api-ts/src/lib/utils.ts @@ -1,437 +1,56 @@ /** - * Shared utility functions used across multiple server modules. + * Compatibility facade for shared API utilities. * - * Pure/stateless helpers only — no module-level caches live here. - * Depends on `./constants.js` only. + * New code should prefer the focused modules in this directory. */ -import { execSync } from "node:child_process"; -import { - mkdir, - readFile, - readdir, - stat, - open, -} from "node:fs/promises"; -import path from "node:path"; -import { z } from "zod"; -import { - ApiEnvelope, - SCHEMA_VERSION, -} from "@threadlens/shared-contracts"; - -/* ── Types ────────────────────────────────────────────────────────── */ - -export type QueryMap = Record; - -/* ── Zod schemas ──────────────────────────────────────────────────── */ - -export const bulkRequestSchema = z.object({ - action: z.enum(["pin", "unpin", "archive_local", "unarchive_local", "resume_command"]), - thread_ids: z.array(z.string().min(1)).min(1).max(500), -}); - -/* ── API envelope ─────────────────────────────────────────────────── */ - -export function envelope( - data: T | null, - error: string | null = null, -): ApiEnvelope { - return { - ok: !error, - schema_version: SCHEMA_VERSION, - data, - error, - }; -} - -export function withSchemaVersion(payload: unknown): unknown { - if (payload && typeof payload === "object" && !Array.isArray(payload)) { - const record = payload as Record; - if (!record.schema_version) { - return { ...record, schema_version: SCHEMA_VERSION }; - } - return payload; - } - return envelope(payload, null); -} - -/* ── HTTP / fetch ─────────────────────────────────────────────────── */ - -export async function fetchWithTimeout( - url: string, - init: RequestInit = {}, - timeoutMs = 2500, -): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetch(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timer); - } -} - -/* ── JSON ─────────────────────────────────────────────────────────── */ - -export function safeJsonParse(raw: string): unknown { - try { - return JSON.parse(raw); - } catch { - return null; - } -} - -export async function readJsonFile(filePath: string): Promise { - try { - const raw = await readFile(filePath, "utf-8"); - return JSON.parse(raw); - } catch { - return {}; - } -} - -/* ── Primitives / guards ──────────────────────────────────────────── */ - -export function isRecord(value: unknown): value is Record { - return Boolean(value) && typeof value === "object" && !Array.isArray(value); -} - -export function nowIsoUtc(): string { - return new Date().toISOString(); -} - -export function cleanTitleText(text: string, maxLen = 280): string { - const t = String(text || "") - .replace(/\n/g, " ") - .replace(/\s+/g, " ") - .trim(); - if (!t) return ""; - if (t.length <= maxLen) return t; - return `${t.slice(0, maxLen - 1).trim()}…`; -} - -export function parseNumber(value: unknown, fallback = 0): number { - const n = Number(value); - return Number.isFinite(n) ? n : fallback; -} - -export function parseQueryString( - value: string | string[] | undefined, -): string { - if (Array.isArray(value)) return String(value[0] ?? ""); - return String(value ?? ""); -} - -export function parseQueryNumber( - value: string | string[] | undefined, - fallback: number, -): number { - const n = Number(parseQueryString(value)); - return Number.isFinite(n) ? n : fallback; -} - -export function canonicalizeQuery(query?: QueryMap): string { - if (!query) return ""; - const keys = Object.keys(query).sort(); - const parts: string[] = []; - for (const key of keys) { - const value = query[key]; - if (value === undefined) continue; - if (Array.isArray(value)) { - [...value].sort().forEach((item) => { - parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(item)}`); - }); - continue; - } - parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`); - } - return parts.join("&"); -} - -/* ── Shell / process ──────────────────────────────────────────────── */ - -export function runCmdText(command: string, timeout = 4000): string { - try { - const out = execSync(command, { - encoding: "utf-8", - stdio: ["ignore", "pipe", "ignore"], - timeout, - shell: "/bin/zsh", - }); - return String(out || "").trim(); - } catch { - return ""; - } -} - -export function getTmuxSessions(): string[] { - const out = runCmdText("tmux ls -F '#S'", 700); - if (!out) return []; - return out - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -/* ── Filesystem ───────────────────────────────────────────────────── */ - -export async function pathExists(p: string): Promise { - try { - await stat(p); - return true; - } catch { - return false; - } -} - -export async function readFileHead( - filePath: string, - maxBytes = 8192, -): Promise { - let fh: Awaited> | null = null; - try { - fh = await open(filePath, "r"); - const buf = Buffer.alloc(maxBytes); - const { bytesRead } = await fh.read(buf, 0, maxBytes, 0); - return buf.subarray(0, bytesRead).toString("utf-8"); - } catch { - return ""; - } finally { - if (fh) await fh.close(); - } -} - -export async function readHeadLines( - filePath: string, - maxLines = 5, -): Promise { - const text = await readFileHead(filePath, 8192); - if (!text) return []; - return text - .split(/\r?\n/) - .map((line) => line.trimEnd()) - .filter(Boolean) - .slice(0, Math.max(0, maxLines)); -} - -export async function readFileTail( - filePath: string, - maxBytes = 2_097_152, -): Promise<{ text: string; truncated: boolean }> { - let fh: Awaited> | null = null; - try { - const st = await stat(filePath); - const size = Number(st.size); - if (!Number.isFinite(size) || size <= 0) - return { text: "", truncated: false }; - const readBytes = Math.max(1, Math.min(maxBytes, size)); - const start = Math.max(0, size - readBytes); - fh = await open(filePath, "r"); - const buf = Buffer.alloc(readBytes); - const { bytesRead } = await fh.read(buf, 0, readBytes, start); - return { - text: buf.subarray(0, bytesRead).toString("utf-8"), - truncated: start > 0, - }; - } catch { - return { text: "", truncated: false }; - } finally { - if (fh) await fh.close(); - } -} - -export async function walkFiles( - root: string, - maxItems = Number.MAX_SAFE_INTEGER, -): Promise { - const out: string[] = []; - let stop = false; - async function walk(dir: string): Promise { - if (stop) return; - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - if (stop) return; - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - await walk(full); - } else if (entry.isFile()) { - out.push(full); - if (out.length >= maxItems) { - stop = true; - return; - } - } - } - } - await walk(root); - return out; -} - -export async function walkFilesByExt( - root: string, - exts: string[], - maxItems = 1000, -): Promise { - const out: string[] = []; - const extSet = new Set(exts.map((x) => x.toLowerCase())); - async function walk(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - await walk(full); - } else if (entry.isFile()) { - if (extSet.has(path.extname(entry.name).toLowerCase())) { - out.push(full); - if (out.length >= maxItems) return; - } - } - if (out.length >= maxItems) return; - } - } - try { - await walk(root); - } catch { - return out; - } - return out; -} - -/* ── Counting / stats ─────────────────────────────────────────────── */ - -export async function countDirsWithPrefix( - root: string, - prefix: string, -): Promise { - try { - const entries = await readdir(root, { withFileTypes: true }); - return entries.filter( - (entry) => entry.isDirectory() && entry.name.startsWith(prefix), - ).length; - } catch { - return 0; - } -} - -export async function quickFileCount(root: string): Promise { - try { - const entries = await readdir(root, { withFileTypes: true }); - return entries.length; - } catch { - return 0; - } -} - -export async function countJsonlFilesRecursive( - root: string, - limit = 5000, -): Promise { - let count = 0; - async function walk(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - await walk(full); - } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { - count += 1; - } - if (count >= limit) return; - } - } - try { - await walk(root); - } catch { - return count; - } - return count; -} - -export async function countFilesRecursiveByExt( - root: string, - exts: string[], - limit = 5000, -): Promise { - let count = 0; - const extSet = new Set(exts.map((ext) => String(ext || "").toLowerCase())); - async function walk(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - await walk(full); - } else if (entry.isFile()) { - const ext = path.extname(entry.name).toLowerCase(); - if (extSet.has(ext)) count += 1; - } - if (count >= limit) return; - } - } - try { - await walk(root); - } catch { - return count; - } - return count; -} - -export function matchesPattern(fileName: string, pattern: string): boolean { - if (!pattern || pattern === "*") return true; - if (pattern.startsWith("*.")) { - return fileName.endsWith(pattern.slice(1)); - } - return fileName === pattern; -} - -export async function scanPathStatsTs( - targetPath: string, - recursive = true, - filePattern = "*", -) { - const out = { - path: targetPath, - exists: false, - file_count: 0, - dir_count: 0, - total_bytes: 0, - latest_mtime: "", - }; - - if (!(await pathExists(targetPath))) return out; - out.exists = true; - - const st = await stat(targetPath); - if (st.isFile()) { - out.file_count = 1; - out.total_bytes = Number(st.size); - out.latest_mtime = new Date(Number(st.mtimeMs)).toISOString(); - return out; - } - - let latestMtime = 0; - async function walk(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - out.dir_count += 1; - if (recursive) await walk(full); - continue; - } - if (!entry.isFile()) continue; - if (!matchesPattern(entry.name, filePattern)) continue; - try { - const fs = await stat(full); - out.file_count += 1; - out.total_bytes += Number(fs.size); - latestMtime = Math.max(latestMtime, Number(fs.mtimeMs)); - } catch { - // no-op - } - } - } - await walk(targetPath); - out.latest_mtime = latestMtime ? new Date(latestMtime).toISOString() : ""; - return out; -} +export { + envelope, + withSchemaVersion, +} from "./envelope.js"; +export { + fetchWithTimeout, +} from "./http.js"; +export { + safeJsonParse, + readJsonFile, +} from "./json.js"; +export { + isRecord, +} from "./guards.js"; +export { + nowIsoUtc, +} from "./time.js"; +export { + cleanTitleText, +} from "./text.js"; +export { + parseNumber, + parseQueryString, + parseQueryNumber, + canonicalizeQuery, +} from "./query.js"; +export type { + QueryMap, +} from "./query.js"; +export { + runCmdText, + getTmuxSessions, +} from "./process.js"; +export { + pathExists, + readFileHead, + readHeadLines, + readFileTail, + walkFiles, + walkFilesByExt, + countDirsWithPrefix, + quickFileCount, + countJsonlFilesRecursive, + countFilesRecursiveByExt, + matchesPattern, + scanPathStatsTs, +} from "./fs.js"; +export { + bulkRequestSchema, +} from "./schemas.js"; diff --git a/apps/api-ts/src/platform/paths.ts b/apps/api-ts/src/platform/paths.ts new file mode 100644 index 0000000..f3730e3 --- /dev/null +++ b/apps/api-ts/src/platform/paths.ts @@ -0,0 +1,93 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const THIS_DIR = process.env.THREADLENS_PROJECT_ROOT + ? path.join(process.env.THREADLENS_PROJECT_ROOT, ".api-root") + : path.dirname(fileURLToPath(import.meta.url)); + +// platform/ -> src/ -> api-ts/ -> apps/ -> project root +export const PROJECT_ROOT = + process.env.THREADLENS_PROJECT_ROOT ?? + path.resolve(THIS_DIR, "../../../.."); + +export function resolveAppVersion(projectRoot = PROJECT_ROOT) { + const envVersion = process.env.APP_VERSION?.trim(); + if (envVersion) return envVersion; + + try { + const packageJsonPath = path.join(projectRoot, "package.json"); + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as { + version?: unknown; + }; + const packageVersion = typeof packageJson.version === "string" + ? packageJson.version.trim() + : ""; + if (packageVersion) return packageVersion; + } catch { + // Fall back to a stable placeholder when package metadata is unavailable. + } + + return "0.1.0"; +} + +export const DEFAULT_PORT = Number(process.env.API_TS_PORT ?? 8788); +export const APP_VERSION = resolveAppVersion(); +export const START_TS = Date.now(); + +const STATE_DIR_OVERRIDE = String( + process.env.THREADLENS_STATE_DIR ?? "", +).trim(); + +export const STATE_DIR = path.resolve( + PROJECT_ROOT, + STATE_DIR_OVERRIDE || path.join(".run", "state"), +); + +export function resolvePlatformHomeDir( + platform = process.platform, + env: Record = process.env, +) { + if (platform === "win32") { + const userProfile = env.USERPROFILE?.trim(); + if (userProfile) return userProfile; + const homeDrive = env.HOMEDRIVE?.trim() ?? ""; + const homePath = env.HOMEPATH?.trim() ?? ""; + if (homeDrive && homePath) return `${homeDrive}${homePath}`; + } + return env.HOME ?? ""; +} + +export const HOME_DIR = resolvePlatformHomeDir(); + +export function resolvePlatformAppDataDir( + platform = process.platform, + env: Record = process.env, +) { + const homeDir = resolvePlatformHomeDir(platform, env); + if (platform === "darwin") { + return path.join(homeDir, "Library", "Application Support"); + } + if (platform === "win32") { + return env.APPDATA ?? path.join(homeDir, "AppData", "Roaming"); + } + return env.XDG_CONFIG_HOME ?? path.join(homeDir, ".config"); +} + +export function resolvePlatformDocumentsDir( + platform = process.platform, + env: Record = process.env, +) { + return path.join(resolvePlatformHomeDir(platform, env), "Documents"); +} + +export function resolvePlatformDownloadsDir( + platform = process.platform, + env: Record = process.env, +) { + return path.join(resolvePlatformHomeDir(platform, env), "Downloads"); +} + +export const APP_DATA_DIR = resolvePlatformAppDataDir(); +export const DOCUMENTS_DIR = resolvePlatformDocumentsDir(); +export const DOWNLOADS_DIR = resolvePlatformDownloadsDir(); diff --git a/apps/api-ts/src/server.test.ts b/apps/api-ts/src/server.test.ts index 673580d..3e728db 100644 --- a/apps/api-ts/src/server.test.ts +++ b/apps/api-ts/src/server.test.ts @@ -471,7 +471,7 @@ describe("api-ts direct endpoints", () => { ).toBe(true); }); - it("POST /api/provider-session-action blocks cleanup on read-only providers", async () => { + it("POST /api/provider-session-action keeps read-only providers outside the action route", async () => { const res = await app.inject({ method: "POST", url: "/api/provider-session-action", @@ -486,7 +486,8 @@ describe("api-ts direct endpoints", () => { const payload = res.json(); const root = payload.data ?? payload; expect(root.ok).toBe(false); - expect(root.error).toBe("cleanup-disabled-provider"); + expect(String(root.error)).toContain("Invalid option"); + expect(String(root.error)).not.toContain("chatgpt"); }); it("POST /api/provider-session-action rejects invalid provider id", async () => { diff --git a/docs/ADDING_PROVIDER.md b/docs/ADDING_PROVIDER.md index ec3a839..1cba6df 100644 --- a/docs/ADDING_PROVIDER.md +++ b/docs/ADDING_PROVIDER.md @@ -39,10 +39,21 @@ pnpm docs:provider-support ## 2. Add Local Roots Provider roots define where ThreadLens is allowed to read provider files from. -Update the provider adapter/root definitions in -`apps/api-ts/src/domains/providers/`. +Update the explicit api-ts provider registry and the provider-specific adapter +folder under `apps/api-ts/src/domains/providers/`. -Add roots for: +Touch these files deliberately: + +- `apps/api-ts/src/domains/providers/registry.ts` for the explicit implemented + adapter registration +- `apps/api-ts/src/domains/providers/capabilities.ts` for route/search/report + exposure policy +- `apps/api-ts/src/domains/providers/adapters//` for provider-specific + root discovery, title, transcript, or health evidence +- `apps/api-ts/src/domains/providers/shared/` only for provider-neutral path/root + helpers + +Add or validate roots for: - live session files - archived session files, if supported @@ -50,11 +61,12 @@ Add roots for: - provider-specific cache locations, if needed Path safety must fail closed. Unknown provider ids must not fall through to -another provider's roots. +another provider's roots. `ProviderRootSpec.source` is used in backup-relative +paths, so it must be deterministic and validation-safe. Do not derive `source` +from arbitrary local directory names. -The current first step toward cleaner extension is the internal adapter registry. Adapters identify the provider and expose root discovery. Keep this boundary -small at first: +small: ```ts export type ProviderAdapter = { @@ -80,24 +92,20 @@ export type ProviderSessionLocator = ## 3. Add Search and Transcript Support -Most providers can use the existing JSON/JSONL transcript flow. In the current -code, custom storage shapes still touch these areas: +Most providers can use the existing JSON/JSONL transcript flow. Provider-neutral +engines live under `apps/api-ts/src/domains/providers/services/`; provider-specific +logic belongs under `apps/api-ts/src/domains/providers/adapters//`. -- `apps/api-ts/src/domains/providers/search/constants.ts` -- `apps/api-ts/src/domains/providers/search.ts` -- `apps/api-ts/src/domains/providers/transcript.ts` -- `apps/api-ts/src/domains/providers/probe.ts` - -`search/constants.ts` contains the default searchable providers and optional -scan budget weights. New providers use a default weight of `1` when omitted, so -only add a custom weight when the provider needs a different scan budget. +Do not add a new provider by hardcoding it into search or matrix entry points. +The default search/report/matrix surfaces should flow through +`capabilities.ts` and the explicit adapter registry. If a provider needs a +custom scan budget, add that weight in the search service only after the +provider is registered and tested. Add fixtures for the real local file shape. Do not infer a provider format from docs alone; use sample session files. -The intended direction is to move provider-specific session discovery and -transcript building behind adapters over follow-up PRs. For database-backed -providers, start read-only: +For database-backed providers, start read-only: - open the database in read-only mode - use a busy timeout or retry path for live provider databases @@ -107,8 +115,9 @@ providers, start read-only: ## 4. Add Provider Diagnostics -Update `apps/api-ts/src/domains/providers/matrix.ts` so the provider appears in -the provider health matrix with useful evidence. +Update the provider adapter/health evidence so the provider appears in the +provider health matrix with useful evidence. Matrix rows should stay aligned +with the explicit adapter registry and capability helpers. The matrix should show: @@ -119,8 +128,10 @@ The matrix should show: ## 5. Add Archive, Backup, and Delete Behavior -If the provider supports destructive file actions, update -`apps/api-ts/src/domains/providers/actions.ts`. +If the provider supports destructive file actions, update the provider action +service under `apps/api-ts/src/domains/providers/services/actions/` only when +the action is truly cross-provider. Provider-specific root/target differences +belong in `adapters//` or `shared/` helpers. Destructive actions must keep the existing safety rules: @@ -130,7 +141,8 @@ Destructive actions must keep the existing safety rules: - backup behavior must not be weakened If the provider should be read-only, set `safe_cleanup: false` and -`hard_delete: false`. +`hard_delete: false`. Read-only providers are excluded from provider session +actions, including `backup_local`. ## 6. Add Tests @@ -149,18 +161,19 @@ Useful existing tests: - `apps/api-ts/src/domains/providers/path-safety.test.ts` - `apps/api-ts/src/domains/providers/parser-fixtures.test.ts` - `apps/api-ts/src/domains/providers/search.test.ts` +- `apps/api-ts/src/domains/providers/search-policy.test.ts` - `apps/api-ts/src/domains/providers/transcript.test.ts` - `apps/api-ts/src/domains/providers/matrix.test.ts` - `apps/api-ts/src/domains/providers/actions.test.ts` +- `apps/api-ts/src/domains/providers/adapters.test.ts` ## Adapter Direction -The current code still has provider behavior spread across registry, path -safety, search, transcript, matrix, and actions. The intended direction is an -internal provider adapter registry, not an external plugin system. +The provider boundary is an internal provider adapter registry, not an external +plugin system. -The adapter boundary starts with provider identity and root discovery. Later PRs -can move provider-specific behavior behind that boundary, such as: +The adapter boundary starts with provider identity and root discovery. Keep +provider-specific behavior behind that boundary when possible, such as: - health evidence - session discovery @@ -171,6 +184,10 @@ Capabilities should remain in `packages/shared-contracts/src/index.ts`; adapter code should read them through `getProviderCapability(adapter.id)` instead of storing a second copy. +`capabilities.ts` in api-ts is not a second capability source of truth. It is +the policy layer that intersects shared-contract facts with api-ts implemented +adapters and route exposure rules. + Do not add dynamic loading of third-party provider code. ThreadLens reads and can mutate local AI session files, so provider support should stay in reviewed source code unless a separate security model is designed. @@ -183,6 +200,7 @@ Before opening a provider PR: pnpm docs:provider-support pnpm --filter @threadlens/shared-contracts test pnpm --filter @threadlens/api test +pnpm lint:deps ``` If provider UI copy changes are included, also run the relevant web or TUI tests. diff --git a/docs/PROVIDER_SUPPORT.md b/docs/PROVIDER_SUPPORT.md index e216fdc..ea6ddd6 100644 --- a/docs/PROVIDER_SUPPORT.md +++ b/docs/PROVIDER_SUPPORT.md @@ -9,7 +9,7 @@ This document distinguishes between: - `read-only cache sources` that can still appear in diagnostics or provider-specific inspection The primary search/session workflow currently covers `Codex`, `Claude`, `Gemini`, `Copilot`. -`ChatGPT` is currently treated as a read-only desktop cache source. It remains available to the provider registry, but stays outside the default search scope and destructive cleanup workflow. +`ChatGPT` is currently treated as a read-only desktop cache source. It remains available to the provider registry, but stays outside the default search scope and provider session action workflow. ## Capability Registry @@ -34,7 +34,7 @@ The primary search/session workflow currently covers `Codex`, `Claude`, `Gemini` ## Local Path Notes - `Codex` indexes session logs from `CODEX_HOME` plus Codex-managed archive, recovery, cwd-backup, and cleanup-backup roots; thread-id transcript lookup can also fall back to Codex home mirrors such as `~/.codex` and `~/.codex-cli`. -- `ChatGPT` reads the local desktop cache for the installed app; this provider remains read-only and stays outside the default search and cleanup flow. +- `ChatGPT` reads the local desktop cache for the installed app; this provider remains read-only and stays outside the default search and provider session action workflow. - `Claude` reads local session stores from dot-home roots such as `~/.claude`. - `Gemini` reads local session stores from dot-home roots such as `~/.gemini`. - `Copilot` resolves local app-data roots by platform: macOS `~/Library/Application Support`, Windows `%APPDATA%`, and Linux `XDG_CONFIG_HOME` or `~/.config`. @@ -51,7 +51,7 @@ The primary search/session workflow currently covers `Codex`, `Claude`, `Gemini` - Read-only desktop cache source. - Useful for desktop cache discovery and provider diagnostics. -- Excluded from the default search scope and destructive cleanup workflow. +- Excluded from the default search scope and provider session action workflow. ### Claude diff --git a/package.json b/package.json index a9c8aea..3694525 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "package:desktop:win": "pnpm build:desktop && pnpm --filter @threadlens/desktop-electron dist:win", "package:desktop:linux": "pnpm build:desktop && pnpm --filter @threadlens/desktop-electron dist:linux", "test": "pnpm -r test", - "lint": "pnpm -r lint", + "lint": "pnpm -r --filter @threadlens/shared-contracts --filter @threadlens/api --filter @threadlens/web --filter @threadlens/tui --filter @threadlens/desktop-electron lint && pnpm lint:deps", "lint:deps": "depcruise --config .dependency-cruiser.cjs apps/api-ts/src" }, "devDependencies": { diff --git a/packages/shared-contracts/provider-support/generate-provider-support.mjs b/packages/shared-contracts/provider-support/generate-provider-support.mjs index 46c1324..a5dda05 100644 --- a/packages/shared-contracts/provider-support/generate-provider-support.mjs +++ b/packages/shared-contracts/provider-support/generate-provider-support.mjs @@ -20,7 +20,7 @@ const PROVIDER_PATH_NOTES = { copilot: "resolves local app-data roots by platform: macOS `~/Library/Application Support`, Windows `%APPDATA%`, and Linux `XDG_CONFIG_HOME` or `~/.config`.", chatgpt: - "reads the local desktop cache for the installed app; this provider remains read-only and stays outside the default search and cleanup flow.", + "reads the local desktop cache for the installed app; this provider remains read-only and stays outside the default search and provider session action workflow.", }; const PROVIDER_WORKFLOW_NOTES = { @@ -47,7 +47,7 @@ const PROVIDER_WORKFLOW_NOTES = { chatgpt: [ "Read-only desktop cache source.", "Useful for desktop cache discovery and provider diagnostics.", - "Excluded from the default search scope and destructive cleanup workflow.", + "Excluded from the default search scope and provider session action workflow.", ], }; @@ -145,7 +145,7 @@ This document distinguishes between: - \`read-only cache sources\` that can still appear in diagnostics or provider-specific inspection The primary search/session workflow currently covers ${publicScope.map((label) => `\`${label}\``).join(", ")}. -${internalScope.length > 0 ? `${internalScope.map((label) => `\`${label}\``).join(", ")} is currently treated as a read-only desktop cache source. It remains available to the provider registry, but stays outside the default search scope and destructive cleanup workflow.` : ""} +${internalScope.length > 0 ? `${internalScope.map((label) => `\`${label}\``).join(", ")} is currently treated as a read-only desktop cache source. It remains available to the provider registry, but stays outside the default search scope and provider session action workflow.` : ""} ## Capability Registry