diff --git a/src/__tests__/trajectory/trajectory.test.ts b/src/__tests__/trajectory/trajectory.test.ts index 1df178d..c4f953e 100644 --- a/src/__tests__/trajectory/trajectory.test.ts +++ b/src/__tests__/trajectory/trajectory.test.ts @@ -2,19 +2,19 @@ // // 用临时目录构造 session jsonl, 跑 collect → export 三格式, 校验输出结构。 -import { describe, expect, it, beforeEach, afterEach } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { promises as fs } from "node:fs"; -import path from "node:path"; import os from "node:os"; +import path from "node:path"; import { collectTrajectories, readManifest, } from "../../services/trajectory/collector.js"; import { + buildDPOPairs, exportTrajectories, - toSFTSample, toGRPOSample, - buildDPOPairs, + toSFTSample, } from "../../services/trajectory/exporters.js"; import type { CollectedTrajectory, @@ -265,6 +265,30 @@ describe("exportTrajectories", () => { const rewards = out.map((l) => JSON.parse(l).reward as number).sort(); expect(rewards).toEqual([0, 1]); }); + + // 回归 #58: sourceDir 指向 session 源目录 (无 manifest), destDir 指向汇聚库 + // 应通过 destDir 回退定位 manifest, 不再返回 count=0 + it("#58 export falls back to destDir when sourceDir has no manifest", async () => { + await seedTwoSessions(); + const res = await exportTrajectories({ + sourceDir, // session 源目录 (无 manifest) + destDir, // 汇聚库 (有 manifest) + format: "grpo", + }); + expect(res.count).toBe(2); // 不再是 0 + }); + + // 显式 storeDir 优先 + it("export uses storeDir when provided", async () => { + await seedTwoSessions(); + const res = await exportTrajectories({ + sourceDir, // 无 manifest, 不应被使用 + destDir, // 输出目录 + storeDir: destDir, // 汇聚库 + format: "sft", + }); + expect(res.count).toBe(1); + }); }); describe("exporter unit transforms", () => { diff --git a/src/cli/handlers/trajectory.ts b/src/cli/handlers/trajectory.ts index d8653bc..d953c42 100644 --- a/src/cli/handlers/trajectory.ts +++ b/src/cli/handlers/trajectory.ts @@ -10,10 +10,10 @@ import { collectTrajectories, - exportTrajectories, - readManifest, DEFAULT_DEST_DIR, DEFAULT_SOURCE_DIR, + exportTrajectories, + readManifest, } from "../../services/trajectory/index.js"; interface ParsedFlags { diff --git a/src/entrypoints/cli.tsx b/src/entrypoints/cli.tsx index 101d4fd..f0423db 100644 --- a/src/entrypoints/cli.tsx +++ b/src/entrypoints/cli.tsx @@ -301,9 +301,7 @@ async function main(): Promise { // 收集 session jsonl → 汇聚标注 → 导出 SFT/DPO/GRPO 训练集。 if (args[0] === "trajectory") { profileCheckpoint("cli_trajectory_path"); - const { trajectoryMain } = await import( - "../cli/handlers/trajectory.js" - ); + const { trajectoryMain } = await import("../cli/handlers/trajectory.js"); await trajectoryMain(args.slice(1)); gracefulShutdownSync(0); } diff --git a/src/services/trajectory/exporters.ts b/src/services/trajectory/exporters.ts index 8bd4024..90e21db 100644 --- a/src/services/trajectory/exporters.ts +++ b/src/services/trajectory/exporters.ts @@ -133,14 +133,30 @@ export function buildDPOPairs(trajectories: CollectedTrajectory[]): DPOPair[] { return pairs; } +// 在候选目录中找到第一个含 manifest.json 的, 返回 {storeDir, manifest}; 都没有则 null +async function resolveStore( + candidates: string[], +): Promise<{ + storeDir: string; + manifest: NonNullable>>; +} | null> { + for (const dir of candidates) { + if (!dir) continue; + const m = await readManifest(dir); + if (m) return { storeDir: dir, manifest: m }; + } + return null; +} + // 加载所有汇聚轨迹 (或按 sessionId 过滤) +// storeDir = 汇聚库目录 (manifest.json + raw/ 所在) export async function loadAll( - destDir: string, + storeDir: string, sessionId?: string, ): Promise { - const manifest = await readManifest(destDir); + const manifest = await readManifest(storeDir); if (!manifest) { - log("no manifest at " + destDir + ", run collect first"); + log("no manifest at " + storeDir + ", run collect first"); return []; } const out: CollectedTrajectory[] = []; @@ -148,7 +164,7 @@ export async function loadAll( if (sessionId && entry.sessionId !== sessionId) continue; const safeName = entry.sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); const rawFile = path.join( - destDir, + storeDir, "raw", entry.product + "-" + safeName + ".jsonl", ); @@ -168,12 +184,27 @@ async function writeJsonl(filePath: string, records: unknown[]): Promise { } // 主入口 +// 汇聚库定位优先级: storeDir > sourceDir > destDir (取首个含 manifest 者) export async function exportTrajectories( options: ExportOptions, ): Promise<{ count: number; format: string; destFile: string }> { - const { sourceDir, destDir, format, sessionId } = options; - log("export format=" + format + " source=" + sourceDir + " dest=" + destDir); - const trajectories = await loadAll(sourceDir, sessionId); + const { sourceDir, destDir, format, sessionId, storeDir } = options; + const resolved = await resolveStore([storeDir, sourceDir, destDir]); + if (!resolved) { + log( + "no manifest found in any of storeDir/sourceDir/destDir " + + "(storeDir=" + + (storeDir ?? "-") + + ", sourceDir=" + + sourceDir + + ", destDir=" + + destDir + + "), run collect first", + ); + } + const store = resolved?.storeDir ?? sourceDir ?? destDir; + log("export format=" + format + " store=" + store + " dest=" + destDir); + const trajectories = await loadAll(store, sessionId); log("loaded " + trajectories.length + " trajectories"); let records: unknown[] = []; diff --git a/src/services/trajectory/index.ts b/src/services/trajectory/index.ts index af2ecac..c19e6ee 100644 --- a/src/services/trajectory/index.ts +++ b/src/services/trajectory/index.ts @@ -2,33 +2,33 @@ export { collectTrajectories, - readManifest, - loadCollectedTrajectory, - DEFAULT_SOURCE_DIR, DEFAULT_DEST_DIR, + DEFAULT_SOURCE_DIR, + loadCollectedTrajectory, MANIFEST_VERSION, + readManifest, } from "./collector.js"; export { - exportTrajectories, - toSFTSample, - toGRPOSample, buildDPOPairs, + exportTrajectories, loadAll, + toGRPOSample, + toSFTSample, } from "./exporters.js"; export type { + CollectedTrajectory, + CollectOptions, + DPOPair, + ExportFormat, + ExportOptions, + GRPOSample, + ManifestEntry, + SFTSample, ToolCall, ToolResult, - TrajectoryStep, TrajectoryLabel, - CollectedTrajectory, - ManifestEntry, TrajectoryManifest, - ExportFormat, - SFTSample, - DPOPair, - GRPOSample, - CollectOptions, - ExportOptions, + TrajectoryStep, } from "./types.js"; diff --git a/src/services/trajectory/types.ts b/src/services/trajectory/types.ts index 823fea8..33b3d2f 100644 --- a/src/services/trajectory/types.ts +++ b/src/services/trajectory/types.ts @@ -106,9 +106,13 @@ export interface CollectOptions { } // 导出选项 +// storeDir = 汇聚库目录 (manifest.json + raw/ 所在), 优先使用 +// sourceDir = 兼容旧调用: 若 storeDir 未传, 作为汇聚库回退 +// destDir = 输出目录 (sft/dpo/grpo.jsonl 写入处) export interface ExportOptions { sourceDir: string; destDir: string; format: ExportFormat; sessionId?: string; + storeDir?: string; }