Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions src/__tests__/trajectory/trajectory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@
//
// 用临时目录构造 session jsonl, 跑 collect → export 三格式, 校验输出结构。

import { describe, expect, it, beforeEach, afterEach } from "bun:test";
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { promises as fs } from "node:fs";
import path from "node:path";
import os from "node:os";
import path from "node:path";
import {
collectTrajectories,
readManifest,
} from "../../services/trajectory/collector.js";
import {
buildDPOPairs,
exportTrajectories,
toSFTSample,
toGRPOSample,
buildDPOPairs,
toSFTSample,
} from "../../services/trajectory/exporters.js";
import type {
CollectedTrajectory,
Expand Down Expand Up @@ -265,6 +265,30 @@ describe("exportTrajectories", () => {
const rewards = out.map((l) => JSON.parse(l).reward as number).sort();
expect(rewards).toEqual([0, 1]);
});

// 回归 #58: sourceDir 指向 session 源目录 (无 manifest), destDir 指向汇聚库
// 应通过 destDir 回退定位 manifest, 不再返回 count=0
it("#58 export falls back to destDir when sourceDir has no manifest", async () => {
await seedTwoSessions();
const res = await exportTrajectories({
sourceDir, // session 源目录 (无 manifest)
destDir, // 汇聚库 (有 manifest)
format: "grpo",
});
expect(res.count).toBe(2); // 不再是 0
});

// 显式 storeDir 优先
it("export uses storeDir when provided", async () => {
await seedTwoSessions();
const res = await exportTrajectories({
sourceDir, // 无 manifest, 不应被使用
destDir, // 输出目录
storeDir: destDir, // 汇聚库
format: "sft",
});
expect(res.count).toBe(1);
});
});

describe("exporter unit transforms", () => {
Expand Down
4 changes: 2 additions & 2 deletions src/cli/handlers/trajectory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@

import {
collectTrajectories,
exportTrajectories,
readManifest,
DEFAULT_DEST_DIR,
DEFAULT_SOURCE_DIR,
exportTrajectories,
readManifest,
} from "../../services/trajectory/index.js";

interface ParsedFlags {
Expand Down
4 changes: 1 addition & 3 deletions src/entrypoints/cli.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,7 @@ async function main(): Promise<void> {
// 收集 session jsonl → 汇聚标注 → 导出 SFT/DPO/GRPO 训练集。
if (args[0] === "trajectory") {
profileCheckpoint("cli_trajectory_path");
const { trajectoryMain } = await import(
"../cli/handlers/trajectory.js"
);
const { trajectoryMain } = await import("../cli/handlers/trajectory.js");
await trajectoryMain(args.slice(1));
gracefulShutdownSync(0);
}
Expand Down
45 changes: 38 additions & 7 deletions src/services/trajectory/exporters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,22 +133,38 @@ export function buildDPOPairs(trajectories: CollectedTrajectory[]): DPOPair[] {
return pairs;
}

// 在候选目录中找到第一个含 manifest.json 的, 返回 {storeDir, manifest}; 都没有则 null
async function resolveStore(
candidates: string[],
): Promise<{
storeDir: string;
manifest: NonNullable<Awaited<ReturnType<typeof readManifest>>>;
} | null> {
for (const dir of candidates) {
if (!dir) continue;
const m = await readManifest(dir);
if (m) return { storeDir: dir, manifest: m };
}
return null;
}

// 加载所有汇聚轨迹 (或按 sessionId 过滤)
// storeDir = 汇聚库目录 (manifest.json + raw/ 所在)
export async function loadAll(
destDir: string,
storeDir: string,
sessionId?: string,
): Promise<CollectedTrajectory[]> {
const manifest = await readManifest(destDir);
const manifest = await readManifest(storeDir);
if (!manifest) {
log("no manifest at " + destDir + ", run collect first");
log("no manifest at " + storeDir + ", run collect first");
return [];
}
const out: CollectedTrajectory[] = [];
for (const entry of manifest.sessions) {
if (sessionId && entry.sessionId !== sessionId) continue;
const safeName = entry.sessionId.replace(/[^a-zA-Z0-9_-]/g, "_");
const rawFile = path.join(
destDir,
storeDir,
"raw",
entry.product + "-" + safeName + ".jsonl",
);
Expand All @@ -168,12 +184,27 @@ async function writeJsonl(filePath: string, records: unknown[]): Promise<void> {
}

// 主入口
// 汇聚库定位优先级: storeDir > sourceDir > destDir (取首个含 manifest 者)
export async function exportTrajectories(
options: ExportOptions,
): Promise<{ count: number; format: string; destFile: string }> {
const { sourceDir, destDir, format, sessionId } = options;
log("export format=" + format + " source=" + sourceDir + " dest=" + destDir);
const trajectories = await loadAll(sourceDir, sessionId);
const { sourceDir, destDir, format, sessionId, storeDir } = options;
const resolved = await resolveStore([storeDir, sourceDir, destDir]);
if (!resolved) {
log(
"no manifest found in any of storeDir/sourceDir/destDir " +
"(storeDir=" +
(storeDir ?? "-") +
", sourceDir=" +
sourceDir +
", destDir=" +
destDir +
"), run collect first",
);
}
const store = resolved?.storeDir ?? sourceDir ?? destDir;
log("export format=" + format + " store=" + store + " dest=" + destDir);
const trajectories = await loadAll(store, sessionId);
log("loaded " + trajectories.length + " trajectories");

let records: unknown[] = [];
Expand Down
30 changes: 15 additions & 15 deletions src/services/trajectory/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,33 @@

export {
collectTrajectories,
readManifest,
loadCollectedTrajectory,
DEFAULT_SOURCE_DIR,
DEFAULT_DEST_DIR,
DEFAULT_SOURCE_DIR,
loadCollectedTrajectory,
MANIFEST_VERSION,
readManifest,
} from "./collector.js";

export {
exportTrajectories,
toSFTSample,
toGRPOSample,
buildDPOPairs,
exportTrajectories,
loadAll,
toGRPOSample,
toSFTSample,
} from "./exporters.js";

export type {
CollectedTrajectory,
CollectOptions,
DPOPair,
ExportFormat,
ExportOptions,
GRPOSample,
ManifestEntry,
SFTSample,
ToolCall,
ToolResult,
TrajectoryStep,
TrajectoryLabel,
CollectedTrajectory,
ManifestEntry,
TrajectoryManifest,
ExportFormat,
SFTSample,
DPOPair,
GRPOSample,
CollectOptions,
ExportOptions,
TrajectoryStep,
} from "./types.js";
4 changes: 4 additions & 0 deletions src/services/trajectory/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,13 @@ export interface CollectOptions {
}

// 导出选项
// storeDir = 汇聚库目录 (manifest.json + raw/ 所在), 优先使用
// sourceDir = 兼容旧调用: 若 storeDir 未传, 作为汇聚库回退
// destDir = 输出目录 (sft/dpo/grpo.jsonl 写入处)
export interface ExportOptions {
sourceDir: string;
destDir: string;
format: ExportFormat;
sessionId?: string;
storeDir?: string;
}
Loading