diff --git a/README.md b/README.md index 72203cb..f9efb41 100644 --- a/README.md +++ b/README.md @@ -419,6 +419,20 @@ bun run ./scripts/build.ts --dev --feature-set=dev-full --feature=BRIDGE_MODE bun run dev # Run from source ``` +### Trajectory Flywheel (D1) + +Collect session trajectories and export training datasets (SFT/DPO/GRPO) for fusion-trainer: + +```bash +./fusion-code trajectory collect # 汇聚 session → ~/.fusion/trajectories +./fusion-code trajectory export --format sft # 导出 SFT 训练集 +./fusion-code trajectory export --format dpo # 导出 DPO 偏好对 +./fusion-code trajectory export --format grpo # 导出 GRPO reward 集 +./fusion-code trajectory manifest # 查看汇聚清单 +``` + +See [docs/trajectory-pipeline.md](docs/trajectory-pipeline.md) for details. + ### Permission Modes Press **Shift+Tab** to cycle modes: diff --git a/docs/trajectory-pipeline.md b/docs/trajectory-pipeline.md new file mode 100644 index 0000000..b397759 --- /dev/null +++ b/docs/trajectory-pipeline.md @@ -0,0 +1,122 @@ +# D1 轨迹飞轮 (Trajectory Flywheel) + +> Issues: #50 (统一汇聚目录 + is_error 标注) · #51 (三格式导出 CLI: SFT/DPO/GRPO) + +fusion-code 在每次会话中产生丰富的工具调用轨迹 (tool_use ↔ tool_result 配对, 含 `is_error` 信号)。 +D1 轨迹飞轮把这些散落的 session jsonl 清洗、汇聚、标注, 导出为 fusion-trainer 可消费的标准训练格式, +形成"使用 → 采集 → 训练 → 更强模型 → 更好使用"的数据飞轮。 + +## 数据流 + +``` +~/.fusion-code/projects//.jsonl (源: 每次会话产生) + │ collect + ▼ +~/.fusion/trajectories/ + ├── manifest.json (汇聚清单 + 统计) + └── raw/-.jsonl (清洗后的 TrajectoryStep 序列) + │ export --format + ▼ +~/.fusion/trajectories/{sft,dpo,grpo}.jsonl (训练集, fusion-trainer 消费) +``` + +## 标注规则 + +每条 session 轨迹按 `tool_result.is_error` 标注: + +- **positive** — 全程无任何 `is_error=true` 的 tool_result。理想成功轨迹, 进入 SFT。 +- **self_correction** — 至少一次 tool 失败后模型自我纠正。进入 DPO (失败作 rejected / 最终成功作 chosen) 与 GRPO (reward=0)。 + +## CLI 用法 + +`trajectory` 是 fusion-code 的顶层快速子命令 (无需进入 REPL): + +```bash +# 1. 收集: 扫描 session jsonl, 清洗配对, 汇聚标注 +fusion-code trajectory collect [--source DIR] [--dest DIR] [--product NAME] +# --source session jsonl 根目录, 默认 ~/.fusion-code/projects +# --dest 汇聚目录, 默认 ~/.fusion/trajectories +# --product 产品标记, 默认 fusion-code + +# 2. 导出: 从汇聚库导出训练集 +fusion-code trajectory export --format sft|dpo|grpo [--dest DIR] [--session ID] +# --format 必填: sft | dpo | grpo +# --dest 汇聚库目录 (collect 的 --dest), 输出也写入该目录 +# --session 仅导出指定 session + +# 3. 查看 +fusion-code trajectory manifest [--dest DIR] # 汇聚清单 + 逐 session 统计 +fusion-code trajectory list [--dest DIR] # 轻量列表 +``` + +## 输出格式 + +### SFT (ShareGPT messages-JSONL) + +仅 `positive` 轨迹。每行: + +```json +{ + "messages": [ + {"role": "system", "content": "You are a helpful coding assistant."}, + {"role": "user", "content": "<首轮 user prompt>"}, + {"role": "assistant", "content": "<整段 assistant 轨迹: thinking + tool_call + final>"} + ], + "source": "<原始 session jsonl 路径>" +} +``` + +### DPO (偏好对) + +仅 `self_correction` 轨迹。每行: + +```json +{ + "prompt": "<首轮 user prompt>", + "chosen": "<最终成功 answer (理想正确响应)>", + "rejected": "<含失败 tool_result 的整段 assistant 轨迹>", + "source": "<原始 session jsonl 路径>" +} +``` + +### GRPO (prompt + reward) + +全部轨迹。每行: + +```json +{ + "prompt": "<首轮 user prompt>", + "completion": "<整段 assistant 轨迹>", + "reward": 1, + "source": "<原始 session jsonl 路径>" +} +``` + +## 模块结构 + +``` +src/services/trajectory/ + types.ts 共享类型 (ToolCall/ToolResult/TrajectoryStep/CollectedTrajectory/...) + collector.ts #50 汇聚器: collectTrajectories / readManifest / loadCollectedTrajectory + exporters.ts #51 导出器: exportTrajectories / toSFTSample / toGRPOSample / buildDPOPairs + index.ts 统一出口 +src/cli/handlers/trajectory.ts CLI 处理器 (trajectoryMain) +src/entrypoints/cli.tsx 快速路径注册 (args[0] === "trajectory") +src/__tests__/trajectory/trajectory.test.ts 11 用例 +``` + +## 测试 + +```bash +bun test src/__tests__/trajectory/trajectory.test.ts +# 11 pass — collect 正例/自纠正标注, raw+manifest 落盘, 空会话跳过, +# SFT 仅正例, DPO 自纠正对, GRPO 全量 reward 0/1, 单元变换 +``` + +## 与 fusion-trainer 的衔接 + +导出的 `sft.jsonl` / `dpo.jsonl` / `grpo.jsonl` 落在 `~/.fusion/trajectories/`, +fusion-trainer 直接读取作为训练数据集: +- SFT → 监督微调 (教模型复现成功轨迹) +- DPO → 偏好优化 (惩罚失败轨迹, 强化正确回答) +- GRPO → 强化学习 (reward 信号驱动策略优化) diff --git a/src/__tests__/trajectory/trajectory.test.ts b/src/__tests__/trajectory/trajectory.test.ts new file mode 100644 index 0000000..1df178d --- /dev/null +++ b/src/__tests__/trajectory/trajectory.test.ts @@ -0,0 +1,338 @@ +// D1 轨迹飞轮 — 收集 + 导出 测试 (issue #50/#51) +// +// 用临时目录构造 session jsonl, 跑 collect → export 三格式, 校验输出结构。 + +import { describe, expect, it, beforeEach, afterEach } from "bun:test"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { + collectTrajectories, + readManifest, +} from "../../services/trajectory/collector.js"; +import { + exportTrajectories, + toSFTSample, + toGRPOSample, + buildDPOPairs, +} from "../../services/trajectory/exporters.js"; +import type { + CollectedTrajectory, + TrajectoryStep, +} from "../../services/trajectory/types.js"; + +let tmpRoot: string; +let sourceDir: string; +let destDir: string; + +// 构造一条 assistant 事件 (text + 可选 tool_use) +function assistantEvent( + text: string, + toolUse?: { id: string; name: string; input: unknown }, +) { + const content: unknown[] = []; + if (text) content.push({ type: "text", text }); + if (toolUse) content.push({ type: "tool_use", ...toolUse }); + return { type: "assistant", message: { role: "assistant", content } }; +} + +// 构造一条 user 事件 (纯文本 或 tool_result) +function userEvent( + text: string, + toolResult?: { toolUseId: string; isError: boolean; content: string }, +) { + if (toolResult) { + return { + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: toolResult.toolUseId, + is_error: toolResult.isError, + content: toolResult.content, + }, + ], + }, + }; + } + return { type: "user", message: { role: "user", content: text } }; +} + +function line(obj: unknown): string { + return JSON.stringify(obj); +} + +async function writeSession( + cwdSlug: string, + sessionId: string, + events: unknown[], +): Promise { + const dir = path.join(sourceDir, cwdSlug); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile( + path.join(dir, sessionId + ".jsonl"), + events.map((e) => line(e)).join("\n") + "\n", + "utf8", + ); +} + +beforeEach(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "traj-")); + sourceDir = path.join(tmpRoot, "projects"); + destDir = path.join(tmpRoot, "trajectories"); + await fs.mkdir(sourceDir, { recursive: true }); +}); + +afterEach(async () => { + await fs.rm(tmpRoot, { recursive: true, force: true }); +}); + +describe("collectTrajectories", () => { + it("labels error-free session as positive", async () => { + await writeSession("proj-a", "sess-ok", [ + userEvent("fix the bug in foo.ts"), + assistantEvent("I will read the file", { + id: "tu1", + name: "Read", + input: { file_path: "/p/foo.ts" }, + }), + userEvent("", { + toolUseId: "tu1", + isError: false, + content: "file contents here", + }), + assistantEvent("The fix is to add a null check."), + ]); + + const manifest = await collectTrajectories({ + sourceDir, + destDir, + product: "fusion-code", + }); + + expect(manifest.totals.sessions).toBe(1); + expect(manifest.totals.positive).toBe(1); + expect(manifest.totals.selfCorrection).toBe(0); + expect(manifest.totals.toolUse).toBe(1); + expect(manifest.totals.toolError).toBe(0); + expect(manifest.sessions[0].label).toBe("positive"); + }); + + it("labels session with tool_error as self_correction", async () => { + await writeSession("proj-b", "sess-err", [ + userEvent("run the tests"), + assistantEvent("running tests", { + id: "tu1", + name: "Bash", + input: { command: "pytest" }, + }), + userEvent("", { + toolUseId: "tu1", + isError: true, + content: "command not found: pytest", + }), + assistantEvent("pytest is not installed. I will use bun test."), + ]); + + const manifest = await collectTrajectories({ + sourceDir, + destDir, + }); + + expect(manifest.totals.sessions).toBe(1); + expect(manifest.totals.positive).toBe(0); + expect(manifest.totals.selfCorrection).toBe(1); + expect(manifest.totals.toolError).toBe(1); + expect(manifest.sessions[0].label).toBe("self_correction"); + }); + + it("writes raw jsonl and manifest to dest", async () => { + await writeSession("proj-c", "sess-1", [ + userEvent("hello"), + assistantEvent("hi there"), + ]); + + await collectTrajectories({ sourceDir, destDir }); + + const manifest = await readManifest(destDir); + expect(manifest).not.toBeNull(); + expect(manifest?.version).toBe(1); + const rawFile = path.join(destDir, "raw", "fusion-code-sess-1.jsonl"); + const raw = await fs.readFile(rawFile, "utf8"); + const steps = raw + .trim() + .split("\n") + .map((l) => JSON.parse(l) as TrajectoryStep); + expect(steps.length).toBe(2); + expect(steps[0].role).toBe("user"); + expect(steps[1].role).toBe("assistant"); + }); + + it("skips empty sessions", async () => { + await writeSession("proj-d", "sess-empty", [ + { type: "system", message: { role: "system", content: "init" } }, + ]); + const manifest = await collectTrajectories({ sourceDir, destDir }); + expect(manifest.totals.sessions).toBe(0); + }); +}); + +describe("exportTrajectories", () => { + async function seedTwoSessions(): Promise { + await writeSession("proj-a", "sess-ok", [ + userEvent("explain closures"), + assistantEvent("A closure captures variables.", { + id: "tu1", + name: "Read", + input: { file_path: "/p/x.ts" }, + }), + userEvent("", { + toolUseId: "tu1", + isError: false, + content: "ok", + }), + assistantEvent( + "Final answer: a closure is a function with captured scope.", + ), + ]); + await writeSession("proj-b", "sess-err", [ + userEvent("fix bug"), + assistantEvent("trying edit", { + id: "tu1", + name: "Edit", + input: { file_path: "/p/y.ts" }, + }), + userEvent("", { + toolUseId: "tu1", + isError: true, + content: "old_string not found", + }), + assistantEvent("Let me re-read and retry with correct old_string."), + ]); + await collectTrajectories({ sourceDir, destDir }); + } + + // export 的 sourceDir = 轨迹汇聚库 (collect 的 destDir), destDir = 输出目录 + it("SFT exports only positive trajectories as ShareGPT messages", async () => { + await seedTwoSessions(); + const res = await exportTrajectories({ + sourceDir: destDir, + destDir, + format: "sft", + }); + expect(res.format).toBe("sft"); + expect(res.count).toBe(1); // only the positive one + const out = (await fs.readFile(res.destFile, "utf8")).trim().split("\n"); + const sample = JSON.parse(out[0]); + expect(sample.messages).toBeDefined(); + expect(sample.messages.length).toBe(3); + expect(sample.messages[0].role).toBe("system"); + expect(sample.messages[1].role).toBe("user"); + expect(sample.messages[2].role).toBe("assistant"); + }); + + it("DPO exports self_correction pairs", async () => { + await seedTwoSessions(); + const res = await exportTrajectories({ + sourceDir: destDir, + destDir, + format: "dpo", + }); + expect(res.format).toBe("dpo"); + expect(res.count).toBe(1); // one self_correction session + const out = (await fs.readFile(res.destFile, "utf8")).trim().split("\n"); + const pair = JSON.parse(out[0]); + expect(pair.prompt).toBeDefined(); + expect(pair.chosen).toBeDefined(); + expect(pair.rejected).toBeDefined(); + // chosen is the final successful answer, rejected contains the failed tool + expect(pair.chosen).toContain("re-read"); + expect(pair.rejected).toContain("trying edit"); + }); + + it("GRPO exports every trajectory with reward 0/1", async () => { + await seedTwoSessions(); + const res = await exportTrajectories({ + sourceDir: destDir, + destDir, + format: "grpo", + }); + expect(res.format).toBe("grpo"); + expect(res.count).toBe(2); // all trajectories + const out = (await fs.readFile(res.destFile, "utf8")).trim().split("\n"); + const rewards = out.map((l) => JSON.parse(l).reward as number).sort(); + expect(rewards).toEqual([0, 1]); + }); +}); + +describe("exporter unit transforms", () => { + function mkTraj( + label: "positive" | "self_correction", + steps: TrajectoryStep[], + ): CollectedTrajectory { + return { + source: "/fake/sess.jsonl", + sessionId: "sess-x", + product: "fusion-code", + steps, + label, + toolUseCount: 0, + toolErrorCount: label === "self_correction" ? 1 : 0, + hasSubagents: false, + }; + } + + it("toSFTSample returns null for self_correction", () => { + const traj = mkTraj("self_correction", [ + { role: "user", text: "q" }, + { role: "assistant", text: "a" }, + ]); + expect(toSFTSample(traj)).toBeNull(); + }); + + it("toSFTSample returns messages for positive", () => { + const traj = mkTraj("positive", [ + { role: "user", text: "what is 1+1" }, + { role: "assistant", text: "2" }, + ]); + const s = toSFTSample(traj); + expect(s).not.toBeNull(); + expect(s?.messages[1].content).toBe("what is 1+1"); + expect(s?.messages[2].content).toBe("2"); + }); + + it("toGRPOSample reward matches label", () => { + const pos = mkTraj("positive", [ + { role: "user", text: "q" }, + { role: "assistant", text: "a" }, + ]); + const neg = mkTraj("self_correction", [ + { role: "user", text: "q" }, + { role: "assistant", text: "a" }, + ]); + expect(toGRPOSample(pos)?.reward).toBe(1); + expect(toGRPOSample(neg)?.reward).toBe(0); + }); + + it("buildDPOPairs only emits for self_correction", () => { + const pos = mkTraj("positive", [ + { role: "user", text: "q" }, + { role: "assistant", text: "good" }, + ]); + const sc = mkTraj("self_correction", [ + { role: "user", text: "q" }, + { role: "assistant", text: "bad attempt" }, + { + role: "user", + text: "", + toolResults: [{ toolUseId: "1", isError: true, content: "err" }], + }, + { role: "assistant", text: "correct final answer" }, + ]); + const pairs = buildDPOPairs([pos, sc]); + expect(pairs.length).toBe(1); + expect(pairs[0].chosen).toBe("correct final answer"); + }); +}); diff --git a/src/cli/handlers/trajectory.ts b/src/cli/handlers/trajectory.ts new file mode 100644 index 0000000..d8653bc --- /dev/null +++ b/src/cli/handlers/trajectory.ts @@ -0,0 +1,157 @@ +// D1 轨迹飞轮 — CLI 处理器 (issue #50/#51) +// +// 子命令: +// fusion-code trajectory collect [--source DIR] [--dest DIR] [--product NAME] +// fusion-code trajectory export --format sft|dpo|grpo [--source DIR] [--dest DIR] [--session ID] +// fusion-code trajectory manifest [--dest DIR] +// fusion-code trajectory list [--source DIR] +// +// 默认 source = ~/.fusion-code/projects, dest = ~/.fusion/trajectories + +import { + collectTrajectories, + exportTrajectories, + readManifest, + DEFAULT_DEST_DIR, + DEFAULT_SOURCE_DIR, +} from "../../services/trajectory/index.js"; + +interface ParsedFlags { + source: string; + dest: string; + product: string; + format: string; + session: string; + positional: string[]; +} + +function parseFlags(args: string[]): ParsedFlags { + const out: ParsedFlags = { + source: DEFAULT_SOURCE_DIR, + dest: DEFAULT_DEST_DIR, + product: "fusion-code", + format: "", + session: "", + positional: [], + }; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === "--source") out.source = args[++i] ?? ""; + else if (a === "--dest") out.dest = args[++i] ?? ""; + else if (a === "--product") out.product = args[++i] ?? ""; + else if (a === "--format") out.format = args[++i] ?? ""; + else if (a === "--session") out.session = args[++i] ?? ""; + else if (a) out.positional.push(a); + } + return out; +} + +function usage(): void { + console.log("Usage:"); + console.log( + " fusion-code trajectory collect [--source DIR] [--dest DIR] [--product NAME]", + ); + console.log( + " fusion-code trajectory export --format sft|dpo|grpo [--dest DIR] [--session ID]", + ); + console.log(" fusion-code trajectory manifest [--dest DIR]"); + console.log(" fusion-code trajectory list [--source DIR]"); +} + +export async function trajectoryMain(args: string[]): Promise { + const sub = args[0]?.toLowerCase(); + const flags = parseFlags(args.slice(1)); + + if (sub === "collect") { + const manifest = await collectTrajectories({ + sourceDir: flags.source, + destDir: flags.dest, + product: flags.product, + }); + console.log( + "collected " + + manifest.totals.sessions + + " sessions, " + + manifest.totals.positive + + " positive / " + + manifest.totals.selfCorrection + + " self_correction → " + + flags.dest, + ); + return; + } + + if (sub === "export") { + if (!flags.format) { + console.error("Error: --format sft|dpo|grpo is required"); + usage(); + process.exitCode = 1; + return; + } + if ( + flags.format !== "sft" && + flags.format !== "dpo" && + flags.format !== "grpo" + ) { + console.error( + "Error: format must be one of sft|dpo|grpo, got " + flags.format, + ); + process.exitCode = 1; + return; + } + // export 的输入是汇聚库 (collect 的 --dest), 输出 sft/dpo/grpo.jsonl 也写入该库 + const result = await exportTrajectories({ + sourceDir: flags.dest, + destDir: flags.dest, + format: flags.format, + sessionId: flags.session || undefined, + }); + console.log( + "exported " + + result.count + + " " + + result.format + + " samples → " + + result.destFile, + ); + return; + } + + if (sub === "manifest") { + const manifest = await readManifest(flags.dest); + if (!manifest) { + console.log("No manifest at " + flags.dest + ". Run `collect` first."); + return; + } + console.log(JSON.stringify(manifest.totals, null, 2)); + for (const s of manifest.sessions) { + console.log( + " " + + s.sessionId + + " label=" + + s.label + + " steps=" + + s.stepCount + + " tools=" + + s.toolUseCount + + " errors=" + + s.toolErrorCount, + ); + } + return; + } + + if (sub === "list") { + const manifest = await readManifest(flags.dest); + if (!manifest) { + console.log("No manifest at " + flags.dest + ". Run `collect` first."); + return; + } + for (const s of manifest.sessions) { + console.log(s.sessionId + " " + s.label + " " + s.stepCount + " steps"); + } + return; + } + + usage(); +} diff --git a/src/entrypoints/cli.tsx b/src/entrypoints/cli.tsx index b36d386..101d4fd 100644 --- a/src/entrypoints/cli.tsx +++ b/src/entrypoints/cli.tsx @@ -297,6 +297,17 @@ async function main(): Promise { return; } + // Fast-path for `fusion-code trajectory ...`: D1 轨迹飞轮 (issue #50/#51). + // 收集 session jsonl → 汇聚标注 → 导出 SFT/DPO/GRPO 训练集。 + if (args[0] === "trajectory") { + profileCheckpoint("cli_trajectory_path"); + const { trajectoryMain } = await import( + "../cli/handlers/trajectory.js" + ); + await trajectoryMain(args.slice(1)); + gracefulShutdownSync(0); + } + // Fast-path for template job commands. if ( feature("TEMPLATES") && diff --git a/src/services/trajectory/collector.ts b/src/services/trajectory/collector.ts new file mode 100644 index 0000000..e5c4742 --- /dev/null +++ b/src/services/trajectory/collector.ts @@ -0,0 +1,393 @@ +// D1 轨迹飞轮 — 汇聚器 (issue #50) +// +// 扫描 ~/.fusion-code/projects/**/*.jsonl + subagents/, 解析配对 tool_use/tool_result, +// 按 is_error 标注 Positive / SelfCorrection, 写入统一汇聚目录 ~/.fusion/trajectories/。 + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import type { + CollectedTrajectory, + CollectOptions, + ManifestEntry, + ToolCall, + ToolResult, + TrajectoryManifest, + TrajectoryStep, +} from "./types.js"; + +const log = (...args: unknown[]) => console.error("[trajectory]", ...args); + +// jsonl 原始事件的最小子集 +interface RawEvent { + type?: string; + message?: { + role?: string; + content?: unknown; + }; + cwd?: string; +} + +interface ContentBlock { + type?: string; + text?: string; + thinking?: string; + id?: string; + name?: string; + input?: unknown; + tool_use_id?: string; + is_error?: boolean; + content?: unknown; +} + +// 默认源目录: fusion-code session jsonl +export const DEFAULT_SOURCE_DIR = path.join( + process.env.HOME ?? "~", + ".fusion-code", + "projects", +); + +// 默认汇聚目录 +export const DEFAULT_DEST_DIR = path.join( + process.env.HOME ?? "~", + ".fusion", + "trajectories", +); + +export const MANIFEST_VERSION = 1; + +// 读取并解析单个 jsonl 文件为 RawEvent 列表 (跳过无法解析的行) +async function readJsonl(filePath: string): Promise { + const content = await fs.readFile(filePath, "utf8"); + const events: RawEvent[] = []; + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + events.push(JSON.parse(trimmed) as RawEvent); + } catch (e) { + log(`skip unparseable line in ${filePath}: ${(e as Error).message}`); + } + } + return events; +} + +// 把 content (string | block[]) 归一为 block[] +function toBlocks(content: unknown): ContentBlock[] { + if (typeof content === "string") { + return content ? [{ type: "text", text: content }] : []; + } + if (Array.isArray(content)) { + return content as ContentBlock[]; + } + return []; +} + +// block 内容转字符串 (tool_result.content 可能是 string 或 block[]) +function blockContentToString(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((b) => + typeof b === "string" + ? b + : ((b as ContentBlock)?.text ?? JSON.stringify(b)), + ) + .join(""); + } + return content == null ? "" : JSON.stringify(content); +} + +// 解析单 session 事件序列为清洗后的 steps + 统计 +function parseSession( + events: RawEvent[], + _source: string, + _sessionId: string, + _product: string, + _cwd?: string, +): { steps: TrajectoryStep[]; toolUseCount: number; toolErrorCount: number } { + const steps: TrajectoryStep[] = []; + let toolUseCount = 0; + let toolErrorCount = 0; + + for (const ev of events) { + if (ev.type !== "user" && ev.type !== "assistant") continue; + const role = ev.message?.role as "user" | "assistant" | undefined; + if (role !== "user" && role !== "assistant") continue; + + const blocks = toBlocks(ev.message?.content); + const toolCalls: ToolCall[] = []; + const toolResults: ToolResult[] = []; + let text = ""; + let thinking = ""; + + for (const b of blocks) { + const bt = b.type; + if (bt === "text" && b.text) { + text += b.text; + } else if (bt === "thinking" && b.thinking) { + thinking += b.thinking; + } else if (bt === "tool_use" && b.id && b.name) { + toolCalls.push({ id: b.id, name: b.name, input: b.input }); + toolUseCount++; + } else if (bt === "tool_result" && b.tool_use_id) { + const isError = b.is_error === true; + toolResults.push({ + toolUseId: b.tool_use_id, + isError, + content: blockContentToString(b.content), + }); + if (isError) toolErrorCount++; + } + } + + // 跳过空步骤 (无文本、无工具调用、无工具结果) + if ( + !text && + !thinking && + toolCalls.length === 0 && + toolResults.length === 0 + ) { + continue; + } + + const step: TrajectoryStep = { role, text }; + if (thinking) step.thinking = thinking; + if (toolCalls.length) step.toolCalls = toolCalls; + if (toolResults.length) step.toolResults = toolResults; + steps.push(step); + } + + return { steps, toolUseCount, toolErrorCount }; +} + +// 标注: 全程无 tool_result.is_error → positive; 有任一 error → self_correction +function labelTrajectory( + toolErrorCount: number, +): "positive" | "self_correction" { + return toolErrorCount === 0 ? "positive" : "self_correction"; +} + +// 枚举所有 session jsonl (含 subagents) +async function listSessionFiles( + sourceDir: string, +): Promise< + { file: string; sessionId: string; product: string; isSubagent: boolean }[] +> { + const out: { + file: string; + sessionId: string; + product: string; + isSubagent: boolean; + }[] = []; + let topEntries: string[] = []; + try { + topEntries = await fs.readdir(sourceDir); + } catch (e) { + log(`source dir not readable: ${sourceDir}: ${(e as Error).message}`); + return out; + } + + for (const entry of topEntries) { + const cwdDir = path.join(sourceDir, entry); + let stat: Awaited>; + try { + stat = await fs.stat(cwdDir); + } catch { + continue; + } + if (!stat.isDirectory()) continue; + let files: string[] = []; + try { + files = await fs.readdir(cwdDir); + } catch { + continue; + } + for (const f of files) { + if (!f.endsWith(".jsonl")) continue; + const sessionId = f.replace(/\.jsonl$/, ""); + out.push({ + file: path.join(cwdDir, f), + sessionId, + product: "fusion-code", + isSubagent: false, + }); + } + // subagents: //subagents/agent-*.jsonl + let subDirs: string[] = []; + try { + subDirs = await fs.readdir(cwdDir); + } catch { + subDirs = []; + } + for (const sd of subDirs) { + const subPath = path.join(cwdDir, sd); + let sstat: Awaited>; + try { + sstat = await fs.stat(subPath); + } catch { + continue; + } + if (!sstat.isDirectory()) continue; + const subAgentsDir = path.join(subPath, "subagents"); + let agentFiles: string[] = []; + try { + agentFiles = await fs.readdir(subAgentsDir); + } catch { + continue; + } + for (const af of agentFiles) { + if (!af.endsWith(".jsonl")) continue; + out.push({ + file: path.join(subAgentsDir, af), + sessionId: `${sd}::${af.replace(/\.jsonl$/, "")}`, + product: "fusion-code", + isSubagent: true, + }); + } + } + } + return out; +} + +// 写单个汇聚轨迹到 dest/raw/-.jsonl +async function writeTrajectory( + destDir: string, + traj: CollectedTrajectory, +): Promise { + const rawDir = path.join(destDir, "raw"); + await fs.mkdir(rawDir, { recursive: true }); + const safeName = traj.sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); + const outFile = path.join(rawDir, `${traj.product}-${safeName}.jsonl`); + const lines = traj.steps.map((s) => JSON.stringify(s)).join("\n"); + await fs.writeFile(outFile, lines + (lines ? "\n" : ""), "utf8"); +} + +// 主入口: 收集并汇聚 +export async function collectTrajectories( + options: CollectOptions, +): Promise { + const { sourceDir, destDir } = options; + const product = options.product ?? "fusion-code"; + log(`scanning source=${sourceDir} dest=${destDir} product=${product}`); + + const files = await listSessionFiles(sourceDir); + log(`found ${files.length} session files (incl subagents)`); + + const sessions: ManifestEntry[] = []; + let totalSteps = 0; + let totalToolUse = 0; + let totalToolError = 0; + let positive = 0; + let selfCorrection = 0; + + for (const f of files) { + let events: RawEvent[] = []; + try { + events = await readJsonl(f.file); + } catch (e) { + log(`skip unreadable ${f.file}: ${(e as Error).message}`); + continue; + } + const { steps, toolUseCount, toolErrorCount } = parseSession( + events, + f.file, + f.sessionId, + product, + ); + if (steps.length === 0) continue; + + const label = labelTrajectory(toolErrorCount); + const traj: CollectedTrajectory = { + source: f.file, + sessionId: f.sessionId, + product: f.product, + steps, + label, + toolUseCount, + toolErrorCount, + hasSubagents: f.isSubagent, + }; + await writeTrajectory(destDir, traj); + + sessions.push({ + source: f.file, + sessionId: f.sessionId, + product: f.product, + label, + toolUseCount, + toolErrorCount, + stepCount: steps.length, + hasSubagents: f.isSubagent, + }); + totalSteps += steps.length; + totalToolUse += toolUseCount; + totalToolError += toolErrorCount; + if (label === "positive") positive++; + else selfCorrection++; + } + + const manifest: TrajectoryManifest = { + version: MANIFEST_VERSION, + generatedAt: new Date().toISOString(), + destDir, + sessions, + totals: { + sessions: sessions.length, + steps: totalSteps, + toolUse: totalToolUse, + toolError: totalToolError, + positive, + selfCorrection, + }, + }; + + await fs.mkdir(destDir, { recursive: true }); + const manifestPath = path.join(destDir, "manifest.json"); + await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8"); + + log( + `collected ${sessions.length} sessions, ${totalSteps} steps, ` + + `${totalToolUse} tool pairs (${totalToolError} errors), ` + + `${positive} positive / ${selfCorrection} self_correction → ${destDir}`, + ); + return manifest; +} + +// 读取已有 manifest (exporter 复用) +export async function readManifest( + destDir: string, +): Promise { + const manifestPath = path.join(destDir, "manifest.json"); + try { + const raw = await fs.readFile(manifestPath, "utf8"); + return JSON.parse(raw) as TrajectoryManifest; + } catch { + return null; + } +} + +// 从 raw/ 目录读回单条汇聚轨迹 (exporter 消费) +export async function loadCollectedTrajectory( + rawFile: string, + entry: ManifestEntry, +): Promise { + const content = await fs.readFile(rawFile, "utf8"); + const steps: TrajectoryStep[] = []; + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + steps.push(JSON.parse(trimmed) as TrajectoryStep); + } catch {} + } + return { + source: entry.source, + sessionId: entry.sessionId, + product: entry.product, + steps, + label: entry.label, + toolUseCount: entry.toolUseCount, + toolErrorCount: entry.toolErrorCount, + hasSubagents: entry.hasSubagents, + }; +} diff --git a/src/services/trajectory/exporters.ts b/src/services/trajectory/exporters.ts new file mode 100644 index 0000000..8bd4024 --- /dev/null +++ b/src/services/trajectory/exporters.ts @@ -0,0 +1,199 @@ +// D1 轨迹飞轮 — 三格式导出 (issue #51) +// +// 消费汇聚后的轨迹, 输出训练可用的标准格式 (fusion-trainer 消费): +// SFT — ShareGPT messages-JSONL, 仅 is_error=false 成功轨迹 +// DPO — 偏好对 {prompt, chosen, rejected}, 失败作 rejected / 成功作 chosen +// GRPO — {prompt, completion, reward}, reward = is_error ? 0 : 1 + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { loadCollectedTrajectory, readManifest } from "./collector.js"; +import type { + CollectedTrajectory, + DPOPair, + ExportOptions, + GRPOSample, + SFTSample, + TrajectoryStep, +} from "./types.js"; + +const log = (...args: unknown[]) => console.error("[trajectory]", ...args); + +// 闭合标签拆分构造, 避免源码中出现裸闭合标签序列 +const CLOSE_THINKING = "<" + "/thinking>"; +const CLOSE_TOOL_CALL = "<" + "/tool_call>"; +const CLOSE_TOOL_RESULT = "<" + "/tool_result>"; + +// 把单步轨迹渲染为对话文本 +function stepToText(step: TrajectoryStep): string { + const parts: string[] = []; + if (step.thinking) parts.push("" + step.thinking + CLOSE_THINKING); + if (step.toolCalls?.length) { + for (const tc of step.toolCalls) { + parts.push( + '' + + JSON.stringify(tc.input) + + CLOSE_TOOL_CALL, + ); + } + } + if (step.toolResults?.length) { + for (const tr of step.toolResults) { + parts.push( + '' + + tr.content + + CLOSE_TOOL_RESULT, + ); + } + } + if (step.text) parts.push(step.text); + return parts.join("\n"); +} + +// 提取首轮 user prompt (跳过纯 tool_result 的 user 步骤) +function extractPrompt(traj: CollectedTrajectory): string { + for (const s of traj.steps) { + if (s.role === "user" && s.text && !s.toolResults?.length) { + return s.text; + } + } + const firstUser = traj.steps.find((s) => s.role === "user"); + return firstUser?.text ?? ""; +} + +// 提取 assistant 最终回答文本 (最后一条含 text 的 assistant 步骤) +function extractFinalAnswer(traj: CollectedTrajectory): string { + for (let i = traj.steps.length - 1; i >= 0; i--) { + const s = traj.steps[i]; + if (s.role === "assistant" && s.text) return s.text; + } + return ""; +} + +// 渲染整条 assistant 轨迹 (含 thought/tool_call/tool_response/final) +function renderAssistantTurn(traj: CollectedTrajectory): string { + const parts: string[] = []; + for (const s of traj.steps) { + if (s.role !== "assistant") continue; + const t = stepToText(s); + if (t) parts.push(t); + } + return parts.join("\n"); +} + +// SFT: 仅 positive 轨迹, 输出 ShareGPT messages +export function toSFTSample(traj: CollectedTrajectory): SFTSample | null { + if (traj.label !== "positive") return null; + const prompt = extractPrompt(traj); + const answer = renderAssistantTurn(traj); + if (!prompt || !answer) return null; + return { + messages: [ + { role: "system", content: "You are a helpful coding assistant." }, + { role: "user", content: prompt }, + { role: "assistant", content: answer }, + ], + source: traj.source, + }; +} + +// GRPO: 每条轨迹一个样本, reward = is_error ? 0 : 1 +export function toGRPOSample(traj: CollectedTrajectory): GRPOSample | null { + const prompt = extractPrompt(traj); + const completion = renderAssistantTurn(traj); + if (!prompt || !completion) return null; + return { + prompt, + completion, + reward: traj.label === "positive" ? 1 : 0, + source: traj.source, + }; +} + +// DPO: 自纠正候选成对 (失败渲染作 rejected / 最终成功 answer 作 chosen) +export function buildDPOPairs(trajectories: CollectedTrajectory[]): DPOPair[] { + const pairs: DPOPair[] = []; + for (const traj of trajectories) { + if (traj.label !== "self_correction") continue; + const prompt = extractPrompt(traj); + const finalAnswer = extractFinalAnswer(traj); + if (!prompt || !finalAnswer) continue; + const rejected = renderAssistantTurn(traj); + pairs.push({ + prompt, + chosen: finalAnswer, + rejected, + source: traj.source, + }); + } + return pairs; +} + +// 加载所有汇聚轨迹 (或按 sessionId 过滤) +export async function loadAll( + destDir: string, + sessionId?: string, +): Promise { + const manifest = await readManifest(destDir); + if (!manifest) { + log("no manifest at " + destDir + ", run collect first"); + return []; + } + const out: CollectedTrajectory[] = []; + for (const entry of manifest.sessions) { + if (sessionId && entry.sessionId !== sessionId) continue; + const safeName = entry.sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); + const rawFile = path.join( + destDir, + "raw", + entry.product + "-" + safeName + ".jsonl", + ); + try { + out.push(await loadCollectedTrajectory(rawFile, entry)); + } catch (e) { + log("skip missing raw " + rawFile + ": " + (e as Error).message); + } + } + return out; +} + +async function writeJsonl(filePath: string, records: unknown[]): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const lines = records.map((r) => JSON.stringify(r)).join("\n"); + await fs.writeFile(filePath, lines + (lines ? "\n" : ""), "utf8"); +} + +// 主入口 +export async function exportTrajectories( + options: ExportOptions, +): Promise<{ count: number; format: string; destFile: string }> { + const { sourceDir, destDir, format, sessionId } = options; + log("export format=" + format + " source=" + sourceDir + " dest=" + destDir); + const trajectories = await loadAll(sourceDir, sessionId); + log("loaded " + trajectories.length + " trajectories"); + + let records: unknown[] = []; + let outFile: string; + if (format === "sft") { + records = trajectories + .map(toSFTSample) + .filter((s): s is SFTSample => s !== null); + outFile = path.join(destDir, "sft.jsonl"); + } else if (format === "dpo") { + records = buildDPOPairs(trajectories); + outFile = path.join(destDir, "dpo.jsonl"); + } else { + records = trajectories + .map(toGRPOSample) + .filter((s): s is GRPOSample => s !== null); + outFile = path.join(destDir, "grpo.jsonl"); + } + + await writeJsonl(outFile, records); + log("exported " + records.length + " " + format + " samples -> " + outFile); + return { count: records.length, format, destFile: outFile }; +} diff --git a/src/services/trajectory/index.ts b/src/services/trajectory/index.ts new file mode 100644 index 0000000..af2ecac --- /dev/null +++ b/src/services/trajectory/index.ts @@ -0,0 +1,34 @@ +// D1 轨迹飞轮 — 模块统一出口 (issue #50/#51) + +export { + collectTrajectories, + readManifest, + loadCollectedTrajectory, + DEFAULT_SOURCE_DIR, + DEFAULT_DEST_DIR, + MANIFEST_VERSION, +} from "./collector.js"; + +export { + exportTrajectories, + toSFTSample, + toGRPOSample, + buildDPOPairs, + loadAll, +} from "./exporters.js"; + +export type { + ToolCall, + ToolResult, + TrajectoryStep, + TrajectoryLabel, + CollectedTrajectory, + ManifestEntry, + TrajectoryManifest, + ExportFormat, + SFTSample, + DPOPair, + GRPOSample, + CollectOptions, + ExportOptions, +} from "./types.js"; diff --git a/src/services/trajectory/types.ts b/src/services/trajectory/types.ts new file mode 100644 index 0000000..823fea8 --- /dev/null +++ b/src/services/trajectory/types.ts @@ -0,0 +1,114 @@ +// D1 轨迹飞轮 — 共享类型 (issue #50/#51) +// +// 数据源: ~/.fusion-code/projects//.jsonl +// 每行一条事件, type ∈ {user, assistant, queue-operation, attachment, ...} +// assistant.message.content = block[]: text | tool_use | thinking +// user.message.content = block[]: tool_result (is_error 天然存在) | str +// 配对: assistant.tool_use.id ↔ user.tool_result.tool_use_id + +// 单条 tool_use (来自 assistant) +export interface ToolCall { + id: string; + name: string; + input: unknown; +} + +// 单条 tool_result (来自 user, 与 ToolCall 按 id 配对) +export interface ToolResult { + toolUseId: string; + isError: boolean; + content: string; +} + +// 清洗后的单步轨迹: 一轮 user→assistant→(tool 循环)→final +export interface TrajectoryStep { + role: "user" | "assistant"; + text: string; + thinking?: string; + toolCalls?: ToolCall[]; + toolResults?: ToolResult[]; +} + +// 标注类别: 成功正例 / 自纠正候选 +export type TrajectoryLabel = "positive" | "self_correction"; + +// 汇聚后的单 session 轨迹 +export interface CollectedTrajectory { + source: string; + sessionId: string; + product: string; + cwd?: string; + steps: TrajectoryStep[]; + label: TrajectoryLabel; + toolUseCount: number; + toolErrorCount: number; + hasSubagents: boolean; +} + +// manifest 单条记录 +export interface ManifestEntry { + source: string; + sessionId: string; + product: string; + label: TrajectoryLabel; + toolUseCount: number; + toolErrorCount: number; + stepCount: number; + hasSubagents: boolean; +} + +// manifest 文件结构 +export interface TrajectoryManifest { + version: number; + generatedAt: string; + destDir: string; + sessions: ManifestEntry[]; + totals: { + sessions: number; + steps: number; + toolUse: number; + toolError: number; + positive: number; + selfCorrection: number; + }; +} + +// 导出格式 +export type ExportFormat = "sft" | "dpo" | "grpo"; + +// SFT 样本 (ShareGPT messages-JSONL) +export interface SFTSample { + messages: { role: "system" | "user" | "assistant"; content: string }[]; + source: string; +} + +// DPO 偏好对 +export interface DPOPair { + prompt: string; + chosen: string; + rejected: string; + source: string; +} + +// GRPO prompt + reward 信号 +export interface GRPOSample { + prompt: string; + completion: string; + reward: number; + source: string; +} + +// 收集选项 +export interface CollectOptions { + sourceDir: string; + destDir: string; + product?: string; +} + +// 导出选项 +export interface ExportOptions { + sourceDir: string; + destDir: string; + format: ExportFormat; + sessionId?: string; +}