diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..679b164 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto eol=lf +*.ps1 text eol=crlf +*.png binary +*.pdf binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb0edd..d7e8201 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[dev,llm]" + python -m pip install -e ".[dev,llm,gui]" - name: Run test suite run: python -m pytest diff --git a/P0_CHANGES.zh-CN.md b/P0_CHANGES.zh-CN.md new file mode 100644 index 0000000..d3b37db --- /dev/null +++ b/P0_CHANGES.zh-CN.md @@ -0,0 +1,86 @@ +# P0 修改说明（2026-10-06，2026-10-07 复核修订） + +本包基于官方最新版 `SlideNote-main`（GitHub commit `3fca464`）制作，包含两部分工作： +移植旧版本里未同步的本地改进 + 落实 ROADMAP 的 P0 三项。全部改动已通过验证： +**354 个测试全绿（基线 337 + 新增 17）、无 API 冒烟通过、评测与排版工具实测跑通。** + +## 一、从旧版本移植的改动（微信包里未提交的本地改进） + +| 文件 | 改动 | +| --- | --- | +| `slidenote/notes/structure.py`（新） | 讲义结构契约评估：必备栏目（本讲目标/主题章节/本讲总结/章节自测）+ 推荐栏目（例子/易错点），检测"第 N 页"式逐页标题和重复模板栏目 | +| `slidenote/notes/document_frame.py`（新） | 确定性文档框架：从 deck brief 提取内容补齐缺失的全局栏目，幂等、不改章节正文、不新增课件外事实 | +| `slidenote/notes/prompt_templates.py` | 单页生成改为"中间证据卡"定位；weave/教学增强按上下文选择全文结构契约或自适应章节规则；收紧事实边界（禁止新增无依据数字/结论）；新增结构修复 prompt | +| `slidenote/notes/repair.py` | 新增 `_repair_note_structure_once` 并接线到 finalize。复核版要求修订后契约通过、正文至少保留 80%、精确保留来源标记/图片目标、coverage 不回退且输出未截断；任一不满足即保留原稿 | +| `slidenote/notes/finalize.py` | lecture-weave 收尾时应用文档框架；契约仍不通过时触发一次结构修复（带守护） | +| `slidenote/notes/quality.py` | 质量报告新增 `structure_contract`、`human_note_structure_score`、`mechanical_structure_pass` 等字段与对应改进建议 | +| `slidenote/notes/versions.py` | prompt 版本号升级（page-lecture-v5 / weave-v5 / teaching-enrichment-v3 / structure-repair-v2），旧缓存自动失效 | +| `slidenote/notes/usage.py` | 用量报告增加 `repair_calls` 计数 | +| `slidenote/build/config.py`、`notes/__init__.py` | 默认 `temperature` 0.2→0.0（可复现评测）、`weave_dedup` soft→normal（配合新去重文案） | +| `tests/test_notes.py` | 断言适配新 prompt 文案 | +| `tests/test_note_structure.py`（新） | 结构契约/文档框架/结构修复守护/质量报告的 9 个测试，包含假阳性和严重截断回归测试 | + +## 二、P0 三项落地（ROADMAP） + +### P0-1 真实课件评测与内容核对 + +- `benchmarks/README.md`：样本集要求、评测流程、硬门槛与启发式/人工复核的边界。 +- `benchmarks/samples.manifest.json`：固定样本清单（改动前后必须用同一份）。 +- `benchmarks/rubric.template.json`：8 维度人工评分 rubric（1-5 分，含锚点）。 +- `benchmarks/spot-check.template.md`：人工抽查模板（关键事实 10 条/漏项/图文对应/来源定位 10 处/耗时成本）。 +- `scripts/make_sample_deck.py`：生成固定合成样例（文字密集页、表格、公式、折线图、扫描页、18 页），可重复构建、可提交仓库。 +- `scripts/eval_decks.py`：评测 harness——按 manifest 逐例运行 build，记录耗时/阶段耗时/tokens/成本估算/coverage/启发式分数；硬门槛失败自动把 notes、coverage、质量报告留档到 `runs//failures//`；`--baseline` 与上次运行逐项对比。 + +用法： + +```powershell +python scripts/make_sample_deck.py +python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/baseline +# 改动后： +python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/after --baseline benchmarks/runs/baseline +``` + +### P0-2 笔记阅读与导出排版验收 + +- `scripts/verify_note_layout.py`：三层检查，每个问题定位到样本/文件/页码。 + - Markdown：标题跳级、多个 H1、空章节、图片无 alt/无邻近解释、超宽表格、行列不齐、代码围栏/公式未闭合、缺来源标记。 + - Word（pandoc 导出）：标题/表格/图片数量与 Markdown 对比、表格行列错位。 + - PDF：文本溢出页边界、图片与文字重叠（图文错位）、空白页、页尾孤立标题。 + - 缺 pandoc/LibreOffice 时对应层显示 skipped 并注明原因。 + +```powershell +python scripts/verify_note_layout.py outputs\你的输出目录 +python scripts/verify_note_layout.py benchmarks\runs\after\outputs +``` + +### P0-3 来源与质量提示说清边界 + +- GUI Quality 标签页：新增三层信号说明（coverage=结构线索、quality_report=启发式、人工复核=最终结论），新增"建议人工复核"清单（合并 coverage 缺项、图片缺失、幻觉风险、结构契约失败，每行注明信号来源和查看位置）。 +- `docs/quality-and-guard.zh-CN.md`：三层质量信号对照表（各自能回答/不能回答什么）、结构契约字段说明。 +- `docs/benchmark.zh-CN.md`：新工具用法。 +- `ROADMAP.zh-CN.md`：标注 P0 工具已就绪，真实课件样本与人工评分待收集。 + +## 三、验证结论 + +- `python -m pytest`：354 passed（完整安装 dev/llm/gui，0 skipped）。 +- `python scripts/smoke_first_run.py`：通过。 +- 评测 harness 实测：合成样例约 0.9s，仓库外绝对 `--out` 与 `--baseline` 正常；报告会明确显示 local 产物的逐页标题，但不把它作为 lecture 硬门槛。 +- 排版验收：Markdown/docx/PDF 三层规则均有自动回归测试；当前合成运行只请求 markdown-zip，因此实际 docx/PDF 导出仍按环境显示 skipped。 +- GUI 导入与复核面板冒烟通过。 + +## 四、2026-10-07 独立复核修复 + +- 阻止结构修复把长讲义截断成短提纲后仍被接受。 +- 校验 manifest `case_id` 并证明输出目录位于评测根目录内，消除路径穿越删除风险。 +- `coverage.json`、`quality_report.json`、`run_summary.json`、`export_report.json` 缺失或关键字段无效时评测失败，不再默认当作 0 漏项。 +- Markdown 报告直接使用实际 `page_listing_hits`，不再把检测到的逐页标题显示成“无”。 +- 收紧结构标题匹配，识别 `Slide 1 - Topic`，排除“这不是总结”“例子与应用被当作主题”等假阳性，并让重复模板信号参与机械结构判定。 +- 支持仓库外绝对输出路径；CI 安装 GUI 依赖并执行全部测试。 +- 新增 `.gitattributes`，已有文本统一为 LF，PowerShell 脚本保留 CRLF，避免全仓库伪 diff。 +- 详细说明见 `REVIEW_FIXES.zh-CN.md`。 + +## 五、待你完成的事 + +1. 真实课件（文字密集/图表/表格/公式/扫描页/50 页以上）按 `benchmarks/README.md` 加入 manifest——合成样例不能替代真实课件。 +2. 用 `spot-check.template.md` 对 lecture 产物做一次人工抽查（需要 API key）。 +3. 本机安装 LibreOffice 后 PDF 导出与 PDF 层排版检查才可用（当前会优雅跳过）。 diff --git a/REVIEW_FIXES.zh-CN.md b/REVIEW_FIXES.zh-CN.md new file mode 100644 index 0000000..654b408 --- /dev/null +++ b/REVIEW_FIXES.zh-CN.md @@ -0,0 +1,36 @@ +# 2026-10-07 复核修复说明 + +本文件记录 `SlideNote-main-P0-reviewed-fixed` 相对初始 P0 修改包的第二轮独立复核修复。 + +## 已修复问题 + +1. **结构修复防截断**：全文修订必须通过结构契约，正文保留率不得低于 80%，并精确保留原来源标记、图片目标及既有 coverage；修订后重新修复图片链接并执行 figure grounding。 +2. **评测路径安全**：`case_id` 使用白名单格式，所有 case 输出/失败目录在删除或写入前验证仍位于预期根目录内。 +3. **评测失败关闭**：关键 artifact 缺失、JSON 无效或关键字段缺失时直接判定失败，不再把缺失值当作 0。 +4. **报告一致性**：逐页标题列直接依据实际检测结果；local 模式显示“有（未作为硬门槛）”。 +5. **结构契约误判**：支持 `Slide 1 - Topic` 等格式；全局结构栏目必须是 H2；“这不是总结”和单独“例子与应用”不再被误判为合格的 summary/topic；成组重复 H3 模板会使机械结构检查失败。 +6. **路径兼容性**：`--out`、manifest 和 baseline 可位于仓库外，不再因 `relative_to()` 抛错。 +7. **CI 完整性**：CI 安装 `gui` extra，Streamlit 测试不再被跳过。 +8. **排版检查证据**：新增 Markdown、DOCX、PDF 三层自动回归测试。 +9. **干净 Git diff**：文本统一 LF，新增 `.gitattributes`；PowerShell 脚本声明 CRLF。 +10. **空章节误报**：父级标题下直接进入有正文的子标题不再被错误标记为空章节。 + +## 验证结果 + +- `python -m pytest -q`：354 passed。 +- `python scripts/smoke_first_run.py`：通过。 +- 合成样本 baseline/after：1/1 通过，绝对输出路径与 baseline 对比正常。 +- 排版检查：当前 markdown-zip 运行 0 error、0 warning；docx/PDF 实际导出因该 manifest 未请求而跳过，但三层检测器均已有自动 fixture 覆盖。 + +## 仍需真实环境完成 + +合成 local 样本不能证明 LLM 讲义内容更准确。发布“质量已提升”的对外结论前，仍需使用固定模型和参数，对至少 3–5 份真实 lecture 课件完成修改前/修改后生成、双人独立 rubric、关键事实与来源抽查。该项需要真实课件和 API key，不应以合成测试替代。 + +## 2026-10-08 维护者复核修复 + +- 全文结构修复使用文档专用后处理，保留原 H1；只接受保留唯一、相同课程标题的候选。结构契约增加 `h1_count` / `title_pass`，缺失或重复主标题不再假通过。 +- 文档框架和结构契约共用标题解析与栏目匹配，带主题后缀、编号或闭合井号的既有栏目不会被重复补齐。 +- 结构、评测和排版检查忽略反引号或波浪号代码围栏里的伪标题。框架按真实 H1 定位插入位置，保留代码正文；排版检查仍报告未闭合围栏。 +- 连续 Markdown 表格按相邻行分组，正常表格不会被误拆，列数不齐的行会准确报出行号。 +- 文档级编织/增强 prompt 明确 H1 由系统添加，正文使用 H2；章节级 prompt 保持 H3。消除同时要求输出 H1/H2 和禁止输出这些标题的冲突，并更新相关缓存版本。 +- 新增 41 个回归用例，完整测试 **395 passed**；无 API 首次运行冒烟通过；原 PR 的固定合成样本 local 评测 **1/1 通过**，严格 Markdown 排版检查 **0 error / 0 warning**。该样本未请求 DOCX/PDF，因此这两层实际导出仍未验收。 diff --git a/ROADMAP.zh-CN.md b/ROADMAP.zh-CN.md index 5156961..db95d09 100644 --- a/ROADMAP.zh-CN.md +++ b/ROADMAP.zh-CN.md @@ -28,6 +28,7 @@ Ingest -> Understand -> Write -> Guard -> Export - 已生成 `notes.md`、coverage、source map 和启发式质量报告。Coverage 能提示部分漏项和来源标记问题，不能证明解释准确或学习效果。 - 独立的 `study-pack` 命令可生成 review、exam、交互式 `exam.html`、结构化学习包与错题复盘提示词；答题记录尚未形成持久化闭环。 - 已有 GUI 查看与下载、进度报告，以及 Markdown、Word、PDF、LaTeX 等输出能力；导出排版仍需用真实课件系统验收。 +- 已有固定样本评测脚手架（`benchmarks/` + `scripts/eval_decks.py`，记录耗时/tokens/成本/硬门槛并留档失败案例）、导出排版验收（`scripts/verify_note_layout.py`），以及 lecture 讲义结构契约与确定性文档框架（`structure_contract` / `document_frame`）；真实课件样本与人工评分仍在收集。 ## 建议优先级 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..c93428e --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,102 @@ +# SlideNote 固定样本集与评测流程（P0） + +目的：让"单份课件的笔记可靠、好读"成为**可复核、可比较**的工程目标，而不是 +只看 coverage 分数或单次人工感觉。本目录是 ROADMAP P0-1（真实课件评测与内容 +核对）的落地工具。 + +## 目录结构 + +```text +benchmarks/ + README.md 本文件 + samples.manifest.json 固定样本清单（评测的输入，改动前后必须用同一份） + samples/ 样本课件 + synthetic_course.pdf 固定合成样例（scripts/make_sample_deck.py 生成） + rubric.template.json 人工 1-5 分评分 rubric（结构/连贯/解释深度等 8 维度） + spot-check.template.md 人工抽查记录模板（关键事实/漏项/图文对应/来源定位） + runs// 每次评测的输出（eval_report.json/md、outputs/、failures/） +``` + +## 固定样本集要求 + +按 ROADMAP P0 的特征矩阵收集，每类至少一份，真实课件优先： + +| 特征 | 要求 | 记录在 manifest 的 `features` | +| --- | --- | --- | +| 文字密集页 | 定义/推理段落为主的课件 | `text-dense` | +| 图表 | 流程图、示意图，需要图文对应 | `chart` | +| 表格 | 有对比结论的表格 | `table` | +| 公式 | 含数学公式或代码推导 | `formula` | +| 扫描页 | 图片化、需要 OCR 的页面 | `scanned` | +| 长课件 | 50 页以上 | `long` | + +加入真实课件时，在 `samples.manifest.json` 增加一个 case： + +```json +{ + "case_id": "cs168-lecture05", + "source": "samples/cs168-lecture05.pdf", + "description": "真实课件：xxx 课程第 5 讲", + "features": ["text-dense", "chart"], + "language": "zh", + "expected_pages": 42, + "source_origin": "user-provided", + "preset": "local", + "export": "markdown-zip", + "manual_notes": "samples/notes/cs168-lecture05.human-notes.md（如有，且允许作为参考时不进仓库）", + "allow_fewshot": false +} +``` + +规则： + +- 真实课件与人工笔记**不要提交**到公共仓库（版权/隐私）；`source_origin: + user-provided` 的 case 只保留本地路径并在 `manual_notes` 记录笔记位置。 +- `allow_fewshot: true` 表示该笔记允许摘出去内容化的片段作为风格示例； + 默认 `false`，避免内容泄漏进 prompt。 +- `synthetic_course.pdf` 由 `scripts/make_sample_deck.py` 生成，内容固定， + 覆盖六类特征，用于 CI 和无 API 冒烟；它不能替代真实课件的人工评分。 + +## 评测流程 + +1. **生成/确认样本**：`python scripts/make_sample_deck.py`（或放入真实课件并更新 manifest）。 +2. **跑基线**： + ```powershell + python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/<日期>-baseline + ``` + 无 API 时用 manifest 里的 `preset: local`（离线，确定性指标）；正式评测把 + `preset` 设为 `lecture` 并配置 API key，运行时间和成本才有产品意义。 +3. **改动后再跑同一 manifest**，并和基线比较： + ```powershell + python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/<日期>-after --baseline benchmarks/runs/<日期>-baseline + ``` + `eval_report.md` 会给出每个用例的耗时/tokens/覆盖/启发式分数差值；注意 + 离线 `local` 预设不调用模型，耗时差值主要反映解析与导出。 +4. **人工抽查**：对 lecture 运行结果使用 `spot-check.template.md`（关键事实 + 10 条、漏项、图文对应、来源定位 10 处）+ `rubric.template.json` 打分。 + 两位评审独立打分，不要共享。 +5. **失败案例**：任何硬门槛失败（关键报告缺失或无效、必讲内容漏项 > 0、lecture + 产物出现逐页标题、构建/导出失败、正文为空）会在 `runs//failures//` 留下 notes.md、 + coverage、质量报告和命令记录，保证失败可复核、可进 issue。 + +## 硬门槛与启发式的边界 + +- **硬门槛（自动，失败即失败）**：关键构建/coverage/quality/export 报告存在且 + 关键字段有效、必讲内容漏项为 0、lecture 产物无"第 N 页"式正文标题、构建与 + 导出无阻塞错误。这是回归防线。local 预览允许逐页标题，但报告仍明确显示检测结果。 +- **启发式分数（自动，仅用于比较）**：coherence、explanation_depth、结构契约 + 分数等只能发现明显回归，**不代表内容正确或好读**。 +- **人工抽查（最终质量结论）**：内容是否讲对、漏了什么、图文是否对应，只能 + 通过对照原课件的人工抽查判断。三者不可互相替代，详见 + [docs/quality-and-guard.zh-CN.md](../docs/quality-and-guard.zh-CN.md)。 + +## 排版验收 + +评测通过后，用排版验收工具检查阅读体验（标题层级、段落密度、长表格、图文 +错位、断页/溢出），问题能定位到样本与页： + +```powershell +python scripts/verify_note_layout.py benchmarks/runs/<日期>-after/outputs +``` + +详见该脚本 `--help` 与 [docs/benchmark.zh-CN.md](../docs/benchmark.zh-CN.md)。 diff --git a/benchmarks/rubric.template.json b/benchmarks/rubric.template.json new file mode 100644 index 0000000..1b2834e --- /dev/null +++ b/benchmarks/rubric.template.json @@ -0,0 +1,90 @@ +{ + "$comment": "SlideNote 人工评分 rubric 模板（1-5 分）。复制本文件到 benchmarks/runs//rubric..json 后打分；两位评审分别打分，不要共享分数。分数只用于横向比较，不是产品指标。", + "schema_version": 1, + "case_id": "", + "reviewer": "<姓名/代号>", + "compared_against": "", + "scale": { + "1": "完全不符合", + "2": "大部分不符合", + "3": "一半符合", + "4": "大部分符合", + "5": "完全符合" + }, + "dimensions": [ + { + "id": "structure", + "name": "组织结构", + "question": "正文是否按知识逻辑组织成章节，而不是逐页复述课件？", + "anchor_1": "整篇是'第 N 页讲了什么'的列表。", + "anchor_5": "章节按概念与推理链命名并展开，页码只出现在来源标记里。" + }, + { + "id": "coherence", + "name": "连贯性", + "question": "段落之间是否有真实的逻辑衔接，能被连续阅读？", + "anchor_1": "段落之间没有关联，像独立卡片堆放。", + "anchor_5": "前后概念按依赖顺序展开，过渡自然。" + }, + { + "id": "explanation_depth", + "name": "解释深度", + "question": "核心概念是否讲清了是什么、为什么重要、如何运作？", + "anchor_1": "只罗列术语和原句。", + "anchor_5": "核心概念均有机制、原因与前后关系解释，深度接近讲义。" + }, + { + "id": "information_selection", + "name": "信息取舍", + "question": "是否区分了核心知识与装饰性/结构性内容，没有为了凑覆盖而堆砌？", + "anchor_1": "目录、页眉、重复 bullet 全部照搬。", + "anchor_5": "只保留有学习价值的增量，结构页仅保留来源标记。" + }, + { + "id": "figure_relation", + "name": "图文关系", + "question": "图表是否出现在相关解释附近，并且说明补充了图表支撑的概念？", + "anchor_1": "图片全部堆在页尾或章节末，说明只有'图中展示了'。", + "anchor_5": "每张图靠近使用它的段落，说明指出图表支撑的推理或结论。" + }, + { + "id": "redundancy", + "name": "重复度", + "question": "同一概念是否只解释一次，没有把渐进演示复述成多遍？", + "anchor_1": "同一定义出现多次，练习页被重复讲解。", + "anchor_5": "重复内容合并为一次最清楚的解释。" + }, + { + "id": "note_likeness", + "name": "电子笔记观感", + "question": "整体是否像学生认真整理的电子笔记（层级、密度、标题习惯）？", + "anchor_1": "像 PPT 转录或机器输出。", + "anchor_5": "像可直接留档复习的讲义笔记。" + }, + { + "id": "source_correctness", + "name": "来源正确性", + "question": "抽查 10 处来源标记，是否指向正确的页面与元素？", + "anchor_1": "多数标记错位或缺失。", + "anchor_5": "抽查全部正确。" + } + ], + "scores": { + "structure": null, + "coherence": null, + "explanation_depth": null, + "information_selection": null, + "figure_relation": null, + "redundancy": null, + "note_likeness": null, + "source_correctness": null + }, + "failure_examples": [ + { + "location": "<章节/段落>", + "evidence": "<摘录原文>", + "problem": "<问题描述>", + "suggestion": "<期望的写法>" + } + ] +} diff --git a/benchmarks/samples.manifest.json b/benchmarks/samples.manifest.json new file mode 100644 index 0000000..376d758 --- /dev/null +++ b/benchmarks/samples.manifest.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "description": "SlideNote 固定评测样本清单。每次改动在同一个 manifest 上运行 scripts/eval_decks.py，保证结果可比。", + "cases": [ + { + "case_id": "synthetic-os-basics", + "source": "benchmarks/samples/synthetic_course.pdf", + "description": "固定合成样例：文字密集页、对比表、公式、折线图、扫描页，共 18 页；由 scripts/make_sample_deck.py 生成，可重复生成且可提交仓库。", + "features": ["text-dense", "table", "formula", "chart", "scanned", "long"], + "language": "zh", + "expected_pages": 18, + "source_origin": "generated", + "preset": "local", + "export": "markdown-zip", + "manual_notes": null, + "allow_fewshot": false + } + ] +} diff --git a/benchmarks/samples/synthetic_course.pdf b/benchmarks/samples/synthetic_course.pdf new file mode 100644 index 0000000..aea2d32 Binary files /dev/null and b/benchmarks/samples/synthetic_course.pdf differ diff --git a/benchmarks/spot-check.template.md b/benchmarks/spot-check.template.md new file mode 100644 index 0000000..2835fb4 --- /dev/null +++ b/benchmarks/spot-check.template.md @@ -0,0 +1,52 @@ +# 人工抽查记录： + +> 使用方法：对照**原课件**逐项抽查，不要只看笔记。每个 case 一份记录， +> 存到 `benchmarks/runs//spot-check..md`。抽查结论写入 +> `benchmarks/runs//comparison-notes.md`，作为下次改动的依据。 + +- 评审人： +- 日期： +- 笔记版本（run 目录 / commit）： +- 课件版本（source 文件 + 页数）： +- 预设与模型（preset / provider / model / vision / ocr）： + +## 1. 关键事实抽查（至少 10 条） + +从课件中挑出"必须讲对"的事实，逐条核对笔记。 + +| # | 来源页 | 课件中的关键事实 | 笔记是否讲到 | 笔记表述是否正确 | 来源标记是否指向该页 | 备注 | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | p3 | 例：PCB 记录进程标识与状态 | 是/否 | 是/否 | 是/否 | | +| 2 | | | | | | | + +## 2. 漏项检查 + +| # | 来源页 | 漏掉的内容 | 是否属于必须解释的学习内容 | 建议归属章节 | +| --- | --- | --- | --- | --- | +| 1 | | | | | + +## 3. 图文对应检查 + +| # | 图片/表格（来源页） | 笔记位置是否靠近相关解释 | 说明是否讲出图表支撑的概念 | 错位描述 | +| --- | --- | --- | --- | --- | +| 1 | | | | | + +## 4. 来源定位抽查 + +随机抽 10 处 `` 标记，核对指向的页面与元素： + +| # | 笔记位置 | 标记内容 | 实际对应元素 | 是否正确 | +| --- | --- | --- | --- | --- | +| 1 | | | | | + +## 5. 运行时间与成本（从 eval_report.json 摘录） + +- 总耗时：____ s（各阶段耗时见 eval_report.json `slowest_stages`） +- LLM 调用：____ 次；输入 tokens：____；输出 tokens：____ +- 估算成本：____ USD（cost_report；pricing.template.json 估算，不代表账单） + +## 6. 结论 + +- 结构：____/5　连贯：____/5　解释深度：____/5　信息取舍：____/5　图文关系：____/5　重复度：____/5　笔记观感：____/5　来源正确性：____/5 +- 可复核失败案例（如有）：`failures//` 路径 + 一句话原因 +- 下一轮最值得改的一件事： diff --git a/docs/benchmark.zh-CN.md b/docs/benchmark.zh-CN.md index 1e7d026..6fd5b1a 100644 --- a/docs/benchmark.zh-CN.md +++ b/docs/benchmark.zh-CN.md @@ -52,6 +52,57 @@ python -m slidenote build path\to\lecture.pdf --out outputs\baseline-lecture --e --- +## 固定样本集与自动评测（benchmarks/，P0） + +`benchmarks/` 目录把上面的手工流程固化为可重复执行的评测，改动前后跑同一份 +manifest 即可对比耗时、tokens、覆盖与启发式分数，失败用例自动留档： + +```powershell +# 1. 生成固定合成样例（覆盖文字密集/表格/公式/图表/扫描页/长课件） +python scripts/make_sample_deck.py + +# 2. 跑基线（local 预设，无 API；正式评测改用 lecture 预设并配置 key） +python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/<日期>-baseline + +# 3. 改动后用同一 manifest 复跑并对比 +python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/<日期>-after --baseline benchmarks/runs/<日期>-baseline +``` + +- `eval_report.md` / `eval_report.json`：每个用例的硬门槛结果、耗时、阶段耗时、 + tokens 与成本估算、coverage 与启发式分数；`--baseline` 生成逐项差值。 +- 硬门槛失败（关键报告缺失/无效、必讲内容漏项 > 0、lecture 产物出现"第 N 页"标题、构建/导出失败） + 会把 notes/coverage/质量报告复制到 `runs//failures//`，作为可复核 + 的失败案例。 +- manifest 的 `case_id` 只允许字母、数字、点、下划线和连字符；工具会再次校验 + 输出路径位于评测根目录内。`--out` 和 `--baseline` 可使用仓库外的绝对路径。 +- local 预览按设计可保留逐页标题；评测报告会显示“有（未作为硬门槛）”，避免把 + “不拦截”误写成“未检测到”。 +- 真实课件请按 `benchmarks/README.md` 的说明加入 manifest（不要提交课件本体）； + 人工抽查用 `benchmarks/spot-check.template.md`，评分用 `benchmarks/rubric.template.json`。 +- 启发式分数只用于跨运行比较，不能代表内容正确性；质量结论以人工抽查为准。 + +## 导出排版验收（P0） + +评测通过后，用排版验收工具检查实际阅读排版（标题层级、空章节、图片 alt 与 +邻近解释、超宽表格、代码围栏/公式闭合、Word 表格行列一致、PDF 文本溢出、 +图文重叠、空白页、页尾孤立标题）。每个问题都报告样本、文件和页码： + +```powershell +# 对一个 build 输出目录（docx/pdf 未导出时该项显示 skipped 并给原因） +python scripts/verify_note_layout.py outputs\baseline-lecture + +# 一次检查整个评测运行的所有用例 +python scripts/verify_note_layout.py benchmarks\runs\<日期>-after\outputs + +# 保留结构化报告；--strict 让警告也导致失败 +python scripts/verify_note_layout.py outputs\baseline-lecture --json layout_report.json --strict +``` + +工具只做启发式检查：能发现断页、溢出、图文错位的常见模式并定位到样本与页， +但最终排版是否可读仍建议人工翻阅一次导出文件。 + +--- + ## 实验分支附录：Agent 后端（experiment/dsh-backend） 以下命令只在实验分支存在。`agent-eval` 内置的 `local` 对照适合检查实验流程能否运行及其结构指标；它与使用文本模型的 Agent 输入条件不同，不能据此断言 Agent 的讲义质量优于主线 `lecture`。质量对比应另跑同一课件的 `lecture`，记录所用模型、视觉/OCR 配置、缓存状态，并人工盲评笔记。 diff --git a/docs/quality-and-guard.zh-CN.md b/docs/quality-and-guard.zh-CN.md index 60b0627..e7e8442 100644 --- a/docs/quality-and-guard.zh-CN.md +++ b/docs/quality-and-guard.zh-CN.md @@ -68,6 +68,31 @@ content_guard.json 未来可以增加轻量 LLM 审阅 pass，但不应该让同一个写作模型无约束地自己审自己。 +### 三层质量信号及其边界（重要） + +SlideNote 有三层互相**不可替代**的质量信号，不要把任何一层当作最终结论： + +| 层 | 产物 | 能回答 | 不能回答 | +| --- | --- | --- | --- | +| 结构性 coverage | `coverage.json` / `coverage.md` | 关键元素有没有进入可见正文、来源标记是否成立、哪些页有漏项 | 讲得对不对、讲得好不好、有没有编造 | +| 启发式质量分数 | `quality_report.json` | 笔记是否出现明显回归（逐页复述、缺自测/易错点栏目、来源密度骤降） | 段落之间是否真有逻辑、例子是否有帮助、内容是否正确 | +| 人工内容复核 | `coverage.md` + 原课件 + `source_map.json` | 事实是否正确、遗漏是否关键、图文是否对应 | ——（这是最终结论的唯一来源） | + +`hallucination_risk` 只是复核优先级提示：`high` 表示来源标记密度低且/或有必讲漏项，应优先抽查；它不做事实核查，`low` 也不代表内容正确。 + +GUI 的 Quality 标签页按上述边界展示数据，并把 coverage 缺项、figure 缺失和启发式风险合并成一个"建议人工复核"清单——清单每行都标注信号来源和查看位置，但不合并成任何总分。 + +### 讲义结构契约（structure contract） + +对 `lecture-notes` / `study-guide` profile，`quality_report.json` 额外包含全文结构契约评估（`structure_contract` 字段）： + +- 必备栏目：`orientation`（本讲目标）、`topic_sections`（按真实课程主题命名的 H2 章节）、`summary`（本讲总结）、`self_test`（章节自测），全文各出现一次； +- 推荐栏目：`example_application`（有材料依据的例子/应用）、`pitfalls`（真正必要的易错点）； +- 机械复述检查（`mechanical_repetition_pass`）：正文不应出现"第 N 页"/`Slide N` 式 H2-H4 标题，不应重复出现同名的目标/总结/自测栏目，也不应在多章重复同一组通用 H3 模板； +- `human_note_structure_score` 是上述栏目的覆盖比例（0-1），只用于跨运行比较。 + +生成管线在 lecture-weave 收尾时由确定性代码补齐缺失的文档框架（`document_frame`，从 deck brief 提取，不新增课件外事实）；只有当补齐后契约仍不通过（例如残留逐页标题）时，才会触发一次全文结构修订。修订稿必须让契约通过、正文至少保留 80%、精确保留已有来源标记和图片目标、coverage 不回退且输出未截断，才会被采纳；否则保留原稿并在 `llm_usage.json` 的 `repair_contexts` 记录拒绝原因。 + ## 教师讲义模式 `lecture` preset 的目标是“教学重构”，不是简单总结。 diff --git a/gui/app.py b/gui/app.py index a14c53e..90910aa 100644 --- a/gui/app.py +++ b/gui/app.py @@ -1042,6 +1042,7 @@ def _cached_output_zip(output_dir: str, signature: tuple[tuple[str, int, int], . def _render_quality_panel(output_dir: Path) -> None: coverage = read_json(output_dir / "coverage.json") or {} run_summary = read_json(output_dir / "run_summary.json") or {} + quality = read_json(output_dir / "quality_report.json") or {} summary = coverage_summary(coverage) score = summary["ratio"] * 100 label = "Visible coverage" if summary["visible"] else "Coverage" @@ -1051,6 +1052,13 @@ def _render_quality_panel(output_dir: Path) -> None: q3.metric("Not explained", summary["missing"]) q4.metric("Pages", (run_summary.get("counts") or {}).get("pages", "—")) st.progress(min(max(score / 100, 0), 1)) + st.caption( + "How to read these signals: **coverage** is a structural clue about whether slide elements " + "are referenced and explained — it cannot prove the explanations are correct. The scores in " + "**quality_report** are heuristic signals computed from text patterns — they can only flag " + "obvious regressions. The final quality verdict always requires a **manual review against " + "the original slides** (see the Page explorer tab)." + ) if summary["required_total"]: st.caption(f"Required items not explained: {summary['required_missing']} / {summary['required_total']}") @@ -1064,9 +1072,100 @@ def _render_quality_panel(output_dir: Path) -> None: else: st.success("No missing coverage items reported.") + _render_manual_review_panel(coverage, quality) _render_stage_timings(run_summary) +def _render_manual_review_panel(coverage: dict, quality: dict) -> None: + """Join coverage gaps and heuristic quality signals into one review queue. + + This is the GUI half of the P0 boundary rule: structural coverage, heuristic + quality scores and manual content review are presented side by side but never + merged into a single number, and each review row names where to look. + """ + rows: list[dict] = [] + missing_pages = sorted( + { + int(item["slide_id"]) + for item in coverage_missing_items(coverage) + if item.get("slide_id") + } + ) + required_missing = (coverage.get("required_visible_coverage") or {}).get("missing") or 0 + for page in missing_pages: + rows.append( + { + "Page": page, + "Signal": "coverage", + "Reason": "Elements from this slide are not explained in the visible note text.", + "Where to look": f"Original slide {page} vs the note section referencing p{page} (Page explorer tab).", + } + ) + if quality: + risk = str(quality.get("hallucination_risk") or "") + if risk == "high": + rows.append( + { + "Page": "—", + "Signal": "heuristic", + "Reason": "High hallucination risk: few source markers relative to prose (and/or missing required items).", + "Where to look": "Spot-check expanded claims in notes.md against the slides.", + } + ) + elif risk == "medium": + rows.append( + { + "Page": "—", + "Signal": "heuristic", + "Reason": "Medium hallucination risk: some explanations may lack a source marker.", + "Where to look": "Spot-check a sample of paragraphs against the slides.", + } + ) + if quality.get("mechanical_structure_pass") is False: + rows.append( + { + "Page": "—", + "Signal": "heuristic", + "Reason": "Note still reads like a page listing (page-number headings or repeated template sections).", + "Where to look": "Compare note headings with the deck outline in Page explorer.", + } + ) + contract = quality.get("structure_contract") or {} + if contract.get("enforced") and contract.get("missing_slots"): + labels = ", ".join(str(slot) for slot in contract["missing_slots"]) + rows.append( + { + "Page": "—", + "Signal": "heuristic", + "Reason": f"Lecture structure contract is missing slots: {labels}.", + "Where to look": "Check whether the deck actually supports these sections before adding them.", + } + ) + figure_coverage = coverage.get("figure_coverage") or {} + for figure in figure_coverage.get("figures") or []: + if figure.get("covered"): + continue + page = figure.get("slide_id") + rows.append( + { + "Page": page, + "Signal": "coverage", + "Reason": "A non-decorative image from this slide is missing or unexplained in the note.", + "Where to look": f"Original slide {page}: compare figure {figure.get('id')} with the surrounding note text.", + } + ) + if rows: + st.markdown("#### Suggested manual review") + st.info( + "These rows come from different signals and are **not** a quality score. Work top-down; " + "clear rows disappear as builds improve, and anything not listed here still deserves a read-through." + ) + st.dataframe(rows, use_container_width=True, hide_index=True) + elif quality: + st.markdown("#### Suggested manual review") + st.caption("No high-risk signals flagged. Heuristics cannot certify correctness — do a quick read-through before relying on the notes.") + + def _render_stage_timings(run_summary: dict[str, Any]) -> None: timings = run_summary.get("stage_timings") or {} rows: list[dict[str, Any]] = [] diff --git a/scripts/eval_decks.py b/scripts/eval_decks.py new file mode 100644 index 0000000..d2350c3 --- /dev/null +++ b/scripts/eval_decks.py @@ -0,0 +1,471 @@ +"""Run the SlideNote build pipeline over a fixed sample manifest and record +reproducible evidence for P0 review: wall time, stage timings, token usage, +cost estimate, coverage, heuristic quality and structure-contract metrics. + +Typical use: + + python scripts/make_sample_deck.py --out benchmarks/samples/synthetic_course.pdf + python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/local-baseline + # after a change: + python scripts/eval_decks.py benchmarks/samples.manifest.json --out benchmarks/runs/after-change \ + --baseline benchmarks/runs/local-baseline + +Hard gates (a case fails when any is violated): +- required visible coverage missing items == 0 +- no page-listing headings (lecture structure contract mechanical check) +- notes.md generated and non-empty +- build and requested exports finish without blocking errors + +Heuristic scores are recorded for comparison only; they never replace the +manual spot-check described in benchmarks/README.md. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +API_ENV_KEYS = ( + "DEEPSEEK_API_KEY", + "DASHSCOPE_API_KEY", + "OPENAI_API_KEY", + "OPENAI_API_BASE", + "BAIDU_OCR_API_KEY", + "BAIDU_OCR_SECRET_KEY", +) +CASE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +REQUIRED_ARTIFACTS = ( + "notes.md", + "run_summary.json", + "coverage.json", + "quality_report.json", + "export_report.json", +) + + +def display_path(path: Path, base: Path = REPO_ROOT) -> str: + """Return a readable path without requiring it to live under the repo.""" + resolved = path.resolve() + try: + return str(resolved.relative_to(base.resolve())) + except ValueError: + return str(resolved) + + +def validate_case_id(value: object) -> str: + case_id = str(value or "") + if not CASE_ID_RE.fullmatch(case_id): + raise ValueError( + "case_id must match ^[A-Za-z0-9][A-Za-z0-9._-]*$ " + f"(got {case_id!r})" + ) + return case_id + + +def safe_case_dir(root: Path, case_id: str) -> Path: + """Resolve a case directory and prove it stays below its expected root.""" + root = root.resolve() + candidate = (root / validate_case_id(case_id)).resolve() + if candidate.parent != root: + raise ValueError(f"unsafe case output path: {candidate}") + return candidate + + +def env_without_api_keys() -> dict[str, str]: + env = dict(os.environ) + for key in API_ENV_KEYS: + env.pop(key, None) + return env + + +def read_json(path: Path) -> dict | None: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def structure_page_headings(notes: str) -> list[str]: + from slidenote.notes.structure import _heading_sections, _is_page_heading + + return [ + heading["title"] + for heading in _heading_sections(notes) + if 2 <= heading["level"] <= 4 and _is_page_heading(heading["title"]) + ] + + +def collect_case_metrics(output_dir: Path, wall_seconds: float, returncode: int, stderr_tail: str) -> dict: + run_summary = read_json(output_dir / "run_summary.json") or {} + coverage = read_json(output_dir / "coverage.json") or {} + quality = read_json(output_dir / "quality_report.json") or {} + usage = read_json(output_dir / "llm_usage.json") or {} + export_report = read_json(output_dir / "export_report.json") or {} + notes_path = output_dir / "notes.md" + notes = notes_path.read_text(encoding="utf-8", errors="replace") if notes_path.exists() else "" + required = coverage.get("required_visible_coverage") or {} + artifact_presence = {name: (output_dir / name).is_file() for name in REQUIRED_ARTIFACTS} + + cost_summary = None + try: + from slidenote.costing import write_cost_report + + pricing = REPO_ROOT / "pricing.template.json" + report = write_cost_report(output_dir, pricing if pricing.exists() else None, "USD") + cost_summary = report.get("summary") + except Exception as exc: # cost report is optional evidence + cost_summary = {"error": type(exc).__name__} + + timings = run_summary.get("stage_timings") or {} + slowest = timings.get("slowest_stages") if isinstance(timings, dict) else None + + usage_summary = usage.get("summary") or {} + export_summary = export_report.get("summary") or {} + export_results = export_report.get("results") or [] + export_failures = [ + item + for item in export_results + if isinstance(item, dict) and item.get("status") == "failed" + ] + blocking_export_failures = export_summary.get("blocking_failures") + # The structure contract only governs lecture note profiles. The local + # preset is a mechanical parse preview where per-page headings are by + # design, so its page-listing hits are recorded but not gated. + structure_enforced = bool(quality.get("structure_contract", {}).get("enforced")) + page_listing_hits = structure_page_headings(notes) + required_missing = required.get("missing") + coverage_valid = isinstance(required_missing, int) and not isinstance(required_missing, bool) + quality_valid = isinstance(quality.get("structure_contract"), dict) and "enforced" in quality["structure_contract"] + export_valid = ( + isinstance(blocking_export_failures, int) + and not isinstance(blocking_export_failures, bool) + and isinstance(export_results, list) + ) + run_summary_valid = bool(run_summary) and isinstance(run_summary.get("run"), dict) + gates = { + "build_exit_code_zero": returncode == 0, + "required_artifacts_present": all(artifact_presence.values()), + "run_summary_valid": run_summary_valid, + "coverage_report_valid": coverage_valid, + "quality_report_valid": quality_valid, + "export_report_valid": export_valid, + "notes_nonempty": bool(notes.strip()), + "required_visible_missing_zero": coverage_valid and required_missing == 0, + "no_page_listing_headings": (not page_listing_hits) if structure_enforced else True, + "no_export_blocking_failures": export_valid and blocking_export_failures == 0 and not export_failures, + } + return { + "case_id_placeholder": True, + "wall_seconds": round(wall_seconds, 3), + "build_exit_code": returncode, + "stderr_tail": stderr_tail[-800:], + "artifacts": artifact_presence, + "preset": run_summary.get("run", {}).get("preset"), + "counts": run_summary.get("counts"), + "slowest_stages": slowest[:5] if isinstance(slowest, list) else None, + "coverage": { + "missing": coverage.get("missing"), + "coverage_ratio": coverage.get("coverage_ratio"), + "required_visible_missing": required.get("missing"), + "required_visible_total": required.get("total"), + }, + "quality_heuristics": { + key: quality.get(key) + for key in ( + "coherence_score", + "explanation_depth_score", + "example_score", + "figure_integration_score", + "mechanical_page_listing_score", + "self_test_score", + "pitfall_score", + "human_note_structure_score", + "structure_contract_pass", + "mechanical_structure_pass", + "hallucination_risk", + ) + }, + "structure_contract": quality.get("structure_contract"), + "page_listing_hits": page_listing_hits, + "usage": { + key: usage_summary.get(key) + for key in ( + "llm_calls", + "input_tokens", + "output_tokens", + "total_tokens", + "api_retries", + "failed_contexts", + "page_note_calls", + "weave_calls", + "teaching_enrichment_calls", + "repair_calls", + ) + }, + "cost_summary": cost_summary, + "export_failures": export_failures, + "gates": gates, + "passed": all(gates.values()), + "notes_chars": len(notes), + } + + +def run_case(case: dict, run_dir: Path, preset: str | None, timeout: int) -> dict: + try: + case_id = validate_case_id(case.get("case_id")) + except (AttributeError, ValueError) as exc: + return {"case_id": str(case.get("case_id", "")) if isinstance(case, dict) else "", "error": str(exc), "passed": False} + source_value = case.get("source") + if not isinstance(source_value, str) or not source_value.strip(): + return {"case_id": case_id, "error": "source must be a non-empty path string", "passed": False} + source = (REPO_ROOT / source_value).resolve() + if not source.exists(): + return {"case_id": case_id, "error": f"source not found: {source}", "passed": False} + case_out = safe_case_dir(run_dir / "outputs", case_id) + if case_out.exists(): + shutil.rmtree(case_out) + case_out.mkdir(parents=True, exist_ok=True) + command = [ + sys.executable, + "-m", + "slidenote", + "build", + str(source), + "--out", + str(case_out), + "--preset", + preset or case.get("preset") or "local", + ] + export = case.get("export") or "markdown-zip" + if export: + command += ["--export", export] + env = env_without_api_keys() if (preset or case.get("preset") or "local") == "local" else dict(os.environ) + started = time.perf_counter() + try: + proc = subprocess.run( + command, + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + wall = time.perf_counter() - started + metrics = collect_case_metrics(case_out, wall, proc.returncode, proc.stderr or "") + except subprocess.TimeoutExpired: + wall = time.perf_counter() - started + metrics = { + "wall_seconds": round(wall, 3), + "error": f"timeout after {timeout}s", + "passed": False, + } + except OSError as exc: + wall = time.perf_counter() - started + metrics = { + "wall_seconds": round(wall, 3), + "error": f"failed to launch build: {type(exc).__name__}: {exc}", + "passed": False, + } + metrics["case_id"] = case_id + metrics.pop("case_id_placeholder", None) + metrics["command"] = command[2:] + metrics["output_dir"] = display_path(case_out) + return metrics + + +def archive_failure(case: dict, metrics: dict, run_dir: Path) -> Path | None: + try: + case_id = validate_case_id(case.get("case_id")) + except (AttributeError, ValueError): + return None + out_dir = safe_case_dir(run_dir / "outputs", case_id) + failure_dir = safe_case_dir(run_dir / "failures", case_id) + failure_dir.mkdir(parents=True, exist_ok=True) + for name in ( + "notes.md", + "coverage.md", + "coverage.json", + "quality_report.json", + "export_report.json", + "llm_usage.json", + "run_summary.json", + ): + source = out_dir / name + if source.exists(): + shutil.copy2(source, failure_dir / name) + (failure_dir / "eval_metrics.json").write_text( + json.dumps(metrics, ensure_ascii=False, indent=2), encoding="utf-8" + ) + return failure_dir + + +def compare_with_baseline(report: dict, baseline_dir: Path) -> dict: + baseline = read_json(baseline_dir / "eval_report.json") + if not baseline: + return {"error": f"no eval_report.json under {baseline_dir}"} + baseline_by_case = {item["case_id"]: item for item in baseline.get("cases", []) if item.get("case_id")} + comparisons = [] + for current in report["cases"]: + case_id = current.get("case_id") + previous = baseline_by_case.get(case_id) + if not previous: + comparisons.append({"case_id": case_id, "status": "new_case"}) + continue + deltas = {} + for key in ("wall_seconds", "notes_chars"): + if isinstance(current.get(key), (int, float)) and isinstance(previous.get(key), (int, float)): + deltas[key] = round(current[key] - previous[key], 3) + for key in ("missing", "required_visible_missing"): + cur = (current.get("coverage") or {}).get(key) + prev = (previous.get("coverage") or {}).get(key) + if isinstance(cur, (int, float)) and isinstance(prev, (int, float)): + deltas[f"coverage.{key}"] = cur - prev + cur_usage, prev_usage = current.get("usage") or {}, previous.get("usage") or {} + for key in ("input_tokens", "output_tokens", "total_tokens", "llm_calls"): + if isinstance(cur_usage.get(key), (int, float)) and isinstance(prev_usage.get(key), (int, float)): + deltas[f"usage.{key}"] = cur_usage[key] - prev_usage[key] + cur_h, prev_h = current.get("quality_heuristics") or {}, previous.get("quality_heuristics") or {} + for key, value in cur_h.items(): + if isinstance(value, (int, float)) and isinstance(prev_h.get(key), (int, float)): + deltas[f"quality.{key}"] = round(value - prev_h[key], 4) + comparisons.append( + { + "case_id": case_id, + "status": "regressed" if (current.get("passed") is False and previous.get("passed") is not False) else + ("improved" if (current.get("passed") is True and previous.get("passed") is False) else "same_or_mixed"), + "gates_now": current.get("gates"), + "gates_baseline": previous.get("gates"), + "deltas": deltas, + } + ) + return {"baseline": str(baseline_dir), "cases": comparisons} + + +def write_markdown_report(report: dict, path: Path) -> None: + lines = [ + "# SlideNote 评测报告（eval_decks）", + "", + f"- 运行时间：{report['meta']['generated_at']}", + f"- manifest：`{report['meta']['manifest']}`", + f"- 通过用例：{sum(1 for c in report['cases'] if c.get('passed'))}/{len(report['cases'])}", + "", + "> 以下启发式分数只用于跨运行比较，不能代表内容正确性；内容结论请配合人工抽查模板使用。", + "", + "| 用例 | 结果 | 耗时(s) | 必讲漏项 | 缺失元素 | 逐页标题 | LLM调用 | tokens(入/出) |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for case in report["cases"]: + coverage = case.get("coverage") or {} + usage = case.get("usage") or {} + heuristics = case.get("quality_heuristics") or {} + lines.append( + "| {id} | {status} | {wall} | {req} | {miss} | {page_listing} | {calls} | {inp}/{out} |".format( + id=case.get("case_id", "?"), + status="通过" if case.get("passed") else "失败", + wall=case.get("wall_seconds", "—"), + req=coverage.get("required_visible_missing") if coverage.get("required_visible_missing") is not None else "—", + miss=coverage.get("missing") if coverage.get("missing") is not None else "—", + page_listing=( + "有" if case.get("page_listing_hits") and not (case.get("gates") or {}).get("no_page_listing_headings") + else "有（未作为硬门槛）" if case.get("page_listing_hits") + else "无" + ), + calls=usage.get("llm_calls") if usage.get("llm_calls") is not None else "—", + inp=usage.get("input_tokens") if usage.get("input_tokens") is not None else "—", + out=usage.get("output_tokens") if usage.get("output_tokens") is not None else "—", + ) + ) + failed = [case for case in report["cases"] if not case.get("passed")] + if failed: + lines += ["", "## 失败用例与证据", ""] + for case in failed: + lines.append(f"### {case.get('case_id')}") + lines.append("") + lines.append(f"- 失败门槛：{[k for k, ok in (case.get('gates') or {}).items() if not ok]}") + if case.get("error"): + lines.append(f"- 错误：{case['error']}") + if case.get("stderr_tail"): + lines.append(f"- stderr 末尾：`{case['stderr_tail'][-300:]}`") + lines.append(f"- 证据目录：`{report['meta']['run_dir']}/failures/{case.get('case_id')}`") + lines.append("") + if report.get("comparison", {}).get("cases"): + lines += ["", "## 与基线对比", "", "| 用例 | 状态 | 主要差异 |", "| --- | --- | --- |"] + for item in report["comparison"]["cases"]: + deltas = item.get("deltas") or {} + summary = ", ".join(f"{k}: {v:+.3f}" if isinstance(v, float) else f"{k}: {v:+d}" for k, v in deltas.items()) + lines.append(f"| {item.get('case_id')} | {item.get('status')} | {summary or '—'} |") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path, help="Path to a samples manifest JSON") + parser.add_argument("--out", type=Path, required=True, help="Run directory (e.g. benchmarks/runs/2026-xx-baseline)") + parser.add_argument("--preset", choices=("local", "lecture"), default=None, help="Override each case preset") + parser.add_argument("--baseline", type=Path, default=None, help="Previous run directory to compare against") + parser.add_argument("--timeout", type=int, default=1800, help="Per-case build timeout in seconds") + parser.add_argument("--no-failure-archive", action="store_true", help="Do not copy failing artifacts") + args = parser.parse_args() + + manifest_path = args.manifest.resolve() + manifest = read_json(manifest_path) + if not manifest or not isinstance(manifest.get("cases"), list): + print(f"invalid manifest: {manifest_path}", file=sys.stderr) + return 2 + case_ids: list[str] = [] + try: + for case in manifest["cases"]: + if not isinstance(case, dict): + raise ValueError("every manifest case must be an object") + case_ids.append(validate_case_id(case.get("case_id"))) + except ValueError as exc: + print(f"invalid manifest: {exc}", file=sys.stderr) + return 2 + if len(case_ids) != len(set(case_ids)): + print("invalid manifest: duplicate case_id values", file=sys.stderr) + return 2 + + run_dir = args.out.resolve() + run_dir.mkdir(parents=True, exist_ok=True) + cases = [run_case(case, run_dir, args.preset, args.timeout) for case in manifest["cases"]] + if not args.no_failure_archive: + for case, metric in zip(manifest["cases"], cases): + if not metric.get("passed"): + archive_failure(case, metric, run_dir) + + report = { + "meta": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "manifest": display_path(manifest_path), + "run_dir": display_path(run_dir), + "preset_override": args.preset, + "python": sys.version.split()[0], + "platform": platform.platform(), + }, + "cases": cases, + } + if args.baseline: + report["comparison"] = compare_with_baseline(report, args.baseline.resolve()) + (run_dir / "eval_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + write_markdown_report(report, run_dir / "eval_report.md") + + failed = [case for case in cases if not case.get("passed")] + print(f"cases: {len(cases)}, passed: {len(cases) - len(failed)}, failed: {len(failed)}") + print(f"report: {run_dir / 'eval_report.md'}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/make_sample_deck.py b/scripts/make_sample_deck.py new file mode 100644 index 0000000..684f434 --- /dev/null +++ b/scripts/make_sample_deck.py @@ -0,0 +1,269 @@ +"""Generate a fixed synthetic courseware sample for SlideNote benchmarks. + +The deck deterministically covers the P0 sample-set feature matrix from the +roadmap: text-dense pages, tables, formulas, chart-like figures, scanned +(image-only) pages and a long page count. Run: + + python scripts/make_sample_deck.py --out benchmarks/samples/synthetic_course.pdf + +The output is byte-stable across runs on the same PyMuPDF version, so it can +serve as a fixed benchmark input in CI and in real-courseware evaluations. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import pymupdf + +COURSE_TITLE = "操作系统导出：进程与内存（基准样例课件）" + +TEXT_DENSE_SECTIONS = [ + ( + "进程的概念", + [ + "进程是一个正在执行的程序实例，它不仅包含程序的代码，还包含程序计数器、寄存器内容和进程地址空间等执行状态。", + "操作系统通过进程控制块（PCB）记录每个进程的标识、状态、优先级、调度信息和资源清单；PCB 是操作系统感知进程存在的唯一标志。", + "进程的三种基本状态是就绪、运行和阻塞：就绪进程等待调度器分配 CPU，运行进程正在占用 CPU，阻塞进程等待某个事件完成后回到就绪队列。", + "状态转换由特定事件驱动：运行到就绪通常因为时间片耗尽，运行到阻塞通常因为请求 I/O 或等待锁，阻塞到就绪则由事件完成触发。", + ], + ), + ( + "线程与并发", + [ + "线程是进程内的一条执行路径，同一进程内的线程共享地址空间、打开文件表和信号处理器，但各自拥有独立的栈和寄存器上下文。", + "引入线程的收益来自两个方向：一是切换成本低于进程切换，因为无需更换地址空间；二是同进程线程间的数据共享不需要额外的通信机制。", + "用户级线程由线程库管理，内核不感知它们的存在；内核级线程由内核直接调度，能够利用多核并行，但创建和切换需要陷入内核。", + "并发与并行的区别在于时间维度：并发指多个执行流在同一时间间隔内交替推进，并行指多个执行流在同一时刻同时推进，需要多核硬件支持。", + ], + ), + ( + "虚拟内存", + [ + "虚拟内存为每个进程提供独立的、可比物理内存更大的地址空间，其核心机制是把地址空间切分成固定大小的页，并按需调入物理内存。", + "页表记录虚拟页号到物理页框号的映射，并维护有效位、修改位和访问位；有效位为 0 的页访问会触发缺页异常，由内核负责调入。", + "缺页处理的代价主要来自磁盘 I/O，因此替换策略直接影响性能：LRU 用最近的访问历史近似未来访问，时钟算法用访问位做出低开销的近似 LRU。", + "工作集模型指出，进程在一段时间内稳定访问的页面集合称为工作集；只要物理内存能容纳所有活跃进程的工作集，抖动就可以避免。", + ], + ), + ( + "文件系统", + [ + "文件系统把存储设备组织成文件和目录，并通过元数据描述文件的所有者、权限、大小和数据块位置。", + "索引节点（inode）是大多数类 Unix 文件系统的核心结构，它保存文件元数据和数据块指针，目录项则把文件名映射到 inode 编号。", + "日志式文件系统在写入数据前先把操作记录写入日志，崩溃后可以重放日志恢复一致性，避免文件系统长时间自检。", + "磁盘调度需要权衡吞吐量与公平性：SSTF 总是选择最近的请求以减少寻道时间，但可能让边缘请求长期等待。", + ], + ), +] + +TABLE_PAGE = { + "title": "调度算法对比", + "headers": ["算法", "选择依据", "优点", "风险"], + "rows": [ + ["FCFS", "到达顺序", "实现简单、无饥饿", "平均等待时间长"], + ["SJF", "最短作业优先", "平均等待时间最优", "长作业可能饥饿"], + ["RR", "时间片轮转", "响应快、公平", "时间片选择敏感"], + ["MLFQ", "多级队列", "兼顾交互与吞吐", "参数调优复杂"], + ], +} + +FORMULA_PAGE = { + "title": "关键公式", + "lines": [ + "有效访问时间 = α × (缺页处理时间) + (1 − α) × (内存访问时间)", + "页错误率上限：EAT ≤ 基准访问时间 × (1 + p × t)", + "时钟算法命中率 ≈ 1 − (1 − h)^k", + "工作集窗口：WS(t, Δ) = { 页 x | 最近 Δ 次引用中出现过 x }", + ], +} + +CHART_PAGE = { + "title": "多级队列调度下的吞吐量", + "series": [12, 18, 24, 27, 26, 22, 17], +} + +SCANNED_PAGES = [ + ( + "扫描页：课堂笔记", + [ + "课堂记录：就绪队列过长时，交互进程的响应时间会明显变差。", + "课堂记录：时间片并不是越大越好，过大的时间片会退化为 FCFS。", + "课堂记录：缺页率与工作集大小呈非线性关系。", + ], + ), +] + +TAIL_SECTIONS = [ + ( + "进程通信", + [ + "进程间通信的两种基本模型是共享内存和消息传递：共享内存快但需要用户自己处理同步，消息传递慢但由内核保证互斥。", + "管道是最常见的消息传递形式，匿名管道用于父子进程，命名管道允许无亲缘关系的进程交换数据。", + ], + ), + ( + "死锁", + [ + "死锁的四个必要条件是互斥、持有并等待、不可抢占和循环等待；破坏任意一个条件即可预防死锁。", + "银行家算法通过安全性检查判断一次资源分配是否可能进入不安全状态，从而避免死锁。", + ], + ), + ( + "同步机制", + [ + "信号量用原子 P/V 操作控制资源计数，互斥锁保证临界区互斥进入，条件变量用于等待复杂条件成立。", + "管程把共享数据和同步操作封装在一个模块内，编译器保证同一时刻只有一个线程活跃在管程内。", + ], + ), + ( + "I/O 控制", + [ + "程序直接控制占用 CPU 轮询状态，中断驱动让 CPU 在设备就绪时才介入，DMA 让整块数据的传送脱离 CPU。", + "设备驱动向上屏蔽硬件差异，把统一的读写请求翻译成设备寄存器操作。", + ], + ), +] + + +def _cjk_text(page: pymupdf.Page, y: float, text: str, size: float = 11, width: float = 500) -> float: + """Insert wrapped CJK text, return the y position after the block.""" + font = "china-s" + x, line_height = 60.0, size * 1.75 + line = "" + for char in text: + if y > 760: + break + if pymupdf.get_text_length(char, fontname=font, fontsize=size) + pymupdf.get_text_length( + line, fontname=font, fontsize=size + ) > width: + page.insert_text((x, y), line, fontname=font, fontsize=size) + y += line_height + line = char + else: + line += char + if line: + page.insert_text((x, y), line, fontname=font, fontsize=size) + y += line_height + return y + + +def _page_title(page: pymupdf.Page, title: str) -> float: + page.insert_text((60, 80), title, fontname="china-s", fontsize=17) + return 120.0 + + +def _text_dense_page(doc: pymupdf.Page | None, doc_handle, title: str, paragraphs: list[str], page_no: int) -> None: + page = doc_handle.new_page() + page.insert_text((40, 40), f"{page_no}", fontname="helv", fontsize=9) + y = _page_title(page, title) + for paragraph in paragraphs: + y = _cjk_text(page, y, paragraph) + y += 10 + + +def build_sample_pdf(path: Path) -> Path: + doc = pymupdf.open() + + # 1: front matter (title page) + page = doc.new_page() + page.insert_text((60, 200), COURSE_TITLE, fontname="china-s", fontsize=22) + page.insert_text((60, 250), "基准样例：固定样本集 / 用于可复现评测", fontname="china-s", fontsize=12) + + # 2-5: text-dense sections + page_no = 2 + for title, paragraphs in TEXT_DENSE_SECTIONS: + _text_dense_page(None, doc, title, paragraphs, page_no) + page_no += 1 + + # 6: table page + page = doc.new_page() + page.insert_text((40, 40), f"{page_no}", fontname="helv", fontsize=9) + y = _page_title(page, TABLE_PAGE["title"]) + col_x = [60, 170, 330, 460] + for index, header in enumerate(TABLE_PAGE["headers"]): + page.insert_text((col_x[index], y), header, fontname="china-s", fontsize=11) + y += 24 + for row in TABLE_PAGE["rows"]: + for index, cell in enumerate(row): + page.insert_text((col_x[index], y), cell, fontname="china-s", fontsize=10) + y += 22 + page_no += 1 + + # 7: formula page + page = doc.new_page() + page.insert_text((40, 40), f"{page_no}", fontname="helv", fontsize=9) + y = _page_title(page, FORMULA_PAGE["title"]) + for line in FORMULA_PAGE["lines"]: + y = _cjk_text(page, y, line, size=13) + page_no += 1 + + # 8: chart page (vector line chart) + page = doc.new_page() + page.insert_text((40, 40), f"{page_no}", fontname="helv", fontsize=9) + y = _page_title(page, CHART_PAGE["title"]) + origin = pymupdf.Point(80, 600) + page.draw_line(origin, origin + pymupdf.Point(440, 0)) + page.draw_line(origin, origin + pymupdf.Point(0, -320)) + step = 440 / (len(CHART_PAGE["series"]) - 1) + points = [ + pymupdf.Point(origin.x + index * step, origin.y - value * 11) + for index, value in enumerate(CHART_PAGE["series"]) + ] + page.draw_polyline(points, color=(0.1, 0.3, 0.7), width=1.6) + for index, point in enumerate(points): + page.draw_circle(point, 2.5, color=(0.7, 0.2, 0.1), fill=(0.7, 0.2, 0.1)) + page.insert_text((point.x - 6, origin.y + 18), f"T{index + 1}", fontname="helv", fontsize=8) + page.insert_text((origin.x, origin.y + 40), "观测窗口（单位时间）", fontname="china-s", fontsize=10) + page_no += 1 + + # 9-10: scanned pages: rasterize a rendered text page and embed as image only + for title, lines in SCANNED_PAGES: + temp = doc.new_page() + y = _page_title(temp, title) + for line in lines: + y = _cjk_text(temp, y, line) + pix = temp.get_pixmap(dpi=72) + temp_number = temp.number + scanned = doc.new_page() + scanned.insert_image(scanned.rect, pixmap=pix) + doc.delete_page(temp_number) + page_no += 1 + + # 11-20: tail sections + repeated exercise pages to make the deck long + for section in TAIL_SECTIONS: + _text_dense_page(None, doc, section[0], section[1], page_no) + page_no += 1 + for index in range(1, 6): + _text_dense_page( + None, + doc, + f"练习与思考 {index}", + [ + f"第 {index} 组练习：给定一组到达时间和估计运行时间，分别计算 FCFS、SJF 和 RR（时间片 = 2）的平均周转时间，并比较结论是否随时间片变化。", + f"第 {index} 组练习：假设某进程的页访问序列已知，用 LRU 和时钟算法分别模拟缺页次数，解释两种策略产生差异的页面是哪些。", + ], + page_no, + ) + page_no += 1 + + doc.save(path, deflate=True) + doc.close() + return path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=Path("benchmarks/samples/synthetic_course.pdf")) + args = parser.parse_args() + args.out.parent.mkdir(parents=True, exist_ok=True) + build_sample_pdf(args.out) + doc = pymupdf.open(args.out) + print(f"sample deck written: {args.out} ({len(doc)} pages)") + doc.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_note_layout.py b/scripts/verify_note_layout.py new file mode 100644 index 0000000..2063861 --- /dev/null +++ b/scripts/verify_note_layout.py @@ -0,0 +1,383 @@ +"""Verify the reading layout of generated SlideNote notes (ROADMAP P0-2). + +Checks three layers for one or more SlideNote output directories and reports +every issue with the sample, file and page it belongs to: + +- Markdown (notes.md): heading hierarchy jumps, duplicate H1, empty sections, + images without alt/nearby explanation, oversized tables, unbalanced code + fences or math delimiters, missing source markers. +- Word (notes.docx, built by pandoc): heading/table/image counts against the + Markdown source, and per-table column consistency (misaligned rows cause + broken tables in Word). +- PDF (notes.pdf, built from docx via LibreOffice): clipped or overflowing + text blocks, image/text overlap (misplacement), empty pages and headings + stranded at the bottom of a page (orphan headings), per page. + +Missing export files are reported as ``skipped`` with the reason, so the tool +works on runs that only requested markdown-zip and on machines without +pandoc/LibreOffice. + +Usage: + python scripts/verify_note_layout.py outputs/lecture [--json report.json] + python scripts/verify_note_layout.py benchmarks/runs//outputs/* + +Exit codes: 0 = no errors; 1 = at least one error (or warning with --strict). +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import zipfile +import xml.etree.ElementTree as ET +from pathlib import Path + +import pymupdf + +WORD_NS = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +HEADING_STYLES = {"Heading1", "Heading2", "Heading3", "Heading4", "heading 1", "heading 2", "heading 3", "heading 4"} +MAX_TABLE_COLUMNS = 10 +MAX_TABLE_CELL_CHARS = 120 +IMAGE_EXPLANATION_MIN_CHARS = 30 +PAGE_MARGIN_TOLERANCE = 2.0 +IMAGE_TEXT_OVERLAP_LIMIT = 0.30 + + +def _visible(text: str) -> str: + text = re.sub(r"", "", text, flags=re.DOTALL) + text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", text) + return text.strip() + + +def _mask_fenced_code(text: str) -> tuple[str, bool]: + """Hide code syntax from layout checks while preserving source offsets.""" + masked_lines: list[str] = [] + fence_marker = "" + fence_length = 0 + for line in text.splitlines(keepends=True): + marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line.rstrip("\r\n")) + if fence_marker: + if ( + marker + and marker.group(1)[0] == fence_marker + and len(marker.group(1)) >= fence_length + and not marker.group(2).strip() + ): + fence_marker = "" + masked_lines.append(re.sub(r"[^\r\n]", " ", line)) + elif marker and (marker.group(1)[0] == "~" or "`" not in marker.group(2)): + fence_marker = marker.group(1)[0] + fence_length = len(marker.group(1)) + masked_lines.append(re.sub(r"[^\r\n]", " ", line)) + else: + masked_lines.append(line) + return "".join(masked_lines), bool(fence_marker) + + +def check_markdown(notes_path: Path) -> tuple[list[dict], dict]: + issues: list[dict] = [] + text = notes_path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines() + layout_text, unclosed_fence = _mask_fenced_code(text) + layout_lines = layout_text.splitlines() + sample = notes_path.parent.name + stats = {"headings": 0, "h1": 0, "images": 0, "tables": 0, "chars": len(_visible(text))} + + h1_count = sum(1 for line in layout_lines if re.match(r"^#\s+\S", line)) + stats["h1"] = h1_count + if h1_count == 0: + issues.append(issue("error", sample, "notes.md", None, "missing_h1", "文档缺少一个 H1 标题。")) + elif h1_count > 1: + issues.append(issue("error", sample, "notes.md", None, "multiple_h1", f"发现 {h1_count} 个 H1，应只有一个课程标题。")) + + headings = [(index, len(match.group(1)), match.group(2).strip()) for index, line in enumerate(layout_lines) if (match := re.match(r"^(#{1,6})\s+(.*)$", line))] + heading_lines = {index for index, _level, _title in headings} + stats["headings"] = len(headings) + previous_level = 0 + for index, level, title in headings: + if previous_level and level > previous_level + 1: + issues.append(issue( + "error", sample, "notes.md", None, "heading_level_jump", + f"第 {index + 1} 行标题“{title}”从 H{previous_level} 直接跳到 H{level}。", + )) + previous_level = level + + for pos, (index, level, title) in enumerate(headings): + if level == 1: + # The document H1 is normally followed directly by section headings. + continue + # A parent section may legitimately start with a child heading. Treat + # the whole nested section as its body until the next heading at the + # same or a higher level, and ignore heading text itself. + end = len(lines) + for next_index, next_level, _next_title in headings[pos + 1:]: + if next_level <= level: + end = next_index + break + # Code examples are section content, even when their lines resemble + # headings. Remove only the real Markdown headings identified above. + section_text = "\n".join(lines[line_index] for line_index in range(index + 1, end) if line_index not in heading_lines) + body = _visible(section_text) + if not body: + issues.append(issue( + "warning", sample, "notes.md", None, "empty_section", + f"标题“{title}”（第 {index + 1} 行）下没有正文内容。", + )) + + if unclosed_fence: + issues.append(issue("error", sample, "notes.md", None, "unbalanced_code_fence", "存在未闭合的代码围栏；结束围栏须使用相同字符且长度不少于开始围栏。")) + if layout_text.count("$$") % 2: + issues.append(issue("error", sample, "notes.md", None, "unbalanced_math", "$$ 定界符数量为奇数，公式块可能未闭合。")) + + image_positions = [match for match in re.finditer(r"(?m)^!\[([^\]]*)\]\(([^)]+)\)", layout_text)] + stats["images"] = len(image_positions) + for match in image_positions: + alt, target = match.group(1).strip(), match.group(2).strip() + line_no = layout_text.count("\n", 0, match.start()) + 1 + if not alt: + issues.append(issue("error", sample, "notes.md", None, "image_missing_alt", f"第 {line_no} 行图片缺少 alt 文本：{target}")) + window = _visible(layout_text[match.end(): match.end() + 600]) + before = _visible(layout_text[max(0, match.start() - 600): match.start()]) + if len(window) < IMAGE_EXPLANATION_MIN_CHARS and len(before) < IMAGE_EXPLANATION_MIN_CHARS: + issues.append(issue( + "warning", sample, "notes.md", None, "image_without_explanation", + f"第 {line_no} 行图片“{alt or target}”前后 600 字符内没有讲解文字。", + )) + + table_rows: list[list[list[str]]] = [] + table_starts: list[int] = [] + previous_table_row: int | None = None + for index, line in enumerate(layout_lines): + if re.match(r"^\s*\|.*\|\s*$", line): + if previous_table_row is None or index != previous_table_row + 1: + table_starts.append(index) + table_rows.append([]) + table_rows[-1].append([cell.strip() for cell in line.strip().strip("|").split("|")]) + previous_table_row = index + stats["tables"] = len(table_rows) + for table_index, rows in enumerate(table_starts and table_rows or []): + if not rows: + continue + columns = len(rows[0]) + line_no = table_starts[table_index] + 1 + if columns > MAX_TABLE_COLUMNS: + issues.append(issue( + "warning", sample, "notes.md", None, "wide_table", + f"第 {line_no} 行表格有 {columns} 列（>{MAX_TABLE_COLUMNS}），导出后大概率横向溢出。", + )) + for row_index, row in enumerate(rows): + row_line_no = table_starts[table_index] + row_index + 1 + if len(row) != columns: + issues.append(issue( + "error", sample, "notes.md", None, "ragged_table_row", + f"第 {row_line_no} 行表格行列数不一致（{len(row)} != {columns}）。", + )) + break + for cell in row: + if len(cell) > MAX_TABLE_CELL_CHARS: + issues.append(issue( + "warning", sample, "notes.md", None, "oversized_table_cell", + f"第 {line_no} 行表格单元格超过 {MAX_TABLE_CELL_CHARS} 字符，导出后可能断页错位。", + )) + break + + if "slidenote-source:" not in text and stats["chars"] > 800: + issues.append(issue( + "warning", sample, "notes.md", None, "no_source_markers", + "笔记没有可追溯来源标记；如为 lecture 产物请检查 source_display 配置。", + )) + return issues, stats + + +def check_docx(docx_path: Path, markdown_stats: dict | None) -> list[dict]: + issues: list[dict] = [] + sample = docx_path.parent.name + try: + with zipfile.ZipFile(docx_path) as archive: + document = archive.read("word/document.xml") + media = [name for name in archive.namelist() if name.startswith("word/media/")] + except (OSError, zipfile.BadZipFile, KeyError) as exc: + return [issue("error", sample, "notes.docx", None, "docx_unreadable", f"无法读取 docx：{type(exc).__name__}: {exc}")] + + root = ET.fromstring(document) + body = root.find(f"{WORD_NS}body") + headings = tables = 0 + for element in body.iter(): + if element.tag == f"{WORD_NS}p": + style = element.find(f"{WORD_NS}pPr/{WORD_NS}pStyle") + if style is not None and style.get(f"{WORD_NS}val") in HEADING_STYLES: + headings += 1 + elif element.tag == f"{WORD_NS}tbl": + tables += 1 + if markdown_stats and markdown_stats.get("headings") and headings == 0: + issues.append(issue("error", sample, "notes.docx", None, "docx_no_headings", "Word 文档没有任何标题样式，标题层级在导出中丢失。")) + if markdown_stats and markdown_stats.get("tables") and tables < markdown_stats["tables"]: + issues.append(issue( + "warning", sample, "notes.docx", None, "docx_table_count_mismatch", + f"Word 表格数（{tables}）少于 Markdown（{markdown_stats['tables']}），可能有表格未导出。", + )) + if markdown_stats and markdown_stats.get("images") and len(media) < markdown_stats["images"]: + issues.append(issue( + "warning", sample, "notes.docx", None, "docx_image_count_mismatch", + f"Word 内嵌图片数（{len(media)}）少于 Markdown（{markdown_stats['images']}），可能有图片丢失。", + )) + + for table_index, table in enumerate([element for element in body.iter(f"{WORD_NS}tbl")]): + rows = table.findall(f"{WORD_NS}tr") + widths = [] + for row in rows: + cells = row.findall(f"{WORD_NS}tc") + widths.append(len(cells)) + if widths and (min(widths) != max(widths)): + issues.append(issue( + "error", sample, "notes.docx", None, "docx_ragged_table", + f"Word 第 {table_index + 1} 个表格行内单元格数不一致（{sorted(set(widths))}），渲染会错位。", + )) + return issues + + +def _rects_overlap_area(a: pymupdf.Rect, b: pymupdf.Rect) -> float: + inter = a & b + if inter.is_empty or inter.width <= 0 or inter.height <= 0: + return 0.0 + smaller = min(a.get_area(), b.get_area()) + return (inter.get_area() / smaller) if smaller else 0.0 + + +def check_pdf(pdf_path: Path) -> list[dict]: + issues: list[dict] = [] + sample = pdf_path.parent.name + try: + doc = pymupdf.open(pdf_path) + except Exception as exc: + return [issue("error", sample, "notes.pdf", None, "pdf_unreadable", f"无法打开 PDF：{type(exc).__name__}: {exc}")] + try: + if len(doc) == 0: + return [issue("error", sample, "notes.pdf", None, "pdf_empty", "PDF 没有任何页面。")] + for page_index, page in enumerate(doc): + page_no = page_index + 1 + page_rect = page.rect + blocks = [block for block in page.get_text("dict")["blocks"]] + text_blocks = [block for block in blocks if block["type"] == 0] + image_blocks = [block for block in blocks if block["type"] == 1] + if not text_blocks and not image_blocks: + issues.append(issue("error", sample, "notes.pdf", page_no, "blank_page", "页面没有任何文本或图片。")) + continue + for block in text_blocks: + bbox = pymupdf.Rect(block["bbox"]) + if ( + bbox.x0 < page_rect.x0 - PAGE_MARGIN_TOLERANCE + or bbox.y0 < page_rect.y0 - PAGE_MARGIN_TOLERANCE + or bbox.x1 > page_rect.x1 + PAGE_MARGIN_TOLERANCE + or bbox.y1 > page_rect.y1 + PAGE_MARGIN_TOLERANCE + ): + snippet = " ".join(span["text"] for line in block["lines"] for span in line["spans"])[:40] + issues.append(issue( + "error", sample, "notes.pdf", page_no, "text_overflow", + f"文本块超出页面边界（{bbox}）：“{snippet}…”", + )) + for image in image_blocks: + image_rect = pymupdf.Rect(image["bbox"]) + for block in text_blocks: + if _rects_overlap_area(image_rect, pymupdf.Rect(block["bbox"])) > IMAGE_TEXT_OVERLAP_LIMIT: + snippet = " ".join(span["text"] for line in block["lines"] for span in line["spans"])[:30] + issues.append(issue( + "warning", sample, "notes.pdf", page_no, "image_text_overlap", + f"图片与文本“{snippet}…”重叠超过 {IMAGE_TEXT_OVERLAP_LIMIT:.0%}，疑似图文错位。", + )) + break + if text_blocks: + last = text_blocks[-1] + spans = [span for line in last["lines"] for span in line["spans"]] + if spans: + max_size = max(span["size"] for span in spans) + sizes = [span["size"] for block in text_blocks for line in block["lines"] for span in line["spans"]] + body_size = sorted(sizes)[len(sizes) // 2] + if max_size >= body_size * 1.35 and last["bbox"][3] > page_rect.y1 * 0.92: + text = " ".join(span["text"] for span in spans)[:30] + issues.append(issue( + "warning", sample, "notes.pdf", page_no, "orphan_heading", + f"页尾最后一段是大号标题“{text}…”，内容被分到下一页。", + )) + finally: + doc.close() + return issues + + +def issue(level: str, sample: str, file: str, page: int | None, rule: str, detail: str) -> dict: + return {"level": level, "sample": sample, "file": file, "page": page, "rule": rule, "detail": detail} + + +def verify_output_dir(output_dir: Path) -> list[dict]: + issues: list[dict] = [] + notes = output_dir / "notes.md" + if not notes.exists(): + return [issue("error", output_dir.name, "notes.md", None, "notes_missing", f"{output_dir} 缺少 notes.md，请先运行 build。")] + markdown_issues, markdown_stats = check_markdown(notes) + issues.extend(markdown_issues) + + docx = output_dir / "notes.docx" + pdf = output_dir / "notes.pdf" + if docx.exists(): + issues.extend(check_docx(docx, markdown_stats)) + else: + issues.append(issue("skipped", output_dir.name, "notes.docx", None, "docx_missing", "未请求 docx 导出或 pandoc 缺失，Word 层检查跳过。")) + if pdf.exists(): + issues.extend(check_pdf(pdf)) + else: + reason = "未请求 pdf 导出；或 LibreOffice 缺失导致 docx→pdf 转换未生成。" + issues.append(issue("skipped", output_dir.name, "notes.pdf", None, "pdf_missing", reason)) + return issues + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("outputs", type=Path, nargs="+", help="SlideNote build 输出目录（可传多个或 runs//outputs 下的子目录）") + parser.add_argument("--json", type=Path, default=None, help="同时把结构化报告写入该文件") + parser.add_argument("--strict", action="store_true", help="warning 也视为失败") + args = parser.parse_args() + + dirs: list[Path] = [] + for path in args.outputs: + if path.is_dir() and (path / "notes.md").exists(): + dirs.append(path) + elif path.is_dir(): + dirs.extend(child for child in sorted(path.iterdir()) if child.is_dir() and (child / "notes.md").exists()) + if not dirs: + print("没有找到包含 notes.md 的输出目录。", file=sys.stderr) + return 2 + + all_issues: list[dict] = [] + for directory in dirs: + all_issues.extend(verify_output_dir(directory)) + + errors = [item for item in all_issues if item["level"] == "error"] + warnings = [item for item in all_issues if item["level"] == "warning"] + skipped = [item for item in all_issues if item["level"] == "skipped"] + + report = {"issues": all_issues, "summary": {"samples": len(dirs), "errors": len(errors), "warnings": len(warnings), "skipped": len(skipped)}} + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + print(f"检查样本：{len(dirs)}；错误 {len(errors)}；警告 {len(warnings)}；跳过 {len(skipped)}") + for item in all_issues: + if item["level"] == "skipped": + continue + where = f"{item['file']}" + (f" 第 {item['page']} 页" if item["page"] else "") + print(f"[{item['level'].upper():7s}] {item['sample']} / {where} / {item['rule']}: {item['detail']}") + for item in skipped: + print(f"[skipped] {item['sample']} / {item['rule']}: {item['detail']}") + + failed = bool(errors) or (args.strict and bool(warnings)) + if failed: + print("排版验收未通过。" + ("" if not args.strict else "（--strict：警告也算失败）")) + else: + print("排版验收通过（启发式检查；渲染结论仍建议人工翻阅一次导出文件）。") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/slidenote/build/config.py b/slidenote/build/config.py index a4050e4..c28db32 100644 --- a/slidenote/build/config.py +++ b/slidenote/build/config.py @@ -155,7 +155,7 @@ def _apply_note_profile_defaults(args: argparse.Namespace) -> None: "api_key": None, "base_url": None, "max_output_tokens": 12000, - "temperature": 0.2, + "temperature": 0.0, "content_guard": "auto", "export_toc": "auto", "asset_mode": "bundle", @@ -168,7 +168,7 @@ def _apply_note_profile_defaults(args: argparse.Namespace) -> None: "note_strategy": "lecture-weave", "note_depth": None, "teaching_enrichment": "auto", - "weave_dedup": "soft", + "weave_dedup": "normal", "page_neighborhood": 1, "deck_brief": "auto", "screenshot_policy": "fallback", diff --git a/slidenote/notes/__init__.py b/slidenote/notes/__init__.py index 62ff57a..7e0c3c1 100644 --- a/slidenote/notes/__init__.py +++ b/slidenote/notes/__init__.py @@ -63,7 +63,7 @@ class NoteOptions: note_language: str = "zh" term_policy: str = "bilingual" teaching_enrichment: str = "auto" - weave_dedup: str = "soft" + weave_dedup: str = "normal" page_neighborhood: int = 1 screenshot_policy: str = "fallback" figure_placement: str = "inline" @@ -129,7 +129,7 @@ def generate_notes( note_language: str = "zh", term_policy: str = "bilingual", teaching_enrichment: str = "auto", - weave_dedup: str = "soft", + weave_dedup: str = "normal", page_neighborhood: int = 1, screenshot_policy: str = "fallback", figure_placement: str = "inline", diff --git a/slidenote/notes/document_frame.py b/slidenote/notes/document_frame.py new file mode 100644 index 0000000..a96efad --- /dev/null +++ b/slidenote/notes/document_frame.py @@ -0,0 +1,202 @@ + +import re +from typing import Any + +from .structure import LECTURE_NOTE_STRUCTURE_SLOTS, _heading_matches, _heading_sections + + +_GLOBAL_HEADINGS_ZH = ("本讲目标", "本讲总结", "章节自测") +_GLOBAL_HEADINGS_EN = ("Learning Objectives", "Lecture Summary", "Review Questions") + + +def ensure_lecture_document_frame( + markdown: str, + deck_brief: dict[str, Any] | None, + note_language: str, +) -> str: + """Add a small, source-derived document frame without rewriting chapters. + + The operation is idempotent. It never modifies chapter prose, code blocks, + images, or source markers; it only adds missing document-level sections. + """ + + text = markdown.strip() + if not text: + return markdown + + headings = _h2_titles(text) + objective_title, summary_title, review_title = ( + _GLOBAL_HEADINGS_EN if note_language == "en" else _GLOBAL_HEADINGS_ZH + ) + chapters = [title for title in headings if not _is_global_heading(title, note_language)] + brief = _brief_payload(deck_brief) + outline = _outline_entries(brief) + topic_names = _topic_names(chapters, outline, brief) + + if not chapters: + topic_title = topic_names[0] if topic_names else ("Core Content" if note_language == "en" else "核心内容") + text = _insert_before_first_body(text, f"## {topic_title}\n\n") + chapters = [topic_title] + + if not _has_global_h2(text, "orientation"): + objectives = _objective_items(brief, topic_names, note_language) + section = f"## {objective_title}\n\n" + "\n".join(f"- {item}" for item in objectives) + "\n\n" + text = _insert_after_h1(text, section) + + if not _has_global_h2(text, "summary"): + summaries = _summary_items(outline, topic_names, note_language) + text = text.rstrip() + f"\n\n## {summary_title}\n\n" + "\n".join(f"- {item}" for item in summaries) + + if not _has_global_h2(text, "self_test"): + questions = _review_items(brief, topic_names, note_language) + text = text.rstrip() + f"\n\n## {review_title}\n\n" + "\n".join( + f"{index}. {item}" for index, item in enumerate(questions, start=1) + ) + + return text.rstrip() + "\n" + + +def _brief_payload(report: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(report, dict): + return {} + brief = report.get("brief") + return brief if isinstance(brief, dict) else report + + +def _outline_entries(brief: dict[str, Any]) -> list[dict[str, str]]: + raw = brief.get("chapter_outline") + if not isinstance(raw, list): + return [] + result: list[dict[str, str]] = [] + for item in raw: + if not isinstance(item, dict): + continue + title = str(item.get("title") or "").strip() + summary = str(item.get("summary") or "").strip() + if title: + result.append({"title": title, "summary": summary}) + return result + + +def _topic_names(chapters: list[str], outline: list[dict[str, str]], brief: dict[str, Any]) -> list[str]: + names = [_clean_numbering(title) for title in chapters if _usable_topic(title)] + names.extend(item["title"] for item in outline if _usable_topic(item["title"])) + if not names: + names.extend( + str(item.get("term") or "").strip() + for item in brief.get("key_concepts") or [] + if isinstance(item, dict) and str(item.get("term") or "").strip() + ) + return _unique(names, limit=10) + + +def _objective_items(brief: dict[str, Any], topics: list[str], note_language: str) -> list[str]: + questions = _clean_strings(brief.get("core_questions"), limit=6) + if questions: + return questions + if note_language == "en": + return [f"Explain the central idea, conditions, and use of {topic}." for topic in topics[:6]] or [ + "Explain the lecture's central concepts and how they connect." + ] + return [f"理解“{topic}”的核心思想、适用条件与作用。" for topic in topics[:6]] or [ + "理解本讲核心概念及其前后关系。" + ] + + +def _summary_items(outline: list[dict[str, str]], topics: list[str], note_language: str) -> list[str]: + items: list[str] = [] + for entry in outline: + if not entry["summary"] or not _usable_topic(entry["title"]): + continue + items.append(f"**{entry['title']}**：{entry['summary']}") + if len(items) >= 8: + break + if items: + return items + if note_language == "en": + return [f"Review the core conclusion of **{topic}**." for topic in topics[:8]] or [ + "Review the lecture's central line of reasoning." + ] + return [f"回顾 **{topic}** 的核心结论及其适用条件。" for topic in topics[:8]] or [ + "回顾本讲的核心知识链与关键条件。" + ] + + +def _review_items(brief: dict[str, Any], topics: list[str], note_language: str) -> list[str]: + questions = [_as_question(item, note_language) for item in _clean_strings(brief.get("core_questions"), limit=6)] + if questions: + return questions + if note_language == "en": + return [f"What problem does {topic} solve, and what conditions does it require?" for topic in topics[:6]] or [ + "Can you reconstruct the lecture's main reasoning without looking at the notes?" + ] + return [f"“{topic}”解决了什么问题，又依赖哪些条件？" for topic in topics[:6]] or [ + "你能不看笔记复述本讲的主要知识链吗？" + ] + + +def _insert_after_h1(markdown: str, section: str) -> str: + h1 = next((heading for heading in _heading_sections(markdown) if heading["level"] == 1), None) + if not h1: + return section + markdown.lstrip() + return markdown[: h1["end"]].rstrip() + "\n\n" + section + markdown[h1["end"] :].lstrip("\r\n") + + +def _insert_before_first_body(markdown: str, heading: str) -> str: + return _insert_after_h1(markdown, heading) + + +def _h2_titles(markdown: str) -> list[str]: + return [heading["title"] for heading in _heading_sections(markdown) if heading["level"] == 2] + + +def _has_global_h2(markdown: str, slot: str) -> bool: + return any(_heading_matches(title, LECTURE_NOTE_STRUCTURE_SLOTS[slot]) for title in _h2_titles(markdown)) + + +def _is_global_heading(title: str, note_language: str) -> bool: + del note_language + return any( + _heading_matches(title, LECTURE_NOTE_STRUCTURE_SLOTS[slot]) + for slot in ("orientation", "summary", "self_test") + ) + + +def _usable_topic(title: str) -> bool: + normalized = _normalize(_clean_numbering(title)) + banned = {"", "welcome", "welcome!", "introduction", "引言", "欢迎", "目录", "contents", "总结", "summing up"} + return normalized not in banned and not normalized.startswith("第 ") + + +def _clean_numbering(title: str) -> str: + return re.sub(r"^\s*(?:\d+|[一二三四五六七八九十百]+)[.、\s-]+", "", title).strip() + + +def _normalize(value: str) -> str: + return re.sub(r"[\s：:—\-_/（）()]+", " ", value).strip().lower() + + +def _clean_strings(value: Any, limit: int) -> list[str]: + if not isinstance(value, list): + return [] + return _unique([str(item).strip() for item in value if isinstance(item, str) and item.strip()], limit=limit) + + +def _unique(items: list[str], limit: int) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for item in items: + key = _normalize(item) + if not key or key in seen: + continue + seen.add(key) + result.append(item) + if len(result) >= limit: + break + return result + + +def _as_question(text: str, note_language: str) -> str: + if text.endswith(("?", "？")): + return text + return text + ("?" if note_language == "en" else "？") diff --git a/slidenote/notes/finalize.py b/slidenote/notes/finalize.py index 3efaea7..410e213 100644 --- a/slidenote/notes/finalize.py +++ b/slidenote/notes/finalize.py @@ -11,8 +11,9 @@ from .assets import _repair_markdown_image_links from .compose import _compose_final_markdown from .contexts import NoteContext +from .document_frame import ensure_lecture_document_frame from .figure_blocks import _ensure_grounded_figures -from .repair import _repair_required_markdown_once +from .repair import _repair_note_structure_once, _repair_required_markdown_once def _finalize_notes_markdown( @@ -31,6 +32,11 @@ def _finalize_notes_markdown( Repairs run per context (never on the whole deck) so the rewrite fits the output-token budget; accepted repairs replace the chunk in ``final_chunks``. Returns the final markdown and the LLM usage records of repair calls. + + For the lecture-weave strategy the deterministic document frame (objectives, + summary, self-test) is appended after composition, and a whole-document + structure repair runs only when the frame could not make the lecture + structure contract pass. """ def compose() -> str: @@ -45,37 +51,63 @@ def compose() -> str: return _ensure_grounded_figures(markdown, deck, asset_map, options.source_display, options.figure_placement) markdown = compose() + repair_usage: list[dict[str, Any]] = [] content_guard = options.content_guard - if not content_guard: - return markdown, [] - coverage = analyze_coverage(deck, markdown, content_guard=content_guard) - missing_slide_ids = {item.get("slide_id") for item in missing_required_items(content_guard, coverage)} - if not missing_slide_ids: - return markdown, [] + if content_guard: + coverage = analyze_coverage(deck, markdown, content_guard=content_guard) + missing_slide_ids = {item.get("slide_id") for item in missing_required_items(content_guard, coverage)} + if missing_slide_ids: + refresh_ids = options.refresh_slide_ids or set() + changed = False + for context in contexts: + slide_ids = {page.slide_id for page in context.pages} + if not slide_ids & missing_slide_ids or not final_chunks.get(context.id, "").strip(): + continue + repaired, record = _repair_required_markdown_once( + deck=Deck(source_path=deck.source_path, source_type=deck.source_type, pages=list(context.pages)), + context=context, + markdown=final_chunks[context.id], + output_root=output_root, + cache=cache, + options=options, + stage=stage, + force_refresh=bool(refresh_ids & slide_ids), + ) + if record is None: + continue + record_repair(content_guard, record) + if isinstance(record.get("llm"), dict): + repair_usage.append(record["llm"]) + if record.get("accepted"): + final_chunks[context.id] = repaired + changed = True + if changed: + markdown = compose() - refresh_ids = options.refresh_slide_ids or set() - repair_usage: list[dict[str, Any]] = [] - changed = False - for context in contexts: - slide_ids = {page.slide_id for page in context.pages} - if not slide_ids & missing_slide_ids or not final_chunks.get(context.id, "").strip(): - continue - repaired, record = _repair_required_markdown_once( - deck=Deck(source_path=deck.source_path, source_type=deck.source_type, pages=list(context.pages)), - context=context, - markdown=final_chunks[context.id], - output_root=output_root, - cache=cache, - options=options, - stage=stage, - force_refresh=bool(refresh_ids & slide_ids), - ) - if record is None: - continue - record_repair(content_guard, record) + if options.note_strategy != "lecture-weave": + return markdown, repair_usage + + markdown = ensure_lecture_document_frame(markdown, options.deck_brief, options.note_language) + if not options.use_llm: + return markdown, repair_usage + repaired, record = _repair_note_structure_once( + deck=deck, + context=NoteContext(id="final", kind="final", title="final", pages=deck.pages), + markdown=markdown, + output_root=output_root, + cache=cache, + options=options, + ) + if record is not None: if isinstance(record.get("llm"), dict): repair_usage.append(record["llm"]) if record.get("accepted"): - final_chunks[context.id] = repaired - changed = True - return (compose() if changed else markdown), repair_usage + markdown = _repair_markdown_image_links(repaired, output_root, asset_map) + markdown = _ensure_grounded_figures( + markdown, + deck, + asset_map, + options.source_display, + options.figure_placement, + ) + return markdown, repair_usage diff --git a/slidenote/notes/postprocess.py b/slidenote/notes/postprocess.py index 4508d5f..847e20b 100644 --- a/slidenote/notes/postprocess.py +++ b/slidenote/notes/postprocess.py @@ -8,14 +8,26 @@ def _postprocess_llm_markdown(markdown: str, source_display: str) -> str: - text = _unwrap_code_images(markdown) - text = _fill_empty_image_alts(text) - text = _remove_meta_paragraphs(text) + text = _clean_llm_markdown(markdown) text = _normalize_chunk_headings(text) text = _convert_visible_sources(text, source_display) return text.strip() +def _postprocess_document_markdown(markdown: str, source_display: str) -> str: + """Clean a full-document rewrite without changing its heading hierarchy.""" + text = _clean_llm_markdown(markdown) + text = _convert_visible_sources(text, source_display) + return text.strip() + + +def _clean_llm_markdown(markdown: str) -> str: + text = _unwrap_code_images(markdown) + text = _fill_empty_image_alts(text) + text = _remove_meta_paragraphs(text) + return text + + def _unwrap_code_images(markdown: str) -> str: return re.sub(r"`(!\[[^\]]*]\([^)]+\))`", r"\1", markdown) diff --git a/slidenote/notes/prompt_templates.py b/slidenote/notes/prompt_templates.py index b3d3a74..8ac682f 100644 --- a/slidenote/notes/prompt_templates.py +++ b/slidenote/notes/prompt_templates.py @@ -20,6 +20,21 @@ _source_prompt_rule, _term_policy_prompt_rule, ) +from .structure import lecture_note_structure_prompt_rule, lecture_section_style_prompt_rule + + +def _context_structure_prompt_rule(note_context: str, note_language: str) -> str: + if note_context != "document": + return lecture_section_style_prompt_rule(note_language) + # Composition supplies the document title after this body is generated. + rule = lecture_note_structure_prompt_rule(note_language) + return rule.replace( + "keep exactly one H1 title; ", + "the system supplies the sole H1 title, so do not emit H1; ", + ).replace( + "保留且只保留一个 H1 课程标题；", + "H1 课程标题由系统添加，本阶段不要输出 H1；", + ) def _llm_context_prompt( @@ -182,7 +197,12 @@ def _llm_page_lecture_prompt( } if prompt_brief: payload["deck_brief"] = prompt_brief - depth_rule = _note_depth_rule(note_depth) + depth_rule = { + "concise": "中间证据卡密度：只保留当前页不可替代的结论。", + "balanced": "中间证据卡密度：保留当前页新增的定义、条件、例子、公式和图表结论。", + "detailed": "中间证据卡密度：完整保留当前页独有证据，但不要把一个要点扩写成多组模板栏目。", + "very-detailed": "中间证据卡密度：不得遗漏当前页独有证据；详细讲解留给章节编织阶段，单页仍须短而准确。", + }[note_depth] profile_rule = _note_profile_prompt_rule(note_profile) source_rule = _source_prompt_rule(source_display) language_rule = _language_prompt_rule(note_language) @@ -193,9 +213,9 @@ def _llm_page_lecture_prompt( else "\u8bf7\u5c3d\u91cf\u4fdd\u7559 current_page \u539f\u59cb\u6761\u76ee\u987a\u5e8f\uff0c\u4f46\u4ecd\u8981\u5199\u6210\u53ef\u4ee5\u76f4\u63a5\u9605\u8bfb\u7684\u77e5\u8bc6\u70b9\u8bb2\u89e3\uff0c\u800c\u4e0d\u662f\u590d\u5236\u6e05\u5355\u6216\u63cf\u8ff0\u9875\u9762\u6784\u6210\u3002" ) style_depth_rule = ( - "\u91cd\u8981\uff1aarticle \u4e0d\u662f\u6458\u8981\u6a21\u5f0f\uff0c\u53ea\u6539\u53d8\u7ec4\u7ec7\u65b9\u5f0f\uff0c\u4e0d\u964d\u4f4e\u8bb2\u89e3\u6df1\u5ea6\uff1b\u5f53 note_depth=detailed \u65f6\uff0c\u9ed8\u8ba4\u5199\u6210\u8be6\u7ec6\u8bb2\u4e49\u5f0f\u5b66\u4e60\u7b14\u8bb0\uff0c\u4e0d\u8981\u628a\u6982\u5ff5\u3001\u516c\u5f0f\u3001\u4f8b\u5b50\u3001\u6761\u4ef6\u6216\u56fe\u8868\u7ed3\u8bba\u538b\u6210\u4e00\u53e5\u8bdd\u3002" + "article 在此阶段表示按知识增量整理证据，不表示为每页写一篇完整讲义；不得遗漏独有定义、条件、公式、代码或图表结论。" if note_style == "article" - else "\u91cd\u8981\uff1afaithful \u4f18\u5148\u4fdd\u7559 PPT \u987a\u5e8f\uff1b\u8bb2\u89e3\u8be6\u7ec6\u7a0b\u5ea6\u4ecd\u7531 note_depth \u63a7\u5236\u3002" + else "faithful 优先保留当前页顺序和独有细节，但仍然只输出供章节编织使用的简洁证据。" ) structural_rule = ( "\u5982\u679c current_page \u4e3b\u8981\u662f\u5c01\u9762\u3001\u76ee\u5f55\u3001\u7ae0\u8282\u5bfc\u822a\u3001\u8bb2\u5e08/\u8054\u7cfb\u65b9\u5f0f\u6216\u5176\u4ed6\u975e\u5b66\u4e60\u5185\u5bb9\uff0c" @@ -216,6 +236,12 @@ def _llm_page_lecture_prompt( "\u82e5 current_page.text_blocks \u542b style_runs/color\uff0c\u8fd9\u662f\u5df2\u62bd\u53d6\u7684\u5f69\u8272\u6587\u5b57\uff0c\u4e0d\u662f\u56fe\u7247\uff1b" "\u9700\u8981\u4fdd\u7559\u7ea2\u5b57\u5f3a\u8c03\u65f6\u7528 `...`\uff0c\u9ed1\u5b57\u7528\u666e\u901a\u6b63\u6587\u3002" ) + intermediate_rule = ( + "这是供后续章节编织使用的中间证据卡，不是最终讲义。只记录 current_page 新增且有材料依据的知识：" + "通常使用一到四个短段落；仅当代码、公式或表格确实需要时使用一个自然小标题。" + "不要为单页固定生成学习目标、核心概念、易错点、小结或自测问题，也不要重复 nearby_pages 已经讲过而 current_page 没有新增的信息。" + "代码和伪代码必须完整保真；材料不足时宁可只保留图片和来源标记，也不能凭常识补写具体结论。" + ) return ( "\u8bf7\u53ea\u8bb2\u89e3 JSON \u4e2d\u7684 current_page \u5305\u542b\u7684\u77e5\u8bc6\u5185\u5bb9\uff0c\u4e0d\u8981\u66ff\u5176\u4ed6\u9875\u9762\u5199\u6b63\u6587\u3002\n" "nearby_pages \u53ea\u7528\u4e8e\u7406\u89e3\u524d\u540e\u903b\u8f91\u548c\u51cf\u5c11\u91cd\u590d\uff0c\u4e0d\u80fd\u628a\u90bb\u8fd1\u9875\u5185\u5bb9\u5f53\u4f5c current_page \u7684\u5185\u5bb9\u5c55\u5f00\u3002\n" @@ -227,6 +253,7 @@ def _llm_page_lecture_prompt( f"{table_rule}\n" f"{semantic_rule}\n" f"{text_style_rule}\n" + f"{intermediate_rule}\n" f"{language_rule}\n" f"{term_rule}\n" f"{depth_rule}\n" @@ -288,14 +315,20 @@ def _llm_weave_prompt( profile_rule = _note_profile_prompt_rule(note_profile) language_rule = _language_prompt_rule(note_language) term_rule = _term_policy_prompt_rule(note_language, term_policy) + structure_rule = _context_structure_prompt_rule(note_context, note_language) + heading_rule = ( + "3. 全文正文使用 H2 组织全局栏目和课程主题，章内小标题使用 H3 或更低级；H1 课程标题由系统添加，不要输出 H1。\n" + if note_context == "document" + else "3. 外层章节标题由系统统一添加；正文小标题只能使用 H3 或更低级，不要输出全文 H1，也不要用“课程笔记”当标题。\n" + ) style_rule = ( "\u7b14\u8bb0\u4f18\u5148\uff1a\u6309\u6982\u5ff5\u548c\u63a8\u7406\u94fe\u7ec4\u7ec7\u5c0f\u8282\uff0c\u53ef\u4ee5\u6253\u6563 page_notes \u7684\u9010\u9875\u7ed3\u6784\uff1b\u5c01\u9762\u3001\u76ee\u5f55\u3001\u7ae0\u8282\u5bfc\u822a\u548c\u91cd\u590d\u5b57\u6bb5\u53ea\u4fdd\u7559\u9690\u85cf\u6765\u6e90\u6807\u8bb0\u5373\u53ef\u3002" if note_style == "article" else "\u4fdd\u771f\u4f18\u5148\uff1a\u5c3d\u91cf\u6cbf\u7740 page_notes \u7684\u987a\u5e8f\u7ec4\u7ec7\u5185\u5bb9\uff0c\u4f46\u4ecd\u8981\u5408\u5e76\u91cd\u590d\u6bb5\u843d\u5e76\u5220\u53bb\u7eaf\u7ed3\u6784\u6027\u8bf4\u660e\u3002" ) dedup_rule = { - "soft": "\u53ea\u5408\u5e76\u660e\u663e\u91cd\u590d\u7684\u53e5\u5b50\u548c\u5b8c\u5168\u76f8\u540c\u7684\u5b9a\u4e49\uff1b\u5b81\u53ef\u7565\u957f\uff0c\u4e5f\u4e0d\u8981\u5220\u6389 page_notes \u4e2d\u7684\u5173\u952e\u89e3\u91ca\u3002", - "normal": "\u5408\u5e76\u91cd\u590d\u5b9a\u4e49\u548c\u76f8\u8fd1\u4f8b\u5b50\uff0c\u4f46\u4fdd\u7559\u6bcf\u9875\u7684\u5173\u952e\u77e5\u8bc6\u70b9\u3001\u56fe\u8868\u89e3\u91ca\u548c\u63a8\u7406\u6b65\u9aa4\u3002", + "soft": "合并明显重复的句子、定义和例子；保留每个独有知识增量，但不要为每一页保留一份完整说明。", + "normal": "先建立本节知识点清单；同一概念只解释一次，把不同页面新增的条件、例子、公式和图示合并到该解释中。", "aggressive": "\u53ef\u4ee5\u66f4\u4e3b\u52a8\u5730\u538b\u7f29\u91cd\u590d\u5185\u5bb9\uff0c\u4f46\u4e0d\u5f97\u5220\u9664\u72ec\u6709\u7684\u5b9a\u4e49\u3001\u6761\u4ef6\u3001\u516c\u5f0f\u3001\u4f8b\u5b50\u548c\u56fe\u8868\u89e3\u91ca\u3002", }[weave_dedup] return ( @@ -307,14 +340,17 @@ def _llm_weave_prompt( f"{language_rule}\n" f"{term_rule}\n" f"{depth_rule}\n" + f"{structure_rule}\n" f"\u53bb\u91cd\u7b56\u7565\uff1a{dedup_rule}\n" "\u786c\u6027\u8981\u6c42\uff1a\n" "1. \u4fdd\u7559 page_notes \u4e2d\u5df2\u7ecf\u5199\u51fa\u7684\u56fe\u7247 Markdown \u548c\u9690\u85cf\u6765\u6e90\u6807\u8bb0\u3002\n" "2. \u5bf9\u6b63\u6587\u53ef\u4ee5\u6309\u6982\u5ff5\u8c03\u6574\u987a\u5e8f\u3001\u5408\u5e76\u6bb5\u843d\u3001\u5220\u53bb\u7ed3\u6784\u6027\u8bf4\u660e\uff1b\u4e0d\u8981\u4e3a\u4e86\u4fdd\u7559\u6bcf\u4e2a source marker \u800c\u9010\u5b57\u6bb5\u590d\u8ff0\u3002page_notes.learning_items 中 must_explain=true 且达到阈值的内容不能在 weave 时被删掉。\n" - "3. \u5916\u5c42\u7ae0\u8282\u6807\u9898\u7531\u7cfb\u7edf\u7edf\u4e00\u6dfb\u52a0\uff1b\u6b63\u6587\u5c0f\u6807\u9898\u53ea\u80fd\u4f7f\u7528 ### \u6216\u66f4\u4f4e\u7ea7\u6807\u9898\uff0c\u4e0d\u8981\u8f93\u51fa\u5168\u6587 H1\uff0c\u4e5f\u4e0d\u8981\u7528\u201c\u8bfe\u7a0b\u7b14\u8bb0\u201d\u5f53\u6807\u9898\u3002\n" + f"{heading_rule}" f"4. {source_rule}\n" "5. \u56fe\u7247 alt \u6587\u672c\u4e0d\u80fd\u4e3a\u7a7a\uff1b\u4e25\u7981\u8f93\u51fa\u201c\u597d\u7684\uff0c\u8fd9\u662f\u201d\u201c\u6839\u636e JSON\u201d\u201c\u7b14\u8bb0\u5df2\u4e25\u683c\u9075\u5faa\u201d\u201c\u65e0\u6cd5\u89c6\u89c9\u89e3\u6790\u201d\u7b49\u5143\u53d9\u8ff0\u3002\n" - "6. \u4e25\u7981\u4f7f\u7528\u201c\u8fd9\u4e00\u9875\u201d\u201c\u672c\u9875\u201d\u201c\u5e7b\u706f\u7247\u201d\u201c\u4e0a\u4e00\u9875\u201d\u201c\u4e0b\u4e00\u9875\u201d\u201c\u6b64\u9875\u201d\u7b49 PPT \u9875\u9762\u7ed3\u6784\u8868\u8ff0\uff1b\u76f4\u63a5\u8bb2\u77e5\u8bc6\uff0c\u81ea\u7136\u8fc7\u6e21\u3002\n\n" + "6. \u4e25\u7981\u4f7f\u7528\u201c\u8fd9\u4e00\u9875\u201d\u201c\u672c\u9875\u201d\u201c\u5e7b\u706f\u7247\u201d\u201c\u4e0a\u4e00\u9875\u201d\u201c\u4e0b\u4e00\u9875\u201d\u201c\u6b64\u9875\u201d\u7b49 PPT \u9875\u9762\u7ed3\u6784\u8868\u8ff0\uff1b\u76f4\u63a5\u8bb2\u77e5\u8bc6\uff0c\u81ea\u7136\u8fc7\u6e21\u3002\n" + "7. 丢弃 page_notes 自带的重复标题、重复自测和重复小结；渐进演示页要合成一次完整过程，而不是按步骤重新讲多遍。\n" + "8. 不得新增 page_notes、learning_items 或 deck_brief 未支持的具体事实、数字、代码行为、效率结论或使用建议。\n\n" f"{json.dumps(payload, ensure_ascii=False, indent=2)}" ) @@ -363,6 +399,7 @@ def _llm_teaching_enrichment_prompt( term_rule = _term_policy_prompt_rule(note_language, term_policy) depth_rule = _note_depth_rule(note_depth) profile_rule = _note_profile_prompt_rule(note_profile) + structure_rule = _context_structure_prompt_rule(note_context, note_language) return ( "\u8bf7\u628a current_markdown \u4fee\u8ba2\u6210\u66f4\u50cf\u8001\u5e08\u91cd\u65b0\u8bb2\u4e00\u904d\u7684 Markdown \u8bb2\u4e49\u5c0f\u8282\u3002\n" "\u4f60\u4e0d\u662f\u5728\u603b\u7ed3\u5e7b\u706f\u7247\uff0c\u800c\u662f\u5728\u505a\u6559\u5b66\u91cd\u6784\uff1a\u5148\u7406\u89e3\u672c\u8282\u5171\u540c\u89e3\u51b3\u4ec0\u4e48\u95ee\u9898\uff0c\u518d\u7528\u5b66\u751f\u80fd\u8ddf\u4e0a\u7684\u987a\u5e8f\u8bb2\u6e05\u695a\u3002\n" @@ -370,14 +407,15 @@ def _llm_teaching_enrichment_prompt( f"{language_rule}\n" f"{term_rule}\n" f"{depth_rule}\n" - "\u53ef\u4f7f\u7528\u7684\u8bb2\u4e49\u7ed3\u6784\uff1a### \u672c\u8282\u6838\u5fc3\u95ee\u9898\u3001### \u80cc\u666f\u4e0e\u76f4\u89c9\u3001### \u8be6\u7ec6\u8bb2\u89e3\u3001### \u56fe\u8868/\u516c\u5f0f\u89e3\u8bfb\u3001### \u6613\u9519\u70b9\u3001### \u672c\u8282\u5c0f\u7ed3\u3001### \u81ea\u6d4b\u95ee\u9898\u3002\u5982\u67d0\u9879\u786e\u5b9e\u4e0d\u9002\u7528\uff0c\u53ef\u5408\u5e76\u5230\u76f8\u90bb\u5c0f\u8282\u3002\n" + f"{structure_rule}\n" "\u786c\u6027\u8981\u6c42\uff1a\n" "1. \u4fdd\u7559 current_markdown \u4e2d\u5df2\u6709\u7684 Markdown \u56fe\u7247\u94fe\u63a5\u548c HTML source marker\uff1b\u65b0\u589e\u6216\u79fb\u52a8\u6bb5\u843d\u65f6\u4e5f\u8981\u4fdd\u7559\u5bf9\u5e94 source marker\u3002\n" "2. \u4e0d\u8981\u628a\u7ae0\u8282\u6539\u56de\u201c\u7b2c 1 \u9875\u8bb2 A\u3001\u7b2c 2 \u9875\u8bb2 B\u201d\u7684\u6e05\u5355\uff1bcoverage \u53ea\u662f\u6700\u540e\u8d28\u68c0\uff0c\u4e0d\u662f\u6b63\u6587\u6a21\u677f\u3002\n" "3. \u5bf9\u6838\u5fc3\u6982\u5ff5\uff0c\u89e3\u91ca\u5b83\u662f\u4ec0\u4e48\u3001\u4e3a\u4ec0\u4e48\u91cd\u8981\u3001\u5982\u4f55\u8fd0\u4f5c\u3001\u4e0e\u524d\u540e\u5185\u5bb9\u7684\u5173\u7cfb\u3002\n" "4. \u5bf9\u516c\u5f0f\u3001\u56fe\u8868\u3001\u6d41\u7a0b\u56fe\u3001\u622a\u56fe\uff0c\u4e0d\u8981\u53ea\u5199\u201c\u56fe\u4e2d\u5c55\u793a\u4e86\u201d\uff0c\u8981\u8bf4\u660e\u5b83\u652f\u6491\u4e86\u54ea\u4e2a\u6982\u5ff5\u6216\u63a8\u7406\u6b65\u9aa4\u3002\n" - "5. \u53ef\u4ee5\u8865\u5145\u5fc5\u8981\u7684\u901a\u7528\u80cc\u666f\u3001\u76f4\u89c9\u89e3\u91ca\u3001\u7b80\u77ed\u4f8b\u5b50\u6216\u7c7b\u6bd4\uff0c\u4f46\u4e0d\u5f97\u65b0\u589e\u8bfe\u4ef6\u6ca1\u6709\u4f9d\u636e\u7684\u5177\u4f53\u6570\u5b57\u3001\u5b9e\u9a8c\u7ed3\u679c\u3001\u4f5c\u8005\u89c2\u70b9\u6216\u7ed3\u8bba\u3002\u901a\u7528\u80cc\u666f\u8981\u7528\u201c\u4e3a\u4e86\u5e2e\u52a9\u7406\u89e3\u201d\u8fd9\u7c7b\u8bed\u6c14\u6807\u660e\u3002\n" - f"6. {source_rule}\n\n" + "5. 只允许补充不改变事实含义的直觉解释或最小类比；不得新增材料没有依据的具体数字、实验结果、代码行为、作者观点、效率结论或使用建议。\n" + "6. 把同一概念的重复段落合并为一次最清楚的解释。不同页面只是逐步演示同一过程时，保留最终完整过程和真正新增的步骤，不逐页复述。\n" + f"7. {source_rule}\n\n" f"{json.dumps(payload, ensure_ascii=False, indent=2)}" ) @@ -410,3 +448,31 @@ def _llm_repair_prompt( f"{term_rule}\n" f"{json.dumps(payload, ensure_ascii=False, indent=2)}" ) + + +def _llm_structure_repair_prompt( + markdown: str, + assessment: dict[str, Any], + source_display: str, + note_language: str, + term_policy: str, +) -> str: + payload = { + "task": "repair_human_note_structure", + "current_markdown": markdown, + "missing_structure_slots": assessment.get("missing_slots", []), + "empty_slot_headings": assessment.get("empty_slot_headings", {}), + } + return ( + "请把 current_markdown 修订为自然、完整、像学生认真整理过的电子讲义。只输出修订后的完整 Markdown。\n" + f"{lecture_note_structure_prompt_rule(note_language)}\n" + "优先重组和复用已有内容，不要用同一句话填充多个栏目。必须保留全部 Markdown 图片链接和 HTML source marker。\n" + "不要新增课件没有依据的具体数字、实验结果、人物观点或结论；允许使用不引入新事实的最小理解例。\n" + "不要出现“第几页”“本页”“幻灯片展示”“根据 JSON”“以下是生成的笔记”等机械或元叙述。\n" + "这是完整文档修订：必须保留且只保留一个现有 H1；主题章节使用 H2，章节内部按内容需要使用 H3。\n" + "学习目标、本讲总结和章节自测在全文各出现一次，不能复制到每个主题章节。\n" + f"{_source_prompt_rule(source_display)}\n" + f"{_language_prompt_rule(note_language)}\n" + f"{_term_policy_prompt_rule(note_language, term_policy)}\n\n" + f"{json.dumps(payload, ensure_ascii=False, indent=2)}" + ) diff --git a/slidenote/notes/quality.py b/slidenote/notes/quality.py index f54bc09..dd8195f 100644 --- a/slidenote/notes/quality.py +++ b/slidenote/notes/quality.py @@ -7,6 +7,8 @@ from slidenote.models import Deck from slidenote.utils import round_score +from .structure import assess_lecture_note_structure, structure_missing_labels + def build_note_quality_report( deck: Deck, @@ -34,6 +36,10 @@ def build_note_quality_report( mechanical_page_listing_score = _mechanical_page_listing_score(notes_markdown) self_test_score = _keyword_score(notes_markdown, ["自测", "检查自己", "思考题", "quiz", "self-test"], base=0.0) pitfall_score = _keyword_score(notes_markdown, ["易错", "误解", "注意", "陷阱", "常见错误", "pitfall", "misconception"], base=0.0) + structure_contract = assess_lecture_note_structure(notes_markdown) + structure_contract["enforced"] = note_profile in {"lecture-notes", "study-guide"} + structure_contract_pass = structure_contract["passed"] if structure_contract["enforced"] else None + mechanical_structure_pass = structure_contract.get("mechanical_repetition_pass", True) hallucination_risk = _hallucination_risk( paragraphs=paragraphs, source_markers=source_markers, @@ -49,6 +55,8 @@ def build_note_quality_report( self_test_score=self_test_score, pitfall_score=pitfall_score, image_total=image_total, + missing_structure_labels=structure_missing_labels(structure_contract) if structure_contract["enforced"] else [], + mechanical_structure_pass=mechanical_structure_pass, ) return { "schema_version": 1, @@ -68,6 +76,11 @@ def build_note_quality_report( "mechanical_page_listing_score": mechanical_page_listing_score, "self_test_score": self_test_score, "pitfall_score": pitfall_score, + "human_note_structure_score": structure_contract["score"], + "structure_contract_pass": structure_contract_pass, + "mechanical_structure_pass": mechanical_structure_pass, + "repetitive_subheading_counts": structure_contract.get("repetitive_subheading_counts", {}), + "structure_contract": structure_contract, # Question quality belongs to `study-pack`; build keeps the documented null keys. "question_quality": None, "question_quality_score": None, @@ -83,6 +96,12 @@ def build_note_quality_report( "required_visible_missing": (coverage_report.get("required_visible_coverage") or {}).get("missing") if coverage_report else None, "question_quality_score": None, "suggested_repairs": len(suggested_repairs), + "structure_contract_pass": structure_contract_pass, + "missing_structure_slots": structure_contract["missing_slots"] if structure_contract["enforced"] else [], + "missing_recommended_structure_slots": structure_contract.get("missing_recommended_slots", []) + if structure_contract["enforced"] + else [], + "mechanical_structure_pass": mechanical_structure_pass, }, } @@ -174,6 +193,8 @@ def _suggest_repairs( self_test_score: float, pitfall_score: float, image_total: int, + missing_structure_labels: list[str], + mechanical_structure_pass: bool, ) -> list[str]: suggestions: list[str] = [] if explanation_depth_score < 0.55: @@ -188,4 +209,8 @@ def _suggest_repairs( suggestions.append("增加本节自测问题，帮助学生检查是否真正理解。") if pitfall_score < 0.25: suggestions.append("补充易错点或常见误解。") + if missing_structure_labels: + suggestions.append("补齐人工讲义结构栏目：" + "、".join(missing_structure_labels) + "。") + if not mechanical_structure_pass: + suggestions.append("合并重复的学习目标、小结和自测，并把逐页标题改成按真实课程主题组织的章节。") return suggestions diff --git a/slidenote/notes/repair.py b/slidenote/notes/repair.py index 5062c7a..ff3f791 100644 --- a/slidenote/notes/repair.py +++ b/slidenote/notes/repair.py @@ -10,9 +10,11 @@ from slidenote.models import Deck from .contexts import NoteContext -from .postprocess import _postprocess_llm_markdown +from .postprocess import _postprocess_document_markdown, _postprocess_llm_markdown from .llm_calls import _generate_cached_llm_text -from .prompt_templates import _llm_repair_prompt +from .prompt_templates import _llm_repair_prompt, _llm_structure_repair_prompt +from .structure import _heading_sections, assess_lecture_note_structure +from .versions import STRUCTURE_REPAIR_PROMPT_VERSION _IMAGE_LINK = re.compile(r"!\[[^\]]*]\(([^)]+)\)") @@ -42,6 +44,15 @@ def _image_targets(markdown: str) -> set[str]: return targets +def _source_marker_payloads(markdown: str) -> set[str]: + """Return the exact source-marker payloads carried by a note draft.""" + return { + match.strip() + for match in re.findall(r"", markdown) + if match.strip() + } + + def _body_chars(markdown: str) -> int: """Count prose without source comments, image URLs or heading boilerplate.""" text = re.sub(r"", "", markdown, flags=re.DOTALL) @@ -183,3 +194,140 @@ def _fits_output_budget(markdown: str, max_output_tokens: int | None) -> bool: return True estimated_tokens = len(markdown) * REPAIR_ESTIMATED_TOKENS_PER_CHAR return estimated_tokens <= max_output_tokens * REPAIR_OUTPUT_BUDGET_HEADROOM + + +def _repair_note_structure_once( + deck: Deck, + context: NoteContext, + markdown: str, + output_root: Path, + cache: LLMCache, + options: "NoteOptions", +) -> tuple[str, dict[str, Any] | None]: + """Rewrite the whole document when the lecture structure contract still fails. + + Runs only for lecture note profiles and only when the deterministic document + frame could not make the contract pass (page-listing headings or repeated + global sections). The candidate replaces the draft only when the structure + contract passes, the body is substantially preserved, and no source, + coverage or image target is lost. + """ + if options.note_profile not in {"lecture-notes", "study-guide"}: + return markdown, None + before = assess_lecture_note_structure(markdown) + if before["passed"]: + return markdown, None + + record: dict[str, Any] = { + "stage": "structure_repair", + "context_id": context.id, + "slide_ids": [page.slide_id for page in context.pages], + "before": before, + "accepted": False, + "rejection_reasons": [], + "llm": None, + } + if not _fits_output_budget(markdown, options.max_output_tokens): + record["rejection_reasons"] = ["input_too_long_for_output_budget"] + record["input_chars"] = len(markdown) + return markdown, record + + prompt = _llm_structure_repair_prompt( + markdown=markdown, + assessment=before, + source_display=options.source_display, + note_language=options.note_language, + term_policy=options.term_policy, + ) + try: + repaired, llm_record = _generate_cached_llm_text( + context=NoteContext( + id=f"repair_structure_{context.id}", + kind=f"repair_structure_{context.kind}", + title=context.title, + pages=context.pages, + ), + output_root=output_root, + cache=cache, + cache_mode=options.cache_mode, + provider=options.provider, + model=options.model, + api_key=options.api_key, + base_url=options.base_url, + max_output_tokens=options.max_output_tokens, + temperature=options.temperature, + user_prompt=prompt, + prompt_version=STRUCTURE_REPAIR_PROMPT_VERSION, + generation_stage="structure_repair", + request_options={ + "source_display": options.source_display, + "note_language": options.note_language, + "term_policy": options.term_policy, + "missing_slots": before["missing_slots"], + }, + force_refresh=False, + ) + except Exception as exc: + # This optional repair must not discard an already generated draft. + record["rejection_reasons"] = ["generation_error"] + record["error_type"] = type(exc).__name__ + return markdown, record + + repaired = _postprocess_document_markdown(repaired, source_display=options.source_display) + after = assess_lecture_note_structure(repaired) + original_titles = [heading["title"] for heading in _heading_sections(markdown) if heading["level"] == 1] + candidate_titles = [heading["title"] for heading in _heading_sections(repaired) if heading["level"] == 1] + title_preserved = len(original_titles) == 1 and candidate_titles == original_titles + before_coverage = analyze_coverage(deck, markdown, content_guard=options.content_guard) + after_coverage = analyze_coverage(deck, repaired, content_guard=options.content_guard) + lost_trace = _covered_ids(before_coverage, "trace_covered") - _covered_ids(after_coverage, "trace_covered") + lost_visible = _covered_ids(before_coverage, "visible_covered") - _covered_ids(after_coverage, "visible_covered") + lost_sources = _source_marker_payloads(markdown) - _source_marker_payloads(repaired) + lost_images = _image_targets(markdown) - _image_targets(repaired) + original_chars = _body_chars(markdown) + candidate_chars = _body_chars(repaired) + reasons: list[str] = [] + if not candidate_chars: + reasons.append("empty_repair") + if not after["passed"]: + reasons.append("structure_contract_still_failing") + if not title_preserved: + reasons.append("document_title_not_preserved") + if after["score"] < before["score"]: + reasons.append("structure_score_regression") + if lost_trace or lost_visible: + reasons.append("coverage_regression") + if lost_sources: + reasons.append("missing_source_markers") + if lost_images: + reasons.append("missing_images") + if candidate_chars < original_chars * MIN_BODY_RETENTION: + reasons.append("body_truncated") + response_usage = llm_record.get("provider_usage") or llm_record.get("cached_entry_usage") or {} + finish_reason = str(response_usage.get("finish_reason") or "").lower() + if finish_reason in _INCOMPLETE_FINISH_REASONS: + reasons.append("incomplete_generation") + + record["after"] = after + record["llm"] = llm_record + record["rejection_reasons"] = reasons + record["validation"] = { + "after_score": after["score"], + "before_score": before["score"], + "after_contract_passed": after["passed"], + "original_document_titles": original_titles, + "candidate_document_titles": candidate_titles, + "document_title_preserved": title_preserved, + "lost_trace_items": sorted(lost_trace), + "lost_visible_items": sorted(lost_visible), + "lost_source_markers": sorted(lost_sources), + "lost_image_targets": sorted(lost_images), + "original_body_chars": original_chars, + "candidate_body_chars": candidate_chars, + "finish_reason": finish_reason or None, + } + if reasons: + return markdown, record + + record["accepted"] = True + return repaired, record diff --git a/slidenote/notes/structure.py b/slidenote/notes/structure.py new file mode 100644 index 0000000..acb69da --- /dev/null +++ b/slidenote/notes/structure.py @@ -0,0 +1,312 @@ + +import re +from collections import Counter +from typing import Any + + +LECTURE_NOTE_STRUCTURE_SLOTS: dict[str, tuple[str, ...]] = { + "orientation": ( + "本讲目标", + "学习目标", + "本讲要解决的问题", + "lecture objectives", + "learning objectives", + "core questions", + ), + "example_application": ( + "例子与应用", + "示例与应用", + "运行示例", + "执行示例", + "推导示例", + "代码示例", + "worked example", + "example", + "application", + ), + "pitfalls": ( + "易错点", + "常见误解", + "注意事项", + "常见错误", + "pitfalls", + "common misconceptions", + "common mistakes", + ), + "summary": ( + "本讲总结", + "课程总结", + "总结", + "summing up", + "lecture summary", + ), + "self_test": ( + "章节自测", + "本讲自测", + "综合自测", + "self-test", + "self test", + "review questions", + ), +} + + +LECTURE_NOTE_SLOT_LABELS_ZH: dict[str, str] = { + "orientation": "本讲目标", + "topic_sections": "按课程主题命名的章节", + "example_application": "至少一个有材料依据的例子或应用", + "pitfalls": "至少一处真正必要的易错点", + "summary": "本讲总结", + "self_test": "章节自测", +} + + +_GENERIC_SECTION_ALIASES = { + alias.lower() + for slot in LECTURE_NOTE_STRUCTURE_SLOTS + for alias in LECTURE_NOTE_STRUCTURE_SLOTS[slot] +} + +_GLOBAL_STRUCTURE_SLOTS = {"orientation", "summary", "self_test"} + +_REPETITIVE_SUBHEADINGS = { + "学习目标", + "本节核心问题", + "核心概念", + "核心概念与机制", + "例子与应用", + "示例与应用", + "易错点", + "常见误解", + "本节小结", + "小结", + "自测问题", + "自测题", + "learning objectives", + "core concepts and mechanisms", + "examples and applications", + "common misconceptions", + "section summary", + "self-test", +} + + +def assess_lecture_note_structure(markdown: str) -> dict[str, Any]: + """Assess the structure of a complete lecture note. + + The contract is global rather than a six-field template repeated in every + chapter: one orientation, topic-named chapters, and one closing summary and + review section, with examples and pitfalls where the source supports them. + """ + + headings = _heading_sections(markdown) + h1_titles = [heading["title"] for heading in headings if heading["level"] == 1] + h1_count = len(h1_titles) + title_pass = h1_count == 1 and bool(h1_titles[0]) + matched: dict[str, str] = {} + empty_matches: dict[str, str] = {} + for slot, aliases in LECTURE_NOTE_STRUCTURE_SLOTS.items(): + for heading in headings: + if slot in _GLOBAL_STRUCTURE_SLOTS and heading["level"] != 2: + continue + if not _heading_matches(heading["title"], aliases): + continue + if _visible_length(heading["body"]) >= 12: + matched[slot] = heading["title"] + else: + empty_matches[slot] = heading["title"] + break + + topic_sections = [ + heading["title"] + for heading in headings + if heading["level"] == 2 + and not _is_generic_document_section(heading["title"]) + and not _is_page_heading(heading["title"]) + ] + if topic_sections: + matched["topic_sections"] = topic_sections[0] + + page_headings = [ + heading["title"] + for heading in headings + if 2 <= heading["level"] <= 4 and _is_page_heading(heading["title"]) + ] + repeated = _repetitive_heading_counts(headings) + template_repetition_detected = len(repeated) >= 2 and sum(repeated.values()) >= 4 + repeated_global = { + slot: sum( + 1 + for heading in headings + if heading["level"] == 2 and _heading_matches(heading["title"], aliases) + ) + for slot, aliases in LECTURE_NOTE_STRUCTURE_SLOTS.items() + if slot in _GLOBAL_STRUCTURE_SLOTS + } + mechanical_repetition_pass = ( + not page_headings + and not template_repetition_detected + and all(count <= 1 for count in repeated_global.values()) + ) + + required = ["orientation", "topic_sections", "summary", "self_test"] + recommended = ["example_application", "pitfalls"] + missing = [slot for slot in required if slot not in matched] + scored = required + recommended + score = round(len(matched.keys() & set(scored)) / len(scored), 4) + return { + "schema_version": 3, + "h1_count": h1_count, + "title_pass": title_pass, + "required_slots": required, + "recommended_slots": recommended, + "matched_slots": matched, + "missing_slots": missing, + "missing_recommended_slots": [slot for slot in recommended if slot not in matched], + "empty_slot_headings": empty_matches, + "topic_sections": topic_sections, + "topic_section_count": len(topic_sections), + "page_headings": page_headings, + "repetitive_subheading_counts": repeated, + "template_repetition_detected": template_repetition_detected, + "repeated_global_sections": repeated_global, + "mechanical_repetition_pass": mechanical_repetition_pass, + "score": score, + "passed": title_pass and not missing and mechanical_repetition_pass, + } + + +def lecture_note_structure_prompt_rule(note_language: str) -> str: + if note_language == "en": + return ( + "Stable document contract: keep exactly one H1 title; place one ## Learning Objectives near the start; " + "organize the body into H2 chapters named after the actual course topics; end with exactly one " + "## Lecture Summary and one ## Review Questions. Across the lecture, include at least one source-backed " + "worked example and one genuinely useful pitfalls section. Inside each chapter, choose only the H3 " + "headings the material needs (for example prerequisites, mechanism, pseudocode/code, worked example, " + "complexity, or pitfalls). Never repeat the same template sequence in every chapter." + ) + return ( + "稳定的全文结构契约：保留且只保留一个 H1 课程标题；开头只设置一次 ## 本讲目标；" + "正文使用由真实课程主题命名的 H2 章节；结尾只设置一次 ## 本讲总结和一次 ## 章节自测。" + "全文至少应有一个由材料支持的示例/应用和一处真正有必要的易错点。" + "每章内部只选择材料确实需要的 H3，例如核心思想、使用前提、伪代码/代码、执行示例、复杂度或易错点；" + "不要让每章重复同一组六栏标题，也不要为了填模板制造空泛内容。" + ) + + +def lecture_section_style_prompt_rule(note_language: str) -> str: + if note_language == "en": + return ( + "Chapter structure is adaptive, not a form. Use two to five natural H3 headings only when supported by " + "the material. Preserve exact code and pseudocode. Do not add chapter-level objectives, a chapter summary, " + "or a self-test by default; those belong to the document frame. Avoid reusing the same generic headings " + "and sentence openings in every chapter." + ) + return ( + "章节结构必须随内容变化，而不是套表格：只在材料支持时选用二到五个自然的 H3，" + "例如“核心思路”“使用前提”“伪代码”“执行过程”“复杂度”“代码要点”或“易错点”。" + "代码与伪代码必须原样保真。默认不要在每章重复学习目标、本节小结和自测；这些属于全文框架。" + "不要让不同章节使用完全相同的小标题顺序或相同的开场句。" + ) + + +def structure_missing_labels(assessment: dict[str, Any]) -> list[str]: + labels = [LECTURE_NOTE_SLOT_LABELS_ZH.get(slot, slot) for slot in assessment.get("missing_slots", [])] + if assessment.get("title_pass") is False: + labels.insert(0, "唯一的 H1 课程标题") + return labels + + +def _heading_sections(markdown: str) -> list[dict[str, Any]]: + visible_markdown = _mask_fenced_code(markdown) + matches = list(re.finditer(r"(?m)^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*\r?$", visible_markdown)) + sections: list[dict[str, Any]] = [] + for index, match in enumerate(matches): + start = match.end() + end = matches[index + 1].start() if index + 1 < len(matches) else len(markdown) + title = markdown[match.start(2) : match.end(2)].strip() + title = re.sub(r"(?:^|[ \t]+)#+[ \t]*$", "", title).strip() + sections.append( + { + "level": len(match.group(1)), + "title": title, + "body": markdown[start:end].strip(), + "start": match.start(), + "end": match.end(), + } + ) + return sections + + +def _mask_fenced_code(markdown: str) -> str: + """Hide fenced code from structural parsing without changing text offsets.""" + + fence_char = "" + fence_length = 0 + lines: list[str] = [] + for line in markdown.splitlines(keepends=True): + content = line.rstrip("\r\n") + if fence_char: + lines.append(re.sub(r"[^\r\n]", " ", line)) + closing = re.fullmatch(r" {0,3}([`~]+)[ \t]*", content) + if closing and set(closing.group(1)) == {fence_char} and len(closing.group(1)) >= fence_length: + fence_char = "" + fence_length = 0 + continue + + opening = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", content) + if opening and (opening.group(1)[0] != "`" or "`" not in opening.group(2)): + fence_char = opening.group(1)[0] + fence_length = len(opening.group(1)) + lines.append(re.sub(r"[^\r\n]", " ", line)) + else: + lines.append(line) + return "".join(lines) + + +def _heading_matches(title: str, aliases: tuple[str, ...]) -> bool: + normalized = _normalize_heading(title) + return any( + normalized == alias_normalized or normalized.startswith(alias_normalized + " ") + for alias in aliases + if (alias_normalized := _normalize_heading(alias)) + ) + + +def _normalize_heading(title: str) -> str: + title = re.sub(r"^\s*(?:\d+|[一二三四五六七八九十百]+)[.、\s-]+", "", title.strip()) + return re.sub(r"[\s：:—\-_/（）()]+", " ", title).strip().lower() + + +def _is_generic_document_section(title: str) -> bool: + return _heading_matches(title, tuple(_GENERIC_SECTION_ALIASES)) + + +def _clean_numbering(title: str) -> str: + return re.sub(r"^\s*(?:\d+|[一二三四五六七八九十百]+)[.、\s-]+", "", title.strip()) + + +def _is_page_heading(title: str) -> bool: + raw = _clean_numbering(title) + suffix = r"(?:\s*[:：—–-]\s*.*|\s+.+)?" + return bool(re.fullmatch(rf"(?:第\s*)?\d+\s*(?:页|张){suffix}", raw)) or bool( + re.fullmatch(rf"(?:slide|page|p)\s*\d+{suffix}", raw, flags=re.IGNORECASE) + ) + + +def _repetitive_heading_counts(headings: list[dict[str, Any]]) -> dict[str, int]: + counts = Counter(_normalize_heading(heading["title"]) for heading in headings if heading["level"] >= 3) + repetitive_aliases = {_normalize_heading(alias) for alias in _REPETITIVE_SUBHEADINGS} + return { + title: count + for title, count in sorted(counts.items()) + if count > 1 and title in repetitive_aliases + } + + +def _visible_length(body: str) -> int: + visible = re.sub(r"", "", body, flags=re.DOTALL) + visible = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", visible) + visible = re.sub(r"[`*_>#|\-\s]", "", visible) + return len(visible) diff --git a/slidenote/notes/usage.py b/slidenote/notes/usage.py index 578b092..675e610 100644 --- a/slidenote/notes/usage.py +++ b/slidenote/notes/usage.py @@ -35,6 +35,7 @@ def _build_usage_report( "page_note_calls": sum(1 for context in (page_contexts or []) if context.get("llm_call")), "weave_calls": sum(1 for context in (weave_contexts or []) if context.get("llm_call")), "teaching_enrichment_calls": sum(1 for context in (teaching_enrichment_contexts or []) if context.get("llm_call")), + "repair_calls": sum(1 for context in (repair_contexts or []) if context.get("llm_call")), "local_cache_hits": sum(1 for context in contexts if context.get("cache_status") == "local_hit"), "local_cache_misses": sum(1 for context in contexts if context.get("cache_status") == "miss"), "local_cache_refreshes": sum(1 for context in contexts if context.get("cache_status") == "refresh"), diff --git a/slidenote/notes/versions.py b/slidenote/notes/versions.py index 039a511..26486bf 100644 --- a/slidenote/notes/versions.py +++ b/slidenote/notes/versions.py @@ -1,6 +1,7 @@ from __future__ import annotations NOTE_PROMPT_VERSION = "note-context-v4" -PAGE_LECTURE_PROMPT_VERSION = "page-lecture-v4" -WEAVE_PROMPT_VERSION = "weave-v4" -TEACHING_ENRICHMENT_PROMPT_VERSION = "teaching-enrichment-v1" +PAGE_LECTURE_PROMPT_VERSION = "page-lecture-v5" +WEAVE_PROMPT_VERSION = "weave-v6" +TEACHING_ENRICHMENT_PROMPT_VERSION = "teaching-enrichment-v4" +STRUCTURE_REPAIR_PROMPT_VERSION = "structure-repair-v2" diff --git a/tests/test_eval_decks.py b/tests/test_eval_decks.py new file mode 100644 index 0000000..18de5a9 --- /dev/null +++ b/tests/test_eval_decks.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path + +import pytest + +from scripts.eval_decks import ( + collect_case_metrics, + display_path, + safe_case_dir, + structure_page_headings, + validate_case_id, + write_markdown_report, +) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_metrics_fail_closed_when_required_reports_are_missing(tmp_path): + (tmp_path / "notes.md").write_text("# Demo\n\n## Slide 1\n\ncontent\n", encoding="utf-8") + + metrics = collect_case_metrics(tmp_path, 0.1, 0, "") + + assert metrics["passed"] is False + assert metrics["gates"]["required_artifacts_present"] is False + assert metrics["gates"]["coverage_report_valid"] is False + assert metrics["gates"]["quality_report_valid"] is False + assert metrics["gates"]["export_report_valid"] is False + + +def test_metrics_record_local_page_listing_without_false_report(tmp_path): + (tmp_path / "notes.md").write_text("# Demo\n\n### 第 1 页：Topic\n\ncontent\n", encoding="utf-8") + _write_json(tmp_path / "run_summary.json", {"run": {"preset": "local"}, "stage_timings": {}}) + _write_json( + tmp_path / "coverage.json", + {"missing": 0, "coverage_ratio": 1.0, "required_visible_coverage": {"missing": 0, "total": 0}}, + ) + _write_json( + tmp_path / "quality_report.json", + {"structure_contract": {"enforced": False}, "mechanical_structure_pass": True}, + ) + _write_json( + tmp_path / "export_report.json", + {"summary": {"blocking_failures": 0}, "results": [{"format": "markdown-zip", "status": "ok"}]}, + ) + + metrics = collect_case_metrics(tmp_path, 0.1, 0, "") + metrics["case_id"] = "local-case" + assert metrics["passed"] is True + assert metrics["page_listing_hits"] == ["第 1 页：Topic"] + + report = { + "meta": {"generated_at": "now", "manifest": "m.json", "run_dir": "run"}, + "cases": [metrics], + } + output = tmp_path / "eval_report.md" + write_markdown_report(report, output) + assert "有（未作为硬门槛）" in output.read_text(encoding="utf-8") + + +def test_case_id_validation_blocks_path_traversal(tmp_path): + with pytest.raises(ValueError): + validate_case_id("../../outside") + with pytest.raises(ValueError): + safe_case_dir(tmp_path, "case/subdir") + assert safe_case_dir(tmp_path, "safe-case_1").parent == tmp_path.resolve() + + +def test_display_path_accepts_paths_outside_repository(tmp_path): + assert display_path(tmp_path) == str(tmp_path.resolve()) + + +@pytest.mark.parametrize("fence", ["```markdown", "~~~markdown"]) +def test_evaluation_page_heading_gate_ignores_code_examples(fence): + closing = fence[:3] + notes = f"# Course\n\n{fence}\n## Slide 1 - Code example\n{closing}\n\n## Slide 2 - Actual listing\n" + + assert structure_page_headings(notes) == ["Slide 2 - Actual listing"] diff --git a/tests/test_note_frame_review.py b/tests/test_note_frame_review.py new file mode 100644 index 0000000..2031427 --- /dev/null +++ b/tests/test_note_frame_review.py @@ -0,0 +1,151 @@ +"""Regressions for document-frame and structure checks found in PR review.""" + +import pytest + +from slidenote.notes.document_frame import ensure_lecture_document_frame +from slidenote.notes.structure import ( + _heading_sections, + assess_lecture_note_structure, + structure_missing_labels, +) + + +BODY = "A source-backed explanation of the algorithm, its conditions, and its conclusion." + + +def _complete_note(objectives="本讲目标", summary="本讲总结", review="章节自测"): + return ( + f"# Search Algorithms\n\n## {objectives}\n\n{BODY}\n\n" + f"## Binary Search\n\n{BODY}\n\n" + f"## {summary}\n\n{BODY}\n\n## {review}\n\n{BODY}\n" + ) + + +@pytest.mark.parametrize( + ("note_language", "objectives", "summary", "review"), + [ + ("zh", "一、本讲目标：二分搜索", "本讲总结（搜索算法）", "章节自测 - 搜索算法"), + ("en", "1. Learning Objectives: Binary Search", "Lecture Summary / Search", "Review Questions (Search)"), + ], +) +def test_frame_keeps_valid_suffixed_global_sections(note_language, objectives, summary, review): + original = _complete_note(objectives, summary, review) + assert assess_lecture_note_structure(original)["passed"] is True + + framed = ensure_lecture_document_frame(original, None, note_language) + + assert framed == original + assert ensure_lecture_document_frame(framed, None, note_language) == framed + assessment = assess_lecture_note_structure(framed) + assert assessment["passed"] is True + assert assessment["repeated_global_sections"] == {"orientation": 1, "summary": 1, "self_test": 1} + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_code_headings_do_not_fill_missing_document_sections(fence): + code = ( + f"{fence}markdown\n# Example document\n\n" + f"## 本讲目标\n\n{BODY}\n\n## 本讲总结\n\n{BODY}\n\n" + f"## 章节自测\n\n{BODY}\n{fence}" + ) + original = f"# Real Course\n\n## Binary Search\n\n{code}\n\n{BODY}\n" + before = assess_lecture_note_structure(original) + assert before["h1_count"] == 1 + assert before["missing_slots"] == ["orientation", "summary", "self_test"] + + framed = ensure_lecture_document_frame(original, None, "zh") + + assert code in framed + assessment = assess_lecture_note_structure(framed) + assert assessment["passed"] is True + assert assessment["h1_count"] == 1 + assert assessment["repeated_global_sections"] == {"orientation": 1, "summary": 1, "self_test": 1} + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_code_page_and_template_headings_do_not_fail_valid_structure(fence): + code = ( + f"{fence}markdown\n## 第 1 页：示例\n### 核心概念\n{BODY}\n" + f"### 易错点\n{BODY}\n### 核心概念\n{BODY}\n### 易错点\n{BODY}\n{fence}" + ) + original = _complete_note().replace(f"## Binary Search\n\n{BODY}", f"## Binary Search\n\n{code}\n\n{BODY}") + + assessment = assess_lecture_note_structure(original) + + assert assessment["passed"] is True + assert assessment["page_headings"] == [] + assert assessment["repetitive_subheading_counts"] == {} + assert ensure_lecture_document_frame(original, None, "zh") == original + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_frame_places_objectives_after_real_h1_and_preserves_code(fence): + code = f"{fence}markdown\n# Example H1\n## Example H2\n{fence}" + original = f"{code}\n\n# Real Course\n\n{BODY}\n" + + framed = ensure_lecture_document_frame(original, None, "en") + + assert framed.startswith(code + "\n\n# Real Course\n\n## Learning Objectives") + assert f"## Core Content\n\n{BODY}" in framed + assert assess_lecture_note_structure(framed)["passed"] is True + + +@pytest.mark.parametrize( + "prefix", + [ + "", + "```markdown\n# Code-only title\n```\n\n", + "~~~markdown\n# Code-only title\n~~~\n\n", + "# First title\n\n# Second title\n\n", + ], +) +def test_contract_requires_exactly_one_real_h1(prefix): + original = prefix + _complete_note().split("\n\n", 1)[1] + + assessment = assess_lecture_note_structure(original) + + assert assessment["missing_slots"] == [] + assert assessment["h1_count"] == (2 if prefix.startswith("# First") else 0) + assert assessment["title_pass"] is False + assert assessment["passed"] is False + assert "唯一的 H1 课程标题" in structure_missing_labels(assessment) + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_fence_closer_must_match_character_and_opening_length(fence): + other_fence = "~~~" if fence[0] == "`" else "```" + original = ( + f"# Real Course\r\n\r\n{fence}{fence[0]}markdown\r\n" + f"# Hidden one\r\n{other_fence}\r\n# Hidden two\r\n" + f"{fence}\r\n# Hidden three\r\n{fence}{fence[0]}\r\n" + f"## Visible Topic\r\n\r\n{BODY}\r\n" + ) + + headings = _heading_sections(original) + + assert [(heading["level"], heading["title"]) for heading in headings] == [ + (1, "Real Course"), (2, "Visible Topic") + ] + assert original[headings[1]["start"] : headings[1]["end"]] == "## Visible Topic\r" + assert "# Hidden three" in headings[0]["body"] + + +def test_commonmark_heading_spacing_and_closing_hashes_share_frame_matching(): + original = _complete_note("本讲目标：搜索 ###", "本讲总结 ##", "章节自测 #") + original = original.replace("# Search Algorithms", " # Search Algorithms ###").replace("## Binary Search", " ## Binary Search") + + framed = ensure_lecture_document_frame(original, None, "zh") + + assert assess_lecture_note_structure(framed)["passed"] is True + assert len(_heading_sections(framed)) == 5 + assert [heading["title"] for heading in _heading_sections(framed)][:2] == ["Search Algorithms", "本讲目标：搜索"] + + +def test_empty_atx_title_does_not_satisfy_document_title_contract(): + original = _complete_note().replace("# Search Algorithms", "# ###") + + assessment = assess_lecture_note_structure(original) + + assert assessment["h1_count"] == 1 + assert assessment["title_pass"] is False + assert assessment["passed"] is False diff --git a/tests/test_note_layout_review.py b/tests/test_note_layout_review.py new file mode 100644 index 0000000..a08d821 --- /dev/null +++ b/tests/test_note_layout_review.py @@ -0,0 +1,127 @@ +import pytest + +from scripts.verify_note_layout import check_markdown + + +def _check(tmp_path, text): + notes = tmp_path / "notes.md" + notes.write_text(text, encoding="utf-8") + return check_markdown(notes) + + +def test_contiguous_five_row_table_is_counted_once(tmp_path): + issues, stats = _check( + tmp_path, + "# Course\n\n" + "| Name | Value |\n" + "| --- | --- |\n" + "| A | 1 |\n" + "| B | 2 |\n" + "| C | 3 |\n", + ) + + assert stats["tables"] == 1 + assert not any(item["rule"] == "ragged_table_row" for item in issues) + + +def test_missing_column_in_third_table_row_is_reported(tmp_path): + issues, stats = _check( + tmp_path, + "# Course\n\n" + "| Name | Value |\n" + "| --- | --- |\n" + "| A |\n" + "| B | 2 |\n", + ) + + assert stats["tables"] == 1 + ragged = [item for item in issues if item["rule"] == "ragged_table_row"] + assert len(ragged) == 1 + assert "5" in ragged[0]["detail"] + assert "1 != 2" in ragged[0]["detail"] + + +@pytest.mark.parametrize("separator", ["\n", "\nSeparate discussion.\n\n"]) +def test_separated_tables_have_independent_column_counts(tmp_path, separator): + issues, stats = _check( + tmp_path, + "# Course\n\n" + "| Name | Value |\n" + "| --- | --- |\n" + "| A | 1 |\n" + + separator + + "| Name | Value | Units |\n" + "| --- | --- | --- |\n" + "| B | 2 | kg |\n", + ) + + assert stats["tables"] == 2 + assert not any(item["rule"] == "ragged_table_row" for item in issues) + + +@pytest.mark.parametrize( + ("opening", "closing"), + [("```markdown", "```"), ("````markdown", "````"), ("~~~markdown", "~~~~")], +) +def test_code_examples_do_not_create_layout_errors(tmp_path, opening, closing): + issues, stats = _check( + tmp_path, + "# Course\n\n## Example\n\n" + + opening + + "\n# Example heading\n#### Example jump\n" + "![](not-an-image.png)\n" + "| Name | Value |\n| --- | --- |\n| Incomplete |\n" + "$$\n" + + closing + + "\n\n## Explanation\n\nThe syntax above is a code example.\n", + ) + + assert stats["h1"] == 1 + assert stats["headings"] == 3 + assert stats["images"] == 0 + assert stats["tables"] == 0 + assert issues == [] + + +def test_shorter_fence_inside_long_fence_does_not_close_code(tmp_path): + issues, stats = _check( + tmp_path, + "# Course\n\n## Example\n\n````markdown\n" + "```python\n# This heading belongs to the example\n```\n" + "![](not-an-image.png)\n$$\n````\n", + ) + + assert stats["headings"] == 2 + assert stats["images"] == 0 + assert issues == [] + + +@pytest.mark.parametrize( + ("opening", "invalid_closing"), + [("~~~", "```"), ("````", "```"), ("```", "```with-info")], +) +def test_invalid_closing_fence_remains_unclosed(tmp_path, opening, invalid_closing): + issues, stats = _check( + tmp_path, + "# Course\n\n## Example\n\n" + + opening + + "\ncode\n" + + invalid_closing + + "\n# Still code\n![](not-an-image.png)\n$$\n", + ) + + assert stats["h1"] == 1 + assert stats["images"] == 0 + assert {item["rule"] for item in issues} == {"unbalanced_code_fence"} + + +def test_real_errors_after_code_keep_original_line_numbers(tmp_path): + issues, stats = _check( + tmp_path, + "# Course\n```markdown\n![](example.png)\n$$\n```\n![](real.png)\n$$\n", + ) + + assert stats["images"] == 1 + image_issue = next(item for item in issues if item["rule"] == "image_missing_alt") + assert "6" in image_issue["detail"] + assert any(item["rule"] == "unbalanced_math" for item in issues) diff --git a/tests/test_note_structure.py b/tests/test_note_structure.py new file mode 100644 index 0000000..740bce3 --- /dev/null +++ b/tests/test_note_structure.py @@ -0,0 +1,361 @@ +"""Structure contract, document frame and finalize wiring tests (ported from the +reference-note baseline work).""" + +from pathlib import Path + +from slidenote.coverage import analyze_coverage +from slidenote.models import Deck, SlidePage, TextBlock +from slidenote.notes import NoteOptions, generate_notes_result + + +def test_human_note_structure_contract_matches_two_course_prototypes(): + from slidenote.notes.structure import assess_lecture_note_structure + + prototype_notes = [ + ( + "# Algorithms\n\n## 本讲目标\n\n理解线性搜索与二分搜索的适用条件，并比较它们的时间复杂度。\n\n" + "## 搜索算法\n\n### 核心思路\n\n线性搜索逐项检查，二分搜索依赖有序数据并在每一步排除一半候选范围。\n\n" + "### 运行示例\n\n在有序号码簿中查找姓名时，可以比较从头查找与反复对半缩小范围的过程。\n\n" + "### 易错点\n\n二分搜索不能直接用于无序数据，排序成本也不能在分析整体任务时被忽略。\n\n" + "## 本讲总结\n\n算法选择取决于数据前提、输入规模和需要执行查找的次数。\n\n" + "## 章节自测\n\n为什么二分搜索每次比较后可以安全地丢弃一半候选项？" + ), + ( + "# Number Representation\n\n## 本讲目标\n\n理解有限位宽如何表示无符号数和有符号数，以及运算为何会溢出。\n\n" + "## 二进制补码\n\n### 表示规则\n\n二进制位模式的含义取决于解释规则，二进制补码让加减法共享相同的硬件运算形式。\n\n" + "### 推导示例\n\n可以用固定四位分别解释同一位模式的无符号值和补码值，观察最高位权重的变化。\n\n" + "### 常见误解\n\n位模式本身没有天然正负号；忽略位宽会导致对取值范围和溢出的错误判断。\n\n" + "## 本讲总结\n\n表示范围由位宽和编码约定共同决定，运算结果也必须在同一位宽下解释。\n\n" + "## 章节自测\n\n为什么同一个四位位模式在无符号解释和补码解释下可能代表不同整数？" + ), + ] + + for markdown in prototype_notes: + assessment = assess_lecture_note_structure(markdown) + assert assessment["passed"] is True + assert assessment["score"] == 1.0 + + +def test_document_frame_is_idempotent_and_preserves_course_content(): + from slidenote.notes.document_frame import ensure_lecture_document_frame + from slidenote.notes.structure import assess_lecture_note_structure + + original = ( + "# Algorithms\n\n" + "## 一、Linear Search\n\n" + "### 执行示例\n\n逐项检查数组。 \n\n" + "```c\nfor (int i = 0; i < n; i++) {}\n```\n\n" + "### 易错点\n\n没有排序信息时不能安全丢弃一半候选。\n\n" + "![搜索过程](notes.assets/images/search.png)\n" + ) + brief = { + "brief": { + "core_questions": ["线性搜索与二分搜索的适用条件有什么不同？"], + "chapter_outline": [ + {"title": "Linear Search", "summary": "逐项检查候选，直到命中或遍历结束。"} + ], + } + } + + framed = ensure_lecture_document_frame(original, brief, "zh") + assert ensure_lecture_document_frame(framed, brief, "zh") == framed + assert framed.count("# Algorithms") == 1 + assert framed.count("## 本讲目标") == 1 + assert framed.count("## 本讲总结") == 1 + assert framed.count("## 章节自测") == 1 + assert "for (int i = 0; i < n; i++) {}" in framed + assert "![搜索过程](notes.assets/images/search.png)" in framed + assert "" in framed + assert assess_lecture_note_structure(framed)["passed"] is True + + +def test_structure_contract_rejects_page_listing_and_repeated_global_sections(): + from slidenote.notes.structure import assess_lecture_note_structure + + mechanical = ( + "# Lecture\n\n" + "## 本讲目标\n\n理解搜索。\n\n" + "## 第 1 页：搜索\n\n逐项检查候选。\n\n" + "## 本讲目标\n\n再次理解搜索。\n\n" + "## 本讲总结\n\n搜索需要明确条件。\n\n" + "## 章节自测\n\n搜索何时结束？" + ) + assessment = assess_lecture_note_structure(mechanical) + assert assessment["passed"] is False + assert assessment["mechanical_repetition_pass"] is False + assert assessment["page_headings"] == ["第 1 页：搜索"] + assert assessment["repeated_global_sections"]["orientation"] == 2 + + +def test_structure_contract_rejects_false_positive_headings_and_repeated_templates(): + from slidenote.notes.structure import assess_lecture_note_structure + + body = "这里是一段明显超过十二个字符的有效正文内容，用于结构契约测试。" + slide_heading = ( + f"# X\n\n## 本讲目标\n\n{body}\n\n## Slide 1 - Topic\n\n{body}\n\n" + f"## 本讲总结\n\n{body}\n\n## 章节自测\n\n{body}" + ) + slide_assessment = assess_lecture_note_structure(slide_heading) + assert slide_assessment["passed"] is False + assert slide_assessment["page_headings"] == ["Slide 1 - Topic"] + + example_only = ( + f"# X\n\n## 本讲目标\n\n{body}\n\n## 例子与应用\n\n{body}\n\n" + f"## 本讲总结\n\n{body}\n\n## 章节自测\n\n{body}" + ) + example_assessment = assess_lecture_note_structure(example_only) + assert "topic_sections" in example_assessment["missing_slots"] + + negated_summary = ( + f"# X\n\n## 本讲目标\n\n{body}\n\n## Topic\n\n{body}\n\n" + f"## 这不是总结\n\n{body}\n\n## 章节自测\n\n{body}" + ) + negated_assessment = assess_lecture_note_structure(negated_summary) + assert "summary" in negated_assessment["missing_slots"] + + repeated_template = ( + f"# X\n\n## 本讲目标\n\n{body}\n\n## Topic A\n\n### 核心概念\n\n{body}\n\n" + f"### 易错点\n\n{body}\n\n## Topic B\n\n### 核心概念\n\n{body}\n\n### 易错点\n\n{body}\n\n" + f"## 本讲总结\n\n{body}\n\n## 章节自测\n\n{body}" + ) + repeated_assessment = assess_lecture_note_structure(repeated_template) + assert repeated_assessment["template_repetition_detected"] is True + assert repeated_assessment["mechanical_repetition_pass"] is False + assert repeated_assessment["passed"] is False + + +def test_structure_repair_rejects_severely_truncated_candidate(tmp_path, monkeypatch): + from slidenote.notes.contexts import NoteContext + from slidenote.notes.repair import _body_chars, _repair_note_structure_once + + body = "原始详细内容" * 400 + original = ( + f"# Demo\n\n## 本讲目标\n\n{body}\n\n## Topic\n\n{body}\n\n" + f"![图](img.png)\n\n\n" + f"## 第 1 页：机械标题\n\n{body}\n\n## 本讲总结\n\n{body}\n\n## 章节自测\n\n{body}" + ) + short = "这是足够十二字符以上的简短占位正文。" + candidate = ( + f"# Demo\n\n## 本讲目标\n\n{short}\n\n## Topic\n\n{short}\n\n" + f"![图](img.png)\n\n\n" + f"## 本讲总结\n\n{short}\n\n## 章节自测\n\n{short}" + ) + + def fake_generate(**_kwargs): + return candidate, {"provider_usage": {"finish_reason": "stop"}} + + monkeypatch.setattr("slidenote.notes.repair._generate_cached_llm_text", fake_generate) + deck = Deck( + source_path="lecture.pdf", + source_type="pdf", + pages=[SlidePage(slide_id=1, title="Topic", text_blocks=[TextBlock(id="s1_t1", type="paragraph", content="Topic")])], + ) + context = NoteContext(id="final", kind="final", title="final", pages=deck.pages) + options = NoteOptions( + use_llm=True, + provider="openai", + api_key="test", + cache_mode="off", + max_output_tokens=100_000, + note_profile="lecture-notes", + ) + repaired, record = _repair_note_structure_once(deck, context, original, tmp_path, None, options) + + assert record is not None + assert record["accepted"] is False + assert "body_truncated" in record["rejection_reasons"] + assert repaired == original + assert _body_chars(candidate) < _body_chars(original) * 0.01 + + +def test_section_enrichment_prompt_uses_adaptive_not_six_field_structure(): + from slidenote.notes.contexts import NoteContext + from slidenote.notes.prompt_templates import _llm_teaching_enrichment_prompt + + page = SlidePage(slide_id=1, title="Binary Search") + context = NoteContext(id="sec1", kind="section", title="Binary Search", pages=[page]) + prompt = _llm_teaching_enrichment_prompt( + context=context, + woven_markdown="Binary search halves a sorted range.", + page_markdown_by_slide={1: "Binary search halves a sorted range."}, + source_display="hidden", + note_context="section", + note_profile="lecture-notes", + note_depth="very-detailed", + note_language="zh", + term_policy="bilingual", + ) + + assert "章节结构必须随内容变化" in prompt + assert "默认不要在每章重复学习目标、本节小结和自测" in prompt + assert "每个最终章节都必须包含" not in prompt + + +def test_two_course_benchmarks_receive_deterministic_document_frame_without_extra_repair(tmp_path, monkeypatch): + from slidenote.notes.structure import assess_lecture_note_structure + + prompts = [] + + class FakeClient: + def __init__(self, **kwargs): + pass + + def generate_with_usage(self, prompt): + prompts.append(prompt) + + class Result: + usage = {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7} + + result = Result() + algorithms = "Binary Search" in prompt or "算法只写成" in prompt + if '"task": "page_lecture"' in prompt: + result.text = ( + "Binary Search 依赖有序输入，并通过对半缩小范围降低查找次数。 " + "" + if algorithms + else "Two's Complement 用固定位宽解释有符号整数。 " + ) + elif '"task": "weave_page_lectures"' in prompt: + result.text = ( + "算法只写成逐页摘要，没有形成完整学习结构。 " + if algorithms + else "数值表示只有一段简短摘要。 " + ) + elif '"task": "teaching_enrichment"' in prompt: + result.text = ( + "### 核心概念\n\n二分搜索反复排除一半候选范围。 " + if algorithms + else "### 核心概念\n\n补码用于解释固定位宽的有符号整数。 " + ) + elif '"task": "repair_human_note_structure"' in prompt: + topic = "二分搜索" if algorithms else "二进制补码" + result.text = ( + f"### 学习目标\n\n理解{topic}解决的问题、适用条件和基本推理过程。 \n\n" + f"### 核心概念与机制\n\n{topic}的定义、前提和运作方式需要作为同一条知识链理解。 \n\n" + f"### 例子与应用\n\n用一个不引入额外事实的最小例子检查{topic}如何作用于输入。 \n\n" + "### 易错点\n\n常见错误是忽略适用前提，只记结论而没有检查推理条件。 \n\n" + f"### 本节小结\n\n{topic}应从问题、机制、条件和结果四个方面连贯掌握。 \n\n" + f"### 自测问题\n\n你能说明{topic}成立所依赖的关键条件吗？ " + ) + else: + result.text = "unexpected" + return result + + monkeypatch.setattr("slidenote.notes.llm_calls.LLMClient", FakeClient) + decks = [ + Deck( + source_path="cs50_algorithms.pdf", + source_type="pdf", + pages=[SlidePage(slide_id=1, title="Algorithms", text_blocks=[TextBlock(id="s1_t1", type="paragraph", content="Binary Search requires sorted input and runs in O(log n).")])], + ), + Deck( + source_path="cs61c_number_representation.pptx", + source_type="pptx", + pages=[SlidePage(slide_id=1, title="Number Representation", text_blocks=[TextBlock(id="s1_t1", type="paragraph", content="Two's Complement represents signed integers with a fixed bit width.")])], + ), + ] + + for index, deck in enumerate(decks): + result = generate_notes_result( + deck, + tmp_path / f"benchmark_{index}", + use_llm=True, + provider="openai", + api_key="test", + cache_mode="off", + note_strategy="lecture-weave", + note_profile="lecture-notes", + note_context="document", + ) + assert assess_lecture_note_structure(result.markdown)["passed"] is True + assert result.llm_usage["summary"]["repair_calls"] == 0 + assert result.markdown.count("## 本讲目标") == 1 + assert result.markdown.count("## 本讲总结") == 1 + assert result.markdown.count("## 章节自测") == 1 + + assert sum('"task": "repair_human_note_structure"' in prompt for prompt in prompts) == 0 + + +def test_structure_repair_runs_only_when_contract_fails_and_keeps_safe_candidate(tmp_path, monkeypatch): + """The structure repair fires on mechanical listing and rejects a regressing candidate.""" + from slidenote.notes.structure import assess_lecture_note_structure + + prompts = [] + + class FakeClient: + def __init__(self, **kwargs): + pass + + def generate_with_usage(self, prompt): + prompts.append(prompt) + + class Result: + usage = {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7} + + result = Result() + if '"task": "page_lecture"' in prompt: + result.text = "第 1 页讲了搜索。 " + elif '"task": "weave_page_lectures"' in prompt: + result.text = "## 第 1 页：搜索\n\n逐项检查候选。 " + elif '"task": "teaching_enrichment"' in prompt: + result.text = "## 第 1 页：搜索\n\n逐项检查候选。 " + elif '"task": "repair_human_note_structure"' in prompt: + # A candidate that does NOT improve the structure must be rejected. + result.text = "## 第 1 页：搜索\n\n逐项检查候选。 " + else: + result.text = "unexpected" + return result + + monkeypatch.setattr("slidenote.notes.llm_calls.LLMClient", FakeClient) + deck = Deck( + source_path="lecture.pdf", + source_type="pdf", + pages=[SlidePage(slide_id=1, title="搜索", text_blocks=[TextBlock(id="s1_t1", type="paragraph", content="Binary Search")])], + ) + result = generate_notes_result( + deck, + tmp_path, + use_llm=True, + provider="openai", + api_key="test", + cache_mode="off", + note_strategy="lecture-weave", + note_profile="lecture-notes", + note_context="document", + ) + assert sum('"task": "repair_human_note_structure"' in prompt for prompt in prompts) == 1 + assert result.llm_usage["summary"]["repair_calls"] == 1 + # The regressing candidate was rejected; the frame still guarantees the slots. + assert result.markdown.count("## 本讲目标") == 1 + assessment = assess_lecture_note_structure(result.markdown) + assert assessment["passed"] is False + assert assessment["page_headings"] == ["第 1 页：搜索"] + assert "第 1 页：搜索" in result.markdown + + +def test_quality_report_contains_structure_contract(): + from slidenote.notes.quality import build_note_quality_report + + deck = Deck(source_path="lecture.pdf", source_type="pdf", pages=[SlidePage(slide_id=1, title="T")]) + mechanical = ( + "# Lecture\n\n" + "## 本讲目标\n\n理解搜索算法的适用条件与基本推理过程。\n\n" + "## 第 1 页：搜索\n\n逐项检查候选，直到命中或遍历结束为止。\n\n" + "## 本讲目标\n\n再次理解搜索算法的适用条件与基本推理过程。\n\n" + "## 本讲总结\n\n搜索需要明确条件，命中或遍历结束都算完成。\n\n" + "## 章节自测\n\n搜索何时可以安全地提前结束，何时必须遍历到末尾？" + ) + report = build_note_quality_report( + deck=deck, + notes_markdown=mechanical, + coverage_report=None, + note_profile="lecture-notes", + note_context="document", + note_strategy="lecture-weave", + note_depth="very-detailed", + ) + assert report["structure_contract_pass"] is False + assert report["mechanical_structure_pass"] is False + assert report["summary"]["missing_structure_slots"] == ["topic_sections"] + assert any("结构栏目" in suggestion for suggestion in report["suggested_repairs"]) + assert any("逐页标题" in suggestion for suggestion in report["suggested_repairs"]) diff --git a/tests/test_notes.py b/tests/test_notes.py index 6f8bd11..2e50ce5 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -661,9 +661,10 @@ def test_article_prompt_prefers_study_notes_over_slide_translation(): assert "像学生课后笔记一样" in prompt assert "不要逐字段翻译 PPT" in prompt - assert "article 不是摘要模式" in prompt - assert "详细讲义式学习笔记" in prompt - assert "不降低讲解深度" in prompt + assert "中间证据卡" in prompt + assert "不要为单页固定生成学习目标" in prompt + assert "完整保留当前页独有证据" in prompt + assert "不表示为每页写一篇完整讲义" in prompt assert "封面" in prompt assert "目录" in prompt assert "只保留该页所有元素的来源标记" in prompt @@ -709,7 +710,7 @@ def test_lecture_notes_profile_prompt_requests_teaching_reconstruction(): assert "不是在总结幻灯片" in prompt assert "教学重构" in prompt - assert "是什么、为什么重要、如何运作" in prompt + assert "详细讲解留给章节编织阶段" in prompt assert "coverage 只是最后质检" in prompt @@ -989,11 +990,18 @@ class Result: ) elif '"task": "teaching_enrichment"' in prompt: result.text = ( - "### 本节核心问题\n\n" - "本节要解释为什么复制之后还需要 quorum 来维持读写可见性。 \n\n" + "## 本讲目标\n\n" + "理解复制与 quorum 如何共同影响可用性和读写可见性。 \n\n" + "## 复制与 Quorum\n\n" + "### 两个机制如何衔接\n\n" + "复制提高可用性，而 quorum 通过读写集合相交约束可见性，两者共同构成一致性机制。 \n\n" + "### 检查读写集合的例子\n\n" + "例如，可以检查一次读取所访问的副本集合是否必然与最近一次写入集合相交。 \n\n" "### 易错点\n\n" "不要把副本数量简单等同于一致性；关键是读集合和写集合是否相交。 \n\n" - "### 自测问题\n\n" + "## 本讲总结\n\n" + "复制先提供冗余，quorum 再用集合交集建立读写之间的可见性联系。 \n\n" + "## 章节自测\n\n" "如果读集合和写集合不相交，会出现什么风险？ " ) else: @@ -1025,8 +1033,10 @@ class Result: assert result.llm_usage["summary"]["teaching_enrichment_calls"] == 1 assert result.llm_usage["request"]["note_profile"] == "lecture-notes" assert result.llm_usage["request"]["note_depth"] == "very-detailed" - assert "### 本节核心问题" in result.markdown - assert "### 自测问题" in result.markdown + assert "稳定的全文结构契约" in prompts[-1] + assert "## 本讲目标" in result.markdown + assert "## 复制与 Quorum" in result.markdown + assert "## 章节自测" in result.markdown assert analyze_coverage(deck, result.markdown)["missing"] == 0 diff --git a/tests/test_prompt_context_review.py b/tests/test_prompt_context_review.py new file mode 100644 index 0000000..8cf0500 --- /dev/null +++ b/tests/test_prompt_context_review.py @@ -0,0 +1,50 @@ +import pytest + +from slidenote.models import SlidePage +from slidenote.notes.contexts import NoteContext +from slidenote.notes.prompt_templates import _llm_teaching_enrichment_prompt, _llm_weave_prompt + + +@pytest.mark.parametrize("language", ["zh", "en"]) +def test_document_generation_leaves_h1_to_composition(language): + page = SlidePage(slide_id=1, title="Search") + context = NoteContext(id="document", kind="document", title="Search", pages=[page]) + common = dict( + context=context, + page_markdown_by_slide={1: "Search examines candidate values."}, + source_display="hidden", + note_context="document", + note_profile="lecture-notes", + note_depth="detailed", + note_language=language, + term_policy="bilingual", + ) + woven = _llm_weave_prompt(**common, note_style="article", weave_dedup="normal") + enriched = _llm_teaching_enrichment_prompt(**common, woven_markdown="Search examines candidates.") + + for prompt in (woven, enriched): + assert "keep exactly one H1 title" not in prompt + assert "保留且只保留一个 H1 课程标题" not in prompt + assert "H2" in prompt + assert "do not emit H1" in prompt or "不要输出 H1" in prompt + assert "全文正文使用 H2" in woven + assert "正文小标题只能使用 H3" not in woven + + +def test_section_weaving_keeps_system_outer_heading(): + page = SlidePage(slide_id=1, title="Search") + prompt = _llm_weave_prompt( + context=NoteContext(id="section", kind="section", title="Search", pages=[page]), + page_markdown_by_slide={1: "Search examines candidate values."}, + source_display="hidden", + note_context="section", + note_profile="lecture-notes", + note_depth="detailed", + note_language="zh", + term_policy="bilingual", + note_style="article", + weave_dedup="normal", + ) + + assert "正文小标题只能使用 H3" in prompt + assert "全文正文使用 H2" not in prompt diff --git a/tests/test_structure_repair_review.py b/tests/test_structure_repair_review.py new file mode 100644 index 0000000..e198231 --- /dev/null +++ b/tests/test_structure_repair_review.py @@ -0,0 +1,122 @@ +"""Regression tests for complete-document structure repairs.""" + +from unittest.mock import Mock + +import pytest + +from slidenote.models import Deck, SlidePage, TextBlock +from slidenote.notes import NoteOptions +from slidenote.notes.contexts import NoteContext +from slidenote.notes.repair import _repair_note_structure_once +from slidenote.notes.structure import assess_lecture_note_structure + + +BODY = "Binary search requires sorted input and narrows the candidate range after each comparison." +SOURCE = "" +IMAGE = "![Search](assets/search.png)" +ORIGINAL = ( + f"# 课程笔记：Search\n\n## 本讲目标\n\n{BODY}\n\n" + f"## 第 1 页：搜索\n\n{BODY} {SOURCE}\n\n{IMAGE}\n\n" + f"### 运行示例\n\n{BODY}\n\n### 易错点\n\n{BODY}\n\n" + f"## 本讲总结\n\n{BODY}\n\n## 章节自测\n\n{BODY}" +) +CANDIDATE = ORIGINAL.replace("## 第 1 页：搜索", "## Binary Search") + + +@pytest.fixture +def repair_case(tmp_path): + deck = Deck( + source_path="lecture.pdf", + source_type="pdf", + pages=[SlidePage( + slide_id=1, + title="Search", + text_blocks=[TextBlock(id="s1_t1", type="paragraph", content=BODY)], + )], + ) + return { + "deck": deck, + "context": NoteContext(id="final", kind="final", title="Search", pages=deck.pages), + "output_root": tmp_path, + "cache": None, + "options": NoteOptions(note_profile="lecture-notes", cache_mode="off"), + } + + +def _run_repair(monkeypatch, repair_case, candidate, original=ORIGINAL): + generate = Mock(return_value=(candidate, {"provider_usage": {"finish_reason": "stop"}})) + monkeypatch.setattr("slidenote.notes.repair._generate_cached_llm_text", generate) + markdown, record = _repair_note_structure_once(markdown=original, **repair_case) + generate.assert_called_once() + assert record is not None + return markdown, record + + +def test_successful_structure_repair_preserves_original_document_title(monkeypatch, repair_case): + assert assess_lecture_note_structure(ORIGINAL)["passed"] is False + assert assess_lecture_note_structure(CANDIDATE)["passed"] is True + + markdown, record = _run_repair(monkeypatch, repair_case, CANDIDATE) + + assert record["accepted"] is True + assert record["rejection_reasons"] == [] + assert markdown == CANDIDATE + assert markdown.startswith("# 课程笔记：Search\n") + assert record["validation"]["document_title_preserved"] is True + assert record["validation"]["candidate_document_titles"] == ["课程笔记：Search"] + assert record["validation"]["lost_trace_items"] == [] + assert record["validation"]["lost_visible_items"] == [] + assert record["validation"]["lost_source_markers"] == [] + assert record["validation"]["lost_image_targets"] == [] + + +@pytest.mark.parametrize( + "candidate", + [ + CANDIDATE.replace("# 课程笔记：Search\n\n", ""), + CANDIDATE + "\n\n# Another title", + CANDIDATE.replace("# 课程笔记：Search", "# Renamed course"), + CANDIDATE.replace("# 课程笔记：Search", "```markdown\n# 课程笔记：Search\n```"), + ], + ids=["missing", "duplicate", "changed", "code-only"], +) +def test_structure_repair_rejects_candidate_that_does_not_preserve_single_title( + monkeypatch, repair_case, candidate, +): + markdown, record = _run_repair(monkeypatch, repair_case, candidate) + + assert record["accepted"] is False + assert "document_title_not_preserved" in record["rejection_reasons"] + assert record["validation"]["document_title_preserved"] is False + assert markdown == ORIGINAL + + +def test_structure_repair_preserves_code_heading_without_counting_it_as_document_title( + monkeypatch, repair_case, +): + code = "```markdown\n# Example heading\n```" + original = ORIGINAL.replace(IMAGE, IMAGE + "\n\n" + code) + candidate = CANDIDATE.replace(IMAGE, IMAGE + "\n\n" + code) + + markdown, record = _run_repair(monkeypatch, repair_case, candidate, original=original) + + assert record["accepted"] is True + assert code in markdown + assert record["validation"]["original_document_titles"] == ["课程笔记：Search"] + assert record["validation"]["candidate_document_titles"] == ["课程笔记：Search"] + + +@pytest.mark.parametrize( + "original", + [ + ORIGINAL.replace("# 课程笔记：Search\n\n", ""), + ORIGINAL + "\n\n# Another title", + ], + ids=["missing-original-title", "multiple-original-titles"], +) +def test_structure_repair_requires_one_original_title(monkeypatch, repair_case, original): + markdown, record = _run_repair(monkeypatch, repair_case, CANDIDATE, original=original) + + assert record["accepted"] is False + assert "document_title_not_preserved" in record["rejection_reasons"] + assert markdown == original diff --git a/tests/test_verify_note_layout.py b/tests/test_verify_note_layout.py new file mode 100644 index 0000000..3dc6f68 --- /dev/null +++ b/tests/test_verify_note_layout.py @@ -0,0 +1,59 @@ +import zipfile + +import pymupdf + +from scripts.verify_note_layout import check_docx, check_markdown, check_pdf + + +def test_markdown_layout_checker_flags_broken_structure(tmp_path): + notes = tmp_path / "notes.md" + notes.write_text( + "# One\n\n# Two\n\n### Jumped\n\n![](image.png)\n\n```python\nprint('open')\n", + encoding="utf-8", + ) + + issues, _stats = check_markdown(notes) + rules = {item["rule"] for item in issues} + + assert "multiple_h1" in rules + assert "heading_level_jump" in rules + assert "image_missing_alt" in rules + assert "unbalanced_code_fence" in rules + + +def test_parent_heading_with_populated_subsection_is_not_empty(tmp_path): + notes = tmp_path / "notes.md" + notes.write_text( + "# Course\n\n## Topic\n\n### Concept\n\nA complete explanation lives in the subsection.\n", + encoding="utf-8", + ) + + issues, _stats = check_markdown(notes) + + assert not any(item["rule"] == "empty_section" for item in issues) + + +def test_docx_layout_checker_flags_missing_heading_styles(tmp_path): + docx = tmp_path / "notes.docx" + document_xml = """ + + Plain text only + """ + with zipfile.ZipFile(docx, "w") as archive: + archive.writestr("word/document.xml", document_xml) + + issues = check_docx(docx, {"headings": 1, "tables": 0, "images": 0}) + + assert any(item["rule"] == "docx_no_headings" for item in issues) + + +def test_pdf_layout_checker_flags_blank_page(tmp_path): + pdf = tmp_path / "notes.pdf" + document = pymupdf.open() + document.new_page() + document.save(pdf) + document.close() + + issues = check_pdf(pdf) + + assert any(item["rule"] == "blank_page" for item in issues)