diff --git a/.github/scripts/prepare-cli-release.mjs b/.github/scripts/prepare-cli-release.mjs index e7fefa0..457fcf4 100644 --- a/.github/scripts/prepare-cli-release.mjs +++ b/.github/scripts/prepare-cli-release.mjs @@ -742,8 +742,9 @@ export function collectCliEvidence({ curation_policy: [ "Summarize only user-visible MemOS CLI changes from the evidence.", "Do not present CI, packaging, release automation, or test-only work as product features.", + "Use source_refs only from required_source_refs.accepted_refs; omit every item backed only by other commits.", "Group related commits into concise Added, Improved, or Fixed bullets.", - "Preserve all covered source_refs when commits are grouped.", + "Preserve all covered required source_refs when commits are grouped.", "Do not mention private endpoints, credentials, internal infrastructure, or raw build paths.", ], }, @@ -1188,6 +1189,83 @@ function sourceRefMatches(left, right) { return false; } +export function curateDraftByEvidence(draft, evidence) { + if (!evidence.has_user_facing_product_changes) { + return { + ...draft, + evidence_curation: { + removed_item_count: 0, + removed_source_ref_count: 0, + removed_items: [], + }, + }; + } + + const validRefs = evidence.commits.flatMap( + (commit) => commit.source_refs || [], + ); + for (const pull of evidence.pull_requests || []) { + validRefs.push(`#${pull.number}`); + } + const userFacingRefs = (evidence.required_source_refs || []).flatMap( + (required) => required.accepted_refs || [], + ); + const matchesAny = (ref, candidates) => + candidates.some((candidate) => sourceRefMatches(ref, candidate)); + + const releaseItems = []; + const removedItems = []; + let removedSourceRefCount = 0; + for (const [index, item] of draft.release_items.entries()) { + const knownRefs = item.source_refs.filter((ref) => + matchesAny(ref, validRefs), + ); + const eligibleRefs = item.source_refs.filter((ref) => + matchesAny(ref, userFacingRefs), + ); + const unknownRefs = item.source_refs.filter( + (ref) => !matchesAny(ref, validRefs), + ); + + if ( + item.source_refs.length > 0 && + knownRefs.length === item.source_refs.length && + eligibleRefs.length === 0 + ) { + removedItems.push({ + index, + reason: "only_non_user_facing_source_refs", + source_refs: [...item.source_refs], + }); + removedSourceRefCount += item.source_refs.length; + continue; + } + + if (eligibleRefs.length > 0 && unknownRefs.length === 0) { + removedSourceRefCount += item.source_refs.length - eligibleRefs.length; + releaseItems.push({ + ...item, + source_refs: [...new Set(eligibleRefs)], + }); + continue; + } + + // Preserve empty or unknown refs so deterministic validation still fails + // closed instead of hiding malformed Doc Agent output. + releaseItems.push(item); + } + + return { + ...draft, + release_items: releaseItems, + evidence_curation: { + removed_item_count: removedItems.length, + removed_source_ref_count: removedSourceRefCount, + removed_items: removedItems, + }, + }; +} + export function validateDraft(draft, evidence) { const issues = []; const validRefs = new Set( @@ -1409,13 +1487,16 @@ async function requestOneDraft({ export async function requestDocAgentDraft(evidence) { if (!evidence.has_user_facing_product_changes) { - const draft = normalizeDraft({ - ok: true, - needs_review: false, - confidence: "high", - warnings: [evidence.skip_reason], - release_items: [], - }); + const draft = curateDraftByEvidence( + normalizeDraft({ + ok: true, + needs_review: false, + confidence: "high", + warnings: [evidence.skip_reason], + release_items: [], + }), + evidence, + ); return { ...draft, validation_report: validateDraft(draft, evidence), @@ -1430,7 +1511,10 @@ export async function requestDocAgentDraft(evidence) { process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_TOKEN || "", ).trim(); if ((!url || !token) && process.env.ALLOW_OFFLINE_DOCS_PREVIEW === "true") { - const draft = normalizeDraft(offlineDraft(evidence)); + const draft = curateDraftByEvidence( + normalizeDraft(offlineDraft(evidence)), + evidence, + ); return { ...draft, validation_report: validateDraft(draft, evidence), @@ -1461,11 +1545,14 @@ export async function requestDocAgentDraft(evidence) { repairContext: null, history: [], }); - const draft = injectDraftFault( - normalizeDraft(payload), + const draft = curateDraftByEvidence( + injectDraftFault( + normalizeDraft(payload), + evidence, + faultCase, + { validationRound: 1 }, + ), evidence, - faultCase, - { validationRound: 1 }, ); const validation = validateDraft(draft, evidence); candidates.push({ @@ -1553,7 +1640,9 @@ export async function requestDocAgentDraft(evidence) { "Use only facts present in the evidence.", "Return concise Added, Improved, or Fixed release_items.", "Each item must contain text_cn, text_en, and valid source_refs.", - "Merge duplicate topics while preserving all source_refs.", + "Use source_refs only from required_source_refs.accepted_refs.", + "Delete every item backed only by commits outside required_source_refs.", + "Merge duplicate topics while preserving every eligible source_ref from required_source_refs.", "Do not expose internal infrastructure or copy raw commit subjects.", ], }; @@ -1579,11 +1668,14 @@ export async function requestDocAgentDraft(evidence) { }); throw error; } - const draft = injectDraftFault( - normalizeDraft(payload), + const draft = curateDraftByEvidence( + injectDraftFault( + normalizeDraft(payload), + evidence, + faultCase, + { validationRound: repairAttempt + 1 }, + ), evidence, - faultCase, - { validationRound: repairAttempt + 1 }, ); const validation = validateDraft(draft, evidence); if (!validation.ok) { @@ -2133,6 +2225,7 @@ async function main() { confidence: draft.confidence, release_items: draft.release_items, candidate_selection: draft.candidate_selection, + evidence_curation: draft.evidence_curation, validation_report: draft.validation_report, validation_attempt_count: draft.validation_attempt_count, repair_attempt_count: draft.repair_attempt_count, @@ -2166,6 +2259,7 @@ async function main() { validation: draft.validation_report, coverage: draft.validation_report.coverage, candidate_selection: draft.candidate_selection, + evidence_curation: draft.evidence_curation, methodology: RELEASE_NOTE_METHODS, warnings: [...draft.warnings, releaseNotes.warning].filter(Boolean), }); diff --git a/.github/scripts/prepare-cli-release.test.mjs b/.github/scripts/prepare-cli-release.test.mjs index cfcdd0b..a8513a2 100644 --- a/.github/scripts/prepare-cli-release.test.mjs +++ b/.github/scripts/prepare-cli-release.test.mjs @@ -20,6 +20,7 @@ import { collectCliEvidence, compareSemver, composePublicReleaseNotes, + curateDraftByEvidence, docsPreviewMarkdown, findPreviousTag, generateGitHubReleaseNotes, @@ -945,6 +946,7 @@ test("runs an end-to-end offline dry run and writes the inspection contract", () ); assert.deepEqual(report.product_paths, ["**"]); assert.equal(report.coverage.missing_required_count, 0); + assert.equal(report.evidence_curation.removed_item_count, 0); const evidence = JSON.parse( readFileSync(join(inspection, "evidence.json"), "utf8"), ); @@ -953,6 +955,7 @@ test("runs an end-to-end offline dry run and writes the inspection contract", () readFileSync(join(inspection, "release-notes-draft.json"), "utf8"), ); assert.equal(acceptedDraft.ok, true); + assert.equal(acceptedDraft.evidence_curation.removed_item_count, 0); assert.ok(acceptedDraft.release_items[0].source_refs.length > 0); assert.match( readFileSync(join(inspection, "README.md"), "utf8"), @@ -1215,6 +1218,108 @@ test("requests three independent candidates and deterministically selects the va } }); +test("accepts the real run shape after pruning an extra automation-only item", async () => { + const previousFetch = globalThis.fetch; + const previousUrl = process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_URL; + const previousToken = process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_TOKEN; + const productSha = "a6199ed6c338f8369bf5fccef26c883701ae58bb"; + const automationSha = "9c57e9c60a7cdc3b0ce23de75f1c7a7307138c0d"; + const mergeSha = "cd568a3945fc9d97cd540f2ef97665b88e1cd877"; + const extendedEvidence = { + ...evidence, + commits: [ + { + sha: productSha, + short_sha: "a6199ed", + subject: "feat: add DeepSeek Harness agent integration (#30)", + source_refs: ["a6199ed", productSha, "#30"], + }, + { + sha: automationSha, + short_sha: "9c57e9c", + subject: "ci: fix CLI release trigger and permissions", + source_refs: ["9c57e9c", automationSha], + }, + { + sha: mergeSha, + short_sha: "cd568a3", + subject: "Merge pull request #31 from MemTensor/docs-sync/memos-cli-release-trigger-fix", + source_refs: ["cd568a3", mergeSha, "#31"], + }, + ], + pull_requests: [{ number: "30" }, { number: "31" }], + important_commits: [ + { + sha: productSha, + short_sha: "a6199ed", + subject: "feat: add DeepSeek Harness agent integration (#30)", + }, + ], + required_source_refs: [ + { + short_sha: "a6199ed", + accepted_refs: ["a6199ed", productSha, "#30"], + }, + ], + }; + const response = { + ...validDraft, + release_items: [ + { + ...validDraft.release_items[0], + text_cn: + "**DeepSeek Harness 集成**:初始化时生成适配的代理指引,便于直接连接 MemOS 记忆能力。", + text_en: + "**DeepSeek Harness integration**: Generates compatible agent guidance during initialization so MemOS memory can be connected directly.", + source_refs: ["a6199ed", "#30"], + }, + { + category: "Improved", + text_cn: "**发布流程**:调整内部发布检查,使自动化运行更加稳定。", + text_en: + "**Release workflow**: Adjusted internal release checks for more stable automation.", + source_refs: ["9c57e9c", "cd568a3"], + }, + ], + }; + let requestCount = 0; + try { + process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_URL = + "https://example.invalid/draft"; + process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_TOKEN = "test-token"; + globalThis.fetch = async () => { + requestCount += 1; + return { + ok: true, + status: 200, + text: async () => JSON.stringify(response), + }; + }; + + const result = await requestDocAgentDraft(extendedEvidence); + assert.equal(requestCount, 3); + assert.equal(result.validation_report.ok, true); + assert.equal(result.validation_attempt_count, 1); + assert.equal(result.repair_attempt_count, 0); + assert.equal(result.release_items.length, 1); + assert.deepEqual(result.release_items[0].source_refs, ["a6199ed", "#30"]); + assert.equal(result.evidence_curation.removed_item_count, 1); + assert.equal(result.evidence_curation.removed_source_ref_count, 2); + } finally { + globalThis.fetch = previousFetch; + if (previousUrl === undefined) { + delete process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_URL; + } else { + process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_URL = previousUrl; + } + if (previousToken === undefined) { + delete process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_TOKEN; + } else { + process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_TOKEN = previousToken; + } + } +}); + test("sends deterministic validation feedback to Doc Agent for repair", async () => { const previousFetch = globalThis.fetch; const previousUrl = process.env.DOC_AGENT_RELEASE_NOTES_DRAFT_URL; @@ -1527,6 +1632,144 @@ test("rejects a real source ref when it belongs only to release automation", () ); }); +test("deterministically removes release items backed only by automation evidence", () => { + const automationSha = "def5678000000000000000000000000000000000"; + const extendedEvidence = { + ...evidence, + commits: [ + ...evidence.commits, + { + sha: automationSha, + short_sha: "def5678", + subject: "fix(ci): repair release workflow", + source_refs: ["def5678", automationSha, "#99"], + }, + ], + pull_requests: [...evidence.pull_requests, { number: "99" }], + }; + const result = curateDraftByEvidence( + { + ...validDraft, + release_items: [ + validDraft.release_items[0], + { + category: "Improved", + text_cn: + "**发布流程**:调整内部发布检查,使自动化运行更加稳定。", + text_en: + "**Release workflow**: Adjusted internal release checks for more stable automation.", + source_refs: ["def5678", "#99"], + }, + ], + }, + extendedEvidence, + ); + + assert.equal(result.release_items.length, 1); + assert.deepEqual(result.release_items[0].source_refs, ["abc1234", "#31"]); + assert.equal(result.evidence_curation.removed_item_count, 1); + assert.equal(result.evidence_curation.removed_source_ref_count, 2); + assert.equal(validateDraft(result, extendedEvidence).ok, true); +}); + +test("strips known automation refs from an otherwise user-facing item", () => { + const automationSha = "def5678000000000000000000000000000000000"; + const extendedEvidence = { + ...evidence, + commits: [ + ...evidence.commits, + { + sha: automationSha, + short_sha: "def5678", + subject: "fix(ci): repair release workflow", + source_refs: ["def5678", automationSha], + }, + ], + }; + const result = curateDraftByEvidence( + { + ...validDraft, + release_items: [ + { + ...validDraft.release_items[0], + source_refs: ["abc1234", "def5678", "#31"], + }, + ], + }, + extendedEvidence, + ); + + assert.deepEqual(result.release_items[0].source_refs, ["abc1234", "#31"]); + assert.equal(result.evidence_curation.removed_item_count, 0); + assert.equal(result.evidence_curation.removed_source_ref_count, 1); + assert.equal(validateDraft(result, extendedEvidence).ok, true); +}); + +test("preserves unknown refs so malformed Doc Agent output still fails closed", () => { + const result = curateDraftByEvidence( + { + ...validDraft, + release_items: [ + { + ...validDraft.release_items[0], + source_refs: ["not-real"], + }, + ], + }, + evidence, + ); + + assert.deepEqual(result.release_items[0].source_refs, ["not-real"]); + assert.equal(result.evidence_curation.removed_item_count, 0); + assert.ok( + validateDraft(result, evidence).issues.some( + (item) => item.kind === "invalid_source_ref", + ), + ); +}); + +test("still fails closed when Doc Agent returns only automation items", () => { + const automationSha = "def5678000000000000000000000000000000000"; + const extendedEvidence = { + ...evidence, + commits: [ + ...evidence.commits, + { + sha: automationSha, + short_sha: "def5678", + subject: "fix(ci): repair release workflow", + source_refs: ["def5678", automationSha], + }, + ], + }; + const result = curateDraftByEvidence( + { + ...validDraft, + release_items: [ + { + category: "Improved", + text_cn: + "**发布流程**:调整内部发布检查,使自动化运行更加稳定。", + text_en: + "**Release workflow**: Adjusted internal release checks for more stable automation.", + source_refs: ["def5678"], + }, + ], + }, + extendedEvidence, + ); + const validation = validateDraft(result, extendedEvidence); + + assert.equal(result.release_items.length, 0); + assert.equal(validation.ok, false); + assert.ok( + validation.issues.some((item) => item.kind === "empty_release_items"), + ); + assert.ok( + validation.issues.some((item) => item.kind === "missing_required_ref"), + ); +}); + test("rejects all generated items when the release has no user-facing changes", () => { const result = validateDraft(validDraft, { ...evidence,