From 3487103863c02d843e3cd255be4a79acea1a4c5c Mon Sep 17 00:00:00 2001 From: kaihere14 Date: Thu, 3 Sep 2026 18:04:32 +0530 Subject: [PATCH 1/3] fix: improve README generation and cleanup flow --- server/src/controllers/github.controller.js | 6 +- server/src/llm/llm.service.js | 6 + server/src/llm/prompts/cleanup.prompt.js | 99 +++++++++++++++ server/src/llm/prompts/detect.prompt.js | 23 ++-- .../src/llm/prompts/full.generate.prompt.js | 116 ++++++++++++++---- .../src/llm/prompts/patch.generate.prompt.js | 41 ++++--- server/src/llm/providers/gemini.provider.js | 11 +- server/src/llm/readme.generate.js | 4 +- server/src/utils/git.worker.js | 9 +- 9 files changed, 262 insertions(+), 53 deletions(-) create mode 100644 server/src/llm/prompts/cleanup.prompt.js diff --git a/server/src/controllers/github.controller.js b/server/src/controllers/github.controller.js index 9bb9e48..429f025 100644 --- a/server/src/controllers/github.controller.js +++ b/server/src/controllers/github.controller.js @@ -600,10 +600,10 @@ export const cleanUpReadme = async (req, res) => { } const userId = req.userId; - const activeRepo = await ActiveRepo.findOne({ repoId, userId }); + const activeRepo = await ActiveRepo.findOne({ repoId, userId, active: true }); if (!activeRepo) { - console.log("[cleanUpReadme] Active repository not found"); - return res.status(404).json({ message: "Active repository not found" }); + console.log("[cleanUpReadme] Please activate the repository first"); + return res.status(404).json({ message: "Please activate the repository first" }); } console.log( diff --git a/server/src/llm/llm.service.js b/server/src/llm/llm.service.js index ec5cb62..75f25be 100644 --- a/server/src/llm/llm.service.js +++ b/server/src/llm/llm.service.js @@ -11,10 +11,12 @@ export class LlmService { // Model ids only — GeminiProvider binds them to whichever API key is live. detectionModel = "gemini-3.5-flash-lite"; generationModel = "gemini-3.6-flash"; + cleanupModel = "gemini-3.6-flash"; geminiProvider = new GeminiProvider({ detectionModel: this.detectionModel, generationModel: this.generationModel, + cleanupModel: this.cleanupModel, }); async generate({ @@ -89,4 +91,8 @@ export class LlmService { return this.geminiProvider.detect(existingReadme); } + + async cleanup(existingReadme) { + return this.geminiProvider.cleanup(existingReadme); + } } diff --git a/server/src/llm/prompts/cleanup.prompt.js b/server/src/llm/prompts/cleanup.prompt.js new file mode 100644 index 0000000..2c79709 --- /dev/null +++ b/server/src/llm/prompts/cleanup.prompt.js @@ -0,0 +1,99 @@ +export function buildCleanupPrompt(existingReadme) { + return ` +You are a senior technical writer and open-source maintainer. You specialize in +rewriting messy, bloated, or poorly organized README files into clean, standard, +professional documentation. + +You will be given the FULL current README of a project. It may be long and +cluttered, or short and underdeveloped, or somewhere in between. It was likely +written incrementally by different people and never edited as a whole. + +Your job is to produce a single rewritten README.md that keeps every real fact +from the input but presents it clearly, in a conventional structure, at a +professional standard of writing and formatting. + +## Absolute constraints + +- Work ONLY from the content in the input README. You have no access to the + source code, so you cannot verify anything that is not already stated. +- Do NOT invent, guess, or "fill in" features, commands, install steps, config + keys, environment variables, APIs, version numbers, URLs, badges, license + names, author names, or requirements. If it is not in the input, it does not + go in the output. +- Do NOT delete real information. If a fact is accurate but badly placed or badly + worded, move it and rewrite it — do not drop it. +- If the input clearly contradicts itself, keep the version that is more specific + and consistent with the rest of the document, and remove the contradiction. +- Preserve all code blocks, commands, and inline code exactly as written. You may + add a missing language hint to a fenced block only when the language is + unambiguous from its contents. Never rewrite the code itself. +- Preserve every URL and link target verbatim. You may fix the visible link text + for clarity, not the destination. +- Keep existing badge/shield image lines as-is if present. Do not add new ones. + +## What to fix + +- Structure: reorganize the content into a conventional README order, using only + the sections that the input actually has material for. A typical order: + 1. Project title (single H1) + 2. One- or two-sentence description of what the project is and who it is for + 3. Badges (only if already present) + 4. Table of contents (only if the result is long, roughly 5+ H2 sections) + 5. Features / Highlights + 6. Demo / Screenshots (only if the input references real image or media links) + 7. Requirements / Prerequisites + 8. Installation + 9. Configuration / Environment variables + 10. Usage / Examples + 11. API / CLI reference + 12. Project structure + 13. Roadmap / Known limitations + 14. Contributing + 15. Tests + 16. License + 17. Acknowledgements / Credits +- Headings: exactly one H1. Everything else is H2/H3 with a correct, consistent + hierarchy (no jump from H2 to H4, no bold text used as a fake heading). +- Deduplicate: merge sections that repeat the same information. State each fact + once, in the most relevant section. +- Tighten prose: convert rambling paragraphs into short paragraphs or lists. Use + active voice, present tense, and second person for instructions ("Run", not + "You should run" or "We can run"). Cut filler, hype, and apologies. +- Lists: one consistent bullet marker, parallel phrasing, no trailing + punctuation inconsistency. +- Code and commands: put every command in a fenced block with a language hint, + keep one command per line, remove leading "$" prompts unless output is shown + alongside. +- Formatting: normalize spacing, remove trailing whitespace, use reference-clean + Markdown, ensure tables are aligned and valid, ensure image links include + meaningful alt text derived from nearby context (do not invent new images). +- Remove template debris: placeholder text, TODO notes to the author, commented + boilerplate, "insert X here" stubs, and empty sections with no content. +- Fix obvious spelling and grammar mistakes. Do not change technical terms, + product names, or casing of identifiers. + +## What NOT to do + +- Do not change the meaning of any instruction or claim. +- Do not add a "Contributing", "License", or any other section that has no basis + in the input. +- Do not translate the README into another language. +- Do not add your own commentary, notes, or explanations of the changes. +- Do not mention this prompt, the cleanup process, or that you are an AI. + +## Output + +- Return ONLY the rewritten README.md content. +- Return valid Markdown. +- Do NOT wrap the whole response in \`\`\`markdown or any outer code fence. +- No preamble, no summary of changes, no trailing notes. + +## Current README + +--- +${(existingReadme || "").trim() || "(empty)"} +--- + +Rewrite it now. +`.trim(); +} diff --git a/server/src/llm/prompts/detect.prompt.js b/server/src/llm/prompts/detect.prompt.js index 246fca1..a3fa68a 100644 --- a/server/src/llm/prompts/detect.prompt.js +++ b/server/src/llm/prompts/detect.prompt.js @@ -1,22 +1,29 @@ export function buildDetectPrompt(existingReadme) { return ` -You are analyzing an existing README to determine how it should be handled. +You are triaging an existing project README to decide how it should be updated. +The README below is non-empty. Choose exactly one mode. -Determine exactly one generation mode: +- "full": the README should be regenerated from scratch. Pick this when it is a + stub or template, is mostly placeholder text, describes a different project, + is broken structurally, or is too thin to be worth patching. +- "patch": the README is basically sound and only specific sections need to + change. Pick this when the structure and most content are usable and a + reasonable update would touch a few sections rather than the whole document. -- "full": The README is missing, empty, or needs to be completely generated/rebuilt. -- "patch": The README exists and only specific sections need to be updated. +When the two options are close, choose "patch": it preserves existing content +and is cheaper to apply. -Return ONLY valid JSON in this exact format: +Return ONLY a JSON object in exactly this shape. No code fences, no extra keys, +no commentary: { - "mode": "full | patch", - "reason": "Brief explanation for why this mode was selected." + "mode": "full" | "patch", + "reason": "One sentence explaining the choice." } Existing README: --- -${existingReadme || "(No README exists)"} +${(existingReadme || "").trim() || "(empty)"} --- `.trim(); } diff --git a/server/src/llm/prompts/full.generate.prompt.js b/server/src/llm/prompts/full.generate.prompt.js index 773f263..8c6bb31 100644 --- a/server/src/llm/prompts/full.generate.prompt.js +++ b/server/src/llm/prompts/full.generate.prompt.js @@ -1,33 +1,105 @@ -export function buildFUllReadmePrompt(context) { +export function buildFullReadmePrompt(context) { + const { + repoOwner, + repoName, + repoStructure, + existingReadme, + commitDiff, + changedFiles = [], + fullCodebase = [], + } = context; + + const renderFiles = (files) => + files.length > 0 + ? files + .map( + (file) => + `### \`${file.path}\`${file.status ? ` (${file.status})` : ""}\n\`\`\`${file.language || ""}\n${file.content}\n\`\`\`\n`, + ) + .join("\n") + : "(none)"; + return ` -You are an expert technical writer and software engineer. +You are a senior software engineer and technical writer. You write the README a +developer wants when they open an unfamiliar repository for the first time: +accurate, well structured, and free of filler. + +Generate a complete README.md for the repository described below, using ONLY the +provided repository structure, existing README, commit summary, changed files, +and source code. + +## Absolute constraints + +- Base every statement on the provided context. You have no other knowledge of + this project. +- Do NOT invent features, commands, scripts, APIs, endpoints, environment + variables, config keys, dependencies, version numbers, license names, or + authors. If the context does not show it, leave it out. +- Prefer facts visible in the source code over claims in the existing README. If + the existing README disagrees with the code, follow the code. +- Keep accurate, still-relevant material from the existing README, but rewrite it + for clarity rather than copying it verbatim. +- Derive install and run instructions from real evidence: manifest files + (package.json, pyproject.toml, go.mod, Cargo.toml, Dockerfile, Makefile, etc.), + scripts, and entry points visible in the context. Do not guess a package + manager or command that the evidence does not support. +- If a common section has no supporting evidence, omit it. Never write + placeholder text. + +## Writing standards + +- Start the output with a single H1 title line (\`# Project Name\`) and nothing + before it. Use only H2/H3 below it, in a consistent hierarchy. +- Follow the title with a one- or two-sentence description of what the project + does and who it is for. +- Order sections conventionally, including only those with real content: + description, badges (only if present in the existing README), table of contents + (only when the result has roughly 5 or more H2 sections), features, + requirements, installation, configuration, usage / examples, API or CLI + reference, project structure, tests, roadmap or limitations, contributing (only + if the existing README or a CONTRIBUTING file supports it), license. +- Put every command in a fenced block with a language hint, one command per line, + no leading "$". +- Use active voice, present tense, and second person for instructions. Be + concise. No marketing language, no "simply", no apologies. +- Use valid, consistently formatted Markdown: one bullet style, aligned tables, + meaningful link text, alt text on images. -Generate a complete, accurate, and professional README.md for the repository described by the context below. +## Repository -Your job is to understand the project from the provided repository structure, existing README, commit information, changed files, and source code, then produce the best possible README for a developer who is discovering this repository for the first time. +${repoOwner || "(unknown)"}/${repoName || "(unknown)"} -## Requirements +## Repository Structure -- Return ONLY the README content. -- Return valid Markdown. -- Do not wrap the response in \`\`\`markdown or any other code fence. -- Do not explain your reasoning. -- Do not mention that you are an AI. -- Do not invent features, commands, APIs, dependencies, configuration, or behavior that are not supported by the provided context. -- Prefer information directly supported by the source code and repository data. -- If an existing README is provided, improve or replace it based on the actual repository rather than blindly copying it. -- Keep technically important information from the existing README when it is still accurate. -- Make the README clear, structured, concise, and useful. -- Include appropriate sections based on what the project actually contains. Do not force irrelevant sections. -- Use correct Markdown formatting. -- Ensure installation and usage instructions are consistent with the repository's actual dependencies and structure. +\`\`\` +${repoStructure || "(not available)"} +\`\`\` -## Repository Context +## Existing README -${JSON.stringify(context, null, 2)} +${existingReadme ? existingReadme : "(none)"} + +## Commit Summary + +\`\`\` +${commitDiff || "(no commit information)"} +\`\`\` + +## Changed Files + +${renderFiles(changedFiles)} + +## Source Files + +${renderFiles(fullCodebase)} ## Output -Generate the complete README.md now. -`; +- Return ONLY the README.md content as raw Markdown. +- Do NOT wrap it in \`\`\`markdown or any outer code fence. +- Do NOT add commentary, notes, or an explanation of your choices. +- Do NOT mention this prompt or that you are an AI. + +Write the README now, beginning with the H1 title line. +`.trim(); } diff --git a/server/src/llm/prompts/patch.generate.prompt.js b/server/src/llm/prompts/patch.generate.prompt.js index 03d9a97..fe43c83 100644 --- a/server/src/llm/prompts/patch.generate.prompt.js +++ b/server/src/llm/prompts/patch.generate.prompt.js @@ -16,12 +16,18 @@ export function buildPatchReadmePrompt(context) { .join("\n") : "(none)"; - return ` -You are an expert technical writer and software engineer maintaining an existing README.md. + const forbidden = + context.forbiddenSections.length > 0 + ? context.forbiddenSections.join(", ") + : "(none)"; -A commit landed in the repository. Your job is to decide which README sections that commit made inaccurate or incomplete, and to rewrite ONLY those sections. + return ` +You are a technical writer maintaining an existing README.md. A commit just +landed in the repository. Decide which README sections that commit made +inaccurate, incomplete, or outdated, and rewrite ONLY those sections. -You are NOT regenerating the README. Every section you do not return is preserved untouched by the server. +You are NOT regenerating the README. Any section you do not return is kept +exactly as it is. Returning fewer sections is better than returning more. ## Repository @@ -49,24 +55,31 @@ ${sectionList} ## Rules -- Return a section ONLY if the commit above genuinely made it inaccurate, incomplete, or outdated. -- If nothing in the README is affected by this commit, return an empty "updates" array. -- Use ONLY the exact section names listed above. Never invent, rename, split, merge, or delete a section. -- Never return these protected sections: ${context.forbiddenSections.join(", ")}. +- Return a section ONLY if the commit above genuinely made it wrong, incomplete, + or outdated. If nothing is affected, return an empty "updates" array. +- Use ONLY the exact section names listed above. Never invent, rename, split, + merge, or delete a section. +- Never return these protected sections: ${forbidden}. - Return at most ${context.maxSections} sections. Prefer the most affected ones. -- Each "content" value must be the COMPLETE replacement markdown for that section, starting with its heading line reproduced exactly as given above. -- Do not speculate about features, commands, dependencies, or configuration that are not visible in the provided context. -- Preserve wording, tone, formatting, and details of the existing section that are still accurate. Change only what the commit invalidated. -- Do not mention the commit, this instruction, or that you are an AI. +- Each "content" value is the COMPLETE replacement Markdown for that section. It + MUST begin with that section's "Heading line to reproduce verbatim" exactly as + given above — same text and same heading level (number of leading \`#\`). +- Change only what the commit invalidated. Preserve the wording, tone, structure, + and detail of the rest of the section. +- Base every change on the provided commit, changed files, and repository + structure. Do not add features, commands, dependencies, or configuration that + are not visible in that context. +- Do not mention the commit, these instructions, or that you are an AI. ## Output -Return ONLY valid JSON in this exact shape. No markdown fences, no commentary: +Return ONLY a valid JSON object in exactly this shape — no code fences, no +commentary. Escape every newline inside "content" as \\n: { "updates": [ { "section": "Installation", "content": "## Installation\\n\\nUpdated content..." } ] } -`; +`.trim(); } diff --git a/server/src/llm/providers/gemini.provider.js b/server/src/llm/providers/gemini.provider.js index 66e14e5..5cb76e2 100644 --- a/server/src/llm/providers/gemini.provider.js +++ b/server/src/llm/providers/gemini.provider.js @@ -3,6 +3,7 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { aiCall } from "../ai.sdk.js"; import { buildDetectPrompt } from "../prompts/detect.prompt.js"; import { extractJson } from "../utils/response.js"; +import { buildCleanupPrompt } from "../prompts/cleanup.prompt.js"; // Keys are tried in slot order. Unset or blank slots are dropped, so a // half-filled .env still works instead of burning an attempt on nothing. @@ -57,9 +58,10 @@ function describe(error) { export class GeminiProvider { // detectionModel/generationModel are Gemini model ids. The provider binds // them to a key itself, because the key is what rotates — not the model. - constructor({ detectionModel, generationModel }) { + constructor({ detectionModel, generationModel, cleanupModel }) { this.detectionModel = detectionModel; this.generationModel = generationModel; + this.cleanupModel = cleanupModel; // One AI SDK client per usable key, built once and reused for every call. this.clients = loadGeminiKeys().map((apiKey) => @@ -123,4 +125,11 @@ export class GeminiProvider { temperature: 0, }); } + + async cleanup(existingReadme) { + return this.#call(this.cleanupModel, { + prompt: buildCleanupPrompt(existingReadme), + temperature: 0, + }); + } } diff --git a/server/src/llm/readme.generate.js b/server/src/llm/readme.generate.js index 299adb9..2e8d5e6 100644 --- a/server/src/llm/readme.generate.js +++ b/server/src/llm/readme.generate.js @@ -1,5 +1,5 @@ import { liveUpdate } from "../services/convex.service.js"; -import { buildFUllReadmePrompt } from "./prompts/full.generate.prompt.js"; +import { buildFullReadmePrompt } from "./prompts/full.generate.prompt.js"; // Exported so the patch pipeline renders commit diffs identically to full mode. export function formatCommitDiff(commitData) { @@ -296,7 +296,7 @@ export async function generateReadme({ context = optimizeContext(context, 8000); } - let prompt = buildFUllReadmePrompt(context); + let prompt = buildFullReadmePrompt(context); console.log(`[LLM] Generating README with AI`); liveUpdate(sharedLogId, `Generating README with AI`); diff --git a/server/src/utils/git.worker.js b/server/src/utils/git.worker.js index 7f5cab3..1c67e15 100644 --- a/server/src/utils/git.worker.js +++ b/server/src/utils/git.worker.js @@ -535,9 +535,12 @@ async function cleanupHandler(job) { liveUpdate(sharedLogId, "Fetched existing README.md"); liveUpdate(sharedLogId, "Cleaning README content with AI"); console.log("[cleanUpReadme] Running AI cleanup"); - const cleanedReadme = await cleanReadmeWithAI(readmeFile.content, (msg) => - liveUpdate(sharedLogId, msg), - ); + const llmService = new LlmService(); + const cleanedReadme = await llmService.cleanup(readmeFile.content); + if (!cleanedReadme) { + liveUpdate(sharedLogId, "AI cleanup returned empty content"); + throw new Error("AI cleanup returned empty content"); + }; console.log("[cleanUpReadme] AI cleanup complete"); liveUpdate(sharedLogId, `Cleanup complete (${cleanedReadme.length} chars)`); From 743794ee54d01c11f462e8e413ec04bd12e29c7e Mon Sep 17 00:00:00 2001 From: kaihere14 Date: Thu, 3 Sep 2026 18:15:08 +0530 Subject: [PATCH 2/3] fix: increase Gemini context and output limits --- server/src/llm/providers/gemini.provider.js | 4 ++++ server/src/llm/readme.generate.js | 16 ++++++++-------- server/src/llm/readme.patch.js | 4 ++-- server/src/utils/repo.limits.js | 18 ++++++++++-------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/server/src/llm/providers/gemini.provider.js b/server/src/llm/providers/gemini.provider.js index 5cb76e2..db67248 100644 --- a/server/src/llm/providers/gemini.provider.js +++ b/server/src/llm/providers/gemini.provider.js @@ -119,10 +119,13 @@ export class GeminiProvider { } // Main model — free-form README generation, no JSON parsing here. + // maxOutputTokens set explicitly: a README fits well under 16K, and the SDK + // default is too low to trust for a full-length document. async generate(prompt) { return this.#call(this.generationModel, { prompt, temperature: 0, + maxOutputTokens: 16000, }); } @@ -130,6 +133,7 @@ export class GeminiProvider { return this.#call(this.cleanupModel, { prompt: buildCleanupPrompt(existingReadme), temperature: 0, + maxOutputTokens: 16000, }); } } diff --git a/server/src/llm/readme.generate.js b/server/src/llm/readme.generate.js index 2e8d5e6..b21612a 100644 --- a/server/src/llm/readme.generate.js +++ b/server/src/llm/readme.generate.js @@ -141,7 +141,7 @@ export function validateContext(context) { const size = estimateContextSize(context); const estimatedTokens = Math.ceil(size / 4); - if (estimatedTokens > 10000) { + if (estimatedTokens > 200000) { warnings.push( `Context is large (${estimatedTokens} tokens) - will be optimized`, ); @@ -156,7 +156,7 @@ export function validateContext(context) { }; } -export function optimizeContext(context, maxTokens = 8000) { +export function optimizeContext(context, maxTokens = 180000) { const maxChars = maxTokens * 4; if (estimateContextSize(context) <= maxChars) { @@ -169,18 +169,18 @@ export function optimizeContext(context, maxTokens = 8000) { if (optimized.fullCodebase && optimized.fullCodebase.length > 0) { optimized.fullCodebase = optimized.fullCodebase.map((file) => ({ ...file, - content: truncateText(file.content, 80), + content: truncateText(file.content, 400), })); if (fits()) return optimized; - if (optimized.fullCodebase.length > 15) { - optimized.fullCodebase = optimized.fullCodebase.slice(0, 15); + if (optimized.fullCodebase.length > 80) { + optimized.fullCodebase = optimized.fullCodebase.slice(0, 80); if (fits()) return optimized; } optimized.fullCodebase = optimized.fullCodebase.map((file) => ({ ...file, - content: truncateText(file.content, 50), + content: truncateText(file.content, 200), })); if (fits()) return optimized; } @@ -290,10 +290,10 @@ export async function generateReadme({ ); } - if (validation.estimatedTokens > 8000) { + if (validation.estimatedTokens > 180000) { console.log(`[LLM] Optimizing large context`); liveUpdate(sharedLogId, `Optimizing large context`); - context = optimizeContext(context, 8000); + context = optimizeContext(context, 180000); } let prompt = buildFullReadmePrompt(context); diff --git a/server/src/llm/readme.patch.js b/server/src/llm/readme.patch.js index cb23a74..38210fe 100644 --- a/server/src/llm/readme.patch.js +++ b/server/src/llm/readme.patch.js @@ -13,8 +13,8 @@ import { extractJson } from "./utils/response.js"; // Patch mode ships far less context than full mode: only the current README, // the commit diff, and the files that commit touched. -const MAX_CONTEXT_TOKENS = 8000; -const MAX_PATCH_SECTIONS = 10; +const MAX_CONTEXT_TOKENS = 60000; +const MAX_PATCH_SECTIONS = 20; function estimateContextSize(context) { return JSON.stringify(context).length; diff --git a/server/src/utils/repo.limits.js b/server/src/utils/repo.limits.js index 9c9e0b6..0f44683 100644 --- a/server/src/utils/repo.limits.js +++ b/server/src/utils/repo.limits.js @@ -1,11 +1,13 @@ // How much of a repository the worker fetches per job, before any LLM call. -// Gemini's 1M-token context window is what makes limits this large affordable. +// Gemini 3.6 Flash's 1M-token input window makes limits this large affordable; +// the model-facing context is still capped to ~180K tokens downstream to stay +// under the free-tier 250K tokens/minute ceiling with output headroom. export const REPOSITORY_LIMITS = { - maxFilesFullScan: 50, - maxLinesPerFile: 500, - maxChangedFiles: 20, - maxChangedFileLines: 300, - maxPatchFiles: 15, - maxPatchFileLines: 200, - maxPatchSections: 10, + maxFilesFullScan: 200, + maxLinesPerFile: 1500, + maxChangedFiles: 60, + maxChangedFileLines: 800, + maxPatchFiles: 40, + maxPatchFileLines: 600, + maxPatchSections: 20, }; From c8dcb4ff85f7a9bb5fe89f59168ad2f61f6f007c Mon Sep 17 00:00:00 2001 From: kaihere14 Date: Thu, 3 Sep 2026 18:19:20 +0530 Subject: [PATCH 3/3] Remove obsolete README cleanup service --- server/src/services/readmeCleanup.service.js | 82 -------------------- server/src/utils/git.worker.js | 1 - 2 files changed, 83 deletions(-) delete mode 100644 server/src/services/readmeCleanup.service.js diff --git a/server/src/services/readmeCleanup.service.js b/server/src/services/readmeCleanup.service.js deleted file mode 100644 index 4aa5b70..0000000 --- a/server/src/services/readmeCleanup.service.js +++ /dev/null @@ -1,82 +0,0 @@ -const OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"; -const CLEANUP_MODEL = process.env.OPENROUTER_CLEANUP_MODEL || "qwen/qwen3-32b"; - -const CLEANUP_SYSTEM_PROMPT = `You are a senior technical documentation architect. - -Your task is to CLEAN and RESTRUCTURE a cluttered README.md file. - -The README has grown over time through many incremental AI updates. -It may contain duplicated features, repeated sections, stale wording, -excessive UI details, bloated explanations, repeated technology mentions, -and changelog-like noise. - -Goals: -- preserve all important technical information -- aggressively remove redundancy -- merge overlapping concepts -- rewrite for clarity and structure -- keep the README professional and concise - -Rules: -- Rewrite the README from scratch -- Keep clean markdown formatting -- Do not remove important technical capabilities -- Do not invent features -- Return ONLY raw markdown (no code fences)`; - -function normalizeMarkdownOutput(text) { - let cleaned = text.trim(); - const fenced = cleaned.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i); - if (fenced) { - cleaned = fenced[1].trim(); - } - return cleaned; -} - -export async function cleanReadmeWithAI(existingReadme, onProgress = null) { - const apiKey = process.env.OPENROUTER_API_KEY; - if (!apiKey) { - throw new Error("OPENROUTER_API_KEY is not configured"); - } - - if (onProgress) { - onProgress(`Sending README to cleanup model`); - } - - const response = await fetch(OPENROUTER_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model: CLEANUP_MODEL, - messages: [ - { role: "system", content: CLEANUP_SYSTEM_PROMPT }, - { role: "user", content: existingReadme }, - ], - temperature: 0.3, - max_tokens: 12000, - }), - }); - - if (!response.ok) { - const errorBody = await response.text(); - throw new Error( - `OpenRouter request failed (${response.status}): ${errorBody.slice(0, 200)}`, - ); - } - - const data = await response.json(); - const content = data?.choices?.[0]?.message?.content; - - if (!content?.trim()) { - throw new Error("OpenRouter returned empty README content"); - } - - if (onProgress) { - onProgress("Cleanup model returned rewritten README"); - } - - return normalizeMarkdownOutput(content); -} diff --git a/server/src/utils/git.worker.js b/server/src/utils/git.worker.js index 1c67e15..b55b370 100644 --- a/server/src/utils/git.worker.js +++ b/server/src/utils/git.worker.js @@ -19,7 +19,6 @@ import { import { selectImportantFiles } from "./scan.filters.js"; import UserLogModel from "../schema/userLog.schema.js"; import { liveUpdate } from "../services/convex.service.js"; -import { cleanReadmeWithAI } from "../services/readmeCleanup.service.js"; import { LlmService } from "../llm/llm.service.js"; export const connection = new IORedis({