From f84e68a60d26fba8911c68da4e789ba08587e437 Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 10:31:14 +0100 Subject: [PATCH 1/5] perf(orchestrator): optimize AI calls for rate limits and token usage Reduce token limit from 8000 to 6000 for more headroom under 10k TPM. Add retry mechanism with exponential backoff for rate limit errors. Change batch processing to sequential with 1s delays to respect TPM limits. --- src/orchestrator.ts | 59 +++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 4bd9af8..539a8d3 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -255,7 +255,7 @@ async function runDeepReview(filesToAudit: string[], allFiles: PrFile[], diff: s // Estimate tokens (rough: 4 chars per token) const estimatedTokens = Math.ceil(diff.length / 4); - const TOKEN_LIMIT = 8000; // Safe buffer under 10K TPM + const TOKEN_LIMIT = 6000; // Reduced from 8000 to leave more headroom for 10k TPM if (estimatedTokens <= TOKEN_LIMIT) { // Small diff: use single-shot review (original behavior) @@ -312,11 +312,13 @@ async function runSingleShotReview(filesToAudit: string[], allFiles: PrFile[], d const formattedFiles = allFiles.map(f => `${f.filename} [${f.status}]`); - const { object } = await generateObject({ - model: groq(ANALYST_MODEL), - schema: JStarReviewSchema, - system: enhancedSystemPrompt, - prompt: buildAnalystUserPrompt(filesToAudit, formattedFiles, diff, existingDocs), + const { object } = await callAIWithRetry(async () => { + return await generateObject({ + model: groq(ANALYST_MODEL), + schema: JStarReviewSchema, + system: enhancedSystemPrompt, + prompt: buildAnalystUserPrompt(filesToAudit, formattedFiles, diff, existingDocs), + }); }); return object; @@ -373,8 +375,8 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff console.log(`šŸŽÆ Reviewing ${relevantDiffs.length} critical files`); - // Review each file chunk (parallel in batches of 3 to not exceed RPM) - const BATCH_SIZE = 3; + // Review each file chunk (Sequential to respect 10k TPM) + const BATCH_SIZE = 1; const allChunkResults: ChunkReviewResult[] = []; for (let i = 0; i < relevantDiffs.length; i += BATCH_SIZE) { @@ -389,7 +391,8 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff allChunkResults.push(...batchResults); if (i + BATCH_SIZE < relevantDiffs.length) { - console.log(` ā³ Reviewed ${i + BATCH_SIZE}/${relevantDiffs.length} files...`); + console.log(` ā³ Reviewed ${i + BATCH_SIZE}/${relevantDiffs.length} files... taking a breath šŸ§˜`); + await new Promise(resolve => setTimeout(resolve, 1000)); // 1s delay to cool down TPM } } @@ -397,20 +400,44 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff return aggregateChunkReviews(allChunkResults); } +/** + * Helper: Retry AI calls with exponential backoff on rate limits. + */ +async function callAIWithRetry(operation: () => Promise, retries = 3, delay = 2000): Promise { + try { + return await operation(); + } catch (error: any) { + // Check for Groq/OpenAI rate limit errors (429, 'limit', 'token') + const isRateLimit = error.statusCode === 429 || + error.message?.includes('rate limit') || + error.message?.includes('token') || + error.code === 'rate_limit_exceeded'; + + if (retries > 0 && isRateLimit) { + console.log(` šŸ”ø Rate limit hit. Retrying in ${delay / 1000}s... (${retries} attempts left)`); + await new Promise(resolve => setTimeout(resolve, delay)); + return callAIWithRetry(operation, retries - 1, delay * 2); + } + throw error; + } +} + /** * Review a single file chunk. */ async function reviewFileChunk(filename: string, fileDiff: string, status: string, architectureContext: string, existingDocs: string[]): Promise { try { - const { object } = await generateObject({ - model: groq(ANALYST_MODEL), - schema: ChunkReviewSchema, - system: CHUNK_REVIEW_SYSTEM_PROMPT, - prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), + return await callAIWithRetry(async () => { + const { object } = await generateObject({ + model: groq(ANALYST_MODEL), + schema: ChunkReviewSchema, + system: CHUNK_REVIEW_SYSTEM_PROMPT, + prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), + }); + return object; }); - return object; } catch (error) { - console.log(`āš ļø Failed to review ${filename}, skipping`); + console.log(`āš ļø Failed to review ${filename} after retries, skipping. Error: ${(error as any).message}`); return { file: filename, findings: [], quality_score: 0 }; } } From ba2a1d8e4e7e6b2e293c8576fd4429434069880f Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 10:39:21 +0100 Subject: [PATCH 2/5] feat(orchestrator): add configurable AI performance tuning options Add support for tuning AI review performance through environment variables: - AI_CONCURRENCY: Controls parallel file reviews (default 1) - AI_MAX_RETRIES: Number of retries on rate limits (default 3) - AI_RETRY_DELAY: Initial retry delay in ms (default 2000) - AI_BACKOFF_FACTOR: Exponential backoff multiplier (default 2) Implement retry logic with exponential backoff for AI calls to handle rate limits. Refactor chunked review to use configurable concurrency instead of fixed sequential processing. Update documentation and type schemas to reflect new configuration options. --- README.md | 15 ++++- docs/features/map-reduce.md | 4 +- src/orchestrator.ts | 119 ++++++++++++++++++++---------------- src/types.ts | 5 ++ 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index df52d43..594e4c2 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,20 @@ Go to your repo ā†’ Settings ā†’ Secrets and variables ā†’ Actions: > `GITHUB_TOKEN` is automatically provided by GitHub Actions. -#### 3. Done! +> `GITHUB_TOKEN` is automatically provided by GitHub Actions. + +#### 3. (Optional) Tune AI Performance + +Add these optional secrets or variables to customize the review process: + +| Variable | Default | Description | +|----------|---------|-------------| +| `AI_CONCURRENCY` | `1` | Number of files to review in parallel. Set to `1` for low-tier (10k TPM) keys. Increase to `3-5` for high-tier. | +| `AI_MAX_RETRIES` | `3` | Number of times to retry a failed/rate-limited AI call. | +| `AI_RETRY_DELAY` | `2000` | Initial delay in ms before retrying (exponential backoff applied). | +| `AI_BACKOFF_FACTOR` | `2` | Multiplier for delay after each retry. | + +#### 4. Done! Open a PR and watch J Star review it. diff --git a/docs/features/map-reduce.md b/docs/features/map-reduce.md index 6654364..5b9866d 100644 --- a/docs/features/map-reduce.md +++ b/docs/features/map-reduce.md @@ -17,13 +17,15 @@ const fileDiffs = splitDiffByFile(diff); ### 2. Map (Review Each File) Each file chunk is reviewed independently in parallel batches: ```typescript -const BATCH_SIZE = 3; // Respects RPM limits +const BATCH_SIZE = parseInt(process.env.AI_CONCURRENCY) || 1; // Default 1 (Sequential) for (let i = 0; i < relevantDiffs.length; i += BATCH_SIZE) { const batch = relevantDiffs.slice(i, i + BATCH_SIZE); const batchResults = await Promise.all(batch.map(fd => reviewFileChunk(...))); } ``` +Configurable via `AI_CONCURRENCY`. Set to `1` for strict rate limits (default), or `3-5` for higher tiers. + ### 3. Reduce (Aggregate) All chunk results are combined into a final `JStarReviewResult`: - Findings are merged into a single array. diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 539a8d3..e7a7559 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -5,10 +5,13 @@ import { generateObject } from 'ai'; import { createGroq } from '@ai-sdk/groq'; import { Octokit } from '@octokit/rest'; -import * as fs from 'fs'; -import * as path from 'path'; - -import { TRIAGE_SYSTEM_PROMPT, ANALYST_SYSTEM_PROMPT, buildAnalystUserPrompt, CHUNK_REVIEW_SYSTEM_PROMPT, buildChunkReviewPrompt } from './prompts.js'; +import { + TRIAGE_SYSTEM_PROMPT, + ANALYST_SYSTEM_PROMPT, + buildAnalystUserPrompt, + CHUNK_REVIEW_SYSTEM_PROMPT, + buildChunkReviewPrompt +} from './prompts.js'; import { TriageSchema, JStarReviewSchema, @@ -29,6 +32,12 @@ const groq = createGroq({ const TRIAGE_MODEL = process.env.TRIAGE_MODEL || 'openai/gpt-oss-120b'; const ANALYST_MODEL = process.env.ANALYST_MODEL || 'moonshotai/kimi-k2-instruct-0905'; +// Tuning Configuration +const AI_CONCURRENCY = parseInt(process.env.AI_CONCURRENCY || '1', 10); +const AI_MAX_RETRIES = parseInt(process.env.AI_MAX_RETRIES || '3', 10); +const AI_RETRY_DELAY = parseInt(process.env.AI_RETRY_DELAY || '2000', 10); +const AI_BACKOFF_FACTOR = parseInt(process.env.AI_BACKOFF_FACTOR || '2', 10); + // ============================================================ // ENVIRONMENT & CONTEXT // ============================================================ @@ -178,7 +187,6 @@ async function fetchPRDiff(ctx: GitHubContext): Promise { } // In some Octokit versions/configurations, diffs might return as objects if mediaType isn't respected - // but typically strict 'diff' format returns string. console.warn('āš ļø Unexpected diff format:', typeof data); return String(data || ''); } @@ -231,6 +239,37 @@ async function addReaction(ctx: GitHubContext, reaction: 'eyes' | 'rocket' | 'co } } +// ============================================================ +// GLOBAL HELPERS +// ============================================================ + +/** + * Helper: Retry AI calls with exponential backoff on rate limits. + */ +async function callAIWithRetry(operation: () => Promise, retries = AI_MAX_RETRIES, delay = AI_RETRY_DELAY): Promise { + try { + return await operation(); + } catch (error: any) { + // Robust Error Detection (Prioritize status codes/codes over messages) + const code = error.code || ''; + const status = error.statusCode || 0; + + const isRateLimit = + status === 429 || + code === 'rate_limit_exceeded' || + code === 'insufficient_quota' || + error.message?.toLowerCase().includes('rate limit') || + error.message?.toLowerCase().includes('token'); + + if (retries > 0 && isRateLimit) { + console.log(` šŸ”ø Rate limit hit (${status || code}). Retrying in ${delay / 1000}s... (${retries} attempts left)`); + await new Promise(resolve => setTimeout(resolve, delay)); + return callAIWithRetry(operation, retries - 1, delay * AI_BACKOFF_FACTOR); + } + throw error; + } +} + // ============================================================ // AI STAGES // ============================================================ @@ -255,7 +294,7 @@ async function runDeepReview(filesToAudit: string[], allFiles: PrFile[], diff: s // Estimate tokens (rough: 4 chars per token) const estimatedTokens = Math.ceil(diff.length / 4); - const TOKEN_LIMIT = 6000; // Reduced from 8000 to leave more headroom for 10k TPM + const TOKEN_LIMIT = 6000; // Safe buffer under 10K TPM if (estimatedTokens <= TOKEN_LIMIT) { // Small diff: use single-shot review (original behavior) @@ -361,6 +400,26 @@ function splitDiffByFile(diff: string): FileDiff[] { return fileDiffs; } +/** + * Review a single file chunk. + */ +async function reviewFileChunk(filename: string, fileDiff: string, status: string, architectureContext: string, existingDocs: string[]): Promise { + try { + return await callAIWithRetry(async () => { + const { object } = await generateObject({ + model: groq(ANALYST_MODEL), + schema: ChunkReviewSchema, + system: CHUNK_REVIEW_SYSTEM_PROMPT, + prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), + }); + return object; + }); + } catch (error) { + console.log(`āš ļø Failed to review ${filename} after retries, skipping. Error: ${(error as any).message}`); + return { file: filename, findings: [], quality_score: 0 }; + } +} + /** * Review large diffs by chunking per-file and aggregating. */ @@ -375,8 +434,8 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff console.log(`šŸŽÆ Reviewing ${relevantDiffs.length} critical files`); - // Review each file chunk (Sequential to respect 10k TPM) - const BATCH_SIZE = 1; + // Review each file chunk (Sequential or Concurrent based on Config) + const BATCH_SIZE = AI_CONCURRENCY; const allChunkResults: ChunkReviewResult[] = []; for (let i = 0; i < relevantDiffs.length; i += BATCH_SIZE) { @@ -392,7 +451,7 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff if (i + BATCH_SIZE < relevantDiffs.length) { console.log(` ā³ Reviewed ${i + BATCH_SIZE}/${relevantDiffs.length} files... taking a breath šŸ§˜`); - await new Promise(resolve => setTimeout(resolve, 1000)); // 1s delay to cool down TPM + await new Promise(resolve => setTimeout(resolve, AI_RETRY_DELAY)); // Configurable delay } } @@ -400,48 +459,6 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff return aggregateChunkReviews(allChunkResults); } -/** - * Helper: Retry AI calls with exponential backoff on rate limits. - */ -async function callAIWithRetry(operation: () => Promise, retries = 3, delay = 2000): Promise { - try { - return await operation(); - } catch (error: any) { - // Check for Groq/OpenAI rate limit errors (429, 'limit', 'token') - const isRateLimit = error.statusCode === 429 || - error.message?.includes('rate limit') || - error.message?.includes('token') || - error.code === 'rate_limit_exceeded'; - - if (retries > 0 && isRateLimit) { - console.log(` šŸ”ø Rate limit hit. Retrying in ${delay / 1000}s... (${retries} attempts left)`); - await new Promise(resolve => setTimeout(resolve, delay)); - return callAIWithRetry(operation, retries - 1, delay * 2); - } - throw error; - } -} - -/** - * Review a single file chunk. - */ -async function reviewFileChunk(filename: string, fileDiff: string, status: string, architectureContext: string, existingDocs: string[]): Promise { - try { - return await callAIWithRetry(async () => { - const { object } = await generateObject({ - model: groq(ANALYST_MODEL), - schema: ChunkReviewSchema, - system: CHUNK_REVIEW_SYSTEM_PROMPT, - prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), - }); - return object; - }); - } catch (error) { - console.log(`āš ļø Failed to review ${filename} after retries, skipping. Error: ${(error as any).message}`); - return { file: filename, findings: [], quality_score: 0 }; - } -} - /** * Combine chunk reviews into final JStarReviewResult. */ diff --git a/src/types.ts b/src/types.ts index 625828d..55d89fe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -151,6 +151,11 @@ export const EnvSchema = z.object({ GITHUB_REPOSITORY: z.string().min(1, 'GITHUB_REPOSITORY is required'), PR_NUMBER: z.string().regex(/^\d+$/, 'PR_NUMBER must be a valid number'), COMMENT_ID: z.string().optional(), + // AI Tuning (Optional) + AI_CONCURRENCY: z.string().optional().default('1'), + AI_MAX_RETRIES: z.string().optional().default('3'), + AI_RETRY_DELAY: z.string().optional().default('2000'), + AI_BACKOFF_FACTOR: z.string().optional().default('2'), }); export type Env = z.infer; From 33460aeaebd859c46cf90d55fafea623d86a88c2 Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 10:44:29 +0100 Subject: [PATCH 3/5] refactor(orchestrator): restructure AI config into interface and pass through context - Introduce AIConfig interface for concurrency, retries, delay, and backoff - Add parseAIConfig function to extract config from environment - Update GitHubContext to include config and pass it to dependent functions - Modify callAIWithRetry, runDeepReview, and review functions to use config parameter - Clean up comments and reorder function definitions for better organization --- src/orchestrator.ts | 220 ++++++++++++++++---------------------------- 1 file changed, 81 insertions(+), 139 deletions(-) diff --git a/src/orchestrator.ts b/src/orchestrator.ts index e7a7559..81f1010 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -21,6 +21,7 @@ import { type JStarReviewResult, type ChunkReviewResult, type Finding, + type Env, } from './types.js'; // Initialize Groq provider @@ -32,16 +33,17 @@ const groq = createGroq({ const TRIAGE_MODEL = process.env.TRIAGE_MODEL || 'openai/gpt-oss-120b'; const ANALYST_MODEL = process.env.ANALYST_MODEL || 'moonshotai/kimi-k2-instruct-0905'; -// Tuning Configuration -const AI_CONCURRENCY = parseInt(process.env.AI_CONCURRENCY || '1', 10); -const AI_MAX_RETRIES = parseInt(process.env.AI_MAX_RETRIES || '3', 10); -const AI_RETRY_DELAY = parseInt(process.env.AI_RETRY_DELAY || '2000', 10); -const AI_BACKOFF_FACTOR = parseInt(process.env.AI_BACKOFF_FACTOR || '2', 10); - // ============================================================ -// ENVIRONMENT & CONTEXT +// ENVIRONMENT & CONFIG // ============================================================ +interface AIConfig { + concurrency: number; + maxRetries: number; + retryDelay: number; + backoffFactor: number; +} + function validateEnv() { const result = EnvSchema.safeParse(process.env); if (!result.success) { @@ -54,12 +56,22 @@ function validateEnv() { return result.data; } +function parseAIConfig(env: Env): AIConfig { + return { + concurrency: parseInt(env.AI_CONCURRENCY || '1', 10), + maxRetries: parseInt(env.AI_MAX_RETRIES || '3', 10), + retryDelay: parseInt(env.AI_RETRY_DELAY || '2000', 10), + backoffFactor: parseInt(env.AI_BACKOFF_FACTOR || '2', 10), + }; +} + interface GitHubContext { owner: string; repo: string; prNumber: number; commentId?: number; octokit: Octokit; + config: AIConfig; // Pass AI config through context } interface PrFile { @@ -67,7 +79,7 @@ interface PrFile { status: 'added' | 'modified' | 'removed' | 'renamed' | 'changed' | 'copied' | 'unchanged'; } -function initGitHub(env: ReturnType): GitHubContext { +function initGitHub(env: Env, config: AIConfig): GitHubContext { const [owner, repo] = env.GITHUB_REPOSITORY.split('/'); return { owner, @@ -75,6 +87,7 @@ function initGitHub(env: ReturnType): GitHubContext { prNumber: parseInt(env.PR_NUMBER, 10), commentId: env.COMMENT_ID ? parseInt(env.COMMENT_ID, 10) : undefined, octokit: new Octokit({ auth: env.GITHUB_TOKEN }), + config, }; } @@ -82,10 +95,6 @@ function initGitHub(env: ReturnType): GitHubContext { // REMOTE CONTEXT LOADING (GitHub API) // ============================================================ -/** - * Helper to fetch a single file content from the remote repo. - * Returns null if not found. - */ async function fetchRemoteFile(ctx: GitHubContext, path: string): Promise { try { const { data } = await ctx.octokit.repos.getContent({ @@ -95,7 +104,7 @@ async function fetchRemoteFile(ctx: GitHubContext, path: string): Promise { return contextDocs; } -/** - * Scan the docs/features folder on remote. - * Returns a map of feature names to their doc files. - */ async function loadDocsInventory(ctx: GitHubContext): Promise> { const inventory = new Map(); const targetDirs = ['docs/features']; for (const docsDir of targetDirs) { - // Remove leading/trailing slashes for cleanliness const cleanPath = docsDir.replace(/^\/+|\/+$/g, ''); - try { const { data } = await ctx.octokit.repos.getContent({ owner: ctx.owner, @@ -146,7 +149,6 @@ async function loadDocsInventory(ctx: GitHubContext): Promise "themes") const featureName = file.name.replace('.md', ''); inventory.set(featureName, file.path); } @@ -180,13 +182,11 @@ async function fetchPRDiff(ctx: GitHubContext): Promise { mediaType: { format: 'diff' }, }); - // Runtime check directly without casting immediately const data = response.data; if (typeof data === 'string') { return data; } - // In some Octokit versions/configurations, diffs might return as objects if mediaType isn't respected console.warn('āš ļø Unexpected diff format:', typeof data); return String(data || ''); } @@ -225,7 +225,6 @@ async function addReaction(ctx: GitHubContext, reaction: 'eyes' | 'rocket' | 'co }); console.log(`šŸ‘€ Reacted to comment with ${reaction}`); } else { - // Fallback: React to the PR itself (Issue) await ctx.octokit.reactions.createForIssue({ owner: ctx.owner, repo: ctx.repo, @@ -246,11 +245,13 @@ async function addReaction(ctx: GitHubContext, reaction: 'eyes' | 'rocket' | 'co /** * Helper: Retry AI calls with exponential backoff on rate limits. */ -async function callAIWithRetry(operation: () => Promise, retries = AI_MAX_RETRIES, delay = AI_RETRY_DELAY): Promise { +async function callAIWithRetry(operation: () => Promise, config: AIConfig, retriesOverride?: number, delayOverride?: number): Promise { + const retries = retriesOverride ?? config.maxRetries; + const delay = delayOverride ?? config.retryDelay; + try { return await operation(); } catch (error: any) { - // Robust Error Detection (Prioritize status codes/codes over messages) const code = error.code || ''; const status = error.statusCode || 0; @@ -264,7 +265,8 @@ async function callAIWithRetry(operation: () => Promise, retries = AI_MAX_ if (retries > 0 && isRateLimit) { console.log(` šŸ”ø Rate limit hit (${status || code}). Retrying in ${delay / 1000}s... (${retries} attempts left)`); await new Promise(resolve => setTimeout(resolve, delay)); - return callAIWithRetry(operation, retries - 1, delay * AI_BACKOFF_FACTOR); + // RECURSIVE AWAIT FIX (Audit Item 1) + return await callAIWithRetry(operation, config, retries - 1, delay * config.backoffFactor); } throw error; } @@ -289,33 +291,30 @@ async function runTriage(files: PrFile[], diffLength: number): Promise { +async function runDeepReview(filesToAudit: string[], allFiles: PrFile[], diff: string, architectureContext: string, existingDocs: string[], config: AIConfig): Promise { console.log(`šŸ§  Running Deep Review with ${ANALYST_MODEL}...`); // Estimate tokens (rough: 4 chars per token) const estimatedTokens = Math.ceil(diff.length / 4); - const TOKEN_LIMIT = 6000; // Safe buffer under 10K TPM + + // TOKEN LIMIT RATIONALE (Audit Item 2) + // We reduce this to 6000 to leave a safe buffer for 120b or Kimi models which often have + // 8k-10k TPM limits on lower tiers. + const TOKEN_LIMIT = 6000; if (estimatedTokens <= TOKEN_LIMIT) { - // Small diff: use single-shot review (original behavior) - const rawResult = await runSingleShotReview(filesToAudit, allFiles, diff, architectureContext, existingDocs); + const rawResult = await runSingleShotReview(filesToAudit, allFiles, diff, architectureContext, existingDocs, config); return adjustScoreForSkippedFiles(rawResult, filesToAudit.length, allFiles.length); } - // Large diff: use chunked map-reduce console.log(`šŸ“¦ Diff too large (${estimatedTokens} est. tokens), using chunked review`); - const rawResult = await runChunkedReview(filesToAudit, allFiles, diff, architectureContext, existingDocs); + const rawResult = await runChunkedReview(filesToAudit, allFiles, diff, architectureContext, existingDocs, config); return adjustScoreForSkippedFiles(rawResult, filesToAudit.length, allFiles.length); } -/** - * Adjusts the score to account for files that were skipped by Triage (assumed safe/100). - * Rule: Final Score = ((RawScore * AuditedCount) + (100 * SkippedCount)) / TotalCount - */ function adjustScoreForSkippedFiles(result: JStarReviewResult, auditedCount: number, totalCount: number): JStarReviewResult { if (totalCount === 0) return result; - // Cap auditedCount at totalCount to prevent negative skipped (e.g. if filesToAudit includes deleted files) const effectiveAudited = Math.min(auditedCount, totalCount); const skippedCount = totalCount - effectiveAudited; @@ -327,10 +326,9 @@ function adjustScoreForSkippedFiles(result: JStarReviewResult, auditedCount: num ); console.log(`āš–ļø Weighted Score Adjustment:`); - console.log(` - Raw Score (Audited Files): ${currentScore}`); - console.log(` - Audited Files: ${effectiveAudited}`); - console.log(` - Skipped Files (Assumed 100): ${skippedCount}`); - console.log(` - New Weighted Score: ${weightedScore}`); + console.log(` - Raw Score: ${currentScore}, Audited: ${effectiveAudited}`); + console.log(` - Skipped (100): ${skippedCount}`); + console.log(` - New Score: ${weightedScore}`); return { ...result, @@ -341,10 +339,7 @@ function adjustScoreForSkippedFiles(result: JStarReviewResult, auditedCount: num }; } -/** - * Original single-shot review for small diffs. - */ -async function runSingleShotReview(filesToAudit: string[], allFiles: PrFile[], diff: string, architectureContext: string, existingDocs: string[]): Promise { +async function runSingleShotReview(filesToAudit: string[], allFiles: PrFile[], diff: string, architectureContext: string, existingDocs: string[], config: AIConfig): Promise { const enhancedSystemPrompt = architectureContext ? `${ANALYST_SYSTEM_PROMPT}\n\n--- PROJECT CONTEXT ---\n${architectureContext}` : ANALYST_SYSTEM_PROMPT; @@ -358,7 +353,7 @@ async function runSingleShotReview(filesToAudit: string[], allFiles: PrFile[], d system: enhancedSystemPrompt, prompt: buildAnalystUserPrompt(filesToAudit, formattedFiles, diff, existingDocs), }); - }); + }, config); return object; } @@ -372,12 +367,8 @@ interface FileDiff { diff: string; } -/** - * Parse unified diff into per-file chunks. - */ function splitDiffByFile(diff: string): FileDiff[] { const fileDiffs: FileDiff[] = []; - // Match diff headers: "diff --git a/path b/path" or "--- a/path" const diffPattern = /^diff --git a\/(.+?) b\/\1/gm; let match; @@ -387,7 +378,6 @@ function splitDiffByFile(diff: string): FileDiff[] { positions.push({ filename: match[1], start: match.index }); } - // Extract each file's diff for (let i = 0; i < positions.length; i++) { const start = positions[i].start; const end = i + 1 < positions.length ? positions[i + 1].start : diff.length; @@ -400,42 +390,17 @@ function splitDiffByFile(diff: string): FileDiff[] { return fileDiffs; } -/** - * Review a single file chunk. - */ -async function reviewFileChunk(filename: string, fileDiff: string, status: string, architectureContext: string, existingDocs: string[]): Promise { - try { - return await callAIWithRetry(async () => { - const { object } = await generateObject({ - model: groq(ANALYST_MODEL), - schema: ChunkReviewSchema, - system: CHUNK_REVIEW_SYSTEM_PROMPT, - prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), - }); - return object; - }); - } catch (error) { - console.log(`āš ļø Failed to review ${filename} after retries, skipping. Error: ${(error as any).message}`); - return { file: filename, findings: [], quality_score: 0 }; - } -} - -/** - * Review large diffs by chunking per-file and aggregating. - */ -async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff: string, architectureContext: string, existingDocs: string[]): Promise { +async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff: string, architectureContext: string, existingDocs: string[], config: AIConfig): Promise { const fileDiffs = splitDiffByFile(diff); console.log(`šŸ”Ŗ Split into ${fileDiffs.length} file chunks`); - // Filter to only audit files the triage identified as critical const relevantDiffs = fileDiffs.filter(fd => filesToAudit.some(f => fd.filename.endsWith(f) || f.endsWith(fd.filename)) ); console.log(`šŸŽÆ Reviewing ${relevantDiffs.length} critical files`); - // Review each file chunk (Sequential or Concurrent based on Config) - const BATCH_SIZE = AI_CONCURRENCY; + const BATCH_SIZE = config.concurrency; const allChunkResults: ChunkReviewResult[] = []; for (let i = 0; i < relevantDiffs.length; i += BATCH_SIZE) { @@ -444,24 +409,40 @@ async function runChunkedReview(filesToAudit: string[], allFiles: PrFile[], diff batch.map(fd => { const fileInfo = allFiles.find(f => f.filename === fd.filename); const status = fileInfo?.status || 'modified'; - return reviewFileChunk(fd.filename, fd.diff, status, architectureContext, existingDocs); + return reviewFileChunk(fd.filename, fd.diff, status, architectureContext, existingDocs, config); }) ); allChunkResults.push(...batchResults); - if (i + BATCH_SIZE < relevantDiffs.length) { + // DELAY OPTIMIZATION (Audit Item 3) + // Only delay if we have more files to process AND we are running in strict sequential mode (low tier). + // High concurrency tiers (3+) likely don't need this artificial delay as much, or the throughput matters more. + if (i + BATCH_SIZE < relevantDiffs.length && config.concurrency === 1) { console.log(` ā³ Reviewed ${i + BATCH_SIZE}/${relevantDiffs.length} files... taking a breath šŸ§˜`); - await new Promise(resolve => setTimeout(resolve, AI_RETRY_DELAY)); // Configurable delay + await new Promise(resolve => setTimeout(resolve, config.retryDelay)); } } - // Aggregate results return aggregateChunkReviews(allChunkResults); } -/** - * Combine chunk reviews into final JStarReviewResult. - */ +async function reviewFileChunk(filename: string, fileDiff: string, status: string, architectureContext: string, existingDocs: string[], config: AIConfig): Promise { + try { + return await callAIWithRetry(async () => { + const { object } = await generateObject({ + model: groq(ANALYST_MODEL), + schema: ChunkReviewSchema, + system: CHUNK_REVIEW_SYSTEM_PROMPT, + prompt: buildChunkReviewPrompt(filename, fileDiff, status, architectureContext, existingDocs), + }); + return object; + }, config); + } catch (error) { + console.log(`āš ļø Failed to review ${filename} after retries, skipping. Error: ${(error as any).message}`); + return { file: filename, findings: [], quality_score: 0 }; + } +} + function aggregateChunkReviews(chunks: ChunkReviewResult[]): JStarReviewResult { const allFindings: Finding[] = []; let totalQuality = 0; @@ -473,7 +454,6 @@ function aggregateChunkReviews(chunks: ChunkReviewResult[]): JStarReviewResult { const avgQuality = chunks.length > 0 ? Math.round(totalQuality / chunks.length) : 0; - // Determine verdict based on findings const hasCritical = allFindings.some(f => f.severity === 'CRITICAL'); const hasHigh = allFindings.some(f => f.severity === 'HIGH'); @@ -502,121 +482,84 @@ function aggregateChunkReviews(chunks: ChunkReviewResult[]): JStarReviewResult { } // ============================================================ -// FORMATTING +// FORMATTING (Skipped detailed changes, kept as is) // ============================================================ function formatTriageSkipComment(triage: TriageResult): string { - return `## āœØ J Star Triage - -**Risk Level:** ${triage.risk_level} - -${triage.ignore_reason ? `> ${triage.ignore_reason}` : ''} - -No critical files detected. Skipping deep review. šŸŽ‰`; + return `## āœØ J Star Triage\n\n**Risk Level:** ${triage.risk_level}\n\n${triage.ignore_reason ? `> ${triage.ignore_reason}` : ''}\n\nNo critical files detected. Skipping deep review. šŸŽ‰`; } function formatReviewComment(review: JStarReviewResult): string { const score = review.summary.quality_score; const verdict = review.summary.verdict; - // 1. Calculate Metrics const counts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, NITPICK: 0 }; - for (const f of review.findings) { - counts[f.severity]++; - } + for (const f of review.findings) counts[f.severity]++; const totalFindings = review.findings.length; - // 2. Determine Mode - // High-Density Mode if >= 30 findings (Bumped from 15 to show Grouped Fixes more often) + // High-Density Mode if >= 30 findings const isHighDensity = totalFindings >= 30; - // 3. Header (Score + Summary Table) const icon = score > 80 ? 'šŸŸ¢' : score > 50 ? 'šŸŸ”' : 'šŸ”“'; let md = `# ${icon} J Star Code Audit\n\n`; - // Simple metrics table (Canonical Rule 2) md += `| Score | Verdict | šŸšØ Critical | šŸ”¶ High | šŸ”¹ Medium | šŸ”§ Nitpick |\n`; md += `| :--- | :--- | :--- | :--- | :--- | :--- |\n`; md += `| **${score}/100** | **${verdict}** | ${counts.CRITICAL || '-'} | ${counts.HIGH || '-'} | ${counts.MEDIUM || '-'} | ${counts.NITPICK || '-'} |\n\n`; if (totalFindings === 0) { - md += `### āœØ No issues found. Ship it!\n\n---\n\n`; - md += `Powered by J Star Sentinel āš”`; + md += `### āœØ No issues found. Ship it!\n\n---\n\nPowered by J Star Sentinel āš”`; return md; } - // Group findings by file const byFile = new Map(); for (const f of review.findings) { if (!byFile.has(f.file)) byFile.set(f.file, []); byFile.get(f.file)!.push(f); } - // Icons mapping const sevIcons: Record = { CRITICAL: 'šŸšØ', HIGH: 'šŸ”¶', MEDIUM: 'šŸ”¹', NITPICK: 'šŸ”§' }; - // 4. Render Body based on Mode if (isHighDensity) { md += `**SUMMARY MODE** (High finding count detected)\n\n`; - for (const [file, findings] of byFile) { md += `### šŸ“„ ${file}\n\n`; md += `| Sev | Cat | Issue | Fix |\n`; md += `| :--- | :--- | :--- | :--- |\n`; - for (const f of findings) { - const title = f.title || f.message.substring(0, 50); // Fallback if title missing during migration + const title = f.title || f.message.substring(0, 50); const desc = f.message; const fix = f.fix_prompt ? `\`${f.fix_prompt.substring(0, 50)}${f.fix_prompt.length > 50 ? '...' : ''}\`` : 'See comments'; md += `| ${sevIcons[f.severity]} | ${f.category} | **${title}**
${desc} | ${fix} |\n`; } md += `\n`; } - } else { - - // Default Mode (Standard PR Review) for (const [file, findings] of byFile) { md += `## šŸ“„ ${file}\n\n`; - const fixes: string[] = []; - for (const f of findings) { const title = f.title || 'Untitled Issue'; const isAlertAndCritical = f.severity === 'CRITICAL'; const isAlertAndHigh = f.severity === 'HIGH'; - if (f.fix_prompt) { - fixes.push(`**${title}**: ${f.fix_prompt}`); - } + if (f.fix_prompt) fixes.push(`**${title}**: ${f.fix_prompt}`); - // Rule 3: GitHub Alerts for CRITICAL/HIGH if (isAlertAndCritical || isAlertAndHigh) { const alertType = isAlertAndCritical ? 'CAUTION' : 'WARNING'; - - md += `> [!${alertType}]\n`; - md += `> **${title}**\n`; - md += `> ${f.message}\n`; - md += `\n`; // End blockquote + md += `> [!${alertType}]\n> **${title}**\n> ${f.message}\n\n`; } else { - // Standard Rendering for Medium/Nitpick - md += `### ${sevIcons[f.severity]} ${title}\n`; - md += `**Category:** ${f.category}\n\n`; - md += `${f.message}\n\n`; + md += `### ${sevIcons[f.severity]} ${title}\n**Category:** ${f.category}\n\n${f.message}\n\n`; } } - if (fixes.length > 0) { md += `**šŸ› ļø Recommended Fixes**\n\n`; - for (const fix of fixes) { - md += `- ${fix}\n`; - } + for (const fix of fixes) md += `- ${fix}\n`; } md += `\n---\n\n`; } } - // 5. Footer (Canonical Rule 7) md += `Powered by J Star Sentinel āš”`; return md; } @@ -630,7 +573,8 @@ async function main() { console.log('================================\n'); const env = validateEnv(); - const ctx = initGitHub(env); + const config = parseAIConfig(env); + const ctx = initGitHub(env, config); await addReaction(ctx, 'eyes'); @@ -638,7 +582,7 @@ async function main() { const architectureContext = await loadArchitectureContext(ctx); const docsInventory = await loadDocsInventory(ctx); - const existingDocs = [...docsInventory.values()]; // Convert Map to array of doc paths + const existingDocs = [...docsInventory.values()]; const [diff, files] = await Promise.all([fetchPRDiff(ctx), fetchPRFiles(ctx)]); console.log(`šŸ“„ Found ${files.length} files (${diff.length} chars diff)\n`); @@ -652,13 +596,11 @@ async function main() { return; } - // AI handles doc drift detection via the prompt - now with existing docs context - const review = await runDeepReview(triage.files_to_audit, files, diff, architectureContext, existingDocs); + const review = await runDeepReview(triage.files_to_audit, files, diff, architectureContext, existingDocs, config); console.log(`\nšŸ”¬ Review:`, JSON.stringify(review, null, 2), '\n'); await postComment(ctx, formatReviewComment(review)); - // Final Reaction based on Verdict const finalReaction = review?.summary?.verdict === 'REQUEST_CHANGES' ? 'confused' : 'rocket'; await addReaction(ctx, finalReaction); From 14de928705442308c36a9a34ad3ccf2c3a506272 Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 10:45:22 +0100 Subject: [PATCH 4/5] docs: update documentation for rate limit stability and analyst features - Add Phase 8 history entry detailing stability improvements under pressure - Include performance tuning options in spawn guide for handling rate limits - Update analyst feature docs with token limits and chunking details --- docs/HISTORY.md | 14 ++++++++++++++ docs/SPAWN_GUIDE.md | 9 +++++++++ docs/features/analyst.md | 2 ++ 3 files changed, 25 insertions(+) diff --git a/docs/HISTORY.md b/docs/HISTORY.md index 8f927f6..ac9056f 100644 --- a/docs/HISTORY.md +++ b/docs/HISTORY.md @@ -172,3 +172,17 @@ Fixed critical runtime safety issues identified by the bot's own self-audit: | `[latest]` | Fix deleted file logic and add emoji reactions | | `[latest]` | Fix duplicate workflow triggers | | `[latest]` | Harden runtime safety (orchestrator.ts) | + +--- + +## Phase 8: Stability under Pressure +**Date:** Dec 14, 2024 + +### What Happened +Stabilized the bot against **Rate Limit Exceeded** crashes when using lower-tier API keys (e.g., Kimi 10k TPM). + +### Key Decisions +1. **Configurable Concurrency:** Added `AI_CONCURRENCY` (default: 1) to allow users to throttle the bot down to sequential processing or scale it up. +2. **Smart Delay:** Introduced an artificial delay between file chunks, but *only* when running in sequential mode. This prioritizes stability for free-tier users while unblocking speed for pro users. +3. **Recursion Rewrite:** Fixed a critical bug where the retry logic wasn't awaiting its own recursive calls. +4. **Conservative Limits:** Reduced the single-shot token limit from 8000 to 6000 to provide a safer buffer against hard API bursts. diff --git a/docs/SPAWN_GUIDE.md b/docs/SPAWN_GUIDE.md index 1791429..da05ff3 100644 --- a/docs/SPAWN_GUIDE.md +++ b/docs/SPAWN_GUIDE.md @@ -119,10 +119,19 @@ review: secrets: GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} with: + triage_model: 'openai/gpt-oss-120b' # Fast/cheap for initial classification triage_model: 'openai/gpt-oss-120b' # Fast/cheap for initial classification analyst_model: 'moonshotai/kimi-k2-instruct-0905' # Powerful for deep review ``` +### Tuning Performance (Optional) + +If you are hitting rate limits (429 errors), you can tune the concurrency via **Secrets** or **Variables**: + +- `AI_CONCURRENCY`: Set to `1` (default) for safety, or `3-5` for speed. +- `AI_MAX_RETRIES`: Default `3`. +- `AI_RETRY_DELAY`: Default `2000` (ms). + --- ## Manual Review Trigger diff --git a/docs/features/analyst.md b/docs/features/analyst.md index 661d299..ba9ba22 100644 --- a/docs/features/analyst.md +++ b/docs/features/analyst.md @@ -11,6 +11,8 @@ After Triage identifies risky files, the Analyst performs a thorough review usin ## Input - The actual diff content of the `files_to_audit`. + - **Single-Shot Limit:** < 6000 tokens (approx). + - **Chunked:** Used automatically if > 6000 tokens. - Project context (`.jstar/architecture.md`, `.jstar/rules.md`). - Existing docs inventory (`docs/features/*.md`). From 3cfd5d6855c4e5a5a41b3875cc5467a385cfa8cd Mon Sep 17 00:00:00 2001 From: JStaRFilms Date: Sun, 14 Dec 2025 10:51:33 +0100 Subject: [PATCH 5/5] refactor(orchestrator): update AI config schema to use numeric coercion and validation - Replace string-based AI tuning parameters in EnvSchema with z.coerce.number() for automatic type conversion and strict validation - Update parseAIConfig to use coerced values directly, removing manual parseInt calls - Add AI_BACKOFF_FACTOR to documentation with proper type descriptions This enhances type safety and prevents runtime errors from invalid numeric inputs while maintaining backward compatibility with defaults. --- docs/SPAWN_GUIDE.md | 7 ++++--- src/orchestrator.ts | 8 ++++---- src/types.ts | 25 ++++++++++++++++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/SPAWN_GUIDE.md b/docs/SPAWN_GUIDE.md index da05ff3..15df1c5 100644 --- a/docs/SPAWN_GUIDE.md +++ b/docs/SPAWN_GUIDE.md @@ -128,9 +128,10 @@ review: If you are hitting rate limits (429 errors), you can tune the concurrency via **Secrets** or **Variables**: -- `AI_CONCURRENCY`: Set to `1` (default) for safety, or `3-5` for speed. -- `AI_MAX_RETRIES`: Default `3`. -- `AI_RETRY_DELAY`: Default `2000` (ms). +- `AI_CONCURRENCY`: Positive integer. Set to `1` (default) for safety, or `3-5` for speed. +- `AI_MAX_RETRIES`: Non-negative integer. Default `3`. +- `AI_RETRY_DELAY`: Non-negative integer (ms). Default `2000`. +- `AI_BACKOFF_FACTOR`: Number >= 1. Default `2`. --- diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 81f1010..e07786a 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -58,10 +58,10 @@ function validateEnv() { function parseAIConfig(env: Env): AIConfig { return { - concurrency: parseInt(env.AI_CONCURRENCY || '1', 10), - maxRetries: parseInt(env.AI_MAX_RETRIES || '3', 10), - retryDelay: parseInt(env.AI_RETRY_DELAY || '2000', 10), - backoffFactor: parseInt(env.AI_BACKOFF_FACTOR || '2', 10), + concurrency: env.AI_CONCURRENCY, // Zod already coerced to number + maxRetries: env.AI_MAX_RETRIES, + retryDelay: env.AI_RETRY_DELAY, + backoffFactor: env.AI_BACKOFF_FACTOR, }; } diff --git a/src/types.ts b/src/types.ts index 55d89fe..0a2cefb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -151,11 +151,26 @@ export const EnvSchema = z.object({ GITHUB_REPOSITORY: z.string().min(1, 'GITHUB_REPOSITORY is required'), PR_NUMBER: z.string().regex(/^\d+$/, 'PR_NUMBER must be a valid number'), COMMENT_ID: z.string().optional(), - // AI Tuning (Optional) - AI_CONCURRENCY: z.string().optional().default('1'), - AI_MAX_RETRIES: z.string().optional().default('3'), - AI_RETRY_DELAY: z.string().optional().default('2000'), - AI_BACKOFF_FACTOR: z.string().optional().default('2'), + // AI Tuning (Optional) - Strict numeric coercing and validation + AI_CONCURRENCY: z.coerce + .number() + .int() + .min(1, 'AI_CONCURRENCY must be a positive integer') + .default(1), + AI_MAX_RETRIES: z.coerce + .number() + .int() + .min(0, 'AI_MAX_RETRIES must be a non-negative integer') + .default(3), + AI_RETRY_DELAY: z.coerce + .number() + .int() + .min(0, 'AI_RETRY_DELAY must be a non-negative integer') + .default(2000), + AI_BACKOFF_FACTOR: z.coerce + .number() + .min(1, 'AI_BACKOFF_FACTOR must be >= 1') + .default(2), }); export type Env = z.infer;