diff --git a/README.md b/README.md index f6f1a8ca..7644283c 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Dispatch can optionally run issue grooming itself by calling an OpenAI-compatibl | `DISPATCH_GROOMER_MODEL` | Conditional | Model name sent to the chat completions API. Required when hosted grooming is enabled. | | `DISPATCH_GROOMER_TIMEOUT_MS` | No | LLM request timeout. Defaults to a scaled value of `60s + 5s/KB of maxContextBytes`, clamped to 60s–300s. | | `DISPATCH_GROOMER_MAX_CONTEXT_BYTES` | No | Issue context budget sent to the model. Defaults to `8192`. | +| `DISPATCH_GROOMER_CONTEXT_MODE` | No | Exploration budget preset: `small`, `medium` (default), `large`. | +| `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS` | No | The model's real context window in tokens. Derives the exploration budget from it and ignores the preset. Recommended for self-hosted models. | | `DISPATCH_GROOMER_DRY_RUN` | No | Defaults to `true`; when true, returns a mutation plan without GitHub or DB writes. | | `DISPATCH_GROOMER_REPO_CONTEXT_ENABLED` | No | Enables bounded GitHub API repository context. Defaults to `false`. | | `DISPATCH_GROOMER_MAX_CONTEXT_FILES` | No | Maximum files included in repository context. Defaults to `5`. | diff --git a/docs/hosted-groomer.md b/docs/hosted-groomer.md index 3d3e2685..c61b2d3f 100644 --- a/docs/hosted-groomer.md +++ b/docs/hosted-groomer.md @@ -27,9 +27,40 @@ The feature is disabled by default. | `DISPATCH_GROOMER_MAX_SEARCHES` | `3` | Maximum GitHub code searches per grooming run. | | `DISPATCH_GROOMER_MAX_FILE_BYTES` | `4096` | Maximum bytes per fetched file snippet. | | `DISPATCH_GROOMER_COMMENT_COOLDOWN_HOURS` | `24` | Suppresses repeated hosted-groomer comments on the same issue. A comment is skipped (and recorded on the run) when a prior run posted a comment within this window, unless `force` is true. | +| `DISPATCH_GROOMER_TOOL_LOOP_ENABLED` | `true` | Lets the groomer drive its own repository exploration with tools (`search_code`, `read_file`, `list_directory`, `submit_findings`) instead of one pre-computed context block. | +| `DISPATCH_GROOMER_MAX_TOOL_CALLS` | `12` | Tool calls the exploration loop may make per grooming run. | +| `DISPATCH_GROOMER_CONTEXT_MODE` | `medium` | Exploration budget preset: `small`, `medium`, `large`. See below. | +| `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS` | unset | The model's real context window in tokens. When set, the exploration budget is derived from it and `DISPATCH_GROOMER_CONTEXT_MODE` is ignored — recommended for self-hosted models whose windows do not match what a named preset assumes. | +| `DISPATCH_GROOMER_EXPLORE_MAX_BYTES` | from mode | Overrides the exploration byte budget. | +| `DISPATCH_GROOMER_EXPLORE_MAX_FILE_BYTES` | from mode | Overrides bytes per file returned to the model. Never exceeds the total budget. | +| `DISPATCH_GROOMER_EXPLORE_TIMEOUT_MS` | from mode | Overrides the wall-clock cap on the exploration loop. | | `DISPATCH_GROOMER_TOKEN` | unset | Optional bearer token for scheduled or admin groomer invocations. When set, `POST /api/groomer/run` accepts this token in addition to `DISPATCH_AGENT_TOKEN`. | | `DISPATCH_GROOMER_INTERVAL_MS` | 600000 | Interval for the in-process scheduler's `groomer` job. Dispatch still processes at most one issue per run. | +## Exploration budget + +Repository exploration is a multi-turn tool loop, so it needs a budget of its +own rather than the single-call one. Three ways to size it, most specific first: + +1. the individual `DISPATCH_GROOMER_EXPLORE_*` overrides, +2. `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS`, which derives the budget from the + model's real context window and reserves the rest for the system prompt, the + issue context and the model's own output, +3. `DISPATCH_GROOMER_CONTEXT_MODE`. + +| Mode | Total bytes | Per file | Timeout | +| --- | --- | --- | --- | +| `small` | 8 KB | 4 KB | 90s | +| `medium` (default) | 24 KB | 8 KB | 150s | +| `large` | 96 KB | 24 KB | 300s | + +The default suits a modest self-hosted model. If your model's window is much +larger, prefer setting `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS` over guessing a +preset: a starved loop stops mid-investigation and reports fewer files, which +shows up as `repository exploration hit its byte budget` in a run's +`contextWarnings`. The resolved budget and which path produced it are recorded +on every run under `contextSummary.exploration.budget`. + ## Endpoint ```http diff --git a/src/lib/groomer/config.ts b/src/lib/groomer/config.ts index a7ed164d..09b4b8ae 100644 --- a/src/lib/groomer/config.ts +++ b/src/lib/groomer/config.ts @@ -1,3 +1,5 @@ +import { resolveExplorationBudget, type ExplorationBudget } from "./exploration-budget"; + export interface HostedGroomerConfig { enabled: boolean; dryRun: boolean; @@ -16,6 +18,7 @@ export interface HostedGroomerConfig { maxToolCalls: number; maxSearchResults: number; maxDirEntries: number; + exploration: ExplorationBudget; } const parseBool = (value: string | undefined, defaultValue = false): boolean => { @@ -56,6 +59,7 @@ export function getHostedGroomerConfig(): HostedGroomerConfig { maxToolCalls: 0, maxSearchResults: 0, maxDirEntries: 0, + exploration: { maxTotalBytes: 0, maxFileBytes: 0, timeoutMs: 0, source: "small" }, maxContextFiles: 5, maxSearches: 3, maxFileBytes: 4096, @@ -96,6 +100,7 @@ export function getHostedGroomerConfig(): HostedGroomerConfig { maxToolCalls: parseIntEnv(process.env.DISPATCH_GROOMER_MAX_TOOL_CALLS, 12), maxSearchResults: parseIntEnv(process.env.DISPATCH_GROOMER_MAX_SEARCH_RESULTS, 10), maxDirEntries: parseIntEnv(process.env.DISPATCH_GROOMER_MAX_DIR_ENTRIES, 60), + exploration: resolveExplorationBudget(), }; // timeoutMs: env override wins; otherwise scale with maxContextBytes. diff --git a/src/lib/groomer/exploration-budget.test.ts b/src/lib/groomer/exploration-budget.test.ts new file mode 100644 index 00000000..0a4cd21e --- /dev/null +++ b/src/lib/groomer/exploration-budget.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_CONTEXT_MODE, + deriveFromContextTokens, + resolveExplorationBudget, +} from "./exploration-budget"; + +describe("resolveExplorationBudget", () => { + it("defaults to medium when nothing is set", () => { + const b = resolveExplorationBudget({}); + expect(b.source).toBe(DEFAULT_CONTEXT_MODE); + expect(b.maxTotalBytes).toBe(24_576); + expect(b.maxFileBytes).toBe(8_192); + expect(b.timeoutMs).toBe(150_000); + }); + + it("honours each named mode", () => { + expect(resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "small" })).toMatchObject({ + source: "small", + maxTotalBytes: 8_192, + }); + expect(resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "large" })).toMatchObject({ + source: "large", + maxTotalBytes: 98_304, + }); + }); + + it("grows monotonically across the modes", () => { + const s = resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "small" }); + const m = resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "medium" }); + const l = resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "large" }); + expect(s.maxTotalBytes).toBeLessThan(m.maxTotalBytes); + expect(m.maxTotalBytes).toBeLessThan(l.maxTotalBytes); + expect(s.maxFileBytes).toBeLessThan(m.maxFileBytes); + expect(m.maxFileBytes).toBeLessThan(l.maxFileBytes); + expect(s.timeoutMs).toBeLessThanOrEqual(m.timeoutMs); + expect(m.timeoutMs).toBeLessThanOrEqual(l.timeoutMs); + }); + + it("falls back to the default mode on an unrecognised value", () => { + expect(resolveExplorationBudget({ DISPATCH_GROOMER_CONTEXT_MODE: "enormous" }).source).toBe( + DEFAULT_CONTEXT_MODE, + ); + }); + + it("derives from the model's context window when given one, ignoring the mode", () => { + const b = resolveExplorationBudget({ + DISPATCH_GROOMER_CONTEXT_MODE: "small", + DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS: "131072", + }); + expect(b.source).toBe("derived"); + // Well above what "small" would have allowed, and a fraction of the window. + expect(b.maxTotalBytes).toBeGreaterThan(100_000); + expect(b.maxTotalBytes).toBeLessThan(131_072 * 3.5); + }); + + it("derives a small budget for a small window", () => { + const b = resolveExplorationBudget({ DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS: "8192" }); + expect(b.source).toBe("derived"); + expect(b.maxTotalBytes).toBeLessThan(24_576); + expect(b.maxTotalBytes).toBeGreaterThanOrEqual(4_096); + }); + + it("never lets a single file exceed the whole budget", () => { + const b = resolveExplorationBudget({ + DISPATCH_GROOMER_EXPLORE_MAX_BYTES: "10000", + DISPATCH_GROOMER_EXPLORE_MAX_FILE_BYTES: "999999", + }); + expect(b.maxFileBytes).toBe(10_000); + }); + + it("lets individual env vars override a mode", () => { + const b = resolveExplorationBudget({ + DISPATCH_GROOMER_CONTEXT_MODE: "small", + DISPATCH_GROOMER_EXPLORE_MAX_BYTES: "50000", + DISPATCH_GROOMER_EXPLORE_TIMEOUT_MS: "200000", + }); + expect(b.source).toBe("env"); + expect(b.maxTotalBytes).toBe(50_000); + expect(b.timeoutMs).toBe(200_000); + // Unset override still comes from the mode. + expect(b.maxFileBytes).toBe(4_096); + }); + + it("ignores non-numeric and out-of-range overrides", () => { + const b = resolveExplorationBudget({ + DISPATCH_GROOMER_EXPLORE_MAX_BYTES: "not-a-number", + DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS: "0", + }); + expect(b.source).toBe(DEFAULT_CONTEXT_MODE); + expect(b.maxTotalBytes).toBe(24_576); + }); +}); + +describe("deriveFromContextTokens", () => { + it("reserves most of the window for everything exploration does not own", () => { + const d = deriveFromContextTokens(100_000); + expect(d.maxTotalBytes).toBeLessThan(100_000 * 3.5 * 0.5); + }); + + it("keeps a floor so a tiny window still gets a usable budget", () => { + expect(deriveFromContextTokens(1_024).maxTotalBytes).toBeGreaterThanOrEqual(4_096); + }); + + it("caps the timeout at five minutes however large the window", () => { + expect(deriveFromContextTokens(1_000_000).timeoutMs).toBe(300_000); + }); + + it("keeps per-file at a quarter of the budget", () => { + const d = deriveFromContextTokens(131_072); + expect(d.maxFileBytes).toBe(Math.floor(d.maxTotalBytes / 4)); + }); +}); diff --git a/src/lib/groomer/exploration-budget.ts b/src/lib/groomer/exploration-budget.ts new file mode 100644 index 00000000..33d260e5 --- /dev/null +++ b/src/lib/groomer/exploration-budget.ts @@ -0,0 +1,95 @@ +/** + * Budget for the groomer's repository exploration loop. + * + * Exploration used to borrow `maxContextBytes` and `timeoutMs` from the single + * grooming call. Those were sized for one pre-computed context and one request, + * not for a multi-turn tool loop, so the loop routinely exhausted its budget + * partway through an investigation and stopped early. + * + * Three ways to size it, most specific first: + * 1. the individual env overrides, + * 2. `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS` — derive from the model's real + * context window, which is the honest answer for a self-hosted model whose + * window does not match what a named mode assumes, + * 3. `DISPATCH_GROOMER_CONTEXT_MODE` — small | medium | large. + */ + +export type ContextMode = "small" | "medium" | "large"; + +export interface ExplorationBudget { + /** Bytes of tool output the loop may accumulate across all turns. */ + maxTotalBytes: number; + /** Bytes of any single file handed back to the model. */ + maxFileBytes: number; + /** Wall-clock cap on the whole loop. */ + timeoutMs: number; + /** Which of the three sizing paths produced this budget. */ + source: "env" | "derived" | ContextMode; +} + +const MODES: Record> = { + small: { maxTotalBytes: 8_192, maxFileBytes: 4_096, timeoutMs: 90_000 }, + medium: { maxTotalBytes: 24_576, maxFileBytes: 8_192, timeoutMs: 150_000 }, + large: { maxTotalBytes: 98_304, maxFileBytes: 24_576, timeoutMs: 300_000 }, +}; + +export const DEFAULT_CONTEXT_MODE: ContextMode = "medium"; + +/** Bytes per token. Deliberately conservative so a derived budget under-fills + * the window rather than overflowing it. */ +const BYTES_PER_TOKEN = 3.5; +/** Share of the window exploration may occupy. The rest is the system prompt, + * the issue context, the findings block and the model's own output. */ +const WINDOW_SHARE = 0.35; + +function parseMode(raw: string | undefined): ContextMode | null { + const v = (raw ?? "").trim().toLowerCase(); + return v === "small" || v === "medium" || v === "large" ? v : null; +} + +function parseIntEnv(raw: string | undefined, min = 1): number | null { + if (!raw) return null; + const n = parseInt(raw, 10); + return Number.isFinite(n) && n >= min ? n : null; +} + +/** + * Derive a budget from the model's context window, reserving the rest of the + * window for everything exploration does not own. + */ +export function deriveFromContextTokens(tokens: number): Omit { + const usable = Math.floor(tokens * WINDOW_SHARE * BYTES_PER_TOKEN); + const maxTotalBytes = Math.max(4_096, usable); + return { + maxTotalBytes, + // A quarter of the budget, so no single file can consume the whole thing + // and leave nothing for the searches that found it. + maxFileBytes: Math.max(2_048, Math.floor(maxTotalBytes / 4)), + // 60s of headroom plus 5s per KB, matching the grooming call's own curve. + timeoutMs: Math.max(60_000, Math.min(300_000, 60_000 + Math.ceil(maxTotalBytes / 1024) * 5_000)), + }; +} + +export function resolveExplorationBudget( + env: Record = process.env, +): ExplorationBudget { + const mode = parseMode(env.DISPATCH_GROOMER_CONTEXT_MODE) ?? DEFAULT_CONTEXT_MODE; + const contextTokens = parseIntEnv(env.DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS, 1024); + + const base = contextTokens ? deriveFromContextTokens(contextTokens) : MODES[mode]; + const source: ExplorationBudget["source"] = contextTokens ? "derived" : mode; + + const totalOverride = parseIntEnv(env.DISPATCH_GROOMER_EXPLORE_MAX_BYTES, 1024); + const fileOverride = parseIntEnv(env.DISPATCH_GROOMER_EXPLORE_MAX_FILE_BYTES, 512); + const timeoutOverride = parseIntEnv(env.DISPATCH_GROOMER_EXPLORE_TIMEOUT_MS, 1_000); + const anyOverride = totalOverride !== null || fileOverride !== null || timeoutOverride !== null; + + const maxTotalBytes = totalOverride ?? base.maxTotalBytes; + return { + maxTotalBytes, + // Never let a single file exceed the whole budget, however it was set. + maxFileBytes: Math.min(fileOverride ?? base.maxFileBytes, maxTotalBytes), + timeoutMs: timeoutOverride ?? base.timeoutMs, + source: anyOverride ? "env" : source, + }; +} diff --git a/src/lib/groomer/run.test.ts b/src/lib/groomer/run.test.ts index d223a1aa..d20f5585 100644 --- a/src/lib/groomer/run.test.ts +++ b/src/lib/groomer/run.test.ts @@ -132,6 +132,7 @@ const mockConfig: HostedGroomerConfig = { maxToolCalls: 12, maxSearchResults: 10, maxDirEntries: 60, + exploration: { maxTotalBytes: 24576, maxFileBytes: 8192, timeoutMs: 150000, source: "medium" }, }; const mockAutomationRepo = { id: "repo-1", fullName: "org/repo", enabled: true }; diff --git a/src/lib/groomer/run.ts b/src/lib/groomer/run.ts index 96676e9e..cb7211c2 100644 --- a/src/lib/groomer/run.ts +++ b/src/lib/groomer/run.ts @@ -196,11 +196,11 @@ async function executeGroomerRun( model: config.model, repoFullName: candidate.repoFullName, prompt: context, - timeoutMs: config.timeoutMs, + timeoutMs: config.exploration.timeoutMs, maxToolCalls: config.maxToolCalls, - maxTotalBytes: config.maxContextBytes, + maxTotalBytes: config.exploration.maxTotalBytes, maxSearchResults: config.maxSearchResults, - maxFileBytes: config.maxFileBytes, + maxFileBytes: config.exploration.maxFileBytes, maxDirEntries: config.maxDirEntries, }) : null; @@ -218,6 +218,7 @@ async function executeGroomerRun( repositoryQueries: repositoryContext.queries, repositoryBytes: repositoryContext.bytes, exploration: { + budget: config.exploration, files: exploration.files, ask: exploration.ask, sources: exploration.sources,