diff --git a/clearwing/agent/tools/hunt/deep_agent.py b/clearwing/agent/tools/hunt/deep_agent.py index 81c6db8c..e13ead38 100644 --- a/clearwing/agent/tools/hunt/deep_agent.py +++ b/clearwing/agent/tools/hunt/deep_agent.py @@ -1,9 +1,7 @@ -"""Deep agent mode tools: execute, read_file, write_file. +"""Deep agent mode tools: bounded source discovery and sandbox operations. -Replaces the constrained 9-tool hunter set with 3 primitives that give -the model full-shell access inside the sandbox container. The model -uses the same tools a human researcher would — gcc, gdb, strace, make, -etc. — all via ``execute()``. +Pairs repository-scoped search with sandbox tools for source reading, +compilation, debugging, and test execution. Model-side reasoning is captured natively via rust-genai's `capture_reasoning_content=True` on every chat request; the hunter @@ -19,15 +17,20 @@ from __future__ import annotations import difflib +import fnmatch import logging +import os import re import shlex +from pathlib import Path from pydantic import Field from clearwing.llm import NativeToolSpec, ToolInputModel from clearwing.reporting.safety import redact_text +from clearwing.sourcehunt.paths import resolve_repo_directory, resolve_repo_file +from .discovery import build_discovery_tools from .pool_query import build_pool_query_tools from .potentials import build_potential_tools from .reporting import build_reporting_tools @@ -44,7 +47,9 @@ class ExecuteInput(ToolInputModel): class ReadFileInput(ToolInputModel): - path: str = Field(description="Absolute path in the container.") + path: str = Field( + description="Repository-relative path from a search hit, or /workspace/... path." + ) offset: int = Field( default=0, description="Line offset (0-based, default 0). Or use start_line (1-based)." ) @@ -91,6 +96,16 @@ class ReadFunctionInput(ToolInputModel): name: str = Field(description="Exact function name to read (e.g. 'foo_bar_baz').") +class FindSourceInput(ToolInputModel): + query: str = Field(description="Filename or directory name substring, or glob pattern.") + path: str = Field(default=".", description="Repository-relative directory to search.") + kind: str = Field(default="both", description="Match files, directories, or both.") + + +_SEARCH_LIMIT = 40 +_SHELL_SEARCH = re.compile(r"(?:^|[;&|]\s*)\s*(?:find|grep|rg)\b") + + def _cap_output(text: str, label: str = "output") -> str: safe = redact_text(text) if len(safe) <= _OUTPUT_CAP: @@ -113,7 +128,7 @@ def _matches(name: str, tokens: list[str]) -> bool: def build_deep_agent_tools(ctx: HunterContext) -> list[NativeToolSpec]: # noqa: C901 - """Build the deep agent tool set: execute, read_file, write_file, + """Build the deep agent tool set: bounded search, execute, read_file, write_file, plus the shared reporting + findings-pool tools. """ # Deep hunters read source via read_file/execute (cat/sed/grep), not the @@ -121,8 +136,156 @@ def build_deep_agent_tools(ctx: HunterContext) -> list[NativeToolSpec]: # noqa: # context so the reporting guard doesn't reject every trace step for a # file it never saw a read_source_file call for. ctx.agent_mode = "deep" + discovery_grep = next(tool for tool in build_discovery_tools(ctx) if tool.name == "grep_source") + + def grep_source(pattern: str, path: str = ".", file_glob: str = "", **_: object) -> dict: + matches = discovery_grep.handler( + pattern=pattern, path=path, file_glob=file_glob, max_results=_SEARCH_LIMIT + 1 + ) + if matches and "error" in matches[0]: + return { + "status": "error", + "error": matches[0]["error"], + "matches": [], + "truncated": False, + } + truncated = len(matches) > _SEARCH_LIMIT + visible = [ + {**match, "matched_text": str(match.get("matched_text", ""))[:240]} + for match in matches[:_SEARCH_LIMIT] + ] + return { + "status": "truncated" if truncated else "matches" if matches else "no_matches", + "pattern": pattern, + "scope": path, + "matches": visible, + "count": min(len(matches), _SEARCH_LIMIT), + "truncated": truncated, + "next_action": "Read a relevant hit with read_file; refine the query if truncated or empty.", + } + + def find_source(query: str, path: str = ".", kind: str = "both", **_: object) -> dict: # noqa: C901 + if kind not in {"file", "directory", "both"}: + return { + "status": "error", + "error": "kind must be file, directory, or both", + "matches": [], + "truncated": False, + } + if not query.strip(): + return { + "status": "error", + "error": "query must not be empty", + "matches": [], + "truncated": False, + } + base = resolve_repo_directory(ctx.repo_path, path) + if base is None: + return { + "status": "error", + "error": "path is outside the repository or is not a directory", + "matches": [], + "truncated": False, + } + root = Path(ctx.repo_path).resolve() + has_glob = any(char in query for char in "*?[]") + needle = query.lower() + matches: list[dict[str, str]] = [] + + if ctx.sandbox is not None: + relative_base = base.relative_to(root).as_posix() + container_base = "/workspace" if relative_base == "." else f"/workspace/{relative_base}" + name_pattern = query if has_glob else f"*{query}*" + type_filter = { + "file": "-type f", + "directory": "-type d", + "both": r"\( -type f -o -type d \)", + }[kind] + command = ( + f"find {shlex.quote(container_base)} -path '*/.git' -prune -o " + f"\\( -iname {shlex.quote(name_pattern)} -a {type_filter} \\) -print " + "| while IFS= read -r match; do " + 'if [ -d "$match" ]; then printf "directory\\t%s\\n" "$match"; ' + 'else printf "file\\t%s\\n" "$match"; fi; done ' + f"| head -n {_SEARCH_LIMIT + 1}" + ) + result = ctx.sandbox.exec(command, timeout=30) + if result.timed_out: + return { + "status": "error", + "error": "repository path search timed out", + "matches": [], + "truncated": False, + } + if result.stderr and not result.stdout: + return { + "status": "error", + "error": _cap_output(result.stderr.strip(), "find error"), + "matches": [], + "truncated": False, + } + for row in result.stdout.splitlines(): + entry_type, separator, absolute = row.partition("\t") + if not separator or entry_type not in {"file", "directory"}: + continue + if not absolute.startswith("/workspace/"): + continue + matches.append( + {"path": redact_text(absolute.removeprefix("/workspace/")), "type": entry_type} + ) + else: + + def wanted(name: str) -> bool: + return fnmatch.fnmatch(name.lower(), needle) if has_glob else needle in name.lower() + + for directory, dirnames, filenames in os.walk(base, followlinks=False): + dirnames[:] = sorted( + name + for name in dirnames + if name != ".git" + and resolve_repo_directory( + ctx.repo_path, (Path(directory) / name).relative_to(root).as_posix() + ) + is not None + ) + if kind in {"directory", "both"}: + for name in dirnames: + if wanted(name): + relative = (Path(directory) / name).relative_to(root).as_posix() + matches.append({"path": redact_text(relative), "type": "directory"}) + if len(matches) > _SEARCH_LIMIT: + break + if len(matches) > _SEARCH_LIMIT: + break + if kind in {"file", "both"}: + for name in sorted(filenames): + candidate = Path(directory) / name + relative = candidate.relative_to(root).as_posix() + if wanted(name) and resolve_repo_file(ctx.repo_path, relative) is not None: + matches.append({"path": redact_text(relative), "type": "file"}) + if len(matches) > _SEARCH_LIMIT: + break + if len(matches) > _SEARCH_LIMIT: + break + truncated = len(matches) > _SEARCH_LIMIT + return { + "status": "truncated" if truncated else "matches" if matches else "no_matches", + "query": query, + "scope": path, + "matches": matches[:_SEARCH_LIMIT], + "count": min(len(matches), _SEARCH_LIMIT), + "truncated": truncated, + "next_action": "Read a relevant file with read_file; refine the query if truncated or empty.", + } def execute(command: str, timeout: int = 300, **_: object) -> dict: + if _SHELL_SEARCH.search(command): + return { + "error": ( + "Shell search blocked. Use grep_source for repository " + "content or find_source for repository paths, then read_file a hit." + ) + } if ctx.sandbox is None: return {"error": "no sandbox available"} result = ctx.sandbox.exec(command, timeout=timeout) @@ -315,11 +478,32 @@ def read_function(name: str, **_: object) -> dict: ) return [ + NativeToolSpec( + name="grep_source", + description=( + "Search source content inside the repository. Returns at most 40 file:line " + "matches and an explicit no_matches or truncated state. Read a relevant hit " + "with read_file; repeated results call for a different approach or a finish." + ), + schema=discovery_grep.schema, + handler=grep_source, + ), + NativeToolSpec( + name="find_source", + description=( + "Find repository files or directories by name substring or glob. Returns at " + "most 40 labeled paths and an explicit no_matches or truncated state. " + "Read a relevant file with read_file." + ), + schema=FindSourceInput.model_json_schema(), + handler=find_source, + ), NativeToolSpec( name="execute", description=( - "Run a shell command inside the sandbox container. " - "Use for compilation, debugging, running tests, etc." + "Run a shell command inside the sandbox container for compilation, " + "debugging, and tests. Use grep_source for source content and " + "find_source for paths instead of shell grep/find searches." ), schema=ExecuteInput.model_json_schema(), handler=execute, @@ -327,7 +511,8 @@ def read_function(name: str, **_: object) -> dict: NativeToolSpec( name="read_file", description=( - "Read lines from a file in the container. " + "Read lines from a relevant grep_source or find_source hit. " + "Repository-relative paths resolve inside /workspace. " "Parameters: path (required), offset (line offset, default 0), " "limit (max lines, default 2000). No other parameters exist." ), diff --git a/clearwing/agent/tools/hunt/discovery.py b/clearwing/agent/tools/hunt/discovery.py index 5b5da721..ffa3e406 100644 --- a/clearwing/agent/tools/hunt/discovery.py +++ b/clearwing/agent/tools/hunt/discovery.py @@ -16,6 +16,7 @@ import logging import os import re +import shlex from pathlib import Path from pydantic import Field @@ -109,7 +110,7 @@ def _container_path(rel_path: str) -> str: return f"/workspace/{rel_path}".replace("//", "/") -def _parse_rg_output(stdout: str, default_file: str = "") -> list[dict]: +def _parse_rg_output(stdout: str, default_file: str = "", max_results: int = 100) -> list[dict]: """Turn ripgrep's `--no-heading --line-number` output into match dicts.""" matches: list[dict] = [] for line in stdout.splitlines(): @@ -133,7 +134,7 @@ def _parse_rg_output(stdout: str, default_file: str = "") -> list[dict]: "matched_text": _safe_source_output(text.rstrip(), "grep match"), } ) - if len(matches) >= 100: + if len(matches) >= max_results: break return matches @@ -143,6 +144,7 @@ def _grep_python_fallback( rel_dir: str, pattern: str, file_glob: str, + max_results: int = 100, ) -> list[dict]: """Pure-Python fallback when no sandbox is attached (test mode).""" try: @@ -186,7 +188,7 @@ def _grep_python_fallback( "matched_text": _safe_source_output(line.rstrip(), "grep match"), } ) - if len(matches) >= 100: + if len(matches) >= max_results: return matches except OSError: continue @@ -292,7 +294,13 @@ def list_source_tree(dir_path: str = ".", max_depth: int = 2) -> list[str]: return out return out - def grep_source(pattern: str, path: str = ".", file_glob: str = "") -> list[dict]: + def grep_source( + pattern: str, + path: str = ".", + file_glob: str = "", + *, + max_results: int = 100, + ) -> list[dict]: """ripgrep-style search for a pattern. Returns up to 100 matches. Args: @@ -320,15 +328,31 @@ def grep_source(pattern: str, path: str = ".", file_glob: str = "") -> list[dict if ctx.sandbox is not None: # Run rg inside the sandbox so we don't depend on the host having it target = _container_path(rel) - argv = ["rg", "--no-heading", "--line-number", "--with-filename", "--max-count", "100"] + argv = ["rg", "--no-heading", "--line-number", "--with-filename"] if file_glob and not path_is_file: argv += ["-g", file_glob] - argv += [pattern, target] - result = ctx.sandbox.exec(argv, timeout=30) - return _parse_rg_output(result.stdout, default_file=rel if path_is_file else "") + argv += ["--", pattern, target] + if max_results == 100: + # Preserve the constrained hunter's existing command shape. + argv[4:4] = ["--max-count", "100"] + result = ctx.sandbox.exec(argv, timeout=30) + else: + # Deep mode asks for one extra match to distinguish a complete + # result from a truncated one. Bound output across all files. + command = f"{shlex.join(argv)} | head -n {max_results}" + result = ctx.sandbox.exec(command, timeout=30) + if result.timed_out: + return [{"error": "repository content search timed out"}] + if result.stderr and not result.stdout: + return [{"error": _safe_source_output(result.stderr.strip(), "grep error")}] + return _parse_rg_output( + result.stdout, + default_file=rel if path_is_file else "", + max_results=max_results, + ) else: # Fallback: use Python re on the host file tree (slower but works in tests) - return _grep_python_fallback(ctx.repo_path, rel, pattern, file_glob) + return _grep_python_fallback(ctx.repo_path, rel, pattern, file_glob, max_results) def find_callers(symbol: str) -> list[dict]: """Find files/lines that reference a symbol. Wraps grep_source. diff --git a/clearwing/sourcehunt/hunter.py b/clearwing/sourcehunt/hunter.py index ef58527d..60df7cd4 100644 --- a/clearwing/sourcehunt/hunter.py +++ b/clearwing/sourcehunt/hunter.py @@ -1,8 +1,7 @@ """Per-file hunter runtime for sourcehunt. -This module now uses a native async tool-calling loop backed by genai-pyo3, -not LangChain/LangGraph. The prompts and tool set are unchanged; the -execution model is simpler: assistant response -> tool calls -> tool results -> +This module uses a native async tool-calling loop backed by genai-pyo3. +The execution model is assistant response -> tool calls -> tool results -> next assistant response, repeated until the model stops calling tools or the step budget is exhausted. """ @@ -993,12 +992,14 @@ def _build_propagation_prompt(file_target: FileTarget) -> str: # --- Deep agent mode ---------------------------------------------------------- -DEEP_AGENT_PROMPT = """You are a security researcher with full shell access inside a sandboxed container. +DEEP_AGENT_PROMPT = """You are a security researcher with shell access inside a sandboxed container. Your shell starts in /workspace, a writable copy of the source tree. Use repository-relative paths; do not prepend `cd /workspace`. Modify source, add debug printfs, recompile, and use `git diff` to track your changes. ASan is enabled by default. UBSan is also available. Tools: -- execute(command): Run any shell command. gcc, gdb, strace, valgrind, make are all available. +- execute(command): Build, debug, and test with gcc, gdb, strace, valgrind, make, and similar commands. Shell grep/find/rg searches are blocked. +- grep_source(pattern, path, file_glob): Search repository source content with bounded results. +- find_source(query, path, kind): Find repository files and directories by name. - read_file(path): Read a file from the container. - write_file(path, contents): Write a file in the container. - record_trace_step(file, line, function, code_snippet, note): Record one step in the vulnerability dataflow trace as you read code. Build the trace incrementally from attacker entry to sink. @@ -1013,6 +1014,16 @@ def _build_propagation_prompt(file_target: FileTarget) -> str: If you find nothing after thorough analysis, say so explicitly. """ +DEEP_SEARCH_GUIDANCE = """ +For source navigation, use grep_source for content and find_source for filenames +or directories. Then read_file on a relevant hit before drawing a conclusion. +Both searches are limited to the repository. A no_matches result means the +query found nothing in that scope; a truncated result needs a narrower scope. +If successive queries return the same locations, change approach or finish. +Shell execute is for builds, debugging, tests, and verification, not broad +grep/find searches. Search results alone are leads, not proof of a finding. +""" + _DEEP_SPECIALIST_FOCUS = { "general": "", "memory_safety": ( @@ -1113,7 +1124,7 @@ def _build_deep_agent_prompt( if count > 0: prompt += "\n" + POOL_ACCESS_BLOCK.format(count=count) - return prompt + DEEP_TRACE_INSTRUCTIONS + return prompt + DEEP_SEARCH_GUIDANCE + DEEP_TRACE_INSTRUCTIONS def _build_unconstrained_prompt( @@ -1187,6 +1198,8 @@ def _build_unconstrained_prompt( prompt += "\n" + ( DEEP_TRACE_INSTRUCTIONS if agent_mode == "deep" else TRACE_BUILDING_INSTRUCTIONS ) + if agent_mode == "deep": + prompt += DEEP_SEARCH_GUIDANCE return prompt @@ -1437,7 +1450,7 @@ def _build_subsystem_prompt( # Subsystem hunt runs in deep-agent mode (read_file/execute), so use the # trace block that references read_file — NOT read_source_file, which the # deep agent doesn't have. - return prompt + DEEP_TRACE_INSTRUCTIONS + return prompt + DEEP_SEARCH_GUIDANCE + DEEP_TRACE_INSTRUCTIONS def build_subsystem_hunter_agent( @@ -1547,8 +1560,8 @@ class NativeHunter: initial_user_message: str = "" # spec 006: override default first message max_repeated_skips: int = 15 # hard cap on total skipped degenerate-loop calls before giving up lead_checkpoint_calls: int = 4 - # Stop when this many consecutive steps make no new progress — no new - # finding, no new potential, and no first read of an unread file. 0 disables. + # Stop when this many consecutive steps make no new progress — findings, + # potentials, source reads, or at most two distinct search discoveries. # Complements degenerate_loop (repeated identical calls) and empty_response # (empty turns), which do not catch varied-but-unproductive reading. max_steps_without_progress: int = 8 @@ -1611,25 +1624,28 @@ async def arun(self) -> HunterRunResult: # write_file does not count — otherwise a stream of varied failing # commands would silently reset the stall counter. # When nothing advances for max_steps_without_progress steps the hunter - # stops (see _should_stop). Steps that only reissued an already- - # throttled call are left to the degenerate_loop terminal, so they do - # not count toward the stall. + # stops (see _should_stop). Repeated structured searches count toward + # this guard; other throttled calls retain the degenerate_loop terminal. last_progress_sig = ( len(self.ctx.findings), - len(self.ctx.potentials), + tuple(sorted(str(p.get("id")) for p in self.ctx.potentials)), len(self.ctx.files_read), len(self.ctx.deep_files_read), 0, ) deep_read_ranges: dict[str, list[tuple[int, int]]] = {} deep_read_range_progress = 0 + seen_search_locations: set[tuple[str, str, int]] = set() + search_progress_epoch = 0 + last_search_progress_epoch = 0 + search_resets = 0 steps_since_progress = 0 prev_step_had_skip = False while True: step += 1 progress_sig = ( len(self.ctx.findings), - len(self.ctx.potentials), + tuple(sorted(str(p.get("id")) for p in self.ctx.potentials)), len(self.ctx.files_read), len(self.ctx.deep_files_read), deep_read_range_progress, @@ -1637,9 +1653,47 @@ async def arun(self) -> HunterRunResult: if progress_sig != last_progress_sig: last_progress_sig = progress_sig steps_since_progress = 0 + last_search_progress_epoch = search_progress_epoch + elif search_progress_epoch != last_search_progress_epoch and search_resets < 2: + # A new source location buys one chance to inspect it. More + # searches cannot indefinitely postpone the eight-step guard. + steps_since_progress = 0 + search_resets += 1 + last_search_progress_epoch = search_progress_epoch elif not prev_step_had_skip: steps_since_progress += 1 prev_step_had_skip = False + final_decision_turn = ( + self.max_steps_without_progress > 0 + and steps_since_progress == self.max_steps_without_progress - 1 + and bool(self.ctx.potentials) + and step < self.max_steps + ) + if final_decision_turn: + messages.append( + ChatMessage( + "user", + "FINAL LEAD DECISION: The no-progress guard will stop this hunt " + "after this turn. Choose one action for each active lead: call " + "record_finding only if the source evidence supports the claim; " + "call dismiss_potential or defer_potential if it does not; or " + "give a concise tool-free final response beginning UNVERIFIED: " + "and naming exactly what remains unverified. An update_potential or another search is " + "not a final decision. Do not promote an uncertain lead.", + ) + ) + elif ( + self.max_steps_without_progress > 0 + and steps_since_progress == self.max_steps_without_progress - 2 + ): + messages.append( + ChatMessage( + "user", + "The hunt is near its no-progress limit. Read a relevant new " + "source location, record a supported lead or finding, or finish. " + "Repeated search results do not extend the hunt.", + ) + ) # Forced fresh synthesis: when approaching the stop boundary, # inject a user message demanding the model consolidate findings # before we cut it off. @@ -1780,6 +1834,12 @@ async def arun(self) -> HunterRunResult: provider_name = getattr(self.llm, "provider_name", None) active_tools = [] if final_synthesis_turn else self.tools + if final_decision_turn and active_tools: + active_tools = [ + tool + for tool in active_tools + if tool.name in {"record_finding", "dismiss_potential", "defer_potential"} + ] if active_tools and not self.ctx.potentials: inactive_potential_tools = { "update_potential", @@ -1970,6 +2030,11 @@ async def arun(self) -> HunterRunResult: tool_arguments, ) ) + final_decision_blocked = final_decision_turn and tool_call.fn_name not in { + "record_finding", + "dismiss_potential", + "defer_potential", + } # Keyed on a normalized prefix rather than the full argument # string: models stuck in a degenerate loop often reissue the # same call with a growing/mutating tail (e.g. appending @@ -2019,7 +2084,24 @@ async def arun(self) -> HunterRunResult: repeated_tool_calls[key] = repeated_tool_calls.get(key, 0) + 1 skipped = repeated_tool_calls[key] > 3 - if dynamic_verification_blocked: + if final_decision_blocked: + tool_output = { + "error": "final lead decision required: record_finding, dismiss_potential, defer_potential, or give a tool-free explanation" + } + tool_summary = _tool_output_text( + tool_call.fn_name, tool_arguments, tool_output + ) + trajectory.log( + "tool_result", + { + "step": step, + "tool_call": _serialize_tool_call(tool_call), + "tool_output": tool_output, + "tool_summary": tool_summary, + "final_decision_blocked": True, + }, + ) + elif dynamic_verification_blocked: tool_output = { "error": ( "dynamic verification requires an active potential. " @@ -2045,7 +2127,7 @@ async def arun(self) -> HunterRunResult: ) elif skipped: total_repeated_skips += 1 - prev_step_had_skip = True + prev_step_had_skip = tool_call.fn_name not in {"grep_source", "find_source"} tool_output = { "error": ( "tool call skipped: you already made this exact call and it " @@ -2079,6 +2161,25 @@ async def arun(self) -> HunterRunResult: }, ) tool_output = await self._run_tool(tools_by_name, tool_call) + if ( + self.agent_mode == "deep" + and tool_call.fn_name in {"grep_source", "find_source"} + and isinstance(tool_output, dict) + and tool_output.get("status") in {"matches", "truncated"} + ): + locations = { + ( + str(hit.get("file") or hit.get("path")), + str(hit.get("type") or "file"), + int(hit.get("line_number") or 0), + ) + for hit in tool_output.get("matches", []) + if isinstance(hit, dict) + and (hit.get("file") or hit.get("type") == "file") + } + if locations - seen_search_locations: + search_progress_epoch += 1 + seen_search_locations.update(locations) if tool_call.fn_name == "read_file" and isinstance(tool_output, str): reread_path = str(tool_arguments.get("path") or "") tool_summary, returned_range = _read_file_tool_response( @@ -2327,11 +2428,20 @@ async def arun(self) -> HunterRunResult: step, len(self.ctx.findings), ) + terminal_reason = "completed" + if ( + final_decision_turn + and self.ctx.potentials + and not last_assistant_text.lstrip().startswith("UNVERIFIED:") + ): + # A generic tool-free reply must not make an unresolved lead + # look completed. The model had its final decision turn. + terminal_reason = "stalled" trajectory.log( "finish", { "step": step, - "status": "completed", + "status": terminal_reason, "findings": [self._serialize_finding(f) for f in self.ctx.findings], "total_input_tokens": total_input_tokens, "total_output_tokens": total_output_tokens, @@ -2342,7 +2452,7 @@ async def arun(self) -> HunterRunResult: findings=list(self.ctx.findings), cost_usd=total_cost_usd, tokens_used=total_input_tokens + total_output_tokens, - stop_reason="completed", + stop_reason=terminal_reason, transcript_summary=last_assistant_text[-500:], potentials=[*self.ctx.potential_history, *self.ctx.potentials], ) @@ -2665,6 +2775,31 @@ def _summarize_read_source(arguments: dict[str, Any], value: str) -> str: def _tool_output_text(tool_name: str, arguments: dict[str, Any], value: Any) -> str: + if tool_name in {"grep_source", "find_source"} and isinstance(value, dict): + status = value.get("status", "error") + matches = value.get("matches") or [] + lines = [ + f"{tool_name}: status={status}; showing={len(matches)}; " + f"scope={value.get('scope', '.')}; truncated={bool(value.get('truncated'))}" + ] + if status == "error": + lines.append(f"error: {value.get('error', 'unknown search error')}") + elif status == "no_matches": + lines.append("No matches in the requested repository scope.") + else: + for match in matches[:20]: + if tool_name == "grep_source": + lines.append( + f"{match.get('file', '?')}:{match.get('line_number', '?')}: " + f"{str(match.get('matched_text', ''))[:100]}" + ) + else: + lines.append(f"[{match.get('type', '?')}] {match.get('path', '?')}") + if len(matches) > 20: + lines.append(f"... {len(matches) - 20} additional hits omitted; narrow the query.") + if value.get("next_action"): + lines.append(str(value["next_action"])) + return _clip_text("\n".join(lines), 3000) if isinstance(value, str): if tool_name == "read_source_file": return _summarize_read_source(arguments, value) diff --git a/tests/test_deep_agent_loop.py b/tests/test_deep_agent_loop.py index 8def6586..fc7284b9 100644 --- a/tests/test_deep_agent_loop.py +++ b/tests/test_deep_agent_loop.py @@ -11,10 +11,11 @@ from genai_pyo3 import ToolCall from clearwing.agent.tools.hunt import build_reporting_tools +from clearwing.agent.tools.hunt.deep_agent import build_deep_agent_tools from clearwing.agent.tools.hunt.potentials import build_potential_tools from clearwing.agent.tools.hunt.sandbox import HunterContext from clearwing.llm.native import NativeToolSpec -from clearwing.sourcehunt.hunter import NativeHunter, _read_file_tool_response +from clearwing.sourcehunt.hunter import NativeHunter, _read_file_tool_response, _tool_output_text @dataclass @@ -84,6 +85,24 @@ def test_read_metadata_does_not_treat_source_word_as_truncation(): assert "Truncated: False" in summary +def test_structured_search_summary_preserves_status_and_next_action(): + result = { + "status": "truncated", + "scope": "src", + "truncated": True, + "matches": [ + {"file": "src/gfx.c", "line_number": i, "matched_text": "x" * 240} for i in range(40) + ], + "next_action": "Read a relevant hit with read_file.", + } + + summary = _tool_output_text("grep_source", {}, result) + + assert summary.startswith("grep_source: status=truncated") + assert "additional hits omitted" in summary + assert "read_file" in summary + + @pytest.mark.asyncio async def test_near_limit_synthesis_requires_a_tool_free_final_response(): hunter, llm = _make_hunter(agent_mode="constrained", max_steps=3) @@ -995,3 +1014,242 @@ async def test_hunter_stalls_when_no_progress(): assert result.stop_reason == "stalled" assert llm.achat.call_count < 20 # stops well short of max_steps + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("symbol", "filename"), + [ + ("RDPGFX_SURFACE_COMMAND", "gfx.c"), + ("struct sc_file", "profile.c"), + ], +) +async def test_repeated_source_searches_stall_without_shell_loops(tmp_path, symbol, filename): + """FreeRDP/OpenSC pattern: varied queries returning the same source hit.""" + source = tmp_path / filename + source.write_text(f"{symbol}\n") + llm = AsyncMock() + ctx = HunterContext(repo_path=str(tmp_path)) + hunter = NativeHunter( + llm=llm, + prompt="test prompt", + tools=build_deep_agent_tools(ctx), + ctx=ctx, + max_steps=100, + agent_mode="deep", + ) + assert hunter.max_steps_without_progress == 8 + queries = itertools.count() + llm.achat.side_effect = lambda **_: FakeResponse( + tool_calls_list=[ + _make_tool_call("grep_source", {"pattern": f"{symbol}{'(?:)' * next(queries)}"}) + ] + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "stalled" + assert result.findings == [] + assert llm.achat.call_count <= 12 + assert all( + "grep_source" in {tool.name for tool in call.kwargs["tools"]} + for call in llm.achat.call_args_list + ) + assert all("find /" not in str(call.kwargs["messages"]) for call in llm.achat.call_args_list) + + +@pytest.mark.asyncio +async def test_only_two_new_search_locations_can_reset_stall(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + search = NativeToolSpec( + name="grep_source", + description="search", + schema={"type": "object", "properties": {"pattern": {"type": "string"}}}, + handler=lambda pattern: { + "status": "matches", + "matches": [{"file": f"src/{pattern}.c", "line_number": 1}], + }, + ) + hunter = NativeHunter( + llm=llm, prompt="test", tools=[search], ctx=ctx, max_steps=100, agent_mode="deep" + ) + queries = itertools.count() + llm.achat.side_effect = lambda **_: FakeResponse( + tool_calls_list=[_make_tool_call("grep_source", {"pattern": chr(97 + next(queries))})] + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "stalled" + assert llm.achat.call_count == 10 # 2 discoveries plus the eight-step guard + + +@pytest.mark.asyncio +async def test_exact_repeated_search_still_reaches_eight_step_guard(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + search = NativeToolSpec( + name="grep_source", + description="search", + schema={"type": "object", "properties": {"pattern": {"type": "string"}}}, + handler=lambda pattern: { + "status": "matches", + "matches": [{"file": "src/gfx.c", "line_number": 12}], + }, + ) + hunter = NativeHunter( + llm=llm, prompt="test", tools=[search], ctx=ctx, max_steps=100, agent_mode="deep" + ) + llm.achat.side_effect = lambda **_: FakeResponse( + tool_calls_list=[_make_tool_call("grep_source", {"pattern": "RDPGFX_SURFACE_COMMAND"})] + ) + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "stalled" + assert llm.achat.call_count <= 10 + + +def _decision_hunter(): + llm = AsyncMock() + ctx = HunterContext(repo_path="/tmp/repo") + ctx.potentials.append({"id": "lead-1", "file": "src/Archive.java", "status": "open"}) + executed: list[str] = [] + + def execute(command): + executed.append(command) + return "PATH TRAVERSAL BYPASS CONFIRMED" + + def record_finding(description): + ctx.findings.append({"description": description}) + return "Finding recorded" + + tools = [ + NativeToolSpec( + name=name, + description=name, + schema={"type": "object", "properties": {field: {"type": "string"}}}, + handler=handler, + ) + for name, field, handler in ( + ("execute", "command", execute), + ("update_potential", "observation", lambda observation: "Updated potential"), + ("record_finding", "description", record_finding), + ("defer_potential", "reason", lambda reason: "Deferred potential"), + ("dismiss_potential", "resolution", lambda resolution: "Dismissed potential"), + ) + ] + hunter = NativeHunter( + llm=llm, prompt="test", tools=tools, ctx=ctx, max_steps=50, agent_mode="deep" + ) + hunter.max_steps_without_progress = 4 + return hunter, llm, executed + + +@pytest.mark.asyncio +async def test_junrar_pattern_gets_final_record_finding_choice(): + hunter, llm, executed = _decision_hunter() + llm.achat.side_effect = [ + FakeResponse( + tool_calls_list=[_make_tool_call("execute", {"command": "javac PoC.java && java PoC"})] + ), + FakeResponse( + tool_calls_list=[_make_tool_call("update_potential", {"observation": "PoC confirmed"})] + ), + FakeResponse( + tool_calls_list=[ + _make_tool_call("record_finding", {"description": "Confirmed path traversal"}) + ] + ), + FakeResponse(text="Finding recorded and investigation complete."), + ] + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "completed" + assert len(result.findings) == 1 + assert executed == ["javac PoC.java && java PoC"] + decision_call = llm.achat.call_args_list[2] + assert {tool.name for tool in decision_call.kwargs["tools"]} == { + "record_finding", + "dismiss_potential", + "defer_potential", + } + assert any( + "FINAL LEAD DECISION" in str(message.to_dict().get("content", "")) + for message in decision_call.kwargs["messages"] + ) + + +@pytest.mark.asyncio +async def test_uncertain_final_lead_is_explained_without_promotion(): + hunter, llm, _ = _decision_hunter() + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("execute", {"command": "inspect one"})]), + FakeResponse( + tool_calls_list=[_make_tool_call("update_potential", {"observation": "unknown"})] + ), + FakeResponse(text="UNVERIFIED: The caller's path to this sink is not established."), + ] + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "completed" + assert result.findings == [] + assert result.potentials[0]["id"] == "lead-1" + assert "not established" in result.transcript_summary + + +@pytest.mark.asyncio +async def test_generic_final_reply_keeps_unresolved_lead_stalled(): + hunter, llm, _ = _decision_hunter() + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("execute", {"command": "inspect one"})]), + FakeResponse( + tool_calls_list=[_make_tool_call("update_potential", {"observation": "unknown"})] + ), + FakeResponse(text="done"), + ] + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "stalled" + assert result.findings == [] + + +@pytest.mark.asyncio +async def test_final_decision_rejects_another_potential_update(): + hunter, llm, _ = _decision_hunter() + update = next(tool for tool in hunter.tools if tool.name == "update_potential") + original_handler = update.handler + update.handler = MagicMock(side_effect=original_handler) + llm.achat.side_effect = [ + FakeResponse(tool_calls_list=[_make_tool_call("execute", {"command": "inspect one"})]), + FakeResponse( + tool_calls_list=[_make_tool_call("update_potential", {"observation": "PoC confirmed"})] + ), + FakeResponse( + tool_calls_list=[_make_tool_call("update_potential", {"observation": "same claim"})] + ), + ] + + with patch("clearwing.sourcehunt.hunter.HunterTrajectoryLogger") as mock_traj: + mock_traj.for_hunter.return_value = MagicMock() + result = await hunter.arun() + + assert result.stop_reason == "stalled" + assert result.findings == [] + assert update.handler.call_count == 1 diff --git a/tests/test_deep_agent_tools.py b/tests/test_deep_agent_tools.py index 664a0d7b..0cda7f0c 100644 --- a/tests/test_deep_agent_tools.py +++ b/tests/test_deep_agent_tools.py @@ -1,7 +1,9 @@ -"""Unit tests for deep agent mode tools (execute, read_file, write_file).""" +"""Unit tests for deep agent mode search and sandbox tools.""" from __future__ import annotations +import subprocess +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -40,7 +42,142 @@ def test_build_deep_agent_tools_set(ctx): names = {t.name for t in tools} assert "think" not in names assert "semgrep_scan" not in names - assert {"execute", "read_file", "write_file", "record_finding"} <= names + assert { + "grep_source", + "find_source", + "execute", + "read_file", + "write_file", + "record_finding", + } <= names + + +def _repo_tools(repo: Path): + return {t.name: t for t in build_deep_agent_tools(HunterContext(repo_path=str(repo)))} + + +def test_deep_grep_source_bounded_and_explicit_empty(tmp_path): + source = tmp_path / "src" / "codec.c" + source.parent.mkdir() + source.write_text("".join(f"needle {line}\n" for line in range(45))) + tools = _repo_tools(tmp_path) + + found = tools["grep_source"].invoke({"pattern": "needle", "path": "src"}) + assert found["status"] == "truncated" + assert found["truncated"] is True + assert found["count"] == 40 + assert found["matches"][0] == { + "file": "src/codec.c", + "line_number": 1, + "matched_text": "needle 0", + } + assert "read_file" in found["next_action"] + + empty = tools["grep_source"].invoke({"pattern": "absent", "path": "src"}) + assert empty["status"] == "no_matches" + assert empty["matches"] == [] + assert empty["truncated"] is False + + +def test_deep_find_source_matches_files_directories_and_stays_in_repo(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + target = repo / "src" / "parser" + target.mkdir(parents=True) + for index in range(45): + (target / f"parser_{index:02}.c").write_text("int x;\n") + outside = tmp_path / "outside_secret.c" + outside.write_text("secret\n") + (repo / "outside_secret.c").symlink_to(outside) + tools = _repo_tools(repo) + + files = tools["find_source"].invoke({"query": "parser_", "kind": "file"}) + assert files["status"] == "truncated" + assert files["count"] == 40 + assert all(hit["type"] == "file" for hit in files["matches"]) + directories = tools["find_source"].invoke({"query": "pars*", "kind": "directory"}) + assert directories["matches"] == [{"path": "src/parser", "type": "directory"}] + assert tools["find_source"].invoke({"query": "missing"})["status"] == "no_matches" + assert tools["find_source"].invoke({"query": "secret"})["status"] == "no_matches" + assert tools["find_source"].invoke({"query": "x", "path": "../"})["status"] == "error" + + +def test_deep_grep_sandbox_command_has_global_result_cap(tmp_path): + source = tmp_path / "src.c" + source.write_text("needle\n") + sandbox = MagicMock() + sandbox.exec.return_value = ExecResult( + exit_code=0, + stdout="".join(f"/workspace/src.c:{line}:needle\n" for line in range(1, 42)), + stderr="", + duration_seconds=0.01, + ) + tools = {t.name: t for t in build_deep_agent_tools(HunterContext(str(tmp_path), sandbox))} + + result = tools["grep_source"].invoke({"pattern": "needle"}) + + assert result["status"] == "truncated" + assert len(result["matches"]) == 40 + command = sandbox.exec.call_args.args[0] + assert "head -n 41" in command + assert "/workspace" in command + + +def test_deep_find_uses_bounded_sandbox_repository_search(tmp_path): + (tmp_path / "src").mkdir() + sandbox = MagicMock() + sandbox.exec.return_value = ExecResult( + exit_code=0, + stdout="directory\t/workspace/src/parser\nfile\t/workspace/src/parser.c\n", + stderr="", + duration_seconds=0.01, + ) + tools = {t.name: t for t in build_deep_agent_tools(HunterContext(str(tmp_path), sandbox))} + + result = tools["find_source"].invoke({"query": "parser", "path": "src"}) + + assert result["status"] == "matches" + assert result["matches"] == [ + {"path": "src/parser", "type": "directory"}, + {"path": "src/parser.c", "type": "file"}, + ] + command = sandbox.exec.call_args.args[0] + assert "find /workspace/src" in command + assert "head -n 41" in command + assert sandbox.exec.call_args.kwargs["timeout"] == 30 + + +def test_deep_find_sandbox_command_returns_real_paths(tmp_path): + (tmp_path / "src" / "parser").mkdir(parents=True) + (tmp_path / "src" / "parser.c").write_text("int parse(void);\n") + sandbox = MagicMock() + + def run_in_test_repo(command, timeout): + completed = subprocess.run( + command.replace("/workspace", str(tmp_path)), + shell=True, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + return ExecResult( + exit_code=completed.returncode, + stdout=completed.stdout.replace(str(tmp_path), "/workspace"), + stderr=completed.stderr, + duration_seconds=0.01, + ) + + sandbox.exec.side_effect = run_in_test_repo + tools = {t.name: t for t in build_deep_agent_tools(HunterContext(str(tmp_path), sandbox))} + + result = tools["find_source"].invoke({"query": "pars*", "path": "src"}) + + assert result["status"] == "matches" + assert result["matches"] == [ + {"path": "src/parser", "type": "directory"}, + {"path": "src/parser.c", "type": "file"}, + ] def test_execute_runs_command(tools, mock_sandbox): @@ -57,6 +194,23 @@ def test_execute_custom_timeout(tools, mock_sandbox): mock_sandbox.exec.assert_called_once_with("make", timeout=600) +@pytest.mark.parametrize( + "command", + [ + "find / -name evp.h", + "cd /workspace && find /workspace -name internal.h", + "grep -rn cipher_final / 2>/dev/null | head", + "grep -rn RDPGFX_SURFACE_COMMAND --include=*.h .", + "find . -name pkcs15.h", + ], +) +def test_execute_redirects_broad_search_to_repository_tools(tools, mock_sandbox, command): + result = tools["execute"].handler(command=command) + assert "grep_source" in result["error"] + assert "find_source" in result["error"] + mock_sandbox.exec.assert_not_called() + + def test_execute_ignores_unexpected_arguments(tools, mock_sandbox): result = tools["execute"].invoke({"command": "ls -la", "_comment": "list files"}) mock_sandbox.exec.assert_called_once_with("ls -la", timeout=300) diff --git a/tests/test_sourcehunt_subsystem.py b/tests/test_sourcehunt_subsystem.py index cc3ada24..65896ee2 100644 --- a/tests/test_sourcehunt_subsystem.py +++ b/tests/test_sourcehunt_subsystem.py @@ -321,6 +321,8 @@ def test_subsystem_prompt_includes_file_listing(): assert "Lifecycle/state" in prompt assert "dominates every subsequent" in prompt assert "Map validation, mutation, and use by lifecycle stage" in prompt + assert "grep_source for content" in prompt + assert "find_source for filenames" in prompt def test_subsystem_prompt_cross_file_calls(): diff --git a/tests/test_unconstrained_prompt.py b/tests/test_unconstrained_prompt.py index e354dee2..69875c66 100644 --- a/tests/test_unconstrained_prompt.py +++ b/tests/test_unconstrained_prompt.py @@ -4,14 +4,7 @@ from unittest.mock import MagicMock -import pytest - from clearwing.sourcehunt.hunter import ( - DISCOVERY_PROMPT, - EXPLOIT_EXTENSION, - HUNTER_EXECUTION_RULES, - MITIGATION_REASONING, - SELF_CHECK, _build_unconstrained_prompt, build_hunter_agent, ) @@ -51,6 +44,15 @@ def test_has_self_check(self): assert "Before you record a finding" in prompt assert "attacker would actually trigger" in prompt + def test_deep_mode_guides_search_to_source_read(self): + prompt = _build_unconstrained_prompt( + _make_file_target(), "test-project", None, None, agent_mode="deep" + ) + assert "grep_source for content" in prompt + assert "find_source for filenames" in prompt + assert "read_file on a relevant hit" in prompt + assert "If successive queries return the same locations" in prompt + def test_has_file_and_project(self): ft = _make_file_target("lib/parser.c") prompt = _build_unconstrained_prompt(ft, "my-project", None, None) @@ -59,25 +61,24 @@ def test_has_file_and_project(self): def test_exploit_mode_appends_extension_and_mitigation(self): ft = _make_file_target() - prompt = _build_unconstrained_prompt( - ft, "test-project", None, None, exploit_mode=True - ) + prompt = _build_unconstrained_prompt(ft, "test-project", None, None, exploit_mode=True) assert "please write exploits" in prompt assert "defensive mitigation" in prompt assert "int32_t[] gets no canary" in prompt def test_no_exploit_mode_omits_extension(self): ft = _make_file_target() - prompt = _build_unconstrained_prompt( - ft, "test-project", None, None, exploit_mode=False - ) + prompt = _build_unconstrained_prompt(ft, "test-project", None, None, exploit_mode=False) assert "please write exploits" not in prompt assert "defensive mitigation" not in prompt def test_campaign_hint_formatted(self): ft = _make_file_target() prompt = _build_unconstrained_prompt( - ft, "test-project", None, None, + ft, + "test-project", + None, + None, campaign_hint="bugs reachable from unauthenticated remote input", ) assert "bugs reachable from unauthenticated remote input" in prompt @@ -111,7 +112,10 @@ def test_all_options_combined(self): crash = {"report": "UBSan: signed-integer-overflow"} hints = [{"line": 10, "description": "width truncation"}] prompt = _build_unconstrained_prompt( - ft, "test-project", crash, hints, + ft, + "test-project", + crash, + hints, campaign_hint="integer overflow bugs", exploit_mode=True, )