fix: orchestration reliability, security, and retry quality (9 targeted fixes)

misterdev/core/economics/free_models.py

@@ -21,6 +21,11 @@
 
 _MODELS_URL = "https://openrouter.ai/api/v1/models"
 _DAY_SECONDS = 24 * 3600
+# Bound the body materialized from the (semi-trusted) OpenRouter endpoint. Real
+# model lists are well under this; the cap only rejects a compromised/misbehaving
+# endpoint slow-dripping an oversized body. Callers already treat a fetch that
+# raises as a transient failure and fall back to the cached list.
+_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
 
 # Per-process cache of fetched list endpoints, keyed by URL. Free-model
 # harvesting, the model catalog, and each failover client's catalog all hit the
@@ -41,7 +46,10 @@ def _http_fetch(url: str = _MODELS_URL) -> list:
 
     req = urllib.request.Request(url, headers={"User-Agent": "misterdev"})
     with urllib.request.urlopen(req, timeout=10) as resp:
-        payload = json.loads(resp.read().decode("utf-8"))
+        raw = resp.read(_MAX_RESPONSE_BYTES + 1)
+    if len(raw) > _MAX_RESPONSE_BYTES:
+        raise ValueError(f"model-list response exceeded {_MAX_RESPONSE_BYTES} bytes")
+    payload = json.loads(raw.decode("utf-8"))
     data = payload.get("data", []) if isinstance(payload, dict) else []
     _FETCH_CACHE[url] = data
     return data

misterdev/core/economics/llm_cache.py

@@ -33,6 +33,11 @@ class LLMCache:
     def __init__(self, dir_path: Path, max_entries: int = DEFAULT_MAX_ENTRIES):
         self.dir = Path(dir_path)
         self.max_entries = max_entries
+        # Running count of stored entries, seeded by one real scan on the first
+        # new put and re-synced whenever an eviction scan runs. Lets a put below
+        # the cap skip the O(entries) scandir+stat that would otherwise run on
+        # every write (O(entries^2) filesystem work across a build).
+        self._count_estimate: Optional[int] = None
 
     @staticmethod
     def _key(system_prompt: str, prompt: str) -> str:
@@ -71,35 +76,58 @@ def put(
 
         key = self._key(system_prompt, prompt)
         path = self._path(key)
+        is_new = not path.exists()
         atomic_write_json(
             path,
             {"output": output, "model": model, "created_at": timestamp, "key": key},
             indent=2,
         )
-        self._evict_if_needed()
+        if is_new:
+            self._note_new_entry()
 
-    def _evict_if_needed(self) -> None:
-        """Drop the oldest entries (by mtime) once the cap is exceeded.
+    def _note_new_entry(self) -> None:
+        """Account for a newly-stored entry, scanning only when eviction is due.
 
-        Best-effort: any filesystem error degrades to leaving the cache as-is
-        rather than raising into the build, like the rest of this module.
+        The full scandir+stat sweep runs only when the running count first needs
+        seeding or crosses the cap — not on every put — so a long build's writes
+        cost O(entries) filesystem work total rather than O(entries^2).
         """
         if self.max_entries <= 0:
             return
+        if self._count_estimate is None:
+            self._count_estimate = self._evict_if_needed()
+            return
+        self._count_estimate += 1
+        if self._count_estimate > self.max_entries:
+            self._count_estimate = self._evict_if_needed()
+
+    def _evict_if_needed(self) -> int:
+        """Drop the oldest entries (by mtime) once the cap is exceeded.
+
+        Returns the entry count after any eviction, so the caller can re-sync its
+        running estimate. Best-effort: any filesystem error degrades to leaving
+        the cache as-is rather than raising into the build, like the rest of this
+        module.
+        """
+        if self.max_entries <= 0:
+            return 0
         try:
             entries = [
                 (e.stat().st_mtime, e.path)
                 for e in os.scandir(self.dir)
                 if e.name.endswith(".json") and e.is_file()
             ]
         except OSError:
-            return
+            return 0
         excess = len(entries) - self.max_entries
         if excess <= 0:
-            return
+            return len(entries)
         entries.sort(key=lambda t: t[0])
+        removed = 0
         for _mtime, victim in entries[:excess]:
             try:
                 os.remove(victim)
+                removed += 1
             except OSError:
                 continue
+        return len(entries) - removed

misterdev/core/evolution/partial_credit.py

@@ -20,6 +20,7 @@
 regressions==0 + resolved_rate (see fitness.py). Pure and deterministic.
 """
 
+import math
 from dataclasses import dataclass
 from typing import Iterable, Optional, Sequence, Tuple
 
@@ -99,8 +100,8 @@ def variance(self) -> float:
         """Sample variance of the per-task credits (population form)."""
         if self.total < 1:
             return 0.0
-        m = self.mean_credit
-        return sum((c - m) ** 2 for c in self.credits) / self.total
+        m = math.fsum(self.credits) / self.total
+        return math.fsum((c - m) ** 2 for c in self.credits) / self.total
 
     @property
     def stderr(self) -> float:

misterdev/core/execution/parallel.py

@@ -429,7 +429,9 @@ def run_one(item):
                 # to delete a branch still checked out in a worktree, which otherwise
                 # leaks even merged branches. The task's commits live on the branch
                 # ref, so the merge below still sees them after the dir is gone.
-                git.worktree_remove(project, str(wt_path))
+                ok, msg = git.worktree_remove(project, str(wt_path))
+                if not ok:
+                    logger.warning(f"Failed to remove worktree {wt_path}: {msg}")
                 merged = False
                 if (
                     result is not None

misterdev/core/integration/mcp_gather.py

@@ -275,7 +275,10 @@ def _emit(kind: str, **details: Any) -> None:
             gathered.append(f"### {server}.{tool} -> (no result / error)")
             continue
         tag = _UNVETTED_TAG if server in provisioned else ""
-        gathered.append(f"### {server}.{tool}{tag}\n{result}")
+        gathered.append(
+            f"### {server}.{tool}{tag}\n"
+            f'<tool_result name="{server}.{tool}">\n{result}\n</tool_result>'
+        )
 
     if not gathered:
         return ""

misterdev/core/integration/mcp_registry.py

@@ -60,6 +60,13 @@
     "https://raw.githubusercontent.com/Himanshu507/cgcone/main/public/registry.json"
 )
 _DEFAULT_TIMEOUT = 10.0
+# Bound the body materialized from a (semi-trusted) registry/GitHub/npm/PyPI
+# endpoint. Real registry and model-list payloads are well under this; the cap
+# only rejects a compromised/misbehaving endpoint slow-dripping an oversized body.
+_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
+# PEP 503 normalized distribution name; anything else (path separators, spaces)
+# is not a real PyPI package and must not be interpolated raw into a request path.
+_PYPI_NAME_RE = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$")
 _DEFAULT_MAX_SERVERS = 3
 _DEFAULT_MIN_TRUST = 0.5
 # Cap network scoring per build so a wide search cannot hammer the GitHub API
@@ -172,7 +179,11 @@ def _work() -> Optional[Any]:
             with urllib.request.urlopen(req, timeout=timeout) as resp:  # noqa: S310
                 if resp.status != 200:
                     return None
-                return json.loads(resp.read().decode("utf-8"))
+                raw = resp.read(_MAX_RESPONSE_BYTES + 1)
+                if len(raw) > _MAX_RESPONSE_BYTES:
+                    logger.debug(f"MCP registry GET oversized (> cap): {url}")
+                    return None
+                return json.loads(raw.decode("utf-8"))
         except (urllib.error.URLError, ValueError, OSError) as e:
             logger.debug(f"MCP registry GET failed ({url}): {e}")
             return None
@@ -205,6 +216,10 @@ def __init__(self, path: Path, ttl: float = _CACHE_TTL_SECONDS):
         with RegistryCache._locks_guard:
             self._lock = RegistryCache._locks.setdefault(key, threading.Lock())
         self._data: Dict[str, Dict[str, Any]] = self._load()
+        try:
+            self._mtime: float = self.path.stat().st_mtime
+        except OSError:
+            self._mtime = 0.0
 
     def _load(self) -> Dict[str, Dict[str, Any]]:
         try:
@@ -224,13 +239,31 @@ def _flush(self) -> None:
             merged = {**self._load(), **self._data}
             self._data = merged
             tmp = self.path.with_suffix(f".{os.getpid()}.{id(self)}.tmp")
-            tmp.write_text(json.dumps(merged), encoding="utf-8")
-            os.replace(tmp, self.path)
+            try:
+                tmp.write_text(json.dumps(merged), encoding="utf-8")
+                os.replace(tmp, self.path)
+                try:
+                    self._mtime = self.path.stat().st_mtime
+                except OSError:
+                    pass
+            except Exception:
+                try:
+                    tmp.unlink(missing_ok=True)
+                except OSError:
+                    pass
+                raise
         except (OSError, ValueError) as e:
             logger.debug(f"MCP registry cache flush failed: {e}")
 
     def get(self, key: str) -> Optional[Any]:
         with self._lock:
+            try:
+                mtime = self.path.stat().st_mtime
+            except OSError:
+                mtime = 0.0
+            if mtime != self._mtime:
+                self._data = self._load()
+                self._mtime = mtime
             entry = self._data.get(key)
             if not entry:
                 return None
@@ -681,8 +714,11 @@ def _resolve_version(runtime: str, identifier: str, timeout: float, cache) -> st
         quoted = urllib.parse.quote(identifier, safe="@")
         data = _http_get_json(f"https://registry.npmjs.org/{quoted}/latest", timeout)
         version = data.get("version", "") if isinstance(data, dict) else ""
+    elif not _PYPI_NAME_RE.match(identifier):
+        version = ""
     else:
-        data = _http_get_json(f"https://pypi.org/pypi/{identifier}/json", timeout)
+        quoted = urllib.parse.quote(identifier, safe="")
+        data = _http_get_json(f"https://pypi.org/pypi/{quoted}/json", timeout)
         version = (
             (data.get("info") or {}).get("version", "")
             if isinstance(data, dict)

misterdev/core/planning/decomposer.py

@@ -4,7 +4,7 @@
 Uses topological sort to determine execution order.
 """
 
-from collections import deque
+import heapq
 from pathlib import Path
 
 from misterdev.core.models import Task
@@ -494,7 +494,8 @@ def _add_implicit_dependencies(tasks: list[Task]) -> None:
 def topological_sort(tasks: list[Task]) -> list[Task]:
     """Sort tasks respecting dependencies. Falls back to category order on ties.
 
-    Returns tasks in execution order. Raises ValueError on cycles.
+    Returns tasks in execution order. On a dependency cycle it does not raise:
+    the cyclic remainder is appended in category order with a warning.
     """
     task_map = {t.id: t for t in tasks}
     in_degree: dict[str, int] = {t.id: 0 for t in tasks}
@@ -506,20 +507,28 @@ def topological_sort(tasks: list[Task]) -> list[Task]:
                 in_degree[t.id] += 1
                 dependents[dep_id].append(t.id)
 
-    # Seed queue with zero-dependency tasks, sorted by category order
-    queue: deque[str] = deque()
-    ready = [tid for tid, deg in in_degree.items() if deg == 0]
-    ready.sort(key=lambda tid: CATEGORY_ORDER.get(task_map[tid].category, 99))
-    queue.extend(ready)
+    def _rank(tid: str) -> int:
+        return CATEGORY_ORDER.get(task_map[tid].category, 99)
+
+    # Min-heap keyed on (category_rank, insertion_index) so every ready task —
+    # the initial seed and tasks unblocked mid-traversal alike — is dequeued in
+    # category order, with declaration order as a stable tie-break.
+    counter = 0
+    heap: list[tuple[int, int, str]] = []
+    for tid, deg in in_degree.items():
+        if deg == 0:
+            heapq.heappush(heap, (_rank(tid), counter, tid))
+            counter += 1
 
     result = []
-    while queue:
-        tid = queue.popleft()
+    while heap:
+        _, _, tid = heapq.heappop(heap)
         result.append(task_map[tid])
         for dep_tid in dependents[tid]:
             in_degree[dep_tid] -= 1
             if in_degree[dep_tid] == 0:
-                queue.append(dep_tid)
+                heapq.heappush(heap, (_rank(dep_tid), counter, dep_tid))
+                counter += 1
 
     if len(result) != len(tasks):
         # Cycle detected; append remaining tasks anyway with a warning

misterdev/core/planning/tasklist_parser.py

@@ -205,9 +205,25 @@ def _extract_dependency_table(text: str) -> Dict[str, List[str]]:
         ),
         None,
     )
-    task_col = next((i for i, h in enumerate(header) if "task" in h or "id" in h), 0)
+    task_col = next(
+        (
+            i
+            for i, h in enumerate(header)
+            if any(
+                k in h for k in ("task", "id", "step", "feature", "item", "name", "#")
+            )
+        ),
+        None,
+    )
     if dep_col is None:
         return deps
+    if task_col is None or task_col == dep_col:
+        # No recognizable task header (or it collided with the dep column):
+        # take the first column that is not the dependency column rather than
+        # blindly keying on index 0 and manufacturing a self-edge.
+        task_col = next((i for i in range(len(header)) if i != dep_col), None)
+    if task_col is None:
+        return deps
     for row in rows[1:]:
         cells = [c.strip() for c in row.strip("|").split("|")]
         if len(cells) <= max(task_col, dep_col) or set(cells[task_col]) <= {

misterdev/core/verification/gatekeeper/__init__.py

@@ -290,6 +290,12 @@ def run_gates(
                 if mutation.evidence:
                     health.test_output = mutation.evidence
                 return False, issues, health
+            elif mutation.status == "skip" and (mutation.reason or "").startswith(
+                "error:"
+            ):
+                logger.warning(
+                    f"G3.6: Mutation gate errored and was skipped: {mutation.reason}"
+                )
 
         # G4: Type check (optional). Blocking like G1/G3: a configured
         # typecheck command that fails short-circuits the gate so broken types

misterdev/core/verification/web_verify.py

@@ -213,7 +213,7 @@ def _drive_browser(
     deadline: float,
 ) -> WebResult:
     """Load ``url`` headless, run ``checks``, always capture a screenshot."""
-    nav_timeout_ms = max(1, int((deadline - time.monotonic()) * 1000))
+    nav_timeout_ms = max(1000, int((deadline - time.monotonic()) * 1000))
     console_errors: List[str] = []
     results: List[Tuple[str, str]] = []
     failures: List[str] = []

misterdev/llm/client/providers.py

@@ -158,7 +158,7 @@ class OpenRouterLLMClient(BaseLLMClient):
     def __init__(self, config: dict):
         super().__init__(config)
         llm_config = config.get("llm", {})
-        self.client, self.api_key = _openrouter_sdk(llm_config)
+        self.client, _ = _openrouter_sdk(llm_config)
         self.model = get_section_setting("llm", llm_config, "model")
         self.temperature = get_section_setting("llm", llm_config, "temperature")
         self.sampling = dict(get_section_setting("llm", llm_config, "sampling") or {})

misterdev/task_executors/markdown_plan_executor/execute_mixin.py

@@ -861,7 +861,7 @@ def execute(
                     locations = resolver.resolve_errors(output)
                     attributed_error = resolver.format_for_llm(locations, output)
                     classified = format_classified_error(output)
-                    error_logs = self._build_error_context(
+                    error_logs = "[BUILD] " + self._build_error_context(
                         prior_errors,
                         attempt,
                         output,
@@ -886,7 +886,7 @@ def execute(
                     locations = resolver.resolve_errors(output)
                     attributed_error = resolver.format_for_llm(locations, output)
                     classified = format_classified_error(output)
-                    error_logs = self._build_error_context(
+                    error_logs = "[TYPECHECK] " + self._build_error_context(
                         prior_errors,
                         attempt,
                         output,
@@ -1016,7 +1016,7 @@ def execute(
                     locations = resolver.resolve_errors(output)
                     attributed_error = resolver.format_for_llm(locations, output)
                     classified = format_classified_error(output)
-                    error_logs = self._build_error_context(
+                    error_logs = "[TEST] " + self._build_error_context(
                         prior_errors,
                         attempt,
                         output,

misterdev/task_executors/markdown_plan_executor/gates_mixin.py

@@ -112,10 +112,12 @@ def _apply_reflection(
             reflections.append(new_reflection)
         if not reflections:
             return error_logs
+        window = reflections[-3:]
         header = "### Reflection on why previous attempts failed\n" + "\n\n".join(
-            reflections
+            window
         )
-        return f"{header}\n\n{error_logs}"
+        trimmed = error_logs[:20000] if len(error_logs) > 20000 else error_logs
+        return f"{header}\n\n{trimmed}"
 
     def _build_error_context(
         self,