feat(guidelines): add consistency-fast pipeline, simplify configuration

.env.example

@@ -11,7 +11,7 @@ OPENAI_API_KEY="<YOUR_OPENAI_API_KEY>"
 # OPENAI_BASE_URL=https://api.openai.com/v1
 
 # Optional: Guideline Generation
-# EVOLVE_GUIDELINES_MODE=regular  # Options: regular, consistency, both
+# EVOLVE_GUIDELINES_MODE=standard  # Options: standard, consistency, all
 # EVOLVE_GUIDELINES_MODEL=gpt-4o  # Model used to generate guidelines
 
 # Optional: Debug mode

altk_evolve/cli/cli.py

@@ -540,19 +540,29 @@ def sync_phoenix(
     include_errors: Annotated[bool, typer.Option("--include-errors", help="Include failed/error spans")] = False,
     guidelines_mode: Annotated[
         Optional[str],
-        typer.Option("--guidelines-mode", help="Guideline generation mode: regular, consistency, or both"),
+        typer.Option("--guidelines-mode", help="Guideline generation mode: standard, consistency, or all"),
+    ] = None,
+    consistency_method: Annotated[
+        Optional[str],
+        typer.Option("--consistency-method", help="Consistency pipeline: fast (LLM self-judged, default) or accurate (resampling)"),
     ] = None,
 ):
     """Sync trajectories from Arize Phoenix and generate guidelines."""
     from altk_evolve.config.guidelines import guidelines_settings
     from altk_evolve.sync.phoenix_sync import PhoenixSync
 
     if guidelines_mode is not None:
-        if guidelines_mode not in ("regular", "consistency", "both"):
-            console.print(f"[red]Invalid --guidelines-mode '{guidelines_mode}'. Choose: regular, consistency, both.[/red]")
+        if guidelines_mode not in ("standard", "consistency", "all"):
+            console.print(f"[red]Invalid --guidelines-mode '{guidelines_mode}'. Choose: standard, consistency, all.[/red]")
             raise typer.Exit(1)
         guidelines_settings.guidelines_mode = guidelines_mode
 
+    if consistency_method is not None:
+        if consistency_method not in ("accurate", "fast"):
+            console.print(f"[red]Invalid --consistency-method '{consistency_method}'. Choose: accurate, fast.[/red]")
+            raise typer.Exit(1)
+        guidelines_settings.consistency_method = consistency_method
+
     syncer = PhoenixSync(
         phoenix_url=phoenix_url,
         namespace_id=namespace,
@@ -565,6 +575,8 @@ def sync_phoenix(
     console.print(f"  Namespace: {syncer.namespace_id}")
     console.print(f"  Limit: {limit}")
     console.print(f"  Guidelines mode: {guidelines_settings.guidelines_mode}")
+    if guidelines_settings.guidelines_mode in ("consistency", "all"):
+        console.print(f"  Consistency method: {guidelines_settings.consistency_method}")
     console.print()
 
     try:

altk_evolve/config/guidelines.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from pathlib import Path
 from typing import Optional
 
@@ -8,33 +9,55 @@
 
 logger = logging.getLogger(__name__)
 
+# Replaced by agent_config.yaml's high_uncertainty_threshold / low_uncertainty_threshold /
+# skip_on_no_uncertainty (accurate-method-only knobs). extra="ignore" below means a
+# deployment that still sets these silently loses them with no error — warn instead.
+_REMOVED_UNCERTAINTY_ENV_VARS = (
+    "EVOLVE_HIGH_UNCERTAINTY_THRESHOLD",
+    "EVOLVE_LOW_UNCERTAINTY_THRESHOLD",
+    "EVOLVE_SKIP_ON_NO_UNCERTAINTY",
+)
+
 
 class GuidelinesSettings(BaseSettings):
+    """Guideline-generation settings, read from `EVOLVE_`-prefixed environment variables."""
+
     model_config = SettingsConfigDict(env_prefix="EVOLVE_", env_file=".env", extra="ignore")
 
-    guidelines_mode: str = "regular"
+    guidelines_mode: str = "standard"
+    consistency_method: str = "fast"
     debug_dir: Optional[Path] = Field(default=None)
-    skip_on_no_uncertainty: bool = True
-    high_uncertainty_threshold: float = Field(default=0.2, ge=0.0, le=1.0)
-    low_uncertainty_threshold: float = Field(default=0.1, ge=0.0, le=1.0)
-
-    @model_validator(mode="after")
-    def low_must_not_exceed_high(self) -> Self:
-        if self.low_uncertainty_threshold > self.high_uncertainty_threshold:
-            raise ValueError(
-                f"EVOLVE_LOW_UNCERTAINTY_THRESHOLD ({self.low_uncertainty_threshold}) "
-                f"must be <= EVOLVE_HIGH_UNCERTAINTY_THRESHOLD ({self.high_uncertainty_threshold})"
-            )
-        return self
 
     @field_validator("guidelines_mode", mode="before")
     @classmethod
     def coerce_invalid_mode(cls, v: str) -> str:
-        if v not in ("regular", "consistency", "both"):
-            logger.warning(f"Unrecognised EVOLVE_GUIDELINES_MODE value '{v}', defaulting to 'regular'")
-            return "regular"
+        """Fall back to 'standard' when EVOLVE_GUIDELINES_MODE is set to an unrecognized value."""
+        if v not in ("standard", "consistency", "all"):
+            logger.warning(f"Unrecognised EVOLVE_GUIDELINES_MODE value '{v}', defaulting to 'standard'")
+            return "standard"
         return v
 
+    @field_validator("consistency_method", mode="before")
+    @classmethod
+    def coerce_invalid_consistency_method(cls, v: str) -> str:
+        """Fall back to 'fast' when EVOLVE_CONSISTENCY_METHOD is set to an unrecognized value."""
+        if v not in ("accurate", "fast"):
+            logger.warning(f"Unrecognised EVOLVE_CONSISTENCY_METHOD value '{v}', defaulting to 'fast'")
+            return "fast"
+        return v
+
+    @model_validator(mode="after")
+    def warn_on_removed_uncertainty_env_vars(self) -> Self:
+        stale = [name for name in _REMOVED_UNCERTAINTY_ENV_VARS if os.getenv(name) is not None]
+        if stale:
+            logger.warning(
+                f"{', '.join(stale)} are no longer read (extra='ignore' silently drops them) — "
+                "the accurate consistency method's uncertainty tuning now lives in "
+                "agent_config.yaml (high_uncertainty_threshold / low_uncertainty_threshold / "
+                "skip_on_no_uncertainty), passed via generate_consistency_guidelines(config_path=...)."
+            )
+        return self
+
 
 # to reload settings call guidelines_settings.__init__()
 guidelines_settings = GuidelinesSettings()

altk_evolve/frontend/mcp/mcp_server.py

@@ -593,7 +593,7 @@ def save_trajectory(
     # then merge before the single update_entities call.
     guideline_entities = []
 
-    if guidelines_mode in ("regular", "both"):
+    if guidelines_mode in ("standard", "all"):
         try:
             regular_results = generate_guidelines(messages)
             guideline_entities += [
@@ -607,7 +607,7 @@ def save_trajectory(
                         "rationale": guideline.rationale,
                         "trigger": guideline.trigger,
                         "implementation_steps": guideline.implementation_steps,
-                        "generation_method": "regular",
+                        "generation_method": "standard",
                         "support": 1,
                     },
                 )
@@ -616,20 +616,27 @@ def save_trajectory(
             ]
         except Exception:
             logger.error(
-                f"Regular guideline generation failed for task {task_id}, skipping",
+                f"Standard guideline generation failed for task {task_id}, skipping",
                 exc_info=True,
             )
 
-    if guidelines_mode in ("consistency", "both"):
+    if guidelines_mode in ("consistency", "all"):
         try:
-            from altk_evolve.llm.guidelines.consistency_guidelines import generate_consistency_guidelines
-
             trajectory = {
                 "messages": messages,
                 "trace_id": task_id,
                 "tools": json.loads(tools) if tools else None,
             }
-            consistency_results = generate_consistency_guidelines(trajectory)
+            if guidelines_settings.consistency_method == "fast":
+                from altk_evolve.llm.guidelines.consistency_guidelines import generate_consistency_guidelines_fast
+
+                consistency_results = generate_consistency_guidelines_fast(trajectory)
+                consistency_method_tag = "consistency-fast"
+            else:
+                from altk_evolve.llm.guidelines.consistency_guidelines import generate_consistency_guidelines
+
+                consistency_results = generate_consistency_guidelines(trajectory)
+                consistency_method_tag = "consistency"
             guideline_entities += [
                 Entity(
                     type="guideline",
@@ -641,7 +648,7 @@ def save_trajectory(
                         "rationale": guideline.rationale,
                         "trigger": guideline.trigger,
                         "implementation_steps": guideline.implementation_steps,
-                        "generation_method": "consistency",
+                        "generation_method": consistency_method_tag,
                         "support": 1,
                     },
                 )

altk_evolve/llm/guidelines/consistency_analyzer/agent_config.yaml

@@ -2,10 +2,15 @@
 # Step names produced by transform_trajectory_to_IR() are "OpenAIAgent_content"
 # (assistant text turns) and "OpenAIAgent_tool_calls" (assistant tool-call turns).
 # Users can override by passing config_path= to generate_consistency_guidelines().
-name: agent consistency configuration 
+name: agent consistency configuration
 aggregation: mean
 max_samples: 5
 max_steps: 15
+# Advanced tuning knobs for how step_uncertainty scores are interpreted when
+# generating guidelines (consistency-accurate method only).
+high_uncertainty_threshold: 0.2
+low_uncertainty_threshold: 0.1
+skip_on_no_uncertainty: true
 agents:
   - name: OpenAIAgent_content
     response_type: text

altk_evolve/llm/guidelines/consistency_analyzer/resampling.py

@@ -67,11 +67,12 @@ def resample_trajectory(
 
         prompt = step["messages"]
         step_model = step["llm_params"].get("model")
+        # Prefer the model actually used at this step in the original trajectory;
+        # fall back to the configured default only when the step genuinely has none.
         model = step_model or model_name
-        # Only forward the configured provider when falling back to the configured
-        # model. For per-step models from the traced trajectory, let litellm infer
-        # the provider to avoid misrouting (e.g. a claude model to the openai endpoint).
-        provider = None if step_model else custom_llm_provider
+        # custom_llm_provider is never recorded per-step in the trajectory (only
+        # `model` is) — it's always a deployment-wide routing setting, so it applies
+        # regardless of which model name is used for this step.
         tools = step.get("tools", None)
 
         response_samples = get_response_sampling(
@@ -80,7 +81,7 @@ def resample_trajectory(
             temperature=temperature,
             samples=samples,
             tools=tools,
-            custom_llm_provider=provider,
+            custom_llm_provider=custom_llm_provider,
         )
 
         step["sampling"] = extract_raw_samples(response_samples)