0.6.16 — preprocessor naming and fixture taxonomy cleanup

docs/DescriptionAndMilestones.md

@@ -93,7 +93,7 @@ Long-term memory remains a host persistence responsibility, not an engine-owned
 
 ### 0.6.x
 
-The 0.6.x line completed checkpoint support, precompiler boundary hardening, and
+The 0.6.x line completed checkpoint support, preprocessor boundary hardening, and
 regression/conformance surfaces that prepare the project for the next milestone.
 
 ### 0.7 — Auditability & Boundary Hardening
@@ -119,7 +119,7 @@ Make engine behavior inspectable and externally controllable without guessing.
   - requires `--with-precompiler`
   - never implicit
   - inspectable via preview / JSON output
-- Explicit precompiler policy for multi-line, multi-sentence, and conversational-prefix input
+- Explicit preprocessor policy for multi-line, multi-sentence, and conversational-prefix input
   (for example `ok. prohibit peanuts`, `sure - use docker`, mixed conversational + directive content)
   that is rule-based, fixture-covered, and inspectable
 - Define policy for directive-adjacent mixed-intent payloads
@@ -131,7 +131,7 @@ Make engine behavior inspectable and externally controllable without guessing.
 - No expansion of authoritative state model
 - No implicit behavior
 - No heuristic-heavy parsing
-- Preserve separation between engine, precompiler, and host/controller layers
+- Preserve separation between engine, preprocessor, and host/controller layers
 
 ### Post-0.7 Direction
 
@@ -145,7 +145,7 @@ Conceptual completion is a stable minimal contract, not feature accumulation.
 
 - Stable minimal engine contract
 - Deterministic and inspectable behavior
-- Strict compiler / precompiler / host separation
+- Strict compiler / preprocessor / host separation
 - No implicit behavior
 - No authoritative state-model expansion
 - Cross-language consistency with Python as source of truth

examples/integrations/litellm/with_preprocessor.py

@@ -28,7 +28,7 @@
 from context_compiler.engine import Engine
 from experimental.preprocessor import (
     PRECOMPILE_OUTCOME_DIRECTIVE,
-    parse_precompiler_output,
+    parse_preprocessor_output,
     precompile_heuristic,
     render_prompt,
 )
@@ -236,7 +236,7 @@ def _llm_fallback_precompile(message: str, state: State) -> str | None:
     except Exception:
         return None
 
-    parsed = parse_precompiler_output(raw_output, source_input=message)
+    parsed = parse_preprocessor_output(raw_output, source_input=message)
     if parsed is None:
         return None
     return parsed
@@ -251,7 +251,7 @@ def _precompile_user_input(message: str, state: State) -> str | None:
             heuristic_result["outcome"] == PRECOMPILE_OUTCOME_DIRECTIVE
             and heuristic_result["directive"]
         ):
-            parsed = parse_precompiler_output(heuristic_result["directive"])
+            parsed = parse_preprocessor_output(heuristic_result["directive"])
             logger.debug("preprocessor: heuristic_directive=%r", heuristic_result["directive"])
             if parsed is not None:
                 return parsed

examples/integrations/litellm_proxy/context_compiler_precall_hook_with_preprocessor.py

@@ -34,7 +34,7 @@ class CustomLogger:  # type: ignore[no-redef]
 )
 from experimental.preprocessor import (
     PRECOMPILE_OUTCOME_DIRECTIVE,
-    parse_precompiler_output,
+    parse_preprocessor_output,
     precompile_heuristic,
     render_prompt,
 )
@@ -180,7 +180,7 @@ def _llm_fallback_precompile(message: str, state: State) -> str | None:
     except Exception:
         return None
 
-    parsed = parse_precompiler_output(raw_output, source_input=message)
+    parsed = parse_preprocessor_output(raw_output, source_input=message)
     if parsed is None:
         return None
     return parsed
@@ -203,7 +203,7 @@ def _precompile_last_user_message(message: str, state: State | None) -> str | No
             heuristic_result["outcome"] == PRECOMPILE_OUTCOME_DIRECTIVE
             and heuristic_result["directive"]
         ):
-            parsed = parse_precompiler_output(heuristic_result["directive"])
+            parsed = parse_preprocessor_output(heuristic_result["directive"])
             if parsed is not None:
                 return parsed
     except Exception:

examples/integrations/openwebui/open_webui_pipe_with_preprocessor.py

@@ -51,7 +51,7 @@ def Field(*, default: Any, description: str = "") -> Any:  # type: ignore[no-red
 from context_compiler.engine import Engine
 from experimental.preprocessor import (
     PRECOMPILE_OUTCOME_DIRECTIVE,
-    parse_precompiler_output,
+    parse_preprocessor_output,
     precompile_heuristic,
     render_prompt,
 )
@@ -671,7 +671,7 @@ async def _llm_fallback_precompile(
             return None, normalized_error
 
         raw_output = _extract_completion_content(response)
-        parsed = parse_precompiler_output(raw_output, source_input=message)
+        parsed = parse_preprocessor_output(raw_output, source_input=message)
         if parsed is None:
             return None, None
         return parsed, None
@@ -694,7 +694,7 @@ async def _precompile_user_input(
             heuristic_result["outcome"] == PRECOMPILE_OUTCOME_DIRECTIVE
             and heuristic_result["directive"]
         ):
-            parsed = parse_precompiler_output(heuristic_result["directive"])
+            parsed = parse_preprocessor_output(heuristic_result["directive"])
             if parsed is not None:
                 return parsed, None
 

experimental/preprocessor/README.md

@@ -11,6 +11,11 @@ Recommended install for integrations using this package:
 Integrations should import this package from the installed environment rather
 than using repo-relative preprocessor paths.
 
+Compatibility note:
+- Prefer `heuristic_preprocessor.py` and `parse_preprocessor_output(...)`.
+- `heuristic_precompiler.py` and `parse_precompiler_output(...)` remain
+  supported compatibility aliases in 0.6.x.
+
 ## Modules
 
 - `heuristic_preprocessor.py`: conservative structural preprocessing pass.

experimental/preprocessor/output_validation.py

@@ -1,9 +1,9 @@
 """Shared preprocessor output normalization and validation helpers.
 
 Public API:
-- validate_precompiler_output
-- parse_precompiler_output
 - parse_preprocessor_output
+- parse_precompiler_output (compatibility alias)
+- validate_precompiler_output
 
 Internal helpers are implementation details and may change.
 """
@@ -146,7 +146,7 @@ def _validate_text_output(raw_output: str) -> PrecompilerValidationResult:
 def validate_precompiler_output(
     raw_output: object, *, source_input: str | None = None
 ) -> PrecompilerValidationResult:
-    """Validate raw precompiler output into a strict classification/output result.
+    """Validate raw preprocessor output into a strict classification/output result.
 
     Contract:
         - directive: output is a canonical directive string

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "context-compiler"
-version = "0.6.15"
+version = "0.6.16"
 description = "Deterministic conversational state engine for LLM applications."
 readme = "README.md"
 requires-python = ">=3.11"

src/context_compiler/repl.py

@@ -1,7 +1,7 @@
 import sys
 from typing import TextIO
 
-from experimental.preprocessor.output_validation import parse_precompiler_output
+from experimental.preprocessor.output_validation import parse_preprocessor_output
 
 from . import __version__, create_engine, get_policy_items, get_premise_value
 from .engine import Decision, DecisionKind, Engine, State
@@ -107,7 +107,7 @@ def _compile_input(raw_input: str, engine: Engine, *, use_precompiler: bool) ->
         return raw_input
     if _has_pending_clarification(engine):
         return raw_input
-    parsed = parse_precompiler_output(raw_input, source_input=raw_input)
+    parsed = parse_preprocessor_output(raw_input, source_input=raw_input)
     return parsed if parsed is not None else raw_input
 
 

tests/fixtures/README.md

@@ -1,16 +1,19 @@
-# Conformance Fixtures
+# Fixture Suites
 
-These fixtures define the cross-language conformance contract for the Context Compiler.
+This directory contains multiple fixture suites with different contracts.
 
-## Layout
+## Fixture types
 
-[`conformance/`](conformance/)
+* [`conformance/`](conformance/) — core engine cross-language conformance contract.
+* [`engine-regression/structured/`](engine-regression/structured/) — deterministic per-turn engine regression fixtures (including checkpoint snapshots).
+* [`preprocessor/`](preprocessor/) — preprocessor heuristic and validation fixtures.
 
-* [`step/`](conformance/step/)
-* [`transcript/`](conformance/transcript/)
+`conformance/` and `engine-regression/structured/` both cover engine behavior at different layers; preprocessor fixtures are intentionally separate from the core engine conformance contract.
 
 ## Step fixtures
 
+For [`conformance/step/`](conformance/step/):
+
 Each step fixture runs:
 
 1. optional `prelude` (array of prior user inputs)
@@ -27,6 +30,8 @@ Then asserts:
 
 ## Transcript fixtures
 
+For [`conformance/transcript/`](conformance/transcript/):
+
 Replay messages using `compile_transcript(messages)`.
 
 Results are normalized to:
@@ -36,13 +41,44 @@ Results are normalized to:
 
 ## Prompt matching
 
+For conformance transcript fixtures:
+
 * If `prompt_to_user` is a string → exact match
 * If `prompt_to_user` is `null` → any non-empty string is accepted
 
 ## Source of truth
 
 Fixtures reflect current Python behavior and tests.
 
+## Engine regression fixtures
+
+[`engine-regression/structured/`](engine-regression/structured/)
+
+These fixtures capture deterministic per-turn engine behavior, including checkpoint snapshots, and are exercised by [`tests/test_structured_regression.py`](../test_structured_regression.py).
+
+They validate:
+
+* per-turn input handling
+* `Decision.kind` outcomes
+* clarification prompt behavior
+* checkpoint export parity against expected snapshots
+* continuation state restoration from checkpoints
+
+## Preprocessor fixtures
+
+[`preprocessor/`](preprocessor/)
+
+These fixtures cover preprocessor behavior (heuristic classification plus output validation), separate from the core engine conformance contract above.
+
+They are exercised by [`tests/test_preprocessor_conformance.py`](../test_preprocessor_conformance.py), including deterministic replay and validation-boundary checks (only validated directive output may pass through).
+
+They validate:
+
+* heuristic classification determinism
+* directive extraction and normalization
+* output validation boundaries
+* reject/unknown safety handling for ambiguous and near-miss inputs
+
 ## Test runner
 
 See [`tests/test_fixtures.py`](../test_fixtures.py) for execution details.

tests/fixtures/engine-regression/structured/README.md

@@ -48,12 +48,12 @@ They do **not** cover:
 
 - REPL / user-facing formatting
 - LLM integration behavior
-- precompiler / heuristic directive generation
+- preprocessor / heuristic directive generation
 
 These surfaces are tested separately because:
 
 - REPL output may intentionally differ from the underlying state representation
-- precompiler behavior is non-deterministic and outside the engine contract
+- preprocessor behavior is non-deterministic and outside the engine contract
 
 This fixture set is the **canonical engine-level conformance surface**, and may be reused by other implementations (e.g., TypeScript) to validate identical engine behavior.
 

tests/test_litellm_preprocessor_model_config.py

@@ -52,7 +52,7 @@ def _completion(**kwargs):
 
     monkeypatch.setattr(module, "_get_litellm_completion", lambda: _completion)
     monkeypatch.setattr(module, "render_prompt", lambda *_: "prompt")
-    monkeypatch.setattr(module, "parse_precompiler_output", lambda value, **_kwargs: value)
+    monkeypatch.setattr(module, "parse_preprocessor_output", lambda value, **_kwargs: value)
 
     result = module._llm_fallback_precompile("please use docker", None)
 
@@ -75,7 +75,7 @@ def _completion(**kwargs):
 
     monkeypatch.setattr(module, "_get_litellm_completion", lambda: _completion)
     monkeypatch.setattr(module, "render_prompt", lambda *_: "prompt")
-    monkeypatch.setattr(module, "parse_precompiler_output", lambda value, **_kwargs: value)
+    monkeypatch.setattr(module, "parse_preprocessor_output", lambda value, **_kwargs: value)
 
     result = module._llm_fallback_precompile("please use docker", None)
 
@@ -100,7 +100,7 @@ def _completion(**kwargs):
 
     monkeypatch.setattr(module, "_get_litellm_completion", lambda: _completion)
     monkeypatch.setattr(module, "render_prompt", lambda *_: "prompt")
-    monkeypatch.setattr(module, "parse_precompiler_output", lambda value, **_kwargs: value)
+    monkeypatch.setattr(module, "parse_preprocessor_output", lambda value, **_kwargs: value)
 
     result = module._llm_fallback_precompile("please use docker", None)
 
@@ -125,7 +125,7 @@ def _completion(**kwargs):
 
     monkeypatch.setattr(module, "_get_litellm_completion", lambda: _completion)
     monkeypatch.setattr(module, "render_prompt", lambda *_: "prompt")
-    monkeypatch.setattr(module, "parse_precompiler_output", lambda value, **_kwargs: value)
+    monkeypatch.setattr(module, "parse_preprocessor_output", lambda value, **_kwargs: value)
 
     result = module._llm_fallback_precompile("please use docker", None)