Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions lab/test_identity_binding_floor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
"""Floor guards for family c1 -- candidate identity binding (MCP path).

Falsify-first: these guards are written to FAIL against the current code and to
turn green only once the fix lands. They encode the c1 Definition of Done:

* STRUCTURAL GATE -- ``candidate_name`` is ABSENT from the MCP input schema of
the four tools that expose it (the free identity channel is removed), AND the
wrapper injects it from ``CANDIDATE_PROFILE`` (manifest promise true by
construction). We remove the channel; we do not scan symptoms.
* The channel is the single discrete identity field ``candidate_name``, present
identically on the four md-generators -- strategic_playbook,
application_summary, interview_prep, quick_reference. cover_letter is already
bound (sender_* injected from the profile, absent from its schema), so it is a
VERIFICATION here, not a change (see test_cover_letter_already_bound).

The free-text career/analysis dimension (pain_points, pitch, ...) is an
UNDECIDABLE channel -- out of scope for this structural floor; it is a candidate
follow-up family, decided on evidence via the negative-sanity observation.
"""

import json
import sys
import tempfile
import types
from pathlib import Path

# Resolve chain_core the same way the lab floor does (server/ is a sibling of lab/).
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "server"))
import chain_core as core # noqa: E402

# The fictional identity the wrapper must inject (the frozen project decision).
FICTIONAL_NAME = "Robin Mercier"
# A stand-in for a real identity that must NEVER survive into the payload. If a
# caller supplies this, injection must override it with FICTIONAL_NAME.
SENTINEL_REAL = "Sentinel Realname Doe"

# The four tools that expose candidate_name as a free input field.
IDENTITY_TOOL_SCHEMAS = {
core.PLAYBOOK_NAME: core.PLAYBOOK_SCHEMA,
core.SUMMARY_NAME: core.SUMMARY_SCHEMA,
core.INTERVIEW_NAME: core.INTERVIEW_SCHEMA,
core.REFCARD_NAME: core.REFCARD_SCHEMA,
}
IDENTITY_BUILDERS = {
core.PLAYBOOK_NAME: core.build_playbook,
core.SUMMARY_NAME: core.build_summary,
core.INTERVIEW_NAME: core.build_interview,
core.REFCARD_NAME: core.build_refcard,
}

# Minimal legitimate TARGET fields (these stay free -- user supplied).
_TARGET_DATA = {
"job_title": "Staff Engineer",
"company_name": "Globex",
"date": "2026-06-28",
"language": "en",
}


def _capture_payload(build_fn):
"""Call a build_* function with a sentinel candidate_name and capture the
--data-json payload, WITHOUT running the real candidate-suite generator.

resolve_suite_paths and subprocess.run are stubbed; the build function still
composes the payload exactly as in production, which is what we inspect.
"""
captured = {}
orig_resolve = core.resolve_suite_paths
orig_run = core.subprocess.run

def fake_resolve():
dummy = Path("/nonexistent/candidate-suite/script.py")
keys = (
"fill_script",
"template",
"brief_script",
"playbook_script",
"summary_script",
"interview_script",
"refcard_script",
)
return {k: dummy for k in keys}

def fake_run(cmd, **kwargs):
i = cmd.index("--data-json")
captured["payload"] = json.loads(cmd[i + 1])
return types.SimpleNamespace(returncode=0, stdout="", stderr="")

core.resolve_suite_paths = fake_resolve
core.subprocess.run = fake_run
try:
with tempfile.TemporaryDirectory() as td:
data = dict(_TARGET_DATA, candidate_name=SENTINEL_REAL)
build_fn(data, output_dir=td)
finally:
core.resolve_suite_paths = orig_resolve
core.subprocess.run = orig_run
return captured.get("payload", {})


def test_candidate_name_absent_from_schema():
"""STRUCTURAL GATE (channel removed): candidate_name must not appear on the
MCP input surface of the four generators. RED today -- it is present."""
failures = []
for tool, schema in IDENTITY_TOOL_SCHEMAS.items():
if "candidate_name" in schema.get("properties", {}):
failures.append(
f"{tool}: 'candidate_name' in inputSchema.properties (free channel open)"
)
if "candidate_name" in schema.get("required", []):
failures.append(f"{tool}: 'candidate_name' in inputSchema.required")
assert not failures, (
"candidate_name still on the MCP tool surface:\n " + "\n ".join(failures)
)


def test_candidate_name_injected_from_profile():
"""STRUCTURAL GATE (binding): the composed payload must carry the fictional
profile name regardless of caller input. RED today -- the caller's value
passes through unbound (no profile override)."""
failures = []
for tool, build_fn in IDENTITY_BUILDERS.items():
payload = _capture_payload(build_fn)
got = payload.get("candidate_name")
if got != FICTIONAL_NAME:
failures.append(
f"{tool}: payload candidate_name = {got!r}, expected {FICTIONAL_NAME!r} (not injected)"
)
assert not failures, (
"candidate_name not injected from CANDIDATE_PROFILE:\n "
+ "\n ".join(failures)
)


def test_cover_letter_already_bound():
"""VERIFICATION (green today): cover_letter is already correct -- sender_*
injected from the profile and absent from its schema. Guards against a
regression of the existing binding."""
leaked = [
k for k in core.LETTER_SCHEMA.get("properties", {}) if k.startswith("sender_")
]
assert not leaked, f"sender_* leaked onto cover_letter surface: {leaked}"
missing = [
k
for k in ("sender_full_name", "sender_email")
if k not in core.CANDIDATE_PROFILE
]
assert not missing, f"CANDIDATE_PROFILE missing sender fields: {missing}"


if __name__ == "__main__":
_tests = [
test_candidate_name_absent_from_schema,
test_candidate_name_injected_from_profile,
test_cover_letter_already_bound,
]
red = 0
for _t in _tests:
try:
_t()
print(f"[PASS] {_t.__name__}")
except AssertionError as exc:
red += 1
print(
f"[FAIL] {_t.__name__}\n " + str(exc).replace("\n", "\n ")
)
print(
f"\n{red} guard(s) RED / {len(_tests)} total "
"(falsify-first: candidate_name guards are EXPECTED red until the fix lands)"
)
sys.exit(1 if red else 0)
133 changes: 133 additions & 0 deletions lab/test_identity_observed_e2e.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
"""Observed (end-to-end) floor for family c1 -- the "biting" half of the DoD.

The structural floor (test_identity_binding_floor.py) proves the channel is
removed and candidate_name is injected at payload-composition time. This test
goes one step further: it RENDERS each of the five deliverables through the real
candidate-suite generators and asserts the rendered artifact carries the
fictional identity ``Robin Mercier`` and NOT the identity present in the input.

It feeds candidate-suite's own complete sample payloads (which carry the sample
identity ``Jordan Lee-Carter``) through the agent-candidate wrappers. Because the
wrapper injects from CANDIDATE_PROFILE and the profile wins, the rendered output
must show ``Robin Mercier`` -- a strictly stronger check than "target fields
only": even a caller-supplied identity is overridden end-to-end.

Requires candidate-suite checked out and CANDIDATE_SUITE_DIR set (as in the Floor
CI gate). When the suite is absent (unit-only runs) the tests skip cleanly.
"""

import os
import sys
import tempfile
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "server"))
import chain_core as core # noqa: E402

SAMPLE_NAME = "Jordan Lee-Carter" # the identity present in the suite's sample input
FICTIONAL = "Robin Mercier" # what the wrapper must force into every output

# candidate-suite _GENERATORS keys -> agent-candidate build_* function names.
MD_MAP = {
"strategic_playbook": ("05_strategic_playbook", "build_playbook"),
"application_summary": ("03_application_summary", "build_summary"),
"interview_prep": ("04_interview_prep", "build_interview"),
"quick_reference": ("06_quick_reference", "build_refcard"),
}


class _SuiteUnavailable(RuntimeError):
pass


def _load_build_samples():
"""Import candidate-suite's build_samples (the complete fictional fixtures).
Returns the module, or None if the suite is not checked out."""
suite = os.environ.get("CANDIDATE_SUITE_DIR")
if not suite:
return None
tooling = Path(suite) / "tooling"
if not (tooling / "build_samples.py").exists():
return None
sys.path.insert(0, str(tooling))
try:
import build_samples

return build_samples
except Exception:
return None


def _require_samples():
bs = _load_build_samples()
if bs is None:
try:
import pytest

pytest.skip("CANDIDATE_SUITE_DIR / build_samples not available")
except ImportError:
raise _SuiteUnavailable(
"SKIP: CANDIDATE_SUITE_DIR / build_samples not available"
)
return bs


def test_md_generators_render_fictional_identity():
"""The four corrected generators render Robin Mercier; the input identity
does not survive into the output."""
bs = _require_samples()
failures = []
with tempfile.TemporaryDirectory() as td:
for tool, (key, fn_name) in MD_MAP.items():
sample = bs._GENERATORS[key]
data = dict(sample["data"], labels=sample["labels"], language="en")
out = getattr(core, fn_name)(data, output_dir=td)
text = Path(out).read_text(encoding="utf-8")
if FICTIONAL not in text:
failures.append(f"{tool}: rendered output missing {FICTIONAL!r}")
if SAMPLE_NAME in text:
failures.append(
f"{tool}: caller-supplied {SAMPLE_NAME!r} leaked into output"
)
assert not failures, "identity not forced in rendered output:\n " + "\n ".join(
failures
)


def test_cover_letter_renders_fictional_identity():
"""The already-bound letter renders Robin Mercier as the signatory; the
caller-supplied sender identity does not survive."""
bs = _require_samples()
from docx import Document

data = dict(bs.lorem_letter_data(signed=False), language="en")
with tempfile.TemporaryDirectory() as td:
out = core.build_letter(data, output_dir=td)
text = "\n".join(p.text for p in Document(out).paragraphs)
assert FICTIONAL in text, f"cover_letter missing {FICTIONAL!r}"
assert SAMPLE_NAME not in text, (
f"caller-supplied {SAMPLE_NAME!r} leaked into cover_letter"
)


if __name__ == "__main__":
_tests = [
test_md_generators_render_fictional_identity,
test_cover_letter_renders_fictional_identity,
]
red = 0
skipped = 0
for _t in _tests:
try:
_t()
print(f"[PASS] {_t.__name__}")
except _SuiteUnavailable as exc:
skipped += 1
print(f"[SKIP] {_t.__name__}: {exc}")
except AssertionError as exc:
red += 1
print(
f"[FAIL] {_t.__name__}\n " + str(exc).replace("\n", "\n ")
)
print(f"\n{red} failure(s), {skipped} skipped / {len(_tests)} observed e2e checks")
sys.exit(1 if red else 0)
Loading