diff --git a/lab/test_identity_binding_floor.py b/lab/test_identity_binding_floor.py new file mode 100644 index 0000000..ff30b5d --- /dev/null +++ b/lab/test_identity_binding_floor.py @@ -0,0 +1,171 @@ +"""Floor guards for family c1 -- candidate identity binding (MCP path). + +Falsify-first: these guards are written to FAIL against the current code and to +turn green only once the fix lands. They encode the c1 Definition of Done: + + * STRUCTURAL GATE -- ``candidate_name`` is ABSENT from the MCP input schema of + the four tools that expose it (the free identity channel is removed), AND the + wrapper injects it from ``CANDIDATE_PROFILE`` (manifest promise true by + construction). We remove the channel; we do not scan symptoms. + * The channel is the single discrete identity field ``candidate_name``, present + identically on the four md-generators -- strategic_playbook, + application_summary, interview_prep, quick_reference. cover_letter is already + bound (sender_* injected from the profile, absent from its schema), so it is a + VERIFICATION here, not a change (see test_cover_letter_already_bound). + +The free-text career/analysis dimension (pain_points, pitch, ...) is an +UNDECIDABLE channel -- out of scope for this structural floor; it is a candidate +follow-up family, decided on evidence via the negative-sanity observation. +""" + +import json +import sys +import tempfile +import types +from pathlib import Path + +# Resolve chain_core the same way the lab floor does (server/ is a sibling of lab/). +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "server")) +import chain_core as core # noqa: E402 + +# The fictional identity the wrapper must inject (the frozen project decision). +FICTIONAL_NAME = "Robin Mercier" +# A stand-in for a real identity that must NEVER survive into the payload. If a +# caller supplies this, injection must override it with FICTIONAL_NAME. +SENTINEL_REAL = "Sentinel Realname Doe" + +# The four tools that expose candidate_name as a free input field. +IDENTITY_TOOL_SCHEMAS = { + core.PLAYBOOK_NAME: core.PLAYBOOK_SCHEMA, + core.SUMMARY_NAME: core.SUMMARY_SCHEMA, + core.INTERVIEW_NAME: core.INTERVIEW_SCHEMA, + core.REFCARD_NAME: core.REFCARD_SCHEMA, +} +IDENTITY_BUILDERS = { + core.PLAYBOOK_NAME: core.build_playbook, + core.SUMMARY_NAME: core.build_summary, + core.INTERVIEW_NAME: core.build_interview, + core.REFCARD_NAME: core.build_refcard, +} + +# Minimal legitimate TARGET fields (these stay free -- user supplied). +_TARGET_DATA = { + "job_title": "Staff Engineer", + "company_name": "Globex", + "date": "2026-06-28", + "language": "en", +} + + +def _capture_payload(build_fn): + """Call a build_* function with a sentinel candidate_name and capture the + --data-json payload, WITHOUT running the real candidate-suite generator. + + resolve_suite_paths and subprocess.run are stubbed; the build function still + composes the payload exactly as in production, which is what we inspect. + """ + captured = {} + orig_resolve = core.resolve_suite_paths + orig_run = core.subprocess.run + + def fake_resolve(): + dummy = Path("/nonexistent/candidate-suite/script.py") + keys = ( + "fill_script", + "template", + "brief_script", + "playbook_script", + "summary_script", + "interview_script", + "refcard_script", + ) + return {k: dummy for k in keys} + + def fake_run(cmd, **kwargs): + i = cmd.index("--data-json") + captured["payload"] = json.loads(cmd[i + 1]) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + core.resolve_suite_paths = fake_resolve + core.subprocess.run = fake_run + try: + with tempfile.TemporaryDirectory() as td: + data = dict(_TARGET_DATA, candidate_name=SENTINEL_REAL) + build_fn(data, output_dir=td) + finally: + core.resolve_suite_paths = orig_resolve + core.subprocess.run = orig_run + return captured.get("payload", {}) + + +def test_candidate_name_absent_from_schema(): + """STRUCTURAL GATE (channel removed): candidate_name must not appear on the + MCP input surface of the four generators. RED today -- it is present.""" + failures = [] + for tool, schema in IDENTITY_TOOL_SCHEMAS.items(): + if "candidate_name" in schema.get("properties", {}): + failures.append( + f"{tool}: 'candidate_name' in inputSchema.properties (free channel open)" + ) + if "candidate_name" in schema.get("required", []): + failures.append(f"{tool}: 'candidate_name' in inputSchema.required") + assert not failures, ( + "candidate_name still on the MCP tool surface:\n " + "\n ".join(failures) + ) + + +def test_candidate_name_injected_from_profile(): + """STRUCTURAL GATE (binding): the composed payload must carry the fictional + profile name regardless of caller input. RED today -- the caller's value + passes through unbound (no profile override).""" + failures = [] + for tool, build_fn in IDENTITY_BUILDERS.items(): + payload = _capture_payload(build_fn) + got = payload.get("candidate_name") + if got != FICTIONAL_NAME: + failures.append( + f"{tool}: payload candidate_name = {got!r}, expected {FICTIONAL_NAME!r} (not injected)" + ) + assert not failures, ( + "candidate_name not injected from CANDIDATE_PROFILE:\n " + + "\n ".join(failures) + ) + + +def test_cover_letter_already_bound(): + """VERIFICATION (green today): cover_letter is already correct -- sender_* + injected from the profile and absent from its schema. Guards against a + regression of the existing binding.""" + leaked = [ + k for k in core.LETTER_SCHEMA.get("properties", {}) if k.startswith("sender_") + ] + assert not leaked, f"sender_* leaked onto cover_letter surface: {leaked}" + missing = [ + k + for k in ("sender_full_name", "sender_email") + if k not in core.CANDIDATE_PROFILE + ] + assert not missing, f"CANDIDATE_PROFILE missing sender fields: {missing}" + + +if __name__ == "__main__": + _tests = [ + test_candidate_name_absent_from_schema, + test_candidate_name_injected_from_profile, + test_cover_letter_already_bound, + ] + red = 0 + for _t in _tests: + try: + _t() + print(f"[PASS] {_t.__name__}") + except AssertionError as exc: + red += 1 + print( + f"[FAIL] {_t.__name__}\n " + str(exc).replace("\n", "\n ") + ) + print( + f"\n{red} guard(s) RED / {len(_tests)} total " + "(falsify-first: candidate_name guards are EXPECTED red until the fix lands)" + ) + sys.exit(1 if red else 0) diff --git a/lab/test_identity_observed_e2e.py b/lab/test_identity_observed_e2e.py new file mode 100644 index 0000000..428b094 --- /dev/null +++ b/lab/test_identity_observed_e2e.py @@ -0,0 +1,133 @@ +"""Observed (end-to-end) floor for family c1 -- the "biting" half of the DoD. + +The structural floor (test_identity_binding_floor.py) proves the channel is +removed and candidate_name is injected at payload-composition time. This test +goes one step further: it RENDERS each of the five deliverables through the real +candidate-suite generators and asserts the rendered artifact carries the +fictional identity ``Robin Mercier`` and NOT the identity present in the input. + +It feeds candidate-suite's own complete sample payloads (which carry the sample +identity ``Jordan Lee-Carter``) through the agent-candidate wrappers. Because the +wrapper injects from CANDIDATE_PROFILE and the profile wins, the rendered output +must show ``Robin Mercier`` -- a strictly stronger check than "target fields +only": even a caller-supplied identity is overridden end-to-end. + +Requires candidate-suite checked out and CANDIDATE_SUITE_DIR set (as in the Floor +CI gate). When the suite is absent (unit-only runs) the tests skip cleanly. +""" + +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "server")) +import chain_core as core # noqa: E402 + +SAMPLE_NAME = "Jordan Lee-Carter" # the identity present in the suite's sample input +FICTIONAL = "Robin Mercier" # what the wrapper must force into every output + +# candidate-suite _GENERATORS keys -> agent-candidate build_* function names. +MD_MAP = { + "strategic_playbook": ("05_strategic_playbook", "build_playbook"), + "application_summary": ("03_application_summary", "build_summary"), + "interview_prep": ("04_interview_prep", "build_interview"), + "quick_reference": ("06_quick_reference", "build_refcard"), +} + + +class _SuiteUnavailable(RuntimeError): + pass + + +def _load_build_samples(): + """Import candidate-suite's build_samples (the complete fictional fixtures). + Returns the module, or None if the suite is not checked out.""" + suite = os.environ.get("CANDIDATE_SUITE_DIR") + if not suite: + return None + tooling = Path(suite) / "tooling" + if not (tooling / "build_samples.py").exists(): + return None + sys.path.insert(0, str(tooling)) + try: + import build_samples + + return build_samples + except Exception: + return None + + +def _require_samples(): + bs = _load_build_samples() + if bs is None: + try: + import pytest + + pytest.skip("CANDIDATE_SUITE_DIR / build_samples not available") + except ImportError: + raise _SuiteUnavailable( + "SKIP: CANDIDATE_SUITE_DIR / build_samples not available" + ) + return bs + + +def test_md_generators_render_fictional_identity(): + """The four corrected generators render Robin Mercier; the input identity + does not survive into the output.""" + bs = _require_samples() + failures = [] + with tempfile.TemporaryDirectory() as td: + for tool, (key, fn_name) in MD_MAP.items(): + sample = bs._GENERATORS[key] + data = dict(sample["data"], labels=sample["labels"], language="en") + out = getattr(core, fn_name)(data, output_dir=td) + text = Path(out).read_text(encoding="utf-8") + if FICTIONAL not in text: + failures.append(f"{tool}: rendered output missing {FICTIONAL!r}") + if SAMPLE_NAME in text: + failures.append( + f"{tool}: caller-supplied {SAMPLE_NAME!r} leaked into output" + ) + assert not failures, "identity not forced in rendered output:\n " + "\n ".join( + failures + ) + + +def test_cover_letter_renders_fictional_identity(): + """The already-bound letter renders Robin Mercier as the signatory; the + caller-supplied sender identity does not survive.""" + bs = _require_samples() + from docx import Document + + data = dict(bs.lorem_letter_data(signed=False), language="en") + with tempfile.TemporaryDirectory() as td: + out = core.build_letter(data, output_dir=td) + text = "\n".join(p.text for p in Document(out).paragraphs) + assert FICTIONAL in text, f"cover_letter missing {FICTIONAL!r}" + assert SAMPLE_NAME not in text, ( + f"caller-supplied {SAMPLE_NAME!r} leaked into cover_letter" + ) + + +if __name__ == "__main__": + _tests = [ + test_md_generators_render_fictional_identity, + test_cover_letter_renders_fictional_identity, + ] + red = 0 + skipped = 0 + for _t in _tests: + try: + _t() + print(f"[PASS] {_t.__name__}") + except _SuiteUnavailable as exc: + skipped += 1 + print(f"[SKIP] {_t.__name__}: {exc}") + except AssertionError as exc: + red += 1 + print( + f"[FAIL] {_t.__name__}\n " + str(exc).replace("\n", "\n ") + ) + print(f"\n{red} failure(s), {skipped} skipped / {len(_tests)} observed e2e checks") + sys.exit(1 if red else 0) diff --git a/server/chain_core.py b/server/chain_core.py index 2cb0816..5819754 100644 --- a/server/chain_core.py +++ b/server/chain_core.py @@ -274,9 +274,22 @@ def dispatch_suite_run(argv): "sender_email": "robin.mercier@example.org", "sender_phone": "+33 6 12 34 56 78", "sender_linkedin": "linkedin.com/in/robin-mercier-fictif", + # candidate_name: the candidate's display identity, injected into the + # md-generators (playbook/summary/interview/refcard) by the helper below. + "candidate_name": "Robin Mercier", } +def _inject_candidate_identity(payload): + """Profile wins: overwrite the candidate identity field with the fixed + fictional value so the agent can never supply it -- the manifest promise + (fictional sample data) is enforced by code, not by a free tool parameter. + Single binding location: CANDIDATE_PROFILE (destined to become the + deployment MCP resource, like the sender_* fields).""" + payload["candidate_name"] = CANDIDATE_PROFILE["candidate_name"] + return payload + + # Pure tool logic (no SDK import here -> importable and testable without the CLI) # --------------------------------------------------------------------------- def read_offer_file(offer_path): @@ -815,6 +828,7 @@ def build_playbook(data, output_dir=None): ) payload = {f: data[f] for f in PLAYBOOK_MODEL_FIELDS if f in data} + _inject_candidate_identity(payload) labels = data.get("labels") if not isinstance(labels, dict): labels = {} @@ -948,6 +962,7 @@ def _run_md_generator( "(e.g. 'fr', 'en'); got: " + repr(language) ) payload = {f: data[f] for f in model_fields if f in data} + _inject_candidate_identity(payload) labels = data.get("labels") if not isinstance(labels, dict): labels = {} @@ -1152,7 +1167,6 @@ def letter_tool_description(): PLAYBOOK_SCHEMA = { "type": "object", "properties": { - "candidate_name": {"type": "string"}, "job_title": {"type": "string"}, "company_name": {"type": "string"}, "date": {"type": "string"}, @@ -1205,7 +1219,6 @@ def letter_tool_description(): "language": {"type": "string"}, }, "required": [ - "candidate_name", "job_title", "company_name", "date", @@ -1220,7 +1233,7 @@ def playbook_tool_description(): return ( "Create the strategic-playbook dossier (.md) through the real " "candidate-suite generator, for the candidate's OWN interview prep. " - "Required: candidate_name, job_title, company_name, date. Optional: " + "Required: job_title, company_name, date. Optional: " "company_context / org_landscape / thirty_second_pitch (strings); " "questions_to_ask / red_lines (lists of strings); pain_points (list of " "objects, each {title, analysis, your_angle}); tough_questions (list of " @@ -1237,7 +1250,6 @@ def playbook_tool_description(): SUMMARY_SCHEMA = { "type": "object", "properties": { - "candidate_name": {"type": "string"}, "job_title": {"type": "string"}, "company_name": {"type": "string"}, "date": {"type": "string"}, @@ -1288,7 +1300,6 @@ def playbook_tool_description(): "language": {"type": "string"}, }, "required": [ - "candidate_name", "job_title", "company_name", "date", @@ -1305,7 +1316,7 @@ def playbook_tool_description(): def summary_tool_description(): return ( "Create the application-summary dossier (.md) through the real " - "candidate-suite generator. Required: candidate_name, job_title, " + "candidate-suite generator. Required: job_title, " "company_name, date; pitch (list of EXACTLY 5 strings); strengths (list " "of {title, context}); weaknesses (list of {title, approach}); " "talking_points (list of {title, content}). Optional: opening_tip / " @@ -1319,7 +1330,6 @@ def summary_tool_description(): INTERVIEW_SCHEMA = { "type": "object", "properties": { - "candidate_name": {"type": "string"}, "job_title": {"type": "string"}, "company_name": {"type": "string"}, "date": {"type": "string"}, @@ -1352,7 +1362,6 @@ def summary_tool_description(): "language": {"type": "string"}, }, "required": [ - "candidate_name", "job_title", "company_name", "date", @@ -1367,7 +1376,7 @@ def summary_tool_description(): def interview_tool_description(): return ( "Create the interview-prep dossier (.md) through the real candidate-suite " - "generator. Required: candidate_name, job_title, company_name, date; " + "generator. Required: job_title, company_name, date; " "screening_questions and competence_questions (lists of objects, each " "{question, answer}). Optional: opening_tip_screening / " "opening_tip_competence / closing_tip (strings). labels is an object of " @@ -1381,7 +1390,6 @@ def interview_tool_description(): REFCARD_SCHEMA = { "type": "object", "properties": { - "candidate_name": {"type": "string"}, "job_title": {"type": "string"}, "company_name": {"type": "string"}, "date": {"type": "string"}, @@ -1416,7 +1424,6 @@ def interview_tool_description(): "language": {"type": "string"}, }, "required": [ - "candidate_name", "job_title", "company_name", "date", @@ -1430,7 +1437,7 @@ def refcard_tool_description(): return ( "Create the one-page quick-reference card (.md) through the real " "candidate-suite generator. It CONDENSES the other deliverables -- call " - "it AFTER producing them. Required: candidate_name, job_title, " + "it AFTER producing them. Required: job_title, " "company_name, date. Optional: pitch_short (string); key_stats (list of " "stat objects); top_points (list of {point, evidence}); quick_qa (list " "of {question, answer}); questions_to_ask / checklist (lists of "