From 573b344a1fc5e2a4a36f12cbcf36d93ec534662e Mon Sep 17 00:00:00 2001 From: Bill Lakenan Date: Thu, 6 Aug 2026 12:17:57 -0400 Subject: [PATCH 1/2] fix(mcs): send credentials to the measure engine and target the job's MCS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jobs against a remote MCS failed every patient with HTTP 401 "Authorization header missing Bearer token". Two independent breaks: 1. evaluate_measure() had no auth parameter at all — a bare client.get() with no headers. The orchestrator resolved the job's MCS *url* via _get_mcs_url() but there was no equivalent for credentials. 2. push_resources(), wipe_patient_data(), and resolve_evaluated_resource() all defaulted to settings.MEASURE_ENGINE_URL with no auth, so patient data went to the local engine while evaluation ran against the remote one. Fixing only the 401 would have produced clean 200s with every population at zero — a silent wrong answer. Adds _get_mcs_auth_headers(), mirroring _get_cdr_auth_headers(): reads credentials from the live MCSConfig via Job.mcs_id rather than duplicating secrets onto the job row, returns {} when no MCS is linked, and raises when the linked config was deleted instead of silently evaluating unauthenticated. No migration — Job.mcs_id already existed. MCS resolution moves above the wipe in run_job so the wipe targets the same engine the job pushes to and evaluates against; wiping a different server would leave the real target's prior-run data in place and contaminate populations. Verified against the CMS connectathon server: 401s 122 -> 0, all responses 200, and patient data now reaches the remote MCS (0 -> 56 Patients, 131 Encounters, 131 Conditions). Remaining failures there are server-side (HSEARCH800001, Hibernate Search not initialized), not client-side. Note: the wipe is now destructive against a shared remote MCS. Tracked separately in #392. Refs #391 (token refresh) --- backend/app/services/fhir_client.py | 41 +++-- backend/app/services/orchestrator.py | 54 ++++++- backend/tests/test_services_fhir_client.py | 70 +++++++- backend/tests/test_services_orchestrator.py | 167 +++++++++++++++++++- 4 files changed, 311 insertions(+), 21 deletions(-) diff --git a/backend/app/services/fhir_client.py b/backend/app/services/fhir_client.py index a4849483..bab1d5af 100644 --- a/backend/app/services/fhir_client.py +++ b/backend/app/services/fhir_client.py @@ -767,6 +767,7 @@ async def evaluate_measure( period_start: str, period_end: str, measure_engine_url: str | None = None, + auth_headers: dict[str, str] | None = None, ) -> dict[str, Any]: """Call $evaluate-measure on the measure engine for a single patient. @@ -776,6 +777,9 @@ async def evaluate_measure( haven't yet been wired to per-job active-MCS context. The orchestrator passes `job.mcs_url` so jobs run against the MCS that was active at job creation time, not whatever's active now. + auth_headers: Credentials for the MCS, resolved from the job's linked + MCSConfig. Omitted for unauthenticated engines (local HAPI). A + remote MCS rejects every evaluation with 401 without these. Raises FhirOperationError (with the MCS OperationOutcome preserved) on 4xx/5xx responses and on 200 OK where the body is an OperationOutcome @@ -796,7 +800,7 @@ async def evaluate_measure( extra={"measure_id": measure_id, "patient_id": patient_id, "attempt": attempt + 1}, ) start_ms = int(time.monotonic() * 1000) - resp = await client.get(url) + resp = await client.get(url, headers=auth_headers or {}) latency_ms = int(time.monotonic() * 1000) - start_ms try: resp.raise_for_status() @@ -918,7 +922,7 @@ async def delete_measure(measure_id: str) -> None: resp.raise_for_status() -async def wipe_patient_data(*, base_url: str, strict: bool = True) -> None: +async def wipe_patient_data(*, base_url: str, strict: bool = True, auth_headers: dict[str, str] | None = None) -> None: """Delete patient-related data from a FHIR server. Called at the START of a new job (with base_url=MEASURE_ENGINE_URL) to clean @@ -929,6 +933,10 @@ async def wipe_patient_data(*, base_url: str, strict: bool = True) -> None: A timed-out DELETE leaves HAPI's server-side operation still running; pushing new data over it causes the in-flight DELETE to wipe the freshly-pushed resources. The strict parameter is kept for API compatibility but no longer silences failures. + + `auth_headers` credentials the target server. A remote MCS rejects every DELETE + with 401 without them, which would leave the prior job's patients in place and + contaminate the next evaluation. """ # Delete clinical resources before Patient: HAPI returns 409 when Patient is # referenced by Condition/Encounter/etc., so Patient must be last. @@ -965,7 +973,7 @@ async def wipe_patient_data(*, base_url: str, strict: bool = True) -> None: try: # Use conditional delete: DELETE ResourceType?_lastUpdated=gt1900-01-01 delete_url = f"{base_url}/{rt}?_lastUpdated=gt1900-01-01" - resp = await client.delete(delete_url) + resp = await client.delete(delete_url, headers=auth_headers or {}) if resp.status_code < 300: logger.info("Wiped resource type", extra={"resourceType": rt}) else: @@ -1036,21 +1044,36 @@ async def _delete_all_of_type(client: httpx.AsyncClient, resource_type: str, bas break -async def resolve_evaluated_resource(reference: str) -> dict[str, Any]: - """Resolve an evaluatedResource reference from the measure engine.""" - url = f"{settings.MEASURE_ENGINE_URL}/{reference}" +async def resolve_evaluated_resource( + reference: str, + base_url: str | None = None, + auth_headers: dict[str, str] | None = None, +) -> dict[str, Any]: + """Resolve an evaluatedResource reference from the measure engine. + + `base_url`/`auth_headers` target the job's MCS. They default to the env-var + engine with no credentials, which only works for a local unauthenticated HAPI. + """ + url = f"{base_url or settings.MEASURE_ENGINE_URL}/{reference}" async with httpx.AsyncClient(timeout=30.0) as client: - resp = await client.get(url) + resp = await client.get(url, headers=auth_headers or {}) resp.raise_for_status() return resp.json() -async def snapshot_evaluated_resources(measure_report: dict[str, Any]) -> list[dict[str, Any]] | None: +async def snapshot_evaluated_resources( + measure_report: dict[str, Any], + base_url: str | None = None, + auth_headers: dict[str, str] | None = None, +) -> list[dict[str, Any]] | None: """Resolve every evaluatedResource reference in a MeasureReport to a stored snapshot. Returns a list of full FHIR resources. Per-reference failures are skipped and logged rather than raised — partial snapshots are still useful and the caller has already persisted the MeasureReport itself. Returns None when there is nothing to snapshot. + + `base_url`/`auth_headers` identify the MCS the report came from, so snapshots + are read back from the same server that evaluated them. """ refs = [ r.get("reference") @@ -1063,7 +1086,7 @@ async def snapshot_evaluated_resources(measure_report: dict[str, Any]) -> list[d resources: list[dict[str, Any]] = [] for ref in refs: try: - resources.append(await resolve_evaluated_resource(ref)) + resources.append(await resolve_evaluated_resource(ref, base_url, auth_headers)) except Exception as exc: logger.warning( "Failed to snapshot evaluated resource", diff --git a/backend/app/services/orchestrator.py b/backend/app/services/orchestrator.py index 68e34fbb..24df74b4 100644 --- a/backend/app/services/orchestrator.py +++ b/backend/app/services/orchestrator.py @@ -18,6 +18,7 @@ from app.db import async_session from app.models.config import CDRConfig from app.models.job import Batch, BatchStatus, Job, JobStatus, MeasureResult +from app.models.mcs_config import MCSConfig from app.services.fhir_client import ( BatchQueryStrategy, DataRequirementsStrategy, @@ -171,9 +172,16 @@ async def run_job(job_id: int) -> None: await session.commit() try: + # Resolve the MCS up front: the wipe below must target the same engine the + # job will push to and evaluate against, not the env-var default. Pointing + # the wipe at a different server would leave the real target's prior-run + # data in place and silently contaminate this job's populations. + mcs_url = await _get_mcs_url(job_id) + mcs_auth_headers = await _get_mcs_auth_headers(job_id) + # Step 1: Wipe patient data from measure engine (cleanup from prior job) logger.info("Wiping prior patient data from measure engine", extra={"job_id": job_id}) - await wipe_patient_data(base_url=settings.MEASURE_ENGINE_URL, strict=False) + await wipe_patient_data(base_url=mcs_url, strict=False, auth_headers=mcs_auth_headers) if await _stop_or_delete_job(job_id): return @@ -238,9 +246,8 @@ async def run_job(job_id: int) -> None: # Step 5: Process batches with concurrency control semaphore = asyncio.Semaphore(settings.MAX_WORKERS) - # Resolve MCS URL once for the whole job. Falls back to the env-var - # default if Job.mcs_url is NULL (legacy rows pre-dating PR #4). - mcs_url = await _get_mcs_url(job_id) + # mcs_url / mcs_auth_headers were resolved before the wipe above so every + # MCS interaction in this job targets one server with one set of credentials. async def process_batch(batch_id: int) -> None: async with semaphore: @@ -251,6 +258,7 @@ async def process_batch(batch_id: int) -> None: cdr_url=cdr_url, auth_headers=auth_headers, mcs_url=mcs_url, + mcs_auth_headers=mcs_auth_headers, ) # Check for cancellation before starting @@ -345,6 +353,28 @@ async def _get_cdr_url(job_id: int) -> str: return settings.DEFAULT_CDR_URL +async def _get_mcs_auth_headers(job_id: int) -> dict[str, str]: + """Resolve auth headers by reading live credentials from the referenced MCS config. + + Mirrors `_get_cdr_auth_headers`: the job snapshots `mcs_id`, and credentials are + read from the live config rather than duplicated onto the job row, so secrets + live in exactly one place. + + A job with no `mcs_id` predates MCS connections or targets the env-var engine — + unauthenticated, so no headers. A job whose linked config has been deleted has + unrecoverable credentials and raises rather than silently evaluating without + auth, which a remote MCS would reject with 401 on every patient. + """ + async with async_session() as session: + job = await session.get(Job, job_id) + if job is None or job.mcs_id is None: + return {} + cfg = await session.get(MCSConfig, job.mcs_id) + if cfg is None: + raise RuntimeError(f"MCS config {job.mcs_id} referenced by job {job_id} no longer exists.") + return await _build_auth_headers(cfg.auth_type, cfg.auth_credentials) + + async def _get_mcs_url(job_id: int) -> str: """Resolve the MCS URL for a job. @@ -370,6 +400,7 @@ async def _process_single_batch( cdr_url: str, auth_headers: dict[str, str], mcs_url: str, + mcs_auth_headers: dict[str, str] | None = None, ) -> None: """Process a single batch in two phases. @@ -431,7 +462,11 @@ async def _process_single_batch( try: gather_result = await strategy.gather_patient_data(cdr_url, patient_id, auth_headers) if gather_result.resources: - await push_resources(gather_result.resources) + await push_resources( + gather_result.resources, + target_url=mcs_url, + auth_headers=mcs_auth_headers, + ) logger.info( f"Pushed {len(gather_result.resources)} resources for {patient_id[:8]}", extra={"job_id": job_id, "patient_id": patient_id}, @@ -546,7 +581,12 @@ async def _process_single_batch( try: measure_report = await evaluate_measure( - measure_id, patient_id, period_start, period_end, measure_engine_url=mcs_url + measure_id, + patient_id, + period_start, + period_end, + measure_engine_url=mcs_url, + auth_headers=mcs_auth_headers, ) populations = _extract_populations(measure_report) @@ -564,7 +604,7 @@ async def _process_single_batch( # new rows without refs. evaluated_resources_snapshot: list[dict] | None = None try: - snapshot_result = await snapshot_evaluated_resources(measure_report) + snapshot_result = await snapshot_evaluated_resources(measure_report, mcs_url, mcs_auth_headers) evaluated_resources_snapshot = snapshot_result if snapshot_result is not None else [] except Exception as snap_exc: logger.warning( diff --git a/backend/tests/test_services_fhir_client.py b/backend/tests/test_services_fhir_client.py index c7654d8c..fd309469 100644 --- a/backend/tests/test_services_fhir_client.py +++ b/backend/tests/test_services_fhir_client.py @@ -1032,7 +1032,7 @@ async def test_snapshot_evaluated_resources_returns_resolved_list(): "Encounter/e1": {"resourceType": "Encounter", "id": "e1"}, } - async def fake_resolve(ref): + async def fake_resolve(ref, base_url=None, auth_headers=None): return fake_resources[ref] with patch("app.services.fhir_client.resolve_evaluated_resource", side_effect=fake_resolve): @@ -1054,7 +1054,7 @@ async def test_snapshot_evaluated_resources_skips_failed_refs(): ], } - async def fake_resolve(ref): + async def fake_resolve(ref, base_url=None, auth_headers=None): if ref == "Encounter/e1": raise RuntimeError("404 not found") return {"resourceType": ref.split("/")[0], "id": ref.split("/")[1]} @@ -1852,3 +1852,69 @@ def test_wait_for_valueset_expansion_logs_timeout(monkeypatch, caplog): assert expanded == {} assert "ValueSet expansion timed out" in caplog.text + + +# --------------------------------------------------------------------------- +# MCS auth wiring — evaluate_measure / resolve_evaluated_resource +# (regression: remote MCS connections got 401 because no auth was ever sent) +# --------------------------------------------------------------------------- + + +async def test_evaluate_measure_sends_auth_headers(mock_measure_report): + """evaluate_measure forwards auth headers to the MCS.""" + mock_response = _make_response(200, mock_measure_report) + + with patch("app.services.fhir_client.httpx.AsyncClient") as mock_httpx: + mock_ctx = AsyncMock() + mock_ctx.get = AsyncMock(return_value=mock_response) + mock_httpx.return_value.__aenter__ = AsyncMock(return_value=mock_ctx) + mock_httpx.return_value.__aexit__ = AsyncMock(return_value=False) + + await evaluate_measure( + "measure-1", + "patient-1", + "2024-01-01", + "2024-12-31", + measure_engine_url="https://mcs.example.org/fhir", + auth_headers={"Authorization": "Bearer tok-123"}, + ) + + sent = mock_ctx.get.call_args.kwargs.get("headers") + assert sent is not None, "evaluate_measure must pass headers to the MCS" + assert sent.get("Authorization") == "Bearer tok-123" + + +async def test_evaluate_measure_without_auth_sends_no_authorization(mock_measure_report): + """Unauthenticated MCS (local HAPI) keeps working — no Authorization header.""" + mock_response = _make_response(200, mock_measure_report) + + with patch("app.services.fhir_client.httpx.AsyncClient") as mock_httpx: + mock_ctx = AsyncMock() + mock_ctx.get = AsyncMock(return_value=mock_response) + mock_httpx.return_value.__aenter__ = AsyncMock(return_value=mock_ctx) + mock_httpx.return_value.__aexit__ = AsyncMock(return_value=False) + + await evaluate_measure("measure-1", "patient-1", "2024-01-01", "2024-12-31") + + sent = mock_ctx.get.call_args.kwargs.get("headers") or {} + assert "Authorization" not in sent + + +async def test_resolve_evaluated_resource_uses_given_base_and_auth(): + """Snapshot reads target the job's MCS with its credentials, not the env default.""" + mock_response = _make_response(200, {"resourceType": "Condition", "id": "c1"}) + + with patch("app.services.fhir_client.httpx.AsyncClient") as mock_httpx: + mock_ctx = AsyncMock() + mock_ctx.get = AsyncMock(return_value=mock_response) + mock_httpx.return_value.__aenter__ = AsyncMock(return_value=mock_ctx) + mock_httpx.return_value.__aexit__ = AsyncMock(return_value=False) + + await resolve_evaluated_resource( + "Condition/c1", + base_url="https://mcs.example.org/fhir", + auth_headers={"Authorization": "Bearer tok-123"}, + ) + + assert mock_ctx.get.call_args[0][0] == "https://mcs.example.org/fhir/Condition/c1" + assert mock_ctx.get.call_args.kwargs.get("headers", {}).get("Authorization") == "Bearer tok-123" diff --git a/backend/tests/test_services_orchestrator.py b/backend/tests/test_services_orchestrator.py index 33b97b2c..33f74005 100644 --- a/backend/tests/test_services_orchestrator.py +++ b/backend/tests/test_services_orchestrator.py @@ -7,13 +7,16 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings +from app.models.config import AuthType from app.models.job import Job, JobStatus, MeasureResult -from app.services.fhir_client import FailedResourceFetch, GatherResult +from app.models.mcs_config import MCSConfig +from app.services.fhir_client import BatchQueryStrategy, FailedResourceFetch, GatherResult from app.services.orchestrator import ( _error_measure_report, _extract_patient_name, _extract_populations, _get_cdr_auth_headers, + _get_mcs_auth_headers, run_job, ) @@ -237,7 +240,8 @@ async def test_run_job_happy_path(test_session, session_factory, mock_measure_re {"resourceType": "Condition", "id": "cond-1"}, ] - mock_wipe.assert_awaited_once_with(base_url=settings.MEASURE_ENGINE_URL, strict=False) + # Job has no mcs_url, so the wipe falls back to the env-var engine with no credentials. + mock_wipe.assert_awaited_once_with(base_url=settings.MEASURE_ENGINE_URL, strict=False, auth_headers={}) async def test_run_job_stores_empty_list_when_snapshot_helper_returns_none( @@ -421,7 +425,9 @@ async def test_run_job_partial_patient_failure(test_session, session_factory, mo {"resourceType": "Patient", "id": "p2", "name": [{"given": ["Bob"], "family": "Bad"}]}, ] - async def mock_evaluate(measure_id, patient_id, period_start, period_end, measure_engine_url=None): + async def mock_evaluate( + measure_id, patient_id, period_start, period_end, measure_engine_url=None, auth_headers=None + ): if patient_id == "p2": raise Exception("Evaluation failed for p2") return mock_measure_report @@ -1074,3 +1080,158 @@ async def test_run_job_sets_started_at_on_transition_to_running(test_session, se assert job.status == JobStatus.complete # started_at must be before or equal to completed_at assert job.started_at <= job.completed_at + + +# --------------------------------------------------------------------------- +# _get_mcs_auth_headers — MCS credentials must reach the measure engine. +# Regression: remote MCS jobs failed every patient with HTTP 401 +# "Authorization header missing Bearer token" because auth was never resolved. +# --------------------------------------------------------------------------- + + +async def test_get_mcs_auth_headers_builds_from_linked_config(test_session, session_factory): + """A bearer-authed MCS config yields an Authorization header for $evaluate-measure.""" + cfg = MCSConfig( + name="Remote MCS", + mcs_url="https://mcs.example.org/fhir", + auth_type=AuthType.bearer, + auth_credentials={"token": "tok-123"}, + ) + test_session.add(cfg) + await test_session.commit() + await test_session.refresh(cfg) + + job = Job( + measure_id="m-1", + period_start="2024-01-01", + period_end="2024-12-31", + cdr_url="http://cdr.example.com/fhir", + status=JobStatus.queued, + mcs_url=cfg.mcs_url, + mcs_id=cfg.id, + ) + test_session.add(job) + await test_session.commit() + await test_session.refresh(job) + + with ( + patch("app.services.orchestrator.async_session", session_factory), + patch( + "app.services.orchestrator._build_auth_headers", + new_callable=AsyncMock, + return_value={"Authorization": "Bearer tok-123"}, + ) as mock_auth, + ): + headers = await _get_mcs_auth_headers(job.id) + + assert headers == {"Authorization": "Bearer tok-123"} + assert mock_auth.call_args[0][0] == AuthType.bearer + + +async def test_get_mcs_auth_headers_empty_when_no_mcs_linked(test_session, session_factory): + """Local/legacy jobs with no mcs_id need no credentials.""" + job = Job( + measure_id="m-local", + period_start="2024-01-01", + period_end="2024-12-31", + cdr_url="http://cdr.example.com/fhir", + status=JobStatus.queued, + mcs_id=None, + ) + test_session.add(job) + await test_session.commit() + await test_session.refresh(job) + + with patch("app.services.orchestrator.async_session", session_factory): + assert await _get_mcs_auth_headers(job.id) == {} + + +async def test_get_mcs_auth_headers_raises_when_config_deleted(test_session, session_factory): + """Credentials are unrecoverable if the MCS config was deleted — fail loudly, not silently.""" + job = Job( + measure_id="m-orphan", + period_start="2024-01-01", + period_end="2024-12-31", + cdr_url="http://cdr.example.com/fhir", + status=JobStatus.queued, + mcs_url="https://mcs.example.org/fhir", + mcs_id=99999, # points at a config that does not exist + ) + test_session.add(job) + await test_session.commit() + await test_session.refresh(job) + + with patch("app.services.orchestrator.async_session", session_factory): + with pytest.raises(RuntimeError, match="no longer exists"): + await _get_mcs_auth_headers(job.id) + + +async def test_run_job_targets_job_mcs_with_credentials(test_session, session_factory, mock_measure_report): + """Wipe, push, and evaluate all target the job's MCS with its credentials. + + Regression: jobs against a remote MCS pushed patient data to the env-var + engine (so the remote never received it) and evaluated without auth (so every + patient 401'd). Both had to be true for a remote MCS job to produce results. + """ + cfg = MCSConfig( + name="Remote MCS", + mcs_url="https://mcs.example.org/fhir", + auth_type=AuthType.bearer, + auth_credentials={"token": "tok-123"}, + ) + test_session.add(cfg) + await test_session.commit() + await test_session.refresh(cfg) + + job = Job( + measure_id="m-1", + period_start="2024-01-01", + period_end="2024-12-31", + cdr_url="http://cdr.example.com/fhir", + status=JobStatus.queued, + mcs_url=cfg.mcs_url, + mcs_id=cfg.id, + ) + test_session.add(job) + await test_session.commit() + await test_session.refresh(job) + job_id = job.id + + patients = [{"resourceType": "Patient", "id": "p1", "name": [{"given": ["A"], "family": "B"}]}] + expected_auth = {"Authorization": "Bearer tok-123"} + + with ( + _make_session_factory_patch(session_factory), + patch("app.services.orchestrator.wipe_patient_data", new_callable=AsyncMock) as mock_wipe, + patch("app.services.orchestrator.push_resources", new_callable=AsyncMock) as mock_push, + patch("app.services.orchestrator._get_cdr_auth_headers", new_callable=AsyncMock, return_value={}), + patch( + "app.services.orchestrator._get_cdr_url", + new_callable=AsyncMock, + return_value="http://cdr.example.com/fhir", + ), + patch.object(BatchQueryStrategy, "gather_patients", new_callable=AsyncMock, return_value=patients), + patch.object( + BatchQueryStrategy, + "gather_patient_data", + new_callable=AsyncMock, + return_value=GatherResult(resources=[{"resourceType": "Patient", "id": "p1"}]), + ), + patch( + "app.services.orchestrator.evaluate_measure", + new_callable=AsyncMock, + return_value=mock_measure_report, + ) as mock_eval, + ): + await run_job(job_id) + + # Wipe cleans the MCS this job will actually use — not the env-var engine. + mock_wipe.assert_awaited_once_with( + base_url="https://mcs.example.org/fhir", strict=False, auth_headers=expected_auth + ) + # Patient data is pushed to that same MCS, authenticated. + assert mock_push.await_args.kwargs["target_url"] == "https://mcs.example.org/fhir" + assert mock_push.await_args.kwargs["auth_headers"] == expected_auth + # Evaluation carries the credentials that were missing in the 401 regression. + assert mock_eval.await_args.kwargs["measure_engine_url"] == "https://mcs.example.org/fhir" + assert mock_eval.await_args.kwargs["auth_headers"] == expected_auth From df53641bbc8c6fd8347c1389225c2543ddead977 Mon Sep 17 00:00:00 2001 From: Bill Lakenan Date: Thu, 6 Aug 2026 14:47:31 -0400 Subject: [PATCH 2/2] docs: add ADR-011 for MCS credential resolution and wipe targeting --- docs/decisions.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/decisions.md b/docs/decisions.md index 406bf93f..4596ad48 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -107,3 +107,21 @@ This log records significant technical and process choices with their rationale. **Defense-in-depth:** `_assert_no_canonical_url_clash()` queries `Measure?url={canonical}` before every `push_resources` call during bundle upload. If HAPI already has a Measure at that canonical URL under a different FHIR ID, the upload fails with a clear error message referencing this ADR. **Alternatives considered:** (a) Canonical-URL normalisation in the comparison endpoint (would mask drift rather than prevent it). (b) Dedup-by-canonical-URL in `push_resources` (silent PUT-overwrite masks the problem and loses the original resource). (c) No seed-file change — runtime normalisation only (would not fix the prebaked images; a rebake would still leave two resources). + +--- + +## ADR-011: MCS credentials are read from the live config, and every MCS interaction targets the job's MCS (2026-08-06) + +**Decision:** Add `_get_mcs_auth_headers(job_id)` resolving credentials from the live `MCSConfig` via `Job.mcs_id`, and route all four measure-engine interactions — wipe, push, `$evaluate-measure`, and evaluated-resource snapshot — at the job's MCS with those credentials. + +**Root cause:** MCS connections wired the *URL* through to jobs (`Job.mcs_url`, `_get_mcs_url()`) but never the *credentials*. `evaluate_measure()` had no auth parameter at all, and `push_resources()` / `wipe_patient_data()` / `resolve_evaluated_resource()` all defaulted to `settings.MEASURE_ENGINE_URL`. A job against a remote MCS therefore pushed patient data to the local engine and evaluated against the remote one without auth — every patient failed `HTTP 401 "Authorization header missing Bearer token"`. + +**Why credentials live on the config, not the job:** `Job` snapshots `mcs_url`/`mcs_name`/`mcs_id` so job rendering never depends on current config state. Credentials are deliberately excluded from that snapshot and read live, matching `_get_cdr_auth_headers()`. Duplicating secrets onto every job row would multiply the blast radius of a database disclosure and leave stale tokens scattered across history. The cost is that deleting an MCS config makes its jobs unrunnable — handled by raising a clear error rather than silently evaluating unauthenticated. + +**Why the wipe moved above MCS resolution in `run_job`:** the wipe must clear the same server the job will push to and evaluate against. Wiping a different server leaves the real target's prior-run patients in place, and those inflate the next evaluation's populations — a silent wrong answer rather than a visible failure. + +**Consequence — the wipe is now destructive against shared infrastructure.** Pointing the wipe at the job's MCS means Lenny deletes all patient data on a remote MCS at job start. Correct for a dedicated engine, dangerous for a shared connectathon server. Tracked in #392; a scoped wipe (delete only the IDs this job pushes) is the likely resolution. + +**Alternatives considered:** (a) Snapshot credentials onto `Job` — rejected, secret sprawl. (b) Add `mcs_auth_type` mirroring `cdr_auth_type` — unnecessary: `mcs_id` alone distinguishes "no MCS linked" from "config deleted", so no migration was needed. (c) Fix only the 401 and leave push targeting the local engine — rejected, would have produced clean `200`s with every population at zero. + +**Status:** Verified against the CMS connectathon server — 401s went 122 → 0, all responses 200, and patient data reached the remote MCS (0 → 56 Patients). Remaining failures there are server-side (`HSEARCH800001`, Hibernate Search not initialized), not client-side.