From 990e37b9aa82ad429314c438409c8b04184a485e Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:56:11 -0700 Subject: [PATCH 01/11] feat(ingest): A-2190 KDE coverage + FTL gate (contract v3) Bump INFLOW_CONTRACT_VERSION to 3 and expand REQUIRED_KDES_BY_CTE: - harvesting/cooling: immediate_subsequent_recipient; cooling also harvest_location_name - initial_packing: harvest_date, harvest_location_name, received_date, received_quantity; field_id/growing_area_name; cooling_* when cooled - transformation: input_lot_codes, input_quantities, input_units_of_measure, input_product_descriptions Webhook KDE validation skips Subpart S required-key gates when ftl_covered is explicitly false (or a non-FTL category is supplied); unknown/true still enforce. CSV templates, API sample JSON, and synthetic tests updated. SEL-549/590 holds unchanged. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- .../src/lib/api-samples/required-kdes.json | 66 +++++++-- services/ingestion/app/csv_templates.py | 40 +++++ services/ingestion/app/webhook_models.py | 15 +- .../app/webhook_router_v2/validation.py | 80 +++++++++- .../ingestion/tests/test_a2190_kde_cte_ftl.py | 140 ++++++++++++++++++ .../tests/test_compliance_score_2960_2962.py | 16 +- .../test_harvester_phone_contract_3041.py | 9 +- .../tests/test_inflow_lab_contract.py | 6 + 8 files changed, 348 insertions(+), 24 deletions(-) create mode 100644 services/ingestion/tests/test_a2190_kde_cte_ftl.py diff --git a/frontend/src/lib/api-samples/required-kdes.json b/frontend/src/lib/api-samples/required-kdes.json index ba80f5e61..2c85b9f22 100644 --- a/frontend/src/lib/api-samples/required-kdes.json +++ b/frontend/src/lib/api-samples/required-kdes.json @@ -1,23 +1,60 @@ { "harvesting": { - "top_level": ["location_name"], - "kdes": ["harvest_date", "reference_document"] + "top_level": [ + "location_name" + ], + "kdes": [ + "harvest_date", + "immediate_subsequent_recipient", + "reference_document" + ] }, "cooling": { - "top_level": ["location_name"], - "kdes": ["cooling_date", "reference_document"] + "top_level": [ + "location_name" + ], + "kdes": [ + "cooling_date", + "immediate_subsequent_recipient", + "harvest_location_name", + "reference_document" + ] }, "initial_packing": { - "top_level": ["location_name"], - "kdes": ["packing_date", "reference_document", "harvester_business_name", "harvester_business_phone"] + "top_level": [ + "location_name" + ], + "kdes": [ + "packing_date", + "reference_document", + "harvester_business_name", + "harvester_business_phone", + "harvest_date", + "harvest_location_name", + "received_date", + "received_quantity", + "growing_area_name_or_field_id", + "cooling_location_name_when_cooled", + "cooling_date_when_cooled" + ] }, "first_land_based_receiving": { "top_level": [], - "kdes": ["landing_date", "receiving_location", "reference_document"] + "kdes": [ + "landing_date", + "receiving_location", + "reference_document" + ] }, "shipping": { "top_level": [], - "kdes": ["ship_date", "ship_from_location", "ship_to_location", "reference_document", "tlc_source_reference"] + "kdes": [ + "ship_date", + "ship_from_location", + "ship_to_location", + "reference_document", + "tlc_source_reference" + ] }, "receiving": { "top_level": [], @@ -30,7 +67,16 @@ ] }, "transformation": { - "top_level": ["location_name"], - "kdes": ["transformation_date", "reference_document"] + "top_level": [ + "location_name" + ], + "kdes": [ + "transformation_date", + "reference_document", + "input_lot_codes", + "input_quantities", + "input_units_of_measure", + "input_product_descriptions" + ] } } diff --git a/services/ingestion/app/csv_templates.py b/services/ingestion/app/csv_templates.py index 1cd96138b..b33daad01 100644 --- a/services/ingestion/app/csv_templates.py +++ b/services/ingestion/app/csv_templates.py @@ -86,6 +86,11 @@ ("growing_coordinates", "36.9107,-121.7569", "GPS coordinates of growing area (optional)"), ("grower_name", "Sunrise Organic Farm LLC", "Business name of grower"), ("harvest_date", "2026-04-01", "Date of harvest (YYYY-MM-DD) — growing rows ingest as harvesting"), + ( + "immediate_subsequent_recipient", + "Sunrise Packhouse", + "Immediate subsequent recipient (who receives the harvested food next)", + ), ("reference_document", "HARV-2026-0401-001", "Harvest record or reference document number"), ], "harvesting": [ @@ -99,6 +104,11 @@ ("location_gln", "0614141000005", "GS1 GLN (optional, 13 digits)"), ("field_id", "FIELD-A7", "Field or growing area identifier"), ("harvester_name", "Valley Fresh Farms LLC", "Business name of harvester"), + ( + "immediate_subsequent_recipient", + "Valley Fresh Packhouse", + "Immediate subsequent recipient (who receives the harvested food next)", + ), ("reference_document", "HARV-2026-0226-001", "Harvest record or reference document number"), ], "cooling": [ @@ -111,6 +121,16 @@ ("location_name", "Valley Fresh Cooler #2", "Cooling facility name"), ("location_gln", "0614141000005", "GS1 GLN (optional)"), ("temperature_celsius", "2.1", "Temperature at cooling (optional but recommended)"), + ( + "immediate_subsequent_recipient", + "Valley Fresh Packhouse", + "Immediate subsequent recipient (who receives the cooled food next)", + ), + ( + "harvest_location_name", + "Valley Fresh Farms, Salinas CA", + "Farm location where the food was harvested", + ), ("reference_document", "COOL-2026-0226-014", "Cooling record or reference document number"), ], "initial_packing": [ @@ -132,6 +152,21 @@ ("reference_document", "PACK-2026-0226-007", "Packing record or reference document number"), ("harvester_business_name", "Valley Fresh Farms LLC", "Business name of the harvester"), ("harvester_business_phone", "+1-202-555-0100", "Phone number of the harvester; sample number only"), + ("harvest_date", "2026-02-26", "Date the food was harvested (YYYY-MM-DD)"), + ( + "harvest_location_name", + "Valley Fresh Farms, Salinas CA", + "Farm location where the food was harvested", + ), + ("field_id", "FIELD-A7", "Field or growing area identifier (or use growing_area_name)"), + ("received_date", "2026-02-26", "Date the packer received the food (YYYY-MM-DD)"), + ("received_quantity", "500", "Quantity received (distinct from packed quantity)"), + ( + "cooling_location_name", + "Valley Fresh Cooler #2", + "Cooling location (required when the food was cooled)", + ), + ("cooling_date", "2026-02-26", "Date of cooling (required when the food was cooled)"), ], "shipping": [ ("traceability_lot_code", "TOM-0226-F3-001", "Lot code being shipped"), @@ -191,6 +226,11 @@ "cases,cases", "Unit for each input lot, comma-separated (one value applies to all)", ), + ( + "input_product_descriptions", + "Roma Tomatoes,Romaine Lettuce", + "Product description for EACH input lot, comma-separated in input_lot_codes order", + ), ("reference_document", "PROD-2026-0228-003", "Production record or reference document number"), ], "first_land_based_receiving": [ diff --git a/services/ingestion/app/webhook_models.py b/services/ingestion/app/webhook_models.py index 5abf96ed0..a66a30140 100644 --- a/services/ingestion/app/webhook_models.py +++ b/services/ingestion/app/webhook_models.py @@ -93,7 +93,7 @@ class WebhookCTEType(str, Enum): # validation semantics. Exposed via /health as inflow_contract_version so # deployed instances can detect skew; the Inflow Lab Contract CI asserts # cross-repo equality. -INFLOW_CONTRACT_VERSION = "2" +INFLOW_CONTRACT_VERSION = "3" # A-2190: harvest/cool recipient, packing carry-in, transform inputs, FTL gate # Required KDEs per CTE type (§1.1325–§1.1350) @@ -105,6 +105,7 @@ class WebhookCTEType(str, Enum): "unit_of_measure", "harvest_date", "location_name", + "immediate_subsequent_recipient", # §1.1325(a)(1); contract v3 / A-2190 "reference_document", # §1.1325(a)(1)(viii) ], WebhookCTEType.COOLING: [ @@ -114,6 +115,8 @@ class WebhookCTEType(str, Enum): "unit_of_measure", "cooling_date", "location_name", + "immediate_subsequent_recipient", # §1.1325(b)(1); contract v3 / A-2190 + "harvest_location_name", # farm where harvested; contract v3 / A-2190 "reference_document", # §1.1325(b)(1)(vii) ], WebhookCTEType.INITIAL_PACKING: [ @@ -126,6 +129,12 @@ class WebhookCTEType(str, Enum): "reference_document", # §1.1330(a)(16) "harvester_business_name", # §1.1330(a)(7) "harvester_business_phone", # §1.1330(a)(7); contract v2 + "harvest_date", # §1.1330(a); contract v3 / A-2190 + "harvest_location_name", # farm where harvested; contract v3 / A-2190 + "received_date", # date food received; contract v3 / A-2190 + "received_quantity", # qty received (distinct from packed); contract v3 / A-2190 + # cooling_location_name + cooling_date required when cooled — see validation.py + # growing_area_name OR field_id required for produce — see validation.py ], WebhookCTEType.FIRST_LAND_BASED_RECEIVING: [ "traceability_lot_code", @@ -166,6 +175,10 @@ class WebhookCTEType(str, Enum): "transformation_date", "location_name", "reference_document", # §1.1350(a)(2)(vi) + "input_lot_codes", # §1.1350(a)(1); contract v3 / A-2190 + "input_quantities", # §1.1350(a)(1)(iii); contract v3 / A-2190 + "input_units_of_measure", # §1.1350(a)(1)(iii); contract v3 / A-2190 + "input_product_descriptions", # §1.1350(a)(1); contract v3 / A-2190 ], } diff --git a/services/ingestion/app/webhook_router_v2/validation.py b/services/ingestion/app/webhook_router_v2/validation.py index b94c0c220..6a9b10f87 100644 --- a/services/ingestion/app/webhook_router_v2/validation.py +++ b/services/ingestion/app/webhook_router_v2/validation.py @@ -50,20 +50,94 @@ def _authoritative_kdes(event: IngestEvent) -> dict[str, object]: return available +def _kde_present(val: object) -> bool: + """True when a KDE value is non-empty (string, list, or other scalar).""" + if val is None: + return False + if isinstance(val, str): + return val.strip() != "" + if isinstance(val, (list, tuple, set)): + return any(_kde_present(item) for item in val) + if isinstance(val, dict): + return any(_kde_present(item) for item in val.values()) + return True + + +def _event_ftl_payload(event: IngestEvent) -> dict: + """Build the dict shape ``is_ftl_food`` expects from an IngestEvent.""" + payload: dict = {"kdes": dict(event.kdes or {})} + # Top-level FTL hints (rare) still count if present on the event object. + for key in ("ftl_covered", "ftl_category"): + if hasattr(event, key): + val = getattr(event, key) + if val is not None: + payload[key] = val + return payload + + def _validate_event_kdes(event: IngestEvent) -> list[str]: - """Validate that an event has all required KDEs for its CTE type.""" + """Validate that an event has all required KDEs for its CTE type. + + A-2190 / contract v3: Subpart S required-KDE gates apply when the product + is FTL or FTL status is unknown. Explicit ``ftl_covered=False`` (or a + non-FTL category) skips the required-KDE list so non-FTL foods are not + scored as gaps. Keep A-2170 location_incomplete behaviour untouched. + """ errors: list[str] = [] - required = _pkg.REQUIRED_KDES_BY_CTE.get(event.cte_type, []) + # FTL gate (fix 5). Import locally so monkeypatches on shared.rules.ftl + # and missing optional deps in unit tests stay isolated. + try: + from shared.rules.ftl import is_ftl_food + except ImportError: # pragma: no cover - shared always present in service + is_ftl_food = None # type: ignore[assignment] + + if is_ftl_food is not None: + ftl_status = is_ftl_food(_event_ftl_payload(event)) + if ftl_status is False: + return [] + + required = _pkg.REQUIRED_KDES_BY_CTE.get(event.cte_type, []) available = _authoritative_kdes(event) for kde_name in required: val = available.get(kde_name) if kde_name == "harvester_business_phone" and not isinstance(val, str): errors.append("Harvester business phone must be a non-empty string") - elif val is None or (isinstance(val, str) and val.strip() == ""): + elif not _kde_present(val): errors.append(f"Missing required KDE '{kde_name}' for {event.cte_type.value} CTE") + # Initial packing conditionals (A-2190 fix 2): field/growing area, and + # cooling location+date only when the food was cooled. + cte_value = getattr(event.cte_type, "value", event.cte_type) + if cte_value == "initial_packing": + has_field = _kde_present(available.get("growing_area_name")) or _kde_present( + available.get("field_id") + ) + if not has_field: + errors.append( + "Missing required KDE 'growing_area_name' or 'field_id' for initial_packing CTE" + ) + + cooled_flag = available.get("cooled") + cooled = cooled_flag is True or ( + isinstance(cooled_flag, str) and cooled_flag.strip().lower() in {"1", "true", "yes"} + ) + cooling_hint = _kde_present(available.get("cooling_date")) or _kde_present( + available.get("cooling_location_name") + ) + if cooled or cooling_hint: + if not _kde_present(available.get("cooling_location_name")): + errors.append( + "Missing required KDE 'cooling_location_name' for initial_packing CTE " + "(required when cooled)" + ) + if not _kde_present(available.get("cooling_date")): + errors.append( + "Missing required KDE 'cooling_date' for initial_packing CTE " + "(required when cooled)" + ) + return errors diff --git a/services/ingestion/tests/test_a2190_kde_cte_ftl.py b/services/ingestion/tests/test_a2190_kde_cte_ftl.py new file mode 100644 index 000000000..b5daec5f2 --- /dev/null +++ b/services/ingestion/tests/test_a2190_kde_cte_ftl.py @@ -0,0 +1,140 @@ +"""A-2190: FSMA 204 KDE coverage fixes (contract v3) + FTL ingest gate. + +Synthetic fixtures only. SEL-549/590 holds unchanged. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from services.ingestion.app.webhook_models import INFLOW_CONTRACT_VERSION, REQUIRED_KDES_BY_CTE, WebhookCTEType +from services.ingestion.app.webhook_router_v2.validation import _validate_event_kdes + + +def _event(cte: WebhookCTEType, kdes: dict, **top) -> SimpleNamespace: + base = { + "cte_type": cte, + "traceability_lot_code": "TLC-A2190-001", + "product_description": "Roma Tomatoes", + "quantity": 10, + "unit_of_measure": "cases", + "location_name": "Valley Fresh Farms", + "location_gln": None, + "kdes": kdes, + } + base.update(top) + return SimpleNamespace(**base) + + +def _fill_required(cte: WebhookCTEType, **overrides) -> dict: + kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[cte]} + kdes.update(overrides) + return kdes + + +def test_contract_version_is_v3(): + assert str(INFLOW_CONTRACT_VERSION) == "3" + + +def test_harvesting_requires_immediate_subsequent_recipient(): + kdes = _fill_required(WebhookCTEType.HARVESTING) + kdes.pop("immediate_subsequent_recipient") + errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) + assert any("immediate_subsequent_recipient" in e for e in errors) + + +def test_cooling_requires_recipient_and_harvest_location(): + kdes = _fill_required(WebhookCTEType.COOLING) + kdes.pop("immediate_subsequent_recipient") + kdes.pop("harvest_location_name") + errors = _validate_event_kdes(_event(WebhookCTEType.COOLING, kdes)) + assert any("immediate_subsequent_recipient" in e for e in errors) + assert any("harvest_location_name" in e for e in errors) + + +def test_initial_packing_requires_harvest_and_received_kdes(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING, field_id="FIELD-A7") + for key in ("harvest_date", "harvest_location_name", "received_date", "received_quantity"): + kdes.pop(key) + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + for key in ("harvest_date", "harvest_location_name", "received_date", "received_quantity"): + assert any(key in e for e in errors), errors + + +def test_initial_packing_requires_field_or_growing_area(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING) + # no field_id / growing_area_name + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + assert any("growing_area_name" in e or "field_id" in e for e in errors) + + +def test_initial_packing_cooling_required_when_cooled(): + kdes = _fill_required( + WebhookCTEType.INITIAL_PACKING, + field_id="FIELD-A7", + cooled=True, + ) + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + assert any("cooling_location_name" in e for e in errors) + assert any("cooling_date" in e for e in errors) + + kdes["cooling_location_name"] = "Cooler #2" + kdes["cooling_date"] = "2026-02-26" + assert _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) == [] + + +def test_initial_packing_cooling_optional_when_not_cooled(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING, field_id="FIELD-A7") + assert _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) == [] + + +def test_transformation_requires_input_lots_qty_desc(): + kdes = _fill_required(WebhookCTEType.TRANSFORMATION) + for key in ( + "input_lot_codes", + "input_quantities", + "input_units_of_measure", + "input_product_descriptions", + ): + kdes.pop(key) + errors = _validate_event_kdes(_event(WebhookCTEType.TRANSFORMATION, kdes)) + for key in ( + "input_lot_codes", + "input_quantities", + "input_units_of_measure", + "input_product_descriptions", + ): + assert any(key in e for e in errors), errors + + +def test_transformation_accepts_list_valued_input_lots(): + kdes = _fill_required( + WebhookCTEType.TRANSFORMATION, + input_lot_codes=["LOT-A", "LOT-B"], + input_quantities=["10", "5"], + input_units_of_measure=["cases", "cases"], + input_product_descriptions=["Tomatoes", "Lettuce"], + ) + assert _validate_event_kdes(_event(WebhookCTEType.TRANSFORMATION, kdes)) == [] + + +def test_ftl_false_skips_required_kde_gate(): + # Missing everything that would fail — but explicit non-FTL skips the gate. + kdes = {"ftl_covered": False} + assert _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) == [] + + +def test_ftl_unknown_still_enforces(): + kdes = _fill_required(WebhookCTEType.HARVESTING) + kdes.pop("immediate_subsequent_recipient") + errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) + assert any("immediate_subsequent_recipient" in e for e in errors) + + +def test_ftl_true_enforces(): + kdes = _fill_required(WebhookCTEType.HARVESTING, ftl_covered=True) + kdes.pop("reference_document") + errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) + assert any("reference_document" in e for e in errors) diff --git a/services/ingestion/tests/test_compliance_score_2960_2962.py b/services/ingestion/tests/test_compliance_score_2960_2962.py index 70d5ff257..6e8ad3a1c 100644 --- a/services/ingestion/tests/test_compliance_score_2960_2962.py +++ b/services/ingestion/tests/test_compliance_score_2960_2962.py @@ -141,12 +141,12 @@ def test_chain_integrity_dimension_still_reports_tamper(self): class TestRequiredKdeCoverageDenominator: def test_denominator_is_per_cte_type_not_flat(self): - """Harvesting requires 7 KDEs, shipping 9. A single flat required + """Harvesting requires 8 KDEs (contract v3), shipping 9. A single flat required list for every CTE type would be a different bug.""" harvesting = cs._required_kde_coverage({"harvesting": 1}, {}) shipping = cs._required_kde_coverage({"shipping": 1}, {}) - assert harvesting["required_kde_checks_total"] == len(REQUIRED_KDES_BY_CTE[WebhookCTEType.HARVESTING]) == 7 + assert harvesting["required_kde_checks_total"] == len(REQUIRED_KDES_BY_CTE[WebhookCTEType.HARVESTING]) == 8 assert shipping["required_kde_checks_total"] == len(REQUIRED_KDES_BY_CTE[WebhookCTEType.SHIPPING]) == 9 def test_two_of_seven_required_present(self): @@ -154,13 +154,13 @@ def test_two_of_seven_required_present(self): {"harvesting": 1}, {("harvesting", "traceability_lot_code"): 1, ("harvesting", "quantity"): 1}, ) - assert cov["required_kde_checks_total"] == 7 + assert cov["required_kde_checks_total"] == 8 assert cov["required_kde_checks_passed"] == 2 assert cov["events_unchecked_type"] == 0 def test_denominator_scales_with_event_count(self): cov = cs._required_kde_coverage({"harvesting": 4}, {("harvesting", "quantity"): 3}) - assert cov["required_kde_checks_total"] == 28 # 4 events x 7 required + assert cov["required_kde_checks_total"] == 32 # 4 events x 8 required assert cov["required_kde_checks_passed"] == 3 def test_unknown_cte_type_is_unchecked_not_fully_covered(self): @@ -179,13 +179,13 @@ def test_enum_member_without_checklist_does_not_raise(self): def test_event_type_matching_is_case_insensitive(self): cov = cs._required_kde_coverage({"HARVESTING": 1}, {("HARVESTING", "quantity"): 1}) - assert cov["required_kde_checks_total"] == 7 + assert cov["required_kde_checks_total"] == 8 assert cov["required_kde_checks_passed"] == 1 class TestKdeCompletenessScoredAgainstRequiredSet: def test_partial_required_set_scores_materially_below_100(self): - """A shipping/harvesting tenant recording 2 of 7 required KDEs must + """A shipping/harvesting tenant recording 2 of 8 required KDEs must not read 100%, and the export KDE credit must be < 30.""" scores = _compute_scores( _data( @@ -302,7 +302,7 @@ def _base_routes(**extra): class TestQueryScoringDataRequiredKdes: def test_query_derives_required_totals_from_the_checklist(self, monkeypatch): - """One harvesting event with 2 of its 7 required KDEs present.""" + """One harvesting event with 2 of its 8 required KDEs present.""" _fake_chain(monkeypatch) session = _RouteSession(route=_base_routes()) # One harvesting event; traceability_lot_code + quantity present. @@ -316,7 +316,7 @@ def test_query_derives_required_totals_from_the_checklist(self, monkeypatch): data = _query_scoring_data(session, "t1") - assert data["required_kde_checks_total"] == 7 + assert data["required_kde_checks_total"] == 8 assert data["required_kde_checks_passed"] == 2 assert data.get("required_kde_data_error") is not True diff --git a/services/ingestion/tests/test_harvester_phone_contract_3041.py b/services/ingestion/tests/test_harvester_phone_contract_3041.py index 046f6c241..4a53137c9 100644 --- a/services/ingestion/tests/test_harvester_phone_contract_3041.py +++ b/services/ingestion/tests/test_harvester_phone_contract_3041.py @@ -11,7 +11,11 @@ @pytest.mark.parametrize("phone", [None, "", " ", 123, {"phone": "555"}]) def test_initial_packing_requires_explicit_string_phone(phone): kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[WebhookCTEType.INITIAL_PACKING]} - kdes.update(harvester_business_phone=phone, from_entity_reference="entity-1") + kdes.update( + harvester_business_phone=phone, + from_entity_reference="entity-1", + field_id="FIELD-A7", + ) event = SimpleNamespace( cte_type=WebhookCTEType.INITIAL_PACKING, traceability_lot_code="L1", @@ -28,6 +32,7 @@ def test_initial_packing_requires_explicit_string_phone(phone): def test_v2_accepts_supplied_phone(): kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[WebhookCTEType.INITIAL_PACKING]} kdes["harvester_business_phone"] = "+1-202-555-0100" + kdes["field_id"] = "FIELD-A7" event = SimpleNamespace( cte_type=WebhookCTEType.INITIAL_PACKING, traceability_lot_code="L1", @@ -38,5 +43,5 @@ def test_v2_accepts_supplied_phone(): location_gln=None, kdes=kdes, ) - assert str(INFLOW_CONTRACT_VERSION) == "2" + assert str(INFLOW_CONTRACT_VERSION) == "3" assert _validate_event_kdes(event) == [] diff --git a/services/ingestion/tests/test_inflow_lab_contract.py b/services/ingestion/tests/test_inflow_lab_contract.py index 0a253dccf..0d9aa6785 100644 --- a/services/ingestion/tests/test_inflow_lab_contract.py +++ b/services/ingestion/tests/test_inflow_lab_contract.py @@ -33,6 +33,7 @@ def _event(cte_type: str, kdes: dict) -> IngestEvent: "harvest_date": "2026-04-22", "farm_location": "Valley Fresh Farms", "location_name": "Valley Fresh Farms", + "immediate_subsequent_recipient": "FreshPack Central", "reference_document": "Harvest Log HAR-20260422-00001", "reference_document_type": "Harvest Log", "reference_document_number": "HAR-20260422-00001", @@ -48,6 +49,11 @@ def _event(cte_type: str, kdes: dict) -> IngestEvent: "location_name": "FreshPack Central", "source_traceability_lot_code": "TLC-LAB-HARVESTING", "farm_location": "Valley Fresh Farms", + "harvest_date": "2026-04-22", + "harvest_location_name": "Valley Fresh Farms", + "field_id": "FIELD-A7", + "received_date": "2026-04-22", + "received_quantity": "24", "reference_document": "Packout Record PACK-20260422-00001", "reference_document_type": "Packout Record", "reference_document_number": "PACK-20260422-00001", From 9753cd95511ffbde46df456518fdc916e5f8dc7f Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:16:29 -0700 Subject: [PATCH 02/11] fix(ingest): Food Safety Lens packing/FTL gates + pin Inflow Lab v3 - Packing: require farm name+phone+address; human-readable growing/field name; explicit cooled yes/no (cooling_* only when cooled=yes). - FTL gate kept; synthetic tests updated. - Alias Lab legacy input_traceability_lot_codes into input_lot_codes for presence checks. - Pin INFLOW_LAB_REF to inflow-lab#258 (contract v3) so Simulator live ingest matches INFLOW_CONTRACT_VERSION 3. Lab PR must land first. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- .github/workflows/inflow-contract-ci.yml | 2 +- .../src/lib/api-samples/required-kdes.json | 9 +- services/ingestion/app/csv_templates.py | 32 ++++- services/ingestion/app/webhook_models.py | 6 +- .../app/webhook_router_v2/validation.py | 131 +++++++++++++++--- .../ingestion/tests/test_a2190_kde_cte_ftl.py | 79 +++++++++-- .../test_harvester_phone_contract_3041.py | 22 ++- .../tests/test_inflow_lab_contract.py | 4 + 8 files changed, 242 insertions(+), 43 deletions(-) diff --git a/.github/workflows/inflow-contract-ci.yml b/.github/workflows/inflow-contract-ci.yml index ce75619f2..88d6b9638 100644 --- a/.github/workflows/inflow-contract-ci.yml +++ b/.github/workflows/inflow-contract-ci.yml @@ -67,7 +67,7 @@ env: # Both sides advertise INFLOW_CONTRACT_VERSION "2". # Bumped to inflow-lab main 919b5282 (SEL-335) after the Contract Pin Drift # check found the pin 18 commits behind; contract version is still 2. - INFLOW_LAB_REF: 919b5282852313479c8eb7fbacf4d79d55bed909 # pragma: allowlist secret + INFLOW_LAB_REF: 22467473744963e75c6e460b2a7b7075aedda125 # pragma: allowlist secret CONTRACT_TENANT_ID: 11111111-1111-1111-1111-111111111111 CONTRACT_API_KEY: ci-inflow-contract-key # pragma: allowlist secret CONTRACT_HMAC_SECRET: ci-inflow-contract-hmac-secret # pragma: allowlist secret diff --git a/frontend/src/lib/api-samples/required-kdes.json b/frontend/src/lib/api-samples/required-kdes.json index 2c85b9f22..d03393685 100644 --- a/frontend/src/lib/api-samples/required-kdes.json +++ b/frontend/src/lib/api-samples/required-kdes.json @@ -33,9 +33,12 @@ "harvest_location_name", "received_date", "received_quantity", - "growing_area_name_or_field_id", - "cooling_location_name_when_cooled", - "cooling_date_when_cooled" + "harvest_location_phone", + "harvest_location_address", + "growing_area_name_or_field_name", + "cooled", + "cooling_location_name_when_cooled_yes", + "cooling_date_when_cooled_yes" ] }, "first_land_based_receiving": { diff --git a/services/ingestion/app/csv_templates.py b/services/ingestion/app/csv_templates.py index b33daad01..a2f17169e 100644 --- a/services/ingestion/app/csv_templates.py +++ b/services/ingestion/app/csv_templates.py @@ -156,17 +156,41 @@ ( "harvest_location_name", "Valley Fresh Farms, Salinas CA", - "Farm location where the food was harvested", + "Farm name where the food was harvested (location description)", + ), + ( + "harvest_location_phone", + "+1-831-555-0147", + "Farm business phone for the harvest location description", + ), + ( + "harvest_location_address", + "100 Farm Rd, Salinas CA 93901", + "Farm physical address for the harvest location description", + ), + ( + "growing_area_name", + "North Field Block A", + "Human-readable growing area or field name (required; field_id alone is not enough)", + ), + ( + "field_id", + "FIELD-A7", + "Optional internal field id; only sufficient when paired with growing_area_name / field_name", ), - ("field_id", "FIELD-A7", "Field or growing area identifier (or use growing_area_name)"), ("received_date", "2026-02-26", "Date the packer received the food (YYYY-MM-DD)"), ("received_quantity", "500", "Quantity received (distinct from packed quantity)"), + ( + "cooled", + "yes", + "Explicit yes/no: was the food cooled before packing? Required for RAC initial packing", + ), ( "cooling_location_name", "Valley Fresh Cooler #2", - "Cooling location (required when the food was cooled)", + "Cooling location (required when cooled=yes)", ), - ("cooling_date", "2026-02-26", "Date of cooling (required when the food was cooled)"), + ("cooling_date", "2026-02-26", "Date of cooling (required when cooled=yes)"), ], "shipping": [ ("traceability_lot_code", "TOM-0226-F3-001", "Lot code being shipped"), diff --git a/services/ingestion/app/webhook_models.py b/services/ingestion/app/webhook_models.py index a66a30140..08017e288 100644 --- a/services/ingestion/app/webhook_models.py +++ b/services/ingestion/app/webhook_models.py @@ -133,8 +133,10 @@ class WebhookCTEType(str, Enum): "harvest_location_name", # farm where harvested; contract v3 / A-2190 "received_date", # date food received; contract v3 / A-2190 "received_quantity", # qty received (distinct from packed); contract v3 / A-2190 - # cooling_location_name + cooling_date required when cooled — see validation.py - # growing_area_name OR field_id required for produce — see validation.py + # Food Safety Lens (A-2190): see validation.py for — + # harvest_location_phone + harvest_location_address (farm description) + # explicit cooled yes/no; cooling_* only when cooled=yes + # human-readable growing_area_name / field_name (field_id alone insufficient) ], WebhookCTEType.FIRST_LAND_BASED_RECEIVING: [ "traceability_lot_code", diff --git a/services/ingestion/app/webhook_router_v2/validation.py b/services/ingestion/app/webhook_router_v2/validation.py index 6a9b10f87..8110c663d 100644 --- a/services/ingestion/app/webhook_router_v2/validation.py +++ b/services/ingestion/app/webhook_router_v2/validation.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +import re import sys from sqlalchemy.exc import SQLAlchemyError @@ -47,6 +48,23 @@ def _authoritative_kdes(event: IngestEvent) -> dict[str, object]: if isinstance(_val, str) and _val.strip() == "": continue available[_key] = _val + # Contract v3 wire name is input_lot_codes; accept Lab's legacy + # input_traceability_lot_codes (kdes or top-level) as a fill-in. + def _present(val: object) -> bool: + if val is None: + return False + if isinstance(val, str): + return val.strip() != "" + if isinstance(val, (list, tuple, set)): + return len(val) > 0 + return True + + if not _present(available.get("input_lot_codes")): + legacy = available.get("input_traceability_lot_codes") + if not _present(legacy): + legacy = event.input_traceability_lot_codes + if _present(legacy): + available["input_lot_codes"] = legacy return available @@ -75,6 +93,58 @@ def _event_ftl_payload(event: IngestEvent) -> dict: return payload + +def _parse_cooled_flag(val: object) -> bool | None: + """Parse explicit cooled yes/no. None means missing / unparseable.""" + if val is None: + return None + if isinstance(val, bool): + return val + if isinstance(val, (int, float)) and val in (0, 1): + return bool(val) + if isinstance(val, str): + normalized = val.strip().lower() + if normalized in {"1", "true", "yes", "y"}: + return True + if normalized in {"0", "false", "no", "n"}: + return False + if normalized == "": + return None + return None + + +def _looks_like_internal_id(val: object) -> bool: + """True for opaque codes (FIELD-A7, UUID-like) rather than display names.""" + if not isinstance(val, str): + return False + s = val.strip() + if not s or " " in s: + return False + # Hyphenated uppercase/digit tokens or UUID-ish strings are internal ids. + if re.fullmatch(r"[A-Z0-9]+(?:-[A-Z0-9]+)+", s): + return True + if re.fullmatch(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", s): + return True + if re.fullmatch(r"[A-Z]{1,12}-?[0-9]{1,8}", s): + return True + return False + + +def _is_human_readable_name(val: object) -> bool: + """True when a value looks like a human-readable place/field name.""" + if not isinstance(val, str): + return False + s = val.strip() + if len(s) < 2: + return False + if _looks_like_internal_id(s): + return False + # Must contain a letter; prefer spaces / mixed content over pure codes. + if not any(ch.isalpha() for ch in s): + return False + return True + + def _validate_event_kdes(event: IngestEvent) -> list[str]: """Validate that an event has all required KDEs for its CTE type. @@ -107,35 +177,64 @@ def _validate_event_kdes(event: IngestEvent) -> list[str]: elif not _kde_present(val): errors.append(f"Missing required KDE '{kde_name}' for {event.cte_type.value} CTE") - # Initial packing conditionals (A-2190 fix 2): field/growing area, and - # cooling location+date only when the food was cooled. + # Initial packing conditionals (A-2190 fix 2; Food Safety Lens answers): + # - farm location description (name + phone + address) separately required + # - field/growing area must be a human-readable name (internal id alone + # only when a readable name is also supplied for export) + # - explicit cooled yes/no required; cooling_* only when cooled=yes + # (do not infer cooled from cooling field presence) cte_value = getattr(event.cte_type, "value", event.cte_type) if cte_value == "initial_packing": - has_field = _kde_present(available.get("growing_area_name")) or _kde_present( - available.get("field_id") + for farm_key in ( + "harvest_location_name", + "harvest_location_phone", + "harvest_location_address", + ): + # harvest_location_name is also in REQUIRED_KDES_BY_CTE; re-check + # phone/address here so the farm description is complete as a set. + if farm_key == "harvest_location_name": + continue + if not _kde_present(available.get(farm_key)): + errors.append( + f"Missing required KDE '{farm_key}' for initial_packing CTE " + "(farm location description: name, phone, and address)" + ) + + growing_name = available.get("growing_area_name") + field_name = available.get("field_name") + field_id = available.get("field_id") + has_readable_name = _is_human_readable_name(growing_name) or _is_human_readable_name( + field_name ) - if not has_field: + has_readable_field_id = _is_human_readable_name(field_id) and not _looks_like_internal_id( + field_id + ) + # Internal id alone is OK only when a human-readable name is also present + # (stand-in for "maps to a name in the export" at ingest time). + has_mapped_internal_id = _kde_present(field_id) and has_readable_name + if not (has_readable_name or has_readable_field_id or has_mapped_internal_id): errors.append( - "Missing required KDE 'growing_area_name' or 'field_id' for initial_packing CTE" + "Missing required human-readable growing area or field name for " + "initial_packing CTE (growing_area_name or field_name; field_id " + "alone only when paired with a readable name for export)" ) - cooled_flag = available.get("cooled") - cooled = cooled_flag is True or ( - isinstance(cooled_flag, str) and cooled_flag.strip().lower() in {"1", "true", "yes"} - ) - cooling_hint = _kde_present(available.get("cooling_date")) or _kde_present( - available.get("cooling_location_name") - ) - if cooled or cooling_hint: + cooled_parsed = _parse_cooled_flag(available.get("cooled")) + if cooled_parsed is None: + errors.append( + "Missing required KDE 'cooled' for initial_packing CTE " + "(explicit yes/no required; do not infer from cooling fields)" + ) + elif cooled_parsed is True: if not _kde_present(available.get("cooling_location_name")): errors.append( "Missing required KDE 'cooling_location_name' for initial_packing CTE " - "(required when cooled)" + "(required when cooled=yes)" ) if not _kde_present(available.get("cooling_date")): errors.append( "Missing required KDE 'cooling_date' for initial_packing CTE " - "(required when cooled)" + "(required when cooled=yes)" ) return errors diff --git a/services/ingestion/tests/test_a2190_kde_cte_ftl.py b/services/ingestion/tests/test_a2190_kde_cte_ftl.py index b5daec5f2..a81643d19 100644 --- a/services/ingestion/tests/test_a2190_kde_cte_ftl.py +++ b/services/ingestion/tests/test_a2190_kde_cte_ftl.py @@ -1,14 +1,14 @@ """A-2190: FSMA 204 KDE coverage fixes (contract v3) + FTL ingest gate. Synthetic fixtures only. SEL-549/590 holds unchanged. +Food Safety Lens answers (2026-10-31) refine packing cooled flag, human-readable +field/growing-area names, and farm location description completeness. """ from __future__ import annotations from types import SimpleNamespace -import pytest - from services.ingestion.app.webhook_models import INFLOW_CONTRACT_VERSION, REQUIRED_KDES_BY_CTE, WebhookCTEType from services.ingestion.app.webhook_router_v2.validation import _validate_event_kdes @@ -30,6 +30,12 @@ def _event(cte: WebhookCTEType, kdes: dict, **top) -> SimpleNamespace: def _fill_required(cte: WebhookCTEType, **overrides) -> dict: kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[cte]} + if cte == WebhookCTEType.INITIAL_PACKING: + # Lens conditionals (not all listed in REQUIRED_KDES_BY_CTE). + kdes.setdefault("harvest_location_phone", "+1-831-555-0147") + kdes.setdefault("harvest_location_address", "100 Farm Rd, Salinas CA") + kdes.setdefault("growing_area_name", "North Field Block A") + kdes.setdefault("cooled", "no") kdes.update(overrides) return kdes @@ -55,7 +61,7 @@ def test_cooling_requires_recipient_and_harvest_location(): def test_initial_packing_requires_harvest_and_received_kdes(): - kdes = _fill_required(WebhookCTEType.INITIAL_PACKING, field_id="FIELD-A7") + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING) for key in ("harvest_date", "harvest_location_name", "received_date", "received_quantity"): kdes.pop(key) errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) @@ -63,19 +69,50 @@ def test_initial_packing_requires_harvest_and_received_kdes(): assert any(key in e for e in errors), errors -def test_initial_packing_requires_field_or_growing_area(): +def test_initial_packing_requires_farm_location_phone_and_address(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING) + kdes.pop("harvest_location_phone") + kdes.pop("harvest_location_address") + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + assert any("harvest_location_phone" in e for e in errors) + assert any("harvest_location_address" in e for e in errors) + + +def test_initial_packing_requires_human_readable_growing_area_or_field_name(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING) + kdes.pop("growing_area_name", None) + kdes.pop("field_name", None) + kdes.pop("field_id", None) + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + assert any("growing area" in e.lower() or "field name" in e.lower() for e in errors) + + # Opaque internal id alone is not enough. + kdes["field_id"] = "FIELD-A7" + errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) + assert any("growing area" in e.lower() or "field name" in e.lower() for e in errors) + + # Internal id + readable name is OK (export can map the id). + kdes["growing_area_name"] = "North Field Block A" + assert _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) == [] + + +def test_initial_packing_requires_explicit_cooled_flag(): kdes = _fill_required(WebhookCTEType.INITIAL_PACKING) - # no field_id / growing_area_name + kdes.pop("cooled", None) + # Presence of cooling fields must NOT satisfy the cooled flag. + kdes["cooling_location_name"] = "Cooler #2" + kdes["cooling_date"] = "2026-02-26" errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) - assert any("growing_area_name" in e or "field_id" in e for e in errors) + assert any("cooled" in e for e in errors) -def test_initial_packing_cooling_required_when_cooled(): +def test_initial_packing_cooling_required_when_cooled_yes(): kdes = _fill_required( WebhookCTEType.INITIAL_PACKING, - field_id="FIELD-A7", - cooled=True, + cooled="yes", ) + kdes.pop("cooling_location_name", None) + kdes.pop("cooling_date", None) errors = _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) assert any("cooling_location_name" in e for e in errors) assert any("cooling_date" in e for e in errors) @@ -85,8 +122,10 @@ def test_initial_packing_cooling_required_when_cooled(): assert _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) == [] -def test_initial_packing_cooling_optional_when_not_cooled(): - kdes = _fill_required(WebhookCTEType.INITIAL_PACKING, field_id="FIELD-A7") +def test_initial_packing_cooling_not_required_when_cooled_no(): + kdes = _fill_required(WebhookCTEType.INITIAL_PACKING, cooled="no") + kdes.pop("cooling_location_name", None) + kdes.pop("cooling_date", None) assert _validate_event_kdes(_event(WebhookCTEType.INITIAL_PACKING, kdes)) == [] @@ -126,9 +165,18 @@ def test_ftl_false_skips_required_kde_gate(): assert _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) == [] -def test_ftl_unknown_still_enforces(): +def test_ftl_known_non_ftl_category_skips_required_kde_gate(): + # Category supplied and not on the FTL catalog → is_ftl_food False → skip. + kdes = {"ftl_category": "Shelf-Stable Bread"} + assert _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) == [] + + +def test_ftl_missing_unknown_still_enforces(): + # No ftl_covered / ftl_category → unknown → enforce required KDEs. kdes = _fill_required(WebhookCTEType.HARVESTING) kdes.pop("immediate_subsequent_recipient") + kdes.pop("ftl_covered", None) + kdes.pop("ftl_category", None) errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) assert any("immediate_subsequent_recipient" in e for e in errors) @@ -138,3 +186,10 @@ def test_ftl_true_enforces(): kdes.pop("reference_document") errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) assert any("reference_document" in e for e in errors) + + +def test_ftl_known_ftl_category_enforces(): + kdes = _fill_required(WebhookCTEType.HARVESTING, ftl_category="Fresh Tomatoes") + kdes.pop("immediate_subsequent_recipient") + errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) + assert any("immediate_subsequent_recipient" in e for e in errors) diff --git a/services/ingestion/tests/test_harvester_phone_contract_3041.py b/services/ingestion/tests/test_harvester_phone_contract_3041.py index 4a53137c9..889a3ddc2 100644 --- a/services/ingestion/tests/test_harvester_phone_contract_3041.py +++ b/services/ingestion/tests/test_harvester_phone_contract_3041.py @@ -7,11 +7,22 @@ from services.ingestion.app.webhook_models import INFLOW_CONTRACT_VERSION, REQUIRED_KDES_BY_CTE, WebhookCTEType from services.ingestion.app.webhook_router_v2.validation import _validate_event_kdes +def _packing_kdes(**overrides): + kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[WebhookCTEType.INITIAL_PACKING]} + kdes.update( + harvest_location_phone="+1-831-555-0147", + harvest_location_address="100 Farm Rd, Salinas CA", + growing_area_name="North Field Block A", + cooled="no", + ) + kdes.update(overrides) + return kdes + + @pytest.mark.parametrize("phone", [None, "", " ", 123, {"phone": "555"}]) def test_initial_packing_requires_explicit_string_phone(phone): - kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[WebhookCTEType.INITIAL_PACKING]} - kdes.update( + kdes = _packing_kdes( harvester_business_phone=phone, from_entity_reference="entity-1", field_id="FIELD-A7", @@ -30,9 +41,10 @@ def test_initial_packing_requires_explicit_string_phone(phone): def test_v2_accepts_supplied_phone(): - kdes = {key: "provided" for key in REQUIRED_KDES_BY_CTE[WebhookCTEType.INITIAL_PACKING]} - kdes["harvester_business_phone"] = "+1-202-555-0100" - kdes["field_id"] = "FIELD-A7" + kdes = _packing_kdes( + harvester_business_phone="+1-202-555-0100", + field_id="FIELD-A7", + ) event = SimpleNamespace( cte_type=WebhookCTEType.INITIAL_PACKING, traceability_lot_code="L1", diff --git a/services/ingestion/tests/test_inflow_lab_contract.py b/services/ingestion/tests/test_inflow_lab_contract.py index 0d9aa6785..8974d632f 100644 --- a/services/ingestion/tests/test_inflow_lab_contract.py +++ b/services/ingestion/tests/test_inflow_lab_contract.py @@ -51,7 +51,11 @@ def _event(cte_type: str, kdes: dict) -> IngestEvent: "farm_location": "Valley Fresh Farms", "harvest_date": "2026-04-22", "harvest_location_name": "Valley Fresh Farms", + "harvest_location_phone": "+1-831-555-0147", + "harvest_location_address": "100 Farm Rd, Salinas CA", + "growing_area_name": "North Field Block A", "field_id": "FIELD-A7", + "cooled": "no", "received_date": "2026-04-22", "received_quantity": "24", "reference_document": "Packout Record PACK-20260422-00001", From d5e1867f07f22cb0fbdb719e533ec98489439f4d Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:33:02 -0700 Subject: [PATCH 03/11] fix(ingest): getattr for legacy input_traceability_lot_codes on duck events _authoritative_kdes read event.input_traceability_lot_codes directly, which AttributeError'd on SimpleNamespace test events that omit the optional typed field. Use getattr with a None default, cover missing/legacy top-level/KDE fill-ins, and re-pin INFLOW_LAB_REF to the latest inflow-lab #258 head (helpers now supply contract-v3 KDEs). Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- .github/workflows/inflow-contract-ci.yml | 2 +- .../app/webhook_router_v2/validation.py | 2 +- .../ingestion/tests/test_a2190_kde_cte_ftl.py | 34 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/inflow-contract-ci.yml b/.github/workflows/inflow-contract-ci.yml index 88d6b9638..3a39b9612 100644 --- a/.github/workflows/inflow-contract-ci.yml +++ b/.github/workflows/inflow-contract-ci.yml @@ -67,7 +67,7 @@ env: # Both sides advertise INFLOW_CONTRACT_VERSION "2". # Bumped to inflow-lab main 919b5282 (SEL-335) after the Contract Pin Drift # check found the pin 18 commits behind; contract version is still 2. - INFLOW_LAB_REF: 22467473744963e75c6e460b2a7b7075aedda125 # pragma: allowlist secret + INFLOW_LAB_REF: 59ae980d601e599e6679a5376a04dc74fd71f8ae # pragma: allowlist secret CONTRACT_TENANT_ID: 11111111-1111-1111-1111-111111111111 CONTRACT_API_KEY: ci-inflow-contract-key # pragma: allowlist secret CONTRACT_HMAC_SECRET: ci-inflow-contract-hmac-secret # pragma: allowlist secret diff --git a/services/ingestion/app/webhook_router_v2/validation.py b/services/ingestion/app/webhook_router_v2/validation.py index 8110c663d..50e8cfb08 100644 --- a/services/ingestion/app/webhook_router_v2/validation.py +++ b/services/ingestion/app/webhook_router_v2/validation.py @@ -62,7 +62,7 @@ def _present(val: object) -> bool: if not _present(available.get("input_lot_codes")): legacy = available.get("input_traceability_lot_codes") if not _present(legacy): - legacy = event.input_traceability_lot_codes + legacy = getattr(event, "input_traceability_lot_codes", None) if _present(legacy): available["input_lot_codes"] = legacy return available diff --git a/services/ingestion/tests/test_a2190_kde_cte_ftl.py b/services/ingestion/tests/test_a2190_kde_cte_ftl.py index a81643d19..43a2768c0 100644 --- a/services/ingestion/tests/test_a2190_kde_cte_ftl.py +++ b/services/ingestion/tests/test_a2190_kde_cte_ftl.py @@ -193,3 +193,37 @@ def test_ftl_known_ftl_category_enforces(): kdes.pop("immediate_subsequent_recipient") errors = _validate_event_kdes(_event(WebhookCTEType.HARVESTING, kdes)) assert any("immediate_subsequent_recipient" in e for e in errors) + + +def test_authoritative_kdes_tolerates_events_without_legacy_input_lot_attr(): + """Duck-typed events (SimpleNamespace) often omit typed optional fields. + + ``_authoritative_kdes`` must use getattr for the Lab legacy + ``input_traceability_lot_codes`` top-level fill-in so missing attributes + do not raise AttributeError during required-KDE checks. + """ + kdes = _fill_required(WebhookCTEType.TRANSFORMATION) + kdes.pop("input_lot_codes") + # No input_traceability_lot_codes attr on the namespace — must not raise. + event = _event(WebhookCTEType.TRANSFORMATION, kdes) + assert not hasattr(event, "input_traceability_lot_codes") + errors = _validate_event_kdes(event) + assert any("input_lot_codes" in e for e in errors) + + +def test_authoritative_kdes_fills_input_lot_codes_from_legacy_top_level(): + kdes = _fill_required(WebhookCTEType.TRANSFORMATION) + kdes.pop("input_lot_codes") + event = _event( + WebhookCTEType.TRANSFORMATION, + kdes, + input_traceability_lot_codes=["LOT-LEGACY-A", "LOT-LEGACY-B"], + ) + assert _validate_event_kdes(event) == [] + + +def test_authoritative_kdes_fills_input_lot_codes_from_legacy_kde(): + kdes = _fill_required(WebhookCTEType.TRANSFORMATION) + kdes.pop("input_lot_codes") + kdes["input_traceability_lot_codes"] = ["LOT-KDE-A"] + assert _validate_event_kdes(_event(WebhookCTEType.TRANSFORMATION, kdes)) == [] From ea209b231e9b6036ed8d60924043dcf977d37212 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:37:49 -0700 Subject: [PATCH 04/11] chore(ci): re-pin INFLOW_LAB_REF to inflow-lab #258 msgpack bump head Keep the pin on the latest #258 head (not a merge SHA) so contract CI tracks the helpers + lockfix until #258 merges elsewhere. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- .github/workflows/inflow-contract-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/inflow-contract-ci.yml b/.github/workflows/inflow-contract-ci.yml index 3a39b9612..c0a46a1d0 100644 --- a/.github/workflows/inflow-contract-ci.yml +++ b/.github/workflows/inflow-contract-ci.yml @@ -67,7 +67,7 @@ env: # Both sides advertise INFLOW_CONTRACT_VERSION "2". # Bumped to inflow-lab main 919b5282 (SEL-335) after the Contract Pin Drift # check found the pin 18 commits behind; contract version is still 2. - INFLOW_LAB_REF: 59ae980d601e599e6679a5376a04dc74fd71f8ae # pragma: allowlist secret + INFLOW_LAB_REF: 384197da1642068549b41c00f98d24db453007ab # pragma: allowlist secret CONTRACT_TENANT_ID: 11111111-1111-1111-1111-111111111111 CONTRACT_API_KEY: ci-inflow-contract-key # pragma: allowlist secret CONTRACT_HMAC_SECRET: ci-inflow-contract-hmac-secret # pragma: allowlist secret From 8285a5b2dc60d9768361d54b18dd2fa1f2d171ad Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:07:32 -0700 Subject: [PATCH 05/11] fix(ingestion): fill contract-v3 KDEs in Live DB fixtures; pin Lab #258 merge COMPLETE_ROMAINE (+ related workbench CSVs) and 01_harvesting were still missing required KDEs that A-2190 / contract v3 added (immediate_subsequent recipient, harvest_location_name, packing farm/cooled fields). Live DB email ingest accepted 2/5 and the 500-event webhook half-rejected as a result. Also re-pin INFLOW_LAB_REF to the Lab #258 merge commit 789becec23f054f5a23d983e8b9352c50ac6c4f6 so contract CI tracks the helpers that already landed on inflow-lab main. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- .github/workflows/inflow-contract-ci.yml | 2 +- services/ingestion/app/inflow_workbench.py | 62 +++++++++++-------- .../tests/fixtures/01_harvesting.csv | 8 +-- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/.github/workflows/inflow-contract-ci.yml b/.github/workflows/inflow-contract-ci.yml index c0a46a1d0..3903dfecf 100644 --- a/.github/workflows/inflow-contract-ci.yml +++ b/.github/workflows/inflow-contract-ci.yml @@ -67,7 +67,7 @@ env: # Both sides advertise INFLOW_CONTRACT_VERSION "2". # Bumped to inflow-lab main 919b5282 (SEL-335) after the Contract Pin Drift # check found the pin 18 commits behind; contract version is still 2. - INFLOW_LAB_REF: 384197da1642068549b41c00f98d24db453007ab # pragma: allowlist secret + INFLOW_LAB_REF: 789becec23f054f5a23d983e8b9352c50ac6c4f6 # pragma: allowlist secret CONTRACT_TENANT_ID: 11111111-1111-1111-1111-111111111111 CONTRACT_API_KEY: ci-inflow-contract-key # pragma: allowlist secret CONTRACT_HMAC_SECRET: ci-inflow-contract-hmac-secret # pragma: allowlist secret diff --git a/services/ingestion/app/inflow_workbench.py b/services/ingestion/app/inflow_workbench.py index 904de7037..996ac91e5 100644 --- a/services/ingestion/app/inflow_workbench.py +++ b/services/ingestion/app/inflow_workbench.py @@ -39,33 +39,41 @@ _STORE_LOCK = threading.Lock() -COMPLETE_ROMAINE_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_ref,harvester_business_phone -harvesting,TLC-FEED-001,Romaine Lettuce,120,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-001,SRC-HARV-001,Valley Fresh Farms,Valley Fresh Farms Field 7, -cooling,TLC-FEED-001,Romaine Lettuce,120,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-001,SRC-COOL-001,,Valley Fresh Farms Field 7, -initial_packing,TLC-FEED-001,Romaine Lettuce,118,cases,Salinas Packhouse,2026-04-26T20:12:00Z,,,2026-04-26,,,,,,,PACK-001,SRC-PACK-001,Valley Fresh Farms,Valley Fresh Farms Field 7,+1-202-555-0100 -shipping,TLC-FEED-001,Romaine Lettuce,118,cases,Salinas Packout Dock,2026-04-26T22:41:00Z,,,,2026-04-26,,Salinas Packhouse,Bay Area DC,,,BOL-001,SRC-PACK-001,,Valley Fresh Farms Field 7, -receiving,TLC-FEED-001,Romaine Lettuce,118,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-001,SRC-PACK-001,,Valley Fresh Farms Field 7,""" - -MISSING_DESTINATION_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_ref,harvester_business_phone -harvesting,TLC-FEED-002,Romaine Lettuce,104,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-002,SRC-HARV-002,Valley Fresh Farms,Valley Fresh Farms Field 11, -cooling,TLC-FEED-002,Romaine Lettuce,104,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-002,SRC-COOL-002,,Valley Fresh Farms Field 11, -initial_packing,TLC-FEED-002,Romaine Lettuce,103,cases,Salinas Packhouse,2026-04-26T20:12:00Z,,,2026-04-26,,,,,,,PACK-002,SRC-PACK-002,Valley Fresh Farms,Valley Fresh Farms Field 11,+1-202-555-0100 -shipping,TLC-FEED-002,Romaine Lettuce,103,cases,Salinas Packout Dock,2026-04-26T22:41:00Z,,,,2026-04-26,,Salinas Packhouse,,,,BOL-002,SRC-PACK-002,,Valley Fresh Farms Field 11, -receiving,TLC-FEED-002,Romaine Lettuce,103,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-002,SRC-PACK-002,,Valley Fresh Farms Field 11,""" - -BROKEN_LINEAGE_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_ref,transformation_date,input_traceability_lot_codes,harvester_business_phone -harvesting,TLC-FEED-003,Romaine Lettuce,220,cases,Desert Bloom Farm,2026-04-26T14:43:00Z,2026-04-26,,,,,,,,,HARV-003,SRC-HARV-003,Desert Bloom Farm,Desert Bloom Farm Block 3,,, -cooling,TLC-FEED-003,Romaine Lettuce,220,cases,Imperial Pre-Cool Facility,2026-04-26T17:51:00Z,,2026-04-26,,,,,,,,COOL-003,SRC-COOL-003,,Desert Bloom Farm Block 3,,, -transformation,TLC-FEED-003-CUT,Romaine Lettuce,218,cases,LA Fresh Cut Kitchen,2026-04-27T00:10:00Z,,,,,,,,,,TRANS-003,SRC-MISSING-PACK,,Desert Bloom Farm Block 3,2026-04-27,,""" - -THREE_MISSING_SHIPTO_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_ref,harvester_business_phone -harvesting,DEMO-LOT-0412,Organic Romaine,180,cases,Coastal Valley Farms,2026-04-12T08:30:00Z,2026-04-12,,,,,,,,,HARV-0412-A,SRC-HARV-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412 -cooling,DEMO-LOT-0412,Organic Romaine,180,cases,Valley Cooling Facility,2026-04-12T12:00:00Z,,2026-04-12,,,,,,,,COOL-0412-A,SRC-COOL-0412,,Coastal Field Block 5, -initial_packing,DEMO-LOT-0412,Organic Romaine,178,cases,Valley Packhouse,2026-04-12T15:30:00Z,,,2026-04-12,,,,,,,PACK-0412-A,SRC-PACK-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412 -shipping,DEMO-LOT-0412,Organic Romaine,178,cases,Valley Packout Dock A,2026-04-12T19:00:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-A,SRC-PACK-0412,,Coastal Field Block 5, -shipping,DEMO-LOT-0412,Organic Romaine,88,cases,Valley Packout Dock B,2026-04-12T19:15:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-B,SRC-PACK-0412,,Coastal Field Block 5, -shipping,DEMO-LOT-0412,Organic Romaine,90,cases,Valley Packout Dock C,2026-04-12T19:30:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-C,SRC-PACK-0412,,Coastal Field Block 5, -receiving,DEMO-LOT-0412,Organic Romaine,178,cases,Regional Distribution Center,2026-04-13T03:45:00Z,,,,,2026-04-13,,,Regional Distribution Center,Valley Packout Dock A,REC-0412,SRC-PACK-0412,,Coastal Field Block 5,""" +COMPLETE_ROMAINE_CSV = """ +cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +harvesting,TLC-FEED-001,Romaine Lettuce,120,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-001,SRC-HARV-001,Valley Fresh Farms,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",+1-202-555-0100,Salinas Cooling Hub,,,,, +cooling,TLC-FEED-001,Romaine Lettuce,120,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-001,SRC-COOL-001,,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",,Salinas Packhouse,,,,, +initial_packing,TLC-FEED-001,Romaine Lettuce,118,cases,Salinas Packhouse,2026-04-26T20:12:00Z,2026-04-26,2026-04-26,2026-04-26,,,,,,,PACK-001,SRC-PACK-001,Valley Fresh Farms,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",+1-202-555-0100,,2026-04-26,120,North Field Block A,yes,Salinas Cooling Hub +shipping,TLC-FEED-001,Romaine Lettuce,118,cases,Salinas Packout Dock,2026-04-26T22:41:00Z,,,,2026-04-26,,Salinas Packhouse,Bay Area DC,,,BOL-001,SRC-PACK-001,,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",,,,,,, +receiving,TLC-FEED-001,Romaine Lettuce,118,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-001,SRC-PACK-001,,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",,,,,,, +""" + +MISSING_DESTINATION_CSV = """ +cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +harvesting,TLC-FEED-002,Romaine Lettuce,104,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-002,SRC-HARV-002,Valley Fresh Farms,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",+1-202-555-0100,Salinas Cooling Hub,,,,, +cooling,TLC-FEED-002,Romaine Lettuce,104,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-002,SRC-COOL-002,,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",,Salinas Packhouse,,,,, +initial_packing,TLC-FEED-002,Romaine Lettuce,103,cases,Salinas Packhouse,2026-04-26T20:12:00Z,2026-04-26,2026-04-26,2026-04-26,,,,,,,PACK-002,SRC-PACK-002,Valley Fresh Farms,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",+1-202-555-0100,,2026-04-26,104,North Field Block B,yes,Salinas Cooling Hub +shipping,TLC-FEED-002,Romaine Lettuce,103,cases,Salinas Packout Dock,2026-04-26T22:41:00Z,,,,2026-04-26,,Salinas Packhouse,,,,BOL-002,SRC-PACK-002,,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",,,,,,, +receiving,TLC-FEED-002,Romaine Lettuce,103,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-002,SRC-PACK-002,,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",,,,,,, +""" + +BROKEN_LINEAGE_CSV = """ +cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name,transformation_date,input_traceability_lot_codes,input_quantities,input_units_of_measure,input_product_descriptions +harvesting,TLC-FEED-003,Romaine Lettuce,220,cases,Desert Bloom Farm,2026-04-26T14:43:00Z,2026-04-26,,,,,,,,,HARV-003,SRC-HARV-003,Desert Bloom Farm,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,Imperial Pre-Cool Facility,,,,,,,,,, +cooling,TLC-FEED-003,Romaine Lettuce,220,cases,Imperial Pre-Cool Facility,2026-04-26T17:51:00Z,,2026-04-26,,,,,,,,COOL-003,SRC-COOL-003,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",,LA Fresh Cut Kitchen,,,,,,,,,, +transformation,TLC-FEED-003-CUT,Romaine Lettuce,218,cases,LA Fresh Cut Kitchen,2026-04-27T00:10:00Z,,,,,,,,,,TRANS-003,SRC-MISSING-PACK,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,,,,,,,2026-04-27,TLC-FEED-003,220,cases,Romaine Lettuce +""" + +THREE_MISSING_SHIPTO_CSV = """ +cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +harvesting,DEMO-LOT-0412,Organic Romaine,180,cases,Coastal Valley Farms,2026-04-12T08:30:00Z,2026-04-12,,,,,,,,,HARV-0412-A,SRC-HARV-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",+1-415-555-0412,Valley Cooling Facility,,,,, +cooling,DEMO-LOT-0412,Organic Romaine,180,cases,Valley Cooling Facility,2026-04-12T12:00:00Z,,2026-04-12,,,,,,,,COOL-0412-A,SRC-COOL-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,Valley Packhouse,,,,, +initial_packing,DEMO-LOT-0412,Organic Romaine,178,cases,Valley Packhouse,2026-04-12T15:30:00Z,2026-04-12,2026-04-12,2026-04-12,,,,,,,PACK-0412-A,SRC-PACK-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",+1-415-555-0412,,2026-04-12,180,Coastal Field Block 5,yes,Valley Cooling Facility +shipping,DEMO-LOT-0412,Organic Romaine,178,cases,Valley Packout Dock A,2026-04-12T19:00:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-A,SRC-PACK-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,,,,,, +shipping,DEMO-LOT-0412,Organic Romaine,88,cases,Valley Packout Dock B,2026-04-12T19:15:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-B,SRC-PACK-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,,,,,, +shipping,DEMO-LOT-0412,Organic Romaine,90,cases,Valley Packout Dock C,2026-04-12T19:30:00Z,,,,2026-04-12,,Valley Packhouse,,,,BOL-0412-C,SRC-PACK-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,,,,,, +receiving,DEMO-LOT-0412,Organic Romaine,178,cases,Regional Distribution Center,2026-04-13T03:45:00Z,,,,,2026-04-13,,,Regional Distribution Center,Valley Packout Dock A,REC-0412,SRC-PACK-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,,,,,, +""" BUILT_IN_SCENARIOS = [ { diff --git a/services/ingestion/tests/fixtures/01_harvesting.csv b/services/ingestion/tests/fixtures/01_harvesting.csv index b6f611b3e..d7fa9292d 100644 --- a/services/ingestion/tests/fixtures/01_harvesting.csv +++ b/services/ingestion/tests/fixtures/01_harvesting.csv @@ -1,4 +1,4 @@ -traceability_lot_code,product_description,quantity,unit_of_measure,harvest_date,location_name,location_gln,field_id,harvester_name,reference_document -LOT-ROM-2026-0312A,Organic Romaine Hearts,5000,lbs,2026-03-12,Salinas Valley Farms,0061414100014,FIELD-SVF-7,Salinas Valley Farms LLC,HARV-SVF-2026-0312-001 -LOT-ROM-2026-0312B,Chopped Romaine Mix,3200,lbs,2026-03-12,Salinas Valley Farms,0061414100014,FIELD-SVF-12,Salinas Valley Farms LLC,HARV-SVF-2026-0312-002 -LOT-ROM-2026-0311C,Romaine Hearts 3-Pack,4100,lbs,2026-03-11,Coastline Ridge Growers,0061414100021,FIELD-CRG-3,Coastline Ridge Growers Inc,HARV-SVF-2026-0312-003 +traceability_lot_code,product_description,quantity,unit_of_measure,harvest_date,location_name,location_gln,field_id,harvester_name,reference_document,immediate_subsequent_recipient +LOT-ROM-2026-0312A,Organic Romaine Hearts,5000,lbs,2026-03-12,Salinas Valley Farms,0061414100014,FIELD-SVF-7,Salinas Valley Farms LLC,HARV-SVF-2026-0312-001,Salinas Cooling Hub +LOT-ROM-2026-0312B,Chopped Romaine Mix,3200,lbs,2026-03-12,Salinas Valley Farms,0061414100014,FIELD-SVF-12,Salinas Valley Farms LLC,HARV-SVF-2026-0312-002,Salinas Cooling Hub +LOT-ROM-2026-0311C,Romaine Hearts 3-Pack,4100,lbs,2026-03-11,Coastline Ridge Growers,0061414100021,FIELD-CRG-3,Coastline Ridge Growers Inc,HARV-SVF-2026-0312-003,Coastline Cooler From f981d84bcdf065c036ea7e5e457d53bc134765e1 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:41:47 -0700 Subject: [PATCH 06/11] fix(ingestion): drop leading newline in workbench CSV literals DictReader treated the blank first line as the header, so _parse_csv_to_events returned [] and Live DB webhook/email proofs posted empty event lists (422) / rejected the CSV attachment. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- services/ingestion/app/inflow_workbench.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/services/ingestion/app/inflow_workbench.py b/services/ingestion/app/inflow_workbench.py index 996ac91e5..0a737f8cc 100644 --- a/services/ingestion/app/inflow_workbench.py +++ b/services/ingestion/app/inflow_workbench.py @@ -39,8 +39,7 @@ _STORE_LOCK = threading.Lock() -COMPLETE_ROMAINE_CSV = """ -cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +COMPLETE_ROMAINE_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name harvesting,TLC-FEED-001,Romaine Lettuce,120,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-001,SRC-HARV-001,Valley Fresh Farms,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",+1-202-555-0100,Salinas Cooling Hub,,,,, cooling,TLC-FEED-001,Romaine Lettuce,120,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-001,SRC-COOL-001,,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",,Salinas Packhouse,,,,, initial_packing,TLC-FEED-001,Romaine Lettuce,118,cases,Salinas Packhouse,2026-04-26T20:12:00Z,2026-04-26,2026-04-26,2026-04-26,,,,,,,PACK-001,SRC-PACK-001,Valley Fresh Farms,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",+1-202-555-0100,,2026-04-26,120,North Field Block A,yes,Salinas Cooling Hub @@ -48,8 +47,7 @@ receiving,TLC-FEED-001,Romaine Lettuce,118,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-001,SRC-PACK-001,,Valley Fresh Farms Field 7,+1-202-555-0100,"123 Farm Rd, Salinas CA",,,,,,, """ -MISSING_DESTINATION_CSV = """ -cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +MISSING_DESTINATION_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name harvesting,TLC-FEED-002,Romaine Lettuce,104,cases,Valley Fresh Farms,2026-04-26T15:20:00Z,2026-04-26,,,,,,,,,HARV-002,SRC-HARV-002,Valley Fresh Farms,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",+1-202-555-0100,Salinas Cooling Hub,,,,, cooling,TLC-FEED-002,Romaine Lettuce,104,cases,Salinas Cooling Hub,2026-04-26T18:12:00Z,,2026-04-26,,,,,,,,COOL-002,SRC-COOL-002,,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",,Salinas Packhouse,,,,, initial_packing,TLC-FEED-002,Romaine Lettuce,103,cases,Salinas Packhouse,2026-04-26T20:12:00Z,2026-04-26,2026-04-26,2026-04-26,,,,,,,PACK-002,SRC-PACK-002,Valley Fresh Farms,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",+1-202-555-0100,,2026-04-26,104,North Field Block B,yes,Salinas Cooling Hub @@ -57,15 +55,13 @@ receiving,TLC-FEED-002,Romaine Lettuce,103,cases,Bay Area DC,2026-04-27T02:04:00Z,,,,,2026-04-27,,,Bay Area DC,Salinas Packout Dock,REC-002,SRC-PACK-002,,Valley Fresh Farms Field 11,+1-202-555-0100,"11 Farm Rd, Salinas CA",,,,,,, """ -BROKEN_LINEAGE_CSV = """ -cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name,transformation_date,input_traceability_lot_codes,input_quantities,input_units_of_measure,input_product_descriptions +BROKEN_LINEAGE_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name,transformation_date,input_traceability_lot_codes,input_quantities,input_units_of_measure,input_product_descriptions harvesting,TLC-FEED-003,Romaine Lettuce,220,cases,Desert Bloom Farm,2026-04-26T14:43:00Z,2026-04-26,,,,,,,,,HARV-003,SRC-HARV-003,Desert Bloom Farm,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,Imperial Pre-Cool Facility,,,,,,,,,, cooling,TLC-FEED-003,Romaine Lettuce,220,cases,Imperial Pre-Cool Facility,2026-04-26T17:51:00Z,,2026-04-26,,,,,,,,COOL-003,SRC-COOL-003,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",,LA Fresh Cut Kitchen,,,,,,,,,, transformation,TLC-FEED-003-CUT,Romaine Lettuce,218,cases,LA Fresh Cut Kitchen,2026-04-27T00:10:00Z,,,,,,,,,,TRANS-003,SRC-MISSING-PACK,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,,,,,,,2026-04-27,TLC-FEED-003,220,cases,Romaine Lettuce """ -THREE_MISSING_SHIPTO_CSV = """ -cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name +THREE_MISSING_SHIPTO_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name harvesting,DEMO-LOT-0412,Organic Romaine,180,cases,Coastal Valley Farms,2026-04-12T08:30:00Z,2026-04-12,,,,,,,,,HARV-0412-A,SRC-HARV-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",+1-415-555-0412,Valley Cooling Facility,,,,, cooling,DEMO-LOT-0412,Organic Romaine,180,cases,Valley Cooling Facility,2026-04-12T12:00:00Z,,2026-04-12,,,,,,,,COOL-0412-A,SRC-COOL-0412,,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",,Valley Packhouse,,,,, initial_packing,DEMO-LOT-0412,Organic Romaine,178,cases,Valley Packhouse,2026-04-12T15:30:00Z,2026-04-12,2026-04-12,2026-04-12,,,,,,,PACK-0412-A,SRC-PACK-0412,Coastal Valley Farms,Coastal Field Block 5,+1-415-555-0412,"5 Coastal Rd, CA",+1-415-555-0412,,2026-04-12,180,Coastal Field Block 5,yes,Valley Cooling Facility From 94577f5c3f0fdcc12e192f05a4cff8a8177fe9e2 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:56:39 -0700 Subject: [PATCH 07/11] fix(ingestion): map received_date distinctly from receive_date Contract v3 requires initial_packing.received_date (carry-in), while receiving uses receive_date. The CSV alias collapsed both to receive_date, so Live DB COMPLETE fixtures lost packing received_date and rejected 1/5 events (400/500). Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- services/ingestion/app/sandbox/csv_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/ingestion/app/sandbox/csv_parser.py b/services/ingestion/app/sandbox/csv_parser.py index e15cbdac9..3be4bc665 100644 --- a/services/ingestion/app/sandbox/csv_parser.py +++ b/services/ingestion/app/sandbox/csv_parser.py @@ -157,9 +157,9 @@ def input_lot_codes(kdes: dict | None) -> list[str]: "date_shipped": "ship_date", "shipping_date": "ship_date", "ship_dt": "ship_date", - # Receive date + # Receive date (receiving CTE) vs received_date (initial_packing carry-in; contract v3) "receive_date": "receive_date", - "received_date": "receive_date", + "received_date": "received_date", "date_received": "receive_date", "receiving_date": "receive_date", "receipt_date": "receive_date", From f510a92f3c53e0cd974dc13d4b1bfe36282e9b80 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:01:07 -0700 Subject: [PATCH 08/11] fix(tests): add harvest recipient to Live DB CSV upload fixture Contract v3 requires immediate_subsequent_recipient on harvesting; the pg evidence CSV helper omitted it so both rows were KDE-rejected (accepted=0) after the A-2190 REQUIRED_KDES update. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- tests/shared/test_postgres_evidence_workflow_pg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/shared/test_postgres_evidence_workflow_pg.py b/tests/shared/test_postgres_evidence_workflow_pg.py index 963720050..00d80bb43 100644 --- a/tests/shared/test_postgres_evidence_workflow_pg.py +++ b/tests/shared/test_postgres_evidence_workflow_pg.py @@ -370,9 +370,9 @@ def _upload_csv(monkeypatch, tenant): day = (datetime.now(timezone.utc).date() - timedelta(days=2)).isoformat() body = ( "Lot #,tlc,Shipped On,product_description,quantity,unit_of_measure,harvest_date,location_name," - "reference_document\n" - f"LOT-1,PG-CSV-{tenant[:8]}-1,{day},Kale,10,lbs,{day},Farm A,REF-1\n" - f"LOT-2,PG-CSV-{tenant[:8]}-2,{day},Chard,5,lbs,{day},Farm A,REF-2\n" + "reference_document,immediate_subsequent_recipient\n" + f"LOT-1,PG-CSV-{tenant[:8]}-1,{day},Kale,10,lbs,{day},Farm A,REF-1,Cooler Co\n" + f"LOT-2,PG-CSV-{tenant[:8]}-2,{day},Chard,5,lbs,{day},Farm A,REF-2,Cooler Co\n" ).encode() with TestClient(app) as client: response = client.post( From aa3f8d1cc5f3e83d5889dd495e83438b9244b6d4 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:11:29 -0700 Subject: [PATCH 09/11] test(shared): sync WEBHOOK_REQUIRED_KDES snapshot with A-2190 contract v3 Update the copied REQUIRED_KDES table only so Test shared matches REQUIRED_KDES_BY_CTE (harvest/cooling recipient + packing/transform fields). Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- services/shared/supplier_data_quality.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/shared/supplier_data_quality.py b/services/shared/supplier_data_quality.py index 65246fad6..65e45dfa6 100644 --- a/services/shared/supplier_data_quality.py +++ b/services/shared/supplier_data_quality.py @@ -72,6 +72,7 @@ "unit_of_measure", "harvest_date", "location_name", + "immediate_subsequent_recipient", "reference_document", ), "cooling": ( @@ -81,6 +82,8 @@ "unit_of_measure", "cooling_date", "location_name", + "immediate_subsequent_recipient", + "harvest_location_name", "reference_document", ), "initial_packing": ( @@ -93,6 +96,10 @@ "reference_document", "harvester_business_name", "harvester_business_phone", + "harvest_date", + "harvest_location_name", + "received_date", + "received_quantity", ), "first_receiver": ( "traceability_lot_code", @@ -133,6 +140,10 @@ "transformation_date", "location_name", "reference_document", + "input_lot_codes", + "input_quantities", + "input_units_of_measure", + "input_product_descriptions", ), } From 012ebaa41d04acaaaef0cbe154c6a5aaf3127716 Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:46:37 -0700 Subject: [PATCH 10/11] fix(ingestion): satisfy A-2190 quantity KDEs via structured inputs Treat complete kdes.inputs (qty+UOM per lot) as satisfying contract-v3 input_quantities/input_units_of_measure so reject-path and completeness do not false-miss after parse folds raw columns. Align harvest fixtures and harvest_location_name seed with REQUIRED_KDES; keep BROKEN_LINEAGE non-compliant by clearing filled inputs; rename jev "Received Date" so it is not auto-mapped as received_date. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- docs/engineering/examples/jev_eval_cases.json | 165 ++++++++++++++++-- services/ingestion/app/csv_templates.py | 27 +++ services/ingestion/app/inflow_workbench.py | 2 +- .../app/webhook_router_v2/validation.py | 12 ++ .../test_csv_ingest_response_model_2892.py | 7 +- .../ingestion/tests/test_csv_templates.py | 2 + .../tests/test_csv_templates_1342.py | 6 +- .../ingestion/tests/test_pilot_diagnostic.py | 7 +- services/shared/rules/seeds.py | 1 + 9 files changed, 206 insertions(+), 23 deletions(-) diff --git a/docs/engineering/examples/jev_eval_cases.json b/docs/engineering/examples/jev_eval_cases.json index 903081e1d..58596faee 100644 --- a/docs/engineering/examples/jev_eval_cases.json +++ b/docs/engineering/examples/jev_eval_cases.json @@ -4,7 +4,18 @@ "cases": [ { "id": "erp-shipping-log", - "headers": ["Ship Dte", "Sales Order", "Cust Name", "Ship-To City", "Item Desc", "Lot No.", "Cases Shipped", "UOM", "BOL No", "Freight Co."], + "headers": [ + "Ship Dte", + "Sales Order", + "Cust Name", + "Ship-To City", + "Item Desc", + "Lot No.", + "Cases Shipped", + "UOM", + "BOL No", + "Freight Co." + ], "cte": "shipping", "mappings": { "Ship Dte": "ship_date", @@ -26,7 +37,17 @@ }, { "id": "distributor-outbound", - "headers": ["Dispatch Dt", "Consignee Name", "Deliver-To GLN", "SKU Description", "Batch #", "Pallets", "Sell Unit", "Invoice Number", "Origin Facility"], + "headers": [ + "Dispatch Dt", + "Consignee Name", + "Deliver-To GLN", + "SKU Description", + "Batch #", + "Pallets", + "Sell Unit", + "Invoice Number", + "Origin Facility" + ], "cte": "shipping", "mappings": { "Dispatch Dt": "ship_date", @@ -45,7 +66,16 @@ }, { "id": "shipping-with-order-date", - "headers": ["Order Date", "Ship Date", "Customer PO", "ShipTo Name", "Product", "TLC", "Qty", "UOM"], + "headers": [ + "Order Date", + "Ship Date", + "Customer PO", + "ShipTo Name", + "Product", + "TLC", + "Qty", + "UOM" + ], "cte": "shipping", "mappings": { "Order Date": "no_match", @@ -55,7 +85,17 @@ }, { "id": "dc-receiving-log", - "headers": ["Date Received", "Rec'd From", "PO #", "Product Name", "Supplier Lot", "Qty Rcvd", "Rcv Unit", "Dock Door", "Temp at Receipt (F)"], + "headers": [ + "Date Received", + "Rec'd From", + "PO #", + "Product Name", + "Supplier Lot", + "Qty Rcvd", + "Rcv Unit", + "Dock Door", + "Temp at Receipt (F)" + ], "cte": "receiving", "mappings": { "Date Received": "receive_date", @@ -74,7 +114,15 @@ }, { "id": "restaurant-receiving", - "headers": ["Delivered On", "Vendor Name", "Invoice #", "Item Name", "Lot/Batch", "Case Count", "Store Location"], + "headers": [ + "Delivered On", + "Vendor Name", + "Invoice #", + "Item Name", + "Lot/Batch", + "Case Count", + "Store Location" + ], "cte": "receiving", "mappings": { "Delivered On": "receive_date", @@ -88,18 +136,34 @@ }, { "id": "receiving-with-supplier-ship-date", - "headers": ["Ship Date", "Received Date", "Shipper Name", "Item Descr.", "Lot Code", "Qty", "Recv Site"], + "headers": [ + "Ship Date", + "Date Received", + "Shipper Name", + "Item Descr.", + "Lot Code", + "Qty", + "Recv Site" + ], "cte": "receiving", "mappings": { - "Received Date": "receive_date", "Shipper Name": "immediate_previous_source", "Item Descr.": "product_description", - "Recv Site": "receiving_location" + "Recv Site": "receiving_location", + "Date Received": "receive_date" } }, { "id": "farm-harvest-log", - "headers": ["Pick Dt", "Ranch", "Block #", "Crop", "Harvest Lot", "Bins", "Crew Lead"], + "headers": [ + "Pick Dt", + "Ranch", + "Block #", + "Crop", + "Harvest Lot", + "Bins", + "Crew Lead" + ], "cte": "harvesting", "mappings": { "Pick Dt": "harvest_date", @@ -119,7 +183,16 @@ }, { "id": "harvest-template-variant", - "headers": ["TLC", "Crop Name", "Qty", "UOM", "Harvest Dt", "Farm Name", "Field #", "Harvest Company"], + "headers": [ + "TLC", + "Crop Name", + "Qty", + "UOM", + "Harvest Dt", + "Farm Name", + "Field #", + "Harvest Company" + ], "cte": "harvesting", "mappings": { "Crop Name": "product_description", @@ -134,7 +207,15 @@ }, { "id": "cooler-log", - "headers": ["Date Cooled", "Cooler ID", "Produce", "Lot Nbr", "Pallet Count", "Pulp Temp F", "Cooling Method"], + "headers": [ + "Date Cooled", + "Cooler ID", + "Produce", + "Lot Nbr", + "Pallet Count", + "Pulp Temp F", + "Cooling Method" + ], "cte": "cooling", "mappings": { "Date Cooled": "cooling_date", @@ -151,7 +232,16 @@ }, { "id": "packhouse-pack-out", - "headers": ["Packed Date", "Packinghouse", "Grower/Harvester", "Harvest Lot #", "Pack Lot", "Commodity Desc", "Cartons", "Pack Style"], + "headers": [ + "Packed Date", + "Packinghouse", + "Grower/Harvester", + "Harvest Lot #", + "Pack Lot", + "Commodity Desc", + "Cartons", + "Pack Style" + ], "cte": "initial_packing", "mappings": { "Packed Date": "packing_date", @@ -168,7 +258,17 @@ }, { "id": "fresh-cut-production", - "headers": ["Production Dt", "Work Order", "Finished Good", "Output Lot", "Output Qty", "Output UOM", "Component Lots", "Component Qtys", "Line"], + "headers": [ + "Production Dt", + "Work Order", + "Finished Good", + "Output Lot", + "Output Qty", + "Output UOM", + "Component Lots", + "Component Qtys", + "Line" + ], "cte": "transformation", "mappings": { "Production Dt": "transformation_date", @@ -187,7 +287,15 @@ }, { "id": "seafood-landing", - "headers": ["Offload Dt", "Vessel Name", "Dock", "Species", "Landing Lot", "Weight (lbs)", "Harvest Area"], + "headers": [ + "Offload Dt", + "Vessel Name", + "Dock", + "Species", + "Landing Lot", + "Weight (lbs)", + "Harvest Area" + ], "cte": "first_land_based_receiving", "mappings": { "Offload Dt": "landing_date", @@ -207,7 +315,15 @@ }, { "id": "mixed-transactions", - "headers": ["Trans Date", "Txn Type", "Material", "Lot ID", "Qty", "From", "To"], + "headers": [ + "Trans Date", + "Txn Type", + "Material", + "Lot ID", + "Qty", + "From", + "To" + ], "cte": "mixed_or_unclear", "mappings": { "Trans Date": "event_date", @@ -223,7 +339,15 @@ }, { "id": "inventory-snapshot", - "headers": ["As Of", "Warehouse", "SKU", "Description", "Lot Ref", "On Hand", "Bin"], + "headers": [ + "As Of", + "Warehouse", + "SKU", + "Description", + "Lot Ref", + "On Hand", + "Bin" + ], "cte": "mixed_or_unclear", "mappings": { "As Of": [ @@ -242,7 +366,14 @@ }, { "id": "per-row-events", - "headers": ["Event Type", "Lot #", "Product", "Qty", "Date", "Site Name"], + "headers": [ + "Event Type", + "Lot #", + "Product", + "Qty", + "Date", + "Site Name" + ], "cte": null, "mappings": { "Lot #": "traceability_lot_code", diff --git a/services/ingestion/app/csv_templates.py b/services/ingestion/app/csv_templates.py index a2f17169e..d55d14077 100644 --- a/services/ingestion/app/csv_templates.py +++ b/services/ingestion/app/csv_templates.py @@ -533,6 +533,30 @@ def _inject_required_kdes(kdes: dict, row_cte: str, row: dict, loc_name: Optiona kdes[kde_key] = value +def _inputs_satisfy_quantity_kdes(kdes: dict) -> bool: + """True when ``kdes.inputs`` carries quantity + UOM for every input lot. + + ``parse_csv_rows`` folds ``input_quantities`` / ``input_units_of_measure`` + into structured ``inputs`` and leaves the raw columns out of ``kdes`` + (see test_csv_input_quantities). Contract v3 still names those raw fields + in REQUIRED_KDES_BY_CTE; treat a complete ``inputs`` list as satisfying + them so template rows and aligned uploads are not false-missing. + """ + inputs = kdes.get("inputs") + if not isinstance(inputs, list) or not inputs: + return False + for item in inputs: + if not isinstance(item, dict): + return False + qty = item.get("quantity") + uom = item.get("unit_of_measure") + if qty is None or str(qty).strip() == "": + return False + if uom is None or str(uom).strip() == "": + return False + return True + + def _validate_kde_completeness(cte_type: str, event: IngestEvent, kdes: dict) -> list[str]: """Check required KDEs for a CTE type. Returns list of missing field names.""" try: @@ -541,6 +565,7 @@ def _validate_kde_completeness(cte_type: str, event: IngestEvent, kdes: dict) -> return [] required = REQUIRED_KDES_BY_CTE.get(cte_enum, []) missing = [] + inputs_ok = _inputs_satisfy_quantity_kdes(kdes) for field in required: # Check in top-level event fields first, then KDEs if field in {"traceability_lot_code", "product_description", "quantity", "unit_of_measure"}: @@ -550,6 +575,8 @@ def _validate_kde_completeness(cte_type: str, event: IngestEvent, kdes: dict) -> elif field == "location_name": if not (event.location_name or kdes.get("location_name")): missing.append(field) + elif field in {"input_quantities", "input_units_of_measure"} and inputs_ok: + continue else: val = kdes.get(field) if val is None or str(val).strip() == "": diff --git a/services/ingestion/app/inflow_workbench.py b/services/ingestion/app/inflow_workbench.py index 0a737f8cc..c57b5a525 100644 --- a/services/ingestion/app/inflow_workbench.py +++ b/services/ingestion/app/inflow_workbench.py @@ -58,7 +58,7 @@ BROKEN_LINEAGE_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name,transformation_date,input_traceability_lot_codes,input_quantities,input_units_of_measure,input_product_descriptions harvesting,TLC-FEED-003,Romaine Lettuce,220,cases,Desert Bloom Farm,2026-04-26T14:43:00Z,2026-04-26,,,,,,,,,HARV-003,SRC-HARV-003,Desert Bloom Farm,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,Imperial Pre-Cool Facility,,,,,,,,,, cooling,TLC-FEED-003,Romaine Lettuce,220,cases,Imperial Pre-Cool Facility,2026-04-26T17:51:00Z,,2026-04-26,,,,,,,,COOL-003,SRC-COOL-003,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",,LA Fresh Cut Kitchen,,,,,,,,,, -transformation,TLC-FEED-003-CUT,Romaine Lettuce,218,cases,LA Fresh Cut Kitchen,2026-04-27T00:10:00Z,,,,,,,,,,TRANS-003,SRC-MISSING-PACK,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,,,,,,,2026-04-27,TLC-FEED-003,220,cases,Romaine Lettuce +transformation,TLC-FEED-003-CUT,Romaine Lettuce,218,cases,LA Fresh Cut Kitchen,2026-04-27T00:10:00Z,,,,,,,,,,TRANS-003,SRC-MISSING-PACK,,Desert Bloom Farm Block 3,+1-760-555-0100,"3 Desert Rd, Imperial CA",+1-760-555-0100,,,,,,,2026-04-27,,,, """ THREE_MISSING_SHIPTO_CSV = """cte_type,traceability_lot_code,product_description,quantity,unit_of_measure,location_name,timestamp,harvest_date,cooling_date,packing_date,ship_date,receive_date,ship_from_location,ship_to_location,receiving_location,immediate_previous_source,reference_document,tlc_source_reference,harvester_business_name,harvest_location_name,harvest_location_phone,harvest_location_address,harvester_business_phone,immediate_subsequent_recipient,received_date,received_quantity,growing_area_name,cooled,cooling_location_name diff --git a/services/ingestion/app/webhook_router_v2/validation.py b/services/ingestion/app/webhook_router_v2/validation.py index 50e8cfb08..2439334c9 100644 --- a/services/ingestion/app/webhook_router_v2/validation.py +++ b/services/ingestion/app/webhook_router_v2/validation.py @@ -65,6 +65,18 @@ def _present(val: object) -> bool: legacy = getattr(event, "input_traceability_lot_codes", None) if _present(legacy): available["input_lot_codes"] = legacy + # Structured ``inputs`` (from CSV input_quantities / input_units_of_measure) + # satisfies the contract-v3 raw field names when every entry has qty+UOM. + inputs = available.get("inputs") + if isinstance(inputs, list) and inputs: + if all( + isinstance(item, dict) + and _present(item.get("quantity")) + and _present(item.get("unit_of_measure")) + for item in inputs + ): + available.setdefault("input_quantities", inputs) + available.setdefault("input_units_of_measure", inputs) return available diff --git a/services/ingestion/tests/test_csv_ingest_response_model_2892.py b/services/ingestion/tests/test_csv_ingest_response_model_2892.py index 443a88157..d9d7fedfb 100644 --- a/services/ingestion/tests/test_csv_ingest_response_model_2892.py +++ b/services/ingestion/tests/test_csv_ingest_response_model_2892.py @@ -35,6 +35,7 @@ "harvest_date", "location_name", "reference_document", + "immediate_subsequent_recipient", ] @@ -135,7 +136,7 @@ def test_new_fields_are_not_required_so_replayed_bodies_stay_valid(self): class TestWireShape: def test_warnings_still_returned_when_present(self): # 'L0T1O23' trips the lot-code character-swap heuristic. - rows = [["L0T1O23", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF"]] + rows = [["L0T1O23", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF", "Packhouse A"]] resp = _post(TestClient(_build_app()), _csv(rows)) assert resp.status_code == 200, resp.text body = resp.json() @@ -143,7 +144,7 @@ def test_warnings_still_returned_when_present(self): assert any("character swap" in w for w in body["kde_warnings"]) def test_clean_upload_reports_zero_warnings_explicitly(self): - resp = _post(TestClient(_build_app()), _csv([["TLC1", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF"]])) + resp = _post(TestClient(_build_app()), _csv([["TLC1", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF", "Packhouse A"]])) assert resp.status_code == 200, resp.text body = resp.json() # An integrator should be able to read the field unconditionally @@ -152,7 +153,7 @@ def test_clean_upload_reports_zero_warnings_explicitly(self): assert body["kde_warning_count"] == 0 def test_base_fields_unchanged(self): - resp = _post(TestClient(_build_app()), _csv([["TLC1", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF"]])) + resp = _post(TestClient(_build_app()), _csv([["TLC1", "Kale", "10", "lbs", "2026-04-17", "Farm A", "REF", "Packhouse A"]])) body = resp.json() assert body["accepted"] == 1 assert body["rejected"] == 0 diff --git a/services/ingestion/tests/test_csv_templates.py b/services/ingestion/tests/test_csv_templates.py index 1fc56c74b..499bb4c80 100644 --- a/services/ingestion/tests/test_csv_templates.py +++ b/services/ingestion/tests/test_csv_templates.py @@ -164,6 +164,7 @@ def test_returns_empty_for_complete_data(self): "harvesting", harvest_date="2026-02-26", reference_document="REF-001", + immediate_subsequent_recipient="Packhouse A", ) kdes = { "traceability_lot_code": "LOT-001", @@ -173,6 +174,7 @@ def test_returns_empty_for_complete_data(self): "harvest_date": "2026-02-26", "location_name": "Test Facility", "reference_document": "REF-001", + "immediate_subsequent_recipient": "Packhouse A", } missing = _validate_kde_completeness("harvesting", event, kdes) assert missing == [] diff --git a/services/ingestion/tests/test_csv_templates_1342.py b/services/ingestion/tests/test_csv_templates_1342.py index fad710cc9..9bdb04f6b 100644 --- a/services/ingestion/tests/test_csv_templates_1342.py +++ b/services/ingestion/tests/test_csv_templates_1342.py @@ -269,7 +269,11 @@ def test_unknown_cte_returns_empty(self): def test_all_present(self): event = _make_event() - kdes = {"harvest_date": "2026-04-17", "reference_document": "REF-1"} + kdes = { + "harvest_date": "2026-04-17", + "reference_document": "REF-1", + "immediate_subsequent_recipient": "Packhouse A", + } missing = _validate_kde_completeness("harvesting", event, kdes) assert missing == [] diff --git a/services/ingestion/tests/test_pilot_diagnostic.py b/services/ingestion/tests/test_pilot_diagnostic.py index 0f97a0fee..f67601061 100644 --- a/services/ingestion/tests/test_pilot_diagnostic.py +++ b/services/ingestion/tests/test_pilot_diagnostic.py @@ -243,7 +243,12 @@ async def review(state): for issue in missing.values(): assert issue["disposition"] == "flagged" and issue["row_indices"] == [2] assert issue["affected_row_count"] == 1 - assert {state["error_cluster"]["field"] for state in states} == set(missing) + # Contract v3 also flags missing_kde for input_lot_codes / input_product_descriptions + # on rows without those columns; only the quantity-data clusters are in scope here. + qty_states = [ + state for state in states if state["error_cluster"]["issue_reason"] == "missing_input_quantity_data" + ] + assert {state["error_cluster"]["field"] for state in qty_states} == set(missing) rendered = response.text + json.dumps(states) + caplog.text for row in rows: for field in ("traceability_lot_code", "product_description", "location_name", "reference_document"): diff --git a/services/shared/rules/seeds.py b/services/shared/rules/seeds.py index fc49fdd02..a8561533f 100644 --- a/services/shared/rules/seeds.py +++ b/services/shared/rules/seeds.py @@ -301,6 +301,7 @@ "fields": [ "kdes.harvest_location_ref", "kdes.harvest_location", + "kdes.harvest_location_name", "kdes.farm_reference", ] }, From 76620fe72f940006e5309f29dbee9165f322999c Mon Sep 17 00:00:00 2001 From: "C.Lee Sellers" <131207452+PetrefiedThunder@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:41:28 -0800 Subject: [PATCH 11/11] fix(ingest): align required-kdes.json with REQUIRED_KDES_BY_CTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop Food Safety Lens packing conditionals from the docs table — harvest_location_phone/address, growing_area_name_or_field_name, cooled, and cooling_*_when_cooled_yes. Those stay enforced in validation.py and are intentionally not listed in REQUIRED_KDES_BY_CTE; the docs drift guard requires an exact match. Signed-off-by: C.Lee Sellers <131207452+PetrefiedThunder@users.noreply.github.com> --- frontend/src/lib/api-samples/required-kdes.json | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/frontend/src/lib/api-samples/required-kdes.json b/frontend/src/lib/api-samples/required-kdes.json index d03393685..bfad38891 100644 --- a/frontend/src/lib/api-samples/required-kdes.json +++ b/frontend/src/lib/api-samples/required-kdes.json @@ -32,13 +32,7 @@ "harvest_date", "harvest_location_name", "received_date", - "received_quantity", - "harvest_location_phone", - "harvest_location_address", - "growing_area_name_or_field_name", - "cooled", - "cooling_location_name_when_cooled_yes", - "cooling_date_when_cooled_yes" + "received_quantity" ] }, "first_land_based_receiving": {