Keep rules: wallet tokens ship as sent (0.17.0, #159487)

CLAUDE.md

@@ -24,6 +24,6 @@ ECS-compliant structured logging with W3C Trace Context support. Framework-agnos
 - Django's `LogContextBinder` must be imported explicitly (not in `__init__.py`) to avoid circular imports
 - `LoggingContextMiddleware` must be placed AFTER auth middleware to capture user_id
 - CVV never carries a token, even when PII is configured — PCI forbids storing CVV in any form, so `[CVV-MASKED]` is always the literal, final output
-- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's value rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack; core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES`, each service's to list
+- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list
 - Card numbers truncate to first 6 + last 4 for 15–19 digits and last 4 only below 15 (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091) — no token alongside, independent of PII configuration, and a second masking pass is a noop
 - `install_maskers()` only sweeps handlers that exist at call time (no stdlib patching) — a handler built after the call needs another call

ecsctx/contrib/net/__init__.py

@@ -35,7 +35,9 @@
 
 from ecsctx.masking.filters import MaskPIIFilter
 from ecsctx.masking.patterns import (
+    _KEPT_HEAD,
     _XML_START,
+    KeptStash,
     _card_field_type,
     _closing,
     _element_text,
@@ -46,9 +48,11 @@
     mask_card_elements,
     mask_objects,
     mask_secret,
+    stash_kept,
     value_rules_in_force,
 )
 from ecsctx.masking.tokens import _MASKED_VALUE, _PLACEHOLDER, make_label
+from ecsctx.masking.value_rules import keep_rules
 
 _SECRET_LABEL = f"[{make_label('secret')}]"
 
@@ -411,12 +415,35 @@ def redact_body(text: str) -> str:
 
     A value a value rule in force matches (``ECSCTX_MASK_VALUE_RULES``), as
     JSON anywhere in the body or as JSON in a JSON string, is the rule's
-    label first, before a credential's rule could hash it.
+    label first, before a credential's rule could hash it -- or, for a keep
+    rule, set aside before any of these rules reads the body and put back as
+    it was sent (``patterns.stash_kept``): a credential's JSON value, an extra
+    secret key such as ``signature`` and a form value's ``=`` never read into
+    it. An element's entity-encoded JSON is read as it decodes, and a value
+    kept there is set aside with the element's text, as it was written.
     """
+    return _redact_body(text)[0]
+
+
+def _redact_body(text: str) -> tuple[str, bool]:
+    """``redact_body``'s result, and whether it set a kept value aside."""
+    stash = None
     if "{" in text and (rules := value_rules_in_force()):
         text = mask_objects(text, rules)
+        stash = stash_kept(text, rules)
+        if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text:
+            # None in the text as written; an element's entity-encoded text
+            # may hold one (_encoded_leaf).
+            stash = KeptStash(text, {})
+    if stash is None:
+        return _redact_body_text(text, None), False
+    return stash.restore(_redact_body_text(stash.text, stash)), bool(stash.originals)
+
+
+def _redact_body_text(text: str, stash: KeptStash | None) -> str:
+    """``redact_body``'s rules, once its value rules have read the body."""
     if "<" in text:
-        text = _mask_xml_elements(text)
+        text = _mask_xml_elements(text, stash)
     if "=" in text:
         text = _FORM_FIELD.sub(_mask_card_form_field, text)
     hint_re, json_re, form_re, _names = _get_compiled()
@@ -443,13 +470,16 @@ def _is_credential_element(name: str, names: frozenset[str]) -> bool:
     return _is_credential_key(name) or name.lower() in names
 
 
-def _mask_xml_elements(text: str) -> str:
+def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str:
     """Each element named as a card, CVV or SAD key is, its text masked by
     that type, as the text rule masks one (`mask_card_elements`) -- first, so
     a CVV inside a credential's element is its label, not part of the text
     the credential's token hashes. Then each element named as a credential
     key is, its text masked as a credential; and entity-encoded text in any
-    other leaf, through the body rules as it decodes."""
+    other leaf, through the body rules as it decodes -- set aside in
+    ``stash`` when it holds a kept value, so the form and JSON rules that
+    read the body next never read into it (base64's `==` after a run holding
+    `cvv` reads as a CVV form field)."""
     text = mask_card_elements(text)
     names = _get_compiled()[3]
     parts, copied, ends = [], 0, None
@@ -464,7 +494,10 @@ def _mask_xml_elements(text: str) -> str:
             continue
         end = closing.start()
         if not _is_credential_element(start.group("local"), names):
-            if (masked := _encoded_leaf(text, start.end(), end)) is not None:
+            if (leaf := _encoded_leaf(text, start.end(), end)) is not None:
+                masked, kept = leaf
+                if kept and stash is not None:
+                    masked = stash.hold(masked)
                 parts += [text[copied : start.end()], masked]
                 copied = end
             continue
@@ -479,23 +512,24 @@ def _mask_xml_elements(text: str) -> str:
     return "".join(parts)
 
 
-def _encoded_leaf(text: str, start: int, end: int) -> str | None:
+def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None:
     """The text of a leaf, ``text[start:end]``, masked by the body rules as
     it decodes and escaped again -- when it is JSON or XML written with
-    entities, and masking changed it. A leaf's raw text needs nothing here,
-    nor a form body's: the body rules read them where they stand, `&amp;` a
-    form's separator either way. Decoded, a form value ran on past `&quot;`,
-    and a leaf the first pass left as a form body read differently on the
-    next."""
+    entities, and masking changed it or kept a value in it (then as written,
+    if nothing else changed) -- and whether it kept one. A leaf's raw text
+    needs nothing here, nor a form body's: the body rules read them where
+    they stand, `&amp;` a form's separator either way. Decoded, a form value
+    ran on past `&quot;`, and a leaf the first pass left as a form body read
+    differently on the next."""
     if text.find("<", start, end) != -1 or text.find("&", start, end) == -1:
         return None
     decoded = html.unescape(text[start:end])
     if decoded.lstrip()[:1] not in ("{", "[", "<"):
         return None
-    masked = redact_body(decoded)
+    masked, kept = _redact_body(decoded)
     if masked == decoded:
-        return None
-    return xml_escape(masked)
+        return (text[start:end], True) if kept else None
+    return xml_escape(masked), kept
 
 
 def _mask_card_form_field(match: re.Match) -> str:

ecsctx/contrib/ottu/masking.py

@@ -1,6 +1,6 @@
 """Ottu's masking vocabulary: key names Ottu services log that are not PII
 (``SAFE_KEYS``, for ``ECSCTX_MASK_SAFE_KEYS``), and the wallet tokens they
-carry, masked by shape (``WALLET_RULES``, for ``ECSCTX_MASK_VALUE_RULES``).
+carry, found by shape (``WALLET_RULES``, for ``ECSCTX_MASK_VALUE_RULES``).
 
 ecsctx's own safe keys hold only names that mean the same in every service. These
 are Ottu's vocabulary, where a generic key rule would mask a value that is not
@@ -14,13 +14,38 @@
 so a service pinned to an older ecsctx keeps working, and because listing a key
 that is already safe costs nothing.
 
-An Apple Pay or Google Pay token carries the device card number and the
-payment cryptogram, encrypted: Sensitive Authentication Data, the SAD label
-under any key and in any text, never hashed. Which shapes those are is Ottu's
-knowledge, not ecsctx's: core names none, and masks one only as a service's
-value rule (``ecsctx.masking.value_rules``). ``WALLET_RULES`` are Ottu's.
-
-A service opts in to both from its settings::
+Wallet tokens ship in logs exactly as sent (#159487, decided 2026-10-01). An
+Apple Pay or Google Pay token is single-use: it pays once, and what it carries
+-- the device card number and the payment cryptogram -- is encrypted for the
+payment processor. When a wallet payment fails, the token as it was sent is
+what tells a wrong merchant certificate, an expired signing key or a token
+altered on the way apart. So ``WALLET_RULES`` are keep rules
+(``ecsctx.masking.value_rules``): a token they match is left as sent under any
+key -- ``paymentData``, TAP's ``token_data``, MPGS's ``paymentToken`` (a JSON
+string), Google Pay's ``tokenizationData.token``, KPay's ``<udf9>`` -- and in
+a message or a body, and no rule reads into it: not a credential rule, not the
+payment-id rule (its ``transactionId``), not a service's extra secret keys
+(``signature``).
+
+What stays masked:
+
+- Saved-card tokens. A card on file is charged again with its token, so it is
+  a credential: no wallet shape matches it, under ``token`` or anywhere.
+- What is not exactly a wallet's shape. The matchers are strict -- an Apple
+  Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method
+  token, and no key besides the ones they write -- so anything else is walked
+  as before: a ``PKPaymentToken`` with a key of its own keeps only its
+  ``paymentData``.
+- Card data. ecsctx keeps nothing under a CVV or SAD key, and nothing that
+  holds a card, CVV or SAD key or a card number (the floor and the guard).
+
+Samsung Pay is not here: its token is a JWE, and no captured token pins its
+shape yet. A rule for any JWE would keep every JWE.
+
+``WALLET_SAD_RULES`` are 0.16.0's: label rules that make the same tokens'
+payment data ``[SAD-MASKED]``, for a service that must not log them.
+
+A service opts in to both lists from its settings::
 
     from ecsctx.contrib.ottu.masking import SAFE_KEYS as OTTU_SAFE_KEYS, WALLET_RULES
 
@@ -36,7 +61,7 @@
 
 from typing import Any
 
-from ecsctx.masking.value_rules import ValueRule
+from ecsctx.masking.value_rules import KeepRule, ValueRule
 
 SAFE_KEYS = frozenset({
     # Ottu PG API fields: a gateway's short name ("mpgs") and flags saying
@@ -88,15 +113,16 @@
 })
 
 
-# Apple Pay's PKPaymentToken `paymentData`: its `version`, and the encrypted
-# `data`. Its `signature` and `header` are useless without `data`, so the whole
-# object is the label, and an unrelated `paymentData` reads through.
+# The versions of Apple Pay's PKPaymentToken `paymentData`.
 _APPLE_PAY_VERSIONS = frozenset({"EC_v1", "RSA_v1"})
-# Google Pay's payment method token: its `protocolVersion` and the
-# `signedMessage` holding the encrypted payload.
+# ...and of Google Pay's payment method token.
 _GOOGLE_PAY_VERSIONS = frozenset({"ECv1", "ECv2", "ECv2SigningOnly"})
 
 
+# 0.16.0's label rules. `paymentData`'s `version` and the encrypted `data`:
+# its `signature` and `header` are useless without `data`, so the whole object
+# is the label, and an unrelated `paymentData` reads through. Google Pay's
+# `protocolVersion` and the `signedMessage` holding the encrypted payload.
 def _is_apple_pay_token(value: Any) -> bool:
     version = value.get("version")
     return isinstance(version, str) and version in _APPLE_PAY_VERSIONS and "data" in value
@@ -109,7 +135,91 @@ def _is_google_pay_token(value: Any) -> bool:
 
 APPLE_PAY_RULE = ValueRule("sad", _is_apple_pay_token, hints=("EC_v1", "RSA_v1"))
 GOOGLE_PAY_RULE = ValueRule("sad", _is_google_pay_token, hints=("ECv1", "ECv2"))
+
+
+# The keep rules' shapes, key for key: what Apple's PKPaymentToken and Google
+# Pay's payment method token are documented to hold, and nothing else. A key
+# besides these is no wallet's, and the object is walked as before.
+_PAYMENT_DATA_KEYS = frozenset({"version", "data", "signature", "header"})
+_PAYMENT_DATA_HEADER_KEYS = frozenset(
+    {"ephemeralPublicKey", "wrappedKey", "publicKeyHash", "transactionId", "applicationData"}
+)
+_PK_PAYMENT_TOKEN_KEYS = frozenset({"paymentData", "paymentMethod", "transactionIdentifier"})
+_PAYMENT_METHOD_KEYS = frozenset({"displayName", "network", "type"})
+_GOOGLE_PAY_KEYS = frozenset({"signature", "intermediateSigningKey", "protocolVersion", "signedMessage"})
+_SIGNING_KEY_KEYS = frozenset({"signedKey", "signatures"})
+
+
+def _text_or_absent(value: Any, key: str) -> bool:
+    """A wallet's field is text where it is written: every leaf a wallet
+    writes is a string, so a container or a number in a slot is no wallet's,
+    and nothing can ride in it -- a `{name, value}` pair among them."""
+    return key not in value or isinstance(value[key], str)
+
+
+def _is_apple_pay_payment_data(value: Any) -> bool:
+    """Apple Pay's `paymentData`: an EC_v1 or RSA_v1 `version`, the
+    encrypted `data` and its `signature` as text, and a `header` of text
+    fields."""
+    version = value.get("version")
+    if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and isinstance(value.get("data"), str)):
+        return False
+    if not (value.keys() <= _PAYMENT_DATA_KEYS and _text_or_absent(value, "signature")):
+        return False
+    header = value.get("header", {})
+    return (
+        isinstance(header, dict)
+        and header.keys() <= _PAYMENT_DATA_HEADER_KEYS
+        and all(isinstance(field, str) for field in header.values())
+    )
+
+
+def _is_pk_payment_token(value: Any) -> bool:
+    """Apple Pay's PKPaymentToken: its `paymentData`, and what the device says
+    about the card -- its display name, network and type -- and the
+    transaction's id, each as text."""
+    payment_data = value.get("paymentData")
+    if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)):
+        return False
+    if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _text_or_absent(value, "transactionIdentifier")):
+        return False
+    method = value.get("paymentMethod", {})
+    return (
+        isinstance(method, dict)
+        and method.keys() <= _PAYMENT_METHOD_KEYS
+        and all(isinstance(field, str) for field in method.values())
+    )
+
+
+def _is_google_pay_payment_token(value: Any) -> bool:
+    """Google Pay's payment method token: its `protocolVersion`, the
+    `signedMessage` and its `signature` as text, and the intermediate signing
+    key ECv2 signs with: a `signedKey` as text and its `signatures`, a list
+    of text."""
+    protocol = value.get("protocolVersion")
+    if not (isinstance(protocol, str) and protocol in _GOOGLE_PAY_VERSIONS):
+        return False
+    if not isinstance(value.get("signedMessage"), str):
+        return False
+    if not (value.keys() <= _GOOGLE_PAY_KEYS and _text_or_absent(value, "signature")):
+        return False
+    signing_key = value.get("intermediateSigningKey", {})
+    if not (isinstance(signing_key, dict) and signing_key.keys() <= _SIGNING_KEY_KEYS):
+        return False
+    signatures = signing_key.get("signatures", [])
+    return (
+        _text_or_absent(signing_key, "signedKey")
+        and isinstance(signatures, list)
+        and all(isinstance(signature, str) for signature in signatures)
+    )
+
+
+APPLE_PAY_TOKEN_KEEP_RULE = KeepRule(_is_pk_payment_token, hints=("EC_v1", "RSA_v1"))
+APPLE_PAY_KEEP_RULE = KeepRule(_is_apple_pay_payment_data, hints=("EC_v1", "RSA_v1"))
+GOOGLE_PAY_KEEP_RULE = KeepRule(_is_google_pay_payment_token, hints=("ECv1", "ECv2"))
 # Found by shape, never by a key name: under `paymentData`, TAP's `token_data`,
 # MPGS's `paymentToken` (a JSON string), Google Pay's `tokenizationData.token`,
-# KPay's `<udf9>`, or in a message.
-WALLET_RULES = (APPLE_PAY_RULE, GOOGLE_PAY_RULE)
+# KPay's `<udf9>`, or in a message. The whole PKPaymentToken first, so it is
+# kept whole where it is one.
+WALLET_RULES = (APPLE_PAY_TOKEN_KEEP_RULE, APPLE_PAY_KEEP_RULE, GOOGLE_PAY_KEEP_RULE)
+WALLET_SAD_RULES = (APPLE_PAY_RULE, GOOGLE_PAY_RULE)

ecsctx/masking/__init__.py

@@ -12,7 +12,9 @@
 for choosing packs (configure_masking_packs, ECSCTX_MASKING_PACKS), a
 service's own safe key names (configure_masking_safe_keys,
 ECSCTX_MASK_SAFE_KEYS), its value rules (configure_masking_value_rules,
-ECSCTX_MASK_VALUE_RULES; ecsctx.masking.value_rules) and skip paths, and
+ECSCTX_MASK_VALUE_RULES; ecsctx.masking.value_rules: ValueRule labels a
+shape, KeepRule ships it as sent -- is_kept and mask_outside_kept let a
+service's own pass after ecsctx's leave a kept value alone) and skip paths, and
 ecsctx.masking.exemptions for the configure_masking() path-exemption API.
 """
 
@@ -32,9 +34,16 @@
 )
 from ecsctx.masking.filters import MaskPIIFilter
 from ecsctx.masking.install import install_maskers, uninstall_maskers
-from ecsctx.masking.patterns import ALL_PACKS, PACK_NAMES, mask_card_value, mask_secret
+from ecsctx.masking.patterns import (
+    ALL_PACKS,
+    PACK_NAMES,
+    is_kept,
+    mask_card_value,
+    mask_outside_kept,
+    mask_secret,
+)
 from ecsctx.masking.tokens import mask_by_field_type, safe_tokenize
-from ecsctx.masking.value_rules import ValueRule
+from ecsctx.masking.value_rules import KeepRule, ValueRule
 
 __all__ = [
     "ALL_PACKS",
@@ -46,6 +55,9 @@
     "configure_masking_value_rules",
     "get_masking_value_rules",
     "ValueRule",
+    "KeepRule",
+    "is_kept",
+    "mask_outside_kept",
     "MaskPIIFilter",
     "install_maskers",
     "uninstall_maskers",