diff --git a/CLAUDE.md b/CLAUDE.md index 72ae4f2..cbbe1b3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,6 @@ ECS-compliant structured logging with W3C Trace Context support. Framework-agnos - Django's `LogContextBinder` must be imported explicitly (not in `__init__.py`) to avoid circular imports - `LoggingContextMiddleware` must be placed AFTER auth middleware to capture user_id - CVV never carries a token, even when PII is configured — PCI forbids storing CVV in any form, so `[CVV-MASKED]` is always the literal, final output -- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's value rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack; core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES`, each service's to list +- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list - Card numbers truncate to first 6 + last 4 for 15–19 digits and last 4 only below 15 (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091) — no token alongside, independent of PII configuration, and a second masking pass is a noop - `install_maskers()` only sweeps handlers that exist at call time (no stdlib patching) — a handler built after the call needs another call diff --git a/README.md b/README.md index 6ba001e..1cfb1ff 100644 --- a/README.md +++ b/README.md @@ -863,7 +863,7 @@ receives the CVV a saved-card payment sends, so the keyed CVV rules are | Pack | Content rules | On by default | |------|---------------|---------------| -| `default` | PEM keys, a JSON object a service's [value rule](#value-rules-a-services-own-shapes) matches (its label), credentials (`token=…`, `"secret": …`, `Bearer …`, `API-Key …`, `…`, `password=['a', 'b']`), keyed CVV (`cvv=123`, `"securityCode": "123"`, `CVV 123`, `vpc_CardSecurityCode=123`, `paymentCvv=123`), an XML element named as a card, CVV or SAD key (`123`, `1234`, `…` truncated), a URL's userinfo, phone numbers, emails, JWTs | always | +| `default` | PEM keys, a JSON object a service's [value rule](#value-rules-a-services-own-shapes) matches (a label rule's label; what a [keep rule](#keep-rules) ships is set aside before every rule and put back as written), credentials (`token=…`, `"secret": …`, `Bearer …`, `API-Key …`, `…`, `password=['a', 'b']`), keyed CVV (`cvv=123`, `"securityCode": "123"`, `CVV 123`, `vpc_CardSecurityCode=123`, `paymentCvv=123`), an XML element named as a card, CVV or SAD key (`123`, `1234`, `…` truncated), a URL's userinfo, phone numbers, emails, JWTs | always | | `pci` | PANs (truncated), a bare 3–4 digit CVV beside a card number or its truncation (`4111111111111111 123`, `411111******1111\|12/27\|123`) or after a CVV word (`the cvv is 123`) | no | | `financial_ids` | IBANs, SSNs, payment/transaction/auth ids (content and key names) | no | @@ -1014,7 +1014,12 @@ from ecsctx.contrib.net import ( A value a service's [value rule](#value-rules-a-services-own-shapes) matches is its label before anything else reads it, as JSON anywhere in the body — an XML element's text included — or as JSON in a JSON - string. + string. What a [keep rule](#keep-rules) ships is then set aside, and put + back as it was sent once the body rules have read the rest: a + credential's JSON value (`"token": "{\"version\"…}"`), an extra secret + key such as `signature` and a form value never read into it. An + element's entity-encoded JSON is read as it decodes, and an element + whose text keeps a value is set aside as it was written. - `redact_url(url, secrets=[token])` also masks literal values anywhere in the URL, longest first — a saved-card token in a path such as `/card//`. @@ -1031,6 +1036,10 @@ from ecsctx.contrib.net import ( - `loggable_request_body(data, json_body)` — the same for the outbound half (`json_body` wins over form `data`); never raises. +Both cap what they return at `ECSCTX_REDACT_BODY_LOG_CAP` characters, 4096 by +default. That cuts an Apple Pay token, roughly 5 KB, short: a service that +logs wallet tokens as sent ([keep rules](#keep-rules)) raises the cap. + Every credential these helpers mask is masked as `mask_secret` masks one: its token where PII tokenization is configured, `[SECRET-MASKED]` where it is not (and for a card-shaped value or a placeholder), so it carries the token the @@ -1117,7 +1126,7 @@ safe keys (the expiry spellings in the whitelist below). | **PANs** | `card`, `pan`, `card_number`, `cardNumber`, `card_no`, `card_num` (MIGS's `vpc_CardNum`) | 12–19 digit runs (`pci`); an XML element so named (`default`) | `411111******1111` | | **CVV** | containing `cvv`, `cvc`, `security code`, `verification value`, or the words `csc`, `cvd`, `cvn`, `card code` — unless what follows names something *about* one (`cvv_required`, `cvvResult`, `cardSecurityCodeError`) | keyed CVV and an XML element so named (`default`), a bare 3–4 digit group beside a card number or its truncation, or after a CVV word (`pci`) | `[CVV-MASKED]` | | **SAD** | track data (`track2`, `trackData`, `raw_track`; not `track_id`), `pin`/`pinBlock`, EMV/chip data, and ending in `cryptogram`, `cavv`, `tavv`, `aav`, `ucaf` | an XML element so named (`default`) | `[SAD-MASKED]` | -| **A service's value rules** | none: found by shape under any key, as a mapping or as JSON text ([value rules](#value-rules-a-services-own-shapes)) | the same shapes as JSON in text, or in a JSON string (`default`) | the rule's label, the whole value | +| **A service's value rules** | none: found by shape under any key, as a mapping or as JSON text ([value rules](#value-rules-a-services-own-shapes)) | the same shapes as JSON in text, or in a JSON string (`default`) | a label rule's label, the whole value; a [keep rule](#keep-rules)'s match as sent | | **National ids** | `civil_id`, `national_id`, `passport`, `iqama`, `qid`, `cpr`, `nid`, `emirates_id`, `ssn`, `tin`, `tax_id`, `aadhaar`, `id_number` | — | `[SSN-MASKED…]` | | **IBAN / SSN / payment ids** | `payment_id`, `transaction_id`, `auth_id` (`financial_ids`) | `financial_ids` | `[IBAN-MASKED…]`, … | @@ -1131,9 +1140,10 @@ are destroyed — collapsing it threw away the one form PCI DSS 3.5.1 permits us to keep. A value a service's **value rule** matches is that rule's label whole, in -every pack and under any key, never hashed; it is found by shape, so anything -else under the same key reads as before. A value encoded into another field -(hex, base64) has no shape to find, and is left to that field's key. +every pack and under any key, never hashed — or, for a keep rule, the value as +sent; it is found by shape, so anything else under the same key reads as +before. A value encoded into another field (hex, base64) has no shape to find, +and is left to that field's key. Since 0.14.0 a **credential, CVV or SAD** key holding a container is walked as well, because it cannot be holding the value itself: a leaf with no rule of its @@ -1196,9 +1206,13 @@ or SAD key, as `redact_body` masks it: `123` is `` truncated — a test card that fails Luhn too, with or without `pci` — and a card object is read field by field. A CVV inside a credential's element is its label before the credential is masked, as under a credential -key. A JSON object a service's value rule matches, written in text or as +key. A JSON object a service's label rule matches, written in text or as JSON in a JSON string, is the rule's label before any credential rule reads -it. +it. One a keep rule matches is set aside before every rule and put back as it +was written: `token={…}`, `"token": {…}`, `Bearer {…}` and +`{…}` keep it, the payment-id rule never reads its +`"transactionId"`, and none is kept right after a CVV or SAD key +(`{…}`, `"securityCode": {…}`). A URL's userinfo in text — a DSN in an exception, `postgresql://user:password@db:5432/app` — is masked part by part as @@ -1345,15 +1359,16 @@ a payment method's encrypted token, say — and which shapes those are is a service's to say: ecsctx names none. A service lists its value rules, and the engine asks them about every mapping the key walk meets (under any key), every string that is JSON text, and every JSON object written in free text or in a -body `redact_body` masks, JSON in an XML element's text included. A matching -value becomes the rule's label, in every pack: never hashed, and left as it is -by a later pass. With none configured, none of this runs. +body `redact_body` masks, JSON in an XML element's text included. A value a +label rule matches becomes the rule's label, in every pack: never hashed, and +left as it is by a later pass; one a [keep rule](#keep-rules) matches ships as +sent. With none configured, none of this runs. -A rule is any object with a `field_type` (the label it becomes: `"sad"` is -`[SAD-MASKED]`), `matches(value)` — called with a mapping, JSON text parsed +A label rule is any object with a `field_type` (the label it becomes: `"sad"` +is `[SAD-MASKED]`), `matches(value)` — called with a mapping, JSON text parsed once — and, optionally, `hints`: literal strings a matching value's text holds -at least one of, so text holding none is never parsed to ask. `ValueRule` is -one: +at least one of, so text holding none is never parsed to ask that rule. +`ValueRule` is one: ```python from ecsctx.masking import ValueRule @@ -1379,20 +1394,134 @@ configure_masking_value_rules([VAULT_BLOB]) An item that does not import or is not a rule makes `configure_masking_value_rules` raise; from the setting or env var it is dropped with a warning and fails the Django boot check, and the others still -apply. A rule that raises leaves the record as `[MASKING-FAILED: …]`, never -an exception out of the log call. - -Ottu's are `ecsctx.contrib.ottu.masking.WALLET_RULES`: an Apple Pay token's -`paymentData` (`version` `EC_v1` or `RSA_v1`, with the encrypted `data`) and -a Google Pay payment method token (`protocolVersion` `ECv1`, `ECv2` or -`ECv2SigningOnly`, with a `signedMessage`) are `[SAD-MASKED]` whole — under -`paymentData`, TAP's `token_data`, MPGS's `paymentToken` (a JSON string), -Google Pay's `tokenizationData.token`, KPay's ``, or in a message. An -Ottu service lists them in `ECSCTX_MASK_VALUE_RULES` next to its -`ECSCTX_MASK_SAFE_KEYS` (above), or sets -`ECSCTX_MASK_VALUE_RULES=ecsctx.contrib.ottu.masking.WALLET_RULES`. Core -masks no wallet token of its own accord: without them, one is read as any -other value is. +apply. A label rule that raises leaves the record as `[MASKING-FAILED: …]`, +never an exception out of the log call. + +The known-clean set the second pass reads is forgotten whenever the value +rules are configured (`configure_masking_value_rules`), so a string masked +under one rule set never comes back as it was under another. + +#### Keep rules + +A **keep rule** ships what it matches exactly as sent: a value a service must +be able to read in its logs, found by shape. It is any object whose `keep` is +`True` (the object itself, not a truthy value), with `matches(value)` and, +optionally, `hints`; it has no `field_type`, since it writes no label. +`KeepRule` is one: + +```python +from ecsctx.masking import KeepRule + +def is_sealed_box(value): + return value.get("kind") == "sealed-box" and set(value) <= {"kind", "box"} + +SEALED_BOX = KeepRule(is_sealed_box, hints=("sealed-box",)) + +ECSCTX_MASK_VALUE_RULES = ["myservice.masking.VAULT_BLOB", "myservice.masking.SEALED_BOX"] +``` + +Keep and label rules go in the one `ECSCTX_MASK_VALUE_RULES` list, and are +asked where label rules are: every mapping the key walk meets, every string +that is JSON text, every JSON object in free text or in a body `redact_body` +masks. They are asked in list order and **the first match wins**: a label +rule listed first labels a value a keep rule would have kept, and an object a +label rule matches is not looked into for one to keep. A keep match the guard +refuses (below) is walked as if nothing had matched: no rule listed after it +is asked, a label rule for the same shape included. A kept value is left +as sent under any key and not descended into — a credential's key, a card's, +or a PII container's above it does not stop it (`token`, `paymentToken`, +`password`, `card.devicePayment`, `customer`): what the rule names is the +value itself, not something the key says about it. In text, what a keep rule +ships is set aside as a `[KEPT--MASKED]` placeholder before every +content rule, and put back as it was written after; a placeholder a rule +destroys — a credential hashed with what was around it — takes the value with +it. Text that already holds `[KEPT-` is masked as if no keep rule were +configured. A keep rule that raises counts as no match: keep rules are asked +on paths that must never raise (`redact_url`, `redact_body`, +`mask_card_value`). + +Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: + +- **The floor.** Nothing is kept under a CVV or SAD key (`cvv`, + `securityCode`, `pin`, `cryptogram`, …), inside a CVV or SAD container + (a key the service lists there included), or as the `value` of a pair + labelled as one; in text, right after such a quoted key, `key=` or XML + start tag. Such a value is masked exactly as without the rule. +- **The guard.** A match that holds, at any depth — JSON text inside it + included — a key the key rules read as a card, a CVV or other SAD, a + `{name, value}` pair whose identifier reads as one (as the key walk reads a + pair), or a leaf that is a card number, is walked as if nothing had + matched. It reads keys, pairs and whole leaves only: a CVV or a card number + written inside a longer string of a kept value (`"note": "cvv=123"`, Track + 2 data, `"card 4111…"`) ships with it. Keeping that out is the matcher's + job, which is why a matcher must be strict about what it matches. A leaf is + a card number when it is an int `int_is_pan` reads as one, or a string + `pan_shaped` reads as one, whatever its prefix and Luhn or not — except + thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's + `keyExpiration`, which `pan_shaped` reads as a card number and every + Google Pay token carries. A leaf that is no JSON value counts as card + data. + +Nothing else in a kept value is read: PII or a credential inside it, under a +key of its own or written in a string, ships with it. + +JSON text under a key that would hash it whole — a credential's — and that +holds a kept value deeper is still hashed whole: a keep never cuts a value out +of a string. A kept mapping is a copy: the record is masked in place. + +A keep decision rests on the text as written, since that is what ships: JSON +that names a key twice in one object (`json.loads` keeps the last) is not +kept, and a JSON-text leaf naming one inside a match is card data to the +guard; a Python repr is kept only where its parse renders back to it exactly +(a comment, or a key written twice, is dropped by the parse). + +A service that masks again after ecsctx — Connect does — asks +`is_kept(value, key=None)` (a mapping or JSON text; the rules in force, first +match, the guard, and the floor on `key`; False with no keep rule configured) +before masking a field, and runs its text masker through +`mask_outside_kept(text, mask)`, which hands `mask` the text with each kept +value set aside and puts back each placeholder it leaves. Both are in +`ecsctx.masking` and never raise. + +In text the floor reads the key right before an object, and every object +the text rules read whole inside it: they read three levels of nesting, so a +kept value two keys below a CVV key (`{"cvv": {"x": {"y": {…}}}}`, the value +itself nested) is not floored there — the guard still holds. An object right +after a CVV key in text is not kept, but no `default` content rule masks a +JSON object there either, with or without keep rules; the key walk masks one +under a CVV key leaf by leaf. + +The cost: with no keep rule configured none of this runs — a string with no +`{` pays one check, one with a `{` a look at the rules in force; with keep +rules alone, the label rule (rule 2) does not run. Text holding none of a keep +rule's hints never asks it; text holding one is read once for the objects in +it. Every mapping the key walk meets asks every rule, once — a mapping has no +text to hint with — so a matcher must be cheap: test a literal field first. +JSON text under a key asks, and its key walk passes on, only the rules whose +hints it holds. On the benchmark's lines (`scripts/bench_masking.py`) 0.17.0 +masks within a few percent of 0.16.0, the run-to-run noise, with no value rule +and with Ottu's `WALLET_RULES` (three keep rules where 0.16.0 had two label +rules): the key walk now asks each mapping once, where it asked twice. + +Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or +Google Pay token is single-use, and the token as it was sent is what debugs a +wallet payment. `ecsctx.contrib.ottu.masking.WALLET_RULES` are keep rules +with strict matchers — an Apple Pay `PKPaymentToken`, its `paymentData` +(`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header` of text +fields) and a Google Pay payment method token (`protocolVersion` `ECv1`, +`ECv2` or `ECv2SigningOnly`, with a `signedMessage`), and no key besides the +ones each writes — found under `paymentData`, TAP's `token_data`, MPGS's +`paymentToken` (a JSON string), Google Pay's `tokenizationData.token`, KPay's +``, or in a message. A `PKPaymentToken` with a key of its own keeps +only its `paymentData`. Saved-card tokens stay masked: no wallet shape +matches one. Samsung Pay's token, a JWE, is not matched yet. An Ottu service +lists them in `ECSCTX_MASK_VALUE_RULES` next to its `ECSCTX_MASK_SAFE_KEYS` +(above), or sets +`ECSCTX_MASK_VALUE_RULES=ecsctx.contrib.ottu.masking.WALLET_RULES`. +`WALLET_SAD_RULES` are 0.16.0's label rules, the same tokens' payment data +`[SAD-MASKED]` whole, for a service that must not log them. Core keeps and +labels no wallet token of its own accord: without either list, one is read as +any other value is. ### Path exemptions diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 2dc9e68..986c051 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -14,10 +14,10 @@ Core structured logging module with ECS field mapping and PII/PCI masking. - `PRIMARY_KEYS` in processors.py defines which fields stay at root vs get pushed to `extra` - Masking has two entry points, both driving the same `MaskPIIFilter`: the filter on every handler `get_logging_config()` builds (masks the record in place — Sentry's logging integration, `handleError` and handlers that never call `format()` read the record) and `mask_sensitive_data` (structlog processor, in the formatter's own chain). The second pass is cheap: `mask_by_patterns` remembers strings already known clean, and a known-clean string is not re-parsed. What a rule set derives -- its known-clean set, `scalar_rules`, its distinct gates -- is cached on the tuple's identity (`patterns._by_identity`), never on its hash: hashing a tuple of rules hashes every compiled pattern's whole program, on every call (30 µs a string for the default pack; lru_cache paid it on every lookup, the 0.15.5 slowdown, #160054). A string that is a JSON object or list (≤ `JSON_PARSE_LIMIT`, 64 KiB) is parsed and walked with the key rules only, re-serialised only if a key rule changed something, then content-scanned once as text — so a JSON number holding a PAN still masks, as an unquoted marker. Never scanned (`DEFAULT_SKIP_KEYS`, `masking/filters.py`): `service`/`project`/`log` and the correlation ids `session_id`, `trace`, `span`. `user.name` is exempt from the name key rule but content-scanned. The record marker stores the packs that masked it, so a filter with more packs still masks. A record `getMessage()` could not render once masked -- a %-style template masking changes (`password=%s`: rule 7 reads the placeholder as the credential), a msg that is not a str (a lazy translation, an exception; `_template_of` is `str(msg)`, bytes decoded as masking decodes them (as `str(bytes)` it never matched the masked template, so every bytes record lost its args), None for a dict or list, a structured record, whose args, if any, stay as they are -- logging cannot format it, so as in stdlib the line is dropped and `handleError` prints the args, masked (README says so)), a number masking turned into text under `%d` (`_as_masked_text` reads that conversion as `%s`, keeping its key, `-`, width and `*`s, so the masked text renders where the number stood; rendered from the raw number, a card number glued to a word stays whole; it finds the conversion by counting them as CPython's `%` reads a template (`_CONVERSION`: `%%`, a key to its balancing parenthesis, flags, width, a precision whose digits may be none -- `%.f` -- one length modifier), else the count slipped and the record was the marker), args that never fit, or a line that masks differently from its masked args rendered (a value split between template and arg, `cvv=%s` with `123`) -- is rendered from the masked args and masked whole, its args `()` (`MaskPIIFilter._mask_rendered`), or is the `[MASKING-FAILED: …]` marker: before, `getMessage()` raised, the handler dropped the line and `handleError` printed the args to stderr. It renders from the masked args, not the raw ones, so a dict arg is still masked by its keys (from the raw ones a card number under `card_number` reached only the text rules, and the default pack has no card rule). A template masking leaves as it is, that renders, and whose masked args render a line with nothing more to mask, keeps its args, and Sentry's grouping by it. Rendered, a record keeps no args for a later handler to mask with more packs, so the render path masks them with every pack (`ALL_PACKS`, only for a record that takes it): a default handler ahead of a pci one rendered `ref4111…` whole, glued to a word, for the pci handler to log, in a pci process and in a default one. A record that keeps its args is masked piece by piece, and the line its masked args render holds nothing more to mask: readers besides the formatter (Sentry's logging integration, `handleError`, a handler that never calls `format()`) render it themselves, and a value split between template and arg (`cvv=%s` with `123`) no longer reaches them as `cvv=123` -- that record takes the render path (`install.py`'s docstring says so). The check renders and masks the line once per record with args; the formatter's `mask_sensitive_data` masks the same line again, a known-clean lookup when it matches. - Two independent detection strategies inside `MaskPIIFilter`: 24 ordered content rules (`masking/patterns.py::RULES` — order is load-bearing, see the comments above `_RULE_TABLE`), grouped into packs (`default` always on, the value rule 2, the XML card, CVV and SAD element 3 and credential element 4, the `API-Key` scheme 6, the credential container 8 and the keyed CVV rules 9, 10 and 14 among its own -- their keyword the key rule's CVV words, after `_`/`-`, `card` or a camelCase word (`vpc_CardSecurityCode`, `paymentCvv`), ending the key -- a CVV must not ship from any service, and Connect receives the saved-card CVV; `pci` -- the card rule and the bare-digit CVV rules, 23 beside a card number or its truncation and 24 after a CVV word -- and `financial_ids` opt-in via `masking/config.py`), each behind a literal pre-check; the credential rules (5, 6, 7, 8, 13) and the CVV word rules (9, 10, 14, 24) are tried only near their words (`_near_words`: a match starts inside the `[\w-]` run holding its word, at most `_CRED_REACH`/`_CVV_REACH` before it, or on the quote before that run) -- `re.sub` tried every word boundary, and a long `a-a-a-…` cost 0.7 s per CVV rule; and key-name matching (`classify_key()`: substring on the lowercased key, fail closed, EXCEPT the three families where a substring made false positives -- a credential word must end the key (`scheme_token` yes, `schemeTokenProvisioningMode` no), `authorization` must be the whole key (`authorizationCode` is a payment verdict), and a `*name` key is a person's only if a person qualifier is in the name (`customer_name` yes, `domainName` no); a CVV word counts anywhere in the key unless its tail names something about one (`cvv_input` yes -- fail closed -- `cvv_required`/`cardSecurityCodeError` no), matched on the separator-free key; `track`/`magnetic` are SAD only when the rest of the key names the data (`raw_track` yes, `track_id` no), and a key ending in `cryptogram`/`cavv`/`ucaf` is SAD; a national id (`civil_id`, `passport`, `qid`, ...) is `ssn` in every pack; SAFE_KEYS is matched in both spellings, separators or camelCase; card keys matched precisely (`card`, a `pan` word, `cardnumber` in the key, or a key ending `cardno`/`cardnum` -- MIGS's `vpc_CardNum`; `card_id` is an id, not one); expiry is NOT classified and is listed in `SAFE_KEYS`; `sad` (track data, PIN blocks, EMV images, cryptograms) is destroyed outright like CVV and can never be listed safe; cached). A matching key masks its value. A scalar under a non-exemptable type is masked whole (a PAN-shaped credential is the label, never truncated); a CONTAINER under `card`, `cvv`, `sad` or `secret` is walked per leaf (`_WALKED_TYPES`) -- unnamed leaves take the container's type, under `cvv`/`sad` nothing is weaker than the container and only a service-listed key escapes, and a card-shaped object under a credential key is walked as `card` -- spelled out in `_mask_dict` rather than made exemptable, because `exemptable` also governs path exemptions and none of these may ever be whitelistable; a `{name, value}` pair masks `value` by the strictest type its identifiers name (`_pair`); a bare `name` under a container named by a thing (`name_context`) is not a person; booleans are never masked; a dataclass/namedtuple is masked by field then rendered back to repr text; the walk stops at `_MAX_DEPTH` and any exception replaces the message with `[MASKING-FAILED: …]` (a log call must never raise); a dict/list under a PII key (`customer`, `billing`, `contact`, `name`, …) keeps its shape and each field is masked on its own — by its own key's type, kept if a safe key (`id`), else as the container's type — so per-field tokens correlate across records. `exc_info`/`stack_info` are never masked (the rendered `error.*` is). Card keys truncate a PAN value and show what is not one (`mask_card_value`: under 12 digits reads through -- unless a group of digits and stars long enough to be a card number (12+) is in no truncation's shape, `_TRUNCATION_SHAPE`: the first six digits at most, four stars or more, the last four at most -- an upstream system's `123456****7890` and a first-four `4111********1111` included -- with a card number's groups -- four to six digits or stars, one card separator (`_CARD_SEP`) apart, as in 4-4-4-4, Amex 4-6-5 and Diners 4-6-4 -- read as one (`_CARD_GROUPS`), so another masker's `45087****001019` (the last six of 15), `4508-750*-****-1019` (read `4508750*****1019`, the first seven of 16) and `3782 822*** *0005` are refused, while a shorter group ends a run: `**** 1111 12 25` is not judged (`****1111` is eight) -- 12+ is content-scanned for a PAN and refused if none is found, if a run of card-number length is left beside the truncations -- joined by dots or slashes too, and a truncation's first six counting with the digits before it -- or if a Luhn-valid reading the rule leaves in free text shows more than its first six and last four, `_overexposes_a_card`; a value that the scan leaves as it is and that holds a truncation the card rule writes (`_WRITTEN_TRUNCATION`: the first six, stars and last four, or eight stars or more and the last four) -- its own output on a later pass -- is refused only for a card-length run, for twelve digits or more showing beside its truncations (`_digits_beside_truncations`) or around one (`_card_digits_showing`), or for digits and stars in any other shape (`_hides_as_truncations`: another masker's `4508750****001019` shows 13 of 17, `****1111 1234 5670` 12 of 16), and the first pass refuses that shape too, so every output is a fixed point (0.15.4 refused its own output there, so the formatter's second pass turned a shown truncation into `[CARD-MASKED]`). A PAN under ANY key is truncated rather than tokenized (`filters._mask_pii_leaf`, gated on `pci` among the call's packs and the process's, as is a bare int card number: in a pci process a default filter hashed one typed into the name box): a keyed hash beside a truncation of the same PAN is what FAQ 1117 forbids, and a card number typed into the name box used to produce exactly that. Under `pci`, a value a PII, secret, ssn or payment-id key would tokenize is its label, not a token, when it holds any 12+ digit run (`holds_pan_run`, fails closed -- a random hex id trips it 2-7% of the time, by length); only in a phone field is a `+`-led run of at most 15 digits (E.164) the phone number. An email or phone number the text rules find is judged the same way (`tokens._PAN_CHECKED_TYPES`: secret, payment_id, email, phone; a phone as written, `phone=True`), so with `pci` `4111111111111111@example.com` and `+4111111111111111` are their labels, not keyed hashes. A canonical UUID (`patterns._UUID`: 8-4-4-4-12 hex with at least one hex letter, nothing alphanumeric touching it) is the one exception to "any 12-19 digit run is a card" (the contract `tests/test_masking_false_positives.py` names): a value that is one holds no run, and the card rule leaves its digits as written in text and reads the rest of the run without them, so a card number beside it is still truncated. An all-digit string in that shape gets no exemption. -- A tokenized value is the bare token (`ptok:v1:…`); `[LABEL]` only where no token can stand (tokenization unconfigured/failing, or CVV/card, which are never tokenized; expiry is not masked at all). A null under a sensitive key stays null. `_WHOLE_TOKEN` keeps the credential text rules off a value that is already a token; like `already_tokenized` and `already_masked`, it takes only the exact shape `tokenize` emits (`ptok:v1:` + 43 base64url characters), so anything else typed after `ptok:` is masked -- and it takes none with a card-number run in its body, which `mask_by_field_type` labels. A token followed by `#` is whole: the `#` opens a URL's fragment (`redact_url`'s `?password=#access_token=…`), though `#` is otherwise part of a value. A credential value in text (rules 5, 7 and 13) runs to its delimiter, as a form value in `redact_body` does, never to the end of a fixed character class (a backslash, `@` or apostrophe left the rest in clear), and never into an end tag (`_NOT_AN_END_TAG`; a quoted value neither, `_quoted_value_body`: in `password=abc` it took `` -- never starting with a quote or an opening bracket (a container, or a label masking wrote) and never running into a label (`_NOT_A_LABEL`: a `]` would cut it, and a label redact_body wrote after a short value made it long enough to mask again); quoted, to its unescaped closing quote, spaces and all (`_quoted_body`), a `"`-quoted one masked as the JSON string it decodes to (`_mask_quoted`), so it carries the key walk's token. Rule 7 reads a quote as opening a quoted value only when a closing one matches it and something is between them -- quotes doubled as CSV and SQL escape one (`7,"password=""s3cret"""`) are structure before an unquoted value -- and the escaped quotes around a JSON-escaped key as the key's; a value that opens with `\"` (JSON in a JSON string) runs to its matching `\"` as the inner string reads it (`_ESCAPED_QUOTED_UNIT`), spaces and all, masked as it decodes twice (`_mask_escaped_quoted`); `_unquoted_value_quote` escapes the quotes it adds around an unquoted value there. After an Authorization keyword (`_AUTH_KEYWORD`: `authorization`, `Proxy-Authorization`, `authorisation_header`; rule 7 only) the value is the scheme and the credential together (`_AUTH_SCHEME`: bearer, basic, digest, token, negotiate, api-key), so `Authorization: Bearer x` carries the token `mask_secret("Bearer x")` -- the one the header gets under its key -- instead of one token for `Bearer` shared by every record and the credential left to the bare-space rule (which needs a digit: `Basic dXNlcjpwYXNz` shipped); a scheme is never the whole value while more follows it (`Bearer [REDACTED]` stays as it is), and a token after the scheme (`Bearer ptok:v1:…`, what masking the credential alone left) is not hashed again with it. Rule 6 does the same for Ottu's `API-Key` scheme word standing alone, without its `Authorization:` (not ending a header name, `X-API-Key`): the scheme and any value to its delimiter -- no digit or length guard, the word is the evidence (1 in 1,550 of Ottu PG's Fernet keys has no digit) -- are one value, `mask_secret("API-Key ")`, the header's token; it runs before rule 7, which would otherwise mask a `password=` inside the value first, and leaves a field masking wrote there (`API-Key password=`). Rule 13 leaves a value whose digits are all in masking's own output (a whole token, a label, a truncation): `Bearer password=`, which redact_body wrote, was hashed again, a token of a token; a credential glued to a key name keeps its own digits (`Token abc123abc123api_key=…`) and is still hashed. Rule 8 masks a container after a credential keyword whole (`password=['a', 'b']`, a rendered `%(pw)s`; a repr's quoted key keeps it a string), bracket-aware and nested three deep, or to the end of the line when never closed -- not after a JSON key (the key walk masked that container by its keys: the saved card under `token`), nor one holding only masking's own output (`_masked_already`, a record's field). Rule 3 masks an XML element named as the key walk names a card, CVV or SAD (`mask_card_elements`: `classify_key` on its local name, every pack on), in every pack: a CVV's or SAD's text, children and all, its label; a card's text a card value (`mask_card_value`: a test card that fails Luhn truncated without `pci`), and a card object (`` holding elements, CyberSource's) field by field -- a CVV or SAD element its label, every other element's text a card value, so `` truncates and an expiry reads through. It runs before rule 4, so a `` inside `` is its label, not part of the text the credential's token hashes. Rule 4 masks an XML element named by a credential keyword (`…`), to its end tag, children and all, as it decodes (`_element_text`: CDATA, entities); a start tag no end tag closes is no element (a route, `/v1/cards//`). Rules 3 and 4 are walks, not regexes (`_mask_elements`; the pattern hands the walk the text from its first `<`): a text's end tags are indexed once (`_end_tags`) and a start tag's found by name -- a lazy regex looked for one from every start tag, and 160 KB of `` took 12 s. Rule 2 gives a JSON object a service's value rule matches (value rules, below), written as JSON or a repr in text or as JSON in a JSON string, the rule's label, quoted where it was a JSON value; the key walk does to one as a mapping or JSON text under any key (`filters._value_label`), and to JSON text holding one deeper under a key that would hash or show it (`label_within`). Other keywords take no scheme: `password=basic more words` is the password `basic`. A credential keyword the key rule reads as a CVV or other SAD (`cvv_token`, `pin_password`, `_credential_type`) masks its value as that type, the label, as under the key and in a URL -- never a keyed hash of a CVV. Rule 17 masks a URL's userinfo in text (`scheme://user:password@`, the password possibly empty, or a token alone -- not a user alone, `ssh://git@host`, unless it is shaped like a card number or holds a card-number run: then `[SECRET-MASKED]`, as redact_url gives it), each part through `_mask_userinfo`, which `contrib.net.redact_url` shares: as it decodes, a token at its head read as the user whole (split at its first colon, pass 2 wrote `ptok:v1:ptok:v1:…`); scheme, host and port kept. It runs before phone and email: a numeric password is no phone number, and the email rule (`_email`) never starts inside masking's own output -- a token, or a userinfo masking wrote (`_mask_userinfo(userinfo) == userinfo`, e.g. redact_url's percent-encoded `%5BSECRET-MASKED%5D`) -- so the host after it survives. A credential keyword as the user (`https://x-access-token:ghp_…@github.com`) is rule 7's first, and the value it runs to takes the host with it. Path exemptions via `configure_masking()`, Django `ECSCTX_MASK_EXEMPT_PATHS`, or env `PII_MASK_EXEMPT_PATHS` (`masking/exemptions.py`, a pattern may start at any depth of the record) apply only to the exemptable field types (email/phone/address/name/generic) and only skip the key-based match — content rules still run on the value (defense in depth). Secrets (cvv/card/secret/payment_id/pem_key/iban/jwt/ssn) are never exemptable. +- A tokenized value is the bare token (`ptok:v1:…`); `[LABEL]` only where no token can stand (tokenization unconfigured/failing, or CVV/card, which are never tokenized; expiry is not masked at all). A null under a sensitive key stays null. `_WHOLE_TOKEN` keeps the credential text rules off a value that is already a token; like `already_tokenized` and `already_masked`, it takes only the exact shape `tokenize` emits (`ptok:v1:` + 43 base64url characters), so anything else typed after `ptok:` is masked -- and it takes none with a card-number run in its body, which `mask_by_field_type` labels. A token followed by `#` is whole: the `#` opens a URL's fragment (`redact_url`'s `?password=#access_token=…`), though `#` is otherwise part of a value. A credential value in text (rules 5, 7 and 13) runs to its delimiter, as a form value in `redact_body` does, never to the end of a fixed character class (a backslash, `@` or apostrophe left the rest in clear), and never into an end tag (`_NOT_AN_END_TAG`; a quoted value neither, `_quoted_value_body`: in `password=abc` it took `` -- never starting with a quote or an opening bracket (a container, or a label masking wrote) and never running into a label (`_NOT_A_LABEL`: a `]` would cut it, and a label redact_body wrote after a short value made it long enough to mask again); quoted, to its unescaped closing quote, spaces and all (`_quoted_body`), a `"`-quoted one masked as the JSON string it decodes to (`_mask_quoted`), so it carries the key walk's token. Rule 7 reads a quote as opening a quoted value only when a closing one matches it and something is between them -- quotes doubled as CSV and SQL escape one (`7,"password=""s3cret"""`) are structure before an unquoted value -- and the escaped quotes around a JSON-escaped key as the key's; a value that opens with `\"` (JSON in a JSON string) runs to its matching `\"` as the inner string reads it (`_ESCAPED_QUOTED_UNIT`), spaces and all, masked as it decodes twice (`_mask_escaped_quoted`); `_unquoted_value_quote` escapes the quotes it adds around an unquoted value there. After an Authorization keyword (`_AUTH_KEYWORD`: `authorization`, `Proxy-Authorization`, `authorisation_header`; rule 7 only) the value is the scheme and the credential together (`_AUTH_SCHEME`: bearer, basic, digest, token, negotiate, api-key), so `Authorization: Bearer x` carries the token `mask_secret("Bearer x")` -- the one the header gets under its key -- instead of one token for `Bearer` shared by every record and the credential left to the bare-space rule (which needs a digit: `Basic dXNlcjpwYXNz` shipped); a scheme is never the whole value while more follows it (`Bearer [REDACTED]` stays as it is), and a token after the scheme (`Bearer ptok:v1:…`, what masking the credential alone left) is not hashed again with it. Rule 6 does the same for Ottu's `API-Key` scheme word standing alone, without its `Authorization:` (not ending a header name, `X-API-Key`): the scheme and any value to its delimiter -- no digit or length guard, the word is the evidence (1 in 1,550 of Ottu PG's Fernet keys has no digit) -- are one value, `mask_secret("API-Key ")`, the header's token; it runs before rule 7, which would otherwise mask a `password=` inside the value first, and leaves a field masking wrote there (`API-Key password=`). Rule 13 leaves a value whose digits are all in masking's own output (a whole token, a label, a truncation): `Bearer password=`, which redact_body wrote, was hashed again, a token of a token; a credential glued to a key name keeps its own digits (`Token abc123abc123api_key=…`) and is still hashed. Rule 8 masks a container after a credential keyword whole (`password=['a', 'b']`, a rendered `%(pw)s`; a repr's quoted key keeps it a string), bracket-aware and nested three deep, or to the end of the line when never closed -- not after a JSON key (the key walk masked that container by its keys: the saved card under `token`), nor one holding only masking's own output (`_masked_already`, a record's field). Rule 3 masks an XML element named as the key walk names a card, CVV or SAD (`mask_card_elements`: `classify_key` on its local name, every pack on), in every pack: a CVV's or SAD's text, children and all, its label; a card's text a card value (`mask_card_value`: a test card that fails Luhn truncated without `pci`), and a card object (`` holding elements, CyberSource's) field by field -- a CVV or SAD element its label, every other element's text a card value, so `` truncates and an expiry reads through. It runs before rule 4, so a `` inside `` is its label, not part of the text the credential's token hashes. Rule 4 masks an XML element named by a credential keyword (`…`), to its end tag, children and all, as it decodes (`_element_text`: CDATA, entities); a start tag no end tag closes is no element (a route, `/v1/cards//`). Rules 3 and 4 are walks, not regexes (`_mask_elements`; the pattern hands the walk the text from its first `<`): a text's end tags are indexed once (`_end_tags`) and a start tag's found by name -- a lazy regex looked for one from every start tag, and 160 KB of `` took 12 s. Rule 2 gives a JSON object a service's label rule matches first (value rules, below), written as JSON or a repr in text or as JSON in a JSON string, the rule's label, quoted where it was a JSON value; the key walk does to one as a mapping or JSON text under any key (`filters._value_ruling`), and to JSON text holding one deeper under a key that would hash or show it (`label_within`). Before every rule, `mask_by_patterns` sets aside what a keep rule ships (`stash_kept`, below). Other keywords take no scheme: `password=basic more words` is the password `basic`. A credential keyword the key rule reads as a CVV or other SAD (`cvv_token`, `pin_password`, `_credential_type`) masks its value as that type, the label, as under the key and in a URL -- never a keyed hash of a CVV. Rule 17 masks a URL's userinfo in text (`scheme://user:password@`, the password possibly empty, or a token alone -- not a user alone, `ssh://git@host`, unless it is shaped like a card number or holds a card-number run: then `[SECRET-MASKED]`, as redact_url gives it), each part through `_mask_userinfo`, which `contrib.net.redact_url` shares: as it decodes, a token at its head read as the user whole (split at its first colon, pass 2 wrote `ptok:v1:ptok:v1:…`); scheme, host and port kept. It runs before phone and email: a numeric password is no phone number, and the email rule (`_email`) never starts inside masking's own output -- a token, or a userinfo masking wrote (`_mask_userinfo(userinfo) == userinfo`, e.g. redact_url's percent-encoded `%5BSECRET-MASKED%5D`) -- so the host after it survives. A credential keyword as the user (`https://x-access-token:ghp_…@github.com`) is rule 7's first, and the value it runs to takes the host with it. Path exemptions via `configure_masking()`, Django `ECSCTX_MASK_EXEMPT_PATHS`, or env `PII_MASK_EXEMPT_PATHS` (`masking/exemptions.py`, a pattern may start at any depth of the record) apply only to the exemptable field types (email/phone/address/name/generic) and only skip the key-based match — content rules still run on the value (defense in depth). Secrets (cvv/card/secret/payment_id/pem_key/iban/jwt/ssn) are never exemptable. - PII tokenization supports two providers (`PII_PROVIDER=file|vault`). File provider reads mounted keysets; Vault provider authenticates via AppRole and fetches from KV v2. Access mode (`PII_ACCESS=tokenize|full`) enforces least privilege. Auto-configures lazily from env vars. - When PII is not configured, `safe_tokenize()` (`masking/tokens.py`, re-exported from `processors`/`ecsctx`; the log-safe wrapper over `pii.tokenize`) returns `[PII_REDACTED]` — `mask_by_field_type()` catches that and falls back to `[LABEL]` rather than embedding it. That fallback is `safe_tokenize`'s documented contract; nothing else emits a fixed marker where a token can stand. -- Every path a credential takes ends in `mask_by_field_type(v, "secret")` — the key walk, the credential text rules, `contrib/django/routes.py`, `contrib/net` — and it returns `[SECRET-MASKED]`, never a token, for a PAN-shaped value (`pan_shaped`, imported inside the function: `patterns` imports `tokens` as it loads) and for a placeholder another masker left (`tokens._PLACEHOLDER`: `[REDACTED]`, `[PII_REDACTED]`, stars, `Bearer ****`), and for a truncation the card rule writes (`tokens._SHOWN_TRUNCATION`, `411111******1111` as written or decoded, or any truncation after an auth scheme, `tokens._TRUNCATION_AFTER_A_SCHEME`, `Bearer 411111******1111`, which was a token of the two: ten digits of a saved card's gateway token, which never passes through as masking's own output -- rendered from its argument under pci, `token=%s` showed them where the key walk gave the label), which would otherwise hash into one token shared by every record. With `pci` among the call's packs, and the process's (`config.packs_in_force()`, imported inside the function as well, since `config` imports `patterns`: the packs of the masking call in progress -- a `MaskPIIFilter`'s own, which every call that builds its context sets in a ContextVar with `config.call_packs` -- joined with the process's `get_masking_packs()`, so in a pci-configured process every filter labels such a value, a filter built with pci labels it in any process, and a default handler ahead of a pci one no longer hashes what the pci handler then logs), a credential, payment id, email or phone number that holds a card-number run anywhere (`holds_pan_run`; a phone's `+`-led run up to 15 digits is the number) is its label too, on every path -- the text rules, `mask_secret`, `contrib/net`, a route parameter -- as the key walk's own `pci` branch (`filters._mask_pii_leaf`, on the call's packs and the process's, `filters._pci_in_force`) makes any PII leaf; without `pci` among the call's packs and the process's it is hashed, an accepted residual: a default-pack service receives no card numbers, and the label would stand where a token can for ~7% of Connect's 64-hex HMAC signatures. Masking's own output passes through (`already_masked`) unless a card-number run is in it: `ptok:v1:` typed before a card number is in a token's exact shape, and is the label, for any type and in every pack (`tokens._CARDLESS_TOKEN` is that shape without one; `_WHOLE_TOKEN` takes only it). Both are judged on the value as `tokenize` hashes it (`normalize_value`: surrounding whitespace and one layer of matching quotes dropped), so `'"4111…"'` is not a keyed hash of the bare card number, and on it as a URL or form encoding decodes it (`unquote_plus`, as `redact_url` and `redact_body` read it), so `%224111…%22` or `4111+1111+…` in text is the label in every pack, as a `%5BSECRET-MASKED%5D` redact_url wrote is masked already; and a value that is masking's own output once normalized (a label, token or truncation in quotes, as the credential text rule leaves `password="[SECRET-MASKED]"`) is returned as it is, never hashed into one shared token. `mask_secret(value)` (`masking/patterns.py`, exported from `ecsctx.masking` and `ecsctx` with `mask_card_value`) is that for a value outside a mapping: `None`/bools pass, anything else is masked as `str(value)`. `_mask_pii_leaf` still asks `pan_shaped` for a secret before the `pci` branch, which would truncate it. `contrib/net` masks with `mask_secret` (no `[REDACTED]` since 0.15.4): `redact_url` masks literal `secrets` in one pass, longest first (a replacement is never scanned again, so a short secret can't mangle the token a longer one became), then masks the userinfo part by part (`_mask_userinfo`, a label's brackets percent-encoded: `urlsplit` refuses brackets in a netloc whose host is not IPv6, so the result would not parse again, nor in `ecs_url`) and rewrites a credential query or fragment param in place, unencoded, leaving every other param byte for byte, and rebuilding the URL only when something changed; a query, fragment or form key the key rules call a card, CVV or SAD (`classify_key(unquote_plus(key), ALL_PACKS)`, `_card_field_type`) has its value masked by that type -- `mask_card_value`, or the label, never a token -- in `redact_url` and in `redact_body`'s form pass (`_FORM_FIELD`, run on any body with an `=`, before the credential-key pre-check); `redact_body` reads a JSON value escape-aware, past any escaped character and to the end of the text when no quote closes it (a body `json.loads` rejects is the one that reaches this rule as text), and masks what it decodes to, or the raw text when it does not decode; its closing quote is looked for, not consumed, since in malformed `""password": "…"` it also opens the next key; and a form value runs to the next `&`, whitespace or end tag (`_FORM_VALUE`), as in 0.15.3 but for the tag (every reading that ended it at a quote left the rest of some secret in clear). `redact_body` first gives every value a service's value rule matches its label (`patterns.mask_objects`, rule 2's pattern), then reads an XML element as a key (`_mask_xml_elements`): its local name, whatever its namespace prefix and attributes -- first a card, CVV or SAD name, as rule 3 reads one (`patterns.mask_card_elements`: a card object field by field, so a `` inside `` is its label), then a credential name (`_is_credential_key`, or a secret body key) as a credential, the label when it holds a card-number run -- one classification across a URL's query and an XML body, so ``, ``, `` and `` are credentials, as those query params are; the text runs to the end tag, children and all, as it decodes; a start tag no end tag closes is no element; a leaf holding JSON or XML written with entities goes through `redact_body` as it decodes and is written back escaped (only those: a form body's `&` is its separator where it stands, and decoded, a form value read differently on the next pass). `_split_form_value` keeps only the structure at its two ends as written -- a quote or an escaped one (at its head, a run of them: `password=""x`, quotes doubled as CSV escapes one, and a run of two or more of either kind, `password=''x` -- a lone `'` is the value's own, a quoted value when another closes it), and `}`, `]`, `,`, a JSON key's `":`, `>`, and `/>` after a quote (after an unquoted value a `/` right after a token reads to the credential text rule as more credential, so only `>` stays) -- and takes a label, token, truncation or placeholder followed only by structure whole (`[SECRET-MASKED],` is not `[SECRET-MASKED` and `],`). The value between them is JSON-unescaped only when it sits between escaped quotes (`\"…\"`, in a JSON string; never in a raw form body), is `[SECRET-MASKED]` if it `holds_pan_run` (asked before any pass-through: a token-shaped value can carry a card number), and otherwise goes through `mask_secret`, being written back exactly as captured when that leaves it as it is (empty, or masked already). It is decoded as form encoding reads it (`unquote_plus`: `+` is a space), so outside a form body a secret holding `+` or `%xx` gets a different token than under a key; nothing leaks. The JSON rule gives every value that `holds_pan_run` the same label, closed or run on to the end of the text, before `mask_secret` (which, without `pci`, hashes one that is not in a token's shape). When its credential passes change the text, `redact_body` runs its card/CVV/SAD field pass once more: masking a credential can uncover a field the first pass read as part of the one holding it (`"password": "password="paymentCvv=…`), so one call leaves nothing a second would mask. The text rules then `redact_body` are a fixed point (0.15.6: the scheme word's rule leaves `password=`, a quoted value stops at an end tag, and only JSON or XML leaves are decoded; on 280k generated shapes none changed on a second pass, where 0.15.5 missed 240 of 120k). +- Every path a credential takes ends in `mask_by_field_type(v, "secret")` — the key walk, the credential text rules, `contrib/django/routes.py`, `contrib/net` — and it returns `[SECRET-MASKED]`, never a token, for a PAN-shaped value (`pan_shaped`, imported inside the function: `patterns` imports `tokens` as it loads) and for a placeholder another masker left (`tokens._PLACEHOLDER`: `[REDACTED]`, `[PII_REDACTED]`, stars, `Bearer ****`), and for a truncation the card rule writes (`tokens._SHOWN_TRUNCATION`, `411111******1111` as written or decoded, or any truncation after an auth scheme, `tokens._TRUNCATION_AFTER_A_SCHEME`, `Bearer 411111******1111`, which was a token of the two: ten digits of a saved card's gateway token, which never passes through as masking's own output -- rendered from its argument under pci, `token=%s` showed them where the key walk gave the label), which would otherwise hash into one token shared by every record. With `pci` among the call's packs, and the process's (`config.packs_in_force()`, imported inside the function as well, since `config` imports `patterns`: the packs of the masking call in progress -- a `MaskPIIFilter`'s own, which every call that builds its context sets in a ContextVar with `config.call_packs` -- joined with the process's `get_masking_packs()`, so in a pci-configured process every filter labels such a value, a filter built with pci labels it in any process, and a default handler ahead of a pci one no longer hashes what the pci handler then logs), a credential, payment id, email or phone number that holds a card-number run anywhere (`holds_pan_run`; a phone's `+`-led run up to 15 digits is the number) is its label too, on every path -- the text rules, `mask_secret`, `contrib/net`, a route parameter -- as the key walk's own `pci` branch (`filters._mask_pii_leaf`, on the call's packs and the process's, `filters._pci_in_force`) makes any PII leaf; without `pci` among the call's packs and the process's it is hashed, an accepted residual: a default-pack service receives no card numbers, and the label would stand where a token can for ~7% of Connect's 64-hex HMAC signatures. Masking's own output passes through (`already_masked`) unless a card-number run is in it: `ptok:v1:` typed before a card number is in a token's exact shape, and is the label, for any type and in every pack (`tokens._CARDLESS_TOKEN` is that shape without one; `_WHOLE_TOKEN` takes only it). Both are judged on the value as `tokenize` hashes it (`normalize_value`: surrounding whitespace and one layer of matching quotes dropped), so `'"4111…"'` is not a keyed hash of the bare card number, and on it as a URL or form encoding decodes it (`unquote_plus`, as `redact_url` and `redact_body` read it), so `%224111…%22` or `4111+1111+…` in text is the label in every pack, as a `%5BSECRET-MASKED%5D` redact_url wrote is masked already; and a value that is masking's own output once normalized (a label, token or truncation in quotes, as the credential text rule leaves `password="[SECRET-MASKED]"`) is returned as it is, never hashed into one shared token. `mask_secret(value)` (`masking/patterns.py`, exported from `ecsctx.masking` and `ecsctx` with `mask_card_value`) is that for a value outside a mapping: `None`/bools pass, anything else is masked as `str(value)`. `_mask_pii_leaf` still asks `pan_shaped` for a secret before the `pci` branch, which would truncate it. `contrib/net` masks with `mask_secret` (no `[REDACTED]` since 0.15.4): `redact_url` masks literal `secrets` in one pass, longest first (a replacement is never scanned again, so a short secret can't mangle the token a longer one became), then masks the userinfo part by part (`_mask_userinfo`, a label's brackets percent-encoded: `urlsplit` refuses brackets in a netloc whose host is not IPv6, so the result would not parse again, nor in `ecs_url`) and rewrites a credential query or fragment param in place, unencoded, leaving every other param byte for byte, and rebuilding the URL only when something changed; a query, fragment or form key the key rules call a card, CVV or SAD (`classify_key(unquote_plus(key), ALL_PACKS)`, `_card_field_type`) has its value masked by that type -- `mask_card_value`, or the label, never a token -- in `redact_url` and in `redact_body`'s form pass (`_FORM_FIELD`, run on any body with an `=`, before the credential-key pre-check); `redact_body` reads a JSON value escape-aware, past any escaped character and to the end of the text when no quote closes it (a body `json.loads` rejects is the one that reaches this rule as text), and masks what it decodes to, or the raw text when it does not decode; its closing quote is looked for, not consumed, since in malformed `""password": "…"` it also opens the next key; and a form value runs to the next `&`, whitespace or end tag (`_FORM_VALUE`), as in 0.15.3 but for the tag (every reading that ended it at a quote left the rest of some secret in clear). `redact_body` first gives every value a service's label rule matches its label (`patterns.mask_objects`, rule 2's pattern) and sets aside what a keep rule ships (`patterns.stash_kept`; an element's entity-encoded text that keeps one is set aside whole, `KeptStash.hold`, since base64's `==` after a run holding `cvv` reads as a CVV form field), then reads an XML element as a key (`_mask_xml_elements`): its local name, whatever its namespace prefix and attributes -- first a card, CVV or SAD name, as rule 3 reads one (`patterns.mask_card_elements`: a card object field by field, so a `` inside `` is its label), then a credential name (`_is_credential_key`, or a secret body key) as a credential, the label when it holds a card-number run -- one classification across a URL's query and an XML body, so ``, ``, `` and `` are credentials, as those query params are; the text runs to the end tag, children and all, as it decodes; a start tag no end tag closes is no element; a leaf holding JSON or XML written with entities goes through `redact_body` as it decodes and is written back escaped (only those: a form body's `&` is its separator where it stands, and decoded, a form value read differently on the next pass). `_split_form_value` keeps only the structure at its two ends as written -- a quote or an escaped one (at its head, a run of them: `password=""x`, quotes doubled as CSV escapes one, and a run of two or more of either kind, `password=''x` -- a lone `'` is the value's own, a quoted value when another closes it), and `}`, `]`, `,`, a JSON key's `":`, `>`, and `/>` after a quote (after an unquoted value a `/` right after a token reads to the credential text rule as more credential, so only `>` stays) -- and takes a label, token, truncation or placeholder followed only by structure whole (`[SECRET-MASKED],` is not `[SECRET-MASKED` and `],`). The value between them is JSON-unescaped only when it sits between escaped quotes (`\"…\"`, in a JSON string; never in a raw form body), is `[SECRET-MASKED]` if it `holds_pan_run` (asked before any pass-through: a token-shaped value can carry a card number), and otherwise goes through `mask_secret`, being written back exactly as captured when that leaves it as it is (empty, or masked already). It is decoded as form encoding reads it (`unquote_plus`: `+` is a space), so outside a form body a secret holding `+` or `%xx` gets a different token than under a key; nothing leaks. The JSON rule gives every value that `holds_pan_run` the same label, closed or run on to the end of the text, before `mask_secret` (which, without `pci`, hashes one that is not in a token's shape). When its credential passes change the text, `redact_body` runs its card/CVV/SAD field pass once more: masking a credential can uncover a field the first pass read as part of the one holding it (`"password": "password="paymentCvv=…`), so one call leaves nothing a second would mask. The text rules then `redact_body` are a fixed point (0.15.6: the scheme word's rule leaves `password=`, a quoted value stops at an end tag, and only JSON or XML leaves are decoded; on 280k generated shapes none changed on a second pass, where 0.15.5 missed 240 of 120k). ## Submodules - `masking/` - `filters.py` (`MaskPIIFilter`, the engine), `patterns.py` (`RULES` content rules and packs, `classify_key()`, `SAFE_KEYS`, `mask_card_value`, `mask_secret`), `config.py` (pack and skip-path selection, a service's safe keys and value rules), `value_rules.py` (`ValueRule`: a service's own shapes of value, masked as their label), `tokens.py` (`safe_tokenize`, `mask_by_field_type`, `already_masked`), `fields_rules.py` (`FieldRule`: tokenizable/exemptable per field type), `exemptions.py` (`configure_masking()` path-exemption state, path-prefix matching) @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: text holding none is never parsed). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_label`; JSON text holding one deeper under a key that would hash it is the label, `label_within`), every JSON object in text (rule 2, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A match is the rule's label, never hashed. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which stops at a kept mapping: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span, a CVV/SAD key right before an object -- quoted, `key=`, an XML start tag -- floors it and all inside it) and restores each placeholder still there; a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/contrib/net/__init__.py b/ecsctx/contrib/net/__init__.py index ccbec73..e866e9f 100644 --- a/ecsctx/contrib/net/__init__.py +++ b/ecsctx/contrib/net/__init__.py @@ -35,7 +35,9 @@ from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ( + _KEPT_HEAD, _XML_START, + KeptStash, _card_field_type, _closing, _element_text, @@ -46,9 +48,11 @@ mask_card_elements, mask_objects, mask_secret, + stash_kept, value_rules_in_force, ) from ecsctx.masking.tokens import _MASKED_VALUE, _PLACEHOLDER, make_label +from ecsctx.masking.value_rules import keep_rules _SECRET_LABEL = f"[{make_label('secret')}]" @@ -411,12 +415,35 @@ def redact_body(text: str) -> str: A value a value rule in force matches (``ECSCTX_MASK_VALUE_RULES``), as JSON anywhere in the body or as JSON in a JSON string, is the rule's - label first, before a credential's rule could hash it. + label first, before a credential's rule could hash it -- or, for a keep + rule, set aside before any of these rules reads the body and put back as + it was sent (``patterns.stash_kept``): a credential's JSON value, an extra + secret key such as ``signature`` and a form value's ``=`` never read into + it. An element's entity-encoded JSON is read as it decodes, and a value + kept there is set aside with the element's text, as it was written. """ + return _redact_body(text)[0] + + +def _redact_body(text: str) -> tuple[str, bool]: + """``redact_body``'s result, and whether it set a kept value aside.""" + stash = None if "{" in text and (rules := value_rules_in_force()): text = mask_objects(text, rules) + stash = stash_kept(text, rules) + if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text: + # None in the text as written; an element's entity-encoded text + # may hold one (_encoded_leaf). + stash = KeptStash(text, {}) + if stash is None: + return _redact_body_text(text, None), False + return stash.restore(_redact_body_text(stash.text, stash)), bool(stash.originals) + + +def _redact_body_text(text: str, stash: KeptStash | None) -> str: + """``redact_body``'s rules, once its value rules have read the body.""" if "<" in text: - text = _mask_xml_elements(text) + text = _mask_xml_elements(text, stash) if "=" in text: text = _FORM_FIELD.sub(_mask_card_form_field, text) hint_re, json_re, form_re, _names = _get_compiled() @@ -443,13 +470,16 @@ def _is_credential_element(name: str, names: frozenset[str]) -> bool: return _is_credential_key(name) or name.lower() in names -def _mask_xml_elements(text: str) -> str: +def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: """Each element named as a card, CVV or SAD key is, its text masked by that type, as the text rule masks one (`mask_card_elements`) -- first, so a CVV inside a credential's element is its label, not part of the text the credential's token hashes. Then each element named as a credential key is, its text masked as a credential; and entity-encoded text in any - other leaf, through the body rules as it decodes.""" + other leaf, through the body rules as it decodes -- set aside in + ``stash`` when it holds a kept value, so the form and JSON rules that + read the body next never read into it (base64's `==` after a run holding + `cvv` reads as a CVV form field).""" text = mask_card_elements(text) names = _get_compiled()[3] parts, copied, ends = [], 0, None @@ -464,7 +494,10 @@ def _mask_xml_elements(text: str) -> str: continue end = closing.start() if not _is_credential_element(start.group("local"), names): - if (masked := _encoded_leaf(text, start.end(), end)) is not None: + if (leaf := _encoded_leaf(text, start.end(), end)) is not None: + masked, kept = leaf + if kept and stash is not None: + masked = stash.hold(masked) parts += [text[copied : start.end()], masked] copied = end continue @@ -479,23 +512,24 @@ def _mask_xml_elements(text: str) -> str: return "".join(parts) -def _encoded_leaf(text: str, start: int, end: int) -> str | None: +def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None: """The text of a leaf, ``text[start:end]``, masked by the body rules as it decodes and escaped again -- when it is JSON or XML written with - entities, and masking changed it. A leaf's raw text needs nothing here, - nor a form body's: the body rules read them where they stand, `&` a - form's separator either way. Decoded, a form value ran on past `"`, - and a leaf the first pass left as a form body read differently on the - next.""" + entities, and masking changed it or kept a value in it (then as written, + if nothing else changed) -- and whether it kept one. A leaf's raw text + needs nothing here, nor a form body's: the body rules read them where + they stand, `&` a form's separator either way. Decoded, a form value + ran on past `"`, and a leaf the first pass left as a form body read + differently on the next.""" if text.find("<", start, end) != -1 or text.find("&", start, end) == -1: return None decoded = html.unescape(text[start:end]) if decoded.lstrip()[:1] not in ("{", "[", "<"): return None - masked = redact_body(decoded) + masked, kept = _redact_body(decoded) if masked == decoded: - return None - return xml_escape(masked) + return (text[start:end], True) if kept else None + return xml_escape(masked), kept def _mask_card_form_field(match: re.Match) -> str: diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index a15b9a9..ab8ff19 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -1,6 +1,6 @@ """Ottu's masking vocabulary: key names Ottu services log that are not PII (``SAFE_KEYS``, for ``ECSCTX_MASK_SAFE_KEYS``), and the wallet tokens they -carry, masked by shape (``WALLET_RULES``, for ``ECSCTX_MASK_VALUE_RULES``). +carry, found by shape (``WALLET_RULES``, for ``ECSCTX_MASK_VALUE_RULES``). ecsctx's own safe keys hold only names that mean the same in every service. These are Ottu's vocabulary, where a generic key rule would mask a value that is not @@ -14,13 +14,38 @@ so a service pinned to an older ecsctx keeps working, and because listing a key that is already safe costs nothing. -An Apple Pay or Google Pay token carries the device card number and the -payment cryptogram, encrypted: Sensitive Authentication Data, the SAD label -under any key and in any text, never hashed. Which shapes those are is Ottu's -knowledge, not ecsctx's: core names none, and masks one only as a service's -value rule (``ecsctx.masking.value_rules``). ``WALLET_RULES`` are Ottu's. - -A service opts in to both from its settings:: +Wallet tokens ship in logs exactly as sent (#159487, decided 2026-10-01). An +Apple Pay or Google Pay token is single-use: it pays once, and what it carries +-- the device card number and the payment cryptogram -- is encrypted for the +payment processor. When a wallet payment fails, the token as it was sent is +what tells a wrong merchant certificate, an expired signing key or a token +altered on the way apart. So ``WALLET_RULES`` are keep rules +(``ecsctx.masking.value_rules``): a token they match is left as sent under any +key -- ``paymentData``, TAP's ``token_data``, MPGS's ``paymentToken`` (a JSON +string), Google Pay's ``tokenizationData.token``, KPay's ```` -- and in +a message or a body, and no rule reads into it: not a credential rule, not the +payment-id rule (its ``transactionId``), not a service's extra secret keys +(``signature``). + +What stays masked: + +- Saved-card tokens. A card on file is charged again with its token, so it is + a credential: no wallet shape matches it, under ``token`` or anywhere. +- What is not exactly a wallet's shape. The matchers are strict -- an Apple + Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method + token, and no key besides the ones they write -- so anything else is walked + as before: a ``PKPaymentToken`` with a key of its own keeps only its + ``paymentData``. +- Card data. ecsctx keeps nothing under a CVV or SAD key, and nothing that + holds a card, CVV or SAD key or a card number (the floor and the guard). + +Samsung Pay is not here: its token is a JWE, and no captured token pins its +shape yet. A rule for any JWE would keep every JWE. + +``WALLET_SAD_RULES`` are 0.16.0's: label rules that make the same tokens' +payment data ``[SAD-MASKED]``, for a service that must not log them. + +A service opts in to both lists from its settings:: from ecsctx.contrib.ottu.masking import SAFE_KEYS as OTTU_SAFE_KEYS, WALLET_RULES @@ -36,7 +61,7 @@ from typing import Any -from ecsctx.masking.value_rules import ValueRule +from ecsctx.masking.value_rules import KeepRule, ValueRule SAFE_KEYS = frozenset({ # Ottu PG API fields: a gateway's short name ("mpgs") and flags saying @@ -88,15 +113,16 @@ }) -# Apple Pay's PKPaymentToken `paymentData`: its `version`, and the encrypted -# `data`. Its `signature` and `header` are useless without `data`, so the whole -# object is the label, and an unrelated `paymentData` reads through. +# The versions of Apple Pay's PKPaymentToken `paymentData`. _APPLE_PAY_VERSIONS = frozenset({"EC_v1", "RSA_v1"}) -# Google Pay's payment method token: its `protocolVersion` and the -# `signedMessage` holding the encrypted payload. +# ...and of Google Pay's payment method token. _GOOGLE_PAY_VERSIONS = frozenset({"ECv1", "ECv2", "ECv2SigningOnly"}) +# 0.16.0's label rules. `paymentData`'s `version` and the encrypted `data`: +# its `signature` and `header` are useless without `data`, so the whole object +# is the label, and an unrelated `paymentData` reads through. Google Pay's +# `protocolVersion` and the `signedMessage` holding the encrypted payload. def _is_apple_pay_token(value: Any) -> bool: version = value.get("version") return isinstance(version, str) and version in _APPLE_PAY_VERSIONS and "data" in value @@ -109,7 +135,91 @@ def _is_google_pay_token(value: Any) -> bool: APPLE_PAY_RULE = ValueRule("sad", _is_apple_pay_token, hints=("EC_v1", "RSA_v1")) GOOGLE_PAY_RULE = ValueRule("sad", _is_google_pay_token, hints=("ECv1", "ECv2")) + + +# The keep rules' shapes, key for key: what Apple's PKPaymentToken and Google +# Pay's payment method token are documented to hold, and nothing else. A key +# besides these is no wallet's, and the object is walked as before. +_PAYMENT_DATA_KEYS = frozenset({"version", "data", "signature", "header"}) +_PAYMENT_DATA_HEADER_KEYS = frozenset( + {"ephemeralPublicKey", "wrappedKey", "publicKeyHash", "transactionId", "applicationData"} +) +_PK_PAYMENT_TOKEN_KEYS = frozenset({"paymentData", "paymentMethod", "transactionIdentifier"}) +_PAYMENT_METHOD_KEYS = frozenset({"displayName", "network", "type"}) +_GOOGLE_PAY_KEYS = frozenset({"signature", "intermediateSigningKey", "protocolVersion", "signedMessage"}) +_SIGNING_KEY_KEYS = frozenset({"signedKey", "signatures"}) + + +def _text_or_absent(value: Any, key: str) -> bool: + """A wallet's field is text where it is written: every leaf a wallet + writes is a string, so a container or a number in a slot is no wallet's, + and nothing can ride in it -- a `{name, value}` pair among them.""" + return key not in value or isinstance(value[key], str) + + +def _is_apple_pay_payment_data(value: Any) -> bool: + """Apple Pay's `paymentData`: an EC_v1 or RSA_v1 `version`, the + encrypted `data` and its `signature` as text, and a `header` of text + fields.""" + version = value.get("version") + if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and isinstance(value.get("data"), str)): + return False + if not (value.keys() <= _PAYMENT_DATA_KEYS and _text_or_absent(value, "signature")): + return False + header = value.get("header", {}) + return ( + isinstance(header, dict) + and header.keys() <= _PAYMENT_DATA_HEADER_KEYS + and all(isinstance(field, str) for field in header.values()) + ) + + +def _is_pk_payment_token(value: Any) -> bool: + """Apple Pay's PKPaymentToken: its `paymentData`, and what the device says + about the card -- its display name, network and type -- and the + transaction's id, each as text.""" + payment_data = value.get("paymentData") + if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): + return False + if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _text_or_absent(value, "transactionIdentifier")): + return False + method = value.get("paymentMethod", {}) + return ( + isinstance(method, dict) + and method.keys() <= _PAYMENT_METHOD_KEYS + and all(isinstance(field, str) for field in method.values()) + ) + + +def _is_google_pay_payment_token(value: Any) -> bool: + """Google Pay's payment method token: its `protocolVersion`, the + `signedMessage` and its `signature` as text, and the intermediate signing + key ECv2 signs with: a `signedKey` as text and its `signatures`, a list + of text.""" + protocol = value.get("protocolVersion") + if not (isinstance(protocol, str) and protocol in _GOOGLE_PAY_VERSIONS): + return False + if not isinstance(value.get("signedMessage"), str): + return False + if not (value.keys() <= _GOOGLE_PAY_KEYS and _text_or_absent(value, "signature")): + return False + signing_key = value.get("intermediateSigningKey", {}) + if not (isinstance(signing_key, dict) and signing_key.keys() <= _SIGNING_KEY_KEYS): + return False + signatures = signing_key.get("signatures", []) + return ( + _text_or_absent(signing_key, "signedKey") + and isinstance(signatures, list) + and all(isinstance(signature, str) for signature in signatures) + ) + + +APPLE_PAY_TOKEN_KEEP_RULE = KeepRule(_is_pk_payment_token, hints=("EC_v1", "RSA_v1")) +APPLE_PAY_KEEP_RULE = KeepRule(_is_apple_pay_payment_data, hints=("EC_v1", "RSA_v1")) +GOOGLE_PAY_KEEP_RULE = KeepRule(_is_google_pay_payment_token, hints=("ECv1", "ECv2")) # Found by shape, never by a key name: under `paymentData`, TAP's `token_data`, # MPGS's `paymentToken` (a JSON string), Google Pay's `tokenizationData.token`, -# KPay's ``, or in a message. -WALLET_RULES = (APPLE_PAY_RULE, GOOGLE_PAY_RULE) +# KPay's ``, or in a message. The whole PKPaymentToken first, so it is +# kept whole where it is one. +WALLET_RULES = (APPLE_PAY_TOKEN_KEEP_RULE, APPLE_PAY_KEEP_RULE, GOOGLE_PAY_KEEP_RULE) +WALLET_SAD_RULES = (APPLE_PAY_RULE, GOOGLE_PAY_RULE) diff --git a/ecsctx/masking/__init__.py b/ecsctx/masking/__init__.py index b2bfc11..3c8043f 100644 --- a/ecsctx/masking/__init__.py +++ b/ecsctx/masking/__init__.py @@ -12,7 +12,9 @@ for choosing packs (configure_masking_packs, ECSCTX_MASKING_PACKS), a service's own safe key names (configure_masking_safe_keys, ECSCTX_MASK_SAFE_KEYS), its value rules (configure_masking_value_rules, -ECSCTX_MASK_VALUE_RULES; ecsctx.masking.value_rules) and skip paths, and +ECSCTX_MASK_VALUE_RULES; ecsctx.masking.value_rules: ValueRule labels a +shape, KeepRule ships it as sent -- is_kept and mask_outside_kept let a +service's own pass after ecsctx's leave a kept value alone) and skip paths, and ecsctx.masking.exemptions for the configure_masking() path-exemption API. """ @@ -32,9 +34,16 @@ ) from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.install import install_maskers, uninstall_maskers -from ecsctx.masking.patterns import ALL_PACKS, PACK_NAMES, mask_card_value, mask_secret +from ecsctx.masking.patterns import ( + ALL_PACKS, + PACK_NAMES, + is_kept, + mask_card_value, + mask_outside_kept, + mask_secret, +) from ecsctx.masking.tokens import mask_by_field_type, safe_tokenize -from ecsctx.masking.value_rules import ValueRule +from ecsctx.masking.value_rules import KeepRule, ValueRule __all__ = [ "ALL_PACKS", @@ -46,6 +55,9 @@ "configure_masking_value_rules", "get_masking_value_rules", "ValueRule", + "KeepRule", + "is_kept", + "mask_outside_kept", "MaskPIIFilter", "install_maskers", "uninstall_maskers", diff --git a/ecsctx/masking/config.py b/ecsctx/masking/config.py index d7805ef..e5e71b6 100644 --- a/ecsctx/masking/config.py +++ b/ecsctx/masking/config.py @@ -15,9 +15,10 @@ way: ``configure_masking_safe_keys()``, else the Django setting, else the environment variable, else none. They extend ``patterns.SAFE_KEYS``, which holds only names that mean the same in every service. So do its value rules -(``ECSCTX_MASK_VALUE_RULES``, ``ecsctx.masking.value_rules``): rule objects or -dotted import paths, ``configure_masking_value_rules()`` first; the environment -variable names import paths, comma-separated. +(``ECSCTX_MASK_VALUE_RULES``, ``ecsctx.masking.value_rules``: label rules and +keep rules, in one list): rule objects or dotted import paths, +``configure_masking_value_rules()`` first; the environment variable names +import paths, comma-separated. The Django setting is read lazily, at log time, because logging is configured while settings are still being imported. Until settings are configured the @@ -33,6 +34,7 @@ from contextvars import ContextVar from functools import lru_cache +from ecsctx.masking import patterns from ecsctx.masking.patterns import ALL_PACKS, classify_key, never_safe from ecsctx.masking.value_rules import load_value_rules @@ -280,9 +282,11 @@ def get_masking_safe_keys() -> frozenset[str]: def configure_masking_value_rules(rules: Iterable | str | None) -> None: - """The value rules this service masks by shape, wherever they are logged - (``ecsctx.masking.value_rules``): rule objects, or dotted import paths to - a rule or to a collection of them. ``None`` goes back to settings/env. + """The value rules this service masks, or ships as sent, by shape, + wherever they are logged (``ecsctx.masking.value_rules``): label and keep + rules in one list, asked in its order, the first match winning. Rule + objects, or dotted import paths to a rule or to a collection of them. + ``None`` goes back to settings/env. Raises on an item that is not one: this runs while logging is being configured, where failing loudly is right.""" global _explicit_rules, _resolved_rules @@ -294,6 +298,15 @@ def configure_masking_value_rules(rules: Iterable | str | None) -> None: raise ValueError("; ".join(problems)) _explicit_rules = loaded _resolved_rules = None + _forget_clean_strings() + + +def _forget_clean_strings() -> None: + """Empty the known-clean set (``patterns._clean``). It is keyed on the + content rules alone, and what is clean depends on the value rules too: a + string masked clean with none configured came back as it was once one + was.""" + patterns._clean.clear() def masking_value_rule_errors() -> list[str]: @@ -356,3 +369,4 @@ def _reset_masking_config() -> None: _explicit_rules = None _resolved_rules = None _warned.clear() + _forget_clean_strings() diff --git a/ecsctx/masking/filters.py b/ecsctx/masking/filters.py index 3bba5d5..054065e 100644 --- a/ecsctx/masking/filters.py +++ b/ecsctx/masking/filters.py @@ -17,6 +17,7 @@ from __future__ import annotations +import copy import dataclasses import json import logging @@ -35,6 +36,7 @@ from ecsctx.masking.patterns import ( _CRED_LITERALS, _CVV_LITERALS, + _FLOOR_TYPES, _KEY_SEPARATORS, _MIN_PAN_DIGITS, _PAYMENT_ID_KEYWORD, @@ -56,7 +58,16 @@ value_rules_in_force, ) from ecsctx.masking.tokens import make_label, mask_by_field_type -from ecsctx.masking.value_rules import label_of, label_within, text_label +from ecsctx.masking.value_rules import ( + KEEP, + applicable, + keep_rules, + label_rules, + parses_as_written, + label_within, + ruling, + text_ruling, +) _IS_MASKED_ = "_IS_MASKED_" @@ -159,8 +170,8 @@ def _value_rules(ctx: _Pass) -> tuple: # object became [CVV-MASKED]. _WALKED_TYPES = frozenset({"card", "cvv", "sad", "secret"}) # Under these nothing is weaker than the container: a CVV must not leave as a -# name token or under a core safe key such as `id`. -_FLOOR_TYPES = frozenset({"cvv", "sad"}) +# name token or under a core safe key such as `id` -- and nothing is kept +# (patterns._FLOOR_TYPES, which the text rules' floor reads too). _CONTAINERS = (dict, list, tuple, set) _CARD_NUMBER_KEYS = frozenset({"number", "pan", "cardnumber", "maskednumber", "maskedpan"}) @@ -502,20 +513,46 @@ def _mask_pii_leaf(text: str, field_type: str, ctx: _Pass) -> str: _JSON_STARTS = frozenset("{[ \t\r\n") -def _value_label(value: Any, ctx: _Pass) -> str | None: - """The label of a value a value rule in force matches, as a mapping or as - JSON text (value_rules): under any key, a credential's or a card's - included, never hashed, never shown.""" +def _value_ruling(value: Any, ctx: _Pass) -> Any: + """What the first value rule in force a value matches, as a mapping or as + JSON text (value_rules), makes of it: its label -- under any key, a + credential's or a card's included, never hashed, never shown -- ``KEEP``, + or None: nothing matched, or a keep rule did and the guard refused it.""" kind = type(value) if kind is dict: rules = _value_rules(ctx) - return label_of(value, rules) if rules else None + return ruling(value, rules) if rules else None if kind is str and value[:1] in _JSON_STARTS: rules = _value_rules(ctx) - return text_label(value, rules) if rules else None + return text_ruling(value, rules) if rules else None return None +def _floored(key: Any, inherited: str | None, pair_type: str | None, ctx: _Pass) -> bool: + """Whether nothing may be kept here: under a CVV or SAD key, inside a CVV + or SAD container, or as the `value` of a pair labelled as one. Such a + value is masked exactly as it would be with no keep rule.""" + if inherited in _FLOOR_TYPES: + return True + lookup_key = str(key) + if pair_type in _FLOOR_TYPES and lookup_key.lower() == "value": + return True + return classify_key(lookup_key, ctx.packs, ctx.safe) in _FLOOR_TYPES + + +def _without_keep(ctx: _Pass) -> _Pass: + """``ctx`` asking only the label rules in force, for what reads through a + CVV or SAD container -- a key the service listed: nothing below it is + kept, however deep, as the floor requires.""" + return ctx._replace(values=[label_rules(_value_rules(ctx))]) + + +def _kept(value: Any) -> Any: + """A value a keep rule ships: a mapping copied, since the record is + masked in place and must not share the caller's; a string as it is.""" + return copy.deepcopy(value) if type(value) is dict else value + + def _mask_card_list_element(value: Any) -> Any: """A bare value in a list under a card key -- `card=(pan, month, cvv)`. @@ -620,11 +657,17 @@ def _mask_dict( if path == () and key in self._skip_keys: result[key] = self._mask_skipped(key, value, ctx) continue - if (label := _value_label(value, ctx)) is not None: - # Under any key, a credential's or a card's included: never - # hashed, never shown. - result[key] = label - continue + if (found := _value_ruling(value, ctx)) is not None: + if found is not KEEP: + # Under any key, a credential's or a card's included: never + # hashed, never shown. + result[key] = found + continue + if not _floored(key, inherited, pair_type, ctx): + # As sent, under any key, a credential's or a card's + # included, and not descended into. + result[key] = _kept(value) + continue if isinstance(value, bool): # One bit: never PII, SAD or a credential, whatever its key or # container. Masking it only destroyed the flag. @@ -634,7 +677,7 @@ def _mask_dict( child_path = path + (lookup_key,) if pair_labels and key in pair_labels: # A field label, not a person: judged by content alone. - result[key] = self._mask_value(value, child_path, ctx) + result[key] = self._mask_value(value, child_path, ctx, ruled=True) continue field_type = classify_key(lookup_key, ctx.packs, ctx.safe) if pair_type is not None and lookup_key.lower() == "value": @@ -652,7 +695,7 @@ def _mask_dict( # reads through; anything else is the container's type, however # it classifies on its own. if field_type is None and _listed(lookup_key, ctx.safe): - result[key] = self._mask_value(value, child_path, ctx) + result[key] = self._mask_value(value, child_path, _without_keep(ctx)) continue field_type = inherited elif field_type is None: @@ -678,7 +721,7 @@ def _mask_dict( and inherited not in _WALKED_TYPES and classify_key(lookup_key, ctx.packs) in (None, "payment_id") ) - result[key] = self._mask_value(value, child_path, ctx, verbatim_digits=verbatim) + result[key] = self._mask_value(value, child_path, ctx, verbatim_digits=verbatim, ruled=True) continue if value is None: # An empty field of a container carries nothing to mask. @@ -706,7 +749,7 @@ def _mask_dict( ): # Never below a card, credential, CVV or SAD container: a broad # exempt prefix must not expose `…token.name_on_card`. - result[key] = self._mask_value(value, child_path, ctx) + result[key] = self._mask_value(value, child_path, ctx, ruled=True) elif ( isinstance(value, str) and value[:1] in _JSON_STARTS @@ -735,7 +778,7 @@ def _mask_dict( # out rather than made exemptable because `exemptable` also # governs ECSCTX_MASK_EXEMPT_PATHS (above), and none of them may # ever be whitelistable. - result[key] = self._mask_value(value, child_path, ctx, inherited=field_type) + result[key] = self._mask_value(value, child_path, ctx, inherited=field_type, ruled=True) else: result[key] = _mask_pii_leaf(str(value), field_type, ctx) return result @@ -769,6 +812,7 @@ def _mask_value( inherited: str | None = None, *, verbatim_digits: bool = False, + ruled: bool = False, ) -> Any: """Apply appropriate masking based on value type. @@ -778,12 +822,14 @@ def _mask_value( level so legitimate values (status codes, counts) are not mangled by the CVV/card patterns — a key-based match still overrides this, see _mask_dict. Inside a PII container (``inherited``) a bare value in a - list is masked as the container's type. + list is masked as the container's type. ``ruled``: the value rules + were asked about this value already, with this ``ctx`` -- by + _mask_dict, for every value it walks -- and are not asked again. """ if ctx is None: ctx = self._context() with call_packs(ctx.packs): - return self._mask_value(value, path, ctx, inherited, verbatim_digits=verbatim_digits) + return self._mask_value(value, path, ctx, inherited, verbatim_digits=verbatim_digits, ruled=ruled) if value is None: return value if len(path) > _MAX_DEPTH: @@ -797,16 +843,18 @@ def _mask_value( if verbatim_digits and _is_reference_number(value): return value # The two exact types nearly every value is, first: this runs for - # every value of every record. Each is a value rule's label before - # anything else reads it (`_value_label`, spelled out here). + # every value of every record. Each is a value rule's label, or kept, + # before anything else reads it (`_value_ruling`, spelled out here). kind = type(value) if kind is str: if ( - value[:1] in _JSON_STARTS + not ruled + and value[:1] in _JSON_STARTS and (rules := _value_rules(ctx)) - and (label := text_label(value, rules)) is not None + and (found := text_ruling(value, rules)) is not None + and (found is not KEEP or inherited not in _FLOOR_TYPES) ): - return label + return value if found is KEEP else found if inherited == "card": return _mask_card_list_element(value) if inherited is not None: @@ -817,8 +865,13 @@ def _mask_value( # is a field value, and its key has already had its say. return self._mask_string(value, ctx, scalar=path != ()) if kind is dict: - if (rules := _value_rules(ctx)) and (label := label_of(value, rules)) is not None: - return label + if ( + not ruled + and (rules := _value_rules(ctx)) + and (found := ruling(value, rules)) is not None + and (found is not KEEP or inherited not in _FLOOR_TYPES) + ): + return _kept(value) if found is KEEP else found return self._mask_dict(value, path, ctx, inherited) if isinstance(value, dict): return self._mask_dict(value, path, ctx, inherited) @@ -897,8 +950,19 @@ def _mask_json_text(self, text: str, parsed: dict | list, path: tuple, ctx: _Pas # Never at path (): the record's skip keys (log, service, session_id) # describe the record, not a payload that happens to use the names. # Key rules only: the content rules run once, on the whole text below, - # not once per value and again on the text. - masked = self._mask_value(parsed, path or (_JSON_TEXT,), ctx._replace(rules=())) + # not once per value and again on the text. And only the value rules + # whose hints the text holds: no other can match a mapping in it -- + # the label rules alone where the text names a key twice, since the + # parse is then not all the text ships. + rules = _value_rules(ctx) + if rules: + asked = applicable(text, rules) + if keep_rules(asked) and not parses_as_written(text): + asked = label_rules(asked) + walk = ctx._replace(rules=(), values=[asked]) + else: + walk = ctx._replace(rules=()) + masked = self._mask_value(parsed, path or (_JSON_TEXT,), walk) if masked == parsed: return self._mask_string(text, ctx) return self._mask_string(json.dumps(masked, ensure_ascii=False, default=str), ctx) @@ -907,6 +971,10 @@ def _mask_args(self, args: Any, ctx: _Pass) -> Any: if isinstance(args, tuple): return tuple(self._mask_arg(arg, ctx) for arg in args) if isinstance(args, dict): + if _value_ruling(args, ctx) is KEEP: + # `log.info("got %s", value)` with one mapping makes it the + # args: rendered whole, it is the value a keep rule ships. + return _kept(args) # "%(amount).3f": key rules still apply, numbers stay numbers. masked = self._mask_dict(args, (), ctx) return { diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 2effd8f..cea3640 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -7,7 +7,9 @@ always on, `pci` and `financial_ids` opt-in (ecsctx.masking.config) — each behind a literal pre-check, applied to every string the filter reaches. Rule order is load-bearing — see the comments on each rule and the - ordering invariants they protect. Do not reorder. + ordering invariants they protect. Do not reorder. Before any of them reads + a string, what a service's keep rule ships as sent is set aside + (stash_kept), and put back after. 2. Key-based (classify_key): in a dict, a key whose words name a sensitive field masks the whole value outright, regardless of the value's type or content. @@ -32,7 +34,7 @@ import re from bisect import bisect_left from collections import deque -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from functools import lru_cache from heapq import merge from itertools import accumulate, pairwise @@ -51,7 +53,19 @@ make_label, mask_by_field_type, ) -from ecsctx.masking.value_rules import hinted, label_of +from ecsctx.masking.value_rules import ( + KEEP, + DuplicateKey, + _first_match, + applicable, + hinted, + holds_card_data, + json_as_written, + keep_rules, + label_rules, + ruling, + text_ruling, +) # --------------------------------------------------------------------------- # Shared keyword/value fragments @@ -1068,9 +1082,11 @@ def _mask_userinfo(userinfo: str) -> str: # Value rules in text # --------------------------------------------------------------------------- # A JSON object written in text -- JSON, a Python repr, or JSON in a JSON -# string -- is asked of the service's value rules (value_rules), and one a -# rule matches is that rule's label, never a token. ecsctx names no shape of -# its own: nothing configured, nothing runs. +# string -- is asked of the service's value rules (value_rules): one a label +# rule matches is that rule's label, never a token (rule 2); one a keep rule +# matches is set aside before any rule reads the text, and put back as it was +# written after (stash_kept). ecsctx names no shape of its own: nothing +# configured, nothing runs. def value_rules_in_force() -> tuple: @@ -1136,16 +1152,29 @@ def _object_value(m: re.Match) -> Any: def _mask_object(m: re.Match, rules: tuple) -> str: - """The label where an object a rule matches stood: a JSON string where it - was a JSON value (after `:`, `,` or `[`), escaped where that JSON is - itself in a string; bare anywhere else -- in prose, an element's text, a - string of its own. An object no rule matches is read member by member, - so one inside it is still found.""" + """The label where an object a label rule matches stood: a JSON string + where it was a JSON value (after `:`, `,` or `[`), escaped where that JSON + is itself in a string; bare anywhere else -- in prose, an element's text, + a string of its own. An object no rule matches is read member by member, + so one inside it is still found; one a keep rule matches first is left as + written and not looked into, unless the guard refuses it or it is + written otherwise than it parses: then it is read as one nothing matches. + Only the rules whose hints the object's text holds are asked.""" written = m.group(0) - label = label_of(_object_value(m), rules) - if label is None: + asked = applicable(written, rules) + if not asked: + return written # nothing in it can match: every match holds a hint + value = _object_value(m) + found = _first_match(value, asked) + if found is not None and found[1] and (holds_card_data(value) or not _written_as_parsed(m)): + found = None + if found is None: inner = mask_objects(written[1:-1], rules) return written if inner == written[1:-1] else f"{{{inner}}}" + rule, keep = found + if keep: + return written + label = f"[{make_label(rule.field_type)}]" text, position = m.string, m.start() - 1 while position >= 0 and text[position].isspace(): position -= 1 @@ -1157,17 +1186,237 @@ def _mask_object(m: re.Match, rules: tuple) -> str: def _value_object(m: re.Match) -> str: rules = value_rules_in_force() - return _mask_object(m, rules) if rules else m.group(0) + return _mask_object(m, rules) if label_rules(rules) else m.group(0) def mask_objects(text: str, rules: tuple) -> str: - """Each JSON object written in ``text`` that one of ``rules`` matches, as - its label: rule 2 alone, for ``contrib.net.redact_body``.""" - if not rules or "{" not in text or not hinted(text, rules): + """Each JSON object written in ``text`` that a label rule among ``rules`` + matches first, as its label: rule 2 alone, for ``contrib.net.redact_body``. + Only the label rules' hints open the text: keep rules alone write no + label.""" + labels = label_rules(rules) + if not labels or "{" not in text or not hinted(text, labels): return text return _OBJECT_RULE.pattern.sub(lambda m: _mask_object(m, rules), text) +# Nothing is kept under these, as the key walk keeps nothing under such a key +# or in such a container (filters): a CVV and the rest of Sensitive +# Authentication Data. +_FLOOR_TYPES = frozenset({"cvv", "sad"}) +# The key right before an object in text, as the floor reads it: a quoted key +# and its colon (JSON, a repr, JSON in a JSON string, its quotes escaped), a +# key and `=` or `:`, or an XML start tag -- then any space and the opening +# quote of a string the object is the JSON text of. +_KEY_BEFORE = re.compile( + r"(?:(?P\\?[\"'])(?P[^\"'\\]{1,128})(?P=quote)\s*:" + r"|(?[\w.-]{1,128})\s*[:=]" + r"|<(?:[A-Za-z_][\w.-]*:)?(?P[A-Za-z_][\w.-]*)(?:\s[^<>]*)?(?)" + r"\s*(?:\\?[\"'])?\s*\Z" +) +# How far back the key is looked for. +_KEY_REACH = 192 + + +def _floored_before(text: str, start: int) -> bool: + """Whether the object at ``start`` sits right after a CVV or SAD key -- + `"cvv": {…}`, `cvv={…}`, `{…}`, `"pin": "{…}"`: nothing under + one is kept, however deep. Every pack on, as the key walk reads a key.""" + found = _KEY_BEFORE.search(text, max(0, start - _KEY_REACH), start) + if found is None: + return False + name = found.group("quoted") or found.group("bare") or found.group("tag") + return classify_key(name, ALL_PACKS) in _FLOOR_TYPES + + +def _written_as_parsed(m: re.Match) -> bool: + """Whether the object a match wrote is all its parse says, as a keep + decision needs: what ships is the text as written. JSON, or JSON in a + JSON string as it decodes, that names no key twice; a Python repr whose + parse renders back to it exactly -- a comment or a key written twice is + dropped by the parse, and would ship unread.""" + written = m.group(0) + try: + if m.group("plain") is None: + json_as_written(json.loads(f'"{written}"')) + return True + try: + json_as_written(written) + except DuplicateKey: + return False + except ValueError: + return repr(ast.literal_eval(written)) == written + return True + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): + return False + + +def _object_as_read(m: re.Match) -> tuple[Any, bool]: + """``_object_value`` and ``_written_as_parsed`` at once, for a keep + decision: JSON parsed strictly first, and as json.loads reads it only + where a key is written twice.""" + written = m.group(0) + try: + if m.group("plain") is None: + written = json.loads(f'"{written}"') + try: + return json_as_written(written), True + except DuplicateKey: + return json.loads(written), False + except ValueError: + if m.group("plain") is None: + return None, True + value = ast.literal_eval(written) + return value, repr(value) == written + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): + return None, True + + +def _object_ruling(m: re.Match, rules: tuple) -> Any: + """``ruling`` of the object a match wrote; a label rule's error reads as a + label -- not kept, not looked into -- so finding kept spans never raises, + and the text is masked as without a keep rule. A keep match written + otherwise than it parses (``_written_as_parsed``) is read as one nothing + matches.""" + try: + value, as_written = _object_as_read(m) + found = ruling(value, rules) + except Exception: # noqa: BLE001 -- a service's matcher; rule 2 reports it where it runs + return "" + return None if found is KEEP and not as_written else found + + +def kept_spans(text: str, rules: tuple) -> list[tuple[int, int]]: + """Where ``text`` writes a value a keep rule among ``rules`` matches first + and may keep: each JSON object (rule 2's reading), top down. An object a + label rule matches first is no span and is not looked into, nor is one + right after a CVV or SAD key; one nothing matches is read member by + member; a keep match the guard refuses (``holds_card_data``) is read as + one nothing matches. In order, none inside another.""" + spans: list[tuple[int, int]] = [] + _find_kept(text, 0, len(text), rules, spans) + return spans + + +def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[int, int]]) -> None: + for m in _OBJECT_RULE.pattern.finditer(text, start, end): + asked = applicable(m.group(0), rules) + if not asked or _floored_before(text, m.start()): + continue + found = _object_ruling(m, asked) + if found is KEEP: + spans.append(m.span()) + elif found is None: + _find_kept(text, m.start() + 1, m.end() - 1, rules, spans) + + +# What a kept span stands as while the rules read the text: a label's shape, +# which every rule leaves as it is, with no digit in it for a rule to read. +_KEPT_HEAD = "[KEPT-" +_KEPT_PLACEHOLDER = re.compile(r"\[KEPT-[A-Z]+-MASKED\]") + + +def _letters(index: int) -> str: + """0 is A, 25 is Z, 26 is AA: a placeholder's own, digit-free name.""" + letters = "" + index += 1 + while index: + index, rest = divmod(index - 1, 26) + letters = chr(ord("A") + rest) + letters + return letters + + +class KeptStash(NamedTuple): + """Text with each kept span set aside (``text``), and what each + placeholder stands for.""" + + text: str + originals: dict[str, str] + + def hold(self, value: str) -> str: + """A new placeholder standing for ``value``, for the caller to put + where it stood: ``contrib.net.redact_body`` sets aside an element's + entity-encoded text that holds a kept value.""" + placeholder = f"{_KEPT_HEAD}{_letters(len(self.originals))}-MASKED]" + self.originals[placeholder] = value + return placeholder + + def restore(self, masked: str) -> str: + """``masked`` with each placeholder that is still in it put back as + the value was written. One a rule destroyed -- hashed or labelled with + what was around it -- is not: the value goes with it.""" + if not self.originals: + return masked + return _KEPT_PLACEHOLDER.sub(lambda m: self.originals.get(m.group(), m.group()), masked) + + +def stash_kept(text: str, rules: tuple | None = None) -> KeptStash | None: + """``text`` with each value a keep rule ships set aside as a placeholder + (``[KEPT-A-MASKED]``), for the rules to read around it -- or None when + there is none: no keep rule among ``rules`` (the rules in force when + None), none whose hints the text holds, no span (``kept_spans``), or text + that already holds a placeholder's head, which is masked as before. + Never raises.""" + if "{" not in text: + return None + if rules is None: + rules = value_rules_in_force() + if not rules or not (keep := keep_rules(rules)) or not hinted(text, keep) or _KEPT_HEAD in text: + return None + spans = kept_spans(text, rules) + if not spans: + return None + parts: list[str] = [] + originals: dict[str, str] = {} + copied = 0 + for index, (start, end) in enumerate(spans): + placeholder = f"{_KEPT_HEAD}{_letters(index)}-MASKED]" + originals[placeholder] = text[start:end] + parts += [text[copied:start], placeholder] + copied = end + parts.append(text[copied:]) + return KeptStash("".join(parts), originals) + + +def is_kept(value: Any, *, key: str | None = None) -> bool: + """Whether masking ships ``value`` as sent: a mapping, or JSON text, that + a keep rule in force matches first (``ECSCTX_MASK_VALUE_RULES``, asked in + their order), that the guard lets through -- no card, CVV or SAD key and + no card number in it, at any depth -- and, given the ``key`` it sits + under, not under a CVV or SAD key. False with no keep rule configured, + for a label rule's match, and for anything else. Never raises. + + For a service's own masking pass after ecsctx's, which must leave such a + value as ecsctx did: ask before masking a field.""" + if not isinstance(value, (str, Mapping)): + return False + rules = value_rules_in_force() + if not keep_rules(rules): + return False + if key is not None and classify_key(str(key), ALL_PACKS) in _FLOOR_TYPES: + return False + try: + if isinstance(value, str): + return text_ruling(value, rules) is KEEP + return ruling(value if isinstance(value, dict) else dict(value), rules) is KEEP + except Exception: # noqa: BLE001 -- a label rule's error: not kept, and a log path never raises + return False + + +def mask_outside_kept(text: str, mask: Callable[[str], str]) -> str: + """``mask(text)`` with each value ecsctx keeps in ``text`` set aside: a + placeholder in a label's shape, ``[KEPT--MASKED]``, stands for + it while ``mask`` reads the rest, and is put back as the value was + written. One ``mask`` destroys is not put back: the value goes with it. + + For a service's own text masking after ecsctx's, which would otherwise + read into what ecsctx ships as sent.""" + stash = stash_kept(text) + if stash is None: + return mask(text) + return stash.restore(mask(stash.text)) + + def _mask_pem(match: re.Match) -> str: """Token computed over the base64 body only — strip the BEGIN/END lines and all whitespace first, so the same key re-wrapped at a different line @@ -1717,7 +1966,9 @@ def _has_credential(_text: str, lowered: str) -> bool: def _has_value_object(text: str, _lowered: str) -> bool: - return "{" in text and bool(rules := value_rules_in_force()) and hinted(text, rules) + # The label rules only: a keep rule writes no label, and what it keeps is + # set aside before this rule reads the text (stash_kept). + return "{" in text and bool(rules := label_rules(value_rules_in_force())) and hinted(text, rules) def _has_credential_container(text: str, lowered: str) -> bool: @@ -1939,6 +2190,12 @@ def _rule(pack, regex, repl, gate, scan=None, prose_only=False): # a `` inside `` was part of the text the credential's # token hashed, where the key walk gives a CVV under a credential key # its label. +# 13. Kept spans are stashed before every rule (mask_by_patterns → +# stash_kept): what a keep rule ships is a placeholder while the rules +# read the text, so none of them reads into it -- a credential rule +# would hash it, the payment-id rule mask its `"transactionId"` -- and +# one that splits text around it never misreads a quote. A placeholder a +# rule destroys takes the value with it. # --------------------------------------------------------------------------- _RULE_TABLE = ( @@ -1950,11 +2207,11 @@ def _rule(pack, regex, repl, gate, scan=None, prose_only=False): _has_pem, ), # 2. Value rules — a JSON object in text, or in a JSON string, that a - # value rule in force matches (`value_rules`, ECSCTX_MASK_VALUE_RULES): - # the rule's label, never a token, in every pack. Nothing configured, - # nothing runs; text holding none of the rules' hints is never parsed. - # Before the credential rules, which would hash one under `"token": …` - # first. + # label rule in force matches first (`value_rules`, + # ECSCTX_MASK_VALUE_RULES): the rule's label, never a token, in every + # pack. Nothing configured, or keep rules alone, nothing runs; text + # holding none of the label rules' hints is never parsed. Before the + # credential rules, which would hash one under `"token": …` first. _rule( "default", _OBJECT_TEXT, @@ -2343,6 +2600,10 @@ def mask_by_patterns(text: str, rules: tuple[Rule, ...]) -> str: clean = _clean_set(rules) if text in clean: return text + if (stash := stash_kept(text)) is not None: + # Not recorded as clean: it is no fixed point of the rules, which + # would read into what it ships. + return stash.restore(mask_by_patterns(stash.text, rules)) for _ in range(_MAX_PASSES): masked = _mask_once(text, rules) if masked == text: diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 8f9464c..b0379fe 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -1,5 +1,5 @@ -"""Value rules: shapes of value a service masks as their label, wherever they -are logged (``ECSCTX_MASK_VALUE_RULES``). +"""Value rules: shapes of value a service masks as their label, or ships as +sent, wherever they are logged (``ECSCTX_MASK_VALUE_RULES``). The key rules read a value by the name it sits under. Some values are known by their shape instead, under whatever key and in whatever text, and which shapes @@ -10,27 +10,40 @@ JSON object written in free text or in a body ``redact_body`` masks. With none configured, none of this runs. -A rule is anything with: +A rule is one of two kinds, both asked in list order, the first match winning: + +- A **label rule** has a ``field_type``: the type whose label a matching value + becomes, as ``make_label`` spells it (``"sad"`` is ``[SAD-MASKED]``). A + matching value is never hashed, and the label is masking's own output, left + as it is. ``ValueRule`` is one, ready made. +- A **keep rule** has ``keep = True`` (the object itself, not a truthy value): + a matching value ships exactly as sent, and the walk does not descend into + it. ``KeepRule`` is one, ready made. Two things it cannot override: nothing + is kept under a CVV or SAD key or inside a CVV or SAD container (the floor, + applied where a key is known: ``filters``, ``patterns.kept_spans``), and a + match holding a card, CVV or SAD key, or a card number, at any depth, is + walked as if nothing matched (the guard, ``holds_card_data``). + +Both have: -- ``field_type``: the type whose label a matching value becomes, as - ``make_label`` spells it (``"sad"`` is ``[SAD-MASKED]``). A matching value is - never hashed, and the label is masking's own output, left as it is. - ``matches(value)``: whether a mapping is one -- a dict the key walk meets, or - JSON text the engine parsed once to ask. + JSON text the engine parsed once to ask. A label rule that raises leaves the + ``[MASKING-FAILED: …]`` marker; a keep rule that raises counts as no match, + since keep rules are asked on paths that must never raise (``redact_url``, + ``redact_body``, ``mask_card_value``). - ``hints``, optionally: literal strings a matching value's text holds at least - one of. Text holding none is never parsed to ask; without hints every JSON - object in text is. - -``ValueRule`` is one, ready made. + one of. A rule is never asked about text holding none of its hints; without + hints it is asked about every JSON object in text. """ from __future__ import annotations import importlib import json +import re from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass -from typing import Any +from typing import Any, ClassVar, NamedTuple from ecsctx.masking.tokens import make_label @@ -44,11 +57,48 @@ class ValueRule: hints: tuple[str, ...] = () -def is_value_rule(rule: Any) -> bool: +@dataclass(frozen=True) +class KeepRule: + """A shape of value that ships as sent wherever it is logged.""" + + matches: Callable[[Any], bool] + hints: tuple[str, ...] = () + keep: ClassVar[bool] = True + + +class _Keep: + """What a value a keep rule matches is ruled: shipped as sent.""" + + __slots__ = () + + def __repr__(self) -> str: + return "KEEP" + + +KEEP = _Keep() + + +def is_keep_rule(rule: Any) -> bool: + return getattr(rule, "keep", None) is True and callable(getattr(rule, "matches", None)) + + +def is_label_rule(rule: Any) -> bool: field_type = getattr(rule, "field_type", None) return isinstance(field_type, str) and bool(field_type) and callable(getattr(rule, "matches", None)) +def _hints_are_text(rule: Any) -> bool: + """Whether a rule's ``hints``, if it has any, are a collection of strings: + each is looked for in text as it is masked, and anything else would raise + there -- in redact_url and redact_body too.""" + hints = getattr(rule, "hints", ()) + return isinstance(hints, (tuple, list, set, frozenset)) and all(isinstance(hint, str) for hint in hints) + + +def is_value_rule(rule: Any) -> bool: + return (is_keep_rule(rule) or is_label_rule(rule)) and _hints_are_text(rule) + + def load_value_rules(value: Iterable[Any] | str) -> tuple[tuple[Any, ...], tuple[str, ...]]: """(rules, problems): the rules a setting names -- rule objects, or dotted import paths to a rule or to a collection of them, comma-separated in one @@ -78,7 +128,11 @@ def _rules_in(item: Any) -> tuple[Any, ...]: members = tuple(found) if all(is_value_rule(member) for member in members): return members - raise ValueError(f"{item!r} is not a value rule or a collection of them: a rule has a `field_type` and `matches(value)`") + raise ValueError( + f"{item!r} is not a value rule or a collection of them: a rule has `matches(value)`, " + "either a `field_type` (its label) or `keep = True` (it ships as sent), and optionally " + "`hints`, a collection of strings" + ) def _imported(path: str) -> Any: @@ -91,62 +145,348 @@ def _imported(path: str) -> Any: raise ValueError(f"{path!r} does not import: {error}") from error -def label_of(value: Any, rules: tuple) -> str | None: - """The label of the first rule a mapping matches; None for anything else.""" - if isinstance(value, dict): - for rule in rules: - if rule.matches(value): - return f"[{make_label(rule.field_type)}]" +class _Split(NamedTuple): + """What masking reads of a rule set, again and again: each rule with + whether it keeps, and its keep and label rules, in their order.""" + + rules: tuple + flagged: tuple[tuple[Any, bool], ...] + keep: tuple + label: tuple + + +# Cached on the rule tuple's identity (as patterns._by_identity caches, never +# on its hash): asked about every mapping and every string masking reads. +# Bounded, and emptied, not evicted. +_SPLITS_LIMIT = 64 +_splits: dict[int, _Split] = {} + + +def _split_of(rules: tuple) -> _Split: + flagged = tuple((rule, is_keep_rule(rule)) for rule in rules) + return _Split( + rules, + flagged, + tuple(rule for rule, keep in flagged if keep), + tuple(rule for rule, keep in flagged if not keep), + ) + + +def _split(rules: Iterable[Any]) -> _Split: + if not isinstance(rules, tuple): + return _split_of(tuple(rules)) + entry = _splits.get(id(rules)) + if entry is None or entry.rules is not rules: + if len(_splits) >= _SPLITS_LIMIT: + _splits.clear() + # Keeping `rules` in the entry keeps its id from being reused. + entry = _splits[id(rules)] = _split_of(rules) + return entry + + +def keep_rules(rules: Iterable[Any]) -> tuple: + """The keep rules of ``rules``, in their order.""" + return _split(rules).keep + + +def label_rules(rules: Iterable[Any]) -> tuple: + """The label rules of ``rules``, in their order.""" + return _split(rules).label + + +def _first_match(value: Any, rules: Iterable[Any]) -> tuple[Any, bool] | None: + """The first of ``rules`` a dict matches, with whether it keeps; None for + no match or anything but a dict. A keep rule that raises counts as no + match; a label rule's error reaches the caller.""" + if not isinstance(value, dict): + return None + for rule, keep in _split(rules).flagged: + if keep: + try: + matched = bool(rule.matches(value)) + except Exception: # noqa: BLE001 -- a service's matcher; a keep rule never raises + matched = False + if matched: + return rule, True + elif rule.matches(value): + return rule, False return None +def rule_of(value: Any, rules: Iterable[Any]) -> Any: + """The first of ``rules`` a mapping matches, or None -- None for anything + but a dict. A keep rule that raises counts as no match; a label rule's + error reaches the caller.""" + found = _first_match(value, rules) + return None if found is None else found[0] + + +def _could_match(rule: Any, text: str) -> bool: + hints = getattr(rule, "hints", ()) + return not hints or any(hint in text for hint in hints) + + +# The tuple of each selection a text's hints make of a rule tuple, so every +# text making it asks the same tuple, and what `_split` caches on its identity +# is found again: a tuple per text would fill `_splits` until it is emptied, +# the configured rules' entry with it. Bounded, and emptied, as `_splits` is. +_selections: dict[tuple[int, int], tuple[tuple, tuple]] = {} + + +def applicable(text: str, rules: Iterable[Any]) -> tuple: + """The rules a value written in ``text`` could match, in their order: one + without hints, or one of whose hints ``text`` holds. ``rules`` itself when + every one could, and for a tuple the same tuple wherever the same ones + could, so what is cached on its identity is found again.""" + if not isinstance(rules, tuple): + return tuple(rule for rule in rules if _could_match(rule, text)) + selected = 0 + for index, rule in enumerate(rules): + if _could_match(rule, text): + selected |= 1 << index + if selected == (1 << len(rules)) - 1: + return rules + if not selected: + return () + entry = _selections.get((id(rules), selected)) + if entry is None or entry[0] is not rules: + if len(_selections) >= _SPLITS_LIMIT: + _selections.clear() + # Keeping `rules` in the entry keeps its id from being reused. + asked = tuple(rule for index, rule in enumerate(rules) if selected >> index & 1) + entry = _selections[(id(rules), selected)] = (rules, asked) + return entry[1] + + def hinted(text: str, rules: tuple) -> bool: """Whether a rule could match something in ``text``: one without hints, or one of whose hints ``text`` holds.""" - for rule in rules: - hints = getattr(rule, "hints", ()) - if not hints or any(hint in text for hint in hints): + return any(_could_match(rule, text) for rule in rules) + + +class DuplicateKey(ValueError): + """JSON text naming a key twice in one object: json.loads keeps the last, + and the text as written keeps both.""" + + +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict: + found = dict(pairs) + if len(found) != len(pairs): + raise DuplicateKey("a key written twice in one object") + return found + + +def json_as_written(text: str) -> Any: + """``text`` parsed as JSON, refusing a key written twice anywhere in it + (``DuplicateKey``, a ValueError). A keep decision rests on the parse, and + the text as written is what ships: a duplicate json.loads drops would + carry what the guard never saw.""" + return json.loads(text, object_pairs_hook=_unique_keys) + + +def parses_as_written(text: str) -> bool: + """Whether JSON text names no key twice: its parse is all it ships.""" + try: + json_as_written(text) + except (ValueError, RecursionError): + return False + return True + + +# Deeper than any payload a rule matches, shallow enough that a cycle or a +# crafted body stops long before Python's recursion limit. +_MAX_DEPTH = 64 +# What the guard refuses to keep: a key the key rules read as one of these. +_CARD_DATA_TYPES = frozenset({"card", "cvv", "sad"}) +# Epoch milliseconds, as written until 2065: thirteen digits from a 1, or from +# a 2 after 18 May 2033. Google Pay's `signedKey` carries one +# (`keyExpiration`), which `pan_shaped` reads as a card number; the one card +# number's shape the guard lets through. No card number is thirteen digits +# from a 1 or a 2. +_EPOCH_MILLISECONDS = re.compile(r"[12][0-9]{12}") + + +def holds_card_data(value: Any) -> bool: + """Whether ``value`` holds, at any depth -- JSON text inside it included -- + a key the key rules read as a card, a CVV or other SAD (every pack on), a + ``{name, value}`` pair whose identifier reads as one (as the key walk reads + a pair, ``filters._pair``), or a leaf that is a card number as a whole: + what no keep rule may ship, not even one that matches everything. Keys, + pairs and whole leaves only: a CVV or a card number written inside a + longer string (``"note": "cvv=123"``) is not read here, and keeping it out + is a matcher's job. + + A leaf is a card number when it is an int ``int_is_pan`` reads as one, or + a string ``pan_shaped`` reads as one -- whatever its prefix, Luhn or not -- + except thirteen bare digits from a 1 or a 2: epoch milliseconds, Google Pay's + ``keyExpiration`` in ``signedKey``, without which no Google Pay token would + ever be kept. Not ``holds_pan_run``, which flags hex ids. + Anything that is no JSON value (an object whose text is not judged here), + nesting past the depth cap, and a leaf of JSON text that names a key twice + (``json_as_written``: what ships would not be what was read) count as + holding card data: such a match is walked, not shipped. + """ + # Imported here: patterns and filters import this module as they load. + from ecsctx.masking.filters import _PAIR_IDENTIFIERS + from ecsctx.masking.patterns import ( + ALL_PACKS, + classify_key, + int_is_pan, + pan_shaped, + ) + + def labels_card_data(pair: Mapping) -> bool: + keys = {str(key).lower(): key for key in pair} + if "value" not in keys: + return False + # Uncached, as the key walk classifies free text: an identifier may be + # a sentence, and must not evict a key name. + return any( + isinstance(label := pair[keys[identifier]], str) + and classify_key.__wrapped__(label, ALL_PACKS) in _CARD_DATA_TYPES + for identifier in _PAIR_IDENTIFIERS + if identifier in keys + ) + + def holds(item: Any, depth: int) -> bool: + if depth > _MAX_DEPTH: return True - return False + if isinstance(item, Mapping): + if labels_card_data(item): + return True + return any( + classify_key(str(key), ALL_PACKS) in _CARD_DATA_TYPES or holds(member, depth + 1) + for key, member in item.items() + ) + if isinstance(item, (list, tuple, set, frozenset)): + return any(holds(member, depth + 1) for member in item) + if item is None or isinstance(item, bool): + return False + if isinstance(item, int): + return int_is_pan(item) + if isinstance(item, float): + return item.is_integer() and int_is_pan(int(item)) + if isinstance(item, str): + if pan_shaped(item) and not _EPOCH_MILLISECONDS.fullmatch(item.strip()): + return True + if item.lstrip()[:1] not in ("{", "["): + return False + try: + parsed = json_as_written(item) + except (DuplicateKey, RecursionError): + return True + except ValueError: + return False + return holds(parsed, depth + 1) + return True + + return holds(value, 0) + + +def ruling(value: Any, rules: Iterable[Any]) -> str | _Keep | None: + """What the first of ``rules`` a mapping matches makes of it: its label, + ``KEEP``, or None -- nothing matched, or a keep rule did and the guard + refused it (``holds_card_data``), so it is walked as if nothing had: no + rule listed after it is asked.""" + found = _first_match(value, rules) + if found is None: + return None + rule, keep = found + if keep: + return None if holds_card_data(value) else KEEP + return f"[{make_label(rule.field_type)}]" + + +def _json_in(text: str) -> Any: + """``text`` parsed, when it is JSON text; None otherwise. Cheap for every + other string: most do not start with a bracket.""" + first = text[:1] + if first.isspace(): + first = text.lstrip()[:1] + if first not in ("{", "["): + return None + try: + return json.loads(text) + except (ValueError, RecursionError): + return None def parsed_json(text: str, rules: tuple) -> Any: """``text`` parsed, when it is JSON text a rule could match; None - otherwise. Cheap for every other string: most do not start with a - bracket.""" + otherwise.""" first = text[:1] if first.isspace(): first = text.lstrip()[:1] if first not in ("{", "[") or not hinted(text, rules): return None + return _json_in(text) + + +def text_ruling(text: str, rules: Iterable[Any]) -> str | _Keep | None: + """``ruling`` for a string that is JSON text, asking only the rules whose + hints it holds. Not ``KEEP`` for text naming a key twice: kept, it would + ship the key json.loads dropped, which the guard never read. A label + rule's reading is 0.16.0's.""" + asked = applicable(text, rules) + if not asked: + return None + if not _split(asked).keep: + return ruling(_json_in(text), asked) + value, as_written = _json_as_read(text) + found = ruling(value, asked) + return None if found is KEEP and not as_written else found + + +def _json_as_read(text: str) -> tuple[Any, bool]: + """``text`` parsed -- None where it is no JSON text -- and whether the + parse is all the text writes: parsed once, strictly, and again as + json.loads reads it only where a key is written twice.""" + first = text[:1] + if first.isspace(): + first = text.lstrip()[:1] + if first not in ("{", "["): + return None, True try: - return json.loads(text) + return json_as_written(text), True + except DuplicateKey: + return _json_in(text), False except (ValueError, RecursionError): - return None + return None, True -def text_label(text: str, rules: tuple) -> str | None: - """The label of a string that is JSON text of a value a rule matches.""" - return label_of(parsed_json(text, rules), rules) +def label_of(value: Any, rules: Iterable[Any]) -> str | None: + """The label of the first rule a mapping matches; None for anything else, + and where that rule is a keep rule.""" + found = _first_match(value, rules) + return None if found is None or found[1] else f"[{make_label(found[0].field_type)}]" -_MAX_DEPTH = 64 +def text_label(text: str, rules: Iterable[Any]) -> str | None: + """The label of a string that is JSON text of a value a rule matches.""" + asked = applicable(text, rules) + return label_of(_json_in(text), asked) if asked else None -def label_within(value: Any, rules: tuple, depth: int = 0) -> str | None: +def label_within(value: Any, rules: Iterable[Any], depth: int = 0) -> str | None: """The label of a value a rule matches anywhere in ``value`` -- a - mapping, a list, or JSON text, however deep -- or None.""" + mapping, a list, or JSON text, however deep -- or None. A kept mapping is + not descended into: what a keep rule ships holds no label.""" if depth > _MAX_DEPTH: return None if isinstance(value, dict): - if (label := label_of(value, rules)) is not None: - return label + found = _first_match(value, rules) + if found is not None: + if not found[1]: + return f"[{make_label(found[0].field_type)}]" + if not holds_card_data(value): + return None items = value.values() elif isinstance(value, (list, tuple)): items = value elif isinstance(value, str): - found = parsed_json(value, rules) - return None if found is None else label_within(found, rules, depth + 1) + asked = applicable(value, rules) + found = _json_in(value) if asked else None + return None if found is None else label_within(found, asked, depth + 1) else: return None for item in items: diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 6852572..c59dd16 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -1,27 +1,21 @@ -"""With Ottu's WALLET_RULES, a wallet token's payment data is `[SAD-MASKED]` -in every pack (#160054). - -An Apple Pay or Google Pay token carries the device card number and the -payment cryptogram, encrypted. It is never hashed: the whole token is the SAD -label, as a cryptogram under its own key is. It is found by its shape, not by -a key name, so an unrelated `paymentData` stays readable and the token is -masked under any key -- `paymentData`, TAP's `token_data`, MPGS's -`paymentToken` (a JSON string), Google Pay's `tokenizationData.token`, KPay's -``. - -The shapes are Ottu's, not ecsctx's: `ecsctx.contrib.ottu.masking.WALLET_RULES`, -value rules a service lists in ECSCTX_MASK_VALUE_RULES. Every test here runs -with them configured, but the last, which runs with nothing configured: core -masks no wallet token of its own accord. - -- Apple Pay (PKPaymentToken's `paymentData`): an object whose `version` is - `EC_v1` or `RSA_v1`, with the encrypted `data`. `signature` and `header` - are useless without it, so the whole object is the label. -- Google Pay (the payment method token): an object, or a JSON string, whose - `protocolVersion` is `ECv1`, `ECv2` or `ECv2SigningOnly`, with a - `signedMessage`. - -Every test runs with a keyset and without one. +"""Ottu's wallet tokens in logs (#159487, decided 2026-10-01). + +Apple Pay and Google Pay tokens ship exactly as sent: a token is +single-use, and the token as it was sent is what debugs a wallet payment. +Saved-card tokens stay masked. + +- `WALLET_RULES` are keep rules: a token they match is left as sent under any + key -- `paymentData`, TAP's `token_data`, MPGS's `paymentToken` (a JSON + string), Google Pay's `tokenizationData.token`, KPay's `` -- and in a + message or a body, and no rule reads into it. The matchers are strict: a + token with a key no wallet writes is walked as before. +- `WALLET_SAD_RULES` are 0.16.0's label rules: the token's payment data is + `[SAD-MASKED]`, for a service that must not log it. + +The shapes are Ottu's, not ecsctx's: value rules a service lists in +ECSCTX_MASK_VALUE_RULES. With nothing configured core keeps and labels no +wallet token. Every test runs with a keyset and without one, in the default +pack and with every pack. """ import json @@ -29,11 +23,21 @@ import pytest -from ecsctx.contrib.net import loggable_request_body, redact_body -from ecsctx.contrib.ottu.masking import WALLET_RULES +from ecsctx.contrib.net import configure_redaction, loggable_request_body, redact_body +from ecsctx.contrib.ottu.masking import ( + APPLE_PAY_KEEP_RULE, + APPLE_PAY_RULE, + APPLE_PAY_TOKEN_KEEP_RULE, + GOOGLE_PAY_KEEP_RULE, + GOOGLE_PAY_RULE, + WALLET_RULES, + WALLET_SAD_RULES, +) +from ecsctx.masking import is_kept, patterns from ecsctx.masking.config import configure_masking_packs, configure_masking_value_rules from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for +from ecsctx.masking.value_rules import is_keep_rule from ecsctx.pii import configure_pii from ecsctx.processors import mask_sensitive_data @@ -71,6 +75,56 @@ } GOOGLE_PAY_V1 = {"signature": "MEQCIA==", "protocolVersion": "ECv1", "signedMessage": '{"encryptedMessage":"ZW5j"}'} NOT_A_WALLET = {"amount": "10.000", "currency": "KWD"} +TOKENS = [APPLE_PAY, APPLE_PAY_RSA, PK_PAYMENT_TOKEN, GOOGLE_PAY, GOOGLE_PAY_V1] +TOKEN_IDS = ["apple-pay", "apple-pay-rsa", "pk-payment-token", "google-pay", "google-pay-v1"] + +# Google Pay's PaymentData: the token is a JSON string under +# tokenizationData.token, beside the payer's own details. +GOOGLE_PAYMENT_DATA = { + "apiVersion": 2, + "apiVersionMinor": 0, + "email": "jane.roe@example.com", + "paymentMethodData": { + "type": "CARD", + "description": "Visa 1234", + "info": { + "cardNetwork": "VISA", + "cardDetails": "1234", + "billingAddress": {"name": "Jane Roe", "postalCode": "13001", "phoneNumber": "+96550000000"}, + }, + "tokenizationData": {"type": "PAYMENT_GATEWAY", "token": json.dumps(GOOGLE_PAY)}, + }, +} +# ottu_pg's webhook sends the saved card under `token`. +SAVED_CARD = { + "token": "tok_saved_card_fake_0001", + "brand": "VISA", + "number": "4111111111111111", + "expiry_month": "12", + "expiry_year": "27", + "pg_code": "mpgs", +} +KPAY_BODY = ( + "TRANPORTAL123S3cretPassw0rd1" + "10.000TRK1{}" +) +PASSWORD = "password=s3cret-Hunter2" +# A token written so that what parses is not what ships: a duplicate key +# json.loads drops, a comment ast.literal_eval drops (review C1). +DUP_DATA = ( + '{"version":"EC_v1","data":"4111111111111111","data":"Zm9v","signature":"c2ln",' + '"header":{"transactionId":"abc"}}' +) +DUP_HEADER = ( + '{"version":"EC_v1","data":"Zm9v","signature":"c2ln",' + '"header":{"cvv":"123"},"header":{"transactionId":"abc"}}' +) +NESTED_DUP = { + "signature": "MEQC", + "protocolVersion": "ECv2", + "signedMessage": '{"encryptedMessage": {"cvv": "123", "number": "4111111111111111"}, "encryptedMessage": "ZW5j"}', +} +REPR_COMMENT = "{'version': 'EC_v1', 'data': 'Zm9v', 'header': {'transactionId': 'abc' # 4111111111111111 cvv=123\n}}" PACKS = [["default"], sorted(ALL_PACKS)] @@ -107,12 +161,226 @@ def _message(msg: str, *args) -> str: return record.getMessage() +def _unconfigured(mask): + """What ``mask()`` gives with no value rule, as before WALLET_RULES.""" + configure_masking_value_rules(()) + try: + return mask() + finally: + configure_masking_value_rules(WALLET_RULES) + + def _holds_no_wallet_data(text: str) -> None: for secret in ("ZGF0YWJsb2JkYXRhYmxvYg==", "c2lnbmF0dXJlLWJsb2I=", "ZXBoZW1lcmFsLWtleQ==", "MEQCIGZha2Utc2lnbmF0dXJl"): assert secret not in text -class TestTheKeyWalk: +class TestTheRules: + def test_wallet_rules_are_the_keep_rules(self): + assert WALLET_RULES == (APPLE_PAY_TOKEN_KEEP_RULE, APPLE_PAY_KEEP_RULE, GOOGLE_PAY_KEEP_RULE) + assert all(is_keep_rule(rule) for rule in WALLET_RULES) + + def test_wallet_sad_rules_are_0_16_0s(self): + assert WALLET_SAD_RULES == (APPLE_PAY_RULE, GOOGLE_PAY_RULE) + assert {rule.field_type for rule in WALLET_SAD_RULES} == {"sad"} + + @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) + def test_each_token_is_kept(self, token): + assert is_kept(token) + assert is_kept(json.dumps(token)) + + @pytest.mark.parametrize( + "value", + [ + {**APPLE_PAY, "extra": "x"}, + {**APPLE_PAY, "version": "EC_v2"}, + {**APPLE_PAY, "data": {"blob": "x"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "extra": "x"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "publicKeyHash": 1}}, + {**APPLE_PAY, "header": "cGto"}, + {key: value for key, value in APPLE_PAY.items() if key != "data"}, + {**PK_PAYMENT_TOKEN, "billingContact": {"givenName": "Jane"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "billingAddress": "x"}}, + {**PK_PAYMENT_TOKEN, "paymentData": NOT_A_WALLET}, + {**GOOGLE_PAY, "extra": "x"}, + {**GOOGLE_PAY, "protocolVersion": "v3"}, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "x": 1}}, + {key: value for key, value in GOOGLE_PAY.items() if key != "signedMessage"}, + NOT_A_WALLET, + # Every leaf a wallet writes is text (review I2): a pair-labelled + # CVV rode in a slot whose type nothing checked. + {**APPLE_PAY, "signature": [{"name": "cvv", "value": "123"}]}, + {**APPLE_PAY, "signature": ["c2ln"]}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": {"name": "securityCode", "value": "123"}}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "network": ["Visa"]}}, + {**PK_PAYMENT_TOKEN, "transactionIdentifier": 7134}, + {"protocolVersion": "ECv2", "signedMessage": {"name": "cvv", "value": "123"}, "signature": "x"}, + {**GOOGLE_PAY, "signedMessage": {"encryptedMessage": "ZW5j"}}, + {**GOOGLE_PAY, "signature": 1}, + {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": {"keyValue": "a2V5"}, "signatures": ["c2ln"]}}, + {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": "c2ln"}}, + {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": [{"x": "c2ln"}]}}, + ], + ids=[ + "apple-extra-key", "apple-version", "apple-data-not-text", "apple-header-extra-key", + "apple-header-not-text", "apple-header-not-a-mapping", "apple-no-data", "pk-extra-key", + "pk-payment-method-extra-key", "pk-no-wallet", "google-extra-key", "google-version", + "google-signing-key-extra-key", "google-no-signed-message", "not-a-wallet", + "apple-signature-pair", "apple-signature-list", "pk-display-name-pair", "pk-network-list", + "pk-transaction-identifier-number", "google-signed-message-pair", "google-signed-message-mapping", + "google-signature-number", "google-signed-key-mapping", "google-signatures-text", + "google-signatures-mappings", + ], + ) + def test_the_matchers_are_strict(self, value): + assert not is_kept(value) + + +class TestWalletTokensShipAsSent: + @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) + def test_under_token(self, token): + assert _walk({"token": token}) == {"token": token} + + def test_under_payload_content(self): + event = {"payload": {"content": PK_PAYMENT_TOKEN, "amount": "10.000"}} + assert _walk(event) == event + + @pytest.mark.parametrize("key", ["paymentData", "token_data", "apple_pay", "payload", "udf9", "token", "card"]) + @pytest.mark.parametrize("token", [APPLE_PAY, APPLE_PAY_RSA], ids=["ec", "rsa"]) + def test_payment_data_under_any_key(self, key, token): + assert _walk({key: token}) == {key: token} + + @pytest.mark.parametrize("key", ["token", "paymentToken", "payload", "note"]) + @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) + def test_the_token_as_json_text(self, key, token): + text = json.dumps(token) + assert _walk({key: text}) == {key: text} + + def test_google_pay_under_tokenization_data_the_rest_masked_as_before(self): + before = _unconfigured(lambda: _walk({"google_pay_payload": GOOGLE_PAYMENT_DATA})) + after = _walk({"google_pay_payload": GOOGLE_PAYMENT_DATA}) + tokenized = after["google_pay_payload"]["paymentMethodData"]["tokenizationData"] + assert tokenized["token"] == json.dumps(GOOGLE_PAY) + assert before["google_pay_payload"]["paymentMethodData"]["tokenizationData"]["token"] != tokenized["token"] + before["google_pay_payload"]["paymentMethodData"]["tokenizationData"]["token"] = tokenized["token"] + assert after == before + assert after["google_pay_payload"]["email"] != "jane.roe@example.com" + + def test_the_mpgs_device_payment_body(self): + body = { + "apiOperation": "PAY", + "sourceOfFunds": { + "type": "CARD", + "provided": {"card": {"devicePayment": {"paymentToken": json.dumps(PK_PAYMENT_TOKEN)}}}, + }, + "order": {"amount": "10.000", "currency": "KWD", "walletProvider": "APPLE_PAY"}, + } + assert _walk({"payload": body}) == {"payload": body} + assert json.loads(loggable_request_body(None, body)) == body + + @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) + def test_a_message_and_a_repr(self, token, packs): + for written in (json.dumps(token), repr(token)): + text = f"apple pay token {written} received, {PASSWORD}" + masked = _text(text, packs) + assert written in masked + assert "s3cret-Hunter2" not in masked + assert _text(masked, packs) == masked + assert _message("token %s", json.dumps(token)) == f"token {json.dumps(token)}" + assert _message("token %r", token) == f"token {token!r}" + assert _walk({"event": f"got {json.dumps(token)}"}) == {"event": f"got {json.dumps(token)}"} + + @pytest.mark.parametrize("token", [APPLE_PAY, PK_PAYMENT_TOKEN], ids=["payment-data", "pk-payment-token"]) + def test_kpay_udf9_through_redact_body_and_the_processor(self, token, packs): + written = json.dumps(token) + body = KPAY_BODY.format(written) + for masked in (redact_body(body), _walk({"http": {"body": body}})["http"]["body"]): + assert f"{written}" in masked + assert '"transactionId": "7134e7d22988391fa183a61a191ae14c"' in masked + assert "S3cretPassw0rd" not in masked + + @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) + def test_entity_encoded_and_with_connects_extra_secret_keys(self, token): + configure_redaction(extra_secret_keys=["signature", "hash"]) + plain = json.dumps(token) + for written in (plain, plain.replace('"', """)): + masked = redact_body(KPAY_BODY.format(written)) + assert f"{written}" in masked + assert "S3cretPassw0rd" not in masked + masked = json.loads(redact_body(json.dumps({"paymentToken": plain, "signature": "outer-signature"}))) + assert masked["paymentToken"] == plain + assert masked["signature"] != "outer-signature" + + def test_a_cvv_beside_it_is_still_masked(self): + masked = _walk({"payload": {"paymentData": APPLE_PAY, "cvv": "123"}}) + assert masked == {"payload": {"paymentData": APPLE_PAY, "cvv": "[CVV-MASKED]"}} + + @pytest.mark.parametrize("written", ["mapping", "json-text"]) + def test_nothing_under_a_cvv_key_is_kept(self, written, packs): + token = APPLE_PAY if written == "mapping" else json.dumps(APPLE_PAY) + _holds_no_wallet_data(json.dumps(_walk({"cvv": token}))) + _holds_no_wallet_data(redact_body(f"{json.dumps(APPLE_PAY)}")) + _holds_no_wallet_data(_text(f"{json.dumps(APPLE_PAY)}", packs)) + if "financial_ids" in packs: + # Not kept, so the payment-id rule reads its transactionId; the + # default pack has no rule for an object after a CVV key in text. + text = f'{{"securityCode": {json.dumps(APPLE_PAY)}}}' + assert "7134e7d22988391fa183a61a191ae14c" not in _text(text, packs) + + @pytest.mark.parametrize("written", [DUP_DATA, DUP_HEADER], ids=["duplicate-data", "duplicate-header"]) + def test_a_token_with_a_duplicate_key_is_masked_as_without_the_rules(self, written, packs): + def every_path(): + return ( + _message("body %s", written), + _message(written), + _walk({"payload": written}), + _walk({"paymentToken": written}), + _text(f"token {written} end", packs), + redact_body(f'{{"token": {written}, "amt": "1"}}'), + redact_body(json.dumps({"paymentToken": written})), + ) + + assert every_path() == _unconfigured(every_path) + assert not is_kept(written) + + def test_a_json_text_leaf_with_a_duplicate_key_is_card_data(self, packs): + assert _walk({"payload": NESTED_DUP}) == _unconfigured(lambda: _walk({"payload": NESTED_DUP})) + assert not is_kept(NESTED_DUP) + text = f"got {json.dumps(NESTED_DUP)}" + assert _text(text, packs) == _unconfigured(lambda: _text(text, packs)) + + def test_a_repr_with_a_comment_is_not_kept(self, packs): + text = f"got {REPR_COMMENT} end" + masked = _text(text, packs) + assert masked == _unconfigured(lambda: _text(text, packs)) + assert "cvv=123" not in masked + + def test_a_payment_data_that_is_no_wallet_is_masked_as_usual(self): + masked = _walk({"paymentData": {"cardNumber": "4111111111111111", "email": "jane.roe@example.com"}}) + assert masked["paymentData"]["cardNumber"] == "411111******1111" + assert masked["paymentData"]["email"] != "jane.roe@example.com" + + def test_a_token_with_extra_keys_is_not_kept_whole(self): + wrapper = {**PK_PAYMENT_TOKEN, "cardholderName": "Jane Roe"} + masked = _walk({"token": wrapper})["token"] + assert masked["paymentData"] == APPLE_PAY + assert masked["cardholderName"] != "Jane Roe" + + @pytest.mark.parametrize("saved", ["tok_saved_card_fake_0001", SAVED_CARD], ids=["string", "card-object"]) + def test_saved_card_tokens_stay_masked(self, saved): + before = _unconfigured(lambda: _walk({"token": saved})) + assert before != {"token": saved} + assert _walk({"token": saved}) == before + assert "4111111111111111" not in json.dumps(before) + + +class TestTheSadVariant: + """0.16.0's assertions, with WALLET_SAD_RULES.""" + + @pytest.fixture(autouse=True) + def wallet_rules(self): + configure_masking_value_rules(WALLET_SAD_RULES) + @pytest.mark.parametrize("key", ["paymentData", "token_data", "apple_pay", "payload", "udf9", "token", "card"]) def test_an_apple_pay_token_under_any_key_is_the_label(self, key): assert _walk({key: APPLE_PAY}) == {key: SAD} @@ -132,8 +400,6 @@ def test_a_google_pay_token_object_is_the_label(self, token): @pytest.mark.parametrize("key", ["token", "paymentToken", "payload"]) @pytest.mark.parametrize("token", [APPLE_PAY, GOOGLE_PAY]) def test_a_token_as_a_json_string_is_the_label(self, key, token): - # MPGS's paymentToken, Google Pay's tokenizationData.token: a JSON - # string under a credential key was hashed. assert _walk({key: json.dumps(token)}) == {key: SAD} def test_the_mpgs_device_payment_body(self): @@ -162,7 +428,6 @@ def test_a_token_in_a_list(self): assert _walk({"tokens": [APPLE_PAY, json.dumps(GOOGLE_PAY)]}) == {"tokens": [SAD, SAD]} def test_a_wrapper_as_a_string_under_a_credential_key_is_the_label(self): - # Never hashed, even with the token inside something else. assert _walk({"token": json.dumps(PK_PAYMENT_TOKEN)}) == {"token": SAD} def test_a_wrapper_as_a_string_under_any_other_key_keeps_its_shape(self): @@ -187,8 +452,6 @@ def test_a_record_with_args(self): assert _message("apple pay token %s", json.dumps(APPLE_PAY)) == f"apple pay token {SAD}" assert _message("token %(t)s", {"t": APPLE_PAY}) == f"token {SAD}" - -class TestText: @pytest.mark.parametrize("token", [APPLE_PAY, APPLE_PAY_RSA, GOOGLE_PAY, GOOGLE_PAY_V1]) def test_a_token_in_a_message_is_the_label(self, token, packs): assert _text(f"apple pay token {json.dumps(token)} received", packs) == f"apple pay token {SAD} received" @@ -227,8 +490,6 @@ def test_masking_twice_masks_once(self, packs): assert "ptok:v1:" not in once assert _text(once, packs) == once - -class TestBodies: def test_redact_body(self): masked = redact_body(json.dumps({"paymentData": APPLE_PAY, "amount": "10.000"})) assert json.loads(masked) == {"paymentData": SAD, "amount": "10.000"} @@ -243,10 +504,7 @@ def test_loggable_request_body(self): assert json.loads(masked) == {"paymentToken": SAD, "amount": "10.000"} def test_kpay_udf9_through_redact_body_and_the_processor(self, packs): - body = ( - "TRANPORTAL123S3cretPassw0rd1" - f"10.000TRK1{json.dumps(PK_PAYMENT_TOKEN)}" - ) + body = KPAY_BODY.format(json.dumps(PK_PAYMENT_TOKEN)) for masked in (redact_body(body), _walk({"http": {"body": body}})["http"]["body"]): _holds_no_wallet_data(masked) udf9 = masked.split("")[1].split("")[0] @@ -254,7 +512,6 @@ def test_kpay_udf9_through_redact_body_and_the_processor(self, packs): assert json.loads(udf9)["paymentMethod"] == PK_PAYMENT_TOKEN["paymentMethod"] def test_kpay_udf9_holding_the_payment_data_itself(self): - # What KPay's client sends: udf9 is the paymentData object, unwrapped. body = f"7134E7D2{json.dumps(APPLE_PAY)}10.000" expected = f"7134E7D2{SAD}10.000" assert redact_body(body) == expected @@ -267,11 +524,21 @@ def test_an_entity_encoded_token_in_an_element(self): def test_a_credential_element_holding_a_token_is_the_label_not_its_hash(self): assert redact_body(f"{json.dumps(APPLE_PAY)}") == f"{SAD}" + def test_the_span_finder_never_runs(self, monkeypatch, packs): + calls: list = [] + monkeypatch.setattr(patterns, "kept_spans", lambda *args: calls.append(args) or []) + text = f"apple pay token {json.dumps(PK_PAYMENT_TOKEN)} {json.dumps(GOOGLE_PAY)}" + _text(text, packs) + _message(text) + redact_body(KPAY_BODY.format(text)) + _walk({"event": text, "token": PK_PAYMENT_TOKEN, "note": json.dumps(GOOGLE_PAY)}) + assert calls == [] + class TestNothingConfigured: - """Core names no wallet shape: without WALLET_RULES in - ECSCTX_MASK_VALUE_RULES, a wallet token is not the label -- the opt-in is - explicit.""" + """Core names no wallet shape: without Ottu's rules in + ECSCTX_MASK_VALUE_RULES, a wallet token is neither kept nor labelled -- + the opt-in is explicit.""" @pytest.fixture(autouse=True) def no_value_rules(self, wallet_rules): @@ -283,6 +550,12 @@ def test_a_token_is_not_labelled_by_core(self, packs): assert SAD not in redact_body(json.dumps({"paymentData": APPLE_PAY})) assert SAD not in _walk({"google_pay": json.dumps(GOOGLE_PAY)})["google_pay"] + def test_a_token_is_not_kept_by_core(self): + assert _walk({"paymentData": APPLE_PAY}) != {"paymentData": APPLE_PAY} + assert not is_kept(APPLE_PAY) + def test_the_setting_opts_in(self): configure_masking_value_rules(["ecsctx.contrib.ottu.masking.WALLET_RULES"]) + assert _walk({"paymentData": APPLE_PAY}) == {"paymentData": APPLE_PAY} + configure_masking_value_rules(["ecsctx.contrib.ottu.masking.WALLET_SAD_RULES"]) assert _walk({"paymentData": APPLE_PAY}) == {"paymentData": SAD} diff --git a/tests/test_masking_golden.py b/tests/test_masking_golden.py index 620a1ea..188e569 100644 --- a/tests/test_masking_golden.py +++ b/tests/test_masking_golden.py @@ -9,6 +9,11 @@ To capture them again, deliberately, when an output is meant to change: `ECSCTX_WRITE_GOLDEN=1 pytest tests/test_masking_golden.py`. + +The file was captured with Ottu's wallet shapes as label rules, which 0.17.0 +calls WALLET_SAD_RULES; WALLET_RULES became keep rules, which ship a wallet +token as sent. With them, every input that holds no wallet hint must still +mask exactly as the file says. """ import base64 @@ -20,7 +25,7 @@ from pathlib import Path from ecsctx.contrib.net import redact_body -from ecsctx.contrib.ottu.masking import WALLET_RULES +from ecsctx.contrib.ottu.masking import WALLET_RULES, WALLET_SAD_RULES from ecsctx.masking.config import configure_masking_packs, configure_masking_value_rules from ecsctx.masking.filters import MaskPIIFilter, _Pass from ecsctx.masking.patterns import ( @@ -179,10 +184,10 @@ def _changed(text: str, masked: str) -> str | None: return None if masked == text else masked -def _outputs(tmp: Path) -> dict: - # As an Ottu service runs: its wallet shapes are value rules it lists, - # no longer core's, and the corpus holds wallet tokens. - configure_masking_value_rules(WALLET_RULES) +def _outputs(tmp: Path, value_rules: tuple = WALLET_SAD_RULES) -> dict: + # As an Ottu service ran on 0.16.0: its wallet shapes are value rules it + # lists, no longer core's, and the corpus holds wallet tokens. + configure_masking_value_rules(value_rules) texts, events = _texts(), _events() outputs: dict = {} for mode in ("no-keyset", "keyset"): @@ -237,3 +242,34 @@ def test_masking_outputs_are_byte_identical_to_the_golden_file(tmp_path): for index, expected in enumerate(golden[key]): assert outputs[key][index] == expected, (key, index) assert text + "\n" == GOLDEN.read_text(encoding="utf-8") + + +# What every wallet token's text holds: Ottu's rules' hints. +WALLET_HINTS = ("EC_v1", "RSA_v1", "ECv1", "ECv2") + + +def _holds_a_wallet_hint(text: str) -> bool: + return any(hint in text for hint in WALLET_HINTS) + + +def test_with_the_keep_rules_every_input_without_a_wallet_hint_masks_as_the_golden_file_says(tmp_path): + outputs = _outputs(tmp_path, WALLET_RULES) + texts = _texts() + inputs = { + "redact_body": texts, + "text": texts, + "prose": texts, + "events": [json.dumps(event) for event in _events()], + "records": [f"{msg} {args!r}" for msg, args in RECORDS], + } + golden = json.loads(GOLDEN.read_text(encoding="utf-8")) + compared = skipped = 0 + for key in sorted(golden): + written = inputs[key.rsplit("/", 1)[1]] + for index, expected in enumerate(golden[key]): + if _holds_a_wallet_hint(written[index]): + skipped += 1 + continue + assert outputs[key][index] == expected, (key, index) + compared += 1 + assert compared > 10 * skipped > 0 diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py new file mode 100644 index 0000000..31b343b --- /dev/null +++ b/tests/test_masking_keep_rules.py @@ -0,0 +1,796 @@ +"""Keep rules: a service's own shapes of value that ship in logs as sent +(ECSCTX_MASK_VALUE_RULES). + +A keep rule is a value rule that writes no label: a value it matches is left +exactly as it came, under any key, a credential's included, and the walk does +not descend into it. Rules are asked in list order and the first match wins, +so a label rule listed first still labels the same value. Two things no keep +rule can override: + +- the floor: nothing is kept under a CVV or SAD key, or inside a CVV or SAD + container, a pair labelled as one included; +- the guard: a match that holds a card, CVV or SAD key, or a card number, at + any depth, is walked as if nothing matched. + +These tests use a shape of their own. Every test runs with a keyset and +without one, and in the default pack and with every pack. +""" + +import copy +import json +import logging + +import pytest +from django.test import override_settings + +from ecsctx.contrib.django.checks import find_masking_errors +from ecsctx.contrib.net import ( + configure_redaction, + loggable_body, + loggable_request_body, + redact_body, + redact_url, +) +from ecsctx.masking import ( + KeepRule, + ValueRule, + configure_masking_value_rules, + get_masking_value_rules, + is_kept, + mask_outside_kept, + patterns, + value_rules, +) +from ecsctx.masking.config import configure_masking_packs, configure_masking_safe_keys +from ecsctx.masking.filters import MaskPIIFilter +from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for +from ecsctx.masking.value_rules import keep_rules, label_rules +from ecsctx.pii import configure_pii +from ecsctx.processors import mask_sensitive_data + +BOX = "c2VhbGVkLWJveC1jb250ZW50cw==" +# What ships as sent: a credential and a payment id the key rules would mask, +# and an IBAN-shaped hex id the IBAN rule would. +SEALED = { + "kind": "sealed-box", + "box": BOX, + "api_key": "sk_test_fake_0123456789", + "transaction_id": "TXN-0000-0001", + "ref": "BE71A9F3C2D4E5F6B7C8", +} +NOT_SEALED = {"kind": "note", "api_key": "sk_test_fake_0123456789"} +ASKED: list = [] + + +def _is_sealed(value) -> bool: + ASKED.append(value) + return value.get("kind") == "sealed-box" and "box" in value + + +KEEP_RULE = KeepRule(_is_sealed, hints=("sealed-box",)) +KEEP_RULES = (KEEP_RULE,) +LABEL_RULE = ValueRule("sad", _is_sealed, hints=("sealed-box",)) +LABEL = "[SAD-MASKED]" +OUTER_RULE = ValueRule("secret", lambda value: value.get("kind") == "outer-box", hints=("outer-box",)) +# A label rule with a hint of its own, which never matches. +OTHER_RULE = ValueRule("secret", lambda value: value.get("kind") == "never", hints=("other-shape",)) +PASSWORD = "password=s3cret-Hunter2" + + +class DuckKeep: + """A keep rule by protocol alone: `keep` is True and `matches` is callable.""" + + keep = True + hints = ("sealed-box",) + + def matches(self, value) -> bool: + return _is_sealed(value) + + +class KeepYes: + """Not a rule: `keep` must be True itself, and there is no field_type.""" + + keep = "yes" + + def matches(self, value) -> bool: + return True + + +DUCK_KEEP = DuckKeep() +KEEP_YES = KeepYes() + +PACKS = [["default"], sorted(ALL_PACKS)] + + +@pytest.fixture(autouse=True, params=["keyset", "no-keyset"]) +def mode(request, token_keyset_path): + if request.param == "keyset": + configure_pii(token_keyset_path=token_keyset_path, env="test") + ASKED.clear() + return request.param + + +@pytest.fixture(autouse=True, params=PACKS, ids=["default", "every-pack"]) +def packs(request): + configure_masking_packs(request.param) + return frozenset(request.param) + + +@pytest.fixture +def keep(): + configure_masking_value_rules(KEEP_RULES) + + +def _walk(event: dict) -> dict: + return mask_sensitive_data(None, "info", json.loads(json.dumps(event))) + + +def _text(text: str, packs: frozenset[str]) -> str: + return mask_by_patterns(text, rules_for(packs | {"default"})) + + +def _message(msg, *args) -> str: + record = logging.LogRecord("t", logging.INFO, __file__, 0, msg, args, None) + assert MaskPIIFilter().filter(record) is True + return record.getMessage() + + +def _holds_no_box(masked) -> None: + text = masked if isinstance(masked, str) else json.dumps(masked) + assert BOX not in text + + +def _masked_as_a_walk(masked: dict) -> None: + """What the key walk makes of SEALED when nothing keeps it.""" + assert masked["kind"] == "sealed-box" + assert masked["api_key"] != SEALED["api_key"] + + +class TestConfiguration: + def test_a_keep_rule_object_loads(self): + configure_masking_value_rules(KEEP_RULES) + assert get_masking_value_rules() == KEEP_RULES + assert _walk({"x": SEALED}) == {"x": SEALED} + + def test_a_dotted_path_to_a_rule_or_a_collection(self): + configure_masking_value_rules(["tests.test_masking_keep_rules.KEEP_RULE"]) + assert get_masking_value_rules() == (KEEP_RULE,) + configure_masking_value_rules("tests.test_masking_keep_rules.KEEP_RULES") + assert _walk({"x": SEALED}) == {"x": SEALED} + + def test_the_environment_variable(self, monkeypatch): + monkeypatch.setenv("ECSCTX_MASK_VALUE_RULES", "tests.test_masking_keep_rules.KEEP_RULES") + assert _walk({"x": SEALED}) == {"x": SEALED} + + def test_the_django_setting(self): + with override_settings(ECSCTX_MASK_VALUE_RULES=["tests.test_masking_keep_rules.KEEP_RULES"]): + assert _walk({"x": SEALED}) == {"x": SEALED} + + def test_nothing_configured_keeps_nothing(self): + _masked_as_a_walk(_walk({"x": SEALED})["x"]) + + def test_a_duck_typed_keep_rule_loads(self): + configure_masking_value_rules([DUCK_KEEP]) + assert get_masking_value_rules() == (DUCK_KEEP,) + assert _walk({"x": SEALED}) == {"x": SEALED} + + def test_keep_must_be_true_itself(self): + with pytest.raises(ValueError, match="keep = True"): + configure_masking_value_rules([KEEP_YES]) + + @pytest.mark.parametrize("hints", [(1,), "sealed-box", None], ids=["not-text", "one-string", "none"]) + def test_hints_must_be_a_collection_of_strings(self, hints): + # Looked for in text as it is masked, anything else raised there. + with pytest.raises(ValueError, match="hints"): + configure_masking_value_rules([KeepRule(_is_sealed, hints=hints)]) + with pytest.raises(ValueError, match="hints"): + configure_masking_value_rules([ValueRule("sad", _is_sealed, hints=hints)]) + + def test_label_and_keep_rules_share_one_list(self): + rules = (LABEL_RULE, KEEP_RULE, DUCK_KEEP) + configure_masking_value_rules(rules) + assert get_masking_value_rules() == rules + assert keep_rules(rules) == (KEEP_RULE, DUCK_KEEP) + assert label_rules(rules) == (LABEL_RULE,) + assert keep_rules(rules) is keep_rules(rules) + + def test_a_bad_item_is_reported_by_the_boot_check(self, monkeypatch): + monkeypatch.setenv( + "ECSCTX_MASK_VALUE_RULES", + "tests.test_masking_keep_rules.KEEP_YES,tests.test_masking_keep_rules.KEEP_RULES", + ) + with pytest.warns(RuntimeWarning, match="KEEP_YES"): + assert _walk({"x": SEALED}) == {"x": SEALED} + [error] = [error for error in find_masking_errors({}) if "ECSCTX_MASK_VALUE_RULES" in error] + assert "KEEP_YES" in error + + +@pytest.mark.usefixtures("keep") +class TestTheKeyWalk: + @pytest.mark.parametrize("key", ["payload", "token", "password", "card", "customer", "udf9"]) + def test_a_mapping_under_any_key_ships_as_sent(self, key): + assert _walk({key: SEALED}) == {key: SEALED} + + @pytest.mark.parametrize("key", ["payload", "token", "password", "card", "customer", "udf9"]) + def test_json_text_under_any_key_ships_as_sent(self, key): + text = json.dumps(SEALED) + assert _walk({key: text}) == {key: text} + + def test_the_message_itself(self): + text = json.dumps(SEALED) + assert _walk({"event": text}) == {"event": text} + assert _message(text) == text + + def test_a_kept_mapping_is_a_copy(self): + sealed = copy.deepcopy(SEALED) + masked = MaskPIIFilter()._mask_value({"payload": sealed}) + assert masked == {"payload": SEALED} + assert masked["payload"] is not sealed + + def test_its_siblings_and_list_items_are_masked_as_before(self): + masked = _walk({"blob": SEALED, "password": "s3cret-Hunter2", "items": [SEALED, "jane@example.com"]}) + assert masked["blob"] == SEALED + assert masked["password"] != "s3cret-Hunter2" + assert masked["items"][0] == SEALED + assert masked["items"][1] != "jane@example.com" + + def test_percent_args(self): + assert _message("got %s", SEALED) == f"got {SEALED}" + assert _message("got %s", json.dumps(SEALED)) == f"got {json.dumps(SEALED)}" + assert _message("got %(t)s", {"t": SEALED}) == f"got {SEALED}" + + @pytest.mark.parametrize( + "event", + [ + {"cvv": "VALUE"}, + {"securityCode": "VALUE"}, + {"pin": "VALUE"}, + {"cryptogram": "VALUE"}, + {"cvv": {"x": "VALUE"}}, + {"cvv": ["VALUE"]}, + {"name": "cvv", "value": "VALUE"}, + ], + ids=["cvv", "securityCode", "pin", "cryptogram", "in-a-cvv-container", "in-a-cvv-list", "cvv-pair"], + ) + @pytest.mark.parametrize("form", ["mapping", "json-text"]) + def test_the_floor_nothing_is_kept_under_a_cvv_or_sad_key(self, event, form): + value = SEALED if form == "mapping" else json.dumps(SEALED) + event = json.loads(json.dumps(event).replace('"VALUE"', json.dumps(value))) + masked = _walk({"payload": event}) + assert masked != {"payload": event} + _holds_no_box(masked) + + def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): + # A listed key reads through a CVV container; nothing below it is kept. + configure_masking_safe_keys(["acquirercode"]) + masked = _walk({"payload": {"cvv": {"acquirercode": SEALED}}}) + _masked_as_a_walk(masked["payload"]["cvv"]["acquirercode"]) + + @pytest.mark.parametrize( + "extra", + [ + {"cvv": "123"}, + {"cardNumber": "4111111111111111"}, + {"note": "4111 1111 1111 1111"}, + {"note": 4111111111111111}, + {"note": json.dumps({"inner": {"securityCode": "123"}})}, + {"deep": [{"pin": "1234"}]}, + ], + ids=["cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list"], + ) + def test_the_guard_a_match_holding_card_data_is_walked(self, extra): + holding = {**SEALED, **extra} + _masked_as_a_walk(_walk({"payload": holding})["payload"]) + assert _walk({"payload": json.dumps(holding)})["payload"] != json.dumps(holding) + + def test_a_rule_that_keeps_everything_still_cannot_ship_a_card(self): + configure_masking_value_rules([KeepRule(lambda value: True)]) + masked = _walk({"payment": {"card": {"number": "4111111111111111", "cvv": "123", "expiry": "12/27"}}}) + card = masked["payment"]["card"] + assert card["number"] == "411111******1111" + assert card["cvv"] == "[CVV-MASKED]" + assert card["expiry"] == "12/27" + + @pytest.mark.parametrize( + "pair", + [ + {"name": "cvv", "value": "123"}, + {"Name": "Security Code", "Value": "123"}, + {"label": "Card Number", "value": "4111111111111111"}, + {"field": "pin", "value": "1234"}, + {"key": "cardCvv", "value": "123"}, + ], + ids=["cvv", "security-code", "card-number-label", "pin-field", "card-cvv-key"], + ) + def test_the_guard_reads_a_pair_labelled_as_card_data(self, pair): + # The key walk masks a pair's value as its identifier names it, so the + # guard reads the pair as a key (review I2). + holding = {**SEALED, "fields": [pair]} + assert not is_kept(holding) + _masked_as_a_walk(_walk({"payload": holding})["payload"]) + configure_masking_value_rules([KeepRule(lambda value: True)]) + event = {"payment": {"fields": [pair]}} + assert _walk(event) == _unconfigured(lambda: _walk(event)) + assert _walk(event) != event + + def test_a_pair_labelled_as_anything_else_is_kept(self): + holding = {**SEALED, "fields": [{"name": "customer_note", "value": "gift"}]} + assert _walk({"payload": holding}) == {"payload": holding} + + @pytest.mark.parametrize( + "leaf", + [ + "8171999927660000", + "122000000000003", + "4111111111111112", + "3542323393147", + "15423233931470", + "1542 3233 9314 7", + ], + ids=["luhn-8-prefix", "uatp-1-prefix", "not-luhn", "13-digits-from-3", "14-digits-from-1", "13-in-groups"], + ) + def test_every_card_number_shape_but_epoch_milliseconds_blocks_keeping(self, leaf): + # Only Google Pay's keyExpiration shape is exempt (review M1): no + # issuer prefix or Luhn check lets another digit run through. + holding = {**SEALED, "note": leaf} + assert not is_kept(holding) + _masked_as_a_walk(_walk({"payload": holding})["payload"]) + + @pytest.mark.parametrize("expiration", ["1542323393147", "2000000000000"], ids=["2018", "after-may-2033"]) + def test_a_timestamp_is_not_a_card_number_to_the_guard(self, expiration): + # Google Pay's signedKey carries keyExpiration, epoch milliseconds: + # thirteen digits, shaped like a card number but no issuer's. From + # 18 May 2033 they start with a 2, and tokens must stay keepable. + holding = {**SEALED, "signedKey": json.dumps({"keyExpiration": expiration})} + assert _walk({"payload": holding}) == {"payload": holding} + + @pytest.mark.parametrize( + "event", + [ + {"token": {"wrapper": "VALUE"}}, + {"paymentToken": "VALUE_TEXT"}, + {"token_data": "VALUE"}, + {"tokenizationData": {"token": "VALUE_TEXT"}}, + {"card": {"devicePayment": {"paymentToken": "VALUE_TEXT"}}}, + ], + ids=["token", "paymentToken", "token_data", "tokenizationData.token", "card.devicePayment"], + ) + def test_a_credential_or_card_key_above_a_kept_value_does_not_stop_it(self, event): + text = json.dumps(event).replace('"VALUE_TEXT"', json.dumps(json.dumps(SEALED))) + event = json.loads(text.replace('"VALUE"', json.dumps(SEALED))) + assert _walk({"payload": event}) == {"payload": event} + + def test_list_order_decides_label_first(self): + configure_masking_value_rules([LABEL_RULE, KEEP_RULE]) + assert _walk({"x": SEALED, "y": json.dumps(SEALED)}) == {"x": LABEL, "y": LABEL} + + def test_list_order_decides_keep_first(self): + configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) + assert _walk({"x": SEALED, "y": json.dumps(SEALED)}) == {"x": SEALED, "y": json.dumps(SEALED)} + + def test_a_keep_match_the_guard_refuses_asks_no_rule_listed_after_it(self, packs): + configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) + holding = {**SEALED, "cvv": "123"} + masked = _walk({"x": holding})["x"] + _masked_as_a_walk(masked) + assert LABEL not in _text(f"got {json.dumps(holding)} back", packs) + + def test_a_keep_rule_that_raises_keeps_nothing_and_raises_nothing(self): + def broken(_value): + raise RuntimeError("rule failed") + + configure_masking_value_rules([KeepRule(broken)]) + _masked_as_a_walk(_walk({"x": SEALED})["x"]) + assert _message("got %s", SEALED) != "[MASKING-FAILED: RuntimeError]" + # ...and counts as no match: the next rule is asked. + configure_masking_value_rules([KeepRule(broken), LABEL_RULE]) + assert _walk({"x": SEALED}) == {"x": LABEL} + + def test_each_mapping_is_asked_once(self): + # _mask_dict asks the rules about every value it walks; the value is + # not asked again on the way down. + ASKED.clear() + _walk({"a": {"b": {"c": 1}}, "items": [{"d": 1}], "customer": {"e": {"f": 1}}}) + assert len(ASKED) == 5 + + def test_json_text_holding_a_kept_value_deeper_under_a_credential_key_is_hashed_whole(self): + # A keep never cuts a value out of a string. + masked = _walk({"password": json.dumps({"wrapper": SEALED})})["password"] + assert masked.startswith("ptok:v1:") or masked == "[SECRET-MASKED]" + + + +def _kept_in(masked: str, kept: str) -> None: + assert kept in masked + assert "s3cret-Hunter2" not in masked + + +@pytest.mark.usefixtures("keep") +class TestText: + def test_a_message_ships_it_as_sent_and_masks_the_rest(self, packs): + text = f"got {json.dumps(SEALED)} with {PASSWORD}" + for masked in (_text(text, packs), _message(text), _walk({"event": text})["event"]): + _kept_in(masked, json.dumps(SEALED)) + + def test_a_python_repr(self, packs): + text = f"got {SEALED!r} with {PASSWORD}" + _kept_in(_text(text, packs), repr(SEALED)) + _kept_in(_message("got %s with %s", SEALED, PASSWORD), repr(SEALED)) + + def test_json_in_a_json_string(self, packs): + text = "sent " + json.dumps({"blob": json.dumps(SEALED), "password": "s3cret-Hunter2"}) + _kept_in(_text(text, packs), json.dumps(json.dumps(SEALED))) + + def test_json_text_walked_by_its_keys_then_read_by_the_text_rules(self): + masked = json.loads(_walk({"body": json.dumps({"wrapper": SEALED, "password": "s3cret-Hunter2"})})["body"]) + assert masked["wrapper"] == SEALED + assert masked["password"] != "s3cret-Hunter2" + + def test_a_template_with_a_credential_word(self): + assert _message("token %s", json.dumps(SEALED)) == f"token {json.dumps(SEALED)}" + assert _message("token %(t)s", {"t": SEALED}) == f"token {SEALED}" + + @pytest.mark.parametrize( + "template", + ["token={}", '{{"token": {}}}', "Authorization: Bearer {}", "{}", "password: {}"], + ids=["token=", "quoted-token", "bearer", "password-element", "password:"], + ) + @pytest.mark.parametrize("separators", [None, (",", ":")], ids=["spaced", "compact"]) + def test_a_credential_before_it_does_not_stop_it(self, template, separators, packs): + sealed = json.dumps(SEALED, separators=separators) + text = f"{template.format(sealed)} {PASSWORD}" + _kept_in(_text(text, packs), sealed) + + @pytest.mark.parametrize( + "template", + ["{}", '{{"securityCode": {}}}', "{{'cvv': {}}}", "cvv={}", '{{"pin": "{}"}}'], + ids=["cvv-element", "securityCode", "repr-cvv", "cvv=", "pin-json-string"], + ) + def test_nothing_under_a_cvv_or_sad_key_is_kept(self, template, packs): + sealed = json.dumps(SEALED) + if template.endswith('"}}'): + sealed = json.dumps(sealed)[1:-1] + elif "'" in template: + sealed = repr(SEALED) + text = template.format(sealed) + assert _text(text, packs) != text + + def test_an_outer_label_match_wins(self, packs): + configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) + text = "got " + json.dumps({"kind": "outer-box", "inner": SEALED}) + assert _text(text, packs) == "got [SECRET-MASKED]" + + def test_an_inner_label_match_is_kept_with_the_kept_object(self, packs): + configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) + sealed = {**SEALED, "inner": {"kind": "outer-box"}} + text = f"got {json.dumps(sealed)}" + assert _text(text, packs) == text + + def test_a_label_rule_listed_first_labels_it(self, packs): + configure_masking_value_rules([LABEL_RULE, KEEP_RULE]) + assert _text(f"got {json.dumps(SEALED)}", packs) == f"got {LABEL}" + + def test_the_guard(self, packs): + holding = json.dumps({**SEALED, "note": json.dumps({"cvv": "123"})}) + assert _text(f"got {holding}", packs) != f"got {holding}" + + def test_a_keep_match_the_guard_refuses_is_read_as_one_nothing_matches(self, packs): + # Rule 2 too: a label match inside it is labelled (review I1). + text = "got " + json.dumps({"outer": 1, "cvv": "123", "inner": SEALED}) + " end" + configure_masking_value_rules([KeepRule(lambda value: "outer" in value), LABEL_RULE]) + masked, body = _text(text, packs), redact_body(text) + configure_masking_value_rules([LABEL_RULE]) + assert masked == _text(text, packs) + assert body == redact_body(text) + assert LABEL in masked + assert LABEL in body + + def test_masking_twice_masks_once(self, packs): + once = _text(f"a {json.dumps(SEALED)} b {PASSWORD} c {SEALED!r}", packs) + assert _text(once, packs) == once + assert "s3cret-Hunter2" not in once + + def test_text_already_holding_the_placeholder_is_masked_as_before(self, packs): + text = f"[KEPT-A-MASKED] {json.dumps(SEALED)}" + assert SEALED["api_key"] not in _text(text, packs) + + def test_a_placeholder_some_rule_destroyed_is_not_restored(self, packs): + # Inside a credential's quoted value the rule hashes the placeholder + # with the rest of the value: the kept object goes with it. + text = '{"password": "x ' + json.dumps(SEALED).replace('"', '\\"') + '"}' + masked = _text(text, packs) + assert BOX not in masked + assert "KEPT-" not in masked + + +@pytest.mark.usefixtures("keep") +class TestCost: + @pytest.fixture + def finder_calls(self, monkeypatch): + calls: list = [] + finder = patterns.kept_spans + + def counting(*args): + calls.append(args) + return finder(*args) + + monkeypatch.setattr(patterns, "kept_spans", counting) + return calls + + @pytest.mark.parametrize("rules", [(), (LABEL_RULE,)], ids=["nothing", "label-rules-only"]) + def test_the_span_finder_is_never_called_without_a_keep_rule(self, rules, finder_calls, packs): + configure_masking_value_rules(rules) + text = f"got {json.dumps(SEALED)} back" + _text(text, packs) + _message(text) + _walk({"event": text, "x": SEALED, "y": json.dumps({"z": SEALED})}) + encoded = json.dumps(SEALED).replace('"', """) + redact_body(f"{text} {encoded}") + assert finder_calls == [] + + def test_the_span_finder_runs_with_one(self, finder_calls, packs): + _text(f"got {json.dumps(SEALED)} back", packs) + assert finder_calls + + def test_a_keep_rule_is_never_asked_about_text_holding_none_of_its_hints(self, packs): + configure_masking_value_rules([KEEP_RULE, OTHER_RULE]) + ASKED.clear() + text = "got " + json.dumps({"kind": "other-shape", "nested": {"kind": "other-shape"}}) + _text(text, packs) + _message(text) + _walk({"x": json.dumps({"kind": "other-shape"}), "y": json.dumps({"z": {"kind": "other-shape"}})}) + encoded = text.replace('"', """) + redact_body(f"{text} {encoded}") + assert ASKED == [] + + def test_texts_whose_hints_select_some_rules_split_them_once(self, monkeypatch, packs): + # A text holding one rule's hints asks a selection of the rules: the + # same tuple for every such text, so its split is cached once and the + # configured rules' split is not emptied out of the cache with it. + configure_masking_value_rules([KEEP_RULE, OTHER_RULE]) + calls: list = [] + split_of = value_rules._split_of + monkeypatch.setattr(value_rules, "_splits", {}) + monkeypatch.setattr(value_rules, "_split_of", lambda rules: calls.append(rules) or split_of(rules)) + for n in range(100): + text = f"got {json.dumps({**SEALED, 'n': n})} back" + _text(text, packs) + _walk({"x": json.dumps({**SEALED, "n": n})}) + redact_body(KPAY.format(json.dumps({**SEALED, "n": n}).replace('"', """))) + assert len(calls) <= 2 + + def test_rule_2_does_not_run_with_keep_rules_only(self, monkeypatch, packs): + calls: list = [] + monkeypatch.setattr(patterns, "_mask_object", lambda m, rules: calls.append(m) or m.group(0)) + text = "got " + json.dumps({"kind": "note", "about": "sealed-box"}) + assert not patterns._has_value_object(text, text.lower()) + _text(text, packs) + assert calls == [] + configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) + assert patterns._has_value_object(text, text.lower()) + + +# Keys Connect lists as extra secret body keys (ECSCTX_REDACT_EXTRA_SECRET_KEYS). +SIGNED = {**SEALED, "signature": "c2lnbmF0dXJlLWJsb2I=", "hash": "aGFzaC1ibG9i"} +# Past the default body cap, as an Apple Pay token is (about 5 KB). +LARGE = {**SEALED, "box": "QUJD" * 1500} +KPAY = "TRANPORTAL123S3cretPassw0rd{}10.000" + + +class _Response: + def __init__(self, text: str) -> None: + self.text = text + self.headers = {"Content-Type": "application/json"} + self.status_code = 200 + + +@pytest.mark.usefixtures("keep") +class TestBodies: + def test_a_json_body(self): + masked = json.loads(redact_body(json.dumps({"blob": SEALED, "password": "s3cret-Hunter2"}))) + assert masked["blob"] == SEALED + assert masked["password"] != "s3cret-Hunter2" + + def test_json_text_in_a_json_string_under_a_credential_key(self): + # Split around the object, the JSON value rule left `""` behind. + body = json.dumps({"token": json.dumps(SEALED), "access_token": "s3cret-Hunter2"}) + masked = redact_body(body) + assert json.loads(masked)["token"] == json.dumps(SEALED) + assert "s3cret-Hunter2" not in masked + assert redact_body(masked) == masked + + def test_connects_extra_secret_keys_leave_it_intact(self): + configure_redaction(extra_secret_keys=["signature", "hash"]) + body = json.dumps({"blob": SIGNED, "signature": "outer-signature"}) + masked = json.loads(redact_body(body)) + assert masked["blob"] == SIGNED + assert masked["signature"] != "outer-signature" + + def test_an_element_holding_it_with_a_password_beside_it(self): + configure_redaction(extra_secret_keys=["signature", "hash"]) + body = KPAY.format(json.dumps(SIGNED)) + masked = redact_body(body) + assert f"{json.dumps(SIGNED)}" in masked + assert "S3cretPassw0rd" not in masked + assert redact_body(masked) == masked + + def test_an_entity_encoded_element(self): + configure_redaction(extra_secret_keys=["signature", "hash"]) + encoded = json.dumps(SIGNED).replace('"', """) + masked = redact_body(KPAY.format(encoded)) + assert f"{encoded}" in masked + assert "S3cretPassw0rd" not in masked + + def test_an_entity_encoded_element_whose_base64_reads_as_a_form_field(self): + # `…cvvXYZ==` is a form field named like a CVV to the form rule: the + # kept text is set aside with its element, so no rule reads it. + padded = {**SEALED, "box": "QUJDcvvXYZ=="} + encoded = json.dumps(padded).replace('"', """) + plain = json.dumps(padded) + assert f"{encoded}" in redact_body(KPAY.format(encoded)) + assert f"{plain}" in redact_body(KPAY.format(plain)) + + def test_loggable_request_body_with_a_raised_cap(self): + configure_redaction(body_log_cap=64 * 1024) + masked = json.loads(loggable_request_body(None, {"blob": LARGE, "password": "s3cret-Hunter2"})) + assert masked["blob"] == LARGE + assert masked["password"] != "s3cret-Hunter2" + + def test_loggable_body_with_a_raised_cap(self): + configure_redaction(body_log_cap=64 * 1024) + masked = json.loads(loggable_body(_Response(json.dumps({"blob": LARGE, "password": "s3cret-Hunter2"})))) + assert masked["blob"] == LARGE + assert masked["password"] != "s3cret-Hunter2" + + def test_the_default_cap_cuts_it(self): + assert len(loggable_request_body(None, {"blob": LARGE})) == 4096 + + def test_nothing_under_a_cvv_key_is_kept(self): + assert SEALED["api_key"] not in redact_body(KPAY.format(json.dumps({"cvv": SEALED}))) + assert BOX not in redact_body(f"{json.dumps(SEALED)}") + + def test_redact_url_never_raises_with_a_keep_rule_that_raises(self): + def broken(_value): + raise RuntimeError("rule failed") + + configure_masking_value_rules([KeepRule(broken)]) + url = "https://pg.example/pay?card_number=" + json.dumps(SEALED).replace(" ", "") + "&password=s3cret" + assert "s3cret" not in redact_url(url) + assert "s3cret-Hunter2" not in redact_body(f"{json.dumps(SEALED)} {PASSWORD}") + + +def _shout(text: str) -> str: + """A service's own pass: every word `secret` masked.""" + return text.replace("secret", "[OWN-MASKED]") + + +class TestHelpers: + @pytest.mark.usefixtures("keep") + def test_is_kept_for_a_mapping_and_for_json_text(self): + assert is_kept(SEALED) + assert is_kept(json.dumps(SEALED)) + assert is_kept(f" {json.dumps(SEALED, indent=2)}") + assert is_kept(SEALED, key="token") + + def test_is_kept_is_false_with_nothing_configured(self): + assert not is_kept(SEALED) + assert not is_kept(json.dumps(SEALED)) + + def test_is_kept_is_false_for_a_label_match(self): + configure_masking_value_rules([LABEL_RULE, KEEP_RULE]) + assert not is_kept(SEALED) + + @pytest.mark.usefixtures("keep") + @pytest.mark.parametrize("key", ["cvv", "securityCode", "pin", "cryptogram"]) + def test_is_kept_is_false_under_a_cvv_or_sad_key(self, key): + assert not is_kept(SEALED, key=key) + assert not is_kept(json.dumps(SEALED), key=key) + + @pytest.mark.usefixtures("keep") + def test_is_kept_is_false_when_the_guard_refuses_it(self): + assert not is_kept({**SEALED, "cvv": "123"}) + assert not is_kept(json.dumps({**SEALED, "note": "4111111111111111"})) + + @pytest.mark.usefixtures("keep") + @pytest.mark.parametrize("value", [NOT_SEALED, "not json", 42, None, ["sealed-box"], json.dumps([SEALED])]) + def test_is_kept_is_false_for_anything_else(self, value): + assert not is_kept(value) + + def test_is_kept_never_raises(self): + def broken(_value): + raise RuntimeError("rule failed") + + configure_masking_value_rules([KeepRule(broken), ValueRule("sad", broken)]) + assert not is_kept(SEALED) + assert not is_kept(json.dumps(SEALED)) + + @pytest.mark.usefixtures("keep") + def test_mask_outside_kept_masks_only_outside(self): + sealed = {**SEALED, "note": "secret"} + text = f"secret {json.dumps(sealed)} and secret" + assert mask_outside_kept(text, _shout) == f"[OWN-MASKED] {json.dumps(sealed)} and [OWN-MASKED]" + + @pytest.mark.usefixtures("keep") + def test_mask_outside_kept_does_not_restore_a_destroyed_placeholder(self): + masked = mask_outside_kept(f"secret {json.dumps(SEALED)}", lambda text: "[OWN-MASKED] everything") + assert masked == "[OWN-MASKED] everything" + masked = mask_outside_kept(f"a {json.dumps(SEALED)}", lambda text: text.replace("[KEPT-", "[GONE-")) + assert BOX not in masked + + def test_mask_outside_kept_with_nothing_configured_masks_it_all(self): + sealed = {**SEALED, "note": "secret"} + assert "secret" not in mask_outside_kept(f"secret {json.dumps(sealed)}", _shout) + + +# What ships is the text as written, so a keep decision rests on it as +# written: a duplicate key json.loads drops, or a comment ast.literal_eval +# drops, must not carry card data past the guard (review C1). +DUP_PAN = '{"kind": "sealed-box", "box": "4111111111111111", "box": "Zm9v", "api_key": "sk_test_fake_0123456789"}' +DUP_CVV = '{"kind": "sealed-box", "box": "Zm9v", "meta": {"cvv": "123"}, "meta": {"n": 1}}' +NESTED_DUP = { + "kind": "sealed-box", + "box": "Zm9v", + "inner": '{"blob": {"cvv": "123", "number": "4111111111111111"}, "blob": "ZW5j"}', +} +REPR_COMMENT = "{'kind': 'sealed-box', 'box': 'Zm9v', 'meta': {'n': 'abc' # 4111111111111111 cvv=123\n}}" + + +def _unconfigured(mask): + """What ``mask()`` gives with no value rule configured.""" + rules = get_masking_value_rules() + configure_masking_value_rules(()) + try: + return mask() + finally: + configure_masking_value_rules(rules) + + +@pytest.mark.usefixtures("keep") +class TestWhatShipsIsWhatWasJudged: + @pytest.mark.parametrize("written", [DUP_PAN, DUP_CVV, json.dumps(NESTED_DUP)], ids=["pan", "cvv", "nested"]) + def test_the_key_walk_keeps_no_json_text_with_a_duplicate_key(self, written): + for event in ({"payload": written}, {"token": written}, {"event": written}): + assert _walk(event) == _unconfigured(lambda event=event: _walk(event)) + assert _message(written) == _unconfigured(lambda: _message(written)) + assert _walk({"payload": written}) != {"payload": written} + + def test_a_json_text_leaf_with_a_duplicate_key_is_card_data(self): + assert _walk({"payload": NESTED_DUP}) == _unconfigured(lambda: _walk({"payload": NESTED_DUP})) + assert _walk({"payload": NESTED_DUP}) != {"payload": NESTED_DUP} + + @pytest.mark.parametrize("written", [DUP_PAN, DUP_CVV, json.dumps(NESTED_DUP), REPR_COMMENT]) + def test_text_keeps_no_object_written_otherwise_than_it_parses(self, written, packs): + text = f"got {written} end" + masked = _text(text, packs) + assert masked == _unconfigured(lambda: _text(text, packs)) + assert masked != text + escaped = "sent " + json.dumps({"blob": written}) + assert _text(escaped, packs) == _unconfigured(lambda: _text(escaped, packs)) + + def test_a_python_repr_that_round_trips_is_still_kept(self, packs): + text = f"got {SEALED!r} end" + assert _text(text, packs) == text + + @pytest.mark.parametrize("written", [DUP_PAN, json.dumps(NESTED_DUP)], ids=["pan", "nested"]) + def test_bodies(self, written): + body = f'{{"token": {written}, "amt": "1"}}' + masked = redact_body(body) + assert masked == _unconfigured(lambda: redact_body(body)) + if written == DUP_PAN: + assert SEALED["api_key"] not in masked + escaped = json.dumps({"paymentToken": written, "amt": "1"}) + assert redact_body(escaped) == _unconfigured(lambda: redact_body(escaped)) + + @pytest.mark.parametrize("written", [DUP_PAN, DUP_CVV, json.dumps(NESTED_DUP)], ids=["pan", "cvv", "nested"]) + def test_is_kept(self, written): + assert not is_kept(written) + + def test_is_kept_reads_a_json_text_leaf_strictly(self): + assert not is_kept(NESTED_DUP) + + def test_mask_outside_kept(self, packs): + def own(text: str) -> str: + return text.replace("123", "[OWN-MASKED]") + + text = f"got {DUP_CVV}" + assert mask_outside_kept(text, own) == own(text) diff --git a/tests/test_masking_value_rules.py b/tests/test_masking_value_rules.py index fe89baa..7e71d23 100644 --- a/tests/test_masking_value_rules.py +++ b/tests/test_masking_value_rules.py @@ -22,7 +22,7 @@ configure_masking_value_rules, get_masking_value_rules, ) -from ecsctx.masking.config import configure_masking_packs +from ecsctx.masking.config import _reset_masking_config, configure_masking_packs from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for from ecsctx.pii import configure_pii @@ -169,6 +169,23 @@ def test_a_setting_item_that_is_not_one_is_dropped_warned_and_reported(self, mon [error] = [error for error in find_masking_errors({}) if "ECSCTX_MASK_VALUE_RULES" in error] assert "no_such_module.RULES" in error + def test_configuring_them_forgets_the_strings_known_clean(self): + # The known-clean set is keyed on the content rules alone: a string + # masked clean with no value rule came back unmasked once one was on. + text = f"got {json.dumps(BLOB)} back" + configure_masking_value_rules(()) + assert _text(text) == text + configure_masking_value_rules(RULES) + assert _text(text) == f"got {LABEL} back" + + def test_a_reset_forgets_them_too(self, monkeypatch): + text = f"got {json.dumps(BLOB)} back" + configure_masking_value_rules(()) + assert _text(text) == text + _reset_masking_config() + monkeypatch.setenv("ECSCTX_MASK_VALUE_RULES", "tests.test_masking_value_rules.RULES") + assert _text(text) == f"got {LABEL} back" + def test_a_rule_that_raises_leaves_the_marker_never_an_exception(self): def broken(_value): raise RuntimeError("rule failed")