diff --git a/CLAUDE.md b/CLAUDE.md index cbbe1b3..d2d6e3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,6 @@ ECS-compliant structured logging with W3C Trace Context support. Framework-agnos - Django's `LogContextBinder` must be imported explicitly (not in `__init__.py`) to avoid circular imports - `LoggingContextMiddleware` must be placed AFTER auth middleware to capture user_id - CVV never carries a token, even when PII is configured — PCI forbids storing CVV in any form, so `[CVV-MASKED]` is always the literal, final output -- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list +- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning, though a value inside a keep match whose own first match is a label rule (one listed before any keep rule matching it) is labelled and the keep refused; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487, each slot text in its alphabet; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list - Card numbers truncate to first 6 + last 4 for 15–19 digits and last 4 only below 15 (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091) — no token alongside, independent of PII configuration, and a second masking pass is a noop - `install_maskers()` only sweeps handlers that exist at call time (no stdlib patching) — a handler built after the call needs another call diff --git a/README.md b/README.md index 1cfb1ff..48e3569 100644 --- a/README.md +++ b/README.md @@ -1406,8 +1406,10 @@ under one rule set never comes back as it was under another. A **keep rule** ships what it matches exactly as sent: a value a service must be able to read in its logs, found by shape. It is any object whose `keep` is `True` (the object itself, not a truthy value), with `matches(value)` and, -optionally, `hints`; it has no `field_type`, since it writes no label. -`KeepRule` is one: +optionally, `hints`; it has no `field_type`, since it writes no label. An +object with both a `field_type` and `keep = True` is neither, rather than +the less safe of the two: it is refused as an item that is not a rule is +(above), the problem naming both. `KeepRule` is one: ```python from ecsctx.masking import KeepRule @@ -1425,42 +1427,72 @@ asked where label rules are: every mapping the key walk meets, every string that is JSON text, every JSON object in free text or in a body `redact_body` masks. They are asked in list order and **the first match wins**: a label rule listed first labels a value a keep rule would have kept, and an object a -label rule matches is not looked into for one to keep. A keep match the guard -refuses (below) is walked as if nothing had matched: no rule listed after it -is asked, a label rule for the same shape included. A kept value is left -as sent under any key and not descended into — a credential's key, a card's, -or a PII container's above it does not stop it (`token`, `paymentToken`, -`password`, `card.devicePayment`, `customer`): what the rule names is the -value itself, not something the key says about it. In text, what a keep rule +label rule matches is not looked into for one to keep. A label rule wins at +any depth inside a keep match, though: a keep match holding a value whose +own first match is a label rule is walked as if nothing had matched, and +that value is labelled — with an Apple Pay label rule listed before +`KeepRule(lambda value: True)`, `{"outer": {"inner": }}` is +`{"outer": {"inner": "[SAD-MASKED]"}}`. Listed after it, the catch-all keep +rule is the inner value's first match too, and the whole value ships as +sent. A keep match the guard refuses +(below) is walked as if nothing had matched too, and either way no rule +listed after it is asked about the value itself, a label rule for the same +shape included. A kept value is left as sent under any key and not +descended into — a credential's key, a card's, or a PII container's above it +does not stop it (`token`, `paymentToken`, `password`, `card.devicePayment`, +`customer`): what the rule names is the value itself, not something the key +says about it. In text, what a keep rule ships is set aside as a `[KEPT--MASKED]` placeholder before every content rule, and put back as it was written after; a placeholder a rule destroys — a credential hashed with what was around it — takes the value with -it. Text that already holds `[KEPT-` is masked as if no keep rule were -configured. A keep rule that raises counts as no match: keep rules are asked -on paths that must never raise (`redact_url`, `redact_body`, -`mask_card_value`). +it. A placeholder is named from a digest of the value it stands for: the same +value has the same name, so a credential hashed around one carries that value +in its token, as under a key, and a name written into the text by anyone +without the value (an element's entity-encoded text decodes to one) is not a +placeholder and restores nothing. Text that already holds `[KEPT-` is masked +as if no keep rule were configured. A keep rule that raises counts as no +match: keep rules are asked on paths that must never raise (`redact_url`, +`redact_body`, `mask_card_value`). So does a label rule asked about what is +inside a keep match: its error refuses the keep and raises nothing there, +and the value is masked as with the label rules alone — whose error reaches +the caller only where the walk itself asks them, as before. Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: - **The floor.** Nothing is kept under a CVV or SAD key (`cvv`, `securityCode`, `pin`, `cryptogram`, …), inside a CVV or SAD container (a key the service lists there included), or as the `value` of a pair - labelled as one; in text, right after such a quoted key, `key=` or XML - start tag. Such a value is masked exactly as without the rule. + labelled as one; in text, anywhere in text that names such a key, element + or pair label. Such a value is masked exactly as without the rule. - **The guard.** A match that holds, at any depth — JSON text inside it included — a key the key rules read as a card, a CVV or other SAD, a `{name, value}` pair whose identifier reads as one (as the key walk reads a - pair), or a leaf that is a card number, is walked as if nothing had - matched. It reads keys, pairs and whole leaves only: a CVV or a card number - written inside a longer string of a kept value (`"note": "cvv=123"`, Track - 2 data, `"card 4111…"`) ships with it. Keeping that out is the matcher's - job, which is why a matcher must be strict about what it matches. A leaf is - a card number when it is an int `int_is_pan` reads as one, or a string - `pan_shaped` reads as one, whatever its prefix and Luhn or not — except - thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's - `keyExpiration`, which `pan_shaped` reads as a card number and every - Google Pay token carries. A leaf that is no JSON value counts as card - data. + pair), a leaf that is a card number, or a text leaf with a card number's + digits in it, is walked as if nothing had matched. A leaf is a card number + when it is an int `int_is_pan` reads as one, or a string `pan_shaped` reads + as one, whatever its prefix and Luhn or not. A text leaf holds a card + number's digits when, read in one pass as the `pci` rules read a value + (`holds_pan_run`), it holds a run of 12 digits or more joined by single + separators at most: `"card 4111…"`, `"1234 4111…"`, `"4111… 123"`, Track 2 + data, 20 bare digits, `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a + hex one. Three leaves are not read for one: a leaf that is thirteen bare + digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's + `keyExpiration`, which `pan_shaped` reads as a card number and every Google + Pay token carries; a leaf that is a canonical UUID (8-4-4-4-12 hex with a + hex letter), as everywhere in ecsctx; and a leaf that is a whole hex id of + 24 characters or more holding a hex letter, such as Apple Pay's 64-hex + `transactionId`: about one random 64-hex id in two hundred holds a 13-19 + digit run that passes Luhn, and reading them cost 1.1% of Apple Pay tokens. + Each is the whole leaf only: a hex id inside base64 or prose + (`"AA/4111…deadbeef/AA"`), digits alone however many, and epoch milliseconds + inside a longer string are all read. A leaf that is no JSON value counts as + card data. What the guard cannot tell from a token's own text ships with + it: a CVV or other short value written inside a longer string + (`"note": "cvv=123"`, or `cvv+123` inside base64, which reads as + ciphertext), and a card number written into a leaf that is a whole hex id + (`4111…` then hex holding a letter, a CVV's digits among it). Keeping that + out is the matcher's job, which is why a matcher must be strict about what + it matches: Ottu's refuse hex alone in a base64 slot. Nothing else in a kept value is read: PII or a credential inside it, under a key of its own or written in a string, ships with it. @@ -1473,7 +1505,16 @@ A keep decision rests on the text as written, since that is what ships: JSON that names a key twice in one object (`json.loads` keeps the last) is not kept, and a JSON-text leaf naming one inside a match is card data to the guard; a Python repr is kept only where its parse renders back to it exactly -(a comment, or a key written twice, is dropped by the parse). +(a comment, or a key written twice, is dropped by the parse). So a repr whose +strings hold a `"` — a Google Pay token's `signedMessage` — written inside +JSON text, where that quote is escaped, is not kept: it no longer renders +back as written (fails closed). + +An entity-encoded value (`{"…"}`, an XML element's text) is set +aside by `redact_body`, which reads such text as it decodes, but not by the +text rules the log processor runs, which read it as written: there it is not +kept and the content rules read into it — with `pci`, about one +entity-encoded Google Pay token in ten comes out altered (fails closed). A service that masks again after ecsctx — Connect does — asks `is_kept(value, key=None)` (a mapping or JSON text; the rules in force, first @@ -1481,15 +1522,37 @@ match, the guard, and the floor on `key`; False with no keep rule configured) before masking a field, and runs its text masker through `mask_outside_kept(text, mask)`, which hands `mask` the text with each kept value set aside and puts back each placeholder it leaves. Both are in -`ecsctx.masking` and never raise. - -In text the floor reads the key right before an object, and every object -the text rules read whole inside it: they read three levels of nesting, so a -kept value two keys below a CVV key (`{"cvv": {"x": {"y": {…}}}}`, the value -itself nested) is not floored there — the guard still holds. An object right -after a CVV key in text is not kept, but no `default` content rule masks a -JSON object there either, with or without keep rules; the key walk masks one -under a CVV key leaf by leaf. +`ecsctx.masking` and never raise of their own: `mask_outside_kept` passes on +an error the caller's `mask` raises. + +In text a container's reach cannot be told — the text rules read an object +three levels deep, and a wallet token's own nesting takes two — so the floor +reads the whole text: text that names a CVV or SAD key anywhere keeps +nothing, and is masked as without a keep rule. A name is a quoted key then +`:`, `=` or `=>` (JSON, a repr, JSON in a JSON string), a bare key then `:` or +`=>`, a key and `=` where a key starts (`cvv=`, `&cvv=`; not base64's `=` +padding inside a string), an XML tag, or a `{name, value}` pair's label +(`"name": "card security code"`), classified as the key walk classifies a +key — every pack on, the service's safe keys honoured. It is read as +written: a quoted key or pair label up to 128 characters (a longer key by +its last 128, before `:` or `=>`), an XML tag of any length, with no +`\uXXXX` escape decoded, in the +text as it stands and, where it holds an entity, as the entities decode. The +key walk reads more: it parses JSON, so a key written with escapes +(`"\u0063vv"`), or longer than 128 characters with its CVV word first, is a +CVV key there and not to the text floor, which can keep a value under it in +text where the key walk would not. Through Ottu's `WALLET_RULES` that gap +ships nothing but a wallet token's own text: their matchers fix every key a +token holds and every slot's alphabet, and the guard reads them, so nothing +else can ride in what they keep. A word in prose (`the cvv check passed`) names +no key. That fails closed: a value beside such a key, not under it +(`{"cvv": "123", "token": {…}}`, or a `` element elsewhere in a body), +is not kept in text, though the key walk keeps it. `redact_body` reads an +element's entity-encoded text the same way, and keeps nothing in it when the +body around it names one. The text is read for names only once a value to +keep is found in it. An object right after a CVV key in text is not kept, but +no `default` content rule masks a JSON object there either, with or without +keep rules; the key walk masks one under a CVV key leaf by leaf. The cost: with no keep rule configured none of this runs — a string with no `{` pays one check, one with a `{` a look at the rules in force; with keep @@ -1498,23 +1561,48 @@ rule's hints never asks it; text holding one is read once for the objects in it. Every mapping the key walk meets asks every rule, once — a mapping has no text to hint with — so a matcher must be cheap: test a literal field first. JSON text under a key asks, and its key walk passes on, only the rules whose -hints it holds. On the benchmark's lines (`scripts/bench_masking.py`) 0.17.0 -masks within a few percent of 0.16.0, the run-to-run noise, with no value rule -and with Ottu's `WALLET_RULES` (three keep rules where 0.16.0 had two label -rules): the key walk now asks each mapping once, where it asked twice. +hints it holds. With Ottu's `WALLET_RULES`, a line carrying no wallet token +costs 1-4% more than with 0.16.0's two label rules, and 5-16% more than with +no value rule (most on a line of many small mappings, each of which asks +every rule; `scripts/bench_masking.py`). A line carrying a token has no +single figure: about 2.3-2.5× 0.16.0's label rules on a short line (an MPGS +line with the token as `paymentToken`: ~73 µs labelled, ~175 µs kept, ~47 µs +with no rule; a message holding a Google Pay token: ~135 / ~300 / ~305 µs), +and less on a body every content rule would otherwise read (KPay ``: +~0.96 ms labelled, ~0.87 ms kept, ~3.1 ms with no rule). The slot checks, +the guard's digit runs and the text floor's scan add 10-35% over 0.17.0 on +such lines. Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or Google Pay token is single-use, and the token as it was sent is what debugs a wallet payment. `ecsctx.contrib.ottu.masking.WALLET_RULES` are keep rules with strict matchers — an Apple Pay `PKPaymentToken`, its `paymentData` -(`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header` of text -fields) and a Google Pay payment method token (`protocolVersion` `ECv1`, -`ECv2` or `ECv2SigningOnly`, with a `signedMessage`), and no key besides the -ones each writes — found under `paymentData`, TAP's `token_data`, MPGS's -`paymentToken` (a JSON string), Google Pay's `tokenizationData.token`, KPay's -``, or in a message. A `PKPaymentToken` with a key of its own keeps -only its `paymentData`. Saved-card tokens stay masked: no wallet shape -matches one. Samsung Pay's token, a JWE, is not matched yet. An Ottu service +(`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header`) and a +Google Pay payment method token (`protocolVersion` `ECv1`, `ECv2` or +`ECv2SigningOnly`, with a `signedMessage`), no key besides the ones each +writes, and every leaf text in its alphabet: standard base64, never hex +alone, for a key, a signature or ciphertext (`data`, `signature`, the header's +`ephemeralPublicKey`, `publicKeyHash` and `wrappedKey`, Google Pay's +`signatures`, and the fields of its `signedMessage` and `signedKey`, each +JSON text of all its fields and nothing else), hex for an id or a hash +(`transactionId`, `applicationData`, `transactionIdentifier`), digits for +`keyExpiration`, and for what `paymentMethod` says about the card words: a +letter and up to 39 more letters, spaces, `.`, `&` or `-` — its `network` +and `type` that alone or nothing (`AmEx`, `debit`), its `displayName` +optionally followed by one space and exactly four digits, the card's last +four (`Visa 0492`, `American Express`) — and no words that read as a CVV or +other SAD (`cvv`, `CVV 1234`: a CID is four digits). A token holding +anything else in a slot — `"data": "cvv=123"`, a spaced card number as its +`signature`, `"network": "cvv=123"` — is not kept, and is masked as before; nor is +one with a card number in a slot's own alphabet (`"4111…+cvv+123"` in +base64), which the guard reads. What neither can tell from ciphertext ships +with a kept token: a CVV or other short value written in a slot's own +alphabet (`cvv+123` inside base64). They are found under `paymentData`, +TAP's `token_data`, MPGS's `paymentToken` (a JSON string), Google Pay's +`tokenizationData.token`, KPay's ``, or in a message. A +`PKPaymentToken` with a key of its own keeps only its `paymentData`. +Saved-card tokens stay masked: no wallet shape matches one. Samsung Pay's +token, a JWE, is not matched yet. An Ottu service lists them in `ECSCTX_MASK_VALUE_RULES` next to its `ECSCTX_MASK_SAFE_KEYS` (above), or sets `ECSCTX_MASK_VALUE_RULES=ecsctx.contrib.ottu.masking.WALLET_RULES`. diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 986c051..9cbd27f 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which stops at a kept mapping: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span, a CVV/SAD key right before an object -- quoted, `key=`, an XML start tag -- floors it and all inside it) and restores each placeholder still there; a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number's digits inside a text leaf that is no JSON text (`patterns.holds_card_run`: `holds_pan_run`, 12+ digits joined by single separators, as the pci rules read a value, one linear pass with a bytes-translate fast path for base64; not read in a leaf that is a whole hex id of 24+ characters holding a hex letter, `_HEX_ID` fullmatch -- Apple Pay's 64-hex ids; Ottu's base64 slots refuse hex alone, 1.1% of which hold a Luhn run -- nor a leaf that is epoch milliseconds); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/contrib/net/__init__.py b/ecsctx/contrib/net/__init__.py index e866e9f..811b0e9 100644 --- a/ecsctx/contrib/net/__init__.py +++ b/ecsctx/contrib/net/__init__.py @@ -44,6 +44,7 @@ _end_tags, _mask_card_field, _mask_userinfo, + floored_text, holds_pan_run, mask_card_elements, mask_objects, @@ -425,16 +426,19 @@ def redact_body(text: str) -> str: return _redact_body(text)[0] -def _redact_body(text: str) -> tuple[str, bool]: - """``redact_body``'s result, and whether it set a kept value aside.""" +def _redact_body(text: str, keep: bool = True) -> tuple[str, bool]: + """``redact_body``'s result, and whether it set a kept value aside -- + never one with ``keep`` False: the text holding this text names a CVV + or SAD key (``patterns.floored_text``).""" stash = None if "{" in text and (rules := value_rules_in_force()): text = mask_objects(text, rules) - stash = stash_kept(text, rules) - if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text: - # None in the text as written; an element's entity-encoded text - # may hold one (_encoded_leaf). - stash = KeptStash(text, {}) + if keep: + stash = stash_kept(text, rules) + if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text: + # None in the text as written; an element's entity-encoded + # text may hold one (_encoded_leaf). + stash = KeptStash(text, {}) if stash is None: return _redact_body_text(text, None), False return stash.restore(_redact_body_text(stash.text, stash)), bool(stash.originals) @@ -479,10 +483,14 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: other leaf, through the body rules as it decodes -- set aside in ``stash`` when it holds a kept value, so the form and JSON rules that read the body next never read into it (base64's `==` after a run holding - `cvv` reads as a CVV form field).""" + `cvv` reads as a CVV form field). No leaf keeps a value without a + ``stash``, nor in text that names a CVV or SAD key, element or pair + label (``floored_text``, read once a leaf keeps one).""" text = mask_card_elements(text) names = _get_compiled()[3] parts, copied, ends = [], 0, None + # Whether a leaf may keep a value: None until one does. + keep: bool | None = None if stash is not None else False for start in _XML_START.finditer(text): if start.start() < copied: continue # inside an element already masked whole @@ -494,11 +502,13 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: continue end = closing.start() if not _is_credential_element(start.group("local"), names): - if (leaf := _encoded_leaf(text, start.end(), end)) is not None: - masked, kept = leaf - if kept and stash is not None: - masked = stash.hold(masked) - parts += [text[copied : start.end()], masked] + leaf = _encoded_leaf(text, start.end(), end, keep is not False) + if leaf is not None and leaf[1]: + if keep is None: + keep = not floored_text(text) + leaf = (stash.hold(leaf[0]), True) if keep else _encoded_leaf(text, start.end(), end, False) + if leaf is not None: + parts += [text[copied : start.end()], leaf[0]] copied = end continue value = _element_text(text[start.end() : end]) @@ -512,7 +522,7 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: return "".join(parts) -def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None: +def _encoded_leaf(text: str, start: int, end: int, keep: bool = True) -> tuple[str, bool] | None: """The text of a leaf, ``text[start:end]``, masked by the body rules as it decodes and escaped again -- when it is JSON or XML written with entities, and masking changed it or kept a value in it (then as written, @@ -520,13 +530,13 @@ def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None: needs nothing here, nor a form body's: the body rules read them where they stand, `&` a form's separator either way. Decoded, a form value ran on past `"`, and a leaf the first pass left as a form body read - differently on the next.""" + differently on the next. Nothing is kept in it with ``keep`` False.""" if text.find("<", start, end) != -1 or text.find("&", start, end) == -1: return None decoded = html.unescape(text[start:end]) if decoded.lstrip()[:1] not in ("{", "[", "<"): return None - masked, kept = _redact_body(decoded) + masked, kept = _redact_body(decoded, keep) if masked == decoded: return (text[start:end], True) if kept else None return xml_escape(masked), kept diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index ab8ff19..2c83725 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -33,9 +33,13 @@ a credential: no wallet shape matches it, under ``token`` or anywhere. - What is not exactly a wallet's shape. The matchers are strict -- an Apple Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method - token, and no key besides the ones they write -- so anything else is walked - as before: a ``PKPaymentToken`` with a key of its own keeps only its - ``paymentData``. + token, no key besides the ones they write, and every leaf text in its + alphabet (base64, hex or digits; short text for what ``paymentMethod`` + says about the card, its display name letters and the last four) + -- so anything else is walked as before: a ``PKPaymentToken`` with a key + of its own keeps only its ``paymentData``, and a token with + ``"data": "cvv=123"`` is masked. A CVV written in a slot's own alphabet + (``cvv+123`` inside base64) cannot be told from ciphertext, and ships. - Card data. ecsctx keeps nothing under a CVV or SAD key, and nothing that holds a card, CVV or SAD key or a card number (the floor and the guard). @@ -59,9 +63,12 @@ service masks stays in its own settings. """ +import re +from collections.abc import Mapping from typing import Any -from ecsctx.masking.value_rules import KeepRule, ValueRule +from ecsctx.masking.patterns import _FLOOR_TYPES, ALL_PACKS, classify_key +from ecsctx.masking.value_rules import KeepRule, ValueRule, json_as_written SAFE_KEYS = frozenset({ # Ottu PG API fields: a gateway's short name ("mpgs") and flags saying @@ -141,77 +148,141 @@ def _is_google_pay_token(value: Any) -> bool: # Pay's payment method token are documented to hold, and nothing else. A key # besides these is no wallet's, and the object is walked as before. _PAYMENT_DATA_KEYS = frozenset({"version", "data", "signature", "header"}) -_PAYMENT_DATA_HEADER_KEYS = frozenset( - {"ephemeralPublicKey", "wrappedKey", "publicKeyHash", "transactionId", "applicationData"} -) _PK_PAYMENT_TOKEN_KEYS = frozenset({"paymentData", "paymentMethod", "transactionIdentifier"}) -_PAYMENT_METHOD_KEYS = frozenset({"displayName", "network", "type"}) _GOOGLE_PAY_KEYS = frozenset({"signature", "intermediateSigningKey", "protocolVersion", "signedMessage"}) _SIGNING_KEY_KEYS = frozenset({"signedKey", "signatures"}) +_SIGNED_MESSAGE_KEYS = frozenset({"encryptedMessage", "ephemeralPublicKey", "tag"}) +_SIGNED_KEY_KEYS = frozenset({"keyValue", "keyExpiration"}) + +# What a wallet writes in a slot: standard base64 for a key, a signature or +# ciphertext, hex for an id or a hash, digits for a time -- nothing else. +# The guard reads keys, pairs, whole leaves and a card number inside one, so +# other text in a kept token's slot (`"data": "cvv=123"`) would ship with it +# (Redmine R3). +# Not hex alone: a whole hex id is not read for a card number (the guard's +# exemption for Apple Pay's ids), and a wallet's base64 never is one. +_BASE64 = re.compile(r"(?![0-9A-Fa-f]*\Z)[A-Za-z0-9+/]+={0,2}") +_HEX = re.compile(r"[0-9A-Fa-f]+") +_DIGITS = re.compile(r"[0-9]+") +_HEADER_ALPHABETS = { + "ephemeralPublicKey": _BASE64, + "publicKeyHash": _BASE64, + "wrappedKey": _BASE64, + "transactionId": _HEX, + "applicationData": _HEX, +} +# PKPaymentToken's `paymentMethod` is what the device says about the card, in +# words: up to 40 letters, spaces, `.`, `&` and `-` from a letter. Its network +# and type are words alone, or nothing ("AmEx", "debit"); its display name +# words and optionally a space and the card's last four ("Visa 0492", +# "American Express"). No other digit or sign, so neither a card number nor an +# email rides in them, and no words that read as a CVV or other SAD: a CID is +# four digits ("CVV 1234"). +_CARD_WORDS = r"[A-Za-z][A-Za-z .&-]{0,39}" +_PAYMENT_METHOD_ALPHABETS = { + "displayName": re.compile(rf"{_CARD_WORDS}(?: [0-9]{{4}})?"), + "network": re.compile(rf"(?:{_CARD_WORDS})?"), + "type": re.compile(rf"(?:{_CARD_WORDS})?"), +} + + +def _written_in(field: Any, alphabet: re.Pattern) -> bool: + """Whether a wallet's field is text in its alphabet: every leaf a wallet + writes is a string, so a container or a number in a slot is no wallet's, + nor is text another alphabet writes -- nothing can ride in it.""" + return isinstance(field, str) and alphabet.fullmatch(field) is not None -def _text_or_absent(value: Any, key: str) -> bool: - """A wallet's field is text where it is written: every leaf a wallet - writes is a string, so a container or a number in a slot is no wallet's, - and nothing can ride in it -- a `{name, value}` pair among them.""" - return key not in value or isinstance(value[key], str) +def _absent_or_in(value: Mapping, key: str, alphabet: re.Pattern) -> bool: + return key not in value or _written_in(value[key], alphabet) def _is_apple_pay_payment_data(value: Any) -> bool: """Apple Pay's `paymentData`: an EC_v1 or RSA_v1 `version`, the - encrypted `data` and its `signature` as text, and a `header` of text - fields.""" + encrypted `data` and its `signature` in base64, and a `header` of its + keys in base64 and its ids in hex.""" version = value.get("version") - if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and isinstance(value.get("data"), str)): + if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and _written_in(value.get("data"), _BASE64)): return False - if not (value.keys() <= _PAYMENT_DATA_KEYS and _text_or_absent(value, "signature")): + if not (value.keys() <= _PAYMENT_DATA_KEYS and _absent_or_in(value, "signature", _BASE64)): return False header = value.get("header", {}) return ( isinstance(header, dict) - and header.keys() <= _PAYMENT_DATA_HEADER_KEYS - and all(isinstance(field, str) for field in header.values()) + and header.keys() <= _HEADER_ALPHABETS.keys() + and all(_written_in(field, _HEADER_ALPHABETS[key]) for key, field in header.items()) ) +def _in_card_words(field: Any, alphabet: re.Pattern) -> bool: + if not _written_in(field, alphabet): + return False + return classify_key(field.rstrip("0123456789 "), ALL_PACKS) not in _FLOOR_TYPES + + def _is_pk_payment_token(value: Any) -> bool: - """Apple Pay's PKPaymentToken: its `paymentData`, and what the device says - about the card -- its display name, network and type -- and the - transaction's id, each as text.""" + """Apple Pay's PKPaymentToken: its `paymentData`, what the device says + about the card -- its network, type and display name in words, the name + with the last four -- and the transaction's id in hex.""" payment_data = value.get("paymentData") if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): return False - if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _text_or_absent(value, "transactionIdentifier")): + if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _absent_or_in(value, "transactionIdentifier", _HEX)): return False method = value.get("paymentMethod", {}) return ( isinstance(method, dict) - and method.keys() <= _PAYMENT_METHOD_KEYS - and all(isinstance(field, str) for field in method.values()) + and method.keys() <= _PAYMENT_METHOD_ALPHABETS.keys() + and all(_in_card_words(field, _PAYMENT_METHOD_ALPHABETS[key]) for key, field in method.items()) ) +def _signed_object(text: Any, keys: frozenset[str]) -> dict | None: + """JSON text a Google Pay token signs, read as the keep decision reads + JSON (a key written twice refused): an object of exactly ``keys``, or + None.""" + if not isinstance(text, str): + return None + try: + parsed = json_as_written(text) + except (ValueError, RecursionError): + return None + return parsed if isinstance(parsed, dict) and parsed.keys() == keys else None + + +def _is_signed_message(text: Any) -> bool: + """`signedMessage`: JSON text of the encrypted message, its ephemeral + public key and tag, each in base64, and nothing else.""" + message = _signed_object(text, _SIGNED_MESSAGE_KEYS) + return message is not None and all(_written_in(field, _BASE64) for field in message.values()) + + +def _is_signed_key(text: Any) -> bool: + """`signedKey`: JSON text of the intermediate signing key in base64 and + its expiration, epoch milliseconds, and nothing else.""" + key = _signed_object(text, _SIGNED_KEY_KEYS) + return key is not None and _written_in(key["keyValue"], _BASE64) and _written_in(key["keyExpiration"], _DIGITS) + + def _is_google_pay_payment_token(value: Any) -> bool: - """Google Pay's payment method token: its `protocolVersion`, the - `signedMessage` and its `signature` as text, and the intermediate signing - key ECv2 signs with: a `signedKey` as text and its `signatures`, a list - of text.""" + """Google Pay's payment method token: its `protocolVersion`, its + `signature` in base64, the intermediate signing key ECv2 signs with -- + a `signedKey` and its `signatures`, a list in base64 -- and the + `signedMessage`. The two signed JSON texts are parsed last.""" protocol = value.get("protocolVersion") if not (isinstance(protocol, str) and protocol in _GOOGLE_PAY_VERSIONS): return False - if not isinstance(value.get("signedMessage"), str): - return False - if not (value.keys() <= _GOOGLE_PAY_KEYS and _text_or_absent(value, "signature")): + if not (value.keys() <= _GOOGLE_PAY_KEYS and _absent_or_in(value, "signature", _BASE64)): return False signing_key = value.get("intermediateSigningKey", {}) if not (isinstance(signing_key, dict) and signing_key.keys() <= _SIGNING_KEY_KEYS): return False signatures = signing_key.get("signatures", []) - return ( - _text_or_absent(signing_key, "signedKey") - and isinstance(signatures, list) - and all(isinstance(signature, str) for signature in signatures) - ) + if not (isinstance(signatures, list) and all(_written_in(signature, _BASE64) for signature in signatures)): + return False + if "signedKey" in signing_key and not _is_signed_key(signing_key["signedKey"]): + return False + return _is_signed_message(value.get("signedMessage")) APPLE_PAY_TOKEN_KEEP_RULE = KeepRule(_is_pk_payment_token, hints=("EC_v1", "RSA_v1")) diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index cea3640..98a186b 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -29,6 +29,7 @@ import ast import contextlib +import hashlib import html import json import re @@ -59,8 +60,8 @@ _first_match, applicable, hinted, - holds_card_data, json_as_written, + keep_refused, keep_rules, label_rules, ruling, @@ -808,6 +809,42 @@ def pan_shaped(text: str) -> bool: _DIGIT_RUN = re.compile(rf"\+?\d(?:{_CARD_SEP}?\d)*") +# ASCII text with each digit marked 0 and each separator the card rule reads +# 2: text with neither twelve 0s in a row nor a 2 -- base64 and hex, a +# wallet's ciphertext -- holds no run, found in one bytes translate where the +# regex costs thirty times as much. +_DIGIT_MARKS = bytes( + 0 if chr(code) in "0123456789" else 2 if code < 128 and re.fullmatch(_CARD_SEP, chr(code)) else code or 1 + for code in range(256) +) +_RUN_MARKS = b"\0" * _MIN_PAN_DIGITS +# A leaf that is a hex id of 24 characters or more -- a 64-hex digest, Apple +# Pay's `transactionId`, an ObjectId -- holds no card number: read like other +# text, about one random 64-hex id in two hundred holds a 13-19 digit run +# that passes Luhn. Only a whole leaf, holding a hex letter: hex inside +# base64 or prose is read, and so are digits alone, which no hex id is. +_HEX_ID = re.compile(r"(?=[0-9]*[A-Fa-f])[0-9A-Fa-f]{24,}") + + +def holds_card_run(text: str) -> bool: + """Whether ``text``, a text leaf of a keep match, holds a run of digits + long enough to be a card number, read as the pci rules read a value + (``holds_pan_run``: 12 digits or more, joined by single separators at + most -- `card 4111…`, `1234 4111…`, `4111… 123`, `4111…+cvv+123` in + base64, `4111…ab` in hex) -- except a leaf that is a whole hex id of 24 + characters or more holding a hex letter (``_HEX_ID``). For the guard + (``value_rules.holds_card_data``), which excepts a leaf that is epoch + milliseconds itself. One pass, linear in the leaf: ASCII text with no + such run of digits and no separator is told by one bytes translate.""" + if _HEX_ID.fullmatch(text): + return False + if text.isascii(): + marked = text.encode("ascii").translate(_DIGIT_MARKS) + if _RUN_MARKS not in marked and b"\2" not in marked: + return False + return holds_pan_run(text) + + def holds_pan_run(text: str, *, phone: bool = False) -> bool: """Whether ``text`` holds, anywhere in it, a run of digits long enough to be a PAN. @@ -1157,16 +1194,17 @@ def _mask_object(m: re.Match, rules: tuple) -> str: is itself in a string; bare anywhere else -- in prose, an element's text, a string of its own. An object no rule matches is read member by member, so one inside it is still found; one a keep rule matches first is left as - written and not looked into, unless the guard refuses it or it is - written otherwise than it parses: then it is read as one nothing matches. - Only the rules whose hints the object's text holds are asked.""" + written and not looked into, unless it is refused (``keep_refused``: the + guard, or a label rule matching inside it) or written otherwise than it + parses: then it is read as one nothing matches. Only the rules whose + hints the object's text holds are asked.""" written = m.group(0) asked = applicable(written, rules) if not asked: return written # nothing in it can match: every match holds a hint value = _object_value(m) found = _first_match(value, asked) - if found is not None and found[1] and (holds_card_data(value) or not _written_as_parsed(m)): + if found is not None and found[1] and (keep_refused(value, asked) or not _written_as_parsed(m)): found = None if found is None: inner = mask_objects(written[1:-1], rules) @@ -1204,29 +1242,116 @@ def mask_objects(text: str, rules: tuple) -> str: # or in such a container (filters): a CVV and the rest of Sensitive # Authentication Data. _FLOOR_TYPES = frozenset({"cvv", "sad"}) -# The key right before an object in text, as the floor reads it: a quoted key -# and its colon (JSON, a repr, JSON in a JSON string, its quotes escaped), a -# key and `=` or `:`, or an XML start tag -- then any space and the opening -# quote of a string the object is the JSON text of. -_KEY_BEFORE = re.compile( - r"(?:(?P\\?[\"'])(?P[^\"'\\]{1,128})(?P=quote)\s*:" - r"|(?[\w.-]{1,128})\s*[:=]" - r"|<(?:[A-Za-z_][\w.-]*:)?(?P[A-Za-z_][\w.-]*)(?:\s[^<>]*)?(?)" - r"\s*(?:\\?[\"'])?\s*\Z" -) -# How far back the key is looked for. -_KEY_REACH = 192 +# A key in text, as the floor reads it, is found from the `:`, `=` or `=>` +# after it and read back from there (``_key_names``): a quoted key (JSON, a +# repr, JSON in a JSON string, its quotes escaped), or a bare one -- after a +# `=`, only where a key starts, never inside a quoted string, where base64's +# `=` padding follows a word (`"…/cvvXYZ=="`). A find per separator, not a +# regex tried at every character: this reads a 7 KB wallet body in +# microseconds. +_NAME_LIMIT = 128 +_QUOTES = "\"'" +# What a key assigned with `=` follows, besides whitespace and the text's start. +_KEY_STARTS = frozenset("&?,({[") +# The name characters a bare key ends with, matched on the text before its +# separator reversed: matched forward, from every position, a long word cost +# its length squared. +_NAME_RUN = re.compile(r"[\w.-]*") +_TAG_NAME = re.compile(r"])") + + +def _key_at(text: str, at: int, assigned: bool) -> str | None: + """The key written before the separator at ``at``: quoted, or a bare name + starting with a letter or `_` (where a key starts, for a bare ``=``).""" + end = at + while end > 0 and text[end - 1].isspace(): + end -= 1 + if end and text[end - 1] in _QUOTES: + close = end - 1 + while close > 0 and text[close - 1] == "\\": + close -= 1 + opening = text.rfind(text[end - 1], max(0, close - _NAME_LIMIT - 1), close) + if opening != -1 and not any(quote in text[opening + 1 : close] for quote in "\"'\\"): + return text[opening + 1 : close] or None + end = close + length = _NAME_RUN.match(text[max(0, end - _NAME_LIMIT - 1) : end][::-1]).end() + if length > _NAME_LIMIT: + # Longer than a key is read: its end, which names what it holds -- + # never after a bare `=`, as base64 padding follows a long word. + return None if assigned else text[end - _NAME_LIMIT : end] + start = end - length + name = text[start:end] + if not name or not (name[0].isalpha() or name[0] == "_"): + return None + if assigned and start and not (text[start - 1].isspace() or text[start - 1] in _KEY_STARTS): + return None + return name -def _floored_before(text: str, start: int) -> bool: - """Whether the object at ``start`` sits right after a CVV or SAD key -- - `"cvv": {…}`, `cvv={…}`, `{…}`, `"pin": "{…}"`: nothing under - one is kept, however deep. Every pack on, as the key walk reads a key.""" - found = _KEY_BEFORE.search(text, max(0, start - _KEY_REACH), start) - if found is None: - return False - name = found.group("quoted") or found.group("bare") or found.group("tag") - return classify_key(name, ALL_PACKS) in _FLOOR_TYPES +def _quoted_after(text: str, at: int) -> str | None: + """The quoted text right after ``at`` (whitespace and escapes skipped), + as a `{name, value}` pair writes its label: None for anything else.""" + while at < len(text) and (text[at].isspace() or text[at] == "\\"): + at += 1 + if at >= len(text) or text[at] not in _QUOTES: + return None + closing = text.find(text[at], at + 1, at + 2 + _NAME_LIMIT) + label = text[at + 1 : closing].rstrip("\\") if closing != -1 else "" + return label if label and not any(quote in label for quote in "\"'\\") else None + + +def _key_names(text: str) -> Iterator[tuple[str, int]]: + """Each key ``text`` writes, and where what follows its separator starts.""" + for separator in ":=": + at = text.find(separator) + while at != -1: + arrow = separator == "=" and text.startswith(">", at + 1) + if (name := _key_at(text, at, separator == "=" and not arrow)) is not None: + yield name, at + 1 + arrow + at = text.find(separator, at + 1) + + +@lru_cache(maxsize=1) +def _pair_identifiers() -> frozenset[str]: + """The identifier keys the key walk reads a `{name, value}` pair by. + Imported here: filters imports this module as it loads.""" + from ecsctx.masking.filters import _PAIR_IDENTIFIERS + + return frozenset(_PAIR_IDENTIFIERS) + + +def _safe_keys_in_force() -> frozenset[str]: + """The service's safe keys (``ECSCTX_MASK_SAFE_KEYS``), as the key walk + reads them. Imported here: config imports this module as it loads.""" + from ecsctx.masking.config import get_masking_safe_keys + + return get_masking_safe_keys() + + +def floored_text(text: str) -> bool: + """Whether ``text`` names, anywhere, a CVV or SAD key, an XML tag named + so, or a `{name, value}` pair labelled so -- an identifier key and the + quoted label after it -- read as the key walk reads a key (every pack + on, the service's safe keys honoured), in the text as written and, + where it holds an entity, as it decodes. Text that does keeps nothing: a + container's reach cannot be told from text the rules read three levels + deep, so the whole text fails closed.""" + safe = _safe_keys_in_force() + identifiers = _pair_identifiers() + for written in (text, html.unescape(text)) if "&" in text else (text,): + for name, after in _key_names(written): + if classify_key(name, ALL_PACKS, safe) in _FLOOR_TYPES: + return True + if ( + name.lower() in identifiers + and (label := _quoted_after(written, after)) is not None + # Uncached, as the key walk classifies a pair's label: free text. + and classify_key.__wrapped__(label, ALL_PACKS, safe) in _FLOOR_TYPES + ): + return True + if any(classify_key(m.group(1), ALL_PACKS, safe) in _FLOOR_TYPES for m in _TAG_NAME.finditer(written)): + return True + return False def _written_as_parsed(m: re.Match) -> bool: @@ -1289,19 +1414,21 @@ def _object_ruling(m: re.Match, rules: tuple) -> Any: def kept_spans(text: str, rules: tuple) -> list[tuple[int, int]]: """Where ``text`` writes a value a keep rule among ``rules`` matches first and may keep: each JSON object (rule 2's reading), top down. An object a - label rule matches first is no span and is not looked into, nor is one - right after a CVV or SAD key; one nothing matches is read member by - member; a keep match the guard refuses (``holds_card_data``) is read as - one nothing matches. In order, none inside another.""" + label rule matches first is no span and is not looked into; one nothing + matches is read member by member; a keep match refused (``keep_refused``: + the guard, or a label rule matching inside it) is read as one nothing + matches. In order, none inside another. None at all in text that names a + CVV or SAD key, element or pair label anywhere (``floored_text``, read + only once a span is found).""" spans: list[tuple[int, int]] = [] _find_kept(text, 0, len(text), rules, spans) - return spans + return [] if spans and floored_text(text) else spans def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[int, int]]) -> None: for m in _OBJECT_RULE.pattern.finditer(text, start, end): asked = applicable(m.group(0), rules) - if not asked or _floored_before(text, m.start()): + if not asked: continue found = _object_ruling(m, asked) if found is KEEP: @@ -1314,16 +1441,24 @@ def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[ # which every rule leaves as it is, with no digit in it for a rule to read. _KEPT_HEAD = "[KEPT-" _KEPT_PLACEHOLDER = re.compile(r"\[KEPT-[A-Z]+-MASKED\]") - - -def _letters(index: int) -> str: - """0 is A, 25 is Z, 26 is AA: a placeholder's own, digit-free name.""" - letters = "" - index += 1 - while index: - index, rest = divmod(index - 1, 26) - letters = chr(ord("A") + rest) + letters - return letters +# A placeholder's letters: its value's 8-byte digest in base 26, always this +# many (26**14 > 2**64), so no two names differ by length alone. +_NAME_LETTERS = 14 + + +def _placeholder(value: str) -> str: + """The placeholder standing for ``value``, named from a digest of it, not + by position: the same value has the same name in any text, so a + credential hashed with one around it carries that value in its token, as + under a key; and only who has the value can write its name, so a name + written into the text (an element's entity-encoded text decodes to one) + restores nothing. Digit-free; never raises (a lone surrogate passes).""" + digest = hashlib.blake2b(value.encode("utf-8", "surrogatepass"), digest_size=8).digest() + number, letters = int.from_bytes(digest, "big"), [] + for _ in range(_NAME_LETTERS): + number, rest = divmod(number, 26) + letters.append(chr(ord("A") + rest)) + return f"{_KEPT_HEAD}{''.join(letters)}-MASKED]" class KeptStash(NamedTuple): @@ -1334,11 +1469,14 @@ class KeptStash(NamedTuple): originals: dict[str, str] def hold(self, value: str) -> str: - """A new placeholder standing for ``value``, for the caller to put + """The placeholder standing for ``value``, for the caller to put where it stood: ``contrib.net.redact_body`` sets aside an element's - entity-encoded text that holds a kept value.""" - placeholder = f"{_KEPT_HEAD}{_letters(len(self.originals))}-MASKED]" - self.originals[placeholder] = value + entity-encoded text that holds a kept value. ``value`` itself where + its name already stands for another value: the rules then read it, + and may mask what is in it.""" + placeholder = _placeholder(value) + if self.originals.setdefault(placeholder, value) != value: + return value return placeholder def restore(self, masked: str) -> str: @@ -1352,11 +1490,12 @@ def restore(self, masked: str) -> str: def stash_kept(text: str, rules: tuple | None = None) -> KeptStash | None: """``text`` with each value a keep rule ships set aside as a placeholder - (``[KEPT-A-MASKED]``), for the rules to read around it -- or None when - there is none: no keep rule among ``rules`` (the rules in force when - None), none whose hints the text holds, no span (``kept_spans``), or text - that already holds a placeholder's head, which is masked as before. - Never raises.""" + named from it (``[KEPT--MASKED]``; the same value twice, one + name), for the rules to read around it -- or None when there is none: no + keep rule among ``rules`` (the rules in force when None), none whose + hints the text holds, no span (``kept_spans``), text that already holds a + placeholder's head, or two values whose names collide; such text is + masked as before. Never raises.""" if "{" not in text: return None if rules is None: @@ -1369,9 +1508,11 @@ def stash_kept(text: str, rules: tuple | None = None) -> KeptStash | None: parts: list[str] = [] originals: dict[str, str] = {} copied = 0 - for index, (start, end) in enumerate(spans): - placeholder = f"{_KEPT_HEAD}{_letters(index)}-MASKED]" - originals[placeholder] = text[start:end] + for start, end in spans: + value = text[start:end] + placeholder = _placeholder(value) + if originals.setdefault(placeholder, value) != value: + return None parts += [text[copied:start], placeholder] copied = end parts.append(text[copied:]) diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index b0379fe..b020f86 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -22,7 +22,11 @@ is kept under a CVV or SAD key or inside a CVV or SAD container (the floor, applied where a key is known: ``filters``, ``patterns.kept_spans``), and a match holding a card, CVV or SAD key, or a card number, at any depth, is - walked as if nothing matched (the guard, ``holds_card_data``). + walked as if nothing matched (the guard, ``holds_card_data``). A match + holding a value whose own first match is a label rule is walked so too + (``keep_refused``): a label rule wins at any depth inside a keep match, the + first match deciding each value -- a catch-all keep rule listed first is + every value's first match. Both have: @@ -123,16 +127,27 @@ def load_value_rules(value: Iterable[Any] | str) -> tuple[tuple[Any, ...], tuple def _rules_in(item: Any) -> tuple[Any, ...]: found = _imported(item) if isinstance(item, str) else item if is_value_rule(found): - return (found,) - if isinstance(found, Iterable) and not isinstance(found, (str, bytes, Mapping)): + members: tuple[Any, ...] | None = (found,) + elif isinstance(found, Iterable) and not isinstance(found, (str, bytes, Mapping)): members = tuple(found) - if all(is_value_rule(member) for member in members): - return members - raise ValueError( - f"{item!r} is not a value rule or a collection of them: a rule has `matches(value)`, " - "either a `field_type` (its label) or `keep = True` (it ships as sent), and optionally " - "`hints`, a collection of strings" - ) + else: + members = None + if members is None or not all(is_value_rule(member) for member in members): + raise ValueError( + f"{item!r} is not a value rule or a collection of them: a rule has `matches(value)`, " + "either a `field_type` (its label) or `keep = True` (it ships as sent), and optionally " + "`hints`, a collection of strings" + ) + for member in members: + # Read as a keep rule it would ship what its author may have meant + # to label: neither reading is taken. + if is_keep_rule(member) and is_label_rule(member): + where = repr(item) if member is found else f"{member!r} in {item!r}" + raise ValueError( + f"{where} has both a `field_type` ({member.field_type!r}: a label rule) and " + "`keep = True` (a keep rule): a rule is one or the other" + ) + return members def _imported(path: str) -> Any: @@ -306,53 +321,65 @@ def parses_as_written(text: str) -> bool: _EPOCH_MILLISECONDS = re.compile(r"[12][0-9]{12}") +def pair_labelled(pair: Mapping, types: frozenset[str]) -> bool: + """Whether a ``{name, value}`` pair's identifier reads as one of ``types`` + (every pack on), as the key walk reads a pair (``filters._pair``), which + masks its ``value`` as that type.""" + keys = {str(key).lower(): key for key in pair} + if "value" not in keys: + return False + # Imported here, past the check nearly every mapping fails: patterns and + # filters import this module as they load. + from ecsctx.masking.filters import _PAIR_IDENTIFIERS + from ecsctx.masking.patterns import ALL_PACKS, classify_key + + # Uncached, as the key walk classifies free text: an identifier may be a + # sentence, and must not evict a key name. + return any( + isinstance(label := pair[keys[identifier]], str) and classify_key.__wrapped__(label, ALL_PACKS) in types + for identifier in _PAIR_IDENTIFIERS + if identifier in keys + ) + + def holds_card_data(value: Any) -> bool: """Whether ``value`` holds, at any depth -- JSON text inside it included -- a key the key rules read as a card, a CVV or other SAD (every pack on), a ``{name, value}`` pair whose identifier reads as one (as the key walk reads - a pair, ``filters._pair``), or a leaf that is a card number as a whole: - what no keep rule may ship, not even one that matches everything. Keys, - pairs and whole leaves only: a CVV or a card number written inside a - longer string (``"note": "cvv=123"``) is not read here, and keeping it out - is a matcher's job. + a pair, ``filters._pair``), a leaf that is a card number as a whole, or a + text leaf holding one: what no keep rule may ship, not even one that + matches everything. A CVV or other short value written inside a longer + string (``"note": "cvv=123"``, ``cvv+123`` in base64) is not read here, + and keeping it out is a matcher's job. A leaf is a card number when it is an int ``int_is_pan`` reads as one, or a string ``pan_shaped`` reads as one -- whatever its prefix, Luhn or not -- except thirteen bare digits from a 1 or a 2: epoch milliseconds, Google Pay's ``keyExpiration`` in ``signedKey``, without which no Google Pay token would - ever be kept. Not ``holds_pan_run``, which flags hex ids. + ever be kept. Any other text leaf that is no JSON text holds one when + ``holds_card_run`` finds a run of 12 digits or more in it, read as the pci + rules read a value -- except a leaf that is a whole hex id of 24 + characters or more holding a hex letter, which ``holds_pan_run`` alone + would read (JSON text is read by its own leaves). Anything that is no JSON value (an object whose text is not judged here), nesting past the depth cap, and a leaf of JSON text that names a key twice (``json_as_written``: what ships would not be what was read) count as holding card data: such a match is walked, not shipped. """ - # Imported here: patterns and filters import this module as they load. - from ecsctx.masking.filters import _PAIR_IDENTIFIERS + # Imported here: patterns imports this module as it loads. from ecsctx.masking.patterns import ( ALL_PACKS, classify_key, + holds_card_run, int_is_pan, pan_shaped, ) - def labels_card_data(pair: Mapping) -> bool: - keys = {str(key).lower(): key for key in pair} - if "value" not in keys: - return False - # Uncached, as the key walk classifies free text: an identifier may be - # a sentence, and must not evict a key name. - return any( - isinstance(label := pair[keys[identifier]], str) - and classify_key.__wrapped__(label, ALL_PACKS) in _CARD_DATA_TYPES - for identifier in _PAIR_IDENTIFIERS - if identifier in keys - ) - def holds(item: Any, depth: int) -> bool: if depth > _MAX_DEPTH: return True if isinstance(item, Mapping): - if labels_card_data(item): + if pair_labelled(item, _CARD_DATA_TYPES): return True return any( classify_key(str(key), ALL_PACKS) in _CARD_DATA_TYPES or holds(member, depth + 1) @@ -367,33 +394,58 @@ def holds(item: Any, depth: int) -> bool: if isinstance(item, float): return item.is_integer() and int_is_pan(int(item)) if isinstance(item, str): - if pan_shaped(item) and not _EPOCH_MILLISECONDS.fullmatch(item.strip()): + if _EPOCH_MILLISECONDS.fullmatch(item.strip()): + return False # Google Pay's `keyExpiration`, a whole leaf + if pan_shaped(item): return True - if item.lstrip()[:1] not in ("{", "["): - return False - try: - parsed = json_as_written(item) - except (DuplicateKey, RecursionError): - return True - except ValueError: - return False - return holds(parsed, depth + 1) + if item.lstrip()[:1] in ("{", "["): + try: + parsed = json_as_written(item) + except (DuplicateKey, RecursionError): + return True + except ValueError: + pass + else: + # Its own leaves are read: `keyExpiration` in Google + # Pay's `signedKey` is a whole leaf there. + return holds(parsed, depth + 1) + return holds_card_run(item) return True return holds(value, 0) +def keep_refused(value: dict, rules: Iterable[Any]) -> bool: + """Whether a keep match is walked as if nothing had matched: the guard + refuses it (``holds_card_data``), or a label rule among ``rules`` labels + something strictly inside it (``label_within``, over its members): a + label rule wins at any depth inside a keep match. For the value itself + the first match still wins. Only with a label rule among ``rules``; one + that raises there refuses the keep, and raises nothing: keep rules are + asked on paths that must never raise, and a member is asked here that + the walk might never ask.""" + if holds_card_data(value): + return True + if not _split(rules).label: + return False + try: + return any(label_within(member, rules, 1) is not None for member in value.values()) + except Exception: # noqa: BLE001 -- a service's label matcher; refusing the keep fails closed + return True + + def ruling(value: Any, rules: Iterable[Any]) -> str | _Keep | None: """What the first of ``rules`` a mapping matches makes of it: its label, - ``KEEP``, or None -- nothing matched, or a keep rule did and the guard - refused it (``holds_card_data``), so it is walked as if nothing had: no - rule listed after it is asked.""" + ``KEEP``, or None -- nothing matched, or a keep rule did and it is + refused (``keep_refused``: the guard, or a label rule matching inside + it), so it is walked as if nothing had: no rule listed after it is + asked.""" found = _first_match(value, rules) if found is None: return None rule, keep = found if keep: - return None if holds_card_data(value) else KEEP + return None if keep_refused(value, rules) else KEEP return f"[{make_label(rule.field_type)}]" @@ -469,17 +521,15 @@ def text_label(text: str, rules: Iterable[Any]) -> str | None: def label_within(value: Any, rules: Iterable[Any], depth: int = 0) -> str | None: """The label of a value a rule matches anywhere in ``value`` -- a - mapping, a list, or JSON text, however deep -- or None. A kept mapping is - not descended into: what a keep rule ships holds no label.""" + mapping, a list, or JSON text, however deep -- or None. A keep match is + looked into like any other mapping: one holding a label match is not kept + (``keep_refused``), and one that is kept holds none.""" if depth > _MAX_DEPTH: return None if isinstance(value, dict): found = _first_match(value, rules) - if found is not None: - if not found[1]: - return f"[{make_label(found[0].field_type)}]" - if not holds_card_data(value): - return None + if found is not None and not found[1]: + return f"[{make_label(found[0].field_type)}]" items = value.values() elif isinstance(value, (list, tuple)): items = value diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index c59dd16..a9d4c51 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -18,8 +18,10 @@ pack and with every pack. """ +import html import json import logging +import time import pytest @@ -33,11 +35,16 @@ WALLET_RULES, WALLET_SAD_RULES, ) -from ecsctx.masking import is_kept, patterns -from ecsctx.masking.config import configure_masking_packs, configure_masking_value_rules +from ecsctx.contrib.ottu.masking import SAFE_KEYS as OTTU_SAFE_KEYS +from ecsctx.masking import is_kept, mask_outside_kept, patterns +from ecsctx.masking.config import ( + configure_masking_packs, + configure_masking_safe_keys, + configure_masking_value_rules, +) from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for -from ecsctx.masking.value_rules import is_keep_rule +from ecsctx.masking.value_rules import KeepRule, is_keep_rule from ecsctx.pii import configure_pii from ecsctx.processors import mask_sensitive_data @@ -73,7 +80,11 @@ "protocolVersion": "ECv2", "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "ephemeralPublicKey": "ZXBr", "tag": "dGFn"}), } -GOOGLE_PAY_V1 = {"signature": "MEQCIA==", "protocolVersion": "ECv1", "signedMessage": '{"encryptedMessage":"ZW5j"}'} +GOOGLE_PAY_V1 = { + "signature": "MEQCIA==", + "protocolVersion": "ECv1", + "signedMessage": '{"encryptedMessage":"ZW5j","ephemeralPublicKey":"ZXBr","tag":"dGFn"}', +} NOT_A_WALLET = {"amount": "10.000", "currency": "KWD"} TOKENS = [APPLE_PAY, APPLE_PAY_RSA, PK_PAYMENT_TOKEN, GOOGLE_PAY, GOOGLE_PAY_V1] TOKEN_IDS = ["apple-pay", "apple-pay-rsa", "pk-payment-token", "google-pay", "google-pay-v1"] @@ -125,6 +136,72 @@ "signedMessage": '{"encryptedMessage": {"cvv": "123", "number": "4111111111111111"}, "encryptedMessage": "ZW5j"}', } REPR_COMMENT = "{'version': 'EC_v1', 'data': 'Zm9v', 'header': {'transactionId': 'abc' # 4111111111111111 cvv=123\n}}" +# Text a wallet never writes in an opaque slot (review N3, Redmine R3): the +# guard reads keys, pairs and whole leaves, so this rode in a kept token. +NOT_TOKEN_TEXT = { + "apple-data-cvv": {**APPLE_PAY, "data": "cvv=123"}, + "apple-signature-card-number": {**APPLE_PAY, "signature": "4111 1111 1111 1111"}, + "google-signed-message-extra-key": { + **GOOGLE_PAY, + "signedMessage": json.dumps({**json.loads(GOOGLE_PAY["signedMessage"]), "note": "cvv=123"}), + }, + "google-signature-cvv": {**GOOGLE_PAY, "signature": "cvv=123"}, + "google-v1-encrypted-message-cvv": { + **GOOGLE_PAY_V1, + "signedMessage": '{"encryptedMessage":"cvv=123","ephemeralPublicKey":"ZXBr","tag":"dGFn"}', + }, + # Text in a slot's own alphabet holding a card number (review I2). + "apple-data-pan-and-cvv-in-base64": {**APPLE_PAY, "data": "4111111111111111+cvv+123"}, + "apple-transaction-id-pan-in-hex": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "transactionId": "4111111111111111ab"}, + }, + # A hex id inside base64 is base64's text, read (review r2, 1 and 4); + # digits beside a card number are read as the pci rules read them (3). + "apple-data-hex-id-holding-a-pan": {**APPLE_PAY, "data": "AA/4111111111111111deadbeef/AA"}, + "apple-data-epoch-run": {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, + "apple-transaction-id-twenty-digits": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "transactionId": "41111111111111110000"}, + }, + # A base64 slot is no hex id: hex alone there is not base64 a wallet + # writes, and a card number would ride in it as a hex id (review r3, 1). + "apple-data-all-hex": {**APPLE_PAY, "data": "4111111111111111abcd1234"}, + "apple-key-hash-all-hex": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "publicKeyHash": "4111111111111111123a0000"}, + }, +} +# What the device says about the card: the network's name and the last four. +NOT_A_DISPLAY_NAME = { + "a-card-number": "Visa 4111111111111111", + "a-cvv": "cvv 123", + "a-cvv-after-the-last-four": "Visa 1234 cvv 123", + "five-digits": "Visa 12345", + "the-last-four-first": "1234 Visa", + "too-long": "V" * 41, + "too-long-before-the-last-four": "V" * 41 + " 1234", + # Letters, spaces, `.`, `&` and `-` only, from a letter (review r2, 6). + "an-email": "jane.roe@example.com", + "a-digit-in-the-name": "Visa2 1234", + "a-space-first": " Visa 1234", + # A name that reads as a CVV or other SAD: a CID is four digits (review r3, 3). + "a-cvv-name": "CVV 1234", + "a-security-code-name": "Security Code 1234", + "a-pin-name": "PIN 1234", +} +# A network or a type is words alone, or nothing (bot review on #72, 1). +NOT_A_NETWORK = { + "a-cvv-setting": "cvv=123", + "a-cvv": "CVV 123", + "a-cvv-name": "cvv", + "a-security-code-name": "Security Code", + "an-email": "john@example.com", + "the-last-four": "Visa 1234", + "a-digit": "1", + "a-space-first": " Visa", + "too-long": "V" * 41, +} PACKS = [["default"], sorted(ALL_PACKS)] @@ -220,6 +297,67 @@ def test_each_token_is_kept(self, token): {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": {"keyValue": "a2V5"}, "signatures": ["c2ln"]}}, {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": "c2ln"}}, {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": [{"x": "c2ln"}]}}, + # Every leaf is text in its alphabet (review N3): base64 for keys, + # signatures and ciphertext, hex for ids, digits for a time. + *NOT_TOKEN_TEXT.values(), + {**APPLE_PAY, "data": ""}, + {**APPLE_PAY, "data": "ZGF0YQ==\n"}, + {**APPLE_PAY, "signature": "c2ln-_"}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "ephemeralPublicKey": "cvv=123"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "publicKeyHash": ""}}, + {**APPLE_PAY_RSA, "header": {**APPLE_PAY_RSA["header"], "wrappedKey": "pin 1234"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "cvv-123"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "ZW5j="}}, + {**PK_PAYMENT_TOKEN, "transactionIdentifier": "cvv 123"}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "displayName": "V" * 65}}, + {**GOOGLE_PAY, "signedMessage": "cvv=123"}, + {**GOOGLE_PAY, "signedMessage": json.dumps(["ZW5j"])}, + {**GOOGLE_PAY, "signedMessage": '{"tag": "dGFn", "tag": "cvv=123"}'}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": {"cvv": "123"}})}, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signatures": ["cvv=1"]}}, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5", "keyExpiration": "1700000000000", "cvv": "123"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "pin=1234", "keyExpiration": "1700000000000"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5", "keyExpiration": "17000000000-0"}), + }, + }, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signedKey": "a2V5"}}, + # Each signed object holds every field it is documented to hold + # (review M7). + {**GOOGLE_PAY, "signedMessage": "{}"}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "ephemeralPublicKey": "ZXBr"})}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "tag": "dGFn"})}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"ephemeralPublicKey": "ZXBr", "tag": "dGFn"})}, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signedKey": "{}"}}, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyExpiration": "1700000000000"}), + }, + }, ], ids=[ "apple-extra-key", "apple-version", "apple-data-not-text", "apple-header-extra-key", @@ -229,12 +367,98 @@ def test_each_token_is_kept(self, token): "apple-signature-pair", "apple-signature-list", "pk-display-name-pair", "pk-network-list", "pk-transaction-identifier-number", "google-signed-message-pair", "google-signed-message-mapping", "google-signature-number", "google-signed-key-mapping", "google-signatures-text", - "google-signatures-mappings", + "google-signatures-mappings", *NOT_TOKEN_TEXT, "apple-data-empty", "apple-data-line-break", + "apple-signature-base64url", "apple-ephemeral-key-cvv", "apple-public-key-hash-empty", + "apple-wrapped-key-pin", "apple-transaction-id-not-hex", "apple-application-data-not-hex", + "pk-transaction-identifier-not-hex", "pk-display-name-too-long", "google-signed-message-not-json", + "google-signed-message-a-list", "google-signed-message-duplicate-key", + "google-signed-message-a-mapping-inside", "google-signatures-cvv", "google-signed-key-extra-key", + "google-key-value-pin", "google-key-expiration-not-digits", "google-signed-key-not-json", + "google-signed-message-empty", "google-signed-message-no-tag", "google-signed-message-no-ephemeral-key", + "google-signed-message-no-encrypted-message", "google-signed-key-empty", "google-signed-key-no-expiration", + "google-signed-key-no-key-value", ], ) def test_the_matchers_are_strict(self, value): assert not is_kept(value) + @pytest.mark.parametrize("path", ["key-walk", "text", "body"]) + def test_a_crafted_400_kb_token_is_read_in_linear_time(self, path, packs): + # Hex ids inside a base64 slot, each holding a Luhn-valid run: 400 KB + # of them cost 6.5 s while a run inside a hex token was skipped by + # rescanning the tokens for each run (review r2, 1). + token = {**APPLE_PAY, "data": "AA" + "/4111111111111111deadbeef" * 16_000} + assert len(token["data"]) > 400_000 + started = time.perf_counter() + assert not is_kept(token) + if path == "key-walk": + assert patterns.kept_spans(json.dumps({"x": token}), WALLET_RULES) == [] + elif path == "text": + assert patterns.kept_spans(f"token {json.dumps(token)} end", WALLET_RULES) == [] + else: + assert patterns.kept_spans(KPAY_BODY.format(json.dumps(token)), WALLET_RULES) == [] + assert time.perf_counter() - started < 1.0 + + @pytest.mark.parametrize("name", NOT_A_DISPLAY_NAME.values(), ids=NOT_A_DISPLAY_NAME.keys()) + def test_a_display_name_is_a_network_and_the_last_four(self, name): + token = {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "displayName": name}} + assert not is_kept(token) + assert not is_kept(json.dumps(token)) + # Its payment data is still kept, on its own. + assert _walk({"token": token})["token"]["paymentData"] == APPLE_PAY + + @pytest.mark.parametrize("field", ["network", "type"]) + @pytest.mark.parametrize("text", NOT_A_NETWORK.values(), ids=NOT_A_NETWORK.keys()) + def test_a_network_and_a_type_are_words(self, field, text): + token = {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], field: text}} + assert not is_kept(token) + assert not is_kept(json.dumps(token)) + assert _walk({"token": token})["token"]["paymentData"] == APPLE_PAY + + @pytest.mark.parametrize("token", NOT_TOKEN_TEXT.values(), ids=NOT_TOKEN_TEXT.keys()) + def test_a_token_with_text_no_wallet_writes_is_masked_as_without_the_rules(self, token, packs): + def every_path(): + return ( + _walk({"payload": token}), + _walk({"paymentToken": json.dumps(token)}), + _text(f"token {json.dumps(token)} end", packs), + _message("token %s", json.dumps(token)), + redact_body(json.dumps({"token": token, "amt": "1"})), + redact_body(KPAY_BODY.format(json.dumps(token))), + ) + + assert every_path() == _unconfigured(every_path) + + @pytest.mark.parametrize( + "token", + [ + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "0a1B" * 16}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "MasterCard 0492", "network": "MasterCard"}}, + {**GOOGLE_PAY, "signature": "MEQCIGZh+2Utc2/n"}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Amex", "network": "AmEx", "type": "credit"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Apple Pay", "network": "Visa", "type": "debit"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 35 + " 1234"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 40 + " 1234"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "American Express 1234", "network": "AmEx"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"network": "CartesBancaires", "type": "prepaid"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"network": "girocard", "type": ""}}, + # A 64-hex id holding a Luhn-valid run of digits, as about one in + # two hundred does: an id, not a card number (fix round 2). + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "a" * 10 + "4111111111111111" + "b" * 38}}, + {**PK_PAYMENT_TOKEN, "transactionIdentifier": "A" * 10 + "4111111111111111" + "B" * 38}, + ], + ids=[ + "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", + "pk-display-name-connects-fixture", "pk-display-name-forty-characters", + "pk-display-name-forty-letters-and-the-last-four", "pk-display-name-in-words", + "pk-network-and-type", "pk-network-and-no-type", + "apple-transaction-id-with-a-luhn-run", "pk-transaction-identifier-with-a-luhn-run", + ], + ) + def test_a_documented_token_is_still_kept(self, token): + assert is_kept(token) + assert is_kept(json.dumps(token)) + class TestWalletTokensShipAsSent: @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) @@ -311,6 +535,15 @@ def test_entity_encoded_and_with_connects_extra_secret_keys(self, token): assert masked["paymentToken"] == plain assert masked["signature"] != "outer-signature" + def test_a_label_rule_inside_a_keep_match_wins(self, packs): + # A keep rule matching the outer mapping first shipped the token a + # label rule names inside it unlabelled (review M8). + configure_masking_value_rules([APPLE_PAY_RULE, KeepRule(lambda value: True)]) + event = {"outer": {"inner": APPLE_PAY}} + assert _walk({"payload": event}) == {"payload": {"outer": {"inner": SAD}}} + assert _text(f"got {json.dumps(event)}", packs) == f'got {{"outer": {{"inner": "{SAD}"}}}}' + assert redact_body(json.dumps(event)) == f'{{"outer": {{"inner": "{SAD}"}}}}' + def test_a_cvv_beside_it_is_still_masked(self): masked = _walk({"payload": {"paymentData": APPLE_PAY, "cvv": "123"}}) assert masked == {"payload": {"paymentData": APPLE_PAY, "cvv": "[CVV-MASKED]"}} @@ -374,6 +607,103 @@ def test_saved_card_tokens_stay_masked(self, saved): assert "4111111111111111" not in json.dumps(before) +# A CVV or SAD container in text around a token, or beside it (review I1): +# the key walk keeps nothing in any of them, and text that names a CVV or SAD +# key, element or pair label anywhere keeps nothing. +FLOORED = { + "a-list-between": '{"cvv": [ TOKEN ]}', + "a-pair-labelled-cvv": '{"name": "cvv", "value": TOKEN}', + "a-pair-in-a-list": '[{"name":"cvv","value": TOKEN}]', + "a-pair-labelled-in-words": '{"name": "card security code", "value": TOKEN}', + "a-pair-labelled-pin-as-json-text": '{"Name": "PIN", "Value": "ESCAPED"}', + "quoted-key-equals": '"cvv" = TOKEN', + "quoted-key-arrow": "'cvv' => TOKEN", + "bare-key-arrow": "securityCode => TOKEN", + "deeper-than-the-text-rules-read": '{"securityCode": {"x": {"y": [TOKEN]}}}', + "a-sibling-of-a-cvv-key": '{"cvv": "123", "token": TOKEN}', + "an-element-elsewhere": "1234 TOKEN", + "a-key-written-after-it": 'TOKEN and then "cryptogram": "AAAB"', +} +FLOOR_TOKENS = {"pk-payment-token": PK_PAYMENT_TOKEN, "apple-pay": APPLE_PAY, "google-pay": GOOGLE_PAY} + + +def _floored(template: str, token: dict) -> str: + escaped = json.dumps(json.dumps(token))[1:-1] + return template.replace("ESCAPED", escaped).replace("TOKEN", json.dumps(token)) + + +def _connect_apple_pay_request() -> dict: + """What Connect's SDK sends to pay with Apple Pay.""" + return { + "payment_method": "apple_pay", + "code": "apple-pay-kwd", + "amount": "100.000", + "currency_code": "KWD", + "session_id": "b1e2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0", + "cvv_required": False, + "apple_pay_payload": {"token": {"token": PK_PAYMENT_TOKEN}}, + } + + +class TestTheFloorInText: + @pytest.fixture(params=FLOOR_TOKENS.values(), ids=FLOOR_TOKENS.keys()) + def token(self, request): + return request.param + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_mask_by_patterns(self, template, token, packs): + text = _floored(template, token) + assert patterns.kept_spans(text, WALLET_RULES) == [] + assert _text(text, packs) == _unconfigured(lambda: _text(text, packs)) + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_redact_body(self, template, token): + # Connect's extra secret keys: what is not kept has its signature masked. + configure_redaction(extra_secret_keys=["signature", "hash"]) + text = _floored(template, token) + for body in ( + text, + f"S3cretPassw0rd{text}", + f"S3cretPassw0rd{html.escape(text)}", + ): + assert redact_body(body) == _unconfigured(lambda body=body: redact_body(body)) + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_mask_outside_kept(self, template, token): + text = _floored(template, token) + assert mask_outside_kept(text, str.upper) == text.upper() + + +class TestStillKeptInText: + @pytest.fixture(autouse=True) + def ottu_safe_keys(self): + configure_masking_safe_keys(OTTU_SAFE_KEYS) + + def test_a_kpay_body(self, packs): + written = json.dumps(PK_PAYMENT_TOKEN) + body = KPAY_BODY.format(written) + for masked in (redact_body(body), _text(body, packs), redact_body(KPAY_BODY.format(html.escape(written)))): + assert "S3cretPassw0rd" not in masked + assert f"{written}" in redact_body(body) + assert f"{written}" in _text(body, packs) + assert f"{html.escape(written)}" in redact_body(KPAY_BODY.format(html.escape(written))) + + def test_connects_sdk_apple_pay_request(self, packs): + request = json.dumps(_connect_apple_pay_request()) + written = json.dumps(PK_PAYMENT_TOKEN) + assert written in _text(f"pay request {request}", packs) + assert written in redact_body(request) + assert written in mask_outside_kept(request, str.upper) + assert _walk({"payload": _connect_apple_pay_request()}) == {"payload": _connect_apple_pay_request()} + + def test_a_google_pay_payment_data_body(self, packs): + body = json.dumps(GOOGLE_PAYMENT_DATA) + escaped = json.dumps(json.dumps(GOOGLE_PAY))[1:-1] + for masked in (_text(f"google pay {body}", packs), redact_body(body)): + assert escaped in masked + assert "jane.roe@example.com" not in _text(f"google pay {body}", packs) + + class TestTheSadVariant: """0.16.0's assertions, with WALLET_SAD_RULES.""" diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 31b343b..6c7a768 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -17,8 +17,10 @@ """ import copy +import html import json import logging +import re import pytest from django.test import override_settings @@ -69,12 +71,16 @@ def _is_sealed(value) -> bool: KEEP_RULE = KeepRule(_is_sealed, hints=("sealed-box",)) KEEP_RULES = (KEEP_RULE,) +KEEP_ALL = KeepRule(lambda value: True) LABEL_RULE = ValueRule("sad", _is_sealed, hints=("sealed-box",)) LABEL = "[SAD-MASKED]" OUTER_RULE = ValueRule("secret", lambda value: value.get("kind") == "outer-box", hints=("outer-box",)) # A label rule with a hint of its own, which never matches. OTHER_RULE = ValueRule("secret", lambda value: value.get("kind") == "never", hints=("other-shape",)) PASSWORD = "password=s3cret-Hunter2" +# A 64-hex digest holding a run of digits that passes Luhn, as about one in +# two hundred random ones does. +DIGEST_WITH_A_RUN = "a" * 10 + "4111111111111111" + "b" * 38 class DuckKeep: @@ -96,8 +102,20 @@ def matches(self, value) -> bool: return True +class Ambiguous: + """A label rule's `field_type` and a keep rule's `keep = True` at once.""" + + field_type = "sad" + keep = True + hints = ("sealed-box",) + + def matches(self, value) -> bool: + return _is_sealed(value) + + DUCK_KEEP = DuckKeep() KEEP_YES = KeepYes() +AMBIGUOUS = Ambiguous() PACKS = [["default"], sorted(ALL_PACKS)] @@ -194,6 +212,30 @@ def test_label_and_keep_rules_share_one_list(self): assert label_rules(rules) == (LABEL_RULE,) assert keep_rules(rules) is keep_rules(rules) + def test_a_rule_with_both_a_field_type_and_keep_is_refused(self): + # It loaded as a keep rule, the less safe reading (review M6). + rules, problems = value_rules.load_value_rules([AMBIGUOUS, KEEP_RULE]) + assert rules == (KEEP_RULE,) + [problem] = problems + assert "`field_type`" in problem + assert "`keep = True`" in problem + with pytest.raises(ValueError, match="`field_type`"): + configure_masking_value_rules([AMBIGUOUS]) + with pytest.raises(ValueError, match="`keep = True`"): + configure_masking_value_rules([(KEEP_RULE, AMBIGUOUS)]) + + def test_an_ambiguous_rule_is_reported_by_the_boot_check(self, monkeypatch): + monkeypatch.setenv( + "ECSCTX_MASK_VALUE_RULES", + "tests.test_masking_keep_rules.AMBIGUOUS,tests.test_masking_keep_rules.KEEP_RULES", + ) + with pytest.warns(RuntimeWarning, match="AMBIGUOUS"): + assert _walk({"x": SEALED}) == {"x": SEALED} + [error] = [error for error in find_masking_errors({}) if "ECSCTX_MASK_VALUE_RULES" in error] + assert "AMBIGUOUS" in error + assert "`field_type`" in error + assert "`keep = True`" in error + def test_a_bad_item_is_reported_by_the_boot_check(self, monkeypatch): monkeypatch.setenv( "ECSCTX_MASK_VALUE_RULES", @@ -275,14 +317,74 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): {"note": 4111111111111111}, {"note": json.dumps({"inner": {"securityCode": "123"}})}, {"deep": [{"pin": "1234"}]}, + # A card number inside a longer text leaf (review I2). + {"note": "card 4111111111111111 exp 12/27"}, + {"box": "4111111111111111+cvv+123"}, + {"ref": "4111111111111111ab"}, + {"note": "Visa 4111 1111 1111 1111 on file"}, + {"memo": ";4111111111111111=25121010000000000000?"}, + {"note": "{not json 4111111111111111"}, + # Hex shorter than an id, or glued to more letters: read. + {"ref": "4111111111111111abcdef"}, + {"ref": "zz" + DIGEST_WITH_A_RUN}, + {"ref": DIGEST_WITH_A_RUN + "z"}, + # Only a leaf that is a whole hex id is not read (review r2, 1 and + # 4): one inside base64, or in prose, is. + {"box": "AA/4111111111111111deadbeef/AA"}, + {"note": "session aaaa4111111111111111bbbb ended"}, + # A run read as the pci rules read one (review r2, 3): a card + # beside another digit group, more digits than a card, no Luhn. + {"note": "1234 4111111111111111"}, + {"note": "4111111111111111 123"}, + {"note": "4111111111111111 12/25"}, + {"ref": "41111111111111110000"}, + {"ref": "order 4111111111111112 retried"}, + # Epoch milliseconds are only a whole leaf's. + {"box": "QUJD1790318444473QUJD"}, + # A hex id holds a hex letter: 24 digits or more are read. + {"digest": "411111111111111112345678"}, + ], + ids=[ + "cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list", + "a-pan-in-prose", "a-pan-in-base64", "a-pan-in-hex", "a-grouped-pan-in-prose", "track-2", + "a-pan-in-text-that-is-no-json", "hex-shorter-than-an-id", "hex-glued-to-letters", "hex-glued-to-a-letter-after", + "a-hex-id-in-base64", "a-24-hex-id-in-prose", "a-group-before-a-card", "a-cvv-after-a-card", + "an-expiry-after-a-card", "twenty-digits", "a-run-failing-luhn", "epoch-milliseconds-in-a-longer-leaf", + "digits-only-in-a-hex-slot", ], - ids=["cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list"], ) def test_the_guard_a_match_holding_card_data_is_walked(self, extra): holding = {**SEALED, **extra} + assert not is_kept(holding) _masked_as_a_walk(_walk({"payload": holding})["payload"]) assert _walk({"payload": json.dumps(holding)})["payload"] != json.dumps(holding) + @pytest.mark.parametrize( + "extra", + [ + {"note": "cvv=123"}, + {"note": "order 12345678901 shipped"}, + {"expiration": "1790318444473"}, + {"note": json.dumps({"keyExpiration": "1790318444473"})}, + # A leaf that is a whole hex id of 24 characters or more: a + # digest, an ObjectId, Apple Pay's transactionId (fix rounds 2, 3). + {"digest": DIGEST_WITH_A_RUN}, + {"digest": DIGEST_WITH_A_RUN.upper()}, + {"digest": "aaaa4111111111111111bbbb"}, + ], + ids=[ + "a-cvv-in-a-string", "eleven-digits", "epoch-milliseconds", "json-text-epoch", + "a-64-hex-digest", "an-uppercase-digest", "a-24-hex-id", + ], + ) + def test_what_the_guard_does_not_read_in_a_text_leaf(self, extra): + # A CVV or other short value inside a longer string is the matcher's + # to keep out; fewer digits than a card number, epoch milliseconds + # and a whole hex id hold none. + holding = {**SEALED, **extra} + assert is_kept(holding) + assert _walk({"payload": holding}) == {"payload": holding} + def test_a_rule_that_keeps_everything_still_cannot_ship_a_card(self): configure_masking_value_rules([KeepRule(lambda value: True)]) masked = _walk({"payment": {"card": {"number": "4111111111111111", "cvv": "123", "expiry": "12/27"}}}) @@ -368,6 +470,30 @@ def test_list_order_decides_keep_first(self): configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) assert _walk({"x": SEALED, "y": json.dumps(SEALED)}) == {"x": SEALED, "y": json.dumps(SEALED)} + def test_a_label_rule_wins_inside_a_keep_match(self, packs): + # A keep rule matching the outer mapping first shipped what a label + # rule names inside it unlabelled (review M8). + configure_masking_value_rules([LABEL_RULE, KEEP_ALL]) + event = {"outer": {"inner": SEALED}, "note": "x"} + labelled = {"outer": {"inner": LABEL}, "note": "x"} + assert _walk({"payload": event}) == {"payload": labelled} + assert json.loads(_walk({"payload": json.dumps(event)})["payload"]) == labelled + assert _walk({"password": json.dumps(event)}) == {"password": LABEL} + assert _text(f"got {json.dumps(event)} back", packs) == f"got {json.dumps(labelled)} back" + assert json.loads(redact_body(json.dumps(event))) == labelled + assert not is_kept(event) + assert not is_kept(json.dumps(event)) + assert is_kept({"outer": {"note": "x"}}) + + def test_a_label_rule_listed_after_a_keep_match_of_the_same_value_is_not_asked(self, packs): + # The first match still wins for the value itself: a keep rule listed + # first that matches every value labels nothing inside. + configure_masking_value_rules([KEEP_ALL, LABEL_RULE]) + event = {"outer": {"inner": SEALED}, "note": "x"} + assert _walk({"payload": event}) == {"payload": event} + assert _text(f"got {json.dumps(event)} back", packs) == f"got {json.dumps(event)} back" + assert is_kept(event) + def test_a_keep_match_the_guard_refuses_asks_no_rule_listed_after_it(self, packs): configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) holding = {**SEALED, "cvv": "123"} @@ -405,6 +531,37 @@ def _kept_in(masked: str, kept: str) -> None: assert "s3cret-Hunter2" not in masked +# CVV and SAD containers in text that are not the key right before the +# object (review M2), and the same keys beside the value (review I1): text +# that names a CVV or SAD key, element or pair label anywhere keeps nothing. +FLOORED = { + "a-list-between": '{"cvv": [ PLAIN ]}', + "a-pair-labelled-cvv": '{"name": "cvv", "value": PLAIN}', + "a-pair-labelled-pin-as-json-text": '{"Name": "PIN", "Value": "ESCAPED"}', + "quoted-key-equals": '"cvv" = PLAIN', + "quoted-key-arrow": "'cvv' => PLAIN", + "bare-key-arrow": "securityCode => PLAIN", + "a-sibling-of-a-cvv-key": '{"cvv": "123", "token": PLAIN}', + "a-pair-whose-name-is-written-twice": '{"name": "cvv", "name": "note", "value": PLAIN}', + "an-object-that-does-not-parse": '{"cvv": undefined, "token": PLAIN}', + "a-cvv-list-beside": '{"payment": {"cvv": ["123"]}, "token": PLAIN}', + "a-cvv-pair-beside": '[{"name": "cvv", "value": "123"}, PLAIN]', + "a-cvv-key-before": '"cvv" = "123", "token" => PLAIN', + "a-form-field-before": "a=1&cvv=123 PLAIN", + "an-element-after": "PLAIN 1234", + "a-key-longer-than-a-key-is-read": '{"' + "x" * 200 + '_cvv": "123", "token": PLAIN}', +} +# Keys about a CVV, not one: text naming them keeps its value. +NOT_FLOORED = { + "a-flag-about-a-cvv": '{"cvv_required": true, "token": PLAIN}', + "a-cvv-word-in-prose": "the cvv check passed for PLAIN", +} + + +def _written(template: str) -> str: + return template.replace("ESCAPED", json.dumps(json.dumps(SEALED))[1:-1]).replace("PLAIN", json.dumps(SEALED)) + + @pytest.mark.usefixtures("keep") class TestText: def test_a_message_ships_it_as_sent_and_masks_the_rest(self, packs): @@ -455,16 +612,35 @@ def test_nothing_under_a_cvv_or_sad_key_is_kept(self, template, packs): text = template.format(sealed) assert _text(text, packs) != text + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, template, packs): + # Not only right after a CVV key: the key walk keeps nothing in any + # of these (review M2). + text = _written(template) + masked = _text(text, packs) + assert masked == _unconfigured(lambda: _text(text, packs)) + assert SEALED["api_key"] not in masked + + @pytest.mark.parametrize("template", NOT_FLOORED.values(), ids=NOT_FLOORED.keys()) + def test_text_naming_no_cvv_key_keeps_its_value(self, template, packs): + assert json.dumps(SEALED) in _text(_written(template), packs) + def test_an_outer_label_match_wins(self, packs): configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) text = "got " + json.dumps({"kind": "outer-box", "inner": SEALED}) assert _text(text, packs) == "got [SECRET-MASKED]" - def test_an_inner_label_match_is_kept_with_the_kept_object(self, packs): + def test_an_inner_label_match_wins_over_the_keep_match_around_it(self, packs): + # 0.17.0 kept it with the kept object (review M8): the keep match is + # now read as one nothing matches, the inner match labelled. configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) sealed = {**SEALED, "inner": {"kind": "outer-box"}} text = f"got {json.dumps(sealed)}" - assert _text(text, packs) == text + masked = _text(text, packs) + assert '"inner": "[SECRET-MASKED]"' in masked + assert SEALED["api_key"] not in masked + configure_masking_value_rules([OUTER_RULE]) + assert masked == _text(text, packs) def test_a_label_rule_listed_first_labels_it(self, packs): configure_masking_value_rules([LABEL_RULE, KEEP_RULE]) @@ -502,6 +678,42 @@ def test_a_placeholder_some_rule_destroyed_is_not_restored(self, packs): assert BOX not in masked assert "KEPT-" not in masked + def test_a_placeholder_is_named_from_the_value_it_holds(self): + # Not by position (review M3, M4): the same value has the same name in + # any text, another value another name, all of one length. + sealed, other = json.dumps(SEALED), json.dumps({**SEALED, "n": 1}) + stash = patterns.stash_kept(f"a {sealed} b {other} c {sealed}", KEEP_RULES) + names = re.findall(r"\[KEPT-[A-Z]+-MASKED\]", stash.text) + assert len(names) == 3 + assert names[0] == names[2] != names[1] + assert len(names[0]) == len(names[1]) + assert stash.originals == {names[0]: sealed, names[1]: other} + assert patterns.stash_kept(f"x {sealed}", KEEP_RULES).text == f"x {names[0]}" + + def test_two_identical_kept_values_are_both_restored(self, packs): + text = f"a {json.dumps(SEALED)} b {PASSWORD} c {json.dumps(SEALED)}" + masked = _text(text, packs) + assert masked.count(json.dumps(SEALED)) == 2 + assert "s3cret-Hunter2" not in masked + + def test_a_credential_hashed_around_a_kept_value_carries_that_value(self, mode, packs): + # The credential's token hashes the placeholder: named by position, + # every kept value there gave one token, where the key walk gives + # each its own (review M3). + def in_a_secret(sealed: dict) -> str: + return json.dumps({"secret": json.dumps({"k": sealed})}) + + first, second = {**SEALED, "n": 1}, {**SEALED, "n": 2} + masked = _text(in_a_secret(first), packs) + assert BOX not in masked + assert masked == _text(in_a_secret(first), packs) + if mode == "keyset": + assert masked != _text(in_a_secret(second), packs) + + def test_a_value_holding_a_lone_surrogate_is_kept_and_nothing_raises(self, packs): + text = "got " + json.dumps({**SEALED, "note": "\ud800"}, ensure_ascii=False) + assert _text(text, packs) == text + @pytest.mark.usefixtures("keep") class TestCost: @@ -649,6 +861,60 @@ def test_nothing_under_a_cvv_key_is_kept(self): assert SEALED["api_key"] not in redact_body(KPAY.format(json.dumps({"cvv": SEALED}))) assert BOX not in redact_body(f"{json.dumps(SEALED)}") + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, template): + body = _written(template) + assert patterns.kept_spans(body, KEEP_RULES) == [] + assert redact_body(body) == _unconfigured(lambda: redact_body(body)) + + @pytest.mark.parametrize("template", NOT_FLOORED.values(), ids=NOT_FLOORED.keys()) + def test_text_naming_no_cvv_key_keeps_its_value(self, template): + assert json.dumps(SEALED) in redact_body(_written(template)) + + def test_an_element_naming_a_placeholder_gets_no_copy_of_a_kept_value(self): + # Its entity-encoded text decodes to a placeholder's name, which the + # outer restore filled with the kept value beside it (review M4). + forged = "{"password":"pw","n":"[KEPT-A-MASKED]"}" + masked = redact_body(f"{forged}{json.dumps(SEALED)}") + element = json.loads(html.unescape(masked.split("")[1].split("")[0])) + assert element["n"] == "[KEPT-A-MASKED]" + assert element["password"] != "pw" + assert f"{json.dumps(SEALED)}" in masked + + def test_two_identical_entity_encoded_elements_are_both_restored(self): + encoded = json.dumps(SEALED).replace('"', """) + masked = redact_body(f"{encoded}S3cretPassw0rd{encoded}") + assert f"{encoded}" in masked + assert f"{encoded}" in masked + assert "S3cretPassw0rd" not in masked + + def test_a_label_rule_that_raises_inside_a_keep_match_refuses_it_and_raises_nothing(self, packs): + # Asked about a keep match's members (review M5): its error refuses + # the keep, and the value is masked as with the label rule alone. + def raises_on_members(value): + if value.get("kind") == "inner-box": + raise RuntimeError("rule failed") + return False + + label_rule = ValueRule("sad", raises_on_members, hints=("sealed-box",)) + sealed = {**SEALED, "inner": {"kind": "inner-box"}} + compact = json.dumps(sealed, separators=(",", ":")) + + def every_path(): + return ( + _text(f"got {json.dumps(sealed)} back", packs), + redact_body(json.dumps(sealed)), + redact_body(KPAY.format(json.dumps(sealed).replace('"', """))), + redact_url(f"https://pg.example/pay?card_number={compact}&password=s3cret"), + mask_outside_kept(f"got {json.dumps(sealed)} back", str.upper), + ) + + configure_masking_value_rules([KEEP_RULE, label_rule]) + masked = every_path() + configure_masking_value_rules([label_rule]) + assert masked == every_path() + assert SEALED["api_key"] not in masked[0] + def test_redact_url_never_raises_with_a_keep_rule_that_raises(self): def broken(_value): raise RuntimeError("rule failed")