From 2550cf43de2a518697a2660cb55752636895ecdc Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:20:53 +0300 Subject: [PATCH 01/22] fix(masking): a kept value's placeholder is named from the value it holds Placeholders were named by position, `[KEPT-A-MASKED]`, `[KEPT-B-MASKED]`. An element whose entity-encoded text decoded to such a name got the kept value beside it copied in by the outer restore (review M4), and a credential hashed around a kept value hashed the placeholder, so every value there gave one token where the key walk gives each its own (M3). A placeholder is now named from a blake2b digest of the value as written: fourteen digit-free letters, base 26. The same value has the same name in any text (two identical kept values share one entry), a different value another, and a name written by anyone without the value restores nothing. Two values whose names collide are masked as without a keep rule. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++++--- ecsctx/masking/patterns.py | 59 ++++++++++++++++++++------------ tests/test_masking_keep_rules.py | 55 +++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1cfb1ff..31f49e6 100644 --- a/README.md +++ b/README.md @@ -1435,10 +1435,14 @@ value itself, not something the key says about it. In text, what a keep rule ships is set aside as a `[KEPT--MASKED]` placeholder before every content rule, and put back as it was written after; a placeholder a rule destroys — a credential hashed with what was around it — takes the value with -it. Text that already holds `[KEPT-` is masked as if no keep rule were -configured. A keep rule that raises counts as no match: keep rules are asked -on paths that must never raise (`redact_url`, `redact_body`, -`mask_card_value`). +it. A placeholder is named from a digest of the value it stands for: the same +value has the same name, so a credential hashed around one carries that value +in its token, as under a key, and a name written into the text by anyone +without the value (an element's entity-encoded text decodes to one) is not a +placeholder and restores nothing. Text that already holds `[KEPT-` is masked +as if no keep rule were configured. A keep rule that raises counts as no +match: keep rules are asked on paths that must never raise (`redact_url`, +`redact_body`, `mask_card_value`). Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index cea3640..6195b5b 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -29,6 +29,7 @@ import ast import contextlib +import hashlib import html import json import re @@ -1314,16 +1315,24 @@ def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[ # which every rule leaves as it is, with no digit in it for a rule to read. _KEPT_HEAD = "[KEPT-" _KEPT_PLACEHOLDER = re.compile(r"\[KEPT-[A-Z]+-MASKED\]") - - -def _letters(index: int) -> str: - """0 is A, 25 is Z, 26 is AA: a placeholder's own, digit-free name.""" - letters = "" - index += 1 - while index: - index, rest = divmod(index - 1, 26) - letters = chr(ord("A") + rest) + letters - return letters +# A placeholder's letters: its value's 8-byte digest in base 26, always this +# many (26**14 > 2**64), so no two names differ by length alone. +_NAME_LETTERS = 14 + + +def _placeholder(value: str) -> str: + """The placeholder standing for ``value``, named from a digest of it, not + by position: the same value has the same name in any text, so a + credential hashed with one around it carries that value in its token, as + under a key; and only who has the value can write its name, so a name + written into the text (an element's entity-encoded text decodes to one) + restores nothing. Digit-free; never raises (a lone surrogate passes).""" + digest = hashlib.blake2b(value.encode("utf-8", "surrogatepass"), digest_size=8).digest() + number, letters = int.from_bytes(digest, "big"), [] + for _ in range(_NAME_LETTERS): + number, rest = divmod(number, 26) + letters.append(chr(ord("A") + rest)) + return f"{_KEPT_HEAD}{''.join(letters)}-MASKED]" class KeptStash(NamedTuple): @@ -1334,11 +1343,14 @@ class KeptStash(NamedTuple): originals: dict[str, str] def hold(self, value: str) -> str: - """A new placeholder standing for ``value``, for the caller to put + """The placeholder standing for ``value``, for the caller to put where it stood: ``contrib.net.redact_body`` sets aside an element's - entity-encoded text that holds a kept value.""" - placeholder = f"{_KEPT_HEAD}{_letters(len(self.originals))}-MASKED]" - self.originals[placeholder] = value + entity-encoded text that holds a kept value. ``value`` itself where + its name already stands for another value: the rules then read it, + and may mask what is in it.""" + placeholder = _placeholder(value) + if self.originals.setdefault(placeholder, value) != value: + return value return placeholder def restore(self, masked: str) -> str: @@ -1352,11 +1364,12 @@ def restore(self, masked: str) -> str: def stash_kept(text: str, rules: tuple | None = None) -> KeptStash | None: """``text`` with each value a keep rule ships set aside as a placeholder - (``[KEPT-A-MASKED]``), for the rules to read around it -- or None when - there is none: no keep rule among ``rules`` (the rules in force when - None), none whose hints the text holds, no span (``kept_spans``), or text - that already holds a placeholder's head, which is masked as before. - Never raises.""" + named from it (``[KEPT--MASKED]``; the same value twice, one + name), for the rules to read around it -- or None when there is none: no + keep rule among ``rules`` (the rules in force when None), none whose + hints the text holds, no span (``kept_spans``), text that already holds a + placeholder's head, or two values whose names collide; such text is + masked as before. Never raises.""" if "{" not in text: return None if rules is None: @@ -1369,9 +1382,11 @@ def stash_kept(text: str, rules: tuple | None = None) -> KeptStash | None: parts: list[str] = [] originals: dict[str, str] = {} copied = 0 - for index, (start, end) in enumerate(spans): - placeholder = f"{_KEPT_HEAD}{_letters(index)}-MASKED]" - originals[placeholder] = text[start:end] + for start, end in spans: + value = text[start:end] + placeholder = _placeholder(value) + if originals.setdefault(placeholder, value) != value: + return None parts += [text[copied:start], placeholder] copied = end parts.append(text[copied:]) diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 31b343b..4da83ae 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -17,8 +17,10 @@ """ import copy +import html import json import logging +import re import pytest from django.test import override_settings @@ -502,6 +504,42 @@ def test_a_placeholder_some_rule_destroyed_is_not_restored(self, packs): assert BOX not in masked assert "KEPT-" not in masked + def test_a_placeholder_is_named_from_the_value_it_holds(self): + # Not by position (review M3, M4): the same value has the same name in + # any text, another value another name, all of one length. + sealed, other = json.dumps(SEALED), json.dumps({**SEALED, "n": 1}) + stash = patterns.stash_kept(f"a {sealed} b {other} c {sealed}", KEEP_RULES) + names = re.findall(r"\[KEPT-[A-Z]+-MASKED\]", stash.text) + assert len(names) == 3 + assert names[0] == names[2] != names[1] + assert len(names[0]) == len(names[1]) + assert stash.originals == {names[0]: sealed, names[1]: other} + assert patterns.stash_kept(f"x {sealed}", KEEP_RULES).text == f"x {names[0]}" + + def test_two_identical_kept_values_are_both_restored(self, packs): + text = f"a {json.dumps(SEALED)} b {PASSWORD} c {json.dumps(SEALED)}" + masked = _text(text, packs) + assert masked.count(json.dumps(SEALED)) == 2 + assert "s3cret-Hunter2" not in masked + + def test_a_credential_hashed_around_a_kept_value_carries_that_value(self, mode, packs): + # The credential's token hashes the placeholder: named by position, + # every kept value there gave one token, where the key walk gives + # each its own (review M3). + def in_a_secret(sealed: dict) -> str: + return json.dumps({"secret": json.dumps({"k": sealed})}) + + first, second = {**SEALED, "n": 1}, {**SEALED, "n": 2} + masked = _text(in_a_secret(first), packs) + assert BOX not in masked + assert masked == _text(in_a_secret(first), packs) + if mode == "keyset": + assert masked != _text(in_a_secret(second), packs) + + def test_a_value_holding_a_lone_surrogate_is_kept_and_nothing_raises(self, packs): + text = "got " + json.dumps({**SEALED, "note": "\ud800"}, ensure_ascii=False) + assert _text(text, packs) == text + @pytest.mark.usefixtures("keep") class TestCost: @@ -649,6 +687,23 @@ def test_nothing_under_a_cvv_key_is_kept(self): assert SEALED["api_key"] not in redact_body(KPAY.format(json.dumps({"cvv": SEALED}))) assert BOX not in redact_body(f"{json.dumps(SEALED)}") + def test_an_element_naming_a_placeholder_gets_no_copy_of_a_kept_value(self): + # Its entity-encoded text decodes to a placeholder's name, which the + # outer restore filled with the kept value beside it (review M4). + forged = "{"password":"pw","n":"[KEPT-A-MASKED]"}" + masked = redact_body(f"{forged}{json.dumps(SEALED)}") + element = json.loads(html.unescape(masked.split("")[1].split("")[0])) + assert element["n"] == "[KEPT-A-MASKED]" + assert element["password"] != "pw" + assert f"{json.dumps(SEALED)}" in masked + + def test_two_identical_entity_encoded_elements_are_both_restored(self): + encoded = json.dumps(SEALED).replace('"', """) + masked = redact_body(f"{encoded}S3cretPassw0rd{encoded}") + assert f"{encoded}" in masked + assert f"{encoded}" in masked + assert "S3cretPassw0rd" not in masked + def test_redact_url_never_raises_with_a_keep_rule_that_raises(self): def broken(_value): raise RuntimeError("rule failed") From 7da3acaed0b15ad0d99df677201afafcb1d04218 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:24:40 +0300 Subject: [PATCH 02/22] fix(masking): the text floor reads a CVV container that is not the key right before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In text the floor read only the key right before an object, so these were kept where the key walk keeps nothing (review M2): `{"cvv": [ {…} ]}` (a list between), `{"name": "cvv", "value": {…}}` (a pair labelled as one, which the README said the floor covered), `"cvv" = {…}` and `'cvv' => {…}`. `_find_kept` no longer looks into an object nothing matched whose parse has a CVV or SAD key at its top level or is a pair labelled as one -- read as the guard reads a pair, now `value_rules.pair_labelled` -- nor into one that does not parse or names a key twice. It fails closed: a value under a sibling key of such an object is masked, not kept. `_KEY_BEFORE` takes `=>`, and a quoted key followed by `=` or `=>`. Co-Authored-By: Claude Opus 5.5 --- README.md | 27 +++++++++++------ ecsctx/masking/patterns.py | 51 ++++++++++++++++++++++---------- ecsctx/masking/value_rules.py | 39 ++++++++++++++---------- tests/test_masking_keep_rules.py | 49 ++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 31f49e6..692d076 100644 --- a/README.md +++ b/README.md @@ -1449,8 +1449,9 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: - **The floor.** Nothing is kept under a CVV or SAD key (`cvv`, `securityCode`, `pin`, `cryptogram`, …), inside a CVV or SAD container (a key the service lists there included), or as the `value` of a pair - labelled as one; in text, right after such a quoted key, `key=` or XML - start tag. Such a value is masked exactly as without the rule. + labelled as one; in text, right after such a key (quoted or not, then `:`, + `=` or `=>`) or XML start tag, and anywhere in an object holding such a key + or labelled as one. Such a value is masked exactly as without the rule. - **The guard.** A match that holds, at any depth — JSON text inside it included — a key the key rules read as a card, a CVV or other SAD, a `{name, value}` pair whose identifier reads as one (as the key walk reads a @@ -1487,13 +1488,21 @@ before masking a field, and runs its text masker through value set aside and puts back each placeholder it leaves. Both are in `ecsctx.masking` and never raise. -In text the floor reads the key right before an object, and every object -the text rules read whole inside it: they read three levels of nesting, so a -kept value two keys below a CVV key (`{"cvv": {"x": {"y": {…}}}}`, the value -itself nested) is not floored there — the guard still holds. An object right -after a CVV key in text is not kept, but no `default` content rule masks a -JSON object there either, with or without keep rules; the key walk masks one -under a CVV key leaf by leaf. +In text the floor reads the key right before an object (`"cvv": {…}`, +`cvv={…}`, `'cvv' => {…}`, `{…}`), and every object the text +rules read whole: one with a CVV or SAD key at its top level, or a +`{name, value}` pair labelled as one, is not looked into for a value to keep, +whatever stands between the key and the value (`{"cvv": [{…}]}`). That fails +closed: a value under another key of such an object +(`{"cvv": "123", "token": {…}}`) is not kept in text, though the key walk +keeps it, and nothing is kept in an object read whole that does not parse or +names a key twice. A value beside such an object, not in it, is kept. The +text rules read three levels of nesting, so a kept value two keys below a CVV +key (`{"cvv": {"x": {"y": {…}}}}`, the value itself nested) is not floored +there — the guard still holds. An object right after a CVV key in text is +not kept, but no `default` content rule masks a JSON object there either, +with or without keep rules; the key walk masks one under a CVV key leaf by +leaf. The cost: with no keep rule configured none of this runs — a string with no `{` pays one check, one with a `{` a look at the rules in force; with keep diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 6195b5b..2e8df44 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -64,6 +64,7 @@ json_as_written, keep_rules, label_rules, + pair_labelled, ruling, text_ruling, ) @@ -1206,12 +1207,12 @@ def mask_objects(text: str, rules: tuple) -> str: # Authentication Data. _FLOOR_TYPES = frozenset({"cvv", "sad"}) # The key right before an object in text, as the floor reads it: a quoted key -# and its colon (JSON, a repr, JSON in a JSON string, its quotes escaped), a -# key and `=` or `:`, or an XML start tag -- then any space and the opening -# quote of a string the object is the JSON text of. +# and its colon (JSON, a repr, JSON in a JSON string, its quotes escaped) or +# `=` or `=>`, a key and `:`, `=` or `=>`, or an XML start tag -- then any +# space and the opening quote of a string the object is the JSON text of. _KEY_BEFORE = re.compile( - r"(?:(?P\\?[\"'])(?P[^\"'\\]{1,128})(?P=quote)\s*:" - r"|(?[\w.-]{1,128})\s*[:=]" + r"(?:(?P\\?[\"'])(?P[^\"'\\]{1,128})(?P=quote)\s*(?::|=>?)" + r"|(?[\w.-]{1,128})\s*(?::|=>?)" r"|<(?:[A-Za-z_][\w.-]*:)?(?P[A-Za-z_][\w.-]*)(?:\s[^<>]*)?(?)" r"\s*(?:\\?[\"'])?\s*\Z" ) @@ -1273,25 +1274,43 @@ def _object_as_read(m: re.Match) -> tuple[Any, bool]: return None, True -def _object_ruling(m: re.Match, rules: tuple) -> Any: - """``ruling`` of the object a match wrote; a label rule's error reads as a - label -- not kept, not looked into -- so finding kept spans never raises, - and the text is masked as without a keep rule. A keep match written - otherwise than it parses (``_written_as_parsed``) is read as one nothing - matches.""" +def _object_ruling(m: re.Match, rules: tuple) -> tuple[Any, bool]: + """``ruling`` of the object a match wrote, and whether nothing inside it + may be kept when nothing matched it (``_floors``). A label rule's error + reads as a label -- not kept, not looked into -- so finding kept spans + never raises, and the text is masked as without a keep rule. A keep + match written otherwise than it parses (``_written_as_parsed``) is read + as one nothing matches.""" try: value, as_written = _object_as_read(m) found = ruling(value, rules) except Exception: # noqa: BLE001 -- a service's matcher; rule 2 reports it where it runs - return "" - return None if found is KEEP and not as_written else found + return "", True + if found is KEEP: + return (KEEP, False) if as_written else (None, True) + return found, found is None and (not as_written or _floors(value)) + + +def _floors(value: Any) -> bool: + """Whether nothing in an object, as parsed, may be kept: one with a CVV + or SAD key at its top level, whatever stands between that key and a + value (`{"cvv": [{…}]}`), or a `{name, value}` pair labelled as one, as + the key walk keeps nothing in such a container or as such a pair's + value. Fails closed: a value under a sibling key is not kept either, nor + anything in an object that does not parse.""" + if not isinstance(value, dict): + return True + return pair_labelled(value, _FLOOR_TYPES) or any( + classify_key(str(key), ALL_PACKS) in _FLOOR_TYPES for key in value + ) def kept_spans(text: str, rules: tuple) -> list[tuple[int, int]]: """Where ``text`` writes a value a keep rule among ``rules`` matches first and may keep: each JSON object (rule 2's reading), top down. An object a label rule matches first is no span and is not looked into, nor is one - right after a CVV or SAD key; one nothing matches is read member by + right after a CVV or SAD key, nor one holding a CVV or SAD key or + labelled as one (``_floors``); one nothing matches is read member by member; a keep match the guard refuses (``holds_card_data``) is read as one nothing matches. In order, none inside another.""" spans: list[tuple[int, int]] = [] @@ -1304,10 +1323,10 @@ def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[ asked = applicable(m.group(0), rules) if not asked or _floored_before(text, m.start()): continue - found = _object_ruling(m, asked) + found, floored = _object_ruling(m, asked) if found is KEEP: spans.append(m.span()) - elif found is None: + elif found is None and not floored: _find_kept(text, m.start() + 1, m.end() - 1, rules, spans) diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index b0379fe..0960be6 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -306,6 +306,27 @@ def parses_as_written(text: str) -> bool: _EPOCH_MILLISECONDS = re.compile(r"[12][0-9]{12}") +def pair_labelled(pair: Mapping, types: frozenset[str]) -> bool: + """Whether a ``{name, value}`` pair's identifier reads as one of ``types`` + (every pack on), as the key walk reads a pair (``filters._pair``), which + masks its ``value`` as that type.""" + keys = {str(key).lower(): key for key in pair} + if "value" not in keys: + return False + # Imported here, past the check nearly every mapping fails: patterns and + # filters import this module as they load. + from ecsctx.masking.filters import _PAIR_IDENTIFIERS + from ecsctx.masking.patterns import ALL_PACKS, classify_key + + # Uncached, as the key walk classifies free text: an identifier may be a + # sentence, and must not evict a key name. + return any( + isinstance(label := pair[keys[identifier]], str) and classify_key.__wrapped__(label, ALL_PACKS) in types + for identifier in _PAIR_IDENTIFIERS + if identifier in keys + ) + + def holds_card_data(value: Any) -> bool: """Whether ``value`` holds, at any depth -- JSON text inside it included -- a key the key rules read as a card, a CVV or other SAD (every pack on), a @@ -326,8 +347,7 @@ def holds_card_data(value: Any) -> bool: (``json_as_written``: what ships would not be what was read) count as holding card data: such a match is walked, not shipped. """ - # Imported here: patterns and filters import this module as they load. - from ecsctx.masking.filters import _PAIR_IDENTIFIERS + # Imported here: patterns imports this module as it loads. from ecsctx.masking.patterns import ( ALL_PACKS, classify_key, @@ -335,24 +355,11 @@ def holds_card_data(value: Any) -> bool: pan_shaped, ) - def labels_card_data(pair: Mapping) -> bool: - keys = {str(key).lower(): key for key in pair} - if "value" not in keys: - return False - # Uncached, as the key walk classifies free text: an identifier may be - # a sentence, and must not evict a key name. - return any( - isinstance(label := pair[keys[identifier]], str) - and classify_key.__wrapped__(label, ALL_PACKS) in _CARD_DATA_TYPES - for identifier in _PAIR_IDENTIFIERS - if identifier in keys - ) - def holds(item: Any, depth: int) -> bool: if depth > _MAX_DEPTH: return True if isinstance(item, Mapping): - if labels_card_data(item): + if pair_labelled(item, _CARD_DATA_TYPES): return True return any( classify_key(str(key), ALL_PACKS) in _CARD_DATA_TYPES or holds(member, depth + 1) diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 4da83ae..448c837 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -407,6 +407,32 @@ def _kept_in(masked: str, kept: str) -> None: assert "s3cret-Hunter2" not in masked +# CVV and SAD containers in text that are not the key right before the +# object (review M2). The last four fail closed: an object holding a CVV key, +# or one whose keys or parse cannot be trusted, is not looked into at all. +FLOORED = { + "a-list-between": '{"cvv": [ PLAIN ]}', + "a-pair-labelled-cvv": '{"name": "cvv", "value": PLAIN}', + "a-pair-labelled-pin-as-json-text": '{"Name": "PIN", "Value": "ESCAPED"}', + "quoted-key-equals": '"cvv" = PLAIN', + "quoted-key-arrow": "'cvv' => PLAIN", + "bare-key-arrow": "securityCode => PLAIN", + "a-sibling-of-a-cvv-key": '{"cvv": "123", "token": PLAIN}', + "a-pair-whose-name-is-written-twice": '{"name": "cvv", "name": "note", "value": PLAIN}', + "an-object-that-does-not-parse": '{"cvv": undefined, "token": PLAIN}', +} +# The same keys beside the kept value, not around it. +BESIDE = { + "a-cvv-list-beside": '{"payment": {"cvv": ["123"]}, "token": PLAIN}', + "a-cvv-pair-beside": '[{"name": "cvv", "value": "123"}, PLAIN]', + "a-cvv-key-before": '"cvv" = "123", "token" => PLAIN', +} + + +def _written(template: str) -> str: + return template.replace("ESCAPED", json.dumps(json.dumps(SEALED))[1:-1]).replace("PLAIN", json.dumps(SEALED)) + + @pytest.mark.usefixtures("keep") class TestText: def test_a_message_ships_it_as_sent_and_masks_the_rest(self, packs): @@ -457,6 +483,19 @@ def test_nothing_under_a_cvv_or_sad_key_is_kept(self, template, packs): text = template.format(sealed) assert _text(text, packs) != text + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, template, packs): + # Not only right after a CVV key: the key walk keeps nothing in any + # of these (review M2). + text = _written(template) + masked = _text(text, packs) + assert masked == _unconfigured(lambda: _text(text, packs)) + assert SEALED["api_key"] not in masked + + @pytest.mark.parametrize("template", BESIDE.values(), ids=BESIDE.keys()) + def test_a_value_beside_a_cvv_container_is_still_kept(self, template, packs): + assert json.dumps(SEALED) in _text(_written(template), packs) + def test_an_outer_label_match_wins(self, packs): configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) text = "got " + json.dumps({"kind": "outer-box", "inner": SEALED}) @@ -687,6 +726,16 @@ def test_nothing_under_a_cvv_key_is_kept(self): assert SEALED["api_key"] not in redact_body(KPAY.format(json.dumps({"cvv": SEALED}))) assert BOX not in redact_body(f"{json.dumps(SEALED)}") + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, template): + body = _written(template) + assert patterns.kept_spans(body, KEEP_RULES) == [] + assert redact_body(body) == _unconfigured(lambda: redact_body(body)) + + @pytest.mark.parametrize("template", BESIDE.values(), ids=BESIDE.keys()) + def test_a_value_beside_a_cvv_container_is_still_kept(self, template): + assert json.dumps(SEALED) in redact_body(_written(template)) + def test_an_element_naming_a_placeholder_gets_no_copy_of_a_kept_value(self): # Its entity-encoded text decodes to a placeholder's name, which the # outer restore filled with the kept value beside it (review M4). From 74fc3c6cbc166e63155295cd27c97cf7fc427847 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:27:23 +0300 Subject: [PATCH 03/22] fix(masking): a label rule wins at any depth inside a keep match With `[APPLE_PAY_RULE, KeepRule(lambda value: True)]`, `{"outer": {"inner": }}` shipped the inner token unlabelled: the keep rule matched the outer mapping first (review M8). A keep match is now refused, walked as if nothing had matched, when a label rule labels something strictly inside it (`keep_refused`: `label_within` over its members, asked only when a label rule is among the rules). The check sits where the guard's does, so the key walk, text, rule 2, bodies and `is_kept` all get it; `label_within` looks into a keep match like any other mapping. For the value itself the first match still wins: a keep rule listed first for the same value keeps it. `test_an_inner_label_match_is_kept_with_the_kept_object` pinned 0.17.0's reading and now pins this one. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 +++++++++------ ecsctx/masking/patterns.py | 16 ++++++------ ecsctx/masking/value_rules.py | 39 +++++++++++++++++++++--------- tests/test_contrib_ottu_wallets.py | 11 ++++++++- tests/test_masking_keep_rules.py | 35 +++++++++++++++++++++++++-- 5 files changed, 92 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 692d076..ac7eef7 100644 --- a/README.md +++ b/README.md @@ -1425,13 +1425,19 @@ asked where label rules are: every mapping the key walk meets, every string that is JSON text, every JSON object in free text or in a body `redact_body` masks. They are asked in list order and **the first match wins**: a label rule listed first labels a value a keep rule would have kept, and an object a -label rule matches is not looked into for one to keep. A keep match the guard -refuses (below) is walked as if nothing had matched: no rule listed after it -is asked, a label rule for the same shape included. A kept value is left -as sent under any key and not descended into — a credential's key, a card's, -or a PII container's above it does not stop it (`token`, `paymentToken`, -`password`, `card.devicePayment`, `customer`): what the rule names is the -value itself, not something the key says about it. In text, what a keep rule +label rule matches is not looked into for one to keep. A label rule wins at +any depth inside a keep match, though: a keep match holding a value a label +rule labels (that value's own first match) is walked as if nothing had +matched, and that value is labelled — with an Apple Pay label rule and +`KeepRule(lambda value: True)`, `{"outer": {"inner": }}` is +`{"outer": {"inner": "[SAD-MASKED]"}}`. A keep match the guard refuses +(below) is walked as if nothing had matched too, and either way no rule +listed after it is asked about the value itself, a label rule for the same +shape included. A kept value is left as sent under any key and not +descended into — a credential's key, a card's, or a PII container's above it +does not stop it (`token`, `paymentToken`, `password`, `card.devicePayment`, +`customer`): what the rule names is the value itself, not something the key +says about it. In text, what a keep rule ships is set aside as a `[KEPT--MASKED]` placeholder before every content rule, and put back as it was written after; a placeholder a rule destroys — a credential hashed with what was around it — takes the value with diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 2e8df44..6921448 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -60,8 +60,8 @@ _first_match, applicable, hinted, - holds_card_data, json_as_written, + keep_refused, keep_rules, label_rules, pair_labelled, @@ -1159,16 +1159,17 @@ def _mask_object(m: re.Match, rules: tuple) -> str: is itself in a string; bare anywhere else -- in prose, an element's text, a string of its own. An object no rule matches is read member by member, so one inside it is still found; one a keep rule matches first is left as - written and not looked into, unless the guard refuses it or it is - written otherwise than it parses: then it is read as one nothing matches. - Only the rules whose hints the object's text holds are asked.""" + written and not looked into, unless it is refused (``keep_refused``: the + guard, or a label rule matching inside it) or written otherwise than it + parses: then it is read as one nothing matches. Only the rules whose + hints the object's text holds are asked.""" written = m.group(0) asked = applicable(written, rules) if not asked: return written # nothing in it can match: every match holds a hint value = _object_value(m) found = _first_match(value, asked) - if found is not None and found[1] and (holds_card_data(value) or not _written_as_parsed(m)): + if found is not None and found[1] and (keep_refused(value, asked) or not _written_as_parsed(m)): found = None if found is None: inner = mask_objects(written[1:-1], rules) @@ -1311,8 +1312,9 @@ def kept_spans(text: str, rules: tuple) -> list[tuple[int, int]]: label rule matches first is no span and is not looked into, nor is one right after a CVV or SAD key, nor one holding a CVV or SAD key or labelled as one (``_floors``); one nothing matches is read member by - member; a keep match the guard refuses (``holds_card_data``) is read as - one nothing matches. In order, none inside another.""" + member; a keep match refused (``keep_refused``: the guard, or a label + rule matching inside it) is read as one nothing matches. In order, none + inside another.""" spans: list[tuple[int, int]] = [] _find_kept(text, 0, len(text), rules, spans) return spans diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 0960be6..cdd221d 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -22,7 +22,10 @@ is kept under a CVV or SAD key or inside a CVV or SAD container (the floor, applied where a key is known: ``filters``, ``patterns.kept_spans``), and a match holding a card, CVV or SAD key, or a card number, at any depth, is - walked as if nothing matched (the guard, ``holds_card_data``). + walked as if nothing matched (the guard, ``holds_card_data``). A match + holding a value a label rule labels is walked so too, wherever that rule + is listed (``keep_refused``): a label rule wins at any depth inside a keep + match, the first match only for the value itself. Both have: @@ -390,17 +393,31 @@ def holds(item: Any, depth: int) -> bool: return holds(value, 0) +def keep_refused(value: dict, rules: Iterable[Any]) -> bool: + """Whether a keep match is walked as if nothing had matched: the guard + refuses it (``holds_card_data``), or a label rule among ``rules`` labels + something strictly inside it (``label_within``, over its members): a + label rule wins at any depth inside a keep match. For the value itself + the first match still wins. Only with a label rule among ``rules``.""" + if holds_card_data(value): + return True + return bool(_split(rules).label) and any( + label_within(member, rules, 1) is not None for member in value.values() + ) + + def ruling(value: Any, rules: Iterable[Any]) -> str | _Keep | None: """What the first of ``rules`` a mapping matches makes of it: its label, - ``KEEP``, or None -- nothing matched, or a keep rule did and the guard - refused it (``holds_card_data``), so it is walked as if nothing had: no - rule listed after it is asked.""" + ``KEEP``, or None -- nothing matched, or a keep rule did and it is + refused (``keep_refused``: the guard, or a label rule matching inside + it), so it is walked as if nothing had: no rule listed after it is + asked.""" found = _first_match(value, rules) if found is None: return None rule, keep = found if keep: - return None if holds_card_data(value) else KEEP + return None if keep_refused(value, rules) else KEEP return f"[{make_label(rule.field_type)}]" @@ -476,17 +493,15 @@ def text_label(text: str, rules: Iterable[Any]) -> str | None: def label_within(value: Any, rules: Iterable[Any], depth: int = 0) -> str | None: """The label of a value a rule matches anywhere in ``value`` -- a - mapping, a list, or JSON text, however deep -- or None. A kept mapping is - not descended into: what a keep rule ships holds no label.""" + mapping, a list, or JSON text, however deep -- or None. A keep match is + looked into like any other mapping: one holding a label match is not kept + (``keep_refused``), and one that is kept holds none.""" if depth > _MAX_DEPTH: return None if isinstance(value, dict): found = _first_match(value, rules) - if found is not None: - if not found[1]: - return f"[{make_label(found[0].field_type)}]" - if not holds_card_data(value): - return None + if found is not None and not found[1]: + return f"[{make_label(found[0].field_type)}]" items = value.values() elif isinstance(value, (list, tuple)): items = value diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index c59dd16..0e366f1 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -37,7 +37,7 @@ from ecsctx.masking.config import configure_masking_packs, configure_masking_value_rules from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for -from ecsctx.masking.value_rules import is_keep_rule +from ecsctx.masking.value_rules import KeepRule, is_keep_rule from ecsctx.pii import configure_pii from ecsctx.processors import mask_sensitive_data @@ -311,6 +311,15 @@ def test_entity_encoded_and_with_connects_extra_secret_keys(self, token): assert masked["paymentToken"] == plain assert masked["signature"] != "outer-signature" + def test_a_label_rule_inside_a_keep_match_wins(self, packs): + # A keep rule matching the outer mapping first shipped the token a + # label rule names inside it unlabelled (review M8). + configure_masking_value_rules([APPLE_PAY_RULE, KeepRule(lambda value: True)]) + event = {"outer": {"inner": APPLE_PAY}} + assert _walk({"payload": event}) == {"payload": {"outer": {"inner": SAD}}} + assert _text(f"got {json.dumps(event)}", packs) == f'got {{"outer": {{"inner": "{SAD}"}}}}' + assert redact_body(json.dumps(event)) == f'{{"outer": {{"inner": "{SAD}"}}}}' + def test_a_cvv_beside_it_is_still_masked(self): masked = _walk({"payload": {"paymentData": APPLE_PAY, "cvv": "123"}}) assert masked == {"payload": {"paymentData": APPLE_PAY, "cvv": "[CVV-MASKED]"}} diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 448c837..9a20fba 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -71,6 +71,7 @@ def _is_sealed(value) -> bool: KEEP_RULE = KeepRule(_is_sealed, hints=("sealed-box",)) KEEP_RULES = (KEEP_RULE,) +KEEP_ALL = KeepRule(lambda value: True) LABEL_RULE = ValueRule("sad", _is_sealed, hints=("sealed-box",)) LABEL = "[SAD-MASKED]" OUTER_RULE = ValueRule("secret", lambda value: value.get("kind") == "outer-box", hints=("outer-box",)) @@ -370,6 +371,30 @@ def test_list_order_decides_keep_first(self): configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) assert _walk({"x": SEALED, "y": json.dumps(SEALED)}) == {"x": SEALED, "y": json.dumps(SEALED)} + def test_a_label_rule_wins_inside_a_keep_match(self, packs): + # A keep rule matching the outer mapping first shipped what a label + # rule names inside it unlabelled (review M8). + configure_masking_value_rules([LABEL_RULE, KEEP_ALL]) + event = {"outer": {"inner": SEALED}, "note": "x"} + labelled = {"outer": {"inner": LABEL}, "note": "x"} + assert _walk({"payload": event}) == {"payload": labelled} + assert json.loads(_walk({"payload": json.dumps(event)})["payload"]) == labelled + assert _walk({"password": json.dumps(event)}) == {"password": LABEL} + assert _text(f"got {json.dumps(event)} back", packs) == f"got {json.dumps(labelled)} back" + assert json.loads(redact_body(json.dumps(event))) == labelled + assert not is_kept(event) + assert not is_kept(json.dumps(event)) + assert is_kept({"outer": {"note": "x"}}) + + def test_a_label_rule_listed_after_a_keep_match_of_the_same_value_is_not_asked(self, packs): + # The first match still wins for the value itself: a keep rule listed + # first that matches every value labels nothing inside. + configure_masking_value_rules([KEEP_ALL, LABEL_RULE]) + event = {"outer": {"inner": SEALED}, "note": "x"} + assert _walk({"payload": event}) == {"payload": event} + assert _text(f"got {json.dumps(event)} back", packs) == f"got {json.dumps(event)} back" + assert is_kept(event) + def test_a_keep_match_the_guard_refuses_asks_no_rule_listed_after_it(self, packs): configure_masking_value_rules([KEEP_RULE, LABEL_RULE]) holding = {**SEALED, "cvv": "123"} @@ -501,11 +526,17 @@ def test_an_outer_label_match_wins(self, packs): text = "got " + json.dumps({"kind": "outer-box", "inner": SEALED}) assert _text(text, packs) == "got [SECRET-MASKED]" - def test_an_inner_label_match_is_kept_with_the_kept_object(self, packs): + def test_an_inner_label_match_wins_over_the_keep_match_around_it(self, packs): + # 0.17.0 kept it with the kept object (review M8): the keep match is + # now read as one nothing matches, the inner match labelled. configure_masking_value_rules([KEEP_RULE, OUTER_RULE]) sealed = {**SEALED, "inner": {"kind": "outer-box"}} text = f"got {json.dumps(sealed)}" - assert _text(text, packs) == text + masked = _text(text, packs) + assert '"inner": "[SECRET-MASKED]"' in masked + assert SEALED["api_key"] not in masked + configure_masking_value_rules([OUTER_RULE]) + assert masked == _text(text, packs) def test_a_label_rule_listed_first_labels_it(self, packs): configure_masking_value_rules([LABEL_RULE, KEEP_RULE]) From 8f90e25af87430d7afc34adf60525e1bb27721a4 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:29:01 +0300 Subject: [PATCH 04/22] fix(masking): a rule with both a field_type and keep = True is refused, not read as a keep rule Such a rule loaded as a keep rule, the less safe reading (review M6). `load_value_rules` now reports it as a problem naming both forms, alone or in a collection: `configure_masking_value_rules` raises, and from the setting or env var it is dropped with a warning and listed by the Django boot check, which surfaces those problems. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++++-- ecsctx/masking/value_rules.py | 29 +++++++++++++++++-------- tests/test_masking_keep_rules.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ac7eef7..3f7a792 100644 --- a/README.md +++ b/README.md @@ -1406,8 +1406,10 @@ under one rule set never comes back as it was under another. A **keep rule** ships what it matches exactly as sent: a value a service must be able to read in its logs, found by shape. It is any object whose `keep` is `True` (the object itself, not a truthy value), with `matches(value)` and, -optionally, `hints`; it has no `field_type`, since it writes no label. -`KeepRule` is one: +optionally, `hints`; it has no `field_type`, since it writes no label. An +object with both a `field_type` and `keep = True` is neither, rather than +the less safe of the two: it is refused as an item that is not a rule is +(above), the problem naming both. `KeepRule` is one: ```python from ecsctx.masking import KeepRule diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index cdd221d..eb6d92b 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -126,16 +126,27 @@ def load_value_rules(value: Iterable[Any] | str) -> tuple[tuple[Any, ...], tuple def _rules_in(item: Any) -> tuple[Any, ...]: found = _imported(item) if isinstance(item, str) else item if is_value_rule(found): - return (found,) - if isinstance(found, Iterable) and not isinstance(found, (str, bytes, Mapping)): + members: tuple[Any, ...] | None = (found,) + elif isinstance(found, Iterable) and not isinstance(found, (str, bytes, Mapping)): members = tuple(found) - if all(is_value_rule(member) for member in members): - return members - raise ValueError( - f"{item!r} is not a value rule or a collection of them: a rule has `matches(value)`, " - "either a `field_type` (its label) or `keep = True` (it ships as sent), and optionally " - "`hints`, a collection of strings" - ) + else: + members = None + if members is None or not all(is_value_rule(member) for member in members): + raise ValueError( + f"{item!r} is not a value rule or a collection of them: a rule has `matches(value)`, " + "either a `field_type` (its label) or `keep = True` (it ships as sent), and optionally " + "`hints`, a collection of strings" + ) + for member in members: + # Read as a keep rule it would ship what its author may have meant + # to label: neither reading is taken. + if is_keep_rule(member) and is_label_rule(member): + where = repr(item) if member is found else f"{member!r} in {item!r}" + raise ValueError( + f"{where} has both a `field_type` ({member.field_type!r}: a label rule) and " + "`keep = True` (a keep rule): a rule is one or the other" + ) + return members def _imported(path: str) -> Any: diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 9a20fba..b8dad3f 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -99,8 +99,20 @@ def matches(self, value) -> bool: return True +class Ambiguous: + """A label rule's `field_type` and a keep rule's `keep = True` at once.""" + + field_type = "sad" + keep = True + hints = ("sealed-box",) + + def matches(self, value) -> bool: + return _is_sealed(value) + + DUCK_KEEP = DuckKeep() KEEP_YES = KeepYes() +AMBIGUOUS = Ambiguous() PACKS = [["default"], sorted(ALL_PACKS)] @@ -197,6 +209,30 @@ def test_label_and_keep_rules_share_one_list(self): assert label_rules(rules) == (LABEL_RULE,) assert keep_rules(rules) is keep_rules(rules) + def test_a_rule_with_both_a_field_type_and_keep_is_refused(self): + # It loaded as a keep rule, the less safe reading (review M6). + rules, problems = value_rules.load_value_rules([AMBIGUOUS, KEEP_RULE]) + assert rules == (KEEP_RULE,) + [problem] = problems + assert "`field_type`" in problem + assert "`keep = True`" in problem + with pytest.raises(ValueError, match="`field_type`"): + configure_masking_value_rules([AMBIGUOUS]) + with pytest.raises(ValueError, match="`keep = True`"): + configure_masking_value_rules([(KEEP_RULE, AMBIGUOUS)]) + + def test_an_ambiguous_rule_is_reported_by_the_boot_check(self, monkeypatch): + monkeypatch.setenv( + "ECSCTX_MASK_VALUE_RULES", + "tests.test_masking_keep_rules.AMBIGUOUS,tests.test_masking_keep_rules.KEEP_RULES", + ) + with pytest.warns(RuntimeWarning, match="AMBIGUOUS"): + assert _walk({"x": SEALED}) == {"x": SEALED} + [error] = [error for error in find_masking_errors({}) if "ECSCTX_MASK_VALUE_RULES" in error] + assert "AMBIGUOUS" in error + assert "`field_type`" in error + assert "`keep = True`" in error + def test_a_bad_item_is_reported_by_the_boot_check(self, monkeypatch): monkeypatch.setenv( "ECSCTX_MASK_VALUE_RULES", From f0c3662ec9b2005b7c02bb556d84c12de47966c6 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:31:47 +0300 Subject: [PATCH 05/22] fix(contrib): Ottu's wallet matchers keep a token only when every slot holds token text The guard reads keys, pairs and whole leaves, so text inside a kept token's string slot shipped with it: `"data": "cvv=123"` (review N3, Redmine R3). Each slot must now be text in its alphabet. Apple Pay `paymentData`: `data`, `signature` and the header's `ephemeralPublicKey`, `publicKeyHash` and `wrappedKey` standard base64, `transactionId` and `applicationData` hex. PKPaymentToken: `transactionIdentifier` hex, `paymentMethod` short text (64 characters at most). Google Pay: `signature` and every `intermediateSigningKey.signatures` base64; `signedMessage` JSON text of `encryptedMessage`, `ephemeralPublicKey` and `tag` in base64 and nothing else, `signedKey` of `keyValue` in base64 and `keyExpiration` in digits and nothing else, both read strictly (a key written twice refused). A token that does not conform is masked as before. Every fixture token conforms. Co-Authored-By: Claude Opus 5.5 --- README.md | 24 ++++-- ecsctx/contrib/ottu/masking.py | 120 ++++++++++++++++++++--------- tests/test_contrib_ottu_wallets.py | 87 ++++++++++++++++++++- 3 files changed, 188 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 3f7a792..5b71c72 100644 --- a/README.md +++ b/README.md @@ -1528,13 +1528,23 @@ Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or Google Pay token is single-use, and the token as it was sent is what debugs a wallet payment. `ecsctx.contrib.ottu.masking.WALLET_RULES` are keep rules with strict matchers — an Apple Pay `PKPaymentToken`, its `paymentData` -(`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header` of text -fields) and a Google Pay payment method token (`protocolVersion` `ECv1`, -`ECv2` or `ECv2SigningOnly`, with a `signedMessage`), and no key besides the -ones each writes — found under `paymentData`, TAP's `token_data`, MPGS's -`paymentToken` (a JSON string), Google Pay's `tokenizationData.token`, KPay's -``, or in a message. A `PKPaymentToken` with a key of its own keeps -only its `paymentData`. Saved-card tokens stay masked: no wallet shape +(`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header`) and a +Google Pay payment method token (`protocolVersion` `ECv1`, `ECv2` or +`ECv2SigningOnly`, with a `signedMessage`), no key besides the ones each +writes, and every leaf text in its alphabet: standard base64 for a key, a +signature or ciphertext (`data`, `signature`, the header's +`ephemeralPublicKey`, `publicKeyHash` and `wrappedKey`, Google Pay's +`signatures`, and the fields of its `signedMessage` and `signedKey`, each +JSON text of its own fields and nothing else), hex for an id or a hash +(`transactionId`, `applicationData`, `transactionIdentifier`), digits for +`keyExpiration`, and short text (64 characters at most) for what +`paymentMethod` says about the card. A token holding anything else in a +slot — `"data": "cvv=123"`, a spaced card number as its `signature` — is +not kept, and is masked as before: the guard reads keys, pairs and whole +leaves, so text riding in a kept slot would ship. They are found under +`paymentData`, TAP's `token_data`, MPGS's `paymentToken` (a JSON string), +Google Pay's `tokenizationData.token`, KPay's ``, or in a message. A +`PKPaymentToken` with a key of its own keeps only its `paymentData`. Saved-card tokens stay masked: no wallet shape matches one. Samsung Pay's token, a JWE, is not matched yet. An Ottu service lists them in `ECSCTX_MASK_VALUE_RULES` next to its `ECSCTX_MASK_SAFE_KEYS` (above), or sets diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index ab8ff19..ba53241 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -33,9 +33,11 @@ a credential: no wallet shape matches it, under ``token`` or anywhere. - What is not exactly a wallet's shape. The matchers are strict -- an Apple Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method - token, and no key besides the ones they write -- so anything else is walked - as before: a ``PKPaymentToken`` with a key of its own keeps only its - ``paymentData``. + token, no key besides the ones they write, and every leaf text in its + alphabet (base64, hex or digits; short text for what ``paymentMethod`` + says about the card) -- so anything else is walked as before: a + ``PKPaymentToken`` with a key of its own keeps only its ``paymentData``, + and a token with ``"data": "cvv=123"`` is masked. - Card data. ecsctx keeps nothing under a CVV or SAD key, and nothing that holds a card, CVV or SAD key or a card number (the floor and the guard). @@ -59,9 +61,11 @@ service masks stays in its own settings. """ +import re +from collections.abc import Mapping from typing import Any -from ecsctx.masking.value_rules import KeepRule, ValueRule +from ecsctx.masking.value_rules import KeepRule, ValueRule, json_as_written SAFE_KEYS = frozenset({ # Ottu PG API fields: a gateway's short name ("mpgs") and flags saying @@ -141,77 +145,123 @@ def _is_google_pay_token(value: Any) -> bool: # Pay's payment method token are documented to hold, and nothing else. A key # besides these is no wallet's, and the object is walked as before. _PAYMENT_DATA_KEYS = frozenset({"version", "data", "signature", "header"}) -_PAYMENT_DATA_HEADER_KEYS = frozenset( - {"ephemeralPublicKey", "wrappedKey", "publicKeyHash", "transactionId", "applicationData"} -) _PK_PAYMENT_TOKEN_KEYS = frozenset({"paymentData", "paymentMethod", "transactionIdentifier"}) _PAYMENT_METHOD_KEYS = frozenset({"displayName", "network", "type"}) _GOOGLE_PAY_KEYS = frozenset({"signature", "intermediateSigningKey", "protocolVersion", "signedMessage"}) _SIGNING_KEY_KEYS = frozenset({"signedKey", "signatures"}) +_SIGNED_MESSAGE_KEYS = frozenset({"encryptedMessage", "ephemeralPublicKey", "tag"}) +_SIGNED_KEY_KEYS = frozenset({"keyValue", "keyExpiration"}) + +# What a wallet writes in a slot: standard base64 for a key, a signature or +# ciphertext, hex for an id or a hash, digits for a time -- nothing else. +# The guard reads keys, pairs and whole leaves, so text in a kept token's +# slot (`"data": "cvv=123"`) would ship with it (Redmine R3). +_BASE64 = re.compile(r"[A-Za-z0-9+/]+={0,2}") +_HEX = re.compile(r"[0-9A-Fa-f]+") +_DIGITS = re.compile(r"[0-9]+") +_HEADER_ALPHABETS = { + "ephemeralPublicKey": _BASE64, + "publicKeyHash": _BASE64, + "wrappedKey": _BASE64, + "transactionId": _HEX, + "applicationData": _HEX, +} +# PKPaymentToken's `paymentMethod` is what the device says about the card, +# free text ("Visa 0492"): text, and short. +_PAYMENT_METHOD_TEXT_LIMIT = 64 + + +def _written_in(field: Any, alphabet: re.Pattern) -> bool: + """Whether a wallet's field is text in its alphabet: every leaf a wallet + writes is a string, so a container or a number in a slot is no wallet's, + nor is text another alphabet writes -- nothing can ride in it.""" + return isinstance(field, str) and alphabet.fullmatch(field) is not None -def _text_or_absent(value: Any, key: str) -> bool: - """A wallet's field is text where it is written: every leaf a wallet - writes is a string, so a container or a number in a slot is no wallet's, - and nothing can ride in it -- a `{name, value}` pair among them.""" - return key not in value or isinstance(value[key], str) +def _absent_or_in(value: Mapping, key: str, alphabet: re.Pattern) -> bool: + return key not in value or _written_in(value[key], alphabet) def _is_apple_pay_payment_data(value: Any) -> bool: """Apple Pay's `paymentData`: an EC_v1 or RSA_v1 `version`, the - encrypted `data` and its `signature` as text, and a `header` of text - fields.""" + encrypted `data` and its `signature` in base64, and a `header` of its + keys in base64 and its ids in hex.""" version = value.get("version") - if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and isinstance(value.get("data"), str)): + if not (isinstance(version, str) and version in _APPLE_PAY_VERSIONS and _written_in(value.get("data"), _BASE64)): return False - if not (value.keys() <= _PAYMENT_DATA_KEYS and _text_or_absent(value, "signature")): + if not (value.keys() <= _PAYMENT_DATA_KEYS and _absent_or_in(value, "signature", _BASE64)): return False header = value.get("header", {}) return ( isinstance(header, dict) - and header.keys() <= _PAYMENT_DATA_HEADER_KEYS - and all(isinstance(field, str) for field in header.values()) + and header.keys() <= _HEADER_ALPHABETS.keys() + and all(_written_in(field, _HEADER_ALPHABETS[key]) for key, field in header.items()) ) def _is_pk_payment_token(value: Any) -> bool: - """Apple Pay's PKPaymentToken: its `paymentData`, and what the device says - about the card -- its display name, network and type -- and the - transaction's id, each as text.""" + """Apple Pay's PKPaymentToken: its `paymentData`, what the device says + about the card -- its display name, network and type, each short text -- + and the transaction's id in hex.""" payment_data = value.get("paymentData") if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): return False - if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _text_or_absent(value, "transactionIdentifier")): + if not (value.keys() <= _PK_PAYMENT_TOKEN_KEYS and _absent_or_in(value, "transactionIdentifier", _HEX)): return False method = value.get("paymentMethod", {}) return ( isinstance(method, dict) and method.keys() <= _PAYMENT_METHOD_KEYS - and all(isinstance(field, str) for field in method.values()) + and all(isinstance(field, str) and len(field) <= _PAYMENT_METHOD_TEXT_LIMIT for field in method.values()) ) +def _signed_object(text: Any, keys: frozenset[str]) -> dict | None: + """JSON text a Google Pay token signs, read as the keep decision reads + JSON (a key written twice refused): an object of ``keys`` at most, or + None.""" + if not isinstance(text, str): + return None + try: + parsed = json_as_written(text) + except (ValueError, RecursionError): + return None + return parsed if isinstance(parsed, dict) and parsed.keys() <= keys else None + + +def _is_signed_message(text: Any) -> bool: + """`signedMessage`: JSON text of the encrypted message, its ephemeral + public key and tag, each in base64.""" + message = _signed_object(text, _SIGNED_MESSAGE_KEYS) + return message is not None and all(_written_in(field, _BASE64) for field in message.values()) + + +def _is_signed_key(text: Any) -> bool: + """`signedKey`: JSON text of the intermediate signing key in base64 and + its expiration, epoch milliseconds.""" + key = _signed_object(text, _SIGNED_KEY_KEYS) + return key is not None and _absent_or_in(key, "keyValue", _BASE64) and _absent_or_in(key, "keyExpiration", _DIGITS) + + def _is_google_pay_payment_token(value: Any) -> bool: - """Google Pay's payment method token: its `protocolVersion`, the - `signedMessage` and its `signature` as text, and the intermediate signing - key ECv2 signs with: a `signedKey` as text and its `signatures`, a list - of text.""" + """Google Pay's payment method token: its `protocolVersion`, its + `signature` in base64, the intermediate signing key ECv2 signs with -- + a `signedKey` and its `signatures`, a list in base64 -- and the + `signedMessage`. The two signed JSON texts are parsed last.""" protocol = value.get("protocolVersion") if not (isinstance(protocol, str) and protocol in _GOOGLE_PAY_VERSIONS): return False - if not isinstance(value.get("signedMessage"), str): - return False - if not (value.keys() <= _GOOGLE_PAY_KEYS and _text_or_absent(value, "signature")): + if not (value.keys() <= _GOOGLE_PAY_KEYS and _absent_or_in(value, "signature", _BASE64)): return False signing_key = value.get("intermediateSigningKey", {}) if not (isinstance(signing_key, dict) and signing_key.keys() <= _SIGNING_KEY_KEYS): return False signatures = signing_key.get("signatures", []) - return ( - _text_or_absent(signing_key, "signedKey") - and isinstance(signatures, list) - and all(isinstance(signature, str) for signature in signatures) - ) + if not (isinstance(signatures, list) and all(_written_in(signature, _BASE64) for signature in signatures)): + return False + if "signedKey" in signing_key and not _is_signed_key(signing_key["signedKey"]): + return False + return _is_signed_message(value.get("signedMessage")) APPLE_PAY_TOKEN_KEEP_RULE = KeepRule(_is_pk_payment_token, hints=("EC_v1", "RSA_v1")) diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 0e366f1..da18a7d 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -125,6 +125,18 @@ "signedMessage": '{"encryptedMessage": {"cvv": "123", "number": "4111111111111111"}, "encryptedMessage": "ZW5j"}', } REPR_COMMENT = "{'version': 'EC_v1', 'data': 'Zm9v', 'header': {'transactionId': 'abc' # 4111111111111111 cvv=123\n}}" +# Text a wallet never writes in an opaque slot (review N3, Redmine R3): the +# guard reads keys, pairs and whole leaves, so this rode in a kept token. +NOT_TOKEN_TEXT = { + "apple-data-cvv": {**APPLE_PAY, "data": "cvv=123"}, + "apple-signature-card-number": {**APPLE_PAY, "signature": "4111 1111 1111 1111"}, + "google-signed-message-extra-key": { + **GOOGLE_PAY, + "signedMessage": json.dumps({**json.loads(GOOGLE_PAY["signedMessage"]), "note": "cvv=123"}), + }, + "google-signature-cvv": {**GOOGLE_PAY, "signature": "cvv=123"}, + "google-v1-encrypted-message-cvv": {**GOOGLE_PAY_V1, "signedMessage": '{"encryptedMessage":"cvv=123"}'}, +} PACKS = [["default"], sorted(ALL_PACKS)] @@ -220,6 +232,46 @@ def test_each_token_is_kept(self, token): {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": {"keyValue": "a2V5"}, "signatures": ["c2ln"]}}, {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": "c2ln"}}, {**GOOGLE_PAY, "intermediateSigningKey": {"signedKey": "{}", "signatures": [{"x": "c2ln"}]}}, + # Every leaf is text in its alphabet (review N3): base64 for keys, + # signatures and ciphertext, hex for ids, digits for a time. + *NOT_TOKEN_TEXT.values(), + {**APPLE_PAY, "data": ""}, + {**APPLE_PAY, "data": "ZGF0YQ==\n"}, + {**APPLE_PAY, "signature": "c2ln-_"}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "ephemeralPublicKey": "cvv=123"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "publicKeyHash": ""}}, + {**APPLE_PAY_RSA, "header": {**APPLE_PAY_RSA["header"], "wrappedKey": "pin 1234"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "cvv-123"}}, + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "ZW5j="}}, + {**PK_PAYMENT_TOKEN, "transactionIdentifier": "cvv 123"}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "displayName": "V" * 65}}, + {**GOOGLE_PAY, "signedMessage": "cvv=123"}, + {**GOOGLE_PAY, "signedMessage": json.dumps(["ZW5j"])}, + {**GOOGLE_PAY, "signedMessage": '{"tag": "dGFn", "tag": "cvv=123"}'}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": {"cvv": "123"}})}, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signatures": ["cvv=1"]}}, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5", "keyExpiration": "1700000000000", "cvv": "123"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "pin=1234", "keyExpiration": "1700000000000"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5", "keyExpiration": "17000000000-0"}), + }, + }, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signedKey": "a2V5"}}, ], ids=[ "apple-extra-key", "apple-version", "apple-data-not-text", "apple-header-extra-key", @@ -229,12 +281,45 @@ def test_each_token_is_kept(self, token): "apple-signature-pair", "apple-signature-list", "pk-display-name-pair", "pk-network-list", "pk-transaction-identifier-number", "google-signed-message-pair", "google-signed-message-mapping", "google-signature-number", "google-signed-key-mapping", "google-signatures-text", - "google-signatures-mappings", + "google-signatures-mappings", *NOT_TOKEN_TEXT, "apple-data-empty", "apple-data-line-break", + "apple-signature-base64url", "apple-ephemeral-key-cvv", "apple-public-key-hash-empty", + "apple-wrapped-key-pin", "apple-transaction-id-not-hex", "apple-application-data-not-hex", + "pk-transaction-identifier-not-hex", "pk-display-name-too-long", "google-signed-message-not-json", + "google-signed-message-a-list", "google-signed-message-duplicate-key", + "google-signed-message-a-mapping-inside", "google-signatures-cvv", "google-signed-key-extra-key", + "google-key-value-pin", "google-key-expiration-not-digits", "google-signed-key-not-json", ], ) def test_the_matchers_are_strict(self, value): assert not is_kept(value) + @pytest.mark.parametrize("token", NOT_TOKEN_TEXT.values(), ids=NOT_TOKEN_TEXT.keys()) + def test_a_token_with_text_no_wallet_writes_is_masked_as_without_the_rules(self, token, packs): + def every_path(): + return ( + _walk({"payload": token}), + _walk({"paymentToken": json.dumps(token)}), + _text(f"token {json.dumps(token)} end", packs), + _message("token %s", json.dumps(token)), + redact_body(json.dumps({"token": token, "amt": "1"})), + redact_body(KPAY_BODY.format(json.dumps(token))), + ) + + assert every_path() == _unconfigured(every_path) + + @pytest.mark.parametrize( + "token", + [ + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "0a1B" * 16}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "MasterCard 0492", "network": "MasterCard"}}, + {**GOOGLE_PAY, "signature": "MEQCIGZh+2Utc2/n"}, + ], + ids=["apple-application-data", "pk-display-name", "google-signature-plus-and-slash"], + ) + def test_a_documented_token_is_still_kept(self, token): + assert is_kept(token) + assert is_kept(json.dumps(token)) + class TestWalletTokensShipAsSent: @pytest.mark.parametrize("token", TOKENS, ids=TOKEN_IDS) From e67140eaa26474d32fb73acd38450c55ee9a4247 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 21:40:11 +0300 Subject: [PATCH 06/22] docs(masking): what the text pass keeps of an entity-encoded or repr'd token, what never raises, what keep rules cost - An entity-encoded wallet token is set aside by redact_body but not by the log processor's text pass: with pci about one entity-encoded Google Pay token in ten comes out altered, failing closed (review M5). - is_kept and mask_outside_kept never raise of their own; mask_outside_kept passes on the caller's mask errors (M7). The cost paragraph says what keep rules cost a line, and a line with a wallet. - A repr of a Google Pay ECv2 token inside JSON-encoded text is not kept: its escaped quotes no longer render back as written (N2). - CLAUDE.md: placeholder names, the text floor's reading of an object read whole, a label rule winning inside a keep match, the refused ambiguous rule, Ottu's slot alphabets. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 25 +++++++++++++++++++------ ecsctx/CLAUDE.md | 2 +- ecsctx/masking/patterns.py | 6 +++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cbbe1b3..e7d1e2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,6 @@ ECS-compliant structured logging with W3C Trace Context support. Framework-agnos - Django's `LogContextBinder` must be imported explicitly (not in `__init__.py`) to avoid circular imports - `LoggingContextMiddleware` must be placed AFTER auth middleware to capture user_id - CVV never carries a token, even when PII is configured — PCI forbids storing CVV in any form, so `[CVV-MASKED]` is always the literal, final output -- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list +- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning, though a label rule wins anywhere inside a keep match; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487, each slot text in its alphabet; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list - Card numbers truncate to first 6 + last 4 for 15–19 digits and last 4 only below 15 (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091) — no token alongside, independent of PII configuration, and a second masking pass is a noop - `install_maskers()` only sweeps handlers that exist at call time (no stdlib patching) — a handler built after the call needs another call diff --git a/README.md b/README.md index 5b71c72..ed54a75 100644 --- a/README.md +++ b/README.md @@ -1486,7 +1486,16 @@ A keep decision rests on the text as written, since that is what ships: JSON that names a key twice in one object (`json.loads` keeps the last) is not kept, and a JSON-text leaf naming one inside a match is card data to the guard; a Python repr is kept only where its parse renders back to it exactly -(a comment, or a key written twice, is dropped by the parse). +(a comment, or a key written twice, is dropped by the parse). So a repr whose +strings hold a `"` — a Google Pay token's `signedMessage` — written inside +JSON text, where that quote is escaped, is not kept: it no longer renders +back as written (fails closed). + +An entity-encoded value (`{"…"}`, an XML element's text) is set +aside by `redact_body`, which reads such text as it decodes, but not by the +text rules the log processor runs, which read it as written: there it is not +kept and the content rules read into it — with `pci`, about one +entity-encoded Google Pay token in ten comes out altered (fails closed). A service that masks again after ecsctx — Connect does — asks `is_kept(value, key=None)` (a mapping or JSON text; the rules in force, first @@ -1494,7 +1503,8 @@ match, the guard, and the floor on `key`; False with no keep rule configured) before masking a field, and runs its text masker through `mask_outside_kept(text, mask)`, which hands `mask` the text with each kept value set aside and puts back each placeholder it leaves. Both are in -`ecsctx.masking` and never raise. +`ecsctx.masking` and never raise of their own: `mask_outside_kept` passes on +an error the caller's `mask` raises. In text the floor reads the key right before an object (`"cvv": {…}`, `cvv={…}`, `'cvv' => {…}`, `{…}`), and every object the text @@ -1519,10 +1529,13 @@ rule's hints never asks it; text holding one is read once for the objects in it. Every mapping the key walk meets asks every rule, once — a mapping has no text to hint with — so a matcher must be cheap: test a literal field first. JSON text under a key asks, and its key walk passes on, only the rules whose -hints it holds. On the benchmark's lines (`scripts/bench_masking.py`) 0.17.0 -masks within a few percent of 0.16.0, the run-to-run noise, with no value rule -and with Ottu's `WALLET_RULES` (three keep rules where 0.16.0 had two label -rules): the key walk now asks each mapping once, where it asked twice. +hints it holds. Keep rules cost 2-5% a line, and about 12% more on a line +that carries a wallet token, which is read for the objects in it and whose +match is guarded. On the benchmark's lines (`scripts/bench_masking.py`) +0.17.0 masks within a few percent of 0.16.0, the run-to-run noise, with no +value rule and with Ottu's `WALLET_RULES` (three keep rules where 0.16.0 had +two label rules): the key walk now asks each mapping once, where it asked +twice. Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or Google Pay token is single-use, and the token as it was sent is what debugs a diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 986c051..ab48a9b 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which stops at a kept mapping: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span, a CVV/SAD key right before an object -- quoted, `key=`, an XML start tag -- floors it and all inside it) and restores each placeholder still there; a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a label rule wins at any depth strictly inside a keep match, wherever it is listed (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span, a CVV/SAD key right before an object -- quoted or bare, then `:`, `=` or `=>`, or an XML start tag -- floors it and all inside it, and an object read whole is not looked into when it holds a CVV/SAD key at its top level, is a pair labelled as one (`value_rules.pair_labelled`, the guard's reading), or does not parse or names a key twice (`_floors`; fails closed, a sibling's value too)) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 6921448..2dccad8 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -1279,9 +1279,9 @@ def _object_ruling(m: re.Match, rules: tuple) -> tuple[Any, bool]: """``ruling`` of the object a match wrote, and whether nothing inside it may be kept when nothing matched it (``_floors``). A label rule's error reads as a label -- not kept, not looked into -- so finding kept spans - never raises, and the text is masked as without a keep rule. A keep - match written otherwise than it parses (``_written_as_parsed``) is read - as one nothing matches.""" + never raises, and the text is masked as without a keep rule. An object + written otherwise than it parses (``_written_as_parsed``) is not looked + into, and a keep match written so is read as one nothing matches.""" try: value, as_written = _object_as_read(m) found = ruling(value, rules) From 2594c7173ddf97f6beb0b6e0b5729c1059c61987 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:11:34 +0300 Subject: [PATCH 07/22] fix(masking): text that names a CVV or SAD key anywhere keeps nothing The text floor read only the object right before a key, or the objects the text rules read whole, three levels deep. A wallet token's own nesting takes two, so `{"cvv": [ ]}`, `{"name": "cvv", "value": }` and a token deeper under `securityCode` were kept for every real Ottu token, where the key walk keeps nothing (review I1). `kept_spans` now keeps nothing in text that names a CVV or SAD key, XML tag or `{name, value}` pair label anywhere (`floored_text`): a quoted key then `:`, `=` or `=>`, a bare key then `:` or `=>`, a key and `=` where a key starts (not base64 padding inside a string), an element's tag, or a pair's label, read with `classify_key(..., ALL_PACKS)` and the service's safe keys, as written and entity-decoded. It is read only once a span is found. `redact_body` keeps nothing in an entity-encoded element of a body that names one. It fails closed: a value beside such a key is not kept in text, though the key walk keeps it. The per-object checks this replaces (`_floored_before`, `_KEY_BEFORE`, `_floors`) add nothing the scan does not, and are gone; an object that does not parse or names a key twice is read member by member again, as in 0.17.0. Ottu's KPay body, Connect's SDK Apple Pay request and a Google Pay PaymentData body still keep their tokens. Co-Authored-By: Claude Opus 5.5 --- README.md | 38 ++++----- ecsctx/CLAUDE.md | 2 +- ecsctx/contrib/net/__init__.py | 42 ++++++---- ecsctx/masking/patterns.py | 125 ++++++++++++++++------------- tests/test_contrib_ottu_wallets.py | 107 +++++++++++++++++++++++- tests/test_masking_keep_rules.py | 22 ++--- 6 files changed, 233 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index ed54a75..23276b6 100644 --- a/README.md +++ b/README.md @@ -1457,9 +1457,8 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: - **The floor.** Nothing is kept under a CVV or SAD key (`cvv`, `securityCode`, `pin`, `cryptogram`, …), inside a CVV or SAD container (a key the service lists there included), or as the `value` of a pair - labelled as one; in text, right after such a key (quoted or not, then `:`, - `=` or `=>`) or XML start tag, and anywhere in an object holding such a key - or labelled as one. Such a value is masked exactly as without the rule. + labelled as one; in text, anywhere in text that names such a key, element + or pair label. Such a value is masked exactly as without the rule. - **The guard.** A match that holds, at any depth — JSON text inside it included — a key the key rules read as a card, a CVV or other SAD, a `{name, value}` pair whose identifier reads as one (as the key walk reads a @@ -1506,21 +1505,24 @@ value set aside and puts back each placeholder it leaves. Both are in `ecsctx.masking` and never raise of their own: `mask_outside_kept` passes on an error the caller's `mask` raises. -In text the floor reads the key right before an object (`"cvv": {…}`, -`cvv={…}`, `'cvv' => {…}`, `{…}`), and every object the text -rules read whole: one with a CVV or SAD key at its top level, or a -`{name, value}` pair labelled as one, is not looked into for a value to keep, -whatever stands between the key and the value (`{"cvv": [{…}]}`). That fails -closed: a value under another key of such an object -(`{"cvv": "123", "token": {…}}`) is not kept in text, though the key walk -keeps it, and nothing is kept in an object read whole that does not parse or -names a key twice. A value beside such an object, not in it, is kept. The -text rules read three levels of nesting, so a kept value two keys below a CVV -key (`{"cvv": {"x": {"y": {…}}}}`, the value itself nested) is not floored -there — the guard still holds. An object right after a CVV key in text is -not kept, but no `default` content rule masks a JSON object there either, -with or without keep rules; the key walk masks one under a CVV key leaf by -leaf. +In text a container's reach cannot be told — the text rules read an object +three levels deep, and a wallet token's own nesting takes two — so the floor +reads the whole text: text that names a CVV or SAD key anywhere keeps +nothing, and is masked as without a keep rule. A name is a quoted key then +`:`, `=` or `=>` (JSON, a repr, JSON in a JSON string), a bare key then `:` or +`=>`, a key and `=` where a key starts (`cvv=`, `&cvv=`; not base64's `=` +padding inside a string), an XML tag, or a `{name, value}` pair's label +(`"name": "card security code"`), read as the key walk reads a key — every +pack on, the service's safe keys honoured — in the text as written and, where +it holds an entity, as it decodes. A word in prose (`the cvv check passed`) +names no key. That fails closed: a value beside such a key, not under it +(`{"cvv": "123", "token": {…}}`, or a `` element elsewhere in a body), +is not kept in text, though the key walk keeps it. `redact_body` reads an +element's entity-encoded text the same way, and keeps nothing in it when the +body around it names one. The text is read for names only once a value to +keep is found in it. An object right after a CVV key in text is not kept, but +no `default` content rule masks a JSON object there either, with or without +keep rules; the key walk masks one under a CVV key leaf by leaf. The cost: with no keep rule configured none of this runs — a string with no `{` pays one check, one with a `{` a look at the rules in force; with keep diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index ab48a9b..4978f21 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a label rule wins at any depth strictly inside a keep match, wherever it is listed (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span, a CVV/SAD key right before an object -- quoted or bare, then `:`, `=` or `=>`, or an XML start tag -- floors it and all inside it, and an object read whole is not looked into when it holds a CVV/SAD key at its top level, is a pair labelled as one (`value_rules.pair_labelled`, the guard's reading), or does not parse or names a key twice (`_floors`; fails closed, a sibling's value too)) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a label rule wins at any depth strictly inside a keep match, wherever it is listed (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/contrib/net/__init__.py b/ecsctx/contrib/net/__init__.py index e866e9f..811b0e9 100644 --- a/ecsctx/contrib/net/__init__.py +++ b/ecsctx/contrib/net/__init__.py @@ -44,6 +44,7 @@ _end_tags, _mask_card_field, _mask_userinfo, + floored_text, holds_pan_run, mask_card_elements, mask_objects, @@ -425,16 +426,19 @@ def redact_body(text: str) -> str: return _redact_body(text)[0] -def _redact_body(text: str) -> tuple[str, bool]: - """``redact_body``'s result, and whether it set a kept value aside.""" +def _redact_body(text: str, keep: bool = True) -> tuple[str, bool]: + """``redact_body``'s result, and whether it set a kept value aside -- + never one with ``keep`` False: the text holding this text names a CVV + or SAD key (``patterns.floored_text``).""" stash = None if "{" in text and (rules := value_rules_in_force()): text = mask_objects(text, rules) - stash = stash_kept(text, rules) - if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text: - # None in the text as written; an element's entity-encoded text - # may hold one (_encoded_leaf). - stash = KeptStash(text, {}) + if keep: + stash = stash_kept(text, rules) + if stash is None and "&" in text and keep_rules(rules) and _KEPT_HEAD not in text: + # None in the text as written; an element's entity-encoded + # text may hold one (_encoded_leaf). + stash = KeptStash(text, {}) if stash is None: return _redact_body_text(text, None), False return stash.restore(_redact_body_text(stash.text, stash)), bool(stash.originals) @@ -479,10 +483,14 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: other leaf, through the body rules as it decodes -- set aside in ``stash`` when it holds a kept value, so the form and JSON rules that read the body next never read into it (base64's `==` after a run holding - `cvv` reads as a CVV form field).""" + `cvv` reads as a CVV form field). No leaf keeps a value without a + ``stash``, nor in text that names a CVV or SAD key, element or pair + label (``floored_text``, read once a leaf keeps one).""" text = mask_card_elements(text) names = _get_compiled()[3] parts, copied, ends = [], 0, None + # Whether a leaf may keep a value: None until one does. + keep: bool | None = None if stash is not None else False for start in _XML_START.finditer(text): if start.start() < copied: continue # inside an element already masked whole @@ -494,11 +502,13 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: continue end = closing.start() if not _is_credential_element(start.group("local"), names): - if (leaf := _encoded_leaf(text, start.end(), end)) is not None: - masked, kept = leaf - if kept and stash is not None: - masked = stash.hold(masked) - parts += [text[copied : start.end()], masked] + leaf = _encoded_leaf(text, start.end(), end, keep is not False) + if leaf is not None and leaf[1]: + if keep is None: + keep = not floored_text(text) + leaf = (stash.hold(leaf[0]), True) if keep else _encoded_leaf(text, start.end(), end, False) + if leaf is not None: + parts += [text[copied : start.end()], leaf[0]] copied = end continue value = _element_text(text[start.end() : end]) @@ -512,7 +522,7 @@ def _mask_xml_elements(text: str, stash: KeptStash | None = None) -> str: return "".join(parts) -def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None: +def _encoded_leaf(text: str, start: int, end: int, keep: bool = True) -> tuple[str, bool] | None: """The text of a leaf, ``text[start:end]``, masked by the body rules as it decodes and escaped again -- when it is JSON or XML written with entities, and masking changed it or kept a value in it (then as written, @@ -520,13 +530,13 @@ def _encoded_leaf(text: str, start: int, end: int) -> tuple[str, bool] | None: needs nothing here, nor a form body's: the body rules read them where they stand, `&` a form's separator either way. Decoded, a form value ran on past `"`, and a leaf the first pass left as a form body read - differently on the next.""" + differently on the next. Nothing is kept in it with ``keep`` False.""" if text.find("<", start, end) != -1 or text.find("&", start, end) == -1: return None decoded = html.unescape(text[start:end]) if decoded.lstrip()[:1] not in ("{", "[", "<"): return None - masked, kept = _redact_body(decoded) + masked, kept = _redact_body(decoded, keep) if masked == decoded: return (text[start:end], True) if kept else None return xml_escape(masked), kept diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 2dccad8..ca308ae 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -64,7 +64,6 @@ keep_refused, keep_rules, label_rules, - pair_labelled, ruling, text_ruling, ) @@ -1207,29 +1206,58 @@ def mask_objects(text: str, rules: tuple) -> str: # or in such a container (filters): a CVV and the rest of Sensitive # Authentication Data. _FLOOR_TYPES = frozenset({"cvv", "sad"}) -# The key right before an object in text, as the floor reads it: a quoted key -# and its colon (JSON, a repr, JSON in a JSON string, its quotes escaped) or -# `=` or `=>`, a key and `:`, `=` or `=>`, or an XML start tag -- then any -# space and the opening quote of a string the object is the JSON text of. -_KEY_BEFORE = re.compile( - r"(?:(?P\\?[\"'])(?P[^\"'\\]{1,128})(?P=quote)\s*(?::|=>?)" - r"|(?[\w.-]{1,128})\s*(?::|=>?)" - r"|<(?:[A-Za-z_][\w.-]*:)?(?P[A-Za-z_][\w.-]*)(?:\s[^<>]*)?(?)" - r"\s*(?:\\?[\"'])?\s*\Z" -) -# How far back the key is looked for. -_KEY_REACH = 192 +# What text names, wherever it is, as the floor reads it: a quoted key (JSON, +# a repr, JSON in a JSON string, its quotes escaped) then `:`, `=` or `=>`, +# looked for at every quote so that a quote inside a string never hides the +# key after it; a bare key, or the word closing a quoted one, then `:` or +# `=>`; a key and `=` where a key starts (`cvv=`, `&cvv=`, `a=1, cvv=`) -- +# not inside a quoted string, where base64's `=` padding follows a word +# (`"…/cvvXYZ=="`); and an XML tag, start or end. +_QUOTED_NAME = re.compile(r"(?=(\\*[\"'])([^\"'\\]{1,128})\1\s*(?::|=>?))") +_BARE_NAME = re.compile(r"(?)") +_ASSIGNED_NAME = re.compile(r"(?:^|(?<=[\s&?,({\[]))([A-Za-z_][\w.-]{0,127})\s*=") +_TAG_NAME = re.compile(r"])") + + +@lru_cache(maxsize=1) +def _pair_label() -> re.Pattern: + """A `{name, value}` pair's label as text writes it: an identifier key + the key walk reads a pair by (``filters._PAIR_IDENTIFIERS``, imported + here: filters imports this module as it loads) and its quoted value.""" + from ecsctx.masking.filters import _PAIR_IDENTIFIERS + + identifiers = "|".join(map(re.escape, _PAIR_IDENTIFIERS)) + return re.compile( + rf"(?=(\\*[\"'])(?i:{identifiers})\1\s*(?::|=>?)\s*(\\*[\"'])([^\"'\\]{{1,128}})\2)" + ) -def _floored_before(text: str, start: int) -> bool: - """Whether the object at ``start`` sits right after a CVV or SAD key -- - `"cvv": {…}`, `cvv={…}`, `{…}`, `"pin": "{…}"`: nothing under - one is kept, however deep. Every pack on, as the key walk reads a key.""" - found = _KEY_BEFORE.search(text, max(0, start - _KEY_REACH), start) - if found is None: - return False - name = found.group("quoted") or found.group("bare") or found.group("tag") - return classify_key(name, ALL_PACKS) in _FLOOR_TYPES +def _safe_keys_in_force() -> frozenset[str]: + """The service's safe keys (``ECSCTX_MASK_SAFE_KEYS``), as the key walk + reads them. Imported here: config imports this module as it loads.""" + from ecsctx.masking.config import get_masking_safe_keys + + return get_masking_safe_keys() + + +def floored_text(text: str) -> bool: + """Whether ``text`` names, anywhere, a CVV or SAD key, an XML tag named + so, or a `{name, value}` pair labelled so -- read as the key walk reads a + key (every pack on, the service's safe keys honoured), in the text as + written and, where it holds an entity, as it decodes. Text that does + keeps nothing: a container's reach cannot be told from text the rules + read three levels deep, so the whole text fails closed.""" + safe = _safe_keys_in_force() + for written in (text, html.unescape(text)) if "&" in text else (text,): + for pattern, group in ((_QUOTED_NAME, 2), (_BARE_NAME, 1), (_ASSIGNED_NAME, 1), (_TAG_NAME, 1)): + for m in pattern.finditer(written): + if classify_key(m.group(group), ALL_PACKS, safe) in _FLOOR_TYPES: + return True + # Uncached, as the key walk classifies a pair's label: free text. + for m in _pair_label().finditer(written): + if classify_key.__wrapped__(m.group(3), ALL_PACKS, safe) in _FLOOR_TYPES: + return True + return False def _written_as_parsed(m: re.Match) -> bool: @@ -1275,60 +1303,43 @@ def _object_as_read(m: re.Match) -> tuple[Any, bool]: return None, True -def _object_ruling(m: re.Match, rules: tuple) -> tuple[Any, bool]: - """``ruling`` of the object a match wrote, and whether nothing inside it - may be kept when nothing matched it (``_floors``). A label rule's error - reads as a label -- not kept, not looked into -- so finding kept spans - never raises, and the text is masked as without a keep rule. An object - written otherwise than it parses (``_written_as_parsed``) is not looked - into, and a keep match written so is read as one nothing matches.""" +def _object_ruling(m: re.Match, rules: tuple) -> Any: + """``ruling`` of the object a match wrote; a label rule's error reads as a + label -- not kept, not looked into -- so finding kept spans never raises, + and the text is masked as without a keep rule. A keep match written + otherwise than it parses (``_written_as_parsed``) is read as one nothing + matches.""" try: value, as_written = _object_as_read(m) found = ruling(value, rules) except Exception: # noqa: BLE001 -- a service's matcher; rule 2 reports it where it runs - return "", True - if found is KEEP: - return (KEEP, False) if as_written else (None, True) - return found, found is None and (not as_written or _floors(value)) - - -def _floors(value: Any) -> bool: - """Whether nothing in an object, as parsed, may be kept: one with a CVV - or SAD key at its top level, whatever stands between that key and a - value (`{"cvv": [{…}]}`), or a `{name, value}` pair labelled as one, as - the key walk keeps nothing in such a container or as such a pair's - value. Fails closed: a value under a sibling key is not kept either, nor - anything in an object that does not parse.""" - if not isinstance(value, dict): - return True - return pair_labelled(value, _FLOOR_TYPES) or any( - classify_key(str(key), ALL_PACKS) in _FLOOR_TYPES for key in value - ) + return "" + return None if found is KEEP and not as_written else found def kept_spans(text: str, rules: tuple) -> list[tuple[int, int]]: """Where ``text`` writes a value a keep rule among ``rules`` matches first and may keep: each JSON object (rule 2's reading), top down. An object a - label rule matches first is no span and is not looked into, nor is one - right after a CVV or SAD key, nor one holding a CVV or SAD key or - labelled as one (``_floors``); one nothing matches is read member by - member; a keep match refused (``keep_refused``: the guard, or a label - rule matching inside it) is read as one nothing matches. In order, none - inside another.""" + label rule matches first is no span and is not looked into; one nothing + matches is read member by member; a keep match refused (``keep_refused``: + the guard, or a label rule matching inside it) is read as one nothing + matches. In order, none inside another. None at all in text that names a + CVV or SAD key, element or pair label anywhere (``floored_text``, read + only once a span is found).""" spans: list[tuple[int, int]] = [] _find_kept(text, 0, len(text), rules, spans) - return spans + return [] if spans and floored_text(text) else spans def _find_kept(text: str, start: int, end: int, rules: tuple, spans: list[tuple[int, int]]) -> None: for m in _OBJECT_RULE.pattern.finditer(text, start, end): asked = applicable(m.group(0), rules) - if not asked or _floored_before(text, m.start()): + if not asked: continue - found, floored = _object_ruling(m, asked) + found = _object_ruling(m, asked) if found is KEEP: spans.append(m.span()) - elif found is None and not floored: + elif found is None: _find_kept(text, m.start() + 1, m.end() - 1, rules, spans) diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index da18a7d..53725d9 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -18,6 +18,7 @@ pack and with every pack. """ +import html import json import logging @@ -33,8 +34,13 @@ WALLET_RULES, WALLET_SAD_RULES, ) -from ecsctx.masking import is_kept, patterns -from ecsctx.masking.config import configure_masking_packs, configure_masking_value_rules +from ecsctx.contrib.ottu.masking import SAFE_KEYS as OTTU_SAFE_KEYS +from ecsctx.masking import is_kept, mask_outside_kept, patterns +from ecsctx.masking.config import ( + configure_masking_packs, + configure_masking_safe_keys, + configure_masking_value_rules, +) from ecsctx.masking.filters import MaskPIIFilter from ecsctx.masking.patterns import ALL_PACKS, mask_by_patterns, rules_for from ecsctx.masking.value_rules import KeepRule, is_keep_rule @@ -468,6 +474,103 @@ def test_saved_card_tokens_stay_masked(self, saved): assert "4111111111111111" not in json.dumps(before) +# A CVV or SAD container in text around a token, or beside it (review I1): +# the key walk keeps nothing in any of them, and text that names a CVV or SAD +# key, element or pair label anywhere keeps nothing. +FLOORED = { + "a-list-between": '{"cvv": [ TOKEN ]}', + "a-pair-labelled-cvv": '{"name": "cvv", "value": TOKEN}', + "a-pair-in-a-list": '[{"name":"cvv","value": TOKEN}]', + "a-pair-labelled-in-words": '{"name": "card security code", "value": TOKEN}', + "a-pair-labelled-pin-as-json-text": '{"Name": "PIN", "Value": "ESCAPED"}', + "quoted-key-equals": '"cvv" = TOKEN', + "quoted-key-arrow": "'cvv' => TOKEN", + "bare-key-arrow": "securityCode => TOKEN", + "deeper-than-the-text-rules-read": '{"securityCode": {"x": {"y": [TOKEN]}}}', + "a-sibling-of-a-cvv-key": '{"cvv": "123", "token": TOKEN}', + "an-element-elsewhere": "1234 TOKEN", + "a-key-written-after-it": 'TOKEN and then "cryptogram": "AAAB"', +} +FLOOR_TOKENS = {"pk-payment-token": PK_PAYMENT_TOKEN, "apple-pay": APPLE_PAY, "google-pay": GOOGLE_PAY} + + +def _floored(template: str, token: dict) -> str: + escaped = json.dumps(json.dumps(token))[1:-1] + return template.replace("ESCAPED", escaped).replace("TOKEN", json.dumps(token)) + + +def _connect_apple_pay_request() -> dict: + """What Connect's SDK sends to pay with Apple Pay.""" + return { + "payment_method": "apple_pay", + "code": "apple-pay-kwd", + "amount": "100.000", + "currency_code": "KWD", + "session_id": "b1e2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0", + "cvv_required": False, + "apple_pay_payload": {"token": {"token": PK_PAYMENT_TOKEN}}, + } + + +class TestTheFloorInText: + @pytest.fixture(params=FLOOR_TOKENS.values(), ids=FLOOR_TOKENS.keys()) + def token(self, request): + return request.param + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_mask_by_patterns(self, template, token, packs): + text = _floored(template, token) + assert patterns.kept_spans(text, WALLET_RULES) == [] + assert _text(text, packs) == _unconfigured(lambda: _text(text, packs)) + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_redact_body(self, template, token): + # Connect's extra secret keys: what is not kept has its signature masked. + configure_redaction(extra_secret_keys=["signature", "hash"]) + text = _floored(template, token) + for body in ( + text, + f"S3cretPassw0rd{text}", + f"S3cretPassw0rd{html.escape(text)}", + ): + assert redact_body(body) == _unconfigured(lambda body=body: redact_body(body)) + + @pytest.mark.parametrize("template", FLOORED.values(), ids=FLOORED.keys()) + def test_mask_outside_kept(self, template, token): + text = _floored(template, token) + assert mask_outside_kept(text, str.upper) == text.upper() + + +class TestStillKeptInText: + @pytest.fixture(autouse=True) + def ottu_safe_keys(self): + configure_masking_safe_keys(OTTU_SAFE_KEYS) + + def test_a_kpay_body(self, packs): + written = json.dumps(PK_PAYMENT_TOKEN) + body = KPAY_BODY.format(written) + for masked in (redact_body(body), _text(body, packs), redact_body(KPAY_BODY.format(html.escape(written)))): + assert "S3cretPassw0rd" not in masked + assert f"{written}" in redact_body(body) + assert f"{written}" in _text(body, packs) + assert f"{html.escape(written)}" in redact_body(KPAY_BODY.format(html.escape(written))) + + def test_connects_sdk_apple_pay_request(self, packs): + request = json.dumps(_connect_apple_pay_request()) + written = json.dumps(PK_PAYMENT_TOKEN) + assert written in _text(f"pay request {request}", packs) + assert written in redact_body(request) + assert written in mask_outside_kept(request, str.upper) + assert _walk({"payload": _connect_apple_pay_request()}) == {"payload": _connect_apple_pay_request()} + + def test_a_google_pay_payment_data_body(self, packs): + body = json.dumps(GOOGLE_PAYMENT_DATA) + escaped = json.dumps(json.dumps(GOOGLE_PAY))[1:-1] + for masked in (_text(f"google pay {body}", packs), redact_body(body)): + assert escaped in masked + assert "jane.roe@example.com" not in _text(f"google pay {body}", packs) + + class TestTheSadVariant: """0.16.0's assertions, with WALLET_SAD_RULES.""" diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index b8dad3f..376f974 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -469,8 +469,8 @@ def _kept_in(masked: str, kept: str) -> None: # CVV and SAD containers in text that are not the key right before the -# object (review M2). The last four fail closed: an object holding a CVV key, -# or one whose keys or parse cannot be trusted, is not looked into at all. +# object (review M2), and the same keys beside the value (review I1): text +# that names a CVV or SAD key, element or pair label anywhere keeps nothing. FLOORED = { "a-list-between": '{"cvv": [ PLAIN ]}', "a-pair-labelled-cvv": '{"name": "cvv", "value": PLAIN}', @@ -481,12 +481,16 @@ def _kept_in(masked: str, kept: str) -> None: "a-sibling-of-a-cvv-key": '{"cvv": "123", "token": PLAIN}', "a-pair-whose-name-is-written-twice": '{"name": "cvv", "name": "note", "value": PLAIN}', "an-object-that-does-not-parse": '{"cvv": undefined, "token": PLAIN}', -} -# The same keys beside the kept value, not around it. -BESIDE = { "a-cvv-list-beside": '{"payment": {"cvv": ["123"]}, "token": PLAIN}', "a-cvv-pair-beside": '[{"name": "cvv", "value": "123"}, PLAIN]', "a-cvv-key-before": '"cvv" = "123", "token" => PLAIN', + "a-form-field-before": "a=1&cvv=123 PLAIN", + "an-element-after": "PLAIN 1234", +} +# Keys about a CVV, not one: text naming them keeps its value. +NOT_FLOORED = { + "a-flag-about-a-cvv": '{"cvv_required": true, "token": PLAIN}', + "a-cvv-word-in-prose": "the cvv check passed for PLAIN", } @@ -553,8 +557,8 @@ def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, templa assert masked == _unconfigured(lambda: _text(text, packs)) assert SEALED["api_key"] not in masked - @pytest.mark.parametrize("template", BESIDE.values(), ids=BESIDE.keys()) - def test_a_value_beside_a_cvv_container_is_still_kept(self, template, packs): + @pytest.mark.parametrize("template", NOT_FLOORED.values(), ids=NOT_FLOORED.keys()) + def test_text_naming_no_cvv_key_keeps_its_value(self, template, packs): assert json.dumps(SEALED) in _text(_written(template), packs) def test_an_outer_label_match_wins(self, packs): @@ -799,8 +803,8 @@ def test_nothing_in_a_cvv_container_is_kept_whatever_stands_between(self, templa assert patterns.kept_spans(body, KEEP_RULES) == [] assert redact_body(body) == _unconfigured(lambda: redact_body(body)) - @pytest.mark.parametrize("template", BESIDE.values(), ids=BESIDE.keys()) - def test_a_value_beside_a_cvv_container_is_still_kept(self, template): + @pytest.mark.parametrize("template", NOT_FLOORED.values(), ids=NOT_FLOORED.keys()) + def test_text_naming_no_cvv_key_keeps_its_value(self, template): assert json.dumps(SEALED) in redact_body(_written(template)) def test_an_element_naming_a_placeholder_gets_no_copy_of_a_kept_value(self): From 6f360032267ebaeb81b045d9a32f8af0aa39fe4a Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:14:14 +0300 Subject: [PATCH 08/22] fix(masking): the guard refuses a keep match one of whose text leaves holds a card number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard read keys, pairs and whole leaves, so a card number written inside a kept value's string shipped with it, even in a slot's own alphabet: `"data": "4111…+cvv+123"` is base64, `"transactionId": "4111…ab"` hex (review I2). A text leaf that is no JSON text now holds card data when a run of 13 to 19 digits in it, joined by single separators at most and bounded by non-digits, passes Luhn (`patterns.holds_card_run`), epoch milliseconds (thirteen digits from a 1 or a 2) excepted as for a whole leaf. JSON text is read by its own leaves, so Google Pay's `keyExpiration` stays exempt. It is in the core guard, so every keep rule gets it. A CVV or other short value inside a string still ships with it, as the README now says. Co-Authored-By: Claude Opus 5.5 --- README.md | 26 ++++++++++++++---------- ecsctx/masking/patterns.py | 22 ++++++++++++++++++++ ecsctx/masking/value_rules.py | 22 ++++++++++++-------- tests/test_contrib_ottu_wallets.py | 13 +++++++++++- tests/test_masking_keep_rules.py | 32 +++++++++++++++++++++++++++++- 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 23276b6..2de3fb6 100644 --- a/README.md +++ b/README.md @@ -1462,17 +1462,21 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: - **The guard.** A match that holds, at any depth — JSON text inside it included — a key the key rules read as a card, a CVV or other SAD, a `{name, value}` pair whose identifier reads as one (as the key walk reads a - pair), or a leaf that is a card number, is walked as if nothing had - matched. It reads keys, pairs and whole leaves only: a CVV or a card number - written inside a longer string of a kept value (`"note": "cvv=123"`, Track - 2 data, `"card 4111…"`) ships with it. Keeping that out is the matcher's - job, which is why a matcher must be strict about what it matches. A leaf is - a card number when it is an int `int_is_pan` reads as one, or a string - `pan_shaped` reads as one, whatever its prefix and Luhn or not — except - thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's - `keyExpiration`, which `pan_shaped` reads as a card number and every - Google Pay token carries. A leaf that is no JSON value counts as card - data. + pair), a leaf that is a card number, or a text leaf with a card number in + it, is walked as if nothing had matched. A leaf is a card number when it is + an int `int_is_pan` reads as one, or a string `pan_shaped` reads as one, + whatever its prefix and Luhn or not. A text leaf holds one when a run of 13 + to 19 digits in it, joined by single separators at most and with no digit + next to either end, passes Luhn: `"card 4111…"`, Track 2 data, + `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a hex one. Both except + thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google + Pay's `keyExpiration`, which `pan_shaped` reads as a card number and every + Google Pay token carries. A leaf that is no JSON value counts as card data. + What the guard cannot tell from a token's own text ships with it: a CVV or + other short value written inside a longer string (`"note": "cvv=123"`, or + `cvv+123` inside base64, which reads as ciphertext), and a run of digits + that fails Luhn. Keeping that out is the matcher's job, which is why a + matcher must be strict about what it matches. Nothing else in a kept value is read: PII or a credential inside it, under a key of its own or written in a string, ships with it. diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index ca308ae..ee92e92 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -55,6 +55,7 @@ mask_by_field_type, ) from ecsctx.masking.value_rules import ( + _EPOCH_MILLISECONDS, KEEP, DuplicateKey, _first_match, @@ -809,6 +810,27 @@ def pan_shaped(text: str) -> bool: _DIGIT_RUN = re.compile(rf"\+?\d(?:{_CARD_SEP}?\d)*") +# A card number anywhere in a value, as the guard reads one: 13 to 19 digits +# joined by single separators at most, as the card rule joins them, with no +# digit next to either end. +_CARD_RUN_IN_TEXT = re.compile(rf"(? bool: + """Whether ``text`` holds, anywhere in it, a run that is a card number: + 13 to 19 digits that pass Luhn -- `card 4111…`, `4111…+cvv+123`, a hex + id holding one -- except thirteen bare digits from a 1 or a 2, epoch + milliseconds. For the guard (``value_rules.holds_card_data``), which + reads every text leaf of a keep match with it. Stricter than + ``holds_pan_run`` about what a card number is (Luhn, 13 digits up), so + ciphertext and hex ids are rarely refused.""" + for m in _CARD_RUN_IN_TEXT.finditer(text): + run = m.group() + if not _EPOCH_MILLISECONDS.fullmatch(run) and _luhn_valid(re.sub(r"\D", "", run)): + return True + return False + + def holds_pan_run(text: str, *, phone: bool = False) -> bool: """Whether ``text`` holds, anywhere in it, a run of digits long enough to be a PAN. diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index eb6d92b..b2ae044 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -345,17 +345,20 @@ def holds_card_data(value: Any) -> bool: """Whether ``value`` holds, at any depth -- JSON text inside it included -- a key the key rules read as a card, a CVV or other SAD (every pack on), a ``{name, value}`` pair whose identifier reads as one (as the key walk reads - a pair, ``filters._pair``), or a leaf that is a card number as a whole: - what no keep rule may ship, not even one that matches everything. Keys, - pairs and whole leaves only: a CVV or a card number written inside a - longer string (``"note": "cvv=123"``) is not read here, and keeping it out - is a matcher's job. + a pair, ``filters._pair``), a leaf that is a card number as a whole, or a + text leaf holding one: what no keep rule may ship, not even one that + matches everything. A CVV or other short value written inside a longer + string (``"note": "cvv=123"``, ``cvv+123`` in base64) is not read here, + and keeping it out is a matcher's job. A leaf is a card number when it is an int ``int_is_pan`` reads as one, or a string ``pan_shaped`` reads as one -- whatever its prefix, Luhn or not -- except thirteen bare digits from a 1 or a 2: epoch milliseconds, Google Pay's ``keyExpiration`` in ``signedKey``, without which no Google Pay token would - ever be kept. Not ``holds_pan_run``, which flags hex ids. + ever be kept. A text leaf that is no JSON text holds one when + ``holds_card_run`` finds a run of 13 to 19 digits passing Luhn in it, + epoch milliseconds again excepted (JSON text is read by its own leaves). + Not ``holds_pan_run``, which flags hex ids. Anything that is no JSON value (an object whose text is not judged here), nesting past the depth cap, and a leaf of JSON text that names a key twice (``json_as_written``: what ships would not be what was read) count as @@ -365,6 +368,7 @@ def holds_card_data(value: Any) -> bool: from ecsctx.masking.patterns import ( ALL_PACKS, classify_key, + holds_card_run, int_is_pan, pan_shaped, ) @@ -391,13 +395,15 @@ def holds(item: Any, depth: int) -> bool: if pan_shaped(item) and not _EPOCH_MILLISECONDS.fullmatch(item.strip()): return True if item.lstrip()[:1] not in ("{", "["): - return False + return holds_card_run(item) try: parsed = json_as_written(item) except (DuplicateKey, RecursionError): return True except ValueError: - return False + return holds_card_run(item) + # Its own leaves are read: `keyExpiration` in Google Pay's + # `signedKey` is a whole leaf there. return holds(parsed, depth + 1) return True diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 53725d9..5914e84 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -142,6 +142,12 @@ }, "google-signature-cvv": {**GOOGLE_PAY, "signature": "cvv=123"}, "google-v1-encrypted-message-cvv": {**GOOGLE_PAY_V1, "signedMessage": '{"encryptedMessage":"cvv=123"}'}, + # Text in a slot's own alphabet holding a card number (review I2). + "apple-data-pan-and-cvv-in-base64": {**APPLE_PAY, "data": "4111111111111111+cvv+123"}, + "apple-transaction-id-pan-in-hex": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "transactionId": "4111111111111111ab"}, + }, } PACKS = [["default"], sorted(ALL_PACKS)] @@ -319,8 +325,13 @@ def every_path(): {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "0a1B" * 16}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "MasterCard 0492", "network": "MasterCard"}}, {**GOOGLE_PAY, "signature": "MEQCIGZh+2Utc2/n"}, + # Thirteen digits from a 1 or a 2 read as epoch milliseconds, + # wherever they stand. + {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, + ], + ids=[ + "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "apple-data-epoch-run", ], - ids=["apple-application-data", "pk-display-name", "google-signature-plus-and-slash"], ) def test_a_documented_token_is_still_kept(self, token): assert is_kept(token) diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 376f974..bc843d5 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -314,14 +314,44 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): {"note": 4111111111111111}, {"note": json.dumps({"inner": {"securityCode": "123"}})}, {"deep": [{"pin": "1234"}]}, + # A card number inside a longer text leaf (review I2). + {"note": "card 4111111111111111 exp 12/27"}, + {"box": "4111111111111111+cvv+123"}, + {"ref": "4111111111111111ab"}, + {"note": "Visa 4111 1111 1111 1111 on file"}, + {"memo": ";4111111111111111=25121010000000000000?"}, + {"note": "{not json 4111111111111111"}, + ], + ids=[ + "cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list", + "a-pan-in-prose", "a-pan-in-base64", "a-pan-in-hex", "a-grouped-pan-in-prose", "track-2", + "a-pan-in-text-that-is-no-json", ], - ids=["cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list"], ) def test_the_guard_a_match_holding_card_data_is_walked(self, extra): holding = {**SEALED, **extra} + assert not is_kept(holding) _masked_as_a_walk(_walk({"payload": holding})["payload"]) assert _walk({"payload": json.dumps(holding)})["payload"] != json.dumps(holding) + @pytest.mark.parametrize( + "extra", + [ + {"note": "cvv=123"}, + {"box": "QUJD1790318444473QUJD"}, + {"ref": "order 4111111111111112 retried"}, + {"ref": "41111111111111111111"}, + {"note": json.dumps({"keyExpiration": "1790318444473"})}, + ], + ids=["a-cvv-in-a-string", "epoch-milliseconds", "a-run-failing-luhn", "twenty-digits", "json-text-epoch"], + ) + def test_what_the_guard_does_not_read_in_a_text_leaf(self, extra): + # A CVV or other short value inside a longer string is the matcher's + # to keep out; a run that is no card number is no card number. + holding = {**SEALED, **extra} + assert is_kept(holding) + assert _walk({"payload": holding}) == {"payload": holding} + def test_a_rule_that_keeps_everything_still_cannot_ship_a_card(self): configure_masking_value_rules([KeepRule(lambda value: True)]) masked = _walk({"payment": {"card": {"number": "4111111111111111", "cvv": "123", "expiry": "12/27"}}}) From 9b69a948003cf7a8e28e02b26370dd82582dd8f8 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:15:13 +0300 Subject: [PATCH 09/22] fix(contrib): a PKPaymentToken's displayName is the network's name and the last four `displayName` was any text of up to 64 characters, so `"Visa 1234 cvv 123"` shipped the CVV in a kept token (review I2). It is now at most 40 characters with no digit but one trailing group of exactly four after a space (`"Visa 0492"`, `"MasterCard 1234"`, `"Amex"`); anything else is no wallet's display name, and the token is not kept whole (its `paymentData` still is). The README says what the slot checks and the guard catch and what they cannot: a CVV written in a slot's own alphabet. Co-Authored-By: Claude Opus 5.5 --- README.md | 24 +++++++++++++++--------- ecsctx/contrib/ottu/masking.py | 26 +++++++++++++++++++------- tests/test_contrib_ottu_wallets.py | 23 ++++++++++++++++++++++- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2de3fb6..0fe722a 100644 --- a/README.md +++ b/README.md @@ -1556,15 +1556,21 @@ signature or ciphertext (`data`, `signature`, the header's `signatures`, and the fields of its `signedMessage` and `signedKey`, each JSON text of its own fields and nothing else), hex for an id or a hash (`transactionId`, `applicationData`, `transactionIdentifier`), digits for -`keyExpiration`, and short text (64 characters at most) for what -`paymentMethod` says about the card. A token holding anything else in a -slot — `"data": "cvv=123"`, a spaced card number as its `signature` — is -not kept, and is masked as before: the guard reads keys, pairs and whole -leaves, so text riding in a kept slot would ship. They are found under -`paymentData`, TAP's `token_data`, MPGS's `paymentToken` (a JSON string), -Google Pay's `tokenizationData.token`, KPay's ``, or in a message. A -`PKPaymentToken` with a key of its own keeps only its `paymentData`. Saved-card tokens stay masked: no wallet shape -matches one. Samsung Pay's token, a JWE, is not matched yet. An Ottu service +`keyExpiration`, and for what `paymentMethod` says about the card short +text: its `network` and `type` at most 64 characters, its `displayName` at +most 40, the network's name and, after a space, the card's last four +(`Visa 0492`, `Amex`) with no other digit. A token holding anything else in a +slot — `"data": "cvv=123"`, a spaced card number as its `signature`, +`"displayName": "cvv 123"` — is not kept, and is masked as before; nor is +one with a card number in a slot's own alphabet (`"4111…+cvv+123"` in +base64), which the guard reads. What neither can tell from ciphertext ships +with a kept token: a CVV or other short value written in a slot's own +alphabet (`cvv+123` inside base64). They are found under `paymentData`, +TAP's `token_data`, MPGS's `paymentToken` (a JSON string), Google Pay's +`tokenizationData.token`, KPay's ``, or in a message. A +`PKPaymentToken` with a key of its own keeps only its `paymentData`. +Saved-card tokens stay masked: no wallet shape matches one. Samsung Pay's +token, a JWE, is not matched yet. An Ottu service lists them in `ECSCTX_MASK_VALUE_RULES` next to its `ECSCTX_MASK_SAFE_KEYS` (above), or sets `ECSCTX_MASK_VALUE_RULES=ecsctx.contrib.ottu.masking.WALLET_RULES`. diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index ba53241..992a5cf 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -35,9 +35,11 @@ Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method token, no key besides the ones they write, and every leaf text in its alphabet (base64, hex or digits; short text for what ``paymentMethod`` - says about the card) -- so anything else is walked as before: a - ``PKPaymentToken`` with a key of its own keeps only its ``paymentData``, - and a token with ``"data": "cvv=123"`` is masked. + says about the card, its display name the network's and the last four) + -- so anything else is walked as before: a ``PKPaymentToken`` with a key + of its own keeps only its ``paymentData``, and a token with + ``"data": "cvv=123"`` is masked. A CVV written in a slot's own alphabet + (``cvv+123`` inside base64) cannot be told from ciphertext, and ships. - Card data. ecsctx keeps nothing under a CVV or SAD key, and nothing that holds a card, CVV or SAD key or a card number (the floor and the guard). @@ -166,9 +168,13 @@ def _is_google_pay_token(value: Any) -> bool: "transactionId": _HEX, "applicationData": _HEX, } -# PKPaymentToken's `paymentMethod` is what the device says about the card, -# free text ("Visa 0492"): text, and short. +# PKPaymentToken's `paymentMethod` is what the device says about the card: +# its network and type, short text, and a display name that is the network's +# name and, after a space, the card's last four ("Visa 0492", "Amex") -- no +# other digit, so neither a card number nor a CVV can ride in it. _PAYMENT_METHOD_TEXT_LIMIT = 64 +_DISPLAY_NAME = re.compile(r"\D*(?: [0-9]{4})?") +_DISPLAY_NAME_LIMIT = 40 def _written_in(field: Any, alphabet: re.Pattern) -> bool: @@ -199,10 +205,15 @@ def _is_apple_pay_payment_data(value: Any) -> bool: ) +def _is_display_name(name: Any) -> bool: + return isinstance(name, str) and len(name) <= _DISPLAY_NAME_LIMIT and _DISPLAY_NAME.fullmatch(name) is not None + + def _is_pk_payment_token(value: Any) -> bool: """Apple Pay's PKPaymentToken: its `paymentData`, what the device says - about the card -- its display name, network and type, each short text -- - and the transaction's id in hex.""" + about the card -- its network and type, each short text, and its display + name, the network's and the last four -- and the transaction's id in + hex.""" payment_data = value.get("paymentData") if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): return False @@ -212,6 +223,7 @@ def _is_pk_payment_token(value: Any) -> bool: return ( isinstance(method, dict) and method.keys() <= _PAYMENT_METHOD_KEYS + and ("displayName" not in method or _is_display_name(method["displayName"])) and all(isinstance(field, str) and len(field) <= _PAYMENT_METHOD_TEXT_LIMIT for field in method.values()) ) diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 5914e84..96777e8 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -149,6 +149,15 @@ "header": {**APPLE_PAY["header"], "transactionId": "4111111111111111ab"}, }, } +# What the device says about the card: the network's name and the last four. +NOT_A_DISPLAY_NAME = { + "a-card-number": "Visa 4111111111111111", + "a-cvv": "cvv 123", + "a-cvv-after-the-last-four": "Visa 1234 cvv 123", + "five-digits": "Visa 12345", + "the-last-four-first": "1234 Visa", + "too-long": "V" * 36 + " 1234", +} PACKS = [["default"], sorted(ALL_PACKS)] @@ -305,6 +314,14 @@ def test_each_token_is_kept(self, token): def test_the_matchers_are_strict(self, value): assert not is_kept(value) + @pytest.mark.parametrize("name", NOT_A_DISPLAY_NAME.values(), ids=NOT_A_DISPLAY_NAME.keys()) + def test_a_display_name_is_a_network_and_the_last_four(self, name): + token = {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "displayName": name}} + assert not is_kept(token) + assert not is_kept(json.dumps(token)) + # Its payment data is still kept, on its own. + assert _walk({"token": token})["token"]["paymentData"] == APPLE_PAY + @pytest.mark.parametrize("token", NOT_TOKEN_TEXT.values(), ids=NOT_TOKEN_TEXT.keys()) def test_a_token_with_text_no_wallet_writes_is_masked_as_without_the_rules(self, token, packs): def every_path(): @@ -325,12 +342,16 @@ def every_path(): {**APPLE_PAY, "header": {**APPLE_PAY["header"], "applicationData": "0a1B" * 16}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "MasterCard 0492", "network": "MasterCard"}}, {**GOOGLE_PAY, "signature": "MEQCIGZh+2Utc2/n"}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Amex", "network": "AmEx", "type": "credit"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Apple Pay", "network": "Visa", "type": "debit"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 35 + " 1234"}}, # Thirteen digits from a 1 or a 2 read as epoch milliseconds, # wherever they stand. {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, ], ids=[ - "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "apple-data-epoch-run", + "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", + "pk-display-name-connects-fixture", "pk-display-name-forty-characters", "apple-data-epoch-run", ], ) def test_a_documented_token_is_still_kept(self, token): From 56f25281e9235ee11df2fe13dd2ce7cbbf37430a Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:16:14 +0300 Subject: [PATCH 10/22] fix(contrib): a Google Pay token's signed message and signed key hold every field `signedMessage` and `signedKey` were read as "at most" their fields, so `"signedMessage": "{}"` was kept (review M7). `signedMessage` must now hold `encryptedMessage`, `ephemeralPublicKey` and `tag`, and `signedKey` `keyValue` and `keyExpiration`, and nothing else. The `GOOGLE_PAY_V1` fixture held only `encryptedMessage`, which no ECv1 token does: it now holds all three. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- ecsctx/contrib/ottu/masking.py | 10 ++++----- tests/test_contrib_ottu_wallets.py | 35 ++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0fe722a..ea39205 100644 --- a/README.md +++ b/README.md @@ -1554,7 +1554,7 @@ writes, and every leaf text in its alphabet: standard base64 for a key, a signature or ciphertext (`data`, `signature`, the header's `ephemeralPublicKey`, `publicKeyHash` and `wrappedKey`, Google Pay's `signatures`, and the fields of its `signedMessage` and `signedKey`, each -JSON text of its own fields and nothing else), hex for an id or a hash +JSON text of all its fields and nothing else), hex for an id or a hash (`transactionId`, `applicationData`, `transactionIdentifier`), digits for `keyExpiration`, and for what `paymentMethod` says about the card short text: its `network` and `type` at most 64 characters, its `displayName` at diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index 992a5cf..2fda17e 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -230,7 +230,7 @@ def _is_pk_payment_token(value: Any) -> bool: def _signed_object(text: Any, keys: frozenset[str]) -> dict | None: """JSON text a Google Pay token signs, read as the keep decision reads - JSON (a key written twice refused): an object of ``keys`` at most, or + JSON (a key written twice refused): an object of exactly ``keys``, or None.""" if not isinstance(text, str): return None @@ -238,21 +238,21 @@ def _signed_object(text: Any, keys: frozenset[str]) -> dict | None: parsed = json_as_written(text) except (ValueError, RecursionError): return None - return parsed if isinstance(parsed, dict) and parsed.keys() <= keys else None + return parsed if isinstance(parsed, dict) and parsed.keys() == keys else None def _is_signed_message(text: Any) -> bool: """`signedMessage`: JSON text of the encrypted message, its ephemeral - public key and tag, each in base64.""" + public key and tag, each in base64, and nothing else.""" message = _signed_object(text, _SIGNED_MESSAGE_KEYS) return message is not None and all(_written_in(field, _BASE64) for field in message.values()) def _is_signed_key(text: Any) -> bool: """`signedKey`: JSON text of the intermediate signing key in base64 and - its expiration, epoch milliseconds.""" + its expiration, epoch milliseconds, and nothing else.""" key = _signed_object(text, _SIGNED_KEY_KEYS) - return key is not None and _absent_or_in(key, "keyValue", _BASE64) and _absent_or_in(key, "keyExpiration", _DIGITS) + return key is not None and _written_in(key["keyValue"], _BASE64) and _written_in(key["keyExpiration"], _DIGITS) def _is_google_pay_payment_token(value: Any) -> bool: diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 96777e8..e8b9a83 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -79,7 +79,11 @@ "protocolVersion": "ECv2", "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "ephemeralPublicKey": "ZXBr", "tag": "dGFn"}), } -GOOGLE_PAY_V1 = {"signature": "MEQCIA==", "protocolVersion": "ECv1", "signedMessage": '{"encryptedMessage":"ZW5j"}'} +GOOGLE_PAY_V1 = { + "signature": "MEQCIA==", + "protocolVersion": "ECv1", + "signedMessage": '{"encryptedMessage":"ZW5j","ephemeralPublicKey":"ZXBr","tag":"dGFn"}', +} NOT_A_WALLET = {"amount": "10.000", "currency": "KWD"} TOKENS = [APPLE_PAY, APPLE_PAY_RSA, PK_PAYMENT_TOKEN, GOOGLE_PAY, GOOGLE_PAY_V1] TOKEN_IDS = ["apple-pay", "apple-pay-rsa", "pk-payment-token", "google-pay", "google-pay-v1"] @@ -141,7 +145,10 @@ "signedMessage": json.dumps({**json.loads(GOOGLE_PAY["signedMessage"]), "note": "cvv=123"}), }, "google-signature-cvv": {**GOOGLE_PAY, "signature": "cvv=123"}, - "google-v1-encrypted-message-cvv": {**GOOGLE_PAY_V1, "signedMessage": '{"encryptedMessage":"cvv=123"}'}, + "google-v1-encrypted-message-cvv": { + **GOOGLE_PAY_V1, + "signedMessage": '{"encryptedMessage":"cvv=123","ephemeralPublicKey":"ZXBr","tag":"dGFn"}', + }, # Text in a slot's own alphabet holding a card number (review I2). "apple-data-pan-and-cvv-in-base64": {**APPLE_PAY, "data": "4111111111111111+cvv+123"}, "apple-transaction-id-pan-in-hex": { @@ -293,6 +300,27 @@ def test_each_token_is_kept(self, token): }, }, {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signedKey": "a2V5"}}, + # Each signed object holds every field it is documented to hold + # (review M7). + {**GOOGLE_PAY, "signedMessage": "{}"}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "ephemeralPublicKey": "ZXBr"})}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"encryptedMessage": "ZW5j", "tag": "dGFn"})}, + {**GOOGLE_PAY, "signedMessage": json.dumps({"ephemeralPublicKey": "ZXBr", "tag": "dGFn"})}, + {**GOOGLE_PAY, "intermediateSigningKey": {**GOOGLE_PAY["intermediateSigningKey"], "signedKey": "{}"}}, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyValue": "a2V5"}), + }, + }, + { + **GOOGLE_PAY, + "intermediateSigningKey": { + **GOOGLE_PAY["intermediateSigningKey"], + "signedKey": json.dumps({"keyExpiration": "1700000000000"}), + }, + }, ], ids=[ "apple-extra-key", "apple-version", "apple-data-not-text", "apple-header-extra-key", @@ -309,6 +337,9 @@ def test_each_token_is_kept(self, token): "google-signed-message-a-list", "google-signed-message-duplicate-key", "google-signed-message-a-mapping-inside", "google-signatures-cvv", "google-signed-key-extra-key", "google-key-value-pin", "google-key-expiration-not-digits", "google-signed-key-not-json", + "google-signed-message-empty", "google-signed-message-no-tag", "google-signed-message-no-ephemeral-key", + "google-signed-message-no-encrypted-message", "google-signed-key-empty", "google-signed-key-no-expiration", + "google-signed-key-no-key-value", ], ) def test_the_matchers_are_strict(self, value): From 4e7ac96a7607cd12693ec0ac77fc708870d3e189 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:17:22 +0300 Subject: [PATCH 11/22] fix(masking): a label rule that raises inside a keep match refuses the keep, raising nothing `keep_refused` asks the label rules about a keep match's members, which 0.17.0 never asked: a label matcher raising there made `redact_body` and `mask_by_patterns` raise where 0.17.0 returned the value kept (review M5). Its error now refuses the keep (fails closed): the value is masked as with the label rules alone, and nothing is raised from that question. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 ++++- ecsctx/masking/value_rules.py | 14 ++++++++++---- tests/test_masking_keep_rules.py | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ea39205..bbb0417 100644 --- a/README.md +++ b/README.md @@ -1450,7 +1450,10 @@ without the value (an element's entity-encoded text decodes to one) is not a placeholder and restores nothing. Text that already holds `[KEPT-` is masked as if no keep rule were configured. A keep rule that raises counts as no match: keep rules are asked on paths that must never raise (`redact_url`, -`redact_body`, `mask_card_value`). +`redact_body`, `mask_card_value`). So does a label rule asked about what is +inside a keep match: its error refuses the keep and raises nothing there, +and the value is masked as with the label rules alone — whose error reaches +the caller only where the walk itself asks them, as before. Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index b2ae044..12cdcaf 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -415,12 +415,18 @@ def keep_refused(value: dict, rules: Iterable[Any]) -> bool: refuses it (``holds_card_data``), or a label rule among ``rules`` labels something strictly inside it (``label_within``, over its members): a label rule wins at any depth inside a keep match. For the value itself - the first match still wins. Only with a label rule among ``rules``.""" + the first match still wins. Only with a label rule among ``rules``; one + that raises there refuses the keep, and raises nothing: keep rules are + asked on paths that must never raise, and a member is asked here that + the walk might never ask.""" if holds_card_data(value): return True - return bool(_split(rules).label) and any( - label_within(member, rules, 1) is not None for member in value.values() - ) + if not _split(rules).label: + return False + try: + return any(label_within(member, rules, 1) is not None for member in value.values()) + except Exception: # noqa: BLE001 -- a service's label matcher; refusing the keep fails closed + return True def ruling(value: Any, rules: Iterable[Any]) -> str | _Keep | None: diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index bc843d5..6ea4436 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -854,6 +854,33 @@ def test_two_identical_entity_encoded_elements_are_both_restored(self): assert f"{encoded}" in masked assert "S3cretPassw0rd" not in masked + def test_a_label_rule_that_raises_inside_a_keep_match_refuses_it_and_raises_nothing(self, packs): + # Asked about a keep match's members (review M5): its error refuses + # the keep, and the value is masked as with the label rule alone. + def raises_on_members(value): + if value.get("kind") == "inner-box": + raise RuntimeError("rule failed") + return False + + label_rule = ValueRule("sad", raises_on_members, hints=("sealed-box",)) + sealed = {**SEALED, "inner": {"kind": "inner-box"}} + compact = json.dumps(sealed, separators=(",", ":")) + + def every_path(): + return ( + _text(f"got {json.dumps(sealed)} back", packs), + redact_body(json.dumps(sealed)), + redact_body(KPAY.format(json.dumps(sealed).replace('"', """))), + redact_url(f"https://pg.example/pay?card_number={compact}&password=s3cret"), + mask_outside_kept(f"got {json.dumps(sealed)} back", str.upper), + ) + + configure_masking_value_rules([KEEP_RULE, label_rule]) + masked = every_path() + configure_masking_value_rules([label_rule]) + assert masked == every_path() + assert SEALED["api_key"] not in masked[0] + def test_redact_url_never_raises_with_a_keep_rule_that_raises(self): def broken(_value): raise RuntimeError("rule failed") From fc0bac983c79ace304e6485c8035c33537c2c52b Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:31:32 +0300 Subject: [PATCH 12/22] perf(masking): the text floor and the guard's card-run check read a wallet in microseconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on a KPay body holding an Apple Pay token (7 KB), `floored_text` took 0.9 ms as regexes tried at every character (a lookahead at each, and a bare name backtracking through every base64 word), and the guard's run check 74 µs a 3 KB leaf. `floored_text` now finds each `:` and `=` and reads the key back from there (a quote's opening found with `rfind`, a bare name matched on the reversed text before it), and reads a pair's label forward from an identifier key: about 20 µs. `holds_card_run` first marks digits and separators with one bytes translate: ASCII text with neither thirteen digits in a row nor a separator (base64, hex) holds no run, about 2.5 µs a 3 KB leaf. A name longer than 128 characters is read by its end, after `:` or `=>`, where it was not read at all. Co-Authored-By: Claude Opus 5.5 --- ecsctx/masking/patterns.py | 130 ++++++++++++++++++++++++------- tests/test_masking_keep_rules.py | 1 + 2 files changed, 101 insertions(+), 30 deletions(-) diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index ee92e92..99a4791 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -814,6 +814,14 @@ def pan_shaped(text: str) -> bool: # joined by single separators at most, as the card rule joins them, with no # digit next to either end. _CARD_RUN_IN_TEXT = re.compile(rf"(? bool: @@ -824,6 +832,10 @@ def holds_card_run(text: str) -> bool: reads every text leaf of a keep match with it. Stricter than ``holds_pan_run`` about what a card number is (Luhn, 13 digits up), so ciphertext and hex ids are rarely refused.""" + if text.isascii(): + marked = text.encode("ascii").translate(_DIGIT_MARKS) + if b"\0" * 13 not in marked and b"\2" not in marked: + return False for m in _CARD_RUN_IN_TEXT.finditer(text): run = m.group() if not _EPOCH_MILLISECONDS.fullmatch(run) and _luhn_valid(re.sub(r"\D", "", run)): @@ -1228,30 +1240,82 @@ def mask_objects(text: str, rules: tuple) -> str: # or in such a container (filters): a CVV and the rest of Sensitive # Authentication Data. _FLOOR_TYPES = frozenset({"cvv", "sad"}) -# What text names, wherever it is, as the floor reads it: a quoted key (JSON, -# a repr, JSON in a JSON string, its quotes escaped) then `:`, `=` or `=>`, -# looked for at every quote so that a quote inside a string never hides the -# key after it; a bare key, or the word closing a quoted one, then `:` or -# `=>`; a key and `=` where a key starts (`cvv=`, `&cvv=`, `a=1, cvv=`) -- -# not inside a quoted string, where base64's `=` padding follows a word -# (`"…/cvvXYZ=="`); and an XML tag, start or end. -_QUOTED_NAME = re.compile(r"(?=(\\*[\"'])([^\"'\\]{1,128})\1\s*(?::|=>?))") -_BARE_NAME = re.compile(r"(?)") -_ASSIGNED_NAME = re.compile(r"(?:^|(?<=[\s&?,({\[]))([A-Za-z_][\w.-]{0,127})\s*=") +# A key in text, as the floor reads it, is found from the `:`, `=` or `=>` +# after it and read back from there (``_key_names``): a quoted key (JSON, a +# repr, JSON in a JSON string, its quotes escaped), or a bare one -- after a +# `=`, only where a key starts, never inside a quoted string, where base64's +# `=` padding follows a word (`"…/cvvXYZ=="`). A find per separator, not a +# regex tried at every character: this reads a 7 KB wallet body in +# microseconds. +_NAME_LIMIT = 128 +_QUOTES = "\"'" +# What a key assigned with `=` follows, besides whitespace and the text's start. +_KEY_STARTS = frozenset("&?,({[") +# The name characters a bare key ends with, matched on the text before its +# separator reversed: matched forward, from every position, a long word cost +# its length squared. +_NAME_RUN = re.compile(r"[\w.-]*") _TAG_NAME = re.compile(r"])") +def _key_at(text: str, at: int, assigned: bool) -> str | None: + """The key written before the separator at ``at``: quoted, or a bare name + starting with a letter or `_` (where a key starts, for a bare ``=``).""" + end = at + while end > 0 and text[end - 1].isspace(): + end -= 1 + if end and text[end - 1] in _QUOTES: + close = end - 1 + while close > 0 and text[close - 1] == "\\": + close -= 1 + opening = text.rfind(text[end - 1], max(0, close - _NAME_LIMIT - 1), close) + if opening != -1 and not any(quote in text[opening + 1 : close] for quote in "\"'\\"): + return text[opening + 1 : close] or None + end = close + length = _NAME_RUN.match(text[max(0, end - _NAME_LIMIT - 1) : end][::-1]).end() + if length > _NAME_LIMIT: + # Longer than a key is read: its end, which names what it holds -- + # never after a bare `=`, as base64 padding follows a long word. + return None if assigned else text[end - _NAME_LIMIT : end] + start = end - length + name = text[start:end] + if not name or not (name[0].isalpha() or name[0] == "_"): + return None + if assigned and start and not (text[start - 1].isspace() or text[start - 1] in _KEY_STARTS): + return None + return name + + +def _quoted_after(text: str, at: int) -> str | None: + """The quoted text right after ``at`` (whitespace and escapes skipped), + as a `{name, value}` pair writes its label: None for anything else.""" + while at < len(text) and (text[at].isspace() or text[at] == "\\"): + at += 1 + if at >= len(text) or text[at] not in _QUOTES: + return None + closing = text.find(text[at], at + 1, at + 2 + _NAME_LIMIT) + label = text[at + 1 : closing].rstrip("\\") if closing != -1 else "" + return label if label and not any(quote in label for quote in "\"'\\") else None + + +def _key_names(text: str) -> Iterator[tuple[str, int]]: + """Each key ``text`` writes, and where what follows its separator starts.""" + for separator in ":=": + at = text.find(separator) + while at != -1: + arrow = separator == "=" and text.startswith(">", at + 1) + if (name := _key_at(text, at, separator == "=" and not arrow)) is not None: + yield name, at + 1 + arrow + at = text.find(separator, at + 1) + + @lru_cache(maxsize=1) -def _pair_label() -> re.Pattern: - """A `{name, value}` pair's label as text writes it: an identifier key - the key walk reads a pair by (``filters._PAIR_IDENTIFIERS``, imported - here: filters imports this module as it loads) and its quoted value.""" +def _pair_identifiers() -> frozenset[str]: + """The identifier keys the key walk reads a `{name, value}` pair by. + Imported here: filters imports this module as it loads.""" from ecsctx.masking.filters import _PAIR_IDENTIFIERS - identifiers = "|".join(map(re.escape, _PAIR_IDENTIFIERS)) - return re.compile( - rf"(?=(\\*[\"'])(?i:{identifiers})\1\s*(?::|=>?)\s*(\\*[\"'])([^\"'\\]{{1,128}})\2)" - ) + return frozenset(_PAIR_IDENTIFIERS) def _safe_keys_in_force() -> frozenset[str]: @@ -1264,21 +1328,27 @@ def _safe_keys_in_force() -> frozenset[str]: def floored_text(text: str) -> bool: """Whether ``text`` names, anywhere, a CVV or SAD key, an XML tag named - so, or a `{name, value}` pair labelled so -- read as the key walk reads a - key (every pack on, the service's safe keys honoured), in the text as - written and, where it holds an entity, as it decodes. Text that does - keeps nothing: a container's reach cannot be told from text the rules - read three levels deep, so the whole text fails closed.""" + so, or a `{name, value}` pair labelled so -- an identifier key and the + quoted label after it -- read as the key walk reads a key (every pack + on, the service's safe keys honoured), in the text as written and, + where it holds an entity, as it decodes. Text that does keeps nothing: a + container's reach cannot be told from text the rules read three levels + deep, so the whole text fails closed.""" safe = _safe_keys_in_force() + identifiers = _pair_identifiers() for written in (text, html.unescape(text)) if "&" in text else (text,): - for pattern, group in ((_QUOTED_NAME, 2), (_BARE_NAME, 1), (_ASSIGNED_NAME, 1), (_TAG_NAME, 1)): - for m in pattern.finditer(written): - if classify_key(m.group(group), ALL_PACKS, safe) in _FLOOR_TYPES: - return True - # Uncached, as the key walk classifies a pair's label: free text. - for m in _pair_label().finditer(written): - if classify_key.__wrapped__(m.group(3), ALL_PACKS, safe) in _FLOOR_TYPES: + for name, after in _key_names(written): + if classify_key(name, ALL_PACKS, safe) in _FLOOR_TYPES: return True + if ( + name.lower() in identifiers + and (label := _quoted_after(written, after)) is not None + # Uncached, as the key walk classifies a pair's label: free text. + and classify_key.__wrapped__(label, ALL_PACKS, safe) in _FLOOR_TYPES + ): + return True + if any(classify_key(m.group(1), ALL_PACKS, safe) in _FLOOR_TYPES for m in _TAG_NAME.finditer(written)): + return True return False diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 6ea4436..22f31f7 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -516,6 +516,7 @@ def _kept_in(masked: str, kept: str) -> None: "a-cvv-key-before": '"cvv" = "123", "token" => PLAIN', "a-form-field-before": "a=1&cvv=123 PLAIN", "an-element-after": "PLAIN 1234", + "a-key-longer-than-a-key-is-read": '{"' + "x" * 200 + '_cvv": "123", "token": PLAIN}', } # Keys about a CVV, not one: text naming them keeps its value. NOT_FLOORED = { From 011ad42ea50b2443d9d9be018e544d7d215bfcd2 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:36:21 +0300 Subject: [PATCH 13/22] docs(masking): what keep rules cost, measured; a label rule wins inside a keep match when it is the first match - The cost paragraph gives the measured figures (review 3): with Ottu's WALLET_RULES a line with no wallet token costs 1-4% more than 0.16.0's label rules and 3-15% more than no value rule; a line with a token has no single figure (an MPGS paymentToken line, a Google Pay message, a KPay body, each labelled / kept / unconfigured), and this release's checks add 15-40% over 0.17.0 there. - The M8 example needs the label rule listed before the catch-all keep rule, which otherwise is the inner value's first match too (review 4); CLAUDE.md and the value_rules docstring said "wherever it is listed". - CLAUDE.md: the guard's card-number run, a label rule's error refusing a keep. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 28 +++++++++++++++++----------- ecsctx/CLAUDE.md | 2 +- ecsctx/contrib/ottu/masking.py | 5 +++-- ecsctx/masking/value_rules.py | 7 ++++--- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e7d1e2d..d2d6e3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,6 +24,6 @@ ECS-compliant structured logging with W3C Trace Context support. Framework-agnos - Django's `LogContextBinder` must be imported explicitly (not in `__init__.py`) to avoid circular imports - `LoggingContextMiddleware` must be placed AFTER auth middleware to capture user_id - CVV never carries a token, even when PII is configured — PCI forbids storing CVV in any form, so `[CVV-MASKED]` is always the literal, final output -- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning, though a label rule wins anywhere inside a keep match; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487, each slot text in its alphabet; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list +- Card number **content** rules (and the bare 3-4 digit CVV rules, which read only a CVV beside a card number or its truncation, or after a CVV word) are the opt-in `pci` pack (`get_logging_config(masking_packs=("pci", "financial_ids"))` or `ECSCTX_MASKING_PACKS`): a PCI service that does not enable it gets no PAN content scanning. The keyed CVV text rules (`cvv=123`, `"securityCode": "123"`, `CVV 123`) are `default`: a CVV must not ship from any service, and Connect (default pack) receives the CVV a saved-card payment sends. Key names (`card`, `pan`, `cvv`, …) are masked in every service; expiry keys are safe keys, never masked. A value a service's label rule matches (`ECSCTX_MASK_VALUE_RULES`, found by shape) is its label in every pack, and one its keep rule matches ships as sent -- rules asked in list order, the first match winning, though a value inside a keep match whose own first match is a label rule (one listed before any keep rule matching it) is labelled and the keep refused; nothing kept under a CVV/SAD key or container (the floor), nor a match holding a card/CVV/SAD key or a card number (the guard); in text a kept value is a `[KEPT-…-MASKED]` placeholder while the content rules run. Core names no shape itself -- Ottu's wallet tokens are `ecsctx.contrib.ottu.masking.WALLET_RULES` (keep rules: Apple Pay and Google Pay tokens ship as sent, decided for #159487, each slot text in its alphabet; `WALLET_SAD_RULES` labels them as 0.16.0 did), each service's to list - Card numbers truncate to first 6 + last 4 for 15–19 digits and last 4 only below 15 (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091) — no token alongside, independent of PII configuration, and a second masking pass is a noop - `install_maskers()` only sweeps handlers that exist at call time (no stdlib patching) — a handler built after the call needs another call diff --git a/README.md b/README.md index bbb0417..08fb247 100644 --- a/README.md +++ b/README.md @@ -1428,11 +1428,13 @@ that is JSON text, every JSON object in free text or in a body `redact_body` masks. They are asked in list order and **the first match wins**: a label rule listed first labels a value a keep rule would have kept, and an object a label rule matches is not looked into for one to keep. A label rule wins at -any depth inside a keep match, though: a keep match holding a value a label -rule labels (that value's own first match) is walked as if nothing had -matched, and that value is labelled — with an Apple Pay label rule and +any depth inside a keep match, though: a keep match holding a value whose +own first match is a label rule is walked as if nothing had matched, and +that value is labelled — with an Apple Pay label rule listed before `KeepRule(lambda value: True)`, `{"outer": {"inner": }}` is -`{"outer": {"inner": "[SAD-MASKED]"}}`. A keep match the guard refuses +`{"outer": {"inner": "[SAD-MASKED]"}}`. Listed after it, the catch-all keep +rule is the inner value's first match too, and the whole value ships as +sent. A keep match the guard refuses (below) is walked as if nothing had matched too, and either way no rule listed after it is asked about the value itself, a label rule for the same shape included. A kept value is left as sent under any key and not @@ -1538,13 +1540,17 @@ rule's hints never asks it; text holding one is read once for the objects in it. Every mapping the key walk meets asks every rule, once — a mapping has no text to hint with — so a matcher must be cheap: test a literal field first. JSON text under a key asks, and its key walk passes on, only the rules whose -hints it holds. Keep rules cost 2-5% a line, and about 12% more on a line -that carries a wallet token, which is read for the objects in it and whose -match is guarded. On the benchmark's lines (`scripts/bench_masking.py`) -0.17.0 masks within a few percent of 0.16.0, the run-to-run noise, with no -value rule and with Ottu's `WALLET_RULES` (three keep rules where 0.16.0 had -two label rules): the key walk now asks each mapping once, where it asked -twice. +hints it holds. With Ottu's `WALLET_RULES`, a line carrying no wallet token +costs 1-4% more than with 0.16.0's two label rules, and 3-15% more than with +no value rule (most on a line of many small mappings, each of which asks +every rule; `scripts/bench_masking.py`). A line carrying a token has no +single figure: about 2.5× 0.16.0's label rules on a short line (an MPGS line +with the token as `paymentToken`: ~72 µs labelled, ~171 µs kept, ~46 µs with +no rule; a message holding a Google Pay token: ~135 / ~341 / ~305 µs), and +less on a body every content rule would otherwise read (KPay ``: +~0.96 ms labelled, ~0.89 ms kept, ~3.1 ms with no rule). The slot checks, +the guard's card-number runs and the text floor's scan add 15-40% over +0.17.0 on such lines. Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or Google Pay token is single-use, and the token as it was sent is what debugs a diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 4978f21..69741b6 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a label rule wins at any depth strictly inside a keep match, wherever it is listed (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs and whole leaves only, a CVV inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, except 13 bare digits from a 1 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number inside a text leaf that is no JSON text (`patterns.holds_card_run`: 13-19 digits joined by single separators at most, bounded by non-digits, passing Luhn; a bytes-translate fast path for base64/hex); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index 2fda17e..f9466c2 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -156,8 +156,9 @@ def _is_google_pay_token(value: Any) -> bool: # What a wallet writes in a slot: standard base64 for a key, a signature or # ciphertext, hex for an id or a hash, digits for a time -- nothing else. -# The guard reads keys, pairs and whole leaves, so text in a kept token's -# slot (`"data": "cvv=123"`) would ship with it (Redmine R3). +# The guard reads keys, pairs, whole leaves and a card number inside one, so +# other text in a kept token's slot (`"data": "cvv=123"`) would ship with it +# (Redmine R3). _BASE64 = re.compile(r"[A-Za-z0-9+/]+={0,2}") _HEX = re.compile(r"[0-9A-Fa-f]+") _DIGITS = re.compile(r"[0-9]+") diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 12cdcaf..500d799 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -23,9 +23,10 @@ applied where a key is known: ``filters``, ``patterns.kept_spans``), and a match holding a card, CVV or SAD key, or a card number, at any depth, is walked as if nothing matched (the guard, ``holds_card_data``). A match - holding a value a label rule labels is walked so too, wherever that rule - is listed (``keep_refused``): a label rule wins at any depth inside a keep - match, the first match only for the value itself. + holding a value whose own first match is a label rule is walked so too + (``keep_refused``): a label rule wins at any depth inside a keep match, the + first match deciding each value -- a catch-all keep rule listed first is + every value's first match. Both have: From 4285de8b30fad4df5f6031cc3ba549a3d7cf2b4b Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 22:42:21 +0300 Subject: [PATCH 14/22] fix(masking): the guard's card-number run skips a whole hex id of 24 characters or more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read like other text, about one random 64-hex id in two hundred holds a 13-19 digit run that passes Luhn, and an Apple Pay token carries two (`transactionId`, `transactionIdentifier`): 1.1% of tokens were refused and masked instead of kept. A run inside a whole hex token of 24 characters or more -- an ObjectId, a 32-hex gateway id, a 64-hex digest, nothing alphanumeric touching it -- is no card number, as Connect's own card scan reads one. Shorter hex (`4111…ab`) and hex glued to more letters are still read, and a whole-leaf card number is still refused first. On 3000 random tokens, none is refused now, Apple Pay or Google Pay. Co-Authored-By: Claude Opus 5.5 --- README.md | 19 +++++++++++++------ ecsctx/CLAUDE.md | 2 +- ecsctx/masking/patterns.py | 26 ++++++++++++++++++++------ ecsctx/masking/value_rules.py | 3 ++- tests/test_contrib_ottu_wallets.py | 5 +++++ tests/test_masking_keep_rules.py | 19 +++++++++++++++++-- 6 files changed, 58 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 08fb247..0b112a2 100644 --- a/README.md +++ b/README.md @@ -1476,12 +1476,19 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a hex one. Both except thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's `keyExpiration`, which `pan_shaped` reads as a card number and every - Google Pay token carries. A leaf that is no JSON value counts as card data. - What the guard cannot tell from a token's own text ships with it: a CVV or - other short value written inside a longer string (`"note": "cvv=123"`, or - `cvv+123` inside base64, which reads as ciphertext), and a run of digits - that fails Luhn. Keeping that out is the matcher's job, which is why a - matcher must be strict about what it matches. + Google Pay token carries. A run inside a whole hex token of 24 characters + or more — an ObjectId, a 32-hex gateway id, a 64-hex digest such as Apple + Pay's `transactionId`, with nothing alphanumeric touching it — is no card + number either, as Connect's own card scan reads one: about one random + 64-hex id in two hundred holds a run that passes Luhn, and read like other + text it cost 1.1% of Apple Pay tokens. Shorter hex, or hex glued to more + letters, is read. A leaf that is no JSON value counts as card data. What + the guard cannot tell from a token's own text ships with it: a CVV or other + short value written inside a longer string (`"note": "cvv=123"`, or + `cvv+123` inside base64, which reads as ciphertext), a run of digits that + fails Luhn, and a card number written into a hex id's own characters. + Keeping that out is the matcher's job, which is why a matcher must be + strict about what it matches. Nothing else in a kept value is read: PII or a credential inside it, under a key of its own or written in a string, ships with it. diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 69741b6..9e95c62 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number inside a text leaf that is no JSON text (`patterns.holds_card_run`: 13-19 digits joined by single separators at most, bounded by non-digits, passing Luhn; a bytes-translate fast path for base64/hex); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number inside a text leaf that is no JSON text (`patterns.holds_card_run`: 13-19 digits joined by single separators at most, bounded by non-digits, passing Luhn, not inside a whole hex token of 24+ characters (`_HEX_ID`, as Connect's card scan reads one: 1.1% of Apple Pay tokens' 64-hex ids hold a Luhn run); a bytes-translate fast path for base64/hex); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index 99a4791..d0483ec 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -824,21 +824,35 @@ def pan_shaped(text: str) -> bool: ) +# An id a run inside which is never a card number: a token of 24 or more hex +# characters -- an ObjectId, a 32-hex gateway id, a 64-hex digest, Apple +# Pay's `transactionId` -- whole, nothing alphanumeric touching it. Connect's +# own card scan reads one so: read like other text, about one 64-hex id in two +# hundred holds a 13-19 digit run that passes Luhn. +_HEX_ID = re.compile(r"(? bool: """Whether ``text`` holds, anywhere in it, a run that is a card number: - 13 to 19 digits that pass Luhn -- `card 4111…`, `4111…+cvv+123`, a hex - id holding one -- except thirteen bare digits from a 1 or a 2, epoch - milliseconds. For the guard (``value_rules.holds_card_data``), which - reads every text leaf of a keep match with it. Stricter than + 13 to 19 digits that pass Luhn -- `card 4111…`, `4111…+cvv+123`, short + hex holding one (`4111…ab`) -- except thirteen bare digits from a 1 or a + 2, epoch milliseconds, and a run inside a whole hex id of 24 characters + or more (``_HEX_ID``). For the guard (``value_rules.holds_card_data``), + which reads every text leaf of a keep match with it. Stricter than ``holds_pan_run`` about what a card number is (Luhn, 13 digits up), so - ciphertext and hex ids are rarely refused.""" + ciphertext is rarely refused.""" if text.isascii(): marked = text.encode("ascii").translate(_DIGIT_MARKS) if b"\0" * 13 not in marked and b"\2" not in marked: return False + ids = None for m in _CARD_RUN_IN_TEXT.finditer(text): run = m.group() - if not _EPOCH_MILLISECONDS.fullmatch(run) and _luhn_valid(re.sub(r"\D", "", run)): + if _EPOCH_MILLISECONDS.fullmatch(run) or not _luhn_valid(re.sub(r"\D", "", run)): + continue + if ids is None: + ids = [found.span() for found in _HEX_ID.finditer(text)] + if not any(start <= m.start() and m.end() <= end for start, end in ids): return True return False diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 500d799..9847d03 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -358,7 +358,8 @@ def holds_card_data(value: Any) -> bool: ``keyExpiration`` in ``signedKey``, without which no Google Pay token would ever be kept. A text leaf that is no JSON text holds one when ``holds_card_run`` finds a run of 13 to 19 digits passing Luhn in it, - epoch milliseconds again excepted (JSON text is read by its own leaves). + epoch milliseconds again excepted, and a run inside a whole hex id of 24 + characters or more (JSON text is read by its own leaves). Not ``holds_pan_run``, which flags hex ids. Anything that is no JSON value (an object whose text is not judged here), nesting past the depth cap, and a leaf of JSON text that names a key twice diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index e8b9a83..df3ca89 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -379,10 +379,15 @@ def every_path(): # Thirteen digits from a 1 or a 2 read as epoch milliseconds, # wherever they stand. {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, + # A 64-hex id holding a Luhn-valid run of digits, as about one in + # two hundred does: an id, not a card number (fix round 2). + {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "a" * 10 + "4111111111111111" + "b" * 38}}, + {**PK_PAYMENT_TOKEN, "transactionIdentifier": "A" * 10 + "4111111111111111" + "B" * 38}, ], ids=[ "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", "pk-display-name-connects-fixture", "pk-display-name-forty-characters", "apple-data-epoch-run", + "apple-transaction-id-with-a-luhn-run", "pk-transaction-identifier-with-a-luhn-run", ], ) def test_a_documented_token_is_still_kept(self, token): diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index 22f31f7..e765955 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -78,6 +78,9 @@ def _is_sealed(value) -> bool: # A label rule with a hint of its own, which never matches. OTHER_RULE = ValueRule("secret", lambda value: value.get("kind") == "never", hints=("other-shape",)) PASSWORD = "password=s3cret-Hunter2" +# A 64-hex digest holding a run of digits that passes Luhn, as about one in +# two hundred random ones does. +DIGEST_WITH_A_RUN = "a" * 10 + "4111111111111111" + "b" * 38 class DuckKeep: @@ -321,11 +324,15 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): {"note": "Visa 4111 1111 1111 1111 on file"}, {"memo": ";4111111111111111=25121010000000000000?"}, {"note": "{not json 4111111111111111"}, + # Hex shorter than an id, or glued to more letters: read. + {"ref": "4111111111111111abcdef"}, + {"ref": "zz" + DIGEST_WITH_A_RUN}, + {"ref": DIGEST_WITH_A_RUN + "z"}, ], ids=[ "cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list", "a-pan-in-prose", "a-pan-in-base64", "a-pan-in-hex", "a-grouped-pan-in-prose", "track-2", - "a-pan-in-text-that-is-no-json", + "a-pan-in-text-that-is-no-json", "hex-shorter-than-an-id", "hex-glued-to-letters", "hex-glued-to-a-letter-after", ], ) def test_the_guard_a_match_holding_card_data_is_walked(self, extra): @@ -342,8 +349,16 @@ def test_the_guard_a_match_holding_card_data_is_walked(self, extra): {"ref": "order 4111111111111112 retried"}, {"ref": "41111111111111111111"}, {"note": json.dumps({"keyExpiration": "1790318444473"})}, + # A whole hex id of 24 characters or more: a digest, an ObjectId + # -- read as Connect's own card scan reads one (fix round 2). + {"digest": DIGEST_WITH_A_RUN}, + {"digest": DIGEST_WITH_A_RUN.upper()}, + {"note": "session aaaa4111111111111111bbbb ended"}, + ], + ids=[ + "a-cvv-in-a-string", "epoch-milliseconds", "a-run-failing-luhn", "twenty-digits", "json-text-epoch", + "a-64-hex-digest", "an-uppercase-digest", "a-24-hex-id-in-prose", ], - ids=["a-cvv-in-a-string", "epoch-milliseconds", "a-run-failing-luhn", "twenty-digits", "json-text-epoch"], ) def test_what_the_guard_does_not_read_in_a_text_leaf(self, extra): # A CVV or other short value inside a longer string is the matcher's From 20a287747a5b32b7b6fad52fd817c98c1da62250 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 23:11:49 +0300 Subject: [PATCH 15/22] fix(masking): the guard reads a text leaf's digits as the pci rules do, in one pass, skipping only a whole hex id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 skipped a Luhn run inside any whole hex token in a leaf, and found the tokens again for every run: a crafted base64 `data` of `/4111111111111111deadbeef` repeated kept its token and cost 6.5 s at 400 KB (review r2, 1). Bounded by `/`, the hex inside base64 also shipped its card number (4). And the run was narrower than the pci rules': a card beside another digit group (`1234 4111…`, `4111… 123`) and 20 bare digits shipped (3). A leaf that is a whole hex id of 24 characters or more -- Apple Pay's `transactionId`, `transactionIdentifier`, `applicationData` -- is not read for a run; any other text leaf is read once with `holds_pan_run`, as the pci rules read a value: 12 digits or more joined by single separators. A leaf that is epoch milliseconds stays exempt, and a whole-leaf card number is still refused first. 400 KB of the crafted `data` is refused in 9 ms (the keep decision on its text 33 ms), linear in its size; of 3000 random Apple Pay tokens none is refused for a hex id. Co-Authored-By: Claude Opus 5.5 --- README.md | 45 ++++++++++++------------- ecsctx/CLAUDE.md | 2 +- ecsctx/masking/patterns.py | 53 +++++++++++------------------- ecsctx/masking/value_rules.py | 33 ++++++++++--------- tests/test_contrib_ottu_wallets.py | 31 ++++++++++++++--- tests/test_masking_keep_rules.py | 33 ++++++++++++++----- 6 files changed, 113 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 0b112a2..7e9024a 100644 --- a/README.md +++ b/README.md @@ -1467,28 +1467,29 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: - **The guard.** A match that holds, at any depth — JSON text inside it included — a key the key rules read as a card, a CVV or other SAD, a `{name, value}` pair whose identifier reads as one (as the key walk reads a - pair), a leaf that is a card number, or a text leaf with a card number in - it, is walked as if nothing had matched. A leaf is a card number when it is - an int `int_is_pan` reads as one, or a string `pan_shaped` reads as one, - whatever its prefix and Luhn or not. A text leaf holds one when a run of 13 - to 19 digits in it, joined by single separators at most and with no digit - next to either end, passes Luhn: `"card 4111…"`, Track 2 data, - `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a hex one. Both except - thirteen bare digits from a 1 or a 2 (epoch milliseconds until 2065), Google - Pay's `keyExpiration`, which `pan_shaped` reads as a card number and every - Google Pay token carries. A run inside a whole hex token of 24 characters - or more — an ObjectId, a 32-hex gateway id, a 64-hex digest such as Apple - Pay's `transactionId`, with nothing alphanumeric touching it — is no card - number either, as Connect's own card scan reads one: about one random - 64-hex id in two hundred holds a run that passes Luhn, and read like other - text it cost 1.1% of Apple Pay tokens. Shorter hex, or hex glued to more - letters, is read. A leaf that is no JSON value counts as card data. What - the guard cannot tell from a token's own text ships with it: a CVV or other - short value written inside a longer string (`"note": "cvv=123"`, or - `cvv+123` inside base64, which reads as ciphertext), a run of digits that - fails Luhn, and a card number written into a hex id's own characters. - Keeping that out is the matcher's job, which is why a matcher must be - strict about what it matches. + pair), a leaf that is a card number, or a text leaf with a card number's + digits in it, is walked as if nothing had matched. A leaf is a card number + when it is an int `int_is_pan` reads as one, or a string `pan_shaped` reads + as one, whatever its prefix and Luhn or not. A text leaf holds a card + number's digits when, read in one pass as the `pci` rules read a value + (`holds_pan_run`), it holds a run of 12 digits or more joined by single + separators at most: `"card 4111…"`, `"1234 4111…"`, `"4111… 123"`, Track 2 + data, 20 bare digits, `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a + hex one. Two leaves hold none: a leaf that is thirteen bare digits from a 1 + or a 2 (epoch milliseconds until 2065), Google Pay's `keyExpiration`, which + `pan_shaped` reads as a card number and every Google Pay token carries; and + a leaf that is a whole hex id of 24 characters or more — a 64-hex digest + such as Apple Pay's `transactionId`, an ObjectId — about one random 64-hex + id in two hundred holds a 13-19 digit run that passes Luhn, and read like + other text it cost 1.1% of Apple Pay tokens. Only a whole leaf: a hex id + inside base64 or prose (`"AA/4111…deadbeef/AA"`), or epoch milliseconds + inside a longer string, is read. A leaf that is no JSON value counts as + card data. What the guard cannot tell from a token's own text ships with + it: a CVV or other short value written inside a longer string + (`"note": "cvv=123"`, or `cvv+123` inside base64, which reads as + ciphertext), and a card number written into a leaf that is a whole hex id + (`4111…` and eight hex letters). Keeping that out is the matcher's job, + which is why a matcher must be strict about what it matches. Nothing else in a kept value is read: PII or a credential inside it, under a key of its own or written in a string, ships with it. diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index 9e95c62..ce866af 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number inside a text leaf that is no JSON text (`patterns.holds_card_run`: 13-19 digits joined by single separators at most, bounded by non-digits, passing Luhn, not inside a whole hex token of 24+ characters (`_HEX_ID`, as Connect's card scan reads one: 1.1% of Apple Pay tokens' 64-hex ids hold a Luhn run); a bytes-translate fast path for base64/hex); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number's digits inside a text leaf that is no JSON text (`patterns.holds_card_run`: `holds_pan_run`, 12+ digits joined by single separators, as the pci rules read a value, one linear pass with a bytes-translate fast path for base64; not read in a leaf that is a whole hex id of 24+ characters, `_HEX_ID` fullmatch -- Apple Pay's 64-hex ids, 1.1% of which hold a Luhn run -- nor a leaf that is epoch milliseconds); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index d0483ec..f5a6dfc 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -55,7 +55,6 @@ mask_by_field_type, ) from ecsctx.masking.value_rules import ( - _EPOCH_MILLISECONDS, KEEP, DuplicateKey, _first_match, @@ -810,51 +809,39 @@ def pan_shaped(text: str) -> bool: _DIGIT_RUN = re.compile(rf"\+?\d(?:{_CARD_SEP}?\d)*") -# A card number anywhere in a value, as the guard reads one: 13 to 19 digits -# joined by single separators at most, as the card rule joins them, with no -# digit next to either end. -_CARD_RUN_IN_TEXT = re.compile(rf"(? bool: - """Whether ``text`` holds, anywhere in it, a run that is a card number: - 13 to 19 digits that pass Luhn -- `card 4111…`, `4111…+cvv+123`, short - hex holding one (`4111…ab`) -- except thirteen bare digits from a 1 or a - 2, epoch milliseconds, and a run inside a whole hex id of 24 characters - or more (``_HEX_ID``). For the guard (``value_rules.holds_card_data``), - which reads every text leaf of a keep match with it. Stricter than - ``holds_pan_run`` about what a card number is (Luhn, 13 digits up), so - ciphertext is rarely refused.""" + """Whether ``text``, a text leaf of a keep match, holds a run of digits + long enough to be a card number, read as the pci rules read a value + (``holds_pan_run``: 12 digits or more, joined by single separators at + most -- `card 4111…`, `1234 4111…`, `4111… 123`, `4111…+cvv+123` in + base64, `4111…ab` in hex) -- except a leaf that is a whole hex id of 24 + characters or more (``_HEX_ID``). For the guard + (``value_rules.holds_card_data``), which excepts a leaf that is epoch + milliseconds itself. One pass, linear in the leaf: ASCII text with no + such run of digits and no separator is told by one bytes translate.""" + if _HEX_ID.fullmatch(text): + return False if text.isascii(): marked = text.encode("ascii").translate(_DIGIT_MARKS) - if b"\0" * 13 not in marked and b"\2" not in marked: + if _RUN_MARKS not in marked and b"\2" not in marked: return False - ids = None - for m in _CARD_RUN_IN_TEXT.finditer(text): - run = m.group() - if _EPOCH_MILLISECONDS.fullmatch(run) or not _luhn_valid(re.sub(r"\D", "", run)): - continue - if ids is None: - ids = [found.span() for found in _HEX_ID.finditer(text)] - if not any(start <= m.start() and m.end() <= end for start, end in ids): - return True - return False + return holds_pan_run(text) def holds_pan_run(text: str, *, phone: bool = False) -> bool: diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 9847d03..66b977c 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -356,9 +356,9 @@ def holds_card_data(value: Any) -> bool: a string ``pan_shaped`` reads as one -- whatever its prefix, Luhn or not -- except thirteen bare digits from a 1 or a 2: epoch milliseconds, Google Pay's ``keyExpiration`` in ``signedKey``, without which no Google Pay token would - ever be kept. A text leaf that is no JSON text holds one when - ``holds_card_run`` finds a run of 13 to 19 digits passing Luhn in it, - epoch milliseconds again excepted, and a run inside a whole hex id of 24 + ever be kept. Any other text leaf that is no JSON text holds one when + ``holds_card_run`` finds a run of 12 digits or more in it, read as the pci + rules read a value -- except a leaf that is a whole hex id of 24 characters or more (JSON text is read by its own leaves). Not ``holds_pan_run``, which flags hex ids. Anything that is no JSON value (an object whose text is not judged here), @@ -394,19 +394,22 @@ def holds(item: Any, depth: int) -> bool: if isinstance(item, float): return item.is_integer() and int_is_pan(int(item)) if isinstance(item, str): - if pan_shaped(item) and not _EPOCH_MILLISECONDS.fullmatch(item.strip()): + if _EPOCH_MILLISECONDS.fullmatch(item.strip()): + return False # Google Pay's `keyExpiration`, a whole leaf + if pan_shaped(item): return True - if item.lstrip()[:1] not in ("{", "["): - return holds_card_run(item) - try: - parsed = json_as_written(item) - except (DuplicateKey, RecursionError): - return True - except ValueError: - return holds_card_run(item) - # Its own leaves are read: `keyExpiration` in Google Pay's - # `signedKey` is a whole leaf there. - return holds(parsed, depth + 1) + if item.lstrip()[:1] in ("{", "["): + try: + parsed = json_as_written(item) + except (DuplicateKey, RecursionError): + return True + except ValueError: + pass + else: + # Its own leaves are read: `keyExpiration` in Google + # Pay's `signedKey` is a whole leaf there. + return holds(parsed, depth + 1) + return holds_card_run(item) return True return holds(value, 0) diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index df3ca89..6d02dda 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -21,6 +21,7 @@ import html import json import logging +import time import pytest @@ -155,6 +156,14 @@ **APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "4111111111111111ab"}, }, + # A hex id inside base64 is base64's text, read (review r2, 1 and 4); + # digits beside a card number are read as the pci rules read them (3). + "apple-data-hex-id-holding-a-pan": {**APPLE_PAY, "data": "AA/4111111111111111deadbeef/AA"}, + "apple-data-epoch-run": {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, + "apple-transaction-id-twenty-digits": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "transactionId": "41111111111111110000"}, + }, } # What the device says about the card: the network's name and the last four. NOT_A_DISPLAY_NAME = { @@ -345,6 +354,23 @@ def test_each_token_is_kept(self, token): def test_the_matchers_are_strict(self, value): assert not is_kept(value) + @pytest.mark.parametrize("path", ["key-walk", "text", "body"]) + def test_a_crafted_400_kb_token_is_read_in_linear_time(self, path, packs): + # Hex ids inside a base64 slot, each holding a Luhn-valid run: 400 KB + # of them cost 6.5 s while a run inside a hex token was skipped by + # rescanning the tokens for each run (review r2, 1). + token = {**APPLE_PAY, "data": "AA" + "/4111111111111111deadbeef" * 16_000} + assert len(token["data"]) > 400_000 + started = time.perf_counter() + assert not is_kept(token) + if path == "key-walk": + assert patterns.kept_spans(json.dumps({"x": token}), WALLET_RULES) == [] + elif path == "text": + assert patterns.kept_spans(f"token {json.dumps(token)} end", WALLET_RULES) == [] + else: + assert patterns.kept_spans(KPAY_BODY.format(json.dumps(token)), WALLET_RULES) == [] + assert time.perf_counter() - started < 1.0 + @pytest.mark.parametrize("name", NOT_A_DISPLAY_NAME.values(), ids=NOT_A_DISPLAY_NAME.keys()) def test_a_display_name_is_a_network_and_the_last_four(self, name): token = {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], "displayName": name}} @@ -376,9 +402,6 @@ def every_path(): {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Amex", "network": "AmEx", "type": "credit"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Apple Pay", "network": "Visa", "type": "debit"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 35 + " 1234"}}, - # Thirteen digits from a 1 or a 2 read as epoch milliseconds, - # wherever they stand. - {**APPLE_PAY, "data": "QUJD1790318444473QUJD"}, # A 64-hex id holding a Luhn-valid run of digits, as about one in # two hundred does: an id, not a card number (fix round 2). {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "a" * 10 + "4111111111111111" + "b" * 38}}, @@ -386,7 +409,7 @@ def every_path(): ], ids=[ "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", - "pk-display-name-connects-fixture", "pk-display-name-forty-characters", "apple-data-epoch-run", + "pk-display-name-connects-fixture", "pk-display-name-forty-characters", "apple-transaction-id-with-a-luhn-run", "pk-transaction-identifier-with-a-luhn-run", ], ) diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index e765955..b532c07 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -328,11 +328,26 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): {"ref": "4111111111111111abcdef"}, {"ref": "zz" + DIGEST_WITH_A_RUN}, {"ref": DIGEST_WITH_A_RUN + "z"}, + # Only a leaf that is a whole hex id is not read (review r2, 1 and + # 4): one inside base64, or in prose, is. + {"box": "AA/4111111111111111deadbeef/AA"}, + {"note": "session aaaa4111111111111111bbbb ended"}, + # A run read as the pci rules read one (review r2, 3): a card + # beside another digit group, more digits than a card, no Luhn. + {"note": "1234 4111111111111111"}, + {"note": "4111111111111111 123"}, + {"note": "4111111111111111 12/25"}, + {"ref": "41111111111111110000"}, + {"ref": "order 4111111111111112 retried"}, + # Epoch milliseconds are only a whole leaf's. + {"box": "QUJD1790318444473QUJD"}, ], ids=[ "cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list", "a-pan-in-prose", "a-pan-in-base64", "a-pan-in-hex", "a-grouped-pan-in-prose", "track-2", "a-pan-in-text-that-is-no-json", "hex-shorter-than-an-id", "hex-glued-to-letters", "hex-glued-to-a-letter-after", + "a-hex-id-in-base64", "a-24-hex-id-in-prose", "a-group-before-a-card", "a-cvv-after-a-card", + "an-expiry-after-a-card", "twenty-digits", "a-run-failing-luhn", "epoch-milliseconds-in-a-longer-leaf", ], ) def test_the_guard_a_match_holding_card_data_is_walked(self, extra): @@ -345,24 +360,24 @@ def test_the_guard_a_match_holding_card_data_is_walked(self, extra): "extra", [ {"note": "cvv=123"}, - {"box": "QUJD1790318444473QUJD"}, - {"ref": "order 4111111111111112 retried"}, - {"ref": "41111111111111111111"}, + {"note": "order 12345678901 shipped"}, + {"expiration": "1790318444473"}, {"note": json.dumps({"keyExpiration": "1790318444473"})}, - # A whole hex id of 24 characters or more: a digest, an ObjectId - # -- read as Connect's own card scan reads one (fix round 2). + # A leaf that is a whole hex id of 24 characters or more: a + # digest, an ObjectId, Apple Pay's transactionId (fix rounds 2, 3). {"digest": DIGEST_WITH_A_RUN}, {"digest": DIGEST_WITH_A_RUN.upper()}, - {"note": "session aaaa4111111111111111bbbb ended"}, + {"digest": "aaaa4111111111111111bbbb"}, ], ids=[ - "a-cvv-in-a-string", "epoch-milliseconds", "a-run-failing-luhn", "twenty-digits", "json-text-epoch", - "a-64-hex-digest", "an-uppercase-digest", "a-24-hex-id-in-prose", + "a-cvv-in-a-string", "eleven-digits", "epoch-milliseconds", "json-text-epoch", + "a-64-hex-digest", "an-uppercase-digest", "a-24-hex-id", ], ) def test_what_the_guard_does_not_read_in_a_text_leaf(self, extra): # A CVV or other short value inside a longer string is the matcher's - # to keep out; a run that is no card number is no card number. + # to keep out; fewer digits than a card number, epoch milliseconds + # and a whole hex id hold none. holding = {**SEALED, **extra} assert is_kept(holding) assert _walk({"payload": holding}) == {"payload": holding} From bac465b56454e47540eb30ceaa467e02c9277f50 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 23:12:51 +0300 Subject: [PATCH 16/22] fix(contrib): a displayName is letters, spaces and `.&-`, and optionally the last four `displayName` accepted any non-digit text of up to 40 characters, so `jane.roe@example.com` was kept (review r2, 6). It is now `[A-Za-z][A-Za-z .&-]{0,39}`, optionally followed by one space and exactly four digits (`Visa 0492`, `American Express`, `Amex`), and the README says exactly that. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++--- ecsctx/contrib/ottu/masking.py | 17 ++++++++--------- tests/test_contrib_ottu_wallets.py | 10 +++++++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7e9024a..cd02d5f 100644 --- a/README.md +++ b/README.md @@ -1574,9 +1574,10 @@ signature or ciphertext (`data`, `signature`, the header's JSON text of all its fields and nothing else), hex for an id or a hash (`transactionId`, `applicationData`, `transactionIdentifier`), digits for `keyExpiration`, and for what `paymentMethod` says about the card short -text: its `network` and `type` at most 64 characters, its `displayName` at -most 40, the network's name and, after a space, the card's last four -(`Visa 0492`, `Amex`) with no other digit. A token holding anything else in a +text: its `network` and `type` at most 64 characters, its `displayName` a +letter and up to 39 more letters, spaces, `.`, `&` or `-`, optionally +followed by one space and exactly four digits, the card's last four +(`Visa 0492`, `American Express`, `Amex`). A token holding anything else in a slot — `"data": "cvv=123"`, a spaced card number as its `signature`, `"displayName": "cvv 123"` — is not kept, and is masked as before; nor is one with a card number in a slot's own alphabet (`"4111…+cvv+123"` in diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index f9466c2..d911b48 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -35,7 +35,7 @@ Pay ``paymentData`` or ``PKPaymentToken``, a Google Pay payment method token, no key besides the ones they write, and every leaf text in its alphabet (base64, hex or digits; short text for what ``paymentMethod`` - says about the card, its display name the network's and the last four) + says about the card, its display name letters and the last four) -- so anything else is walked as before: a ``PKPaymentToken`` with a key of its own keeps only its ``paymentData``, and a token with ``"data": "cvv=123"`` is masked. A CVV written in a slot's own alphabet @@ -170,12 +170,12 @@ def _is_google_pay_token(value: Any) -> bool: "applicationData": _HEX, } # PKPaymentToken's `paymentMethod` is what the device says about the card: -# its network and type, short text, and a display name that is the network's -# name and, after a space, the card's last four ("Visa 0492", "Amex") -- no -# other digit, so neither a card number nor a CVV can ride in it. +# its network and type, short text, and a display name: up to 40 letters, +# spaces, `.`, `&` and `-` from a letter, and optionally a space and the +# card's last four ("Visa 0492", "American Express", "Amex") -- no other +# digit or sign, so neither a card number, a CVV nor an email rides in it. _PAYMENT_METHOD_TEXT_LIMIT = 64 -_DISPLAY_NAME = re.compile(r"\D*(?: [0-9]{4})?") -_DISPLAY_NAME_LIMIT = 40 +_DISPLAY_NAME = re.compile(r"[A-Za-z][A-Za-z .&-]{0,39}(?: [0-9]{4})?") def _written_in(field: Any, alphabet: re.Pattern) -> bool: @@ -207,14 +207,13 @@ def _is_apple_pay_payment_data(value: Any) -> bool: def _is_display_name(name: Any) -> bool: - return isinstance(name, str) and len(name) <= _DISPLAY_NAME_LIMIT and _DISPLAY_NAME.fullmatch(name) is not None + return isinstance(name, str) and _DISPLAY_NAME.fullmatch(name) is not None def _is_pk_payment_token(value: Any) -> bool: """Apple Pay's PKPaymentToken: its `paymentData`, what the device says about the card -- its network and type, each short text, and its display - name, the network's and the last four -- and the transaction's id in - hex.""" + name, letters and the last four -- and the transaction's id in hex.""" payment_data = value.get("paymentData") if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): return False diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 6d02dda..47b82aa 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -172,7 +172,12 @@ "a-cvv-after-the-last-four": "Visa 1234 cvv 123", "five-digits": "Visa 12345", "the-last-four-first": "1234 Visa", - "too-long": "V" * 36 + " 1234", + "too-long": "V" * 41, + "too-long-before-the-last-four": "V" * 41 + " 1234", + # Letters, spaces, `.`, `&` and `-` only, from a letter (review r2, 6). + "an-email": "jane.roe@example.com", + "a-digit-in-the-name": "Visa2 1234", + "a-space-first": " Visa 1234", } PACKS = [["default"], sorted(ALL_PACKS)] @@ -402,6 +407,8 @@ def every_path(): {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Amex", "network": "AmEx", "type": "credit"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "Apple Pay", "network": "Visa", "type": "debit"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 35 + " 1234"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 40 + " 1234"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "American Express 1234", "network": "AmEx"}}, # A 64-hex id holding a Luhn-valid run of digits, as about one in # two hundred does: an id, not a card number (fix round 2). {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "a" * 10 + "4111111111111111" + "b" * 38}}, @@ -410,6 +417,7 @@ def every_path(): ids=[ "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", "pk-display-name-connects-fixture", "pk-display-name-forty-characters", + "pk-display-name-forty-letters-and-the-last-four", "pk-display-name-in-words", "apple-transaction-id-with-a-luhn-run", "pk-transaction-identifier-with-a-luhn-run", ], ) From d49e060d4a15ce4621e6833712b4a2a6e7bb9dc4 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 23:13:16 +0300 Subject: [PATCH 17/22] docs(masking): what the text floor reads, and what only the key walk does The README said the text floor reads a name "as the key walk reads a key" (review r2, 2). It reads a quoted key, tag or pair label as written, up to 128 characters (a longer key by its last 128), with no `\uXXXX` escape decoded; the key walk parses JSON and reads more, so a key written with escapes, or longer with its CVV word first, floors there and not in text. Through Ottu's WALLET_RULES that gap ships nothing but a token's own text. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cd02d5f..cea1a74 100644 --- a/README.md +++ b/README.md @@ -1529,10 +1529,19 @@ nothing, and is masked as without a keep rule. A name is a quoted key then `:`, `=` or `=>` (JSON, a repr, JSON in a JSON string), a bare key then `:` or `=>`, a key and `=` where a key starts (`cvv=`, `&cvv=`; not base64's `=` padding inside a string), an XML tag, or a `{name, value}` pair's label -(`"name": "card security code"`), read as the key walk reads a key — every -pack on, the service's safe keys honoured — in the text as written and, where -it holds an entity, as it decodes. A word in prose (`the cvv check passed`) -names no key. That fails closed: a value beside such a key, not under it +(`"name": "card security code"`), classified as the key walk classifies a +key — every pack on, the service's safe keys honoured. It is read as +written: a quoted key, tag or pair label up to 128 characters (a longer key +by its last 128, before `:` or `=>`), with no `\uXXXX` escape decoded, in the +text as it stands and, where it holds an entity, as the entities decode. The +key walk reads more: it parses JSON, so a key written with escapes +(`"\u0063vv"`), or longer than 128 characters with its CVV word first, is a +CVV key there and not to the text floor, which can keep a value under it in +text where the key walk would not. Through Ottu's `WALLET_RULES` that gap +ships nothing but a wallet token's own text: their matchers fix every key a +token holds and every slot's alphabet, and the guard reads them, so nothing +else can ride in what they keep. A word in prose (`the cvv check passed`) names +no key. That fails closed: a value beside such a key, not under it (`{"cvv": "123", "token": {…}}`, or a `` element elsewhere in a body), is not kept in text, though the key walk keeps it. `redact_body` reads an element's entity-encoded text the same way, and keeps nothing in it when the From 83dd6bd7a97e0ba915b28955a8866b5c9c6371b3 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 23:17:54 +0300 Subject: [PATCH 18/22] docs(masking): the cost paragraph's wallet-line figures after the guard's one-pass read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-measured after the guard's read became one pass of `holds_pan_run` (this round): a Google Pay message line kept costs ~300 µs where it cost ~341 µs, and this release's checks add 10-35% over 0.17.0 on a wallet line, where the paragraph said 15-40%. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cea1a74..2dee6c7 100644 --- a/README.md +++ b/README.md @@ -1558,16 +1558,16 @@ it. Every mapping the key walk meets asks every rule, once — a mapping has no text to hint with — so a matcher must be cheap: test a literal field first. JSON text under a key asks, and its key walk passes on, only the rules whose hints it holds. With Ottu's `WALLET_RULES`, a line carrying no wallet token -costs 1-4% more than with 0.16.0's two label rules, and 3-15% more than with +costs 1-4% more than with 0.16.0's two label rules, and 5-16% more than with no value rule (most on a line of many small mappings, each of which asks every rule; `scripts/bench_masking.py`). A line carrying a token has no -single figure: about 2.5× 0.16.0's label rules on a short line (an MPGS line -with the token as `paymentToken`: ~72 µs labelled, ~171 µs kept, ~46 µs with -no rule; a message holding a Google Pay token: ~135 / ~341 / ~305 µs), and -less on a body every content rule would otherwise read (KPay ``: -~0.96 ms labelled, ~0.89 ms kept, ~3.1 ms with no rule). The slot checks, -the guard's card-number runs and the text floor's scan add 15-40% over -0.17.0 on such lines. +single figure: about 2.3-2.5× 0.16.0's label rules on a short line (an MPGS +line with the token as `paymentToken`: ~73 µs labelled, ~175 µs kept, ~47 µs +with no rule; a message holding a Google Pay token: ~135 / ~300 / ~305 µs), +and less on a body every content rule would otherwise read (KPay ``: +~0.96 ms labelled, ~0.87 ms kept, ~3.1 ms with no rule). The slot checks, +the guard's digit runs and the text floor's scan add 10-35% over 0.17.0 on +such lines. Ottu's wallet tokens ship in logs exactly as sent (#159487): an Apple Pay or Google Pay token is single-use, and the token as it was sent is what debugs a From 9cf12af8ebfc13fd464e68fbc8b77cce585edd84 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Thu, 1 Oct 2026 23:20:43 +0300 Subject: [PATCH 19/22] fix(masking): a whole hex id the guard skips holds a hex letter A leaf of digits alone matched the hex-id exemption, so 24 digits or more written into a hex slot (a card number and eight digits in Apple Pay's `transactionId`) were not read for a card number. A hex id holds a hex letter; digits alone are read like any other text (#160102). Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++--- ecsctx/masking/patterns.py | 5 +++-- tests/test_masking_keep_rules.py | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2dee6c7..ebff98e 100644 --- a/README.md +++ b/README.md @@ -1481,9 +1481,10 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: a leaf that is a whole hex id of 24 characters or more — a 64-hex digest such as Apple Pay's `transactionId`, an ObjectId — about one random 64-hex id in two hundred holds a 13-19 digit run that passes Luhn, and read like - other text it cost 1.1% of Apple Pay tokens. Only a whole leaf: a hex id - inside base64 or prose (`"AA/4111…deadbeef/AA"`), or epoch milliseconds - inside a longer string, is read. A leaf that is no JSON value counts as + other text it cost 1.1% of Apple Pay tokens. Only a whole leaf holding a + hex letter: a hex id inside base64 or prose (`"AA/4111…deadbeef/AA"`), + digits alone however many, and epoch milliseconds inside a longer string + are read. A leaf that is no JSON value counts as card data. What the guard cannot tell from a token's own text ships with it: a CVV or other short value written inside a longer string (`"note": "cvv=123"`, or `cvv+123` inside base64, which reads as diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index f5a6dfc..ac81f9b 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -821,8 +821,9 @@ def pan_shaped(text: str) -> bool: # A leaf that is a hex id of 24 characters or more -- a 64-hex digest, Apple # Pay's `transactionId`, an ObjectId -- holds no card number: read like other # text, about one random 64-hex id in two hundred holds a 13-19 digit run -# that passes Luhn. Only a whole leaf: hex inside base64 or prose is read. -_HEX_ID = re.compile(r"[0-9A-Fa-f]{24,}") +# that passes Luhn. Only a whole leaf, holding a hex letter: hex inside +# base64 or prose is read, and so are digits alone, which no hex id is. +_HEX_ID = re.compile(r"(?=[0-9]*[A-Fa-f])[0-9A-Fa-f]{24,}") def holds_card_run(text: str) -> bool: diff --git a/tests/test_masking_keep_rules.py b/tests/test_masking_keep_rules.py index b532c07..6c7a768 100644 --- a/tests/test_masking_keep_rules.py +++ b/tests/test_masking_keep_rules.py @@ -341,6 +341,8 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): {"ref": "order 4111111111111112 retried"}, # Epoch milliseconds are only a whole leaf's. {"box": "QUJD1790318444473QUJD"}, + # A hex id holds a hex letter: 24 digits or more are read. + {"digest": "411111111111111112345678"}, ], ids=[ "cvv", "cardNumber", "a-pan-leaf", "an-int-pan", "a-nested-json-cvv", "a-pin-in-a-list", @@ -348,6 +350,7 @@ def test_the_floor_holds_below_a_key_the_service_lists_in_a_cvv_container(self): "a-pan-in-text-that-is-no-json", "hex-shorter-than-an-id", "hex-glued-to-letters", "hex-glued-to-a-letter-after", "a-hex-id-in-base64", "a-24-hex-id-in-prose", "a-group-before-a-card", "a-cvv-after-a-card", "an-expiry-after-a-card", "twenty-digits", "a-run-failing-luhn", "epoch-milliseconds-in-a-longer-leaf", + "digits-only-in-a-hex-slot", ], ) def test_the_guard_a_match_holding_card_data_is_walked(self, extra): From 9dac34c94af1b09df28549356d1df53f36c66d16 Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Fri, 2 Oct 2026 07:29:20 +0300 Subject: [PATCH 20/22] fix(contrib): a wallet's base64 slot is never hex alone; its displayName never names a CVV A whole hex id of 24 characters or more is not read for a card number (the guard's exemption for Apple Pay's ids), so a base64 slot written in hex alone (`"data": "4111111111111111abcd1234"`) shipped a card number. A wallet's base64 never is: Ottu's matchers refuse it. And a displayName that reads as a CVV or other SAD ("CVV 1234", "Security Code 1234") is no card's: a CID is four digits, the shape of the last four (#160102). Co-Authored-By: Claude Opus 5.5 --- ecsctx/contrib/ottu/masking.py | 12 +++++++++--- tests/test_contrib_ottu_wallets.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index d911b48..6e8d8e6 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -67,6 +67,7 @@ from collections.abc import Mapping from typing import Any +from ecsctx.masking.patterns import _FLOOR_TYPES, ALL_PACKS, classify_key from ecsctx.masking.value_rules import KeepRule, ValueRule, json_as_written SAFE_KEYS = frozenset({ @@ -159,7 +160,9 @@ def _is_google_pay_token(value: Any) -> bool: # The guard reads keys, pairs, whole leaves and a card number inside one, so # other text in a kept token's slot (`"data": "cvv=123"`) would ship with it # (Redmine R3). -_BASE64 = re.compile(r"[A-Za-z0-9+/]+={0,2}") +# Not hex alone: a whole hex id is not read for a card number (the guard's +# exemption for Apple Pay's ids), and a wallet's base64 never is one. +_BASE64 = re.compile(r"(?![0-9A-Fa-f]*\Z)[A-Za-z0-9+/]+={0,2}") _HEX = re.compile(r"[0-9A-Fa-f]+") _DIGITS = re.compile(r"[0-9]+") _HEADER_ALPHABETS = { @@ -173,7 +176,8 @@ def _is_google_pay_token(value: Any) -> bool: # its network and type, short text, and a display name: up to 40 letters, # spaces, `.`, `&` and `-` from a letter, and optionally a space and the # card's last four ("Visa 0492", "American Express", "Amex") -- no other -# digit or sign, so neither a card number, a CVV nor an email rides in it. +# digit or sign, so neither a card number nor an email rides in it, and no +# name that reads as a CVV or other SAD: a CID is four digits ("CVV 1234"). _PAYMENT_METHOD_TEXT_LIMIT = 64 _DISPLAY_NAME = re.compile(r"[A-Za-z][A-Za-z .&-]{0,39}(?: [0-9]{4})?") @@ -207,7 +211,9 @@ def _is_apple_pay_payment_data(value: Any) -> bool: def _is_display_name(name: Any) -> bool: - return isinstance(name, str) and _DISPLAY_NAME.fullmatch(name) is not None + if not (isinstance(name, str) and _DISPLAY_NAME.fullmatch(name)): + return False + return classify_key(name.rstrip("0123456789 "), ALL_PACKS) not in _FLOOR_TYPES def _is_pk_payment_token(value: Any) -> bool: diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 47b82aa..5413304 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -164,6 +164,13 @@ **APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "41111111111111110000"}, }, + # A base64 slot is no hex id: hex alone there is not base64 a wallet + # writes, and a card number would ride in it as a hex id (review r3, 1). + "apple-data-all-hex": {**APPLE_PAY, "data": "4111111111111111abcd1234"}, + "apple-key-hash-all-hex": { + **APPLE_PAY, + "header": {**APPLE_PAY["header"], "publicKeyHash": "4111111111111111123a0000"}, + }, } # What the device says about the card: the network's name and the last four. NOT_A_DISPLAY_NAME = { @@ -178,6 +185,10 @@ "an-email": "jane.roe@example.com", "a-digit-in-the-name": "Visa2 1234", "a-space-first": " Visa 1234", + # A name that reads as a CVV or other SAD: a CID is four digits (review r3, 3). + "a-cvv-name": "CVV 1234", + "a-security-code-name": "Security Code 1234", + "a-pin-name": "PIN 1234", } PACKS = [["default"], sorted(ALL_PACKS)] From 350f017b123df1003bbbadaa1835a4c46703906e Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Fri, 2 Oct 2026 07:29:20 +0300 Subject: [PATCH 21/22] docs(masking): the guard's three exempt leaves, an XML tag's length, the hex letter The README names the canonical UUID as the guard's third exempt leaf and rewrites that paragraph; the text floor reads an XML tag of any length; the docstrings and ecsctx/CLAUDE.md say a skipped hex id holds a hex letter, and the stale "Not holds_pan_run" line is gone (#160102). Co-Authored-By: Claude Opus 5.5 --- README.md | 38 +++++++++++++++++++---------------- ecsctx/CLAUDE.md | 2 +- ecsctx/masking/patterns.py | 2 +- ecsctx/masking/value_rules.py | 4 ++-- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ebff98e..6bb7b17 100644 --- a/README.md +++ b/README.md @@ -1475,22 +1475,24 @@ Two things no keep rule overrides, not even `KeepRule(lambda value: True)`: (`holds_pan_run`), it holds a run of 12 digits or more joined by single separators at most: `"card 4111…"`, `"1234 4111…"`, `"4111… 123"`, Track 2 data, 20 bare digits, `"4111…+cvv+123"` in a base64 slot, `"4111…ab"` in a - hex one. Two leaves hold none: a leaf that is thirteen bare digits from a 1 - or a 2 (epoch milliseconds until 2065), Google Pay's `keyExpiration`, which - `pan_shaped` reads as a card number and every Google Pay token carries; and - a leaf that is a whole hex id of 24 characters or more — a 64-hex digest - such as Apple Pay's `transactionId`, an ObjectId — about one random 64-hex - id in two hundred holds a 13-19 digit run that passes Luhn, and read like - other text it cost 1.1% of Apple Pay tokens. Only a whole leaf holding a - hex letter: a hex id inside base64 or prose (`"AA/4111…deadbeef/AA"`), - digits alone however many, and epoch milliseconds inside a longer string - are read. A leaf that is no JSON value counts as + hex one. Three leaves are not read for one: a leaf that is thirteen bare + digits from a 1 or a 2 (epoch milliseconds until 2065), Google Pay's + `keyExpiration`, which `pan_shaped` reads as a card number and every Google + Pay token carries; a leaf that is a canonical UUID (8-4-4-4-12 hex with a + hex letter), as everywhere in ecsctx; and a leaf that is a whole hex id of + 24 characters or more holding a hex letter, such as Apple Pay's 64-hex + `transactionId`: about one random 64-hex id in two hundred holds a 13-19 + digit run that passes Luhn, and reading them cost 1.1% of Apple Pay tokens. + Each is the whole leaf only: a hex id inside base64 or prose + (`"AA/4111…deadbeef/AA"`), digits alone however many, and epoch milliseconds + inside a longer string are all read. A leaf that is no JSON value counts as card data. What the guard cannot tell from a token's own text ships with it: a CVV or other short value written inside a longer string (`"note": "cvv=123"`, or `cvv+123` inside base64, which reads as ciphertext), and a card number written into a leaf that is a whole hex id - (`4111…` and eight hex letters). Keeping that out is the matcher's job, - which is why a matcher must be strict about what it matches. + (`4111…` then hex holding a letter, a CVV's digits among it). Keeping that + out is the matcher's job, which is why a matcher must be strict about what + it matches: Ottu's refuse hex alone in a base64 slot. Nothing else in a kept value is read: PII or a credential inside it, under a key of its own or written in a string, ships with it. @@ -1532,8 +1534,9 @@ nothing, and is masked as without a keep rule. A name is a quoted key then padding inside a string), an XML tag, or a `{name, value}` pair's label (`"name": "card security code"`), classified as the key walk classifies a key — every pack on, the service's safe keys honoured. It is read as -written: a quoted key, tag or pair label up to 128 characters (a longer key -by its last 128, before `:` or `=>`), with no `\uXXXX` escape decoded, in the +written: a quoted key or pair label up to 128 characters (a longer key by +its last 128, before `:` or `=>`), an XML tag of any length, with no +`\uXXXX` escape decoded, in the text as it stands and, where it holds an entity, as the entities decode. The key walk reads more: it parses JSON, so a key written with escapes (`"\u0063vv"`), or longer than 128 characters with its CVV word first, is a @@ -1577,8 +1580,8 @@ with strict matchers — an Apple Pay `PKPaymentToken`, its `paymentData` (`version` `EC_v1` or `RSA_v1`, the encrypted `data`, a `header`) and a Google Pay payment method token (`protocolVersion` `ECv1`, `ECv2` or `ECv2SigningOnly`, with a `signedMessage`), no key besides the ones each -writes, and every leaf text in its alphabet: standard base64 for a key, a -signature or ciphertext (`data`, `signature`, the header's +writes, and every leaf text in its alphabet: standard base64, never hex +alone, for a key, a signature or ciphertext (`data`, `signature`, the header's `ephemeralPublicKey`, `publicKeyHash` and `wrappedKey`, Google Pay's `signatures`, and the fields of its `signedMessage` and `signedKey`, each JSON text of all its fields and nothing else), hex for an id or a hash @@ -1587,7 +1590,8 @@ JSON text of all its fields and nothing else), hex for an id or a hash text: its `network` and `type` at most 64 characters, its `displayName` a letter and up to 39 more letters, spaces, `.`, `&` or `-`, optionally followed by one space and exactly four digits, the card's last four -(`Visa 0492`, `American Express`, `Amex`). A token holding anything else in a +(`Visa 0492`, `American Express`, `Amex`), and no name that reads as a CVV +or other SAD (`CVV 1234`: a CID is four digits). A token holding anything else in a slot — `"data": "cvv=123"`, a spaced card number as its `signature`, `"displayName": "cvv 123"` — is not kept, and is masked as before; nor is one with a card number in a slot's own alphabet (`"4111…+cvv+123"` in diff --git a/ecsctx/CLAUDE.md b/ecsctx/CLAUDE.md index ce866af..9cbd27f 100644 --- a/ecsctx/CLAUDE.md +++ b/ecsctx/CLAUDE.md @@ -28,7 +28,7 @@ Core structured logging module with ECS field mapping and PII/PCI masking. ## Footguns - `SAFE_KEYS` whitelist (`masking/patterns.py`) prevents masking of non-PII fields containing "name" etc. (e.g., `service_name`, `pathname`); checked before every other key rule. It holds only names that mean the same in every service — never a service's own vocabulary. A service lists its own in `ECSCTX_MASK_SAFE_KEYS` / `configure_masking_safe_keys()` (`masking/config.py`, resolved like the packs, passed to `classify_key` as `safe`); anything the classifier calls a card, CVV, SAD or credential is refused (`never_safe` asks `classify_key`; expiry is not refused, since nothing masks it). A listed key also frees a digits-only reference number of up to 14 digits from the content rules when the key is not PII on its own and not inside a card/credential/CVV/SAD container (15-19 digits always truncate). Ottu's list is `ecsctx.contrib.ottu.masking.SAFE_KEYS`, opted into by each service, never installed implicitly -- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number's digits inside a text leaf that is no JSON text (`patterns.holds_card_run`: `holds_pan_run`, 12+ digits joined by single separators, as the pci rules read a value, one linear pass with a bytes-translate fast path for base64; not read in a leaf that is a whole hex id of 24+ characters, `_HEX_ID` fullmatch -- Apple Pay's 64-hex ids, 1.1% of which hold a Luhn run -- nor a leaf that is epoch milliseconds); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). +- Value rules (`masking/value_rules.py`): a service's own shapes of value, masked as their label (label rules) or shipped as sent (keep rules) wherever they are logged; ecsctx names none. `ECSCTX_MASK_VALUE_RULES` / `configure_masking_value_rules()` (`masking/config.py`, resolved like the safe keys: rule objects, or dotted import paths to a rule or a collection, comma-separated in the env var; an item that is not one -- a rule with both a `field_type` and `keep = True` included -- raises from the call, is dropped with a warning from the setting or env var, and fails the boot check). A label rule has `field_type` (the label, `make_label`), `matches(mapping)` and optional `hints` (literal strings its text holds one of: a rule is never asked about text holding none, `applicable`; the key walk of JSON text passes on only those, `_mask_json_text`). A keep rule (`KeepRule`, or `keep is True` with a callable `matches`) writes no label: what it matches ships as sent. One list, asked in its order, the first match winning (`_first_match`; each rule's kind is cached on the tuple's identity, `_split`) -- for the value itself: a value strictly inside a keep match whose own first match is a label rule (listed before any keep rule matching it) is labelled and the keep refused (`keep_refused`: the guard, or `label_within` over its members, asked only with a label rule among the rules, a label rule's error refusing the keep and raising nothing; called from `ruling` and rule 2). The engine asks them about every mapping the key walk meets under any key, every JSON-text string (`_value_ruling` -> label, `KEEP` or None; JSON text holding a label match deeper under a key that would hash it is the label, `label_within`, which looks into a keep match like any mapping, a kept one holding no label: JSON text holding a kept value deeper is still hashed whole), every JSON object in text (rule 2 for label rules, `_OBJECT_TEXT`: JSON, a repr, or JSON in a JSON string, three levels deep; an object no rule matches is read member by member) and every such object in a body (`redact_body` -> `patterns.mask_objects`, first). A label match is the rule's label, never hashed. A keep match is a deep copy in the key walk, not descended into; in text `mask_by_patterns` replaces it with a `[KEPT--MASKED]` placeholder before every rule (`stash_kept`/`kept_spans`, top down: a label match is no span; and no span at all in text that names a CVV/SAD key anywhere -- `floored_text`, read only once a span is found: a quoted key then `:`/`=`/`=>`, a bare key then `:`/`=>`, a key and `=` where a key starts (not base64 padding in a string), an XML tag, or a `{name, value}` pair's label, `classify_key(..., ALL_PACKS, safe keys)`, as written and entity-decoded; fails closed, a sibling's value too, since the text rules read objects only three levels deep; `redact_body` passes it to an entity-encoded leaf, `_encoded_leaf(keep=False)`) and restores each placeholder still there; a placeholder is named from a blake2b digest of its value (`_placeholder`, 14 letters: the same value one name in any text, so a credential hashed around it carries that value in its token, and `restore` fills only its own names, so a name written into the text restores nothing; two values whose names collide are masked as without a keep rule); a keep decision on text rests on the text as written (`json_as_written` refuses a key written twice, `_written_as_parsed` keeps a repr only where it renders back exactly); restored text is not recorded clean, and text holding `[KEPT-` is masked as before. Never kept: under a CVV/SAD key, inside a CVV/SAD container (a listed key there reads through with the label rules only, `_without_keep`), the `value` of a pair labelled as one (`_floored`); nor a match holding a card/CVV/SAD key, a `{name, value}` pair labelled as one, or a card number at any depth, JSON text inside included -- keys, pairs, whole leaves, and a card number's digits inside a text leaf that is no JSON text (`patterns.holds_card_run`: `holds_pan_run`, 12+ digits joined by single separators, as the pci rules read a value, one linear pass with a bytes-translate fast path for base64; not read in a leaf that is a whole hex id of 24+ characters holding a hex letter, `_HEX_ID` fullmatch -- Apple Pay's 64-hex ids; Ottu's base64 slots refuse hex alone, 1.1% of which hold a Luhn run -- nor a leaf that is epoch milliseconds); a CVV or other short value inside a longer string is the matcher's to keep out (`holds_card_data`: a string leaf is a card number when `pan_shaped`, either way except 13 bare digits from a 1 or 2 -- epoch milliseconds, Google Pay's `keyExpiration`, which `pan_shaped` reads as one), which is walked as if nothing matched. A keep matcher that raises is no match (it runs in `redact_url`/`redact_body`/`mask_card_value`); a label matcher's error leaves the marker. `is_kept`/`mask_outside_kept` (`ecsctx.masking`) serve a service's own pass after ecsctx's. Configuring the value rules empties `patterns._clean`. Nothing configured, nothing runs: the rules are looked up only when a mapping, JSON text or text holding `{` asks, at most once a call in the key walk (`_Pass.values`). - CVV never goes through tokenization at all, even when PII is configured — the CVV rules return the literal `[CVV-MASKED]` directly. PCI forbids storing CVV in any form, not even as an HMAC digest. - SSN/PEM tokens are computed over a normalized value (digits-only for SSNs, base64-body-only for PEM), not the raw matched text — otherwise the same value in a different grouping/wrapping would produce a different token and break correlation. Card numbers carry no token at all: they truncate to first 6 + last 4 from 15 digits up and last 4 only below (`411111******1111`, bare, PCI DSS 3.5.1, FAQ 1091), separators stripped, independent of PII configuration. The star run is the marker: `already_masked()` and `_text_has_card_context()` both recognise that shape, since there is no wrapper left to read. The card rule (`_CardRun`) reads a whole run of digit groups at once: where a separator admits one card or a card beside another number, Luhn decides, and every Luhn-valid 12-19-digit stretch along the groups lies inside a truncated span (overlapping ones merged), so no output shows more than its first six and last four -- except over digits it leaves as something else's: an IBAN's (`_iban_end`: its country's registered length from `_IBAN_LENGTHS`, in groups of four joined by single spaces, check digits holding; after a letter bank code only if the IBAN ends where the run does), a word's own or a truncation's last four (unless they start a Luhn-valid card in card-style groups, which makes them a card's), a phone number's after "+" (at most 15 digits, E.164, even a card's), a range's across a Unicode dash (no stretch crosses it; a Unicode dash is a separator only when the whole run is card-style groups), and a canonical UUID's (`_UUID`; the run is read again without them, `_truncated_outside`). Accepted residuals follow from those: a card glued to a word or to `****` in non-card-style groups, one split by a range's dash, a card completing an IBAN of its country's exact length with check digits holding, a card typed into a UUID's own groups (`41111111-1111-1111-abcd-ef0123456789` stays whole in free text, and under a name key with a keyset it is hashed), and the phone rule (it runs first) taking phone-shaped digits at a card's end (ten before a Unicode dash, or its last digits with those after them). A later pass leaves a truncation's first six and last four alone. A card key never shows those residuals: it refuses instead, since `mask_card_value` shows its scan only while fewer than twelve digits show outside the truncations the card rule writes, however they are joined -- double spaces, commas, another masker's `X`s (`_digits_beside_truncations`, group by group: a search for the shape from every star of a long run was quadratic). Nor may twelve digits show in a row, however joined (`_card_digits_showing`, one running count): bare digits, a truncation's last four and the next one's first six count together, as a card number could be read across them; a truncation's own stars separate its first six from its last four; a word ends a row, but a run of `X`/`x` joins digits as stars do (another masker's `4508750XXX001019`). So `4111111111 111111******1111`, `400001******5018 00 000009******0001` (the Maestro 501800000009), `********4111 ********1111 1111`, `02313379 ********1262`, `411111******3782XX82246310005`, `512345******0008 12 25 512345******0008` and `4111111111111111 12345678901` are refused. The one residual left: a short group beside a truncation, under twelve digits with it (`450875******1019 000` could be a 19-digit number another masker cut oddly). - No dict-level "already masked" marker: a dict re-passed to `_mask_dict()` (e.g. `mask_sensitive_data` running twice in one chain) is fully re-traversed. Idempotency lives in the leaf checks instead — `mask_by_field_type` leaves an already-`[LABEL…]` value untouched via `already_masked()`, and a second content-regex pass over masked markers is a noop, so `_mask_string` always runs the full pass with no early-exit (a whole-string `already_masked()` gate there would let a coincidental `-MASKED]` fragment suppress masking for real PII in the same string). Only `filter()` marks anything, and only on the `LogRecord` (`mark_object_as_masked`/`is_masked_object`, a plain attribute that never ships as output), so a record reaching two handlers skips masking on the second pass. diff --git a/ecsctx/masking/patterns.py b/ecsctx/masking/patterns.py index ac81f9b..98a186b 100644 --- a/ecsctx/masking/patterns.py +++ b/ecsctx/masking/patterns.py @@ -832,7 +832,7 @@ def holds_card_run(text: str) -> bool: (``holds_pan_run``: 12 digits or more, joined by single separators at most -- `card 4111…`, `1234 4111…`, `4111… 123`, `4111…+cvv+123` in base64, `4111…ab` in hex) -- except a leaf that is a whole hex id of 24 - characters or more (``_HEX_ID``). For the guard + characters or more holding a hex letter (``_HEX_ID``). For the guard (``value_rules.holds_card_data``), which excepts a leaf that is epoch milliseconds itself. One pass, linear in the leaf: ASCII text with no such run of digits and no separator is told by one bytes translate.""" diff --git a/ecsctx/masking/value_rules.py b/ecsctx/masking/value_rules.py index 66b977c..b020f86 100644 --- a/ecsctx/masking/value_rules.py +++ b/ecsctx/masking/value_rules.py @@ -359,8 +359,8 @@ def holds_card_data(value: Any) -> bool: ever be kept. Any other text leaf that is no JSON text holds one when ``holds_card_run`` finds a run of 12 digits or more in it, read as the pci rules read a value -- except a leaf that is a whole hex id of 24 - characters or more (JSON text is read by its own leaves). - Not ``holds_pan_run``, which flags hex ids. + characters or more holding a hex letter, which ``holds_pan_run`` alone + would read (JSON text is read by its own leaves). Anything that is no JSON value (an object whose text is not judged here), nesting past the depth cap, and a leaf of JSON text that names a key twice (``json_as_written``: what ships would not be what was read) count as From 9efb3a6a30fc9b7b950139724d6668fcd3c8248e Mon Sep 17 00:00:00 2001 From: Dacian Popute Date: Fri, 2 Oct 2026 08:14:33 +0300 Subject: [PATCH 22/22] fix(contrib): an Apple Pay token's network and type are words, as its display name A PKPaymentToken's paymentMethod network and type were any text of 64 characters at most, so `"network": "cvv=123"` or an email in `type` shipped with the kept token: the guard reads keys, pairs and card numbers, not a CVV written in a string. Both are now words alone or nothing (letters, spaces, `.`, `&`, `-`, from a letter, 40 at most) that read as no CVV or other SAD, as the display name already was (bot review on #72). Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++++++------- ecsctx/contrib/ottu/masking.py | 37 ++++++++++++++++-------------- tests/test_contrib_ottu_wallets.py | 23 +++++++++++++++++++ 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 6bb7b17..48e3569 100644 --- a/README.md +++ b/README.md @@ -1586,14 +1586,14 @@ alone, for a key, a signature or ciphertext (`data`, `signature`, the header's `signatures`, and the fields of its `signedMessage` and `signedKey`, each JSON text of all its fields and nothing else), hex for an id or a hash (`transactionId`, `applicationData`, `transactionIdentifier`), digits for -`keyExpiration`, and for what `paymentMethod` says about the card short -text: its `network` and `type` at most 64 characters, its `displayName` a -letter and up to 39 more letters, spaces, `.`, `&` or `-`, optionally -followed by one space and exactly four digits, the card's last four -(`Visa 0492`, `American Express`, `Amex`), and no name that reads as a CVV -or other SAD (`CVV 1234`: a CID is four digits). A token holding anything else in a -slot — `"data": "cvv=123"`, a spaced card number as its `signature`, -`"displayName": "cvv 123"` — is not kept, and is masked as before; nor is +`keyExpiration`, and for what `paymentMethod` says about the card words: a +letter and up to 39 more letters, spaces, `.`, `&` or `-` — its `network` +and `type` that alone or nothing (`AmEx`, `debit`), its `displayName` +optionally followed by one space and exactly four digits, the card's last +four (`Visa 0492`, `American Express`) — and no words that read as a CVV or +other SAD (`cvv`, `CVV 1234`: a CID is four digits). A token holding +anything else in a slot — `"data": "cvv=123"`, a spaced card number as its +`signature`, `"network": "cvv=123"` — is not kept, and is masked as before; nor is one with a card number in a slot's own alphabet (`"4111…+cvv+123"` in base64), which the guard reads. What neither can tell from ciphertext ships with a kept token: a CVV or other short value written in a slot's own diff --git a/ecsctx/contrib/ottu/masking.py b/ecsctx/contrib/ottu/masking.py index 6e8d8e6..2c83725 100644 --- a/ecsctx/contrib/ottu/masking.py +++ b/ecsctx/contrib/ottu/masking.py @@ -149,7 +149,6 @@ def _is_google_pay_token(value: Any) -> bool: # besides these is no wallet's, and the object is walked as before. _PAYMENT_DATA_KEYS = frozenset({"version", "data", "signature", "header"}) _PK_PAYMENT_TOKEN_KEYS = frozenset({"paymentData", "paymentMethod", "transactionIdentifier"}) -_PAYMENT_METHOD_KEYS = frozenset({"displayName", "network", "type"}) _GOOGLE_PAY_KEYS = frozenset({"signature", "intermediateSigningKey", "protocolVersion", "signedMessage"}) _SIGNING_KEY_KEYS = frozenset({"signedKey", "signatures"}) _SIGNED_MESSAGE_KEYS = frozenset({"encryptedMessage", "ephemeralPublicKey", "tag"}) @@ -172,14 +171,19 @@ def _is_google_pay_token(value: Any) -> bool: "transactionId": _HEX, "applicationData": _HEX, } -# PKPaymentToken's `paymentMethod` is what the device says about the card: -# its network and type, short text, and a display name: up to 40 letters, -# spaces, `.`, `&` and `-` from a letter, and optionally a space and the -# card's last four ("Visa 0492", "American Express", "Amex") -- no other -# digit or sign, so neither a card number nor an email rides in it, and no -# name that reads as a CVV or other SAD: a CID is four digits ("CVV 1234"). -_PAYMENT_METHOD_TEXT_LIMIT = 64 -_DISPLAY_NAME = re.compile(r"[A-Za-z][A-Za-z .&-]{0,39}(?: [0-9]{4})?") +# PKPaymentToken's `paymentMethod` is what the device says about the card, in +# words: up to 40 letters, spaces, `.`, `&` and `-` from a letter. Its network +# and type are words alone, or nothing ("AmEx", "debit"); its display name +# words and optionally a space and the card's last four ("Visa 0492", +# "American Express"). No other digit or sign, so neither a card number nor an +# email rides in them, and no words that read as a CVV or other SAD: a CID is +# four digits ("CVV 1234"). +_CARD_WORDS = r"[A-Za-z][A-Za-z .&-]{0,39}" +_PAYMENT_METHOD_ALPHABETS = { + "displayName": re.compile(rf"{_CARD_WORDS}(?: [0-9]{{4}})?"), + "network": re.compile(rf"(?:{_CARD_WORDS})?"), + "type": re.compile(rf"(?:{_CARD_WORDS})?"), +} def _written_in(field: Any, alphabet: re.Pattern) -> bool: @@ -210,16 +214,16 @@ def _is_apple_pay_payment_data(value: Any) -> bool: ) -def _is_display_name(name: Any) -> bool: - if not (isinstance(name, str) and _DISPLAY_NAME.fullmatch(name)): +def _in_card_words(field: Any, alphabet: re.Pattern) -> bool: + if not _written_in(field, alphabet): return False - return classify_key(name.rstrip("0123456789 "), ALL_PACKS) not in _FLOOR_TYPES + return classify_key(field.rstrip("0123456789 "), ALL_PACKS) not in _FLOOR_TYPES def _is_pk_payment_token(value: Any) -> bool: """Apple Pay's PKPaymentToken: its `paymentData`, what the device says - about the card -- its network and type, each short text, and its display - name, letters and the last four -- and the transaction's id in hex.""" + about the card -- its network, type and display name in words, the name + with the last four -- and the transaction's id in hex.""" payment_data = value.get("paymentData") if not (isinstance(payment_data, dict) and _is_apple_pay_payment_data(payment_data)): return False @@ -228,9 +232,8 @@ def _is_pk_payment_token(value: Any) -> bool: method = value.get("paymentMethod", {}) return ( isinstance(method, dict) - and method.keys() <= _PAYMENT_METHOD_KEYS - and ("displayName" not in method or _is_display_name(method["displayName"])) - and all(isinstance(field, str) and len(field) <= _PAYMENT_METHOD_TEXT_LIMIT for field in method.values()) + and method.keys() <= _PAYMENT_METHOD_ALPHABETS.keys() + and all(_in_card_words(field, _PAYMENT_METHOD_ALPHABETS[key]) for key, field in method.items()) ) diff --git a/tests/test_contrib_ottu_wallets.py b/tests/test_contrib_ottu_wallets.py index 5413304..a9d4c51 100644 --- a/tests/test_contrib_ottu_wallets.py +++ b/tests/test_contrib_ottu_wallets.py @@ -190,6 +190,18 @@ "a-security-code-name": "Security Code 1234", "a-pin-name": "PIN 1234", } +# A network or a type is words alone, or nothing (bot review on #72, 1). +NOT_A_NETWORK = { + "a-cvv-setting": "cvv=123", + "a-cvv": "CVV 123", + "a-cvv-name": "cvv", + "a-security-code-name": "Security Code", + "an-email": "john@example.com", + "the-last-four": "Visa 1234", + "a-digit": "1", + "a-space-first": " Visa", + "too-long": "V" * 41, +} PACKS = [["default"], sorted(ALL_PACKS)] @@ -395,6 +407,14 @@ def test_a_display_name_is_a_network_and_the_last_four(self, name): # Its payment data is still kept, on its own. assert _walk({"token": token})["token"]["paymentData"] == APPLE_PAY + @pytest.mark.parametrize("field", ["network", "type"]) + @pytest.mark.parametrize("text", NOT_A_NETWORK.values(), ids=NOT_A_NETWORK.keys()) + def test_a_network_and_a_type_are_words(self, field, text): + token = {**PK_PAYMENT_TOKEN, "paymentMethod": {**PK_PAYMENT_TOKEN["paymentMethod"], field: text}} + assert not is_kept(token) + assert not is_kept(json.dumps(token)) + assert _walk({"token": token})["token"]["paymentData"] == APPLE_PAY + @pytest.mark.parametrize("token", NOT_TOKEN_TEXT.values(), ids=NOT_TOKEN_TEXT.keys()) def test_a_token_with_text_no_wallet_writes_is_masked_as_without_the_rules(self, token, packs): def every_path(): @@ -420,6 +440,8 @@ def every_path(): {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 35 + " 1234"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "V" * 40 + " 1234"}}, {**PK_PAYMENT_TOKEN, "paymentMethod": {"displayName": "American Express 1234", "network": "AmEx"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"network": "CartesBancaires", "type": "prepaid"}}, + {**PK_PAYMENT_TOKEN, "paymentMethod": {"network": "girocard", "type": ""}}, # A 64-hex id holding a Luhn-valid run of digits, as about one in # two hundred does: an id, not a card number (fix round 2). {**APPLE_PAY, "header": {**APPLE_PAY["header"], "transactionId": "a" * 10 + "4111111111111111" + "b" * 38}}, @@ -429,6 +451,7 @@ def every_path(): "apple-application-data", "pk-display-name", "google-signature-plus-and-slash", "pk-display-name-amex", "pk-display-name-connects-fixture", "pk-display-name-forty-characters", "pk-display-name-forty-letters-and-the-last-four", "pk-display-name-in-words", + "pk-network-and-type", "pk-network-and-no-type", "apple-transaction-id-with-a-luhn-run", "pk-transaction-identifier-with-a-luhn-run", ], )