From ad347a4440698e1de0871f60e817e059ba89b433 Mon Sep 17 00:00:00 2001 From: jansaldo Date: Fri, 22 May 2026 16:40:15 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fix=20handling=20of=20non-alphan?= =?UTF-8?q?umeric=20entities=20by=20returning=20None=20for=20empty=20clean?= =?UTF-8?q?ed=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transforms/anonymization_postprocess/core.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/aymurai/transforms/anonymization_postprocess/core.py b/aymurai/transforms/anonymization_postprocess/core.py index 977fb109..bd596d29 100644 --- a/aymurai/transforms/anonymization_postprocess/core.py +++ b/aymurai/transforms/anonymization_postprocess/core.py @@ -1,10 +1,9 @@ import re from copy import deepcopy -from string import punctuation +from aymurai.meta.pipeline_interfaces import Transform from aymurai.meta.types import DataItem from aymurai.utils.misc import get_element -from aymurai.meta.pipeline_interfaces import Transform class AnonymizationEntityCleaner(Transform): @@ -45,6 +44,9 @@ def process(self, ent: dict) -> dict: # Clean the text cleaned_text = pattern.sub("", original_text) + if not cleaned_text: + return None + # Update the entity's alt text and indices ent["attrs"]["aymurai_alt_text"] = cleaned_text ent["attrs"]["aymurai_alt_start_char"] = start_char + leading_chars_removed @@ -61,11 +63,11 @@ def __call__(self, item: DataItem) -> DataItem: DataItem: processed item """ item = deepcopy(item) - ents = get_element(item, [self.field, "entities"]) or [] - # Filter out predictions that are punctuation marks only - ents = [ent for ent in ents if ent["text"] not in punctuation] - ents = [self.process(ent) for ent in ents] + # Filter out predictions with empty alt text and update the rest + item[self.field]["entities"] = [ + out for ent in ents if (out := self.process(ent)) is not None + ] return item