From 3eac89316055b5ff6f3180004e04c6c9e4872699 Mon Sep 17 00:00:00 2001 From: bijay-odyssey Date: Tue, 18 Aug 2026 23:41:54 +0545 Subject: [PATCH] Add batch support to /anonymize and /deanonymize endpoints presidio-analyzer's /analyze endpoint already accepts a list of texts and returns a matching list of results. presidio-anonymizer's /anonymize and /deanonymize never got the same treatment, even though the core package already ships BatchAnonymizerEngine and BatchDeanonymizeEngine unused by the HTTP layer. Extend both endpoints so 'text' can be a string (existing behavior, unchanged) or an array of strings. In array mode, 'analyzer_results' / 'anonymizer_results' must be an array of the same length, one list of results per text, and the response becomes a matching array of results. A length mismatch raises a 422 with a clear message instead of failing silently or misaligning results. Adds tests/test_app.py (no HTTP-level test file existed for this service before), covering single-text behavior is unchanged, batch anonymize/deanonymize including a full encrypt/decrypt round trip, missing-results defaulting, and length-mismatch validation. Updates docs/api-docs/api-docs.yml (request/response schemas + batch examples) to match how /analyze's existing batch support is already documented there. Closes #1045 --- docs/api-docs/api-docs.yml | 96 ++++++++++++--- presidio-anonymizer/app.py | 74 +++++++++++- presidio-anonymizer/tests/test_app.py | 163 ++++++++++++++++++++++++++ 3 files changed, 315 insertions(+), 18 deletions(-) create mode 100644 presidio-anonymizer/tests/test_app.py diff --git a/docs/api-docs/api-docs.yml b/docs/api-docs/api-docs.yml index 6561e31e53..6fbe213e65 100644 --- a/docs/api-docs/api-docs.yml +++ b/docs/api-docs/api-docs.yml @@ -178,7 +178,13 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/AnonymizeResponse" + oneOf: + - description: "An anonymization result (when input text is a single string)" + $ref: "#/components/schemas/AnonymizeResponse" + - description: "A list of anonymization results (when input text is an array of strings)" + type: array + items: + $ref: "#/components/schemas/AnonymizeResponse" examples: Replace and Redact Anonymizers: value: @@ -186,6 +192,12 @@ paths: Replace as default Anonymizer: value: { "text": "hello world, my name is ANONYMIZED. My number is: ANONYMIZED", "items": [ { "operator": "replace", "entity_type": "PHONE_NUMBER", "start": 50, "end": 60, "text": "ANONYMIZED" }, { "operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text": "ANONYMIZED" } ] } + Batch response (array of texts): + value: + [ + { "text": "hello world, my name is ANONYMIZED.", "items": [ { "operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text": "ANONYMIZED" } ] }, + { "text": "call me at ANONYMIZED", "items": [ { "operator": "replace", "entity_type": "PHONE_NUMBER", "start": 11, "end": 21, "text": "ANONYMIZED" } ] } + ] 400: $ref: "#/components/responses/400BadRequest" @@ -232,11 +244,23 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DeanonymizeResponse" + oneOf: + - description: "A deanonymization result (when input text is a single string)" + $ref: "#/components/schemas/DeanonymizeResponse" + - description: "A list of deanonymization results (when input text is an array of strings)" + type: array + items: + $ref: "#/components/schemas/DeanonymizeResponse" examples: Decrypt Single PII: value: { "text": "text_for_encryption", "items": [ { "start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type": "NUMBER" } ] } + Batch response (array of texts): + value: + [ + { "text": "text_for_encryption_1", "items": [ { "start": 0, "end": 22, "operator": "decrypt", "text": "text_for_encryption_1", "entity_type": "NUMBER" } ] }, + { "text": "text_for_encryption_2", "items": [ { "start": 0, "end": 22, "operator": "decrypt", "text": "text_for_encryption_2", "entity_type": "NUMBER" } ] } + ] 400: $ref: "#/components/responses/400BadRequest" @@ -362,6 +386,18 @@ components: { "start": 24, "end": 32, "score": 0.8, "entity_type": "PERSON" }, ] } + Batch Request (array of texts): + value: + { + "text": ["hello world, my name is Jane Doe.", "call me at 034453334"], + "anonymizers": { + "DEFAULT": { "type": "replace", "new_value": "ANONYMIZED" } + }, + "analyzer_results": [ + [ { "start": 24, "end": 32, "score": 0.8, "entity_type": "PERSON" } ], + [ { "start": 11, "end": 20, "score": 0.95, "entity_type": "PHONE_NUMBER" } ] + ] + } DeanonymizeRequest: required: true @@ -386,6 +422,18 @@ components: "entity_type": "PERSON" } ] } + Batch Request (array of texts): + value: + { + "text": ["S184CMt9Drj7QaKQ21JTrpYzghnboTF9pn/neN8JME0=", "aQ21JTrpYzghnboTF9pn/nOMt9Drj7QaK4CE0M="], + "deanonymizers": { + "PERSON": { "type": "decrypt", "key": "WmZq4t7w!z%C&F)J" } + }, + "anonymizer_results": [ + [ { "start": 11, "end": 55, "entity_type": "PERSON" } ], + [ { "start": 0, "end": 38, "entity_type": "PERSON" } ] + ] + } schemas: @@ -442,8 +490,12 @@ components: - analyzer_results properties: text: - type: string - description: "The text to anonymize" + oneOf: + - type: string + - type: array + items: + type: string + description: "The text to anonymize. Can be a single string or an array of strings; when an array, 'analyzer_results' must be an array of the same length, one list of results per text." example: "hello world, my name is Jane Doe. My number is: 034453334" anonymizers: description: "Object where the key is DEFAULT or the ENTITY_TYPE and the value is the anonymizer definition" @@ -459,10 +511,16 @@ components: { "DEFAULT": { "type": "replace", "new_value": "" } } analyzer_results: - type: array - description: "Array of analyzer detections" - items: - $ref: "#/components/schemas/RecognizerResult" + description: "Array of analyzer detections (single text), or an array of such arrays, one per text (batch request)" + oneOf: + - type: array + items: + $ref: "#/components/schemas/RecognizerResult" + - type: array + items: + type: array + items: + $ref: "#/components/schemas/RecognizerResult" DeanonymizeRequest: type: object @@ -472,8 +530,12 @@ components: - deanonymizers properties: text: - type: string - description: "The anonymized text" + oneOf: + - type: string + - type: array + items: + type: string + description: "The anonymized text. Can be a single string or an array of strings; when an array, 'anonymizer_results' must be an array of the same length, one list of results per text." example: "My name is S184CMt9Drj7QaKQ21JTrpYzghnboTF9pn/neN8JME0=" deanonymizers: description: "Object where the key is DEFAULT or the ENTITY_TYPE and the value is decrypt since it is the only one supported" @@ -484,10 +546,16 @@ components: default: { "DEFAULT": { "type": "decrypt", "key": "3t6w9z$C&F)J@NcR" } } anonymizer_results: - type: array - description: "Array of anonymized PIIs" - items: - $ref: "#/components/schemas/OperatorResult" + description: "Array of anonymized PIIs (single text), or an array of such arrays, one per text (batch request)" + oneOf: + - type: array + items: + $ref: "#/components/schemas/OperatorResult" + - type: array + items: + type: array + items: + $ref: "#/components/schemas/OperatorResult" RecognizerResult: diff --git a/presidio-anonymizer/app.py b/presidio-anonymizer/app.py index 52f75f41a0..45cac0ce8d 100644 --- a/presidio-anonymizer/app.py +++ b/presidio-anonymizer/app.py @@ -1,5 +1,6 @@ """REST API server for anonymizer.""" +import json import logging import os from logging.config import fileConfig @@ -57,11 +58,33 @@ def anonymize() -> Response: if AppEntitiesConvertor.check_custom_operator(anonymizers_config): raise BadRequest("Custom type anonymizer is not supported") + text = content.get("text", "") + if isinstance(text, list): + results = [ + self.anonymizer.anonymize( + text=item_text, + analyzer_results=AppEntitiesConvertor.analyzer_results_from_json( + item_analyzer_results + ), + operators=anonymizers_config, + ) + for item_text, item_analyzer_results in zip( + text, + _batch_items( + "analyzer_results", content.get("analyzer_results"), text + ), + ) + ] + return Response( + json.dumps(results, default=lambda o: o.__dict__), + mimetype="application/json", + ) + analyzer_results = AppEntitiesConvertor.analyzer_results_from_json( content.get("analyzer_results") ) anoymizer_result = self.anonymizer.anonymize( - text=content.get("text", ""), + text=text, analyzer_results=analyzer_results, operators=anonymizers_config, ) @@ -73,12 +96,36 @@ def deanonymize() -> Response: if not content: raise BadRequest("Invalid request json") text = content.get("text", "") - deanonymize_entities = AppEntitiesConvertor.deanonymize_entities_from_json( - content - ) deanonymize_config = AppEntitiesConvertor.operators_config_from_json( content.get("deanonymizers") ) + + if isinstance(text, list): + results = [ + self.deanonymize.deanonymize( + text=item_text, + entities=AppEntitiesConvertor.deanonymize_entities_from_json( + {"anonymizer_results": item_entities} + ), + operators=deanonymize_config, + ) + for item_text, item_entities in zip( + text, + _batch_items( + "anonymizer_results", + content.get("anonymizer_results"), + text, + ), + ) + ] + return Response( + json.dumps(results, default=lambda o: o.__dict__), + mimetype="application/json", + ) + + deanonymize_entities = AppEntitiesConvertor.deanonymize_entities_from_json( + content + ) deanonymized_response = self.deanonymize.deanonymize( text=text, entities=deanonymize_entities, operators=deanonymize_config ) @@ -112,6 +159,25 @@ def server_error(e): self.logger.error(f"A fatal error occurred during execution: {e}") return jsonify(error="Internal server error"), 500 +def _batch_items(field_name, value, texts): + """Align a per-item results field with a batch ``text`` list. + + :param field_name: name of the request field, used in the error message. + :param value: the raw request value for that field (expected to be a list + of lists, one per item in ``texts``), or ``None`` if omitted. + :param texts: the batch ``text`` list the request is being validated against. + """ + if value is None: + return [[] for _ in texts] + if len(value) != len(texts): + raise InvalidParamError( + f"Invalid input, '{field_name}' must contain one list of results " + f"per item in 'text' when 'text' is a list ({len(texts)} text " + f"items, {len(value)} '{field_name}' items)" + ) + return value + + def create_app(): # noqa server = Server() return server.app diff --git a/presidio-anonymizer/tests/test_app.py b/presidio-anonymizer/tests/test_app.py new file mode 100644 index 0000000000..01fe733ca9 --- /dev/null +++ b/presidio-anonymizer/tests/test_app.py @@ -0,0 +1,163 @@ +import json + +import pytest + +from app import create_app + + +@pytest.fixture +def client(): + app = create_app() + app.config["TESTING"] = True + with app.test_client() as client: + yield client + + +def test_anonymize_single_text_unchanged(client): + request_body = { + "text": "hello world, my name is Jane Doe.", + "anonymizers": {"DEFAULT": {"type": "replace", "new_value": "ANONYMIZED"}}, + "analyzer_results": [ + {"start": 24, "end": 32, "score": 0.8, "entity_type": "NAME"} + ], + } + + response = client.post("/anonymize", json=request_body) + body = json.loads(response.data) + + assert response.status_code == 200 + assert isinstance(body, dict) + assert body["text"] == "hello world, my name is ANONYMIZED." + + +def test_anonymize_batch_returns_one_result_per_text(client): + request_body = { + "text": [ + "hello world, my name is Jane Doe.", + "call me at 034453334", + ], + "anonymizers": { + "DEFAULT": {"type": "replace", "new_value": "ANONYMIZED"}, + }, + "analyzer_results": [ + [{"start": 24, "end": 32, "score": 0.8, "entity_type": "NAME"}], + [{"start": 11, "end": 20, "score": 0.9, "entity_type": "PHONE_NUMBER"}], + ], + } + + response = client.post("/anonymize", json=request_body) + body = json.loads(response.data) + + assert response.status_code == 200 + assert isinstance(body, list) + assert len(body) == 2 + assert body[0]["text"] == "hello world, my name is ANONYMIZED." + assert body[1]["text"] == "call me at ANONYMIZED" + + +def test_anonymize_batch_missing_analyzer_results_leaves_text_unchanged(client): + request_body = { + "text": ["hello world!", "nice to meet you"], + "anonymizers": {"DEFAULT": {"type": "replace", "new_value": "ANONYMIZED"}}, + } + + response = client.post("/anonymize", json=request_body) + body = json.loads(response.data) + + assert response.status_code == 200 + assert [item["text"] for item in body] == ["hello world!", "nice to meet you"] + assert all(item["items"] == [] for item in body) + + +def test_anonymize_batch_length_mismatch_returns_422(client): + request_body = { + "text": ["one", "two", "three"], + "anonymizers": {"DEFAULT": {"type": "replace", "new_value": "X"}}, + "analyzer_results": [[]], + } + + response = client.post("/anonymize", json=request_body) + body = json.loads(response.data) + + assert response.status_code == 422 + assert "analyzer_results" in body["error"] + + +def test_deanonymize_single_text_unchanged(client): + key = "1111111111111111" + encrypt_body = { + "text": "my number is 034453334", + "anonymizers": {"DEFAULT": {"type": "encrypt", "key": key}}, + "analyzer_results": [ + {"start": 13, "end": 22, "score": 0.9, "entity_type": "PHONE_NUMBER"} + ], + } + encrypted = json.loads(client.post("/anonymize", json=encrypt_body).data) + + decrypt_body = { + "text": encrypted["text"], + "deanonymizers": {"DEFAULT": {"type": "decrypt", "key": key}}, + "anonymizer_results": [ + { + "start": item["start"], + "end": item["end"], + "entity_type": item["entity_type"], + } + for item in encrypted["items"] + ], + } + response = client.post("/deanonymize", json=decrypt_body) + body = json.loads(response.data) + + assert response.status_code == 200 + assert body["text"] == "my number is 034453334" + + +def test_deanonymize_batch_returns_one_result_per_text(client): + key = "1111111111111111" + encrypt_body = { + "text": ["number is 034453334", "number is 099998888"], + "anonymizers": {"DEFAULT": {"type": "encrypt", "key": key}}, + "analyzer_results": [ + [{"start": 10, "end": 19, "score": 0.9, "entity_type": "PHONE_NUMBER"}], + [{"start": 10, "end": 19, "score": 0.9, "entity_type": "PHONE_NUMBER"}], + ], + } + encrypted = json.loads(client.post("/anonymize", json=encrypt_body).data) + + decrypt_body = { + "text": [item["text"] for item in encrypted], + "deanonymizers": {"DEFAULT": {"type": "decrypt", "key": key}}, + "anonymizer_results": [ + [ + { + "start": item["items"][0]["start"], + "end": item["items"][0]["end"], + "entity_type": item["items"][0]["entity_type"], + } + ] + for item in encrypted + ], + } + + response = client.post("/deanonymize", json=decrypt_body) + body = json.loads(response.data) + + assert response.status_code == 200 + assert isinstance(body, list) + assert body[0]["text"] == "number is 034453334" + assert body[1]["text"] == "number is 099998888" + + +def test_deanonymize_batch_length_mismatch_returns_422(client): + request_body = { + "text": ["one", "two"], + "deanonymizers": {"DEFAULT": {"type": "decrypt", "key": "1111111111111111"}}, + "anonymizer_results": [[]], + } + + response = client.post("/deanonymize", json=request_body) + body = json.loads(response.data) + + assert response.status_code == 422 + assert "anonymizer_results" in body["error"]