Skip to content

Add batch support to /anonymize and /deanonymize endpoints #2233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bijay-odyssey wants to merge 1 commit into
base: main
Choose a base branch
from
+315 −18
Open

Add batch support to /anonymize and /deanonymize endpoints #2233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 82 additions & 14 deletions docs/api-docs/api-docs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,14 +178,26 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/AnonymizeResponse"
oneOf:
- description: "An anonymization result (when input text is a single string)"
$ref: "#/components/schemas/AnonymizeResponse"
- description: "A list of anonymization results (when input text is an array of strings)"
type: array
items:
$ref: "#/components/schemas/AnonymizeResponse"
examples:
Replace and Redact Anonymizers:
value:
{ "text": "hello world, my name is ANONYMIZED. My number is: ", "items": [ { "operator": "redact", "entity_type": "PHONE_NUMBER", "start": 50, "end": 50, "text": "" }, { "operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text": "ANONYMIZED" } ] }
Replace as default Anonymizer:
value:
{ "text": "hello world, my name is ANONYMIZED. My number is: ANONYMIZED", "items": [ { "operator": "replace", "entity_type": "PHONE_NUMBER", "start": 50, "end": 60, "text": "ANONYMIZED" }, { "operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text": "ANONYMIZED" } ] }
Batch response (array of texts):
value:
[
{ "text": "hello world, my name is ANONYMIZED.", "items": [ { "operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text": "ANONYMIZED" } ] },
{ "text": "call me at ANONYMIZED", "items": [ { "operator": "replace", "entity_type": "PHONE_NUMBER", "start": 11, "end": 21, "text": "ANONYMIZED" } ] }
]
400:
$ref: "#/components/responses/400BadRequest"

Expand Down Expand Up @@ -232,11 +244,23 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/DeanonymizeResponse"
oneOf:
- description: "A deanonymization result (when input text is a single string)"
$ref: "#/components/schemas/DeanonymizeResponse"
- description: "A list of deanonymization results (when input text is an array of strings)"
type: array
items:
$ref: "#/components/schemas/DeanonymizeResponse"
examples:
Decrypt Single PII:
value:
{ "text": "text_for_encryption", "items": [ { "start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type": "NUMBER" } ] }
Batch response (array of texts):
value:
[
{ "text": "text_for_encryption_1", "items": [ { "start": 0, "end": 22, "operator": "decrypt", "text": "text_for_encryption_1", "entity_type": "NUMBER" } ] },
{ "text": "text_for_encryption_2", "items": [ { "start": 0, "end": 22, "operator": "decrypt", "text": "text_for_encryption_2", "entity_type": "NUMBER" } ] }
]
400:
$ref: "#/components/responses/400BadRequest"

Expand Down Expand Up @@ -362,6 +386,18 @@ components:
{ "start": 24, "end": 32, "score": 0.8, "entity_type": "PERSON" },
]
}
Batch Request (array of texts):
value:
{
"text": ["hello world, my name is Jane Doe.", "call me at 034453334"],
"anonymizers": {
"DEFAULT": { "type": "replace", "new_value": "ANONYMIZED" }
},
"analyzer_results": [
[ { "start": 24, "end": 32, "score": 0.8, "entity_type": "PERSON" } ],
[ { "start": 11, "end": 20, "score": 0.95, "entity_type": "PHONE_NUMBER" } ]
]
}

DeanonymizeRequest:
required: true
Expand All @@ -386,6 +422,18 @@ components:
"entity_type": "PERSON"
} ]
}
Batch Request (array of texts):
value:
{
"text": ["S184CMt9Drj7QaKQ21JTrpYzghnboTF9pn/neN8JME0=", "aQ21JTrpYzghnboTF9pn/nOMt9Drj7QaK4CE0M="],
"deanonymizers": {
"PERSON": { "type": "decrypt", "key": "WmZq4t7w!z%C&F)J" }
},
"anonymizer_results": [
[ { "start": 11, "end": 55, "entity_type": "PERSON" } ],
[ { "start": 0, "end": 38, "entity_type": "PERSON" } ]
]
}


schemas:
Expand Down Expand Up @@ -442,8 +490,12 @@ components:
- analyzer_results
properties:
text:
type: string
description: "The text to anonymize"
oneOf:
- type: string
- type: array
items:
type: string
description: "The text to anonymize. Can be a single string or an array of strings; when an array, 'analyzer_results' must be an array of the same length, one list of results per text."
example: "hello world, my name is Jane Doe. My number is: 034453334"
anonymizers:
description: "Object where the key is DEFAULT or the ENTITY_TYPE and the value is the anonymizer definition"
Expand All @@ -459,10 +511,16 @@ components:
{ "DEFAULT": { "type": "replace", "new_value": "<ENTITY_TYPE>" } }

analyzer_results:
type: array
description: "Array of analyzer detections"
items:
$ref: "#/components/schemas/RecognizerResult"
description: "Array of analyzer detections (single text), or an array of such arrays, one per text (batch request)"
oneOf:
- type: array
items:
$ref: "#/components/schemas/RecognizerResult"
- type: array
items:
type: array
items:
$ref: "#/components/schemas/RecognizerResult"

DeanonymizeRequest:
type: object
Expand All @@ -472,8 +530,12 @@ components:
- deanonymizers
properties:
text:
type: string
description: "The anonymized text"
oneOf:
- type: string
- type: array
items:
type: string
description: "The anonymized text. Can be a single string or an array of strings; when an array, 'anonymizer_results' must be an array of the same length, one list of results per text."
example: "My name is S184CMt9Drj7QaKQ21JTrpYzghnboTF9pn/neN8JME0="
deanonymizers:
description: "Object where the key is DEFAULT or the ENTITY_TYPE and the value is decrypt since it is the only one supported"
Expand All @@ -484,10 +546,16 @@ components:
default:
{ "DEFAULT": { "type": "decrypt", "key": "3t6w9z$C&F)J@NcR" } }
anonymizer_results:
type: array
description: "Array of anonymized PIIs"
items:
$ref: "#/components/schemas/OperatorResult"
description: "Array of anonymized PIIs (single text), or an array of such arrays, one per text (batch request)"
oneOf:
- type: array
items:
$ref: "#/components/schemas/OperatorResult"
- type: array
items:
type: array
items:
$ref: "#/components/schemas/OperatorResult"


RecognizerResult:
Expand Down
74 changes: 70 additions & 4 deletions presidio-anonymizer/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""REST API server for anonymizer."""

import json
import logging
import os
from logging.config import fileConfig
Expand Down Expand Up @@ -57,11 +58,33 @@ def anonymize() -> Response:
if AppEntitiesConvertor.check_custom_operator(anonymizers_config):
raise BadRequest("Custom type anonymizer is not supported")

text = content.get("text", "")
if isinstance(text, list):
results = [
self.anonymizer.anonymize(
text=item_text,
analyzer_results=AppEntitiesConvertor.analyzer_results_from_json(
item_analyzer_results
),
operators=anonymizers_config,
)
for item_text, item_analyzer_results in zip(
text,
_batch_items(
"analyzer_results", content.get("analyzer_results"), text
),
)
]
return Response(
json.dumps(results, default=lambda o: o.__dict__),
mimetype="application/json",
)

analyzer_results = AppEntitiesConvertor.analyzer_results_from_json(
content.get("analyzer_results")
)
anoymizer_result = self.anonymizer.anonymize(
text=content.get("text", ""),
text=text,
analyzer_results=analyzer_results,
operators=anonymizers_config,
)
Expand All @@ -73,12 +96,36 @@ def deanonymize() -> Response:
if not content:
raise BadRequest("Invalid request json")
text = content.get("text", "")
deanonymize_entities = AppEntitiesConvertor.deanonymize_entities_from_json(
content
)
deanonymize_config = AppEntitiesConvertor.operators_config_from_json(
content.get("deanonymizers")
)

if isinstance(text, list):
results = [
self.deanonymize.deanonymize(
text=item_text,
entities=AppEntitiesConvertor.deanonymize_entities_from_json(
{"anonymizer_results": item_entities}
),
operators=deanonymize_config,
)
for item_text, item_entities in zip(
text,
_batch_items(
"anonymizer_results",
content.get("anonymizer_results"),
text,
),
)
]
return Response(
json.dumps(results, default=lambda o: o.__dict__),
mimetype="application/json",
)

deanonymize_entities = AppEntitiesConvertor.deanonymize_entities_from_json(
content
)
deanonymized_response = self.deanonymize.deanonymize(
text=text, entities=deanonymize_entities, operators=deanonymize_config
)
Expand Down Expand Up @@ -112,6 +159,25 @@ def server_error(e):
self.logger.error(f"A fatal error occurred during execution: {e}")
return jsonify(error="Internal server error"), 500

def _batch_items(field_name, value, texts):
"""Align a per-item results field with a batch ``text`` list.

:param field_name: name of the request field, used in the error message.
:param value: the raw request value for that field (expected to be a list
of lists, one per item in ``texts``), or ``None`` if omitted.
:param texts: the batch ``text`` list the request is being validated against.
"""
if value is None:
return [[] for _ in texts]
if len(value) != len(texts):
raise InvalidParamError(
f"Invalid input, '{field_name}' must contain one list of results "
f"per item in 'text' when 'text' is a list ({len(texts)} text "
f"items, {len(value)} '{field_name}' items)"
)
return value


def create_app(): # noqa
server = Server()
return server.app
Expand Down
Loading