Skip to content

Commit 6116c06

Browse files
kshvgkshvg
authored
feat(anonymizer): expose confidence score in OperatorResult (#2059)
1 parent 2bb88d2 commit 6116c06

7 files changed

Lines changed: 149 additions & 33 deletions

File tree

‎e2e-tests/tests/test_api_anonymizer.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ def test_given_anonymize_called_with_valid_request_then_expected_valid_response_
3131
{
3232
"text": "hello world, my name is ANONYMIZED. My number is: 03445****",
3333
"items": [
34-
{"operator": "mask", "entity_type": "PHONE_NUMBER", "start": 50, "end": 59, "text":"03445****"},
35-
{"operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text":"ANONYMIZED"}
34+
{"operator": "mask", "entity_type": "PHONE_NUMBER", "start": 50, "end": 59, "text":"03445****", "score": 0.95},
35+
{"operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text":"ANONYMIZED", "score": 0.8}
3636
]
3737
}
3838
"""
@@ -146,7 +146,7 @@ def test_given_decrypt_called_with_encrypted_text_then_decrypted_text_returned()
146146

147147
response_status, response_content = deanonymize(json.dumps(request_body))
148148

149-
expected_response = """{"text": "text_for_encryption", "items": [{"start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type":"NUMBER"}]}"""
149+
expected_response = """{"text": "text_for_encryption", "items": [{"start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type":"NUMBER", "score": null}]}"""
150150

151151
assert response_status == 200
152152
assert equal_json_strings(expected_response, response_content)
@@ -298,8 +298,8 @@ def test_keep_name():
298298
{
299299
"text": "hello world, my name is Jane Doe. My number is: <PHONE_NUMBER>",
300300
"items": [
301-
{"operator": "replace", "entity_type": "PHONE_NUMBER", "start": 48, "end": 62, "text":"<PHONE_NUMBER>"},
302-
{"operator": "keep", "entity_type": "NAME", "start": 24, "end": 32, "text":"Jane Doe"}
301+
{"operator": "replace", "entity_type": "PHONE_NUMBER", "start": 48, "end": 62, "text":"<PHONE_NUMBER>", "score": 0.95},
302+
{"operator": "keep", "entity_type": "NAME", "start": 24, "end": 32, "text":"Jane Doe", "score": 0.8}
303303
]
304304
}
305305
"""
@@ -330,8 +330,8 @@ def test_overlapping_keep_first():
330330
{
331331
"text": "I'm George Washington<LOCATION>",
332332
"items": [
333-
{"operator": "replace", "entity_type": "LOCATION", "start": 21, "end": 31, "text":"<LOCATION>"},
334-
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington"}
333+
{"operator": "replace", "entity_type": "LOCATION", "start": 21, "end": 31, "text":"<LOCATION>", "score": 0.8},
334+
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington", "score": 0.8}
335335
]
336336
}
337337
"""
@@ -362,8 +362,8 @@ def test_overlapping_keep_second():
362362
{
363363
"text": "I'm <NAME>Washington Square Park",
364364
"items": [
365-
{"operator": "keep", "entity_type": "LOCATION", "start": 10, "end": 32, "text":"Washington Square Park"},
366-
{"operator": "replace", "entity_type": "NAME", "start": 4, "end": 10, "text":"<NAME>"}
365+
{"operator": "keep", "entity_type": "LOCATION", "start": 10, "end": 32, "text":"Washington Square Park", "score": 0.8},
366+
{"operator": "replace", "entity_type": "NAME", "start": 4, "end": 10, "text":"<NAME>", "score": 0.8}
367367
]
368368
}
369369
"""
@@ -393,8 +393,8 @@ def test_overlapping_keep_both():
393393
{
394394
"text": "I'm George WashingtonWashington Square Park",
395395
"items": [
396-
{"operator": "keep", "entity_type": "LOCATION", "start": 21, "end": 43, "text":"Washington Square Park"},
397-
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington"}
396+
{"operator": "keep", "entity_type": "LOCATION", "start": 21, "end": 43, "text":"Washington Square Park", "score": 0.8},
397+
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington", "score": 0.8}
398398
]
399399
}
400400
"""

‎e2e-tests/tests/test_api_e2e_integration_flows.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def test_given_text_with_pii_then_analyze_and_anonymize_successfully():
5656
"analyzer_results": analyzer_data,
5757
}
5858

59-
expected_response = """{"text": "<PERSON> drivers license is AC43****", "items": [{"operator": "mask", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 36, "text": "AC43****"}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
59+
expected_response = """{"text": "<PERSON> drivers license is AC43****", "items": [{"operator": "mask", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 36, "text": "AC43****", "score": 0.6499999999999999}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
6060

6161
anonymize_and_assert(anonymizer_request, expected_response)
6262

@@ -96,7 +96,7 @@ def test_given_a_correct_analyze_input_high_threashold_then_anonymize_partially(
9696
"analyzer_results": analyzer_data,
9797
}
9898

99-
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
99+
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
100100

101101
anonymize_and_assert(anonymizer_request, expected_response)
102102

@@ -142,7 +142,7 @@ def test_given_a_correct_analyze_input_with_high_threshold_and_unmatched_entitie
142142
"analyzer_results": analyzer_data,
143143
}
144144

145-
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
145+
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
146146

147147
anonymize_and_assert(anonymizer_request, expected_response)
148148

@@ -177,7 +177,7 @@ def test_given_an_unknown_entity_then_anonymize_uses_defaults():
177177
"analyzer_results": analyzer_data,
178178
}
179179

180-
expected_response = """{"text": "<PERSON> drivers license is <US_DRIVER_LICENSE>", "items": [{"operator": "replace", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 47, "text": "<US_DRIVER_LICENSE>"}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
180+
expected_response = """{"text": "<PERSON> drivers license is <US_DRIVER_LICENSE>", "items": [{"operator": "replace", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 47, "text": "<US_DRIVER_LICENSE>", "score": 0.6499999999999999}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
181181

182182
anonymize_and_assert(anonymizer_request, expected_response)
183183

‎presidio-anonymizer/presidio_anonymizer/core/engine_base.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ def _operate(
7373
entity.entity_type,
7474
changed_text,
7575
operator_metadata.operator_name,
76+
score=getattr(entity, "score", None),
7677
)
7778
engine_result.add_item(result_item)
7879

‎presidio-anonymizer/presidio_anonymizer/entities/engine/pii_entity.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,15 @@ def __init__(self, start: int, end: int, entity_type: str):
2222

2323
def __repr__(self):
2424
"""Return a string representation of the object."""
25-
return (
26-
f"start: {self.start}"
27-
f"end: {self.end},"
25+
repr_str = (
26+
f"start: {self.start}, "
27+
f"end: {self.end}, "
2828
f"entity_type: {self.entity_type}"
2929
)
30+
score = getattr(self, "score", None)
31+
if score is not None:
32+
repr_str = repr_str + f", score: {score}"
33+
return repr_str
3034

3135
def __gt__(self, other):
3236
"""Check one entity is greater then other by the text end index."""

‎presidio-anonymizer/presidio_anonymizer/entities/engine/result/operator_result.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Dict
1+
from typing import Dict, Optional
22

33
from presidio_anonymizer.entities import PIIEntity
44

@@ -13,18 +13,20 @@ def __init__(
1313
entity_type: str,
1414
text: str = None,
1515
operator: str = None,
16+
score: Optional[float] = None,
1617
):
1718
PIIEntity.__init__(self, start, end, entity_type)
1819
self.text = text
1920
self.operator = operator
21+
self.score = score
2022

2123
def __repr__(self):
2224
"""Return a string representation of the object."""
2325
return str(self.to_dict())
2426

2527
def to_dict(self) -> Dict:
2628
"""Return object as Dict."""
27-
return self.__dict__
29+
return dict(self.__dict__)
2830

2931
def __str__(self):
3032
"""Return a string representation of the object."""
@@ -58,17 +60,20 @@ def from_json(cls, json: Dict) -> "OperatorResult":
5860
"entity_type":"PERSON",
5961
"text":"resulted_text",
6062
"operator":"encrypt",
63+
"score": 0.85
6164
}
6265
"""
6366
start = json.get("start")
6467
end = json.get("end")
6568
entity_type = json.get("entity_type")
6669
text = json.get("text")
6770
operator = json.get("operator")
71+
score = json.get("score")
6872
return cls(
6973
start=start,
7074
end=end,
7175
entity_type=entity_type,
7276
text=text,
7377
operator=operator,
78+
score=float(score) if score is not None else None,
7479
)

‎presidio-anonymizer/tests/integration/test_anonymize_engine.py‎

Lines changed: 102 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import re
2+
import json
23

34
import pytest
45

@@ -27,7 +28,7 @@ def test_given_url_at_the_end_then_we_redact_is_successfully():
2728
]
2829
expected_result = (
2930
'{"text": "The url is ", "items": [{"start": 11, "end": 11, "entity_type": '
30-
'"URL", "text": "", "operator": "redact"}]}'
31+
'"URL", "text": "", "operator": "redact", "score": 1.0}]}'
3132
)
3233
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
3334

@@ -63,9 +64,9 @@ def test_given_name_and_phone_number_then_we_anonymize_correctly():
6364
expected_result = (
6465
'{"text": "hello world, my name is ********. My number is: '
6566
'03-******4", "items": [{"start": 48, "end": 57, "entity_type": '
66-
'"PHONE_NUMBER", "text": "03-******", "operator": "mask"}, '
67+
'"PHONE_NUMBER", "text": "03-******", "operator": "mask", "score": 0.95}, '
6768
'{"start": 24, "end": 32, "entity_type": "NAME", '
68-
'"text": "********", "operator": "mask"}]}'
69+
'"text": "********", "operator": "mask", "score": 0.8}]}'
6970
)
7071
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
7172

@@ -85,9 +86,9 @@ def test_given_name_and_phone_number_without_anonymizers_then_we_use_default():
8586
'{"text": "hello world, my name is <NAME>. My number is: '
8687
'<PHONE_NUMBER>4", "items": [{"start": 46, "end": 60, '
8788
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
88-
'"operator": "replace"}, {"start": 24, "end": 30, '
89+
'"operator": "replace", "score": 0.95}, {"start": 24, "end": 30, '
8990
'"entity_type": "NAME", "text": "<NAME>", '
90-
'"operator": "replace"}]}'
91+
'"operator": "replace", "score": 0.8}]}'
9192
)
9293
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
9394

@@ -106,9 +107,9 @@ def test_given_redact_and_replace_then_we_anonymize_successfully():
106107
'{"text": "hello world, my name is . My number is: '
107108
'<PHONE_NUMBER>4", "items": [{"start": 40, "end": 54, '
108109
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
109-
'"operator": "replace"}, {"start": 24, "end": 24, '
110+
'"operator": "replace", "score": 0.95}, {"start": 24, "end": 24, '
110111
'"entity_type": "NAME", "text": "", "operator": '
111-
'"redact"}]}'
112+
'"redact", "score": 0.8}]}'
112113
)
113114
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
114115

@@ -128,13 +129,13 @@ def test_given_intersecting_entities_then_we_anonymize_correctly():
128129
'{"text": "hello world, my name is <FULL_NAME><LAST_NAME> My '
129130
'number is: <PHONE_NUMBER><SSN>4", "items": [{"start": 75, '
130131
'"end": 80, "entity_type": "SSN", "text": "<SSN>", '
131-
'"operator": "replace"}, {"start": 61, "end": 75, '
132+
'"operator": "replace", "score": 0.8}, {"start": 61, "end": 75, '
132133
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
133-
'"operator": "replace"}, {"start": 35, "end": 46, '
134+
'"operator": "replace", "score": 0.95}, {"start": 35, "end": 46, '
134135
'"entity_type": "LAST_NAME", "text": "<LAST_NAME>", '
135-
'"operator": "replace"}, {"start": 24, "end": 35, '
136+
'"operator": "replace", "score": 0.6}, {"start": 24, "end": 35, '
136137
'"entity_type": "FULL_NAME", "text": "<FULL_NAME>", '
137-
'"operator": "replace"}]}'
138+
'"operator": "replace", "score": 0.6}]}'
138139
)
139140
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
140141

@@ -149,7 +150,7 @@ def test_given_intersecting_the_same_entities_then_we_anonymize_correctly():
149150
expected_result = (
150151
'{"text": "hello world, my name is <FULL_NAME> My number is: 03-4453334", '
151152
'"items": [{"start": 24, "end": 35, "entity_type": "FULL_NAME",'
152-
' "text": "<FULL_NAME>", "operator": "replace"}]}'
153+
' "text": "<FULL_NAME>", "operator": "replace", "score": 0.6}]}'
153154
)
154155
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
155156

@@ -318,7 +319,6 @@ def test_when_hash_with_user_provided_salt_then_hash_is_reproducible():
318319
)
319320
def test_hash_with_known_salt_produces_expected_output(text, salt, hash_type, expected_hash):
320321
"""Test that hashing with a known salt produces expected deterministic output."""
321-
from presidio_anonymizer import AnonymizerEngine
322322

323323
params = {"hash_type": hash_type, "salt": salt}
324324
anonymizer_config = {"DEFAULT": OperatorConfig("hash", params)}
@@ -339,6 +339,95 @@ def test_hash_with_known_salt_produces_expected_output(text, salt, hash_type, ex
339339
assert result2.items[0].text == expected_hash
340340

341341

342+
def test_given_single_entity_then_score_is_propagated_to_result():
343+
"""Score from analyzer result appears on the corresponding OperatorResult."""
344+
text = "My name is Jane Doe"
345+
anonymizer_config = {"PERSON": OperatorConfig("replace")}
346+
analyzer_results = [
347+
RecognizerResult(start=11, end=19, score=0.85, entity_type="PERSON"),
348+
]
349+
350+
engine = AnonymizerEngine()
351+
result = engine.anonymize(text, analyzer_results, anonymizer_config)
352+
353+
assert result.items[0].score == 0.85
354+
355+
356+
def test_given_multiple_entities_then_each_score_is_propagated_correctly():
357+
"""Each OperatorResult carries the score of its originating RecognizerResult."""
358+
text = "My name is Jane Doe. My number is 034453334"
359+
anonymizer_config = {}
360+
analyzer_results = [
361+
RecognizerResult(start=11, end=19, score=0.8, entity_type="NAME"),
362+
RecognizerResult(start=34, end=43, score=0.95, entity_type="PHONE_NUMBER"),
363+
]
364+
365+
engine = AnonymizerEngine()
366+
result = engine.anonymize(text, analyzer_results, anonymizer_config)
367+
368+
result_by_type = {item.entity_type: item for item in result.items}
369+
assert result_by_type["NAME"].score == 0.8
370+
assert result_by_type["PHONE_NUMBER"].score == 0.95
371+
372+
373+
@pytest.mark.parametrize(
374+
"operator_name,operator_params",
375+
[
376+
("replace", {}),
377+
("redact", {}),
378+
("mask", {"masking_char": "*", "chars_to_mask": 4, "from_end": False}),
379+
("hash", {}),
380+
],
381+
)
382+
def test_given_different_operators_then_score_is_always_propagated(
383+
operator_name, operator_params
384+
):
385+
"""Score is propagated regardless of which operator is applied."""
386+
text = "My name is Jane Doe"
387+
anonymizer_config = {"NAME": OperatorConfig(operator_name, operator_params)}
388+
analyzer_results = [
389+
RecognizerResult(start=11, end=19, score=0.75, entity_type="NAME"),
390+
]
391+
392+
engine = AnonymizerEngine()
393+
result = engine.anonymize(text, analyzer_results, anonymizer_config)
394+
395+
assert result.items[0].score == 0.75
396+
397+
398+
def test_given_conflicting_entities_then_winning_entity_score_is_preserved():
399+
"""When conflict resolution drops an entity, the surviving entity keeps its own score."""
400+
text = "hello world, my name is Jane Doe"
401+
anonymizer_config = {}
402+
analyzer_results = [
403+
RecognizerResult(start=24, end=32, score=0.6, entity_type="FULL_NAME"),
404+
RecognizerResult(start=24, end=28, score=0.9, entity_type="FIRST_NAME"), # loses
405+
RecognizerResult(start=24, end=30, score=0.8, entity_type="NAME"), # loses
406+
]
407+
408+
engine = AnonymizerEngine()
409+
result = engine.anonymize(text, analyzer_results, anonymizer_config)
410+
411+
assert len(result.items) == 1
412+
assert result.items[0].entity_type == "FULL_NAME"
413+
assert result.items[0].score == 0.6 # its own score, not the score of dropped entities
414+
415+
416+
def test_given_score_in_result_then_it_is_present_in_json_output():
417+
"""Score is included in to_json() serialized output."""
418+
text = "My name is Jane Doe"
419+
anonymizer_config = {}
420+
analyzer_results = [
421+
RecognizerResult(start=11, end=19, score=0.85, entity_type="NAME"),
422+
]
423+
424+
engine = AnonymizerEngine()
425+
result = engine.anonymize(text, analyzer_results, anonymizer_config)
426+
427+
output = json.loads(result.to_json())
428+
assert output["items"][0]["score"] == pytest.approx(0.85)
429+
430+
342431
def run_engine_and_validate(
343432
text: str, anonymizers_config, analyzer_results, expected_result
344433
):

‎presidio-anonymizer/tests/test_pii_entity.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from presidio_anonymizer.entities import PIIEntity
2+
3+
4+
def test_pii_entity_repr_includes_score_when_present():
5+
entity = PIIEntity(start=0, end=5, entity_type="PERSON")
6+
entity.score = 0.85
7+
result = repr(entity)
8+
assert "score: 0.85" in result
9+
assert "start: 0" in result
10+
assert "entity_type: PERSON" in result
11+
12+
13+
def test_pii_entity_repr_omits_score_when_absent():
14+
entity = PIIEntity(start=0, end=5, entity_type="PERSON")
15+
result = repr(entity)
16+
assert "score" not in result
17+
assert "start: 0" in result

0 commit comments

Comments
 (0)