Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -341,11 +341,13 @@ def add_pattern_recognizer_from_dict(self, recognizer_dict: Dict) -> None:
""" # noqa: E501

recognizer_config = recognizer_dict.copy()
has_score_thresholds = "score_thresholds" in recognizer_config
score_thresholds = normalize_score_thresholds(
recognizer_config.pop("score_thresholds", None)
)
recognizer = PatternRecognizer.from_dict(recognizer_config)
recognizer.score_thresholds = score_thresholds
if has_score_thresholds:
recognizer.score_thresholds = score_thresholds
self.add_recognizer(recognizer)

def add_recognizers_from_yaml(self, yml_path: Union[str, Path]) -> None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,11 @@ def get(
}
custom_to_exclude = {"enabled", "type", "class_name", "score_thresholds"}
for recognizer_conf in predefined:
# Only override when the key is present. An absent key means the
# configuration says nothing about thresholds, so the value the
# recognizer class set for itself must stand. Assigning the
# normalized ``None`` (an empty mapping) would silently discard it.
has_score_thresholds = "score_thresholds" in recognizer_conf
score_thresholds = normalize_score_thresholds(
recognizer_conf.get("score_thresholds")
)
Expand Down Expand Up @@ -432,11 +437,13 @@ def get(
)

recognizer = recognizer_cls(**kwargs)
recognizer.score_thresholds = score_thresholds
if has_score_thresholds:
recognizer.score_thresholds = score_thresholds
recognizer_instances.append(recognizer)

for recognizer_conf in custom:
if RecognizerListLoader.is_recognizer_enabled(recognizer_conf):
has_score_thresholds = "score_thresholds" in recognizer_conf
score_thresholds = normalize_score_thresholds(
recognizer_conf.get("score_thresholds")
)
Expand All @@ -447,8 +454,9 @@ def get(
recognizer_conf=new_conf,
supported_languages=supported_languages,
)
for recognizer in custom_recognizers:
recognizer.score_thresholds = score_thresholds
if has_score_thresholds:
for recognizer in custom_recognizers:
recognizer.score_thresholds = score_thresholds
recognizer_instances.extend(custom_recognizers)

for recognizer_conf in recognizer_instances:
Expand Down
118 changes: 118 additions & 0 deletions presidio-analyzer/tests/test_recognizer_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
RecognizerRegistry,
)
from presidio_analyzer.predefined_recognizers import SpacyRecognizer, UsSsnRecognizer
from presidio_analyzer.recognizer_registry.recognizers_loader_utils import (
RecognizerListLoader,
)


def create_mock_pattern_recognizer(lang, entity, name):
Expand Down Expand Up @@ -225,6 +228,121 @@ def test_add_recognizers_from_yaml_attaches_thresholds(tmp_path):
}


class RecognizerWithOwnThresholds(PatternRecognizer):
"""Recognizer declaring its own thresholds, for omission tests."""

DEFAULT_THRESHOLDS = {"THRESHOLD_TEST": 0.6}

def __init__(self, supported_language: str = "en", **kwargs):
super().__init__(
supported_entity="THRESHOLD_TEST",
patterns=[Pattern("test", r"\bTHR\d{4}\b", 0.3)],
supported_language=supported_language,
**kwargs,
)
self.score_thresholds = dict(self.DEFAULT_THRESHOLDS)


@pytest.fixture
def threshold_recognizer_registered(monkeypatch):
"""Make the test recognizer resolvable by name from configuration."""
original = RecognizerListLoader.get_existing_recognizer_cls

def resolve(recognizer_name):
if recognizer_name == "RecognizerWithOwnThresholds":
return RecognizerWithOwnThresholds
return original(recognizer_name)

monkeypatch.setattr(
RecognizerListLoader, "get_existing_recognizer_cls", staticmethod(resolve)
)


def _load_predefined(conf):
return RecognizerListLoader.get(
recognizers=[conf],
global_regex_flags=re.DOTALL | re.MULTILINE | re.IGNORECASE,
supported_languages=["en"],
)


def test_predefined_recognizer_keeps_own_thresholds_when_yaml_omits_them(
threshold_recognizer_registered,
):
"""Omitting the key must not discard the recognizer's own thresholds."""
loaded = _load_predefined(
{
"name": "RecognizerWithOwnThresholds",
"supported_languages": ["en"],
"type": "predefined",
"enabled": True,
}
)

assert len(loaded) == 1
assert loaded[0].score_thresholds == {"THRESHOLD_TEST": 0.6}


def test_predefined_recognizer_thresholds_overridden_when_yaml_sets_them(
threshold_recognizer_registered,
):
"""An explicit value in configuration still wins over the class default."""
loaded = _load_predefined(
{
"name": "RecognizerWithOwnThresholds",
"supported_languages": ["en"],
"type": "predefined",
"enabled": True,
"score_thresholds": {"THRESHOLD_TEST": 0.9},
}
)

assert loaded[0].score_thresholds == {"THRESHOLD_TEST": 0.9}


def test_predefined_recognizer_thresholds_cleared_when_yaml_sets_empty_mapping(
threshold_recognizer_registered,
):
"""An explicit empty mapping is a deliberate reset, not an omission."""
loaded = _load_predefined(
{
"name": "RecognizerWithOwnThresholds",
"supported_languages": ["en"],
"type": "predefined",
"enabled": True,
"score_thresholds": {},
}
)

assert loaded[0].score_thresholds == {}


def test_add_recognizer_from_dict_keeps_own_thresholds_when_key_omitted(monkeypatch):
"""The dict path follows the same omission rule.

``PatternRecognizer.from_dict`` returns the base class today, whose
thresholds are always empty, so the constructed recognizer is replaced
with one carrying its own default to make the behavior observable.
"""
monkeypatch.setattr(
PatternRecognizer,
"from_dict",
staticmethod(lambda config: RecognizerWithOwnThresholds()),
)
registry = RecognizerRegistry()

registry.add_pattern_recognizer_from_dict(
{
"name": "Zip code Recognizer",
"supported_language": "de",
"patterns": [{"name": "zip", "regex": r"\d{5}", "score": 0.5}],
"supported_entity": "ZIP",
}
)

assert registry.recognizers[0].score_thresholds == {"THRESHOLD_TEST": 0.6}


@pytest.mark.parametrize("score_thresholds", [False, 0, "", []])
def test_add_recognizer_from_dict_rejects_falsey_non_mapping_thresholds(
score_thresholds,
Expand Down
Loading