refactor: move compare, evaluate_sample, and helpers to TokenEvaluator

compare(), __revert_known_errors(), _adjust_per_entities(), and
evaluate_sample() are token-level concerns only used by TokenEvaluator.
Move them out of BaseEvaluator and into TokenEvaluator where they belong.

- BaseEvaluator: remove compare, __revert_known_errors, _adjust_per_entities,
  evaluate_sample; inline _adjust_per_entities into deprecated get_results_dataframe;
  remove now-unused spacy Token import
- TokenEvaluator: add all four methods; add logging + spacy Token + ErrorType + ModelError imports
- MockEvaluator in test_evaluator.py: inherit from TokenEvaluator (was BaseEvaluator)
- Fix stale Evaluator(, ...) patterns across test files and docs (add model=None)

Author: omri374

docs/evaluation.md

@@ -105,7 +105,7 @@ from presidio_evaluator.models import PresidioAnalyzerWrapper
 from presidio_evaluator.entity_mapping import CanonicalMapper
 
 model = PresidioAnalyzerWrapper(analyzer_engine=analyzer)
-evaluator = SpanEvaluator(model=None)
+evaluator = SpanEvaluator()
 
 # 2. Get predictions as a DataFrame
 results_df = model.predict_dataset(dataset)
@@ -281,7 +281,7 @@ evaluation_result = EvaluationResult(
 )
 
 # Initialize evaluator and evaluate
-evaluator = TokenEvaluator(model=None)
+evaluator = TokenEvaluator()
 final_result = evaluator.calculate_score([evaluation_result])
 
 print(f"Precision: {final_result.pii_precision:.4f}")
@@ -316,7 +316,7 @@ evaluation_result = EvaluationResult(
 )
 
 # Initialize span evaluator with no skip words
-evaluator = SpanEvaluator(iou_threshold=0.5, model=None, skip_words=[])
+evaluator = SpanEvaluator(iou_threshold=0.5, , skip_words=[])
 scores = evaluator.calculate_score([evaluation_result])
 
 print(f"Precision: {scores.pii_precision:.4f}")
@@ -351,7 +351,7 @@ evaluation_result = EvaluationResult(
 )
 
 # Initialize span evaluator with no skip words
-evaluator = SpanEvaluator(iou_threshold=0.5, model=None)
+evaluator = SpanEvaluator(iou_threshold=0.5, )
 scores = evaluator.calculate_score([evaluation_result])
 
 print(f"Precision: {scores.pii_precision:.4f}")

presidio_evaluator/evaluation/base_evaluator.py

@@ -1,14 +1,12 @@
 import logging
 import warnings
 from abc import ABC, abstractmethod
-from collections import Counter
 
 import numpy as np
 import pandas as pd
-from spacy.tokens import Token
 
 from presidio_evaluator import InputSample
-from presidio_evaluator.evaluation import ErrorType, EvaluationResult, ModelError
+from presidio_evaluator.evaluation import EvaluationResult
 from presidio_evaluator.evaluation.skipwords import get_skip_words
 from presidio_evaluator.models import BaseModel
 
@@ -33,7 +31,7 @@ def __init__(
         """
         Evaluate PII detection results.
 
-        :param model: Must be None. Passing a model is no longer supported.
+        :param model: Deprecated. Must be None. Passing a model is no longer supported.
         Use model.predict_dataset(dataset) to obtain a results DataFrame,
         then pass it to calculate_score_on_df().
         :param verbose: Whether to print debug information
@@ -73,161 +71,6 @@ def __init__(
         else:
             self.skip_words = skip_words
 
-    def compare(
-        self,
-        input_sample: InputSample,
-        prediction: list[str],
-    ) -> tuple[Counter, list[ModelError]]:
-        """
-        Compares ground truth tags (annotation) and predicted (prediction)
-        :param input_sample: input sample containing list of tags
-        :param prediction: predicted value for each token
-        """
-        annotation = list(input_sample.tags)
-        tokens = input_sample.tokens
-
-        if len(annotation) != len(prediction):
-            logger.warning(
-                "Annotation and prediction do not have the"
-                f"same length. Sample={input_sample}",
-            )
-            return Counter(), []
-
-        results = Counter()
-        mistakes = []
-
-        if self.entities_to_keep:
-            prediction = self._adjust_per_entities(prediction)
-            annotation = self._adjust_per_entities(annotation)
-
-        for i in range(0, len(annotation)):
-            cur_token = tokens[i]
-            cur_prediction = prediction[i]
-            cur_annotation = annotation[i]
-
-            results[(cur_annotation, cur_prediction)] += 1
-
-            if self.verbose:
-                logger.info("Annotation: %s", cur_annotation)
-                logger.info("Prediction: %s", cur_prediction)
-                logger.info("Results: %s", results)
-
-            is_error = cur_annotation != cur_prediction
-
-            if is_error:
-                reverted = self.__revert_known_errors(
-                    cur_annotation,
-                    cur_prediction,
-                    cur_token,
-                    results,
-                )
-                if reverted:
-                    continue
-
-                if prediction[i] == "O":
-                    mistakes.append(
-                        ModelError(
-                            error_type=ErrorType.FN,
-                            annotation=cur_annotation,
-                            prediction=cur_prediction,
-                            token=cur_token,
-                            full_text=input_sample.full_text,
-                            metadata=input_sample.metadata,
-                        ),
-                    )
-                elif annotation[i] == "O":
-                    mistakes.append(
-                        ModelError(
-                            error_type=ErrorType.FP,
-                            annotation=cur_annotation,
-                            prediction=cur_prediction,
-                            token=cur_token,
-                            full_text=input_sample.full_text,
-                            metadata=input_sample.metadata,
-                        ),
-                    )
-                else:
-                    mistakes.append(
-                        ModelError(
-                            error_type=ErrorType.WrongEntity,
-                            annotation=cur_annotation,
-                            prediction=cur_prediction,
-                            token=cur_token,
-                            full_text=input_sample.full_text,
-                            metadata=input_sample.metadata,
-                        ),
-                    )
-
-        return results, mistakes
-
-    def __revert_known_errors(
-        self,
-        current_annotation: str,
-        current_prediction: str,
-        current_token: str | Token,
-        results: Counter[tuple[str, str]],
-    ) -> bool:
-        reverted = False
-
-        if str(current_token).lower().strip() in self.skip_words:
-            # Ignore cases where the token is a skip word
-            results[(current_annotation, current_prediction)] -= 1
-            reverted = True
-
-        if current_prediction in self.generic_entities and current_annotation != "O":
-            # Ignore cases where the prediction is generic
-            results[(current_annotation, current_prediction)] -= 1
-            # Add a result which assumes the generic equals the specific
-            results[(current_annotation, current_annotation)] += 1
-            reverted = True
-
-        elif current_annotation in self.generic_entities and current_prediction != "O":
-            # Ignore cases where the prediction is generic
-            results[(current_annotation, current_prediction)] -= 1
-            # Add a result which assumes the generic equals the specific
-            results[(current_prediction, current_prediction)] += 1
-            reverted = True
-
-        # Remove temporary keys which should not be counted
-        if results[(current_annotation, current_prediction)] == 0:
-            del results[(current_annotation, current_prediction)]
-
-        return reverted
-
-    def _adjust_per_entities(self, tags: list[str]) -> list[str]:
-        if self.entities_to_keep:
-            return [tag if tag in self.entities_to_keep else "O" for tag in tags]
-        else:
-            return tags
-
-    def evaluate_sample(
-        self,
-        sample: InputSample,
-        prediction: list[str],
-    ) -> EvaluationResult:
-        warnings.warn(
-            "evaluate_sample() is deprecated. Use predict_dataset() + calculate_score_on_df() instead:\n"
-            "  results_df = model.predict_dataset(dataset)\n"
-            "  result = evaluator.calculate_score_on_df(results_df=results_df)",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-
-        if self.verbose:
-            logger.debug(f"Input sentence: {sample.full_text}")
-
-        results, model_errors = self.compare(input_sample=sample, prediction=prediction)
-
-        return EvaluationResult(
-            results=results,
-            model_errors=model_errors,
-            text=sample.full_text,
-            tokens=[str(token) for token in sample.tokens],
-            actual_tags=sample.tags,
-            predicted_tags=prediction,
-            start_indices=sample.start_indices,
-        )
-
     def evaluate_all(
         self,
         dataset: list[InputSample],
@@ -248,7 +91,7 @@ def evaluate_all(
 
                 # Step 3: evaluate
                 from presidio_evaluator.evaluation import SpanEvaluator
-                evaluator = SpanEvaluator(model=None)
+                evaluator = SpanEvaluator()
                 result_per_type = evaluator.calculate_score_on_df(per_type=True, results_df=mapped_df)
                 global_df = SpanEvaluator.create_global_entities_df(mapped_df)
                 result = evaluator.calculate_score_on_df(per_type=False, results_df=global_df, evaluation_result=result_per_type)
@@ -316,8 +159,12 @@ def get_results_dataframe(
             annotations = list(res.actual_tags)
             predictions = list(res.predicted_tags)
             if self.entities_to_keep:
-                annotations = self._adjust_per_entities(annotations)
-                predictions = self._adjust_per_entities(predictions)
+                annotations = [
+                    tag if tag in self.entities_to_keep else "O" for tag in annotations
+                ]
+                predictions = [
+                    tag if tag in self.entities_to_keep else "O" for tag in predictions
+                ]
 
             # Filter to the requested entity subset (e.g. for per-entity scoring)
             annotations = self._filter_entities(annotations, entities)