Implement strict parameter for strict mode (#33)

* add strict mode

* add test for strict mode

* black and mypy fixes

* 📝 adding to docstring

* ✏️ refining logic for strict mode

* 📝 adding note for future dev

* ✏️ failing on first error, not after all errors

---------

Co-authored-by: Wessel Huising <wessel@MacBook-Pro-van-Wessel.local>
Co-authored-by: Xavier Nogueira <xavier.rojas.nogueira@gmail.com>

Author: 3 people

src/pandantic/validators/pandas.py

@@ -23,6 +23,7 @@ def validate(
         self,
         dataframe: pd.DataFrame,
         errors: Literal["skip", "raise", "log"] = "raise",
+        strict: bool = False,
         context: Optional[
             dict[str, Any]
         ] = None,  # pylint: disable=consider-alternative-union-syntax,useless-suppression
@@ -35,13 +36,26 @@ def validate(
             dataframe (pd.DataFrame): The DataFrame to validate.
             errors (Literal["skip", "raise", "log"], optional): How to handle validation errors. Defaults to "raise".
                 NOTE: "skip" and "log" effectively filter the dataframe, excluding invalid rows.
+            strict (bool, default=False): whether to fail validation if extra fields/columns are present.
             context (Optional[dict[str, Any]], optional): The context to use for validation. Defaults to None.
             n_jobs (int, optional): The number of processes to use for validation. Defaults to 1.
             queue (Optional[Queue], optional): A custom Queue object for multiprocessing. Defaults to None.
 
         Returns:
             pd.DataFrame: The original DataFrame if errors="raise" or "log", or a filtered DataFrame with valid rows if errors="skip".
         """
+        # check for extra columns and handle strict mode
+        # NOTE: this will need to be abstracted to handle different types of schema objects
+        if strict:
+            extras = {
+                col for col in dataframe.columns if col not in self.schema.model_fields.keys()
+            }
+            if extras:
+                raise ValueError(
+                    f"Strict mode is enabled but the following extra columns were found in the schema: {extras}."
+                )
+            del extras
+
         if errors not in ["skip", "raise", "log"]:
             raise ValueError("errors must be one of 'skip', 'raise', or 'log'")
 
@@ -101,13 +115,12 @@ def validate(
                 except ValidationError as exc:  # pylint: disable=broad-exception-caught
                     if errors == "log":
                         logging.info("Validation error found at index %s\n%s", index, exc)
-
+                    if errors == "raise":
+                        raise exc
                     errors_index.append(index)
 
         logging.debug("# invalid rows: %s", len(errors_index))
 
-        if len(errors_index) > 0 and errors == "raise":
-            raise ValueError(f"{len(errors_index)} validation errors found in dataframe.")
         if len(errors_index) > 0 and errors in ["skip", "log"]:
             return dataframe[~dataframe.index.isin(list(errors_index))]
         return dataframe
@@ -141,7 +154,8 @@ def _validate_chunk(
             except ValidationError as exc:  # pylint: disable=broad-exception-caught
                 if errors == "log":
                     logging.info("Validation error found at index %s\n%s", index, exc)
-
+                if errors == "raise":
+                    raise exc
                 queue.put(index)
 
         logging.debug("Process ended.")
@@ -159,9 +173,12 @@ def iterate(
         """Iterate over a DataFrame and yield validated schema models."""
         for i, row in dataframe.iterrows():
             try:
-                yield i, self.schema.model_validate(
-                    obj=row.to_dict(),
-                    context=context,
+                yield (
+                    i,
+                    self.schema.model_validate(
+                        obj=row.to_dict(),
+                        context=context,
+                    ),
                 )
             except Exception as e:
                 if verbose:

tests/test_pandas_validator.py

@@ -3,6 +3,7 @@
     * validate() function (full table).
     * validate() function (to skip table).
 """
+
 import logging
 from typing import Optional
 
@@ -155,3 +156,28 @@ class Model(BaseModel):
 
     # THEN
     assert df_skiped.equals(df_example)
+
+
+def test_strict_mode():
+    # GIVEN
+    class Model(BaseModel):
+        a: Optional[int] = None
+        b: str
+
+    df_example = pd.DataFrame({"a": [None, None, None], "b": ["str", "str", "str"], "c": [1, 2, 3]})
+    validator = PandasValidator(schema=Model)
+
+    # WHEN
+    df_skiped = validator.validate(df_example, errors="skip")
+
+    # THEN
+    assert df_skiped.equals(df_example)
+
+    # THEN
+    with pytest.raises(ValueError):
+        # WHEN
+        validator.validate(
+            dataframe=df_example,
+            strict=True,
+            errors="raise",
+        )