fix: change default list of values to be skipped and default behavior of skip falsy to True

Signed-off-by: Nilton Junior <ngm.junior@outlook.com>

Author: NiltonGMJunior

docs/source/settings.rst

@@ -185,7 +185,7 @@ If this setting is not provided or set to ``False``, spider statistics will be:
 
 SPIDERMON_FIELD_COVERAGE_SKIP_FALSY
 -----------------------------------
-Default: ``False``
+Default: ``True``
 
 When enabled, returned fields that have falsy values (empty strings, empty lists, empty tuples, empty dictionaries, zero, False, etc.) will not be counted as fields with a value.
 
@@ -238,9 +238,9 @@ If this setting is not provided or set to ``False``, spider statistics will be:
 
 SPIDERMON_FIELD_COVERAGE_SKIP_VALUES
 ------------------------------------
-Default: ``[]``
+Default: ``["", [], {}, "N/A", "-"]``
 
-A list of custom values that should not be counted as valid field values when calculating field coverage. This is useful when your items contain placeholder values like "N/A", "-", "TBD", etc. that indicate missing data but are not Python falsy values. You can also skip numeric values like ``0`` or ``-1`` if they represent missing data in your use case.
+A list of custom values that should not be counted as valid field values when calculating field coverage. By default, this includes empty string, empty list, empty dict, "N/A", and "-". You can override this to customize which values should be skipped. This is useful when your items contain placeholder values like "TBD", etc. that indicate missing data but are not Python falsy values. You can also skip numeric values like ``0`` or ``-1`` if they represent missing data in your use case.
 
 This setting works in addition to ``SPIDERMON_FIELD_COVERAGE_SKIP_NONE`` and ``SPIDERMON_FIELD_COVERAGE_SKIP_FALSY``. Values are matched using exact equality (``==``), so type matters (e.g., the string ``"0"`` is different from the integer ``0``).
 
@@ -285,7 +285,7 @@ If this setting is set to ``["N/A", "-", "TBD"]``, spider statistics will be:
    'spidermon_field_coverage/dict/field_3': 0.5,  # Ignored "-"
    'spidermon_field_coverage/dict/field_4': 0.5,  # Ignored "TBD"
 
-If this setting is not provided or set to an empty list, spider statistics will be:
+If you want to override the default skip values, you can set this to a custom list. If set to an empty list ``[]``, no custom skip values will be used (only falsy values will be skipped if ``SPIDERMON_FIELD_COVERAGE_SKIP_FALSY`` is enabled). If not provided, the default values will be used. Without the default skip values, spider statistics would be:
 
 .. code-block:: python
 

spidermon/contrib/scrapy/extensions.py

@@ -60,15 +60,31 @@ def __init__(
         self.periodic_tasks = {}
         self.client = Client(self.crawler.settings)
 
+    @staticmethod
+    def _get_default_skip_values():
+        """Get the default skip values array.
+
+        Default skip values: empty string, empty list, empty dict, 'N/A', '-'
+        """
+        return ["", [], {}, "N/A", "-"]
+
     def _get_skip_values_list(self, settings):
         """Get skip values list, supporting Python lists, JSON strings, and
         comma-separated strings.
 
         This allows preserving types (e.g., integers) when provided as Python
         lists or JSON strings, while still supporting comma-separated strings
         for backward compatibility.
+
+        Default skip values: empty string, empty list, empty dict, 'N/A', '-'
         """
-        value = settings.get("SPIDERMON_FIELD_COVERAGE_SKIP_VALUES", [])
+        # Default skip values
+        default_skip_values = self._get_default_skip_values()
+
+        value = settings.get("SPIDERMON_FIELD_COVERAGE_SKIP_VALUES", None)
+        if value is None:
+            return default_skip_values
+
         if not value:
             return []
 
@@ -194,7 +210,7 @@ def _count_item(
             if skip_none_values and value is None:
                 continue
 
-            if skip_falsy_values and not value:
+            if skip_falsy_values and value in self._get_default_skip_values():
                 continue
 
             if value in skip_values:
@@ -264,9 +280,24 @@ def item_scraped(self, item, response, spider):
             "SPIDERMON_FIELD_COVERAGE_SKIP_NONE", False
         )
         skip_falsy_values = spider.crawler.settings.getbool(
-            "SPIDERMON_FIELD_COVERAGE_SKIP_FALSY", False
+            "SPIDERMON_FIELD_COVERAGE_SKIP_FALSY", True
+        )
+        # Check if SPIDERMON_FIELD_COVERAGE_SKIP_VALUES was explicitly set
+        # to empty list. If so, disable default skip values check.
+        skip_values_setting = spider.crawler.settings.get(
+            "SPIDERMON_FIELD_COVERAGE_SKIP_VALUES"
         )
         skip_values = self._get_skip_values_list(spider.crawler.settings)
+        if (
+            skip_values_setting is not None
+            and isinstance(skip_values_setting, list)
+            and skip_values_setting == []
+            and skip_values == []
+        ):
+            # Setting was explicitly set to empty list, disable default
+            # skip values (skip_falsy_values should not apply)
+            skip_falsy_values = False
+
         list_field_coverage_levels = spider.crawler.settings.getint(
             "SPIDERMON_LIST_FIELDS_COVERAGE_LEVELS", 0
         )

tests/test_item_scraped_signal.py

@@ -552,7 +552,8 @@ def test_item_scraped_count_ignore_custom_skip_values():
     )  # Only the valid values (skipped "TBD")
 
 
-def test_item_scraped_count_do_not_ignore_custom_skip_values_by_default():
+def test_item_scraped_count_ignore_default_skip_values():
+    """Test that default skip values (empty string, empty list, empty dict, N/A, -) are applied"""
     settings = {
         "SPIDERMON_ENABLED": True,
         "EXTENSIONS": {"spidermon.contrib.scrapy.extensions.Spidermon": 100},
@@ -562,6 +563,39 @@ def test_item_scraped_count_do_not_ignore_custom_skip_values_by_default():
     crawler = get_crawler(settings_dict=settings)
     spider = Spider.from_crawler(crawler, "example.com")
 
+    returned_items = [
+        {"field1": "value1", "field2": "N/A", "field3": "-", "field4": ""},
+        {"field1": "value1", "field2": "value2", "field3": "value3", "field4": "value4"},
+    ]
+
+    for item in returned_items:
+        spider.crawler.signals.send_catch_log_deferred(
+            signal=signals.item_scraped,
+            item=item,
+            response="",
+            spider=spider,
+        )
+
+    stats = spider.crawler.stats.get_stats()
+
+    assert stats.get("spidermon_item_scraped_count/dict/field1") == 2
+    assert stats.get("spidermon_item_scraped_count/dict/field2") == 1  # "N/A" skipped by default
+    assert stats.get("spidermon_item_scraped_count/dict/field3") == 1  # "-" skipped by default
+    assert stats.get("spidermon_item_scraped_count/dict/field4") == 1  # "" skipped by default
+
+
+def test_item_scraped_count_do_not_ignore_custom_skip_values_when_empty_list():
+    """Test that setting skip_values to empty list disables default skip values"""
+    settings = {
+        "SPIDERMON_ENABLED": True,
+        "EXTENSIONS": {"spidermon.contrib.scrapy.extensions.Spidermon": 100},
+        "SPIDERMON_ADD_FIELD_COVERAGE": True,
+        "SPIDERMON_FIELD_COVERAGE_SKIP_VALUES": [],
+    }
+
+    crawler = get_crawler(settings_dict=settings)
+    spider = Spider.from_crawler(crawler, "example.com")
+
     returned_items = [
         {"field1": "value1", "field2": "N/A"},
         {"field1": "value1", "field2": "-"},
@@ -580,7 +614,7 @@ def test_item_scraped_count_do_not_ignore_custom_skip_values_by_default():
     assert stats.get("spidermon_item_scraped_count/dict/field1") == 2
     assert (
         stats.get("spidermon_item_scraped_count/dict/field2") == 2
-    )  # Counted because skip_values not set
+    )  # Counted because skip_values is empty list
 
 
 def test_item_scraped_count_skip_values_type_sensitive():