fix: update code to dependency updates (#1768)

CI was failing because several dependencies (with no major version pin)
have introduced breaking changes or changed behavior.


- `chardet v7` has different character encoding detections that chardet
v5. Several tests were broken.
- some tests were simply testing the (wrong) detection from chardet.
They are now testing the correct/most likely encoding of the file
(UTF-8).
- test files are really short, with a unique non-ascii character.
Chardet's confidence is therefore very low, and it cannot distinguish
several encodings. Some tests were made less strict ("Check that the
non-ascii is correctly decoded" instead of "check the exact encoding"),
and the sample size to detect encoding has been augmented (x10).
- there was a dependency on `pytz`, which was a (former) transitive
dependency (from `pandas` maybe ?). It has been replaced with
`datetime`.
- Pandas v2 to v3 changes its `dtypes` 
   - adapt to new dtypes collection type
   - make the tests accept former and new dtypes alongside
   - deal with new "nan" handling
- adapt to changes of `pytest-cov` v7. 
- pin TatSu < 5.15 
- pin virtualenv < 21 for hatch ([hatch
issue](pypa/hatch#2193), still applies to 3.8
and 3.9)

Additionnaly :
- add CI tests for python 3.13 and 3.14
- depreciation warning for python 3.8 and 3.9
- CI for py3.8 takes ages, so I removed it.

Author: pierrecamilleri

.github/workflows/general.yaml

@@ -21,16 +21,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Prepare environment
-        run: pip3 install hatch
+        # https://github.com/pypa/hatch/issues/2193
+        run: pip3 install hatch 'virtualenv<21'
       - name: Prepare variables
         run: cp .env.example .env
       - name: Prepare secrets
@@ -78,9 +79,9 @@ jobs:
     runs-on: macos-14
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
       - name: Set up postgresql
@@ -100,9 +101,9 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
       - name: Prepare environment
@@ -119,9 +120,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
       - name: Prepare environment
@@ -145,9 +146,9 @@ jobs:
     needs: [test-linux, test-macos, test-windows]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
       - name: Install dependencies

frictionless/__init__.py

@@ -1,3 +1,14 @@
+import sys
+import warnings
+
+if sys.version_info < (3, 10):
+    warnings.warn(
+        f"Python {sys.version_info.major}.{sys.version_info.minor} has reached end-of-life "
+        "and is no longer supported. Please upgrade to Python 3.10 or later.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
 from .actions import convert as convert
 from .actions import describe as describe
 from .actions import extract as extract

frictionless/__main__.py

@@ -1,4 +1,9 @@
+import warnings
+
 from .console import console
 
+warnings.filterwarnings("default", category=DeprecationWarning, module="frictionless")
+
+
 if __name__ == "__main__":
     console(prog_name="frictionless")

frictionless/conftest.py

@@ -2,7 +2,6 @@
 
 import pytest
 import sqlalchemy as sa
-from pytest_cov.embed import cleanup_on_sigterm
 
 from frictionless import platform
 
@@ -11,8 +10,13 @@
 
 # Cleanups
 
+try:
+    # For python 3.8 only, that does not support pytest_cov v7
+    from pytest_cov.embed import cleanup_on_sigterm
 
-cleanup_on_sigterm()
+    cleanup_on_sigterm()
+except ImportError:
+    pass
 
 
 # Fixtures

frictionless/detector/detector.py

@@ -216,7 +216,15 @@ def detect_encoding(
             encoding = detector.result["encoding"] or settings.DEFAULT_ENCODING
             confidence = detector.result["confidence"] or 0
             if confidence < self.encoding_confidence:
+                # low confidence, so we try UTF8
+                # If decoding fails, we fallback to the detected encoding
+                # despite the low-confidence
+                detected = encoding
                 encoding = settings.DEFAULT_ENCODING
+                try:
+                    buffer.decode(encoding)
+                except UnicodeDecodeError:
+                    encoding = detected
             if encoding == "ascii":
                 encoding = settings.DEFAULT_ENCODING
 
@@ -329,7 +337,7 @@ def detect_schema(
 
             # Handle name/empty
             for index, name in enumerate(names):
-                names[index] = name or f"field{index+1}"
+                names[index] = name or f"field{index + 1}"
 
             # Deduplicate names
             if len(names) != len(set(names)):

frictionless/formats/markdown/mapper.py

@@ -99,7 +99,7 @@ def dicts_to_markdown_table(dicts: List[Dict[str, Any]], **kwargs: Any) -> str:
     if kwargs:
         dicts = [filter_dict(x, **kwargs) for x in dicts]
     df = platform.pandas.DataFrame(dicts)
-    return df.where(df.notnull(), None).to_markdown(index=False)  # type: ignore
+    return df.fillna("").to_markdown(index=False)
 
 
 def filter_dict(

frictionless/formats/pandas/__spec__/test_parser.py

@@ -1,16 +1,18 @@
-from datetime import datetime, time
+from datetime import datetime, time, timezone
 from decimal import Decimal
 
 import isodate
 import numpy as np
 import pandas as pd
-import pytz
 from dateutil.tz import tzoffset, tzutc
-from pandas.core.dtypes.common import is_datetime64_ns_dtype
+from pandas.api.types import is_datetime64_any_dtype
 
 from frictionless import Package, Schema, validate
 from frictionless.resources import TableResource
 
+# Infer dtype from real DataFrame as the type is depending on pandas' version
+STRING_DTYPE = pd.DataFrame({"s": ["a"]}).dtypes["s"]
+
 # Read
 
 
@@ -73,14 +75,14 @@ def test_pandas_parser_from_dataframe_with_primary_key_having_datetime():
         # Assert rows
         assert resource.read_rows() == [
             {
-                "Date": datetime(2004, 1, 5, tzinfo=pytz.utc),
+                "Date": datetime(2004, 1, 5, tzinfo=timezone.utc),
                 "VIXClose": Decimal("17.49"),
                 "VIXHigh": Decimal("18.49"),
                 "VIXLow": Decimal("17.44"),
                 "VIXOpen": Decimal("18.45"),
             },
             {
-                "Date": datetime(2004, 1, 6, tzinfo=pytz.utc),
+                "Date": datetime(2004, 1, 6, tzinfo=timezone.utc),
                 "VIXClose": Decimal("16.73"),
                 "VIXHigh": Decimal("17.67"),
                 "VIXLow": Decimal("16.19"),
@@ -112,14 +114,14 @@ def test_pandas_parser_nan_in_integer_resource_column():
         ]
     )
     df = res.to_pandas()
-    assert all(df.dtypes.values == pd.array([pd.Int64Dtype(), float, object]))  # type: ignore
+    assert list(df.dtypes) == [pd.Int64Dtype(), np.dtype("float64"), STRING_DTYPE]
 
 
 def test_pandas_parser_nan_in_integer_csv_column():
     # see issue 1109
     res = TableResource(path="data/issue-1109.csv")
     df = res.to_pandas()
-    assert all(df.dtypes.values == pd.array([pd.Int64Dtype(), float, object]))  # type: ignore
+    assert list(df.dtypes) == [pd.Int64Dtype(), np.dtype("float64"), STRING_DTYPE]
 
 
 def test_pandas_parser_write_types():
@@ -273,7 +275,7 @@ def test_pandas_parser_nan_with_field_type_information_1143():
     }
     res = TableResource.from_descriptor(descriptor)
     df = res.to_pandas()
-    assert all(df.dtypes.values == pd.array([pd.Int64Dtype(), float, object]))  # type: ignore
+    assert list(df.dtypes) == [pd.Int64Dtype(), np.dtype("float64"), STRING_DTYPE]
 
 
 def test_pandas_parser_nan_without_field_type_information_1143():
@@ -291,7 +293,7 @@ def test_pandas_parser_nan_without_field_type_information_1143():
     }
     res = TableResource.from_descriptor(descriptor)
     df = res.to_pandas()
-    assert all(df.dtypes.values == pd.array([object, object, object]))  # type: ignore
+    assert list(df.dtypes) == [STRING_DTYPE, STRING_DTYPE, STRING_DTYPE]
 
 
 def test_pandas_parser_preserve_datetime_field_type_1138():
@@ -311,7 +313,7 @@ def test_pandas_parser_preserve_datetime_field_type_1138():
     }
     resource = TableResource.from_descriptor(descriptor)
     df = resource.to_pandas()
-    assert is_datetime64_ns_dtype(df.dtypes.values[1])  # type: ignore
+    assert is_datetime64_any_dtype(df.dtypes.values[1])  # type: ignore
 
 
 def test_pandas_parser_test_issue_sample_data_1138():
@@ -338,7 +340,7 @@ def test_pandas_parser_test_issue_sample_data_1138():
     }
     resource = TableResource.from_descriptor(descriptor)
     df = resource.to_pandas()
-    assert is_datetime64_ns_dtype(df.dtypes.values[0])  # type: ignore
+    assert is_datetime64_any_dtype(df.dtypes.values[0])  # type: ignore
 
 
 def test_validate_package_with_in_code_resources_1245():

frictionless/formats/pandas/parser.py

@@ -140,8 +140,7 @@ def write_row_stream(self, source: TableResource):
                         value = float(value)
                     # Convert to UTC for timezone aware datetime
                     # From version 0.24 pandas preserves the dateutil object and doesn't by default
-                    # convert to "UTC" and fastparquet write raises error as it can't handle tzutc()
-                    # object
+                    # convert to "UTC"
                     # https://github.com/pandas-dev/pandas/issues/25423#issuecomment-485784044
                     if isinstance(value, datetime.datetime) and value.tzinfo:
                         value = value.astimezone(datetime.timezone.utc)

frictionless/formats/parquet/parser.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
+from ... import types
 from ...platform import platform
 from ...resources import TableResource
 from ...system import Parser
 from .control import ParquetControl
 
 
 class ParquetParser(Parser):
-    """JSONL parser implementation."""
+    """Parquet parser implementation."""
 
     supported_types = [
         "array",
@@ -24,24 +25,30 @@ class ParquetParser(Parser):
 
     # Read
 
-    def read_cell_stream_create(self):
+    def read_cell_stream_create(self) -> types.ICellStream:
         control = ParquetControl.from_dialect(self.resource.dialect)
         handle = self.resource.normpath
         if self.resource.remote:
             handles = platform.pandas.io.common.get_handle(  # type: ignore
                 self.resource.normpath, "rb", is_text=False
             )
             handle = handles.handle
-        file = platform.fastparquet.ParquetFile(handle)
-        for group, df in enumerate(file.iter_row_groups(**control.to_python()), start=1):
-            with TableResource(data=df, format="pandas") as resource:
-                for line, cells in enumerate(resource.cell_stream, start=1):
-                    # Starting from second group we don't need a header row
-                    if group != 1 and line == 1:
-                        continue
-                    yield cells
+        pq = platform.pyarrow_parquet
+        table = pq.read_table(
+            handle,
+            columns=control.columns,
+            filters=control.filters or None,
+        )
+        df = table.to_pandas(categories=control.categories or None)
+        with TableResource(data=df, format="pandas") as resource:
+            yield from resource.cell_stream
 
     # Write
 
     def write_row_stream(self, source: TableResource):
-        platform.fastparquet.write(self.resource.normpath, source.to_pandas())
+        import pyarrow as pa  # type: ignore[reportMissingTypeStubs]
+
+        pq = platform.pyarrow_parquet
+        df = source.to_pandas()
+        table = pa.Table.from_pandas(df)  # type: ignore[reportUnknownMemberType]
+        pq.write_table(table, self.resource.normpath)

frictionless/formats/spss/__spec__/test_parser.py

@@ -6,7 +6,8 @@
 from frictionless.resources import TableResource
 
 pytestmark = pytest.mark.skipif(
-    platform.type == "darwin" or platform.python in ["3.10", "3.11", "3.12"],
+    platform.type == "darwin"
+    or platform.python in ["3.10", "3.11", "3.12", "3.13", "3.14"],
     reason="Not supported MacOS and Python3.10+",
 )