Fix tests

Author: remi-braun

ci/on_push/test_satellites.py

@@ -498,7 +498,7 @@ def core(prod_path, possible_bands, tmpdir, **kwargs):
 
 test_optical_constellations_cases = [
     pytest.param("*S2*_MSI*_N7*", {}, id="s2_after_04_00"),
-    pytest.param("*S2*_MSI*_N02*", {}, id="s2_before_04_00"),
+    pytest.param("*S2*_MSI*_N0209*", {}, id="s2_before_04_00"),
     pytest.param("*SENTINEL2*", {}, id="s2_theia"),
     pytest.param("*S2A_39KZU*", {}, id="s2_cloud"),
     pytest.param("*S3*_OL_1_*", {}, id="s3_olci"),
@@ -599,6 +599,6 @@ def test_invalid():
 
 
 @s3_env
-def test_s2_before_04_00(eoreader_tests_path):
+def test_s2_after_04_00(eoreader_tests_path):
     """Function testing the support of Sentinel-2 constellation"""
-    _test_core_optical("*S2*_MSI*_N0209*", eoreader_tests_path.tmpdir)
+    _test_core_optical("*S2*_MSI*_N7*", eoreader_tests_path.tmpdir)

ci/scripts_utils.py

@@ -200,25 +200,32 @@ def decorator(function):
         @wraps(function)
         def s3_env_wrapper(*args, **kwargs):
             with rasterio.Env(
-                # Set to TRUE or EMPTY_DIR to avoid listing all files in the directory once a single file is opened (this is highly recommended).
-                GDAL_DISABLE_READDIR_ON_OPEN=True,
-                # # Size of the default block cache, can be set in byte, MB, or as a percentage of available main, memory.
-                # GDAL_CACHEMAX=gdal_cachemax_bytes, # => doesn't seem to improve anything, in fact slows down things a bit
                 # Global cache size for downloads in bytes, defaults to 16 MB.
                 CPL_VSIL_CURL_CACHE_SIZE=mo_to_bytes(200),
+                #
                 # Enable / disable per-file caching by setting to TRUE or FALSE.
                 VSI_CACHE=True,
+                #
                 # Per-file cache size in bytes
                 VSI_CACHE_SIZE=mo_to_bytes(5),
+                #
                 # When set to YES, this attempts to download multiple range requests in parallel, reusing the same TCP connection
                 GDAL_HTTP_MULTIPLEX=True,
+                #
                 # Gives the number of initial bytes GDAL should read when opening a file and inspecting its metadata.
                 GDAL_INGESTED_BYTES_AT_OPEN=ko_to_bytes(32),
                 GDAL_HTTP_VERSION=2,
+                #
                 # Tells GDAL to merge consecutive range GET requests.
                 GDAL_HTTP_MERGE_CONSECUTIVE_RANGES="YES",
+                #
+                # -- useless by experience --
+                # Size of the default block cache, can be set in byte, MB, or as a percentage of available main, memory.
+                # GDAL_CACHEMAX=gdal_cachemax_bytes, # => doesn't seem to improve anything, in fact slows down things a bit
                 # Number of threads GDAL can use for block reads and (de)compression, set to ALL_CPUS to use all available cores.
-                # GDAL_NUM_THREADS="ALL_CPUS", # => doesn't seem to improve anything, in fact slows down things
+                # GDAL_NUM_THREADS="ALL_CPUS", #  => doesn't seem to improve anything, in fact slows down things
+                # Set to TRUE or EMPTY_DIR to avoid listing all files in the directory once a single file is opened (this is highly recommended).
+                # GDAL_DISABLE_READDIR_ON_OPEN=True, #  => seems to have drawbacks especially for s2
             ):
                 return unistra.s3_env(
                     function(*args, **kwargs), use_s3_env_var=CI_EOREADER_S3

eoreader/utils.py

@@ -53,7 +53,44 @@
 DEFAULT_NOF_BANDS_IN_CHUNKS = 1
 UINT16_NODATA = rasters.UINT16_NODATA
 
-read_bit_array = rasters.read_bit_array
+
+# Workaround for now, remove this asap
+def read_bit_array(
+    bit_mask: Union[xr.DataArray, np.ndarray], bit_id: Union[list, int]
+) -> Union[np.ndarray, list]:
+    """
+    Read 8 bit arrays as a succession of binary masks.
+
+    Forces array to :code:`np.uint8`.
+
+    See :py:func:`rasters.read_bit_array`.
+
+    Args:
+        bit_mask (np.ndarray): Bit array to read
+        bit_id (int): Bit ID of the slice to be read
+          Example: read the bit 0 of the mask as a cloud mask (Theia)
+
+    Returns:
+        Union[np.ndarray, list]: Binary mask or list of binary masks if a list of bit_id is given
+    """
+    if misc.compare_version("sertit", "1.47.0", ">="):
+        return rasters.read_bit_array(bit_mask, bit_id)
+    else:
+        # Suppress nan nodata and convert back to original dtype if known
+
+        if isinstance(bit_mask, np.ndarray):
+            bit_mask = np.nan_to_num(bit_mask)
+        elif isinstance(bit_mask, xr.DataArray):
+            orig_dtype = bit_mask.encoding.get("dtype")
+            bit_mask = bit_mask.fillna(0).data
+            if orig_dtype is not None and bit_mask.dtype != orig_dtype:
+                bit_mask = bit_mask.astype(orig_dtype)
+
+        else:
+            bit_mask = bit_mask.fillna(0)
+        from sertit import rasters_rio
+
+        return rasters_rio.read_bit_array(bit_mask, bit_id)
 
 
 def get_src_dir() -> AnyPathType: