141 - added offset filter and level (#142)

* added offset filter and level

* Set ZSTD Filter levels to 7
* added PositiveDeltaFilter for var len attributes in offsets_filters of ArraySchema

* Updated python build versions

* remove matrix python ver 3.11, added 3.14

* updated python dependencies

* python version >= 3.12 and inclusive of 3.14 required

---------

Co-authored-by: GannonGuess <131088734+GannonGuess@users.noreply.github.com>
Co-authored-by: kylemann16 <kyle@hobu.co>

Author: gannon-guess

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.11', '3.12', '3.13']
+        python-version: ['3.12', '3.13', '3.14']
     steps:
       - name: Checkout
         uses: actions/checkout@v4

environment.yml

@@ -2,7 +2,7 @@ name: pdal-sm
 channels:
   - conda-forge
 dependencies:
-  - python>=3.11, <3.14
+  - python>=3.12, <=3.14
   - pip
   - cloudpickle>=3.1.2
   - boto3

src/silvimetric/resources/attribute.py

@@ -42,7 +42,7 @@ def schema(self) -> Attr:
             name=self.name,
             dtype=self.dtype.subtype,
             var=True,
-            filters=FilterList([ZstdFilter()])
+            filters=FilterList([ZstdFilter(level = 7)])
         )
 
     def __eq__(self, other):

src/silvimetric/resources/metric.py

@@ -144,7 +144,7 @@ def schema(self, attr: Attribute) -> Any:
         return Attr(
             name=entry_name,
             dtype=self.dtype,
-            filters=FilterList([ZstdFilter()]),
+            filters=FilterList([ZstdFilter(level = 7)]),
             nullable=True
         )
 

src/silvimetric/resources/storage.py

@@ -102,39 +102,39 @@ def create(config: StorageConfig, ctx: tiledb.Ctx = None):
             domain=(0, xi),
             dtype=np.uint64,
             tile=xsize,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
         )
         dim_col = tiledb.Dim(
             name='Y',
             domain=(0, yi),
             dtype=np.uint64,
             tile=ysize,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
         )
         domain = tiledb.Domain(dim_row, dim_col)
 
         count_att = tiledb.Attr(
             name='count',
             dtype=np.uint32,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
             fill=0,
         )
         proc_att = tiledb.Attr(
             name='shatter_process_num',
             dtype=np.uint16,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
             fill=0,
         )
         start_time_att = tiledb.Attr(
             name='start_time',
             dtype=np.datetime64('', 'D').dtype,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
             fill=np.datetime64(0, 'D'),
         )
         end_time_att = tiledb.Attr(
             name='end_time',
             dtype=np.datetime64('', 'D').dtype,
-            filters=tiledb.FilterList([tiledb.ZstdFilter()]),
+            filters=tiledb.FilterList([tiledb.ZstdFilter(level = 7)]),
             fill=np.datetime64(0, 'D'),
         )
         dim_atts = [attr.schema() for attr in config.attrs]
@@ -171,13 +171,11 @@ def create(config: StorageConfig, ctx: tiledb.Ctx = None):
                 *dim_atts,
                 *metric_atts,
             ],
-            # offsets_filters=tiledb.FilterList(
-            #     [
-            #         tiledb.PositiveDeltaFilter(),
-            #         tiledb.BitWidthReductionFilter(),
-            #         tiledb.ZstdFilter(),
-            #     ]
-            # ),
+            offsets_filters=tiledb.FilterList(
+                [
+                    tiledb.PositiveDeltaFilter(),
+                ]
+            ),
         )
         schema.check()