Updated all pyarrow.Table.to_pandas() calls to reduce memory consumption

This was done using the split_blocks and self_destruct parameters in tandem.

Author: alancleary

apis/python/src/tiledbvcf/dataset.py

@@ -387,7 +387,9 @@ def read_variant_stats(
             "regions": regions,
             "scan_all_samples": scan_all_samples,
         }
-        return self.read_variant_stats_arrow(**kwargs).to_pandas()
+        return self.read_variant_stats_arrow(**kwargs).to_pandas(
+            split_blocks=True, self_destruct=True
+        )
 
     def read_variant_stats_arrow(
         self,
@@ -474,7 +476,9 @@ def read_allele_count(
         if self.mode != "r":
             raise Exception("Dataset not open in read mode")
 
-        return self.read_allele_count_arrow(regions=regions).to_pandas()
+        return self.read_allele_count_arrow(regions=regions).to_pandas(
+            split_blocks=True, self_destruct=True
+        )
 
     def read_allele_count_arrow(
         self,
@@ -732,7 +736,7 @@ def continue_read(self, release_buffers: bool = True) -> pd.DataFrame:
         """
         table = self.continue_read_arrow(release_buffers=release_buffers)
 
-        return table.to_pandas()
+        return table.to_pandas(split_blocks=True, self_destruct=True)
 
     def continue_read_arrow(self, release_buffers: bool = True) -> pa.Table:
         """

apis/python/src/tiledbvcf/sample_qc.py

@@ -26,4 +26,6 @@ def sample_qc(
     pandas.DataFrame
     """
 
-    return clib.sample_qc(dataset_uri, samples=samples, config=config).to_pandas()
+    return clib.sample_qc(dataset_uri, samples=samples, config=config).to_pandas(
+        split_blocks=True, self_destruct=True
+    )