Merge branch 'xan/revert-SOMAContext' of https://github.com/single-cell-data/TileDB-SOMA into xan/revert-SOMAContext

Author: XanthosXanthopoulos

apis/python/HISTORY.md

@@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Fixed
 
+- `PlatformConfig.dense_nd_array_dim_zstd_level` was incorrectly aliased to `sparse_nd_array_dim_zstd_level` and is now bound to the correct field.
+
 ### Security
 
 ## [2.3.0]

apis/python/src/tiledbsoma/_soma_array.py

@@ -34,7 +34,7 @@ def schema(self) -> pa.Schema:
 
     def schema_config_options(self) -> clib.PlatformSchemaConfig:
         """Returns metadata about the array schema that is not encompassed within
-        the Arrow Schema, in the form of a PlatformConfig.
+        the Arrow Schema, in the form of a PlatformSchemaConfig.
 
         Available attributes are:
             * capacity: int

apis/python/src/tiledbsoma/pytiledbsoma.cc

@@ -1,5 +1,7 @@
 #include <tiledbsoma/tiledbsoma>
 
+#include <sstream>
+
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
@@ -154,7 +156,7 @@ PYBIND11_MODULE(pytiledbsoma, m) {
         .def(py::init<>())
         .def_readwrite("dataframe_dim_zstd_level", &PlatformConfig::dataframe_dim_zstd_level)
         .def_readwrite("sparse_nd_array_dim_zstd_level", &PlatformConfig::sparse_nd_array_dim_zstd_level)
-        .def_readwrite("dense_nd_array_dim_zstd_level", &PlatformConfig::sparse_nd_array_dim_zstd_level)
+        .def_readwrite("dense_nd_array_dim_zstd_level", &PlatformConfig::dense_nd_array_dim_zstd_level)
         .def_readwrite("write_X_chunked", &PlatformConfig::write_X_chunked)
         .def_readwrite("goal_chunk_nnz", &PlatformConfig::goal_chunk_nnz)
         .def_readwrite("remote_cap_nbytes", &PlatformConfig::remote_cap_nbytes)
@@ -166,7 +168,45 @@ PYBIND11_MODULE(pytiledbsoma, m) {
         .def_readwrite("allows_duplicates", &PlatformConfig::allows_duplicates)
         .def_readwrite("tile_order", &PlatformConfig::tile_order)
         .def_readwrite("cell_order", &PlatformConfig::cell_order)
-        .def_readwrite("consolidate_and_vacuum", &PlatformConfig::consolidate_and_vacuum);
+        .def_readwrite("consolidate_and_vacuum", &PlatformConfig::consolidate_and_vacuum)
+        .def(
+            "to_dict",
+            [](const PlatformConfig& c) {
+                return py::dict(
+                    "dataframe_dim_zstd_level"_a = c.dataframe_dim_zstd_level,
+                    "sparse_nd_array_dim_zstd_level"_a = c.sparse_nd_array_dim_zstd_level,
+                    "dense_nd_array_dim_zstd_level"_a = c.dense_nd_array_dim_zstd_level,
+                    "write_X_chunked"_a = c.write_X_chunked,
+                    "goal_chunk_nnz"_a = c.goal_chunk_nnz,
+                    "remote_cap_nbytes"_a = c.remote_cap_nbytes,
+                    "capacity"_a = c.capacity,
+                    "offsets_filters"_a = c.offsets_filters,
+                    "validity_filters"_a = c.validity_filters,
+                    "allows_duplicates"_a = c.allows_duplicates,
+                    "tile_order"_a = c.tile_order,
+                    "cell_order"_a = c.cell_order,
+                    "attrs"_a = c.attrs,
+                    "dims"_a = c.dims,
+                    "consolidate_and_vacuum"_a = c.consolidate_and_vacuum);
+            })
+        .def("__repr__", [](const PlatformConfig& c) {
+            auto q = [](const std::optional<std::string>& v) { return v ? ("'" + *v + "'") : std::string("None"); };
+            std::ostringstream os;
+            os << "PlatformConfig("
+               << "dataframe_dim_zstd_level=" << c.dataframe_dim_zstd_level
+               << ", sparse_nd_array_dim_zstd_level=" << c.sparse_nd_array_dim_zstd_level
+               << ", dense_nd_array_dim_zstd_level=" << c.dense_nd_array_dim_zstd_level
+               << ", write_X_chunked=" << (c.write_X_chunked ? "True" : "False")
+               << ", goal_chunk_nnz=" << c.goal_chunk_nnz << ", remote_cap_nbytes=" << c.remote_cap_nbytes
+               << ", capacity=" << c.capacity << ", offsets_filters='" << c.offsets_filters << "'"
+               << ", validity_filters='" << c.validity_filters << "'"
+               << ", allows_duplicates=" << (c.allows_duplicates ? "True" : "False")
+               << ", tile_order=" << q(c.tile_order) << ", cell_order=" << q(c.cell_order) << ", attrs='" << c.attrs
+               << "'"
+               << ", dims='" << c.dims << "'"
+               << ", consolidate_and_vacuum=" << (c.consolidate_and_vacuum ? "True" : "False") << ")";
+            return os.str();
+        });
 
     py::class_<PlatformSchemaConfig>(m, "PlatformSchemaConfig")
         .def(py::init<>())
@@ -177,7 +217,33 @@ PYBIND11_MODULE(pytiledbsoma, m) {
         .def_readwrite("dims", &PlatformSchemaConfig::dims)
         .def_readwrite("allows_duplicates", &PlatformSchemaConfig::allows_duplicates)
         .def_readwrite("tile_order", &PlatformSchemaConfig::tile_order)
-        .def_readwrite("cell_order", &PlatformSchemaConfig::cell_order);
+        .def_readwrite("cell_order", &PlatformSchemaConfig::cell_order)
+        .def(
+            "to_dict",
+            [](const PlatformSchemaConfig& c) {
+                return py::dict(
+                    "capacity"_a = c.capacity,
+                    "offsets_filters"_a = c.offsets_filters,
+                    "validity_filters"_a = c.validity_filters,
+                    "attrs"_a = c.attrs,
+                    "dims"_a = c.dims,
+                    "allows_duplicates"_a = c.allows_duplicates,
+                    "tile_order"_a = c.tile_order,
+                    "cell_order"_a = c.cell_order);
+            })
+        .def("__repr__", [](const PlatformSchemaConfig& c) {
+            auto q = [](const std::optional<std::string>& v) { return v ? ("'" + *v + "'") : std::string("None"); };
+            std::ostringstream os;
+            os << "PlatformSchemaConfig("
+               << "capacity=" << c.capacity << ", allows_duplicates=" << (c.allows_duplicates ? "True" : "False")
+               << ", tile_order=" << q(c.tile_order) << ", cell_order=" << q(c.cell_order) << ", offsets_filters='"
+               << c.offsets_filters << "'"
+               << ", validity_filters='" << c.validity_filters << "'"
+               << ", attrs='" << c.attrs << "'"
+               << ", dims='" << c.dims << "'"
+               << ")";
+            return os.str();
+        });
 
     m.def("_update_dataframe_schema", &SOMADataFrame::update_dataframe_schema);
 

apis/python/tests/test_platform_config.py

@@ -6,6 +6,7 @@
 import tiledbsoma
 import tiledbsoma.io
 import tiledbsoma.options._tiledb_create_write_options as tco
+from tiledbsoma import pytiledbsoma as clib
 
 from ._util import assert_adata_equal
 
@@ -134,3 +135,97 @@ def test_dig_platform_config():
     # Unrecognized type (at tip)
     with pytest.raises(TypeError):
         tco._dig_platform_config({"a": {"b": "invalid"}}, int, ("a", "b"))
+
+
+def test_platform_schema_config_to_dict_keys():
+    cfg = clib.PlatformSchemaConfig()
+    d = cfg.to_dict()
+    assert type(d) is dict
+    assert set(d) == {
+        "capacity",
+        "offsets_filters",
+        "validity_filters",
+        "attrs",
+        "dims",
+        "allows_duplicates",
+        "tile_order",
+        "cell_order",
+    }
+
+
+def test_platform_schema_config_to_dict_values_match_attrs():
+    cfg = clib.PlatformSchemaConfig()
+    cfg.capacity = 1234
+    cfg.tile_order = "row-major"
+    d = cfg.to_dict()
+    assert d["capacity"] == 1234
+    assert d["tile_order"] == "row-major"
+    assert d["cell_order"] is None
+    assert isinstance(d["allows_duplicates"], bool)
+
+
+def test_platform_schema_config_to_dict_json_fields_parse(conftest_pbmc_small, tmp_path):
+    output_path = tmp_path.as_posix()
+    create_cfg = {
+        "capacity": 8888,
+        "offsets_filters": [
+            "RleFilter",
+            {"_type": "GzipFilter", "level": 7},
+            "NoOpFilter",
+        ],
+        "dims": {
+            "soma_dim_0": {"tile": 6, "filters": ["RleFilter"]},
+            "soma_dim_1": {"filters": []},
+        },
+        "attrs": {"soma_data": {"filters": ["NoOpFilter"]}},
+        "cell_order": "row-major",
+        "tile_order": "column-major",
+    }
+    tiledbsoma.io.from_anndata(
+        output_path,
+        conftest_pbmc_small,
+        "RNA",
+        platform_config={"tiledb": {"create": create_cfg}},
+    )
+    x_arr_uri = str(Path(output_path) / "ms" / "RNA" / "X" / "data")
+    with tiledbsoma.SparseNDArray.open(x_arr_uri) as x_arr:
+        cfg = x_arr.schema_config_options()
+        d = cfg.to_dict()
+        assert d["capacity"] == create_cfg["capacity"]
+        assert d["tile_order"] == create_cfg["tile_order"]
+        assert d["cell_order"] == create_cfg["cell_order"]
+        # JSON-string fields should parse cleanly
+        assert isinstance(json.loads(d["offsets_filters"]), list)
+        assert isinstance(json.loads(d["attrs"]), dict)
+        assert isinstance(json.loads(d["dims"]), dict)
+
+
+def test_platform_schema_config_repr():
+    cfg = clib.PlatformSchemaConfig()
+    r = repr(cfg)
+    assert r.startswith("PlatformSchemaConfig(")
+    assert r.endswith(")")
+    assert "capacity=" in r
+    assert "tile_order=None" in r
+
+
+def test_platform_config_to_dict_keys():
+    cfg = clib.PlatformConfig()
+    d = cfg.to_dict()
+    assert type(d) is dict
+    assert {"capacity", "tile_order", "consolidate_and_vacuum"} <= set(d)
+
+
+def test_platform_config_repr():
+    cfg = clib.PlatformConfig()
+    assert repr(cfg).startswith("PlatformConfig(")
+
+
+def test_platform_config_dense_zstd_level_not_aliased():
+    """Regression: pytiledbsoma.cc:157 used to bind dense_nd_array_dim_zstd_level
+    to &PlatformConfig::sparse_nd_array_dim_zstd_level."""
+    cfg = clib.PlatformConfig()
+    cfg.sparse_nd_array_dim_zstd_level = 1
+    cfg.dense_nd_array_dim_zstd_level = 9
+    assert cfg.sparse_nd_array_dim_zstd_level == 1
+    assert cfg.dense_nd_array_dim_zstd_level == 9

apis/r/DESCRIPTION

@@ -6,7 +6,7 @@ Description: Interface for working with 'TileDB'-based Stack of Matrices,
     like those commonly used for single cell data analysis. It is documented at
     <https://github.com/single-cell-data>; a formal specification available is at
     <https://github.com/single-cell-data/SOMA/blob/main/abstract_specification.md>.
-Version: 2.3.99.4
+Version: 2.3.99.5
 Authors@R: c(
     person(given = "Paul", family = "Hoffman",
            role = c("cre", "aut"), email = "tiledb-r@tiledb.com",

apis/r/NEWS.md

@@ -29,7 +29,8 @@
 - Metadata values retrieved from collection-based classes (`SOMACollection`, `SOMAExperiment`, `SOMAMeasurement`) are no longer wrapped in a named list, consistent with array-based classes ([#4429](https://github.com/single-cell-data/TileDB-SOMA/pull/4429)).
 - [BREAKING] Use stored handle to access `SOMAArrayBase` properties rather than re-opening the `SOMAArrayBase`. Array properties can no longer be accessed on an unopened array. ([#4414](https://github.com/single-cell-data/TileDB-SOMA/pull/4414))
 - Use stored handle to read and write to `SOMASparseNDArray`, `SOMADenseNDArray`, and `SOMADataFrame` rather than re-opening the SOMA oobjects for each read/write. ([#4414](https://github.com/single-cell-data/TileDB-SOMA/pull/4414))
-- Properly close `ms` collection handle in `write_soma.SummarizedExperiment()`, ensuring that new measurements added to the ms collection are persisted to disk. ([#4452](https://github.com/single-cell-data/TileDB-SOMA/pull/4452))
+- Properly close `ms` collection handle in `write_soma.SummarizedExperiment()`, ensuring that new measurements added to the `ms` collection are persisted to disk. ([#4452](https://github.com/single-cell-data/TileDB-SOMA/pull/4452))
+- Fix handling of empty arrays when calling `write_soma()` with `ingest_mode = "resume"`. ([#4443](https://github.com/single-cell-data/TileDB-SOMA/pull/4453))
 
 ## Security
 

apis/r/R/write_soma.R

@@ -716,7 +716,13 @@ write_soma.TsparseMatrix <- function(
       j = bit64::as.integer64(methods::slot(x, "j")),
       x = methods::slot(x, "x")
     )
-    tbl <- tbl[-which(tbl$i %in% row_ids & tbl$j %in% col_ids), , drop = FALSE]
+
+    # Only subset the table if there are existing joinid matches
+    idx <- which(tbl$i %in% row_ids & tbl$j %in% col_ids)
+    if (length(idx) > 0) {
+      tbl <- tbl[-idx, , drop = FALSE]
+    }
+
     x <- if (nrow(tbl)) {
       Matrix::sparseMatrix(
         i = as.integer(tbl$i),

apis/r/tests/testthat/test-14-SummarizedExperimentIngest.R

@@ -58,6 +58,17 @@ test_that("Write SummarizedExperiment mechanics", {
     names(SummarizedExperiment::rowData(se))
   )
 
+  # Verify X data values round-trip correctly
+  for (assay in SummarizedExperiment::assayNames(se)) {
+    original <- SummarizedExperiment::assay(se, assay)
+    stored <- ms$X$get(assay)$read()$sparse_matrix()$concat()
+    expect_equal(
+      sum(stored != 0),
+      sum(original != 0),
+      label = sprintf("non-zero count for assay '%s'", assay)
+    )
+  }
+
   # Test ms_name assertions
   expect_error(write_soma(se, uri))
   expect_error(write_soma(se, uri, ""))
@@ -101,4 +112,8 @@ test_that("Resume-mode adds a second measurement to an existing experiment", {
   on.exit(exp$close(), add = TRUE, after = FALSE)
 
   expect_setequal(exp$ms$names(), c("ms1", "ms2"))
+
+  mat2 <- exp$ms$get("ms2")$X$get("counts")$read()$sparse_matrix()$concat()
+  # Verify that the second measurement's X data is not all zeros (CX-279)
+  expect_true(sum(mat2 != 0) > 0)
 })

apis/r/tests/testthat/test-16-write-soma-resume.R

@@ -314,6 +314,36 @@ test_that("Resume-mode sparse arrays", {
   gc()
 })
 
+test_that("Resume-mode sparse array to new URI preserves data", {
+  collection <- SOMACollectionCreate(tempfile(pattern = "sparse-new-resume"))
+  on.exit(collection$close(), add = TRUE, after = FALSE)
+
+  # Create a sparse matrix with known non-zero values
+  mat <- Matrix::rsparsematrix(10L, 8L, 0.5, repr = "T")
+
+  # Writing a new sparse array in "resume" mode should still write all data even
+  # if the array doesn't exist yet.
+  expect_s3_class(
+    ndarray <- write_soma(
+      mat,
+      uri = "new-array",
+      soma_parent = collection,
+      ingest_mode = "resume"
+    ),
+    "SOMASparseNDArray"
+  )
+  on.exit(ndarray$close(), add = TRUE, after = FALSE)
+
+  ndarray$reopen("READ")
+  result <- ndarray$read()$sparse_matrix()$concat()
+  expect_equal(
+    as.matrix(result),
+    as.matrix(mat),
+    label = "result",
+    expected.label = "mat"
+  )
+})
+
 test_that("Resume-mode dense arrays", {
   skip_if(!extended_tests())
   skip_if_not_installed("datasets")