Implement Census Visium Builder Prototype

api/python/cellxgene_census/pyproject.toml

@@ -31,7 +31,10 @@ dependencies= [
     # NOTE: the tiledbsoma version must be >= to the version used in the Census builder, to
     # ensure that the assets are readable (tiledbsoma supports backward compatible reading).
     # Make sure this version does not fall behind the builder's tiledbsoma version.
-    "tiledbsoma~=1.11.4",
+    # TODO (spatial): tiledbsoma pin to a PyPI release is temporarily commented out in favor git commit pin
+    # "tiledbsoma==1.11.4",
+    # TODO (spatial): Pin tiledbsoma dependency to an actual released version after tiledbsoma spatial code has been released
+    "tiledbsoma @ git+https://github.com/single-cell-data/TileDB-SOMA.git@9eb540f#egg=tiledbsoma&subdirectory=apis/python/",
     "anndata",
     "numpy>=1.21,<2.0",
     "requests",

tools/cellxgene_census_builder/Dockerfile

@@ -6,12 +6,14 @@ ARG COMMIT_SHA
 ENV COMMIT_SHA=${COMMIT_SHA}
 
 # Ubuntu 22 contains only the python3.11 RC as of 2023-12-21, so use deadsnakes
+# TODO (spatial): `git` is added to this dockerfile to be able to install python packages from github. Remove when it is not needed.
 RUN apt update && \
   apt install -y software-properties-common && \
   add-apt-repository -y ppa:deadsnakes/ppa && \
   apt update && \
   apt -y full-upgrade && \
   apt -y install python3.11 python3.11-venv python3-pip awscli && \
+  apt -y install git && \
   apt-get clean
 
 # set python3.11 as default

tools/cellxgene_census_builder/SPATIAL-README.md

@@ -0,0 +1,9 @@
+## Development Environment Setup and Run
+
+- `pip install -e tools/cellxgene_census_builder`
+
+  **NOTE:** When running the builder on MacOS, unpin `pyarrow` in [census builder pyproject.toml](./pyproject.toml)
+
+- `pip install -e api/python/cellxgene_census`
+
+- [Dev tools for spatial](./spatial_dev_tools/) contains scripts and notebooks to aid development and testing

tools/cellxgene_census_builder/pyproject.toml

@@ -34,8 +34,12 @@ dependencies= [
     # recent cellxgene-census _readers_ are able to read the results of a Census build (writer).
     # The compatibility matrix is defined here:
     #    https://github.com/TileDB-Inc/TileDB/blob/dev/format_spec/FORMAT_SPEC.md
-    "tiledbsoma==1.9.3",
-    "cellxgene-census==1.12.0",
+    # TODO (spatial): tiledbsoma pin to a PyPI release is temporarily commented out in favor git commit pin
+    # "tiledbsoma==1.9.3",
+    # TODO (spatial): Pin tiledbsoma dependency to an actual released version after tiledbsoma spatial code has been released
+    "tiledbsoma @ git+https://github.com/single-cell-data/TileDB-SOMA.git@9eb540f#egg=tiledbsoma&subdirectory=apis/python/",
+    # TODO (spatial): Uncomment line below to install "cellxgene-census" at an appropriate version when tiledbsoma spatial code has been released 
+    # "cellxgene-census==1.12.0",
     "cellxgene-ontology-guide==0.6.1",
     "scipy==1.12.0",
     "fsspec[http]==2024.3.1",