[c++/python/r] Pushdown SOMA object management to C++

apis/python/src/tiledbsoma/_collection.py

@@ -7,7 +7,6 @@
 from __future__ import annotations
 
 import abc
-import itertools
 from collections.abc import MutableMapping
 from typing import Any, Callable, ClassVar, Final, TypeVar, cast, overload
 
@@ -346,36 +345,11 @@ def members(self) -> dict[str, tuple[str, str]]:
         """Get a mapping of {member_name: (uri, soma_object_type)}."""
         return cast("dict[str, tuple[str, str]]", self._handle.members())
 
-    def __repr__(self) -> str:
-        """Default display for :class:`Collection`."""
-        lines = itertools.chain((self._my_repr(),), self._contents_lines(""))
-        return "<" + "\n".join(lines) + ">"
-
     # ================================================================
     # PRIVATE METHODS FROM HERE ON DOWN
     # ================================================================
 
-    def _my_repr(self) -> str:
-        start = super()._my_repr()
-        if self.closed:
-            return start
-        n = len(self)
-        if n == 0:
-            count = "empty"
-        elif n == 1:
-            count = "1 item"
-        else:
-            count = f"{n} items"
-        return f"{start} ({count})"
-
-    def _set_element(
-        self,
-        key: str,
-        *,
-        uri: str,
-        relative: bool,
-        soma_object: _TDBO,
-    ) -> None:
+    def _set_element(self, key: str, *, uri: str, relative: bool, soma_object: _TDBO, managed: bool = False) -> None:
         """Internal implementation of element setting.
 
         Args:
@@ -389,7 +363,7 @@ def _set_element(
                 The reified SOMA object to store locally.
         """
         self._check_allows_child(key, type(soma_object))
-        super()._set_element(key, uri=uri, relative=relative, soma_object=soma_object)
+        super()._set_element(key, uri=uri, relative=relative, soma_object=soma_object, managed=managed)
 
     @classmethod
     def _check_allows_child(cls, key: str, child_cls: type) -> None:

apis/python/src/tiledbsoma/_dataframe.py

@@ -251,9 +251,7 @@ def create(
             if is_does_not_exist_error(tdbe):
                 raise DoesNotExistError(tdbe) from tdbe
             raise SOMAError(tdbe) from tdbe
-        return cls(
-            handle, uri=uri, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code"
-        )
+        return cls(handle, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code")
 
     def keys(self) -> tuple[str, ...]:
         """Returns the names of the columns when read back as a dataframe.

apis/python/src/tiledbsoma/_factory.py

@@ -160,9 +160,7 @@ def _open_soma_object(
             raise SOMAError(tdbe) from tdbe
         try:
             cls: type[SOMAObject] = _type_name_to_cls(handle.type.lower())
-            return cls(
-                handle, uri=uri, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code"
-            )
+            return cls(handle, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code")
         except KeyError:
             raise SOMAError(f"{uri!r} has unknown storage type {clib_type!r}") from None
 

apis/python/src/tiledbsoma/_geometry_dataframe.py

@@ -295,9 +295,7 @@ def create(
             if is_does_not_exist_error(tdbe):
                 raise DoesNotExistError(tdbe) from tdbe
             raise SOMAError(tdbe) from tdbe
-        return cls(
-            handle, uri=uri, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code"
-        )
+        return cls(handle, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code")
 
     def _parse_special_metadata(self) -> None:
         # Get and validate coordinate space.

apis/python/src/tiledbsoma/_multiscale_image.py

@@ -222,10 +222,8 @@ def create(
             )
             metadata = _tdb_handles.MetadataWrapper.from_handle(handle)
             metadata[SOMA_MULTISCALE_IMAGE_SCHEMA] = image_meta_str_
-            metadata._write()
             multiscale = cls(
                 handle,
-                uri=uri,
                 context=context,
                 _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
             )
@@ -625,7 +623,9 @@ def has_channel_axis(self) -> bool:
 
     def levels(self) -> dict[str, tuple[str, tuple[int, ...]]]:
         """Returns a mapping of {member_name: (uri, shape)}."""
-        return {level.name: (self._contents[level.name].uri, level.shape) for level in self._levels}
+        members = self._handle.members()
+
+        return {level.name: (members[level.name][0], level.shape) for level in self._levels}
 
     @property
     def level_count(self) -> int:
@@ -657,7 +657,7 @@ def level_uri(self, level: int | str) -> str:
         """
         if isinstance(level, int):
             level = self._levels[level].name
-        return self._contents[level].uri
+        return str(self._handle.members()[level][0])
 
     @property
     def nchannels(self) -> int:

apis/python/src/tiledbsoma/_point_cloud_dataframe.py

@@ -277,9 +277,7 @@ def create(
             if is_does_not_exist_error(tdbe):
                 raise DoesNotExistError(tdbe) from tdbe
             raise SOMAError(tdbe) from tdbe
-        return cls(
-            handle, uri=uri, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code"
-        )
+        return cls(handle, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code")
 
     def _parse_special_metadata(self) -> None:
         # Get and validate coordinate space.

apis/python/src/tiledbsoma/_scene.py

@@ -163,9 +163,7 @@ def create(
             if is_does_not_exist_error(tdbe):
                 raise DoesNotExistError(tdbe) from tdbe
             raise SOMAError(tdbe) from tdbe
-        return cls(
-            handle, uri=uri, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code"
-        )
+        return cls(handle, context=context, _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code")
 
     def _parse_special_metadata(self) -> None:
         coord_space = self.metadata.get(SOMA_COORDINATE_SPACE_METADATA_KEY)

apis/python/src/tiledbsoma/_soma_group.py

@@ -5,8 +5,7 @@
 from __future__ import annotations
 
 import warnings
-from collections.abc import Iterable, Iterator
-from threading import Lock
+from collections.abc import Iterator
 from typing import Any, Callable, Generic, TypeVar, cast
 
 import attrs
@@ -17,7 +16,7 @@
 # This package's pybind11 code
 from . import pytiledbsoma as clib
 from ._core_options import OpenMode
-from ._exception import SOMAError, UnsupportedOperationError, is_does_not_exist_error
+from ._exception import DoesNotExistError, SOMAError, UnsupportedOperationError, is_does_not_exist_error
 from ._soma_context import SOMAContext
 from ._soma_object import SOMAObject
 from ._types import OpenTimestamp, SOMABaseTileDBType
@@ -53,56 +52,50 @@ class SOMAGroup(SOMAObject, Generic[CollectionElementType]):
         Experimental.
     """
 
-    __slots__ = ("_contents", "_mutated_keys", "_reify_lock")
-
     def __init__(
         self,
         handle: _tdb_handles.RawHandle,
         *,
-        uri: str,
         context: SOMAContext,
         **kwargs: Any,  # noqa: ANN401
     ) -> None:
-        super().__init__(handle, uri=uri, context=context, **kwargs)
-        self._contents = {
-            name: _CachedElement.from_handle_entry(entry) for name, entry in self._handle.members().items()
-        }
+        super().__init__(handle, context=context, **kwargs)
         """The contents of the persisted TileDB Group.
 
         This is loaded at startup when we have a read handle.
         """
-        self._mutated_keys: set[str] = set()
-        self._reify_lock = Lock()
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._handle
 
     def __len__(self) -> int:
         """Return the number of members in the collection."""
-        return len(self._contents)
+        return int(self._handle.__len__())
 
     def __getitem__(self, key: str) -> CollectionElementType:
         """Gets the value associated with the key."""
         err_str = f"{self.__class__.__name__} has no item {key!r}"
 
-        try:
-            entry = self._contents[key]
-        except KeyError:
+        if not isinstance(key, str) or key not in self._handle:
             raise KeyError(err_str) from None
 
-        with self._reify_lock:
-            if entry.soma is None:
-                from . import _factory  # Delayed binding to resolve circular import.
-
-                entry.soma = _factory._open_soma_object(
-                    uri=entry.uri,
-                    mode=self.mode,
-                    context=self.context,
-                    tiledb_timestamp=self.tiledb_timestamp_ms,
-                    clib_type=None if entry.tiledb_type is None else entry.tiledb_type.name,
-                )
-
-                # Since we just opened this object, we own it and should close it.
-                self._close_stack.enter_context(entry.soma)
+        try:
+            handle: clib = getattr(self._handle, key) if hasattr(self._handle, key) else self._handle.get(key)
+        except (RuntimeError, SOMAError) as err:
+            if is_does_not_exist_error(err):
+                raise DoesNotExistError(err) from err
+            raise err
+
+        from . import _factory  # Delayed binding to resolve circular import.
+
+        cls: type[SOMAObject] = _factory._type_name_to_cls(handle.type.lower())
+        soma_object = cls(
+            handle,
+            context=self.context,
+            _dont_call_this_use_create_or_open_instead="tiledbsoma-internal-code",
+        )
 
-        return cast("CollectionElementType", entry.soma)
+        return cast("CollectionElementType", soma_object)
 
     def __setitem__(self, key: str, value: CollectionElementType) -> None:
         """Default collection __setattr__."""
@@ -118,30 +111,9 @@ def __delitem__(self, key: str) -> None:
         self._del_element(key)
 
     def __iter__(self) -> Iterator[str]:
-        return iter(self._contents)
-
-    def _contents_lines(self, last_indent: str) -> Iterable[str]:
-        indent = last_indent + "    "
-        if self.closed:
-            return
-        for key, entry in self._contents.items():
-            obj = entry.soma
-            if obj is None:
-                # We haven't reified this SOMA object yet. Don't try to open it.
-                yield f"{indent}{key!r}: {entry.uri!r} (unopened)"
-            else:
-                yield f"{indent}{key!r}: {obj._my_repr()}"
-                if isinstance(obj, SOMAGroup):
-                    yield from obj._contents_lines(indent)
+        return iter(self._handle.members())
 
-    def _set_element(
-        self,
-        key: str,
-        *,
-        uri: str,
-        relative: bool,
-        soma_object: _TDBO,
-    ) -> None:
+    def _set_element(self, key: str, *, uri: str, relative: bool, soma_object: _TDBO, managed: bool = False) -> None:
         """Internal implementation of element setting.
 
         Args:
@@ -154,25 +126,20 @@ def _set_element(
             value:
                 The reified SOMA object to store locally.
         """
-        if key in self._mutated_keys.union(self._contents):
-            # TileDB groups currently do not support replacing elements.
-            # If we use a hack to flush writes, corruption is possible.
-            raise SOMAError(f"replacing key {key!r} is unsupported")
-        clib_collection = self._handle
         relative_type = clib.URIType.relative if relative else clib.URIType.absolute
-        if self.context.is_tiledbv2_uri(self.uri):
-            clib_collection.add(
+        try:
+            self._handle.add(
                 uri=uri,
                 uri_type=relative_type,
                 name=key,
                 soma_type=soma_object.soma_type,
+                member=soma_object._handle,
+                managed=managed,
             )
-        self._contents[key] = _CachedElement(uri=soma_object.uri, tiledb_type=None, soma=soma_object)
-        self._mutated_keys.add(key)
+        except ValueError as err:
+            raise SOMAError(err) from err
 
     def _del_element(self, key: str) -> None:
-        if key in self._mutated_keys:
-            raise SOMAError(f"Cannot delete previously-mutated key '{key!r}'.")
         try:
             if self.closed:
                 raise SOMAError(f"Cannot delete '{key!r}'. {self} is closed")
@@ -189,12 +156,12 @@ def _del_element(self, key: str) -> None:
                 raise SOMAError(
                     f"Deleting is not allowed in mode '{self.mode}'. {self} should be reopened with mode='d'."
                 )
+        except ValueError as err:
+            raise SOMAError(err) from err
         except Exception as tdbe:
             if is_does_not_exist_error(tdbe):
                 raise KeyError(tdbe) from tdbe
             raise
-        self._contents.pop(key, None)
-        self._mutated_keys.add(key)
 
     def _add_new_element(
         self,
@@ -229,12 +196,7 @@ def _add_new_element(
             )
 
         child = factory(child_uri.full_uri)
-        self._set_element(
-            key,
-            uri=child_uri.add_uri,
-            relative=child_uri.relative,
-            soma_object=child,
-        )
+        self._set_element(key, uri=child_uri.add_uri, relative=child_uri.relative, soma_object=child, managed=True)
         self._close_stack.enter_context(child)
         return child
 
@@ -285,10 +247,6 @@ def reopen(self, mode: OpenMode, tiledb_timestamp: OpenTimestamp | None = None)
             Experimental.
         """
         super().reopen(mode, tiledb_timestamp)
-        self._contents = {
-            name: _CachedElement.from_handle_entry(entry) for name, entry in self._handle.members().items()
-        }
-        self._mutated_keys = set()
         return self
 
     def set(