Dataset2D from `read_lazy` fails when I try to call an unsupported column

[selmanozleyen]

Please make sure these conditions are met

• I have checked that this issue has not already been reported.
• I have confirmed this bug exists on the latest version of anndata.
• (optional) I have confirmed this bug exists on the main branch of anndata.

Report

Code:

import anndata as ad
import h5py
import zarr
from anndata.experimental import read_lazy
with h5py.File("/lustre/groups/ml01/workspace/100mil/100m_int_indices.h5ad", "r") as f:
    adata_all = ad.AnnData(
        obs=read_lazy(f["obs"]),
        var=read_lazy(f["var"]),
        uns=read_lazy(f["uns"]),
        obsm=read_lazy(f["obsm"]),
    )
adata_all.obs['cell_line'] # fails here (it is categorical data)

This is what it looks like normally

0           CVCL_0131
1           CVCL_0480
2           CVCL_0293
3           CVCL_0397
4           CVCL_1097
              ...    
95624329    CVCL_0504
95624330    CVCL_1693
95624331    CVCL_1381
95624332    CVCL_1285
95624333    CVCL_1550
Name: cell_line, Length: 95624334, dtype: category
Categories (50, object): ['CVCL_0023', 'CVCL_0028', 'CVCL_0069', 'CVCL_0099', ..., 'CVCL_1717', 'CVCL_1724', 'CVCL_1731', 'CVCL_C466']

Traceback:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/IPython/core/formatters.py:770, in PlainTextFormatter.__call__(self, obj)
    763 stream = StringIO()
    764 printer = pretty.RepresentationPrinter(stream, self.verbose,
    765     self.max_width, self.newline,
    766     max_seq_length=self.max_seq_length,
    767     singleton_pprinters=self.singleton_printers,
    768     type_pprinters=self.type_printers,
    769     deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
    771 printer.flush()
    772 return stream.getvalue()

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/IPython/lib/pretty.py:411, in RepresentationPrinter.pretty(self, obj)
    400                         return meth(obj, self, cycle)
    401                 if (
    402                     cls is not object
    403                     # check if cls defines __repr__
   (...)    409                     and callable(_safe_getattr(cls, "__repr__", None))
    410                 ):
--> 411                     return _repr_pprint(obj, self, cycle)
    413     return _default_pprint(obj, self, cycle)
    414 finally:
...

File h5py/h5d.pyx:399, in h5py.h5d.DatasetID.get_type()

ValueError: Invalid dataset identifier (identifier is not of specified type)
Error raised while reading key '??' of <class 'h5py._hl.dataset.Dataset'> from /

Versions

| Package | Version                 |
| ------- | ----------------------- |
| xarray  | 2025.9.0                |
| anndata | 0.13.0.dev28+g1ba19458f |
| h5py    | 3.14.0                  |
| zarr    | 3.1.2                   |
| pandas  | 2.3.2                   |

| Dependency         | Version               |
| ------------------ | --------------------- |
| packaging          | 25.0                  |
| debugpy            | 1.8.12                |
| msgpack            | 1.1.1                 |
| cupy-cuda12x       | 13.6.0                |
| urllib3            | 2.5.0                 |
| setuptools         | 80.9.0                |
| click              | 8.2.1                 |
| Pygments           | 2.19.2                |
| legacy-api-wrap    | 1.4.1                 |
| cloudpickle        | 3.1.1                 |
| donfig             | 0.8.1.post1           |
| executing          | 2.2.1                 |
| certifi            | 2025.8.3 (2025.08.03) |
| jupyter_core       | 5.8.1                 |
| ipywidgets         | 8.1.7                 |
| tqdm               | 4.67.1                |
| jedi               | 0.19.2                |
| ipykernel          | 6.29.5                |
| crc32c             | 2.7.1                 |
| MarkupSafe         | 3.0.2                 |
| attrs              | 25.3.0                |
| stack-data         | 0.6.3                 |
| fastrlock          | 0.8.3                 |
| jupyter_client     | 8.6.3                 |
| wcwidth            | 0.2.13                |
| prompt_toolkit     | 3.0.52                |
| parso              | 0.8.5                 |
| six                | 1.17.0                |
| sortedcontainers   | 2.4.0                 |
| idna               | 3.10                  |
| Jinja2             | 3.1.6                 |
| requests           | 2.32.5                |
| simplejson         | 3.20.1                |
| omegaconf          | 2.3.0                 |
| scipy              | 1.15.3                |
| pyarrow            | 21.0.0                |
| pure_eval          | 0.2.3                 |
| tblib              | 3.1.0                 |
| fsspec             | 2025.9.0              |
| pytz               | 2025.2                |
| toolz              | 1.0.0                 |
| comm               | 0.2.2                 |
| numpy              | 2.2.6                 |
| ipython            | 9.5.0                 |
| typing_extensions  | 4.15.0                |
| torch              | 2.8.0                 |
| locket             | 1.0.0                 |
| numcodecs          | 0.16.2                |
| asttokens          | 3.0.0                 |
| natsort            | 8.4.0                 |
| dask               | 2025.9.1              |
| distributed        | 2025.9.1              |
| scanpy             | 1.11.4                |
| python-dateutil    | 2.9.0.post0           |
| psutil             | 7.0.0                 |
| PyYAML             | 6.0.2                 |
| platformdirs       | 4.4.0                 |
| rich               | 14.1.0                |
| traitlets          | 5.14.3                |
| tornado            | 6.5.2                 |
| session-info2      | 0.2.1                 |
| zict               | 3.0.0                 |
| pyzmq              | 27.1.0                |
| decorator          | 5.2.1                 |
| charset-normalizer | 3.4.3                 |

| Component | Info                                                                           |
| --------- | ------------------------------------------------------------------------------ |
| Python    | 3.12.11 | packaged by conda-forge | (main, Jun  4 2025, 14:45:31) [GCC 13.3.0] |
| OS        | Linux-5.14.0-570.25.1.el9_6.x86_64-x86_64-with-glibc2.34                       |
| Updated   | 2025-10-16 13:10                                                              

[ilan-gold]

@selmanozleyen can you post a longer stack trace?

[ilan-gold]

Separately, why use read_lazy and not read_elem_lazy for the sub-elements?

[ilan-gold]

Setting aside the performance (see #2121), I was able to do

with h5py.File("/lustre/groups/ml01/workspace/100mil/100m_int_indices.h5ad", "r") as f:
     read_elem_lazy(f["obs"])

with your file.

[selmanozleyen]

@ilan-gold it fails when I call obs['cell_line'] also in read_elem_lazy

here is the full tb.

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/IPython/core/formatters.py:406, in BaseFormatter.__call__(self, obj)
    404     method = get_real_method(obj, self.print_method)
    405     if method is not None:
--> 406         return method()
    407     return None
    408 else:

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/common.py:189, in AbstractArray._repr_html_(self)
    187 if OPTIONS["display_style"] == "text":
    188     return f"<pre>{escape(repr(self))}</pre>"
--> 189 return formatting_html.array_repr(self)

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/formatting_html.py:334, in array_repr(arr)
    326 arr_name = f"'{arr.name}'" if getattr(arr, "name", None) else ""
    328 header_components = [
    329     f"<div class='xr-obj-type'>{obj_type}</div>",
    330     f"<div class='xr-array-name'>{arr_name}</div>",
    331     format_dims(dims, indexed_dims),
    332 ]
--> 334 sections = [array_section(arr)]
    336 if hasattr(arr, "coords"):
    337     sections.append(coord_section(arr.coords))

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/formatting_html.py:244, in array_section(obj)
    242 variable = getattr(obj, "variable", obj)
    243 preview = escape(inline_variable_array_repr(variable, max_width=70))
--> 244 data_repr = short_data_repr_html(obj)
    245 data_icon = _icon("icon-database")
    247 return (
    248     "<div class='xr-array-wrap'>"
    249     f"<input id='{data_id}' class='xr-array-in' type='checkbox' {collapsed}>"
   (...)    253     "</div>"
    254 )

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/formatting_html.py:43, in short_data_repr_html(array)
     41 if hasattr(internal_data, "_repr_html_"):
     42     return internal_data._repr_html_()
---> 43 text = escape(short_data_repr(array))
     44 return f"<pre>{text}</pre>"

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/formatting.py:673, in short_data_repr(array)
    671 if isinstance(array, np.ndarray):
    672     return short_array_repr(array)
--> 673 elif is_duck_array(internal_data):
    674     return limit_lines(repr(array.data), limit=40)
    675 elif getattr(array, "_in_memory", None):

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/namedarray/utils.py:81, in is_duck_array(value)
     76 if isinstance(value, np.ndarray):
     77     return True
     78 return (
     79     hasattr(value, "ndim")
     80     and hasattr(value, "shape")
---> 81     and hasattr(value, "dtype")
     82     and (
     83         (hasattr(value, "__array_function__") and hasattr(value, "__array_ufunc__"))
     84         or hasattr(value, "__array_namespace__")
     85     )
     86 )

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/xarray/core/utils.py:661, in NDArrayMixin.dtype(self)
    659 @property
    660 def dtype(self: Any) -> np.dtype:
--> 661     return self.array.dtype

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/functools.py:998, in cached_property.__get__(self, instance, owner)
    996 val = cache.get(self.attrname, _NOT_FOUND)
    997 if val is _NOT_FOUND:
--> 998     val = self.func(instance)
    999     try:
   1000         cache[self.attrname] = val

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/anndata/experimental/backed/_lazy_arrays.py:130, in CategoricalArray.dtype(self)
    128 @cached_property
    129 def dtype(self):
--> 130     return pd.CategoricalDtype(categories=self.categories, ordered=self._ordered)

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/functools.py:998, in cached_property.__get__(self, instance, owner)
    996 val = cache.get(self.attrname, _NOT_FOUND)
    997 if val is _NOT_FOUND:
--> 998     val = self.func(instance)
    999     try:
   1000         cache[self.attrname] = val

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/anndata/experimental/backed/_lazy_arrays.py:115, in CategoricalArray.categories(self)
    112     return self._categories[...]
    113 from ..._io.h5ad import read_dataset
--> 115 return read_dataset(self._categories)

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/anndata/_io/utils.py:210, in report_read_key_on_error.<locals>.func_wrapper(*args, **kwargs)
    208     raise ValueError(msg)
    209 try:
--> 210     return func(*args, **kwargs)
    211 except Exception as e:
    212     path, key = _get_display_path(store).rsplit("/", 1)

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/anndata/_io/h5ad.py:352, in read_dataset(dataset)
    349 @report_read_key_on_error
    350 def read_dataset(dataset: h5py.Dataset):
    351     if H5PY_V3:
--> 352         string_dtype = h5py.check_string_dtype(dataset.dtype)
    353         if (string_dtype is not None) and (string_dtype.encoding == "utf-8"):
    354             dataset = dataset.asstr()

File h5py/_objects.pyx:56, in h5py._objects.with_phil.wrapper()

File h5py/_objects.pyx:57, in h5py._objects.with_phil.wrapper()

File /ictstr01/home/icb/selman.ozleyen/.local/share/mamba/envs/lpert/lib/python3.12/site-packages/h5py/_hl/dataset.py:590, in Dataset.dtype(self)
    586 @property
    587 @with_phil
    588 def dtype(self):
    589     """Numpy dtype representing the datatype"""
--> 590     return self.id.dtype

File h5py/h5d.pyx:196, in h5py.h5d.DatasetID.dtype.__get__()

File h5py/h5d.pyx:198, in h5py.h5d.DatasetID.dtype.__get__()

File h5py/_objects.pyx:56, in h5py._objects.with_phil.wrapper()

File h5py/_objects.pyx:57, in h5py._objects.with_phil.wrapper()

File h5py/h5d.pyx:399, in h5py.h5d.DatasetID.get_type()

ValueError: Invalid dataset identifier (identifier is not of specified type)
Error raised while reading key '??' of <class 'h5py._hl.dataset.Dataset'> from /