Organizing some thoughts here after working with the Catalog for a while, here are some things I wish acted differently.
- PDF previews - PDF previews (re-presented thumbnails?) are rendered at low resolution, making it difficult to use interactively. I understand that lambda's have restrictions in time and size, but I would appreciate 300ppi pdf output >>> 50ppi or whatever blocky resolution is default now. This doesn't seem to be a problem for PNG previews.
- Unknown file type previews - Great to have an early exit if you don't know how to preview a file, but this blocks easy inspection of unknown file types by extension. I would prefer to treat everything as a text file if the format is unknown with clear early binary detection logic to prevent loading nonsense. This would be nice because lots of pipelines write files with weird formats or extensions, but 90% of the time they are just text files. (ex. geneInfo.tab, fasta.fai, h5 (not h5ad), quant.sf, etc.)
- h5ad previews - The preview of small ~ (<5Gb) h5ad files reliably aid in quickly identifying single cell datasets and their features. This is useful for identification of stage of data (raw, normalized, annotated) as well as feature sets available. Pulling out Cell, Gene, and Expression embedding layers is powerful because now I can query that key without downloading the full object.
The problem lies with scale, and I don't think that there is an immediate change that can be made to address this limitation. When reading very large h5ad files in the preview, I time out. Additionally, I never can render tabular data, but a 5x5 preview of each matrix would be enough to satisfy quick inspection. To this end, I propose a new feature branch to write a new h5ad preview parser using h5py tailored to single cell datasets. I have a rough demo of streaming byte loading selectively. It would be great to get more guidance on where this type of extension should be implemented.
class S3HDF5File:
"""Read-only file-like wrapper that serves S3 objects to h5py via range requests.
Each ``read()`` call issues a single ``GetObject`` with a ``Range`` header
so that only the HDF5 chunks actually touched by h5py are transferred.
"""
__slots__ = ("_s3", "_bucket", "_key", "_version_id", "_pos", "_size")
def __init__(
self,
s3_client: Any,
bucket: str,
key: str,
version_id: str | None = None,
file_size: int | None = None,
) -> None:
self._s3 = s3_client
self._bucket = bucket
self._key = key
self._version_id = version_id
self._pos = 0
if file_size is None:
kwargs: dict[str, Any] = {"Bucket": bucket, "Key": key}
if version_id:
kwargs["VersionId"] = version_id
head = s3_client.head_object(**kwargs)
file_size = head["ContentLength"]
self._size = file_size
# -- file-like interface required by h5py ------------------------------
def read(self, size: int = -1) -> bytes:
if size == 0:
return b""
if size < 0:
size = self._size - self._pos
end = min(self._pos + size - 1, self._size - 1)
if self._pos > end:
return b""
kwargs: dict[str, Any] = {
"Bucket": self._bucket,
"Key": self._key,
"Range": f"bytes={self._pos}-{end}",
}
if self._version_id:
kwargs["VersionId"] = self._version_id
resp = self._s3.get_object(**kwargs)
data: bytes = resp["Body"].read()
self._pos += len(data)
return data
def seek(self, pos: int, whence: int = 0) -> int:
if whence == 0:
self._pos = pos
elif whence == 1:
self._pos += pos
elif whence == 2:
self._pos = self._size + pos
return self._pos
def tell(self) -> int:
return self._pos
@property
def size(self) -> int:
return self._size
def seekable(self) -> bool:
return True
def readable(self) -> bool:
return True
def writable(self) -> bool:
return False
def close(self) -> None:
pass
def __enter__(self) -> S3HDF5File:
return self
def __exit__(self, *args: object) -> None:
self.close()
def _extract_from_h5(
h5: h5py.File,
*,
obsm_keys: Sequence[str],
obs_columns: Sequence[str] | None,
) -> pd.DataFrame:
"""Read obs metadata + selected obsm arrays from an open h5py handle."""
obs = h5["obs"]
obsm = h5["obsm"]
obs_index = _read_obs_index(obs)
embedding_arrays: dict[str, np.ndarray] = {}
for key in obsm_keys:
if key not in obsm:
available = list(obsm.keys())
msg = f"Embedding '{key}' not found in obsm. Available: {available}"
raise KeyError(msg)
embedding_arrays[key] = obsm[key][:]
available_obs = set(obs.keys())
columns = list(obs_columns) if obs_columns is not None else DEFAULT_OBS_COLUMNS
columns = [c for c in columns if c in available_obs]
obs_dict: dict[str, np.ndarray | list] = {}
for col in columns:
obs_dict[col] = _read_obs_column(obs, col)
df = pd.DataFrame(obs_dict, index=obs_index)
for key, arr in embedding_arrays.items():
if arr.ndim == 1:
df[key] = arr
else:
for i in range(arr.shape[1]):
df[f"{key}_{i + 1}"] = arr[:, i]
return df
def load_h5ad_metadata(
entry: Any,
*,
obsm_keys: Sequence[str] = ("X_umap_3d",),
obs_columns: Sequence[str] | None = None,
) -> pd.DataFrame:
"""Load only cell metadata and embeddings from a packaged h5ad file.
Uses Quilt's authenticated boto3 session to read the h5ad via **S3
range requests** — only the HDF5 chunks containing ``obs`` and the
requested ``obsm`` keys are transferred. The count matrix (``X``),
layers, and other heavy arrays are never downloaded.
Parameters
----------
entry:
A ``quilt3.PackageEntry`` pointing at an ``.h5ad`` object in S3.
obsm_keys:
Keys from ``adata.obsm`` to include. Each array with *n* columns
produces columns ``{key}_1 … {key}_n``.
obs_columns:
Observation metadata columns to extract. Columns absent from the
file are silently skipped. Defaults to :data:`DEFAULT_OBS_COLUMNS`.
Returns
-------
pd.DataFrame
A DataFrame indexed by cell barcode with the requested metadata
and embedding coordinates.
"""
import quilt3 # noqa: PLC0415 – keep quilt3 lazy for test/import speed
pk = entry.physical_key
sess = quilt3.session.get_boto3_session()
s3 = sess.client("s3")
s3file = S3HDF5File(s3, pk.bucket, pk.path, pk.version_id)
with h5py.File(s3file, "r") as f:
return _extract_from_h5(f, obsm_keys=obsm_keys, obs_columns=obs_columns)Would love to contribute!
Organizing some thoughts here after working with the Catalog for a while, here are some things I wish acted differently.
The problem lies with scale, and I don't think that there is an immediate change that can be made to address this limitation. When reading very large h5ad files in the preview, I time out. Additionally, I never can render tabular data, but a 5x5 preview of each matrix would be enough to satisfy quick inspection. To this end, I propose a new feature branch to write a new h5ad preview parser using
h5pytailored to single cell datasets. I have a rough demo of streaming byte loading selectively. It would be great to get more guidance on where this type of extension should be implemented.Would love to contribute!