added objid functions

pyproject.toml

@@ -19,17 +19,18 @@ authors = [{ "name" = "The HDF Group", "email" = "[email protected]" }]
 keywords = ["json", "hdf5", "multidimensional array", "data", "datacube"]
 requires-python = ">=3.8"
 dependencies = [
-    "h5py >=3.10",
+    "h5py >= 3.10",
     "numpy >= 2.0; python_version>='3.9'",
     "jsonschema >=4.4.0",
     "tomli; python_version<'3.11'",
     "numpy >=1.20,<2.0.0; python_version=='3.8'",
 ]
+
 dynamic = ["version"]
 
 [project.urls]
-Homepage = "https://hdf5-json.readthedocs.io"
-Documentation = "https://hdf5-json.readthedocs.io"
+Homepage = "https://support.hdfgroup.org/documentation/hdf5-json/latest/"
+Documentation = "https://support.hdfgroup.org/documentation/hdf5-json/latest/"
 Source = "https://github.com/HDFGroup/hdf5-json"
 "Bug Reports" = "https://github.com/HDFGroup/hdf5-json/issues"
 Social = "https://twitter.com/hdf5"

src/h5json/__init__.py

@@ -21,6 +21,14 @@
 from .hdf5dtype import getTypeResponse
 from .hdf5dtype import getItemSize
 from .hdf5dtype import createDataType
+from .objid import createObjId
+from .objid import getCollectionForId
+from .objid import isObjId
+from .objid import isS3ObjKey
+from .objid import getS3Key
+from .objid import getObjId
+from .objid import isSchema2Id
+from .objid import isRootObjId
 from .hdf5db import Hdf5db
 from . import _version
 

src/h5json/hdf5db.py

@@ -13,12 +13,12 @@
 import time
 import h5py
 import numpy as np
-import uuid
 import os.path as op
 import os
 import json
 import logging
-from .hdf5dtype import getTypeItem, createDataType, getItemSize
+from .hdf5dtype import getTypeItem, createDataType, getItemSize, Reference, RegionReference
+from .objid import createObjId
 from .apiversion import _apiver
 
 
@@ -73,6 +73,43 @@
 _H5PY_COMPRESSION_FILTERS = ("gzip", "lzf", "szip")
 
 
+def convert_dtype(srcdt):
+    """Return a dtype based on input dtype, converting any Reference types from
+    h5json style to h5py.
+    """
+
+    if len(srcdt) > 0:
+        fields = []
+        for name in srcdt.fields:
+            item = srcdt.fields[name]
+            # item is a tuple of dtype and integer offset
+            field_dt = convert_dtype(item[0])
+            fields.append((name, field_dt))
+        tgt_dt = np.dtype(fields)
+    else:
+        # check if this a "special dtype"
+        if srcdt.metadata and "ref" in srcdt.metadata:
+            if srcdt.metadata['ref'] is Reference:
+                tgt_dt = h5py.special_dtype(ref=h5py.Reference)
+            elif srcdt.metadata['ref'] is RegionReference:
+                tgt_dt = h5py.special_dtype(ref=h5py.RegionReference)
+            else:
+                raise TypeError(f"Unexpected ref type: {srcdt}")
+        elif srcdt.metadata and "vlen" in srcdt.metadata:
+            src_vlen = srcdt.metadata["vlen"]
+            if isinstance(src_vlen, np.dtype):
+                tgt_base = convert_dtype(src_vlen)
+            else:
+                tgt_base = src_vlen
+            tgt_dt = h5py.special_dtype(vlen=tgt_base)
+        elif srcdt.kind == "U":
+            # use vlen for unicode strings
+            tgt_dt = h5py.special_dtype(vlen=str)
+        else:
+            tgt_dt = srcdt  # no conversion needed
+    return tgt_dt
+
+
 def visitObj(path, obj):
     hdf5db = _db[obj.file.filename]
     hdf5db.visit(path, obj)
@@ -561,7 +598,7 @@ def initFile(self):
 
         self.log.info("initializing file")
         if not self.root_uuid:
-            self.root_uuid = str(uuid.uuid1())
+            self.root_uuid = createObjId()
         self.dbGrp.attrs["rootUUID"] = self.root_uuid
         self.dbGrp.create_group("{groups}")
         self.dbGrp.create_group("{datasets}")
@@ -593,21 +630,21 @@ def visit(self, path, obj):
             msg = "Unknown object type: " + __name__ + " found during scan of HDF5 file"
             self.log.error(msg)
             raise IOError(errno.EIO, msg)
-        uuid1 = uuid.uuid1()  # create uuid
-        id = str(uuid1)
+        obj_id = createObjId()  # create uuid
+
         addrGrp = self.dbGrp["{addr}"]
         if not self.readonly:
             # storing db in the file itself, so we can link to the object directly
-            col[id] = obj.ref  # save attribute ref to object
+            col[obj_id] = obj.ref  # save attribute ref to object
         else:
             # store path to object
-            col[id] = obj.name
+            col[obj_id] = obj.name
         addr = h5py.h5o.get_info(obj.id).addr
         # store reverse map as an attribute
-        addrGrp.attrs[str(addr)] = id
+        addrGrp.attrs[str(addr)] = obj_id
 
     #
-    # Get Datset creation properties
+    # Get Dataset creation properties
     #
     def getDatasetCreationProps(self, dset_uuid):
         prop_list = {}
@@ -760,7 +797,7 @@ def getObjByPath(self, path):
     def getObjectByUuid(self, col_type, obj_uuid):
         # col_type should be either "datasets", "groups", or "datatypes"
         if col_type not in ("datasets", "groups", "datatypes"):
-            msg = "Unexpectd error, invalid col_type: [" + col_type + "]"
+            msg = "Unexpected error, invalid col_type: [" + col_type + "]"
             self.log.error(msg)
             raise IOError(errno.EIO, msg)
         if col_type == "groups" and obj_uuid == self.dbGrp.attrs["rootUUID"]:
@@ -1087,7 +1124,7 @@ def createCommittedType(self, datatype, obj_uuid=None):
             raise IOError(errno.EPERM, msg)
         datatypes = self.dbGrp["{datatypes}"]
         if not obj_uuid:
-            obj_uuid = str(uuid.uuid1())
+            obj_uuid = createObjId()
         dt = self.createTypeFromItem(datatype)
 
         datatypes[obj_uuid] = dt
@@ -1476,6 +1513,7 @@ def makeAttribute(self, obj, attr_name, shape, attr_type, value):
                     self.makeNullTermStringAttribute(obj, attr_name, strLength, value)
                 else:
                     typeItem = getTypeItem(dt)
+                    dt = convert_dtype(dt)
                     value = self.toRef(rank, typeItem, value)
 
                     # create numpy array
@@ -1725,6 +1763,7 @@ def toNumPyValue(self, typeItem, src, des):
             baseType = typeItem["base"]
 
             dt = self.createTypeFromItem(baseType)
+            dt = convert_dtype(dt)
             des = np.array(src, dtype=dt)
 
         elif typeClass == "H5T_REFERENCE":
@@ -1901,7 +1940,7 @@ def listToRef(self, data):
             # object reference should be in the form: <collection_name>/<uuid>
             for prefix in ("datasets", "groups", "datatypes"):
                 if data.startswith(prefix):
-                    uuid_ref = data[len(prefix) :]
+                    uuid_ref = data[len(prefix):]
                     if len(uuid_ref) == (UUID_LEN + 1) and uuid_ref.startswith("/"):
                         obj = self.getObjectByUuid(prefix, uuid_ref[1:])
                         if obj:
@@ -2193,7 +2232,8 @@ def getDatasetValuesByUuid(self, obj_uuid, slices=Ellipsis, format="json"):
             raise IOError(errno.EIO, msg)
 
         if isinstance(slices, (list, tuple)) and len(slices) != rank:
-            msg = "Unexpected error: getDatasetValuesByUuid: number of dims in selection not same as rank"
+            msg = "Unexpected error: getDatasetValuesByUuid: "
+            msg += "number of dims in selection not same as rank"
             self.log.error(msg)
             raise IOError(errno.EIO, msg)
 
@@ -2715,7 +2755,7 @@ def createDataset(
             raise IOError(errno.EPERM, msg)
         datasets = self.dbGrp["{datasets}"]
         if not obj_uuid:
-            obj_uuid = str(uuid.uuid1())
+            obj_uuid = createObjId()
         dt = None
         item = {}
         fillvalue = None
@@ -3490,7 +3530,7 @@ def createGroup(self, obj_uuid=None):
             raise IOError(errno.EPERM, msg)
         groups = self.dbGrp["{groups}"]
         if not obj_uuid:
-            obj_uuid = str(uuid.uuid1())
+            obj_uuid = createObjId()
         newGroup = groups.create_group(obj_uuid)
         # store reverse map as an attribute
         addr = h5py.h5o.get_info(newGroup.id).addr