ENH: Data management update to support SUA ifaces for Homogen OneDAL tables (uxlfoundation#2045)

* ENH: Data management update to support SUA ifaces

* TEST: added memory leak tests

* TEST: enabled tests for the dpnp/dpctl inputs

* Support of __sycl_usm_array_interface__ implemented without dpctl as a build dependency.

* Covered by tests:

* refactored onedal/utils/_array_api.py file.

* updated take primitive in sklearnex/tests/test_memory_usage.py.

* Documented code

Author: samir-nasibli

onedal/datatypes/_data_conversion.py

@@ -22,40 +22,24 @@
 from onedal import _backend, _is_dpc_backend
 
 from ..utils import _is_csr
-from ..utils._dpep_helpers import is_dpctl_available
 
-dpctl_available = is_dpctl_available("0.14")
 
-if dpctl_available:
-    import dpctl
-    import dpctl.tensor as dpt
-
-
-def _apply_and_pass(func, *args):
+def _apply_and_pass(func, *args, **kwargs):
     if len(args) == 1:
-        return func(args[0])
-    return tuple(map(func, args))
+        return func(args[0], **kwargs)
+    return tuple(map(lambda arg: func(arg, **kwargs), args))
 
 
-def from_table(*args):
-    return _apply_and_pass(_backend.from_table, *args)
+if _is_dpc_backend:
 
+    from ..utils._dpep_helpers import dpctl_available, dpnp_available
 
-def convert_one_to_table(arg):
     if dpctl_available:
-        if isinstance(arg, dpt.usm_ndarray):
-            return _backend.dpctl_to_table(arg)
-
-    if not _is_csr(arg):
-        arg = make2d(arg)
-    return _backend.to_table(arg)
-
+        import dpctl.tensor as dpt
 
-def to_table(*args):
-    return _apply_and_pass(convert_one_to_table, *args)
+    if dpnp_available:
+        import dpnp
 
-
-if _is_dpc_backend:
     from ..common._policy import _HostInteropPolicy
 
     def _convert_to_supported(policy, *data):
@@ -85,10 +69,78 @@ def convert_or_pass(x):
 
         return _apply_and_pass(func, *data)
 
+    def convert_one_from_table(table, sycl_queue=None, sua_iface=None, xp=None):
+        # Currently only `__sycl_usm_array_interface__` protocol used to
+        # convert into dpnp/dpctl tensors.
+        if sua_iface:
+            if (
+                sycl_queue
+                and sycl_queue.sycl_device.is_cpu
+                and table.__sycl_usm_array_interface__["syclobj"] is None
+            ):
+                # oneDAL returns tables with None sycl queue for CPU sycl queue inputs.
+                # This workaround is necessary for the functional preservation
+                # of the compute-follows-data execution.
+                # Host tables first converted into numpy.narrays and then to array from xp
+                # namespace.
+                return xp.asarray(
+                    _backend.from_table(table), usm_type="device", sycl_queue=sycl_queue
+                )
+            else:
+                xp_name = xp.__name__
+                if dpnp_available and xp_name == "dpnp":
+                    # By default DPNP ndarray created with a copy.
+                    # TODO:
+                    # investigate why dpnp.array(table, copy=False) doesn't work.
+                    # Work around with using dpctl.tensor.asarray.
+                    return dpnp.array(dpt.asarray(table), copy=False)
+                else:
+                    return xp.asarray(table)
+        return _backend.from_table(table)
+
+    def convert_one_to_table(arg, sua_iface=None):
+        # Note: currently only oneDAL homogen tables are supported and the
+        # contiuginity of the input array should be checked in advance.
+        if sua_iface:
+            return _backend.sua_iface_to_table(arg)
+
+        if not _is_csr(arg):
+            arg = make2d(arg)
+        return _backend.to_table(arg)
+
 else:
 
     def _convert_to_supported(policy, *data):
         def func(x):
             return x
 
         return _apply_and_pass(func, *data)
+
+    def convert_one_from_table(table, sycl_queue=None, sua_iface=None, xp=None):
+        # Currently only `__sycl_usm_array_interface__` protocol used to
+        # convert into dpnp/dpctl tensors.
+        if sua_iface:
+            raise RuntimeError(
+                "SYCL usm array conversion from table requires the DPC backend"
+            )
+        return _backend.from_table(table)
+
+    def convert_one_to_table(arg, sua_iface=None):
+        if sua_iface:
+            raise RuntimeError(
+                "SYCL usm array conversion to table requires the DPC backend"
+            )
+
+        if not _is_csr(arg):
+            arg = make2d(arg)
+        return _backend.to_table(arg)
+
+
+def from_table(*args, sycl_queue=None, sua_iface=None, xp=None):
+    return _apply_and_pass(
+        convert_one_from_table, *args, sycl_queue=sycl_queue, sua_iface=sua_iface, xp=xp
+    )
+
+
+def to_table(*args, sua_iface=None):
+    return _apply_and_pass(convert_one_to_table, *args, sua_iface=sua_iface)

onedal/datatypes/data_conversion.cpp

@@ -23,7 +23,7 @@
 #include "oneapi/dal/table/detail/homogen_utils.hpp"
 
 #include "onedal/datatypes/data_conversion.hpp"
-#include "onedal/datatypes/numpy_helpers.hpp"
+#include "onedal/datatypes/utils/numpy_helpers.hpp"
 #include "onedal/version.hpp"
 
 #if ONEDAL_VERSION <= 20230100