Merge pull request #740 from CEED/natalie/device-id

Update device ID selection for HIP/CUDA/MAGMA backends

Author: jeremylt

README.rst

@@ -265,6 +265,11 @@ Currently, each MAGMA library installation is only built for either CUDA or HIP.
 set of libCEED backends (``/gpu/cuda/magma/*`` or ``/gpu/hip/magma/*``) will automatically be built
 for the version of the MAGMA library found in ``MAGMA_DIR``.
 
+Users can specify a device for all CUDA, HIP, and MAGMA backends through adding `:device_id=#` 
+after the resource name.  For example:
+
+ - `/gpu/cuda/gen:device_id=1`
+ 
 The ``/*/occa`` backends rely upon the `OCCA <http://github.com/libocca/occa>`_ package to provide
 cross platform performance. To enable the OCCA backend, the environment variable ``OCCA_DIR`` must point
 to the top-level OCCA directory, with the OCCA library located in the ``${OCCA_DIR}/lib`` (By default,

backends/cuda-gen/ceed-cuda-gen.c

@@ -32,15 +32,15 @@ static int CeedInit_Cuda_gen(const char *resource, Ceed ceed) {
                      "Cuda backend cannot use resource: %s", resource);
   // LCOV_EXCL_STOP
 
-  Ceed ceedshared;
-  CeedInit("/gpu/cuda/shared", &ceedshared);
-  ierr = CeedSetDelegate(ceed, ceedshared); CeedChkBackend(ierr);
-
-  Ceed_Cuda_gen *data;
+  Ceed_Cuda *data;
   ierr = CeedCalloc(1, &data); CeedChkBackend(ierr);
   ierr = CeedSetData(ceed, data); CeedChkBackend(ierr);
   ierr = CeedCudaInit(ceed, resource, nrc); CeedChkBackend(ierr);
 
+  Ceed ceedshared;
+  CeedInit("/gpu/cuda/shared", &ceedshared);
+  ierr = CeedSetDelegate(ceed, ceedshared); CeedChkBackend(ierr);
+
   const char fallbackresource[] = "/gpu/cuda/ref";
   ierr = CeedSetOperatorFallbackResource(ceed, fallbackresource);
   CeedChkBackend(ierr);

backends/cuda-gen/ceed-cuda-gen.h

@@ -44,10 +44,6 @@ typedef struct {
   void *d_c;
 } CeedQFunction_Cuda_gen;
 
-typedef struct {
-  Ceed_Cuda base;
-} Ceed_Cuda_gen;
-
 CEED_INTERN int CeedQFunctionCreate_Cuda_gen(CeedQFunction qf);
 
 CEED_INTERN int CeedOperatorCreate_Cuda_gen(CeedOperator op);

backends/cuda-shared/ceed-cuda-shared-basis.c

@@ -773,7 +773,7 @@ int CeedBasisApplyTensor_Cuda_shared(CeedBasis basis, const CeedInt nelem,
   int ierr;
   Ceed ceed;
   ierr = CeedBasisGetCeed(basis, &ceed); CeedChkBackend(ierr);
-  Ceed_Cuda_shared *ceed_Cuda;
+  Ceed_Cuda *ceed_Cuda;
   CeedGetData(ceed, &ceed_Cuda); CeedChkBackend(ierr);
   CeedBasis_Cuda_shared *data;
   CeedBasisGetData(basis, &data); CeedChkBackend(ierr);

backends/cuda-shared/ceed-cuda-shared.c

@@ -16,7 +16,6 @@
 
 #include <ceed/ceed.h>
 #include <ceed/backend.h>
-#include <stdbool.h>
 #include <string.h>
 #include "ceed-cuda-shared.h"
 #include "../cuda/ceed-cuda.h"
@@ -34,15 +33,15 @@ static int CeedInit_Cuda_shared(const char *resource, Ceed ceed) {
   // LCOV_EXCL_STOP
   ierr = CeedSetDeterministic(ceed, true); CeedChk(ierr);
 
-  Ceed ceedref;
-  CeedInit("/gpu/cuda/ref", &ceedref);
-  ierr = CeedSetDelegate(ceed, ceedref); CeedChk(ierr);
-
-  Ceed_Cuda_shared *data;
+  Ceed_Cuda *data;
   ierr = CeedCalloc(1, &data); CeedChk(ierr);
   ierr = CeedSetData(ceed, data); CeedChk(ierr);
   ierr = CeedCudaInit(ceed, resource, nrc); CeedChk(ierr);
 
+  Ceed ceedref;
+  CeedInit("/gpu/cuda/ref", &ceedref);
+  ierr = CeedSetDelegate(ceed, ceedref); CeedChk(ierr);
+
   ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateTensorH1",
                                 CeedBasisCreateTensorH1_Cuda_shared);
   CeedChk(ierr);

backends/cuda-shared/ceed-cuda-shared.h

@@ -35,10 +35,6 @@ typedef struct {
   CeedScalar *c_G;
 } CeedBasis_Cuda_shared;
 
-typedef struct {
-  Ceed_Cuda base;
-} Ceed_Cuda_shared;
-
 CEED_INTERN int CeedBasisCreateTensorH1_Cuda_shared(CeedInt dim, CeedInt P1d,
     CeedInt Q1d, const CeedScalar *interp1d, const CeedScalar *grad1d,
     const CeedScalar *qref1d, const CeedScalar *qweight1d, CeedBasis basis);

backends/cuda/ceed-cuda.c

@@ -21,7 +21,6 @@
 #include <cuda_runtime.h>
 #include <nvrtc.h>
 #include <stdarg.h>
-#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -150,22 +149,22 @@ static int CeedGetPreferredMemType_Cuda(CeedMemType *type) {
 //------------------------------------------------------------------------------
 int CeedCudaInit(Ceed ceed, const char *resource, int nrc) {
   int ierr;
-  const int rlen = strlen(resource);
-  const bool slash = (rlen>nrc) ? (resource[nrc] == '/') : false;
-  const int deviceID = (slash && rlen > nrc + 1) ? atoi(&resource[nrc + 1]) : -1;
+  const char *device_spec = strstr(resource, ":device_id=");
+  const int deviceID = (device_spec) ? atoi(device_spec+11) : -1;
 
   int currentDeviceID;
   ierr = cudaGetDevice(&currentDeviceID); CeedChk_Cu(ceed,ierr);
   if (deviceID >= 0 && currentDeviceID != deviceID) {
     ierr = cudaSetDevice(deviceID); CeedChk_Cu(ceed,ierr);
+    currentDeviceID = deviceID;
   }
-
   struct cudaDeviceProp deviceProp;
-  ierr = cudaGetDeviceProperties(&deviceProp, deviceID); CeedChk_Cu(ceed,ierr);
+  ierr = cudaGetDeviceProperties(&deviceProp, currentDeviceID);
+  CeedChk_Cu(ceed,ierr);
 
   Ceed_Cuda *data;
   ierr = CeedGetData(ceed, &data); CeedChkBackend(ierr);
-  data->deviceId = deviceID;
+  data->deviceId = currentDeviceID;
   data->optblocksize = deviceProp.maxThreadsPerBlock;
   return CEED_ERROR_SUCCESS;
 }
@@ -210,6 +209,7 @@ static int CeedInit_Cuda(const char *resource, Ceed ceed) {
     return CeedError(ceed, CEED_ERROR_BACKEND,
                      "Cuda backend cannot use resource: %s", resource);
   // LCOV_EXCL_STOP
+  ierr = CeedSetDeterministic(ceed, true); CeedChk(ierr);
 
   Ceed_Cuda *data;
   ierr = CeedCalloc(1, &data); CeedChkBackend(ierr);

backends/hip-gen/ceed-hip-gen.c

@@ -32,15 +32,15 @@ static int CeedInit_Hip_gen(const char *resource, Ceed ceed) {
                      "Hip backend cannot use resource: %s", resource);
   // LCOV_EXCL_STOP
 
-  Ceed ceedshared;
-  CeedInit("/gpu/hip/shared", &ceedshared);
-  ierr = CeedSetDelegate(ceed, ceedshared); CeedChkBackend(ierr);
-
-  Ceed_Hip_gen *data;
+  Ceed_Hip *data;
   ierr = CeedCalloc(1, &data); CeedChkBackend(ierr);
   ierr = CeedSetData(ceed, data); CeedChkBackend(ierr);
   ierr = CeedHipInit(ceed, resource, nrc); CeedChkBackend(ierr);
 
+  Ceed ceedshared;
+  CeedInit("/gpu/hip/shared", &ceedshared);
+  ierr = CeedSetDelegate(ceed, ceedshared); CeedChkBackend(ierr);
+
   const char fallbackresource[] = "/gpu/hip/ref";
   ierr = CeedSetOperatorFallbackResource(ceed, fallbackresource);
   CeedChkBackend(ierr);

backends/hip-gen/ceed-hip-gen.h

@@ -44,10 +44,6 @@ typedef struct {
   void *d_c;
 } CeedQFunction_Hip_gen;
 
-typedef struct {
-  Ceed_Hip base;
-} Ceed_Hip_gen;
-
 CEED_INTERN int CeedQFunctionCreate_Hip_gen(CeedQFunction qf);
 
 CEED_INTERN int CeedOperatorCreate_Hip_gen(CeedOperator op);

backends/hip-shared/ceed-hip-shared-basis.c

@@ -859,7 +859,7 @@ int CeedBasisApplyTensor_Hip_shared(CeedBasis basis, const CeedInt nelem,
   int ierr;
   Ceed ceed;
   ierr = CeedBasisGetCeed(basis, &ceed); CeedChkBackend(ierr);
-  Ceed_Hip_shared *ceed_Hip;
+  Ceed_Hip *ceed_Hip;
   CeedGetData(ceed, &ceed_Hip); CeedChkBackend(ierr);
   CeedBasis_Hip_shared *data;
   CeedBasisGetData(basis, &data); CeedChkBackend(ierr);