Make both src and dst call back when ready

src/Inciter/Discretization.cpp

@@ -322,34 +322,33 @@ Discretization::transfer(
 
   } else {
 
-    m_transfer_complete = cb;
-
     // determine source and destination mesh depending on direction of transfer
     std::size_t fromMesh(0), toMesh(0);
     CkCallback cb_xfer;
     if (dirn == 0) {
       fromMesh = m_mytransfer[m_nsrc].src;
       toMesh = m_mytransfer[m_ndst].dst;
-      cb_xfer = CkCallback( CkIndex_Discretization::to_complete(), thisProxy[thisIndex] );
+      cb_xfer = CkCallback(
+        CkIndex_Discretization::transfer_complete(), thisProxy[thisIndex] );
     }
     else {
       fromMesh = m_mytransfer[m_nsrc].dst;
       toMesh = m_mytransfer[m_ndst].src;
-      cb_xfer = CkCallback( CkIndex_Discretization::from_complete(), thisProxy[thisIndex] );
+      cb_xfer = cb;
     }
 
     // Pass source and destination meshes to mesh transfer lib (if coupled)
     Assert( m_nsrc < m_mytransfer.size(), "Indexing out of mytransfer[src]" );
     if (fromMesh == m_meshid) {
-      exam2m::setSourceTets( thisProxy, thisIndex, &m_inpoel, &m_coord, u );
+      exam2m::setSourceTets( thisProxy, thisIndex, &m_inpoel, &m_coord, u,
+        cb_xfer );
       ++m_nsrc;
     } else {
       m_nsrc = 0;
     }
     Assert( m_ndst < m_mytransfer.size(), "Indexing out of mytransfer[dst]" );
     if (toMesh == m_meshid) {
-      exam2m::setDestPoints( thisProxy, thisIndex, &m_coord, u,
-        cb_xfer );
+      exam2m::setDestPoints( thisProxy, thisIndex, &m_coord, u, cb_xfer );
       ++m_ndst;
     } else {
       m_ndst = 0;
@@ -361,61 +360,16 @@ Discretization::transfer(
   m_ndst = 0;
 }
 
-void Discretization::to_complete()
+void Discretization::transfer_complete()
 // *****************************************************************************
 //! Solution transfer from background to overset mesh completed (from ExaM2M)
 //! \brief This is called by ExaM2M on the destination mesh when the
 //!   transfer completes. Since this is called only on the destination, we find
 //!   and notify the corresponding source of the completion.
 // *****************************************************************************
 {
-  // Lookup the source disc and notify it of completion
-  for (auto& t : m_transfer) {
-    if (m_meshid == t.dst) {
-      m_disc[ t.src ][ thisIndex ].transfer_complete();
-    }
-  }
-
-  thisProxy[ thisIndex ].transfer_complete();
-}
-
-void Discretization::from_complete()
-// *****************************************************************************
-//! Solution transfer from overset to background mesh completed (from ExaM2M)
-//! \brief This is called by ExaM2M on the destination mesh when the
-//!   transfer completes. Since this is called only on the destination, we find
-//!   and notify the corresponding source of the completion.
-// *****************************************************************************
-{
-  // Lookup the source disc and notify it of completion
-  for (auto& t : m_transfer) {
-    if (m_meshid == t.src) {
-      m_disc[ t.dst ][ thisIndex ].transfer_complete_from_dest();
-    }
-  }
-
-  m_transfer_complete.send();
-}
-
-void Discretization::transfer_complete_from_dest()
-// *****************************************************************************
-//! Solution transfer completed (from dest Discretization)
-//! \details Called on the source only by the destination when a back and forth
-//!   transfer step completes.
-// *****************************************************************************
-{
-  m_transfer_complete.send();
-}
-
-void Discretization::transfer_complete()
-// *****************************************************************************
-//! Solution transfer completed (one-way)
-//! \note Single exit point after solution transfer between meshes
-// *****************************************************************************
-{
-  contribute( sizeof(nullptr), nullptr, CkReduction::nop,
-    CkCallback(CkReductionTarget(Transporter,solutionTransferred),
-    m_transporter) );
+  contribute( CkCallback( CkReductionTarget(Transporter,solutionTransferred),
+                           m_transporter) );
 }
 
 std::vector< std::size_t >

src/Inciter/Discretization.hpp

@@ -127,15 +127,6 @@ class Discretization : public CBase_Discretization {
       std::size_t dirn,
       CkCallback cb );
 
-    //! Solution transfer from background to overset mesh completed (from ExaM2M)
-    void to_complete();
-
-    //! Solution transfer from overset to background mesh completed (from ExaM2M)
-    void from_complete();
-
-    //! Solution transfer completed (from dest Discretization)
-    void transfer_complete_from_dest();
-
     //! Solution transfer completed (one-way)
     void transfer_complete();
 
@@ -439,7 +430,6 @@ class Discretization : public CBase_Discretization {
     //! \param[in,out] p Charm++'s PUP::er serializer object reference
     void pup( PUP::er &p ) override {
       p | m_meshid;
-      p | m_transfer_complete;
       p | m_transfer;
       p | m_mytransfer;
       p | m_disc;
@@ -506,9 +496,6 @@ class Discretization : public CBase_Discretization {
 
     //! Mesh ID
     std::size_t m_meshid;
-    //! \brief Charm++ callback of the function to call after a mesh-to-mesh
-    //!   solution transfer (to-and-fro) is complete
-    CkCallback m_transfer_complete;
     //! Solution/mesh transfer (coupling) information coordination propagation
     //! \details This has the same size with the same src/dst information on
     //!   all solvers.

src/Inciter/discretization.ci

@@ -46,9 +46,6 @@ module discretization {
       entry void stat( tk::real mesh_volume );
       entry void transferInit();
       entry void transfer_complete();
-      entry void to_complete();
-      entry void from_complete();
-      entry void transfer_complete_from_dest();
 
       // SDAG code follows. See http://charm.cs.illinois.edu/manuals/html/
       // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".

src/Transfer/M2MTransfer.cpp

@@ -40,8 +40,8 @@ void addMesh(CkArrayID p, int elem, CkCallback cb) {
   m2mtransferProxy[0].addMesh(p, elem, cb);
 }
 
-void setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u) {
-  m2mtransferProxy.ckLocalBranch()->setSourceTets(p, index, inpoel, coords, u);
+void setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u, CkCallback cb) {
+  m2mtransferProxy.ckLocalBranch()->setSourceTets(p, index, inpoel, coords, u, cb);
 }
 
 void setDestPoints(CkArrayID p, int index, tk::UnsMesh::Coords* coords, tk::Fields& u, CkCallback cb) {
@@ -87,11 +87,11 @@ void M2MTransfer::setDestPoints(CkArrayID p, int index, tk::UnsMesh::Coords* coo
   w->setDestPoints(coords, u, cb);
 }
 
-void M2MTransfer::setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u) {
+void M2MTransfer::setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u, CkCallback cb) {
   m_sourcemesh = static_cast<std::size_t>(CkGroupID(p).idx);
   TransferDetails* w = proxyMap[m_sourcemesh].m_proxy[index].ckLocal();
   assert(w);
-  w->setSourceTets(inpoel, coords, u);
+  w->setSourceTets(inpoel, coords, u, cb);
 }
 
 void

src/Transfer/M2MTransfer.hpp

@@ -8,7 +8,7 @@
 namespace exam2m {
 
 void addMesh(CkArrayID p, int elem, CkCallback cb);
-void setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u);
+void setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel, tk::UnsMesh::Coords* coords, const tk::Fields& u, CkCallback cb);
 void setDestPoints(CkArrayID p, int index, tk::UnsMesh::Coords* coords, tk::Fields& u, CkCallback cb);
 
 class LibMain : public CBase_LibMain {
@@ -47,7 +47,7 @@ class M2MTransfer : public CBase_M2MTransfer {
     void addMesh(CkArrayID p, int elem, CkCallback cb);
     void setMesh(CkArrayID p, MeshData d);
     void setSourceTets(CkArrayID p, int index, std::vector< std::size_t >* inpoel,
-                       tk::UnsMesh::Coords* coords, const tk::Fields& u);
+                       tk::UnsMesh::Coords* coords, const tk::Fields& u, CkCallback cb);
     void setDestPoints(CkArrayID p, int index, tk::UnsMesh::Coords* coords,
                        tk::Fields& u, CkCallback cb);
     void distributeCollisions(int nColl, Collision* colls);

src/Transfer/TransferDetails.cpp

@@ -53,17 +53,21 @@ void
 TransferDetails::setSourceTets(
     std::vector< std::size_t>* inpoel,
     tk::UnsMesh::Coords* coords,
-    const tk::Fields& u )
+    const tk::Fields& u,
+    CkCallback cb )
 // *****************************************************************************
 //  Set the data for the source tetrahedrons to be collided
 //! \param[in] inpoel Pointer to the connectivity data for the source mesh
 //! \param[in] coords Pointer to the coordinate data for the source mesh
 //! \param[in] u Pointer to the solution data for the source mesh
+//! \param[in] cb Callback to call when src side of transfer is done
 // *****************************************************************************
 {
   m_coord = coords;
   m_u = const_cast< tk::Fields* >( &u );
   m_inpoel = inpoel;
+  m_donecb = cb;
+  m_srcnotified = 0;
 
   // Send tetrahedron data to the collision detection library
   collideTets();
@@ -78,7 +82,7 @@ TransferDetails::setDestPoints(
 //  Set the data for the destination points to be collided
 //! \param[in] coords Pointer to the coordinate data for the destination mesh
 //! \param[in,out] u Pointer to the solution data for the destination mesh
-//! \param[in] cb Callback to call once this chare received all solution data
+//! \param[in] cb Callback to call when dst side of transfer is done
 // *****************************************************************************
 {
   m_coord = coords;
@@ -221,7 +225,7 @@ TransferDetails::determineActualCollisions(
     CProxy_TransferDetails proxy,
     int index,
     int nColls,
-    PotentialCollision* colls ) const
+    PotentialCollision* colls )
 // *****************************************************************************
 //  Identify actual collisions by calling intet on all possible collisions, and
 //  interpolate solution values to send back to the destination mesh.
@@ -266,6 +270,10 @@ TransferDetails::determineActualCollisions(
   //    thisIndex, numInTet, nColls);
   // Send the solution data for the actual collisions back to the dest mesh
   proxy[index].transferSolution( return_data );
+  if (not m_srcnotified) {
+    m_donecb.send();
+    m_srcnotified = 1;
+  }
 }
 
 void

src/Transfer/TransferDetails.hpp

@@ -57,7 +57,8 @@ class TransferDetails : public CBase_TransferDetails {
     //! Set the source mesh data
     void setSourceTets( std::vector< std::size_t>* inpoel,
                         tk::UnsMesh::Coords* coords,
-                        const tk::Fields& u );
+                        const tk::Fields& u,
+                        CkCallback cb );
 
     //! Set the destination mesh data
     void setDestPoints( tk::UnsMesh::Coords* coords,
@@ -75,7 +76,7 @@ class TransferDetails : public CBase_TransferDetails {
     void determineActualCollisions( CProxy_TransferDetails proxy,
                                     int index,
                                     int nColls,
-                                    PotentialCollision* colls ) const;
+                                    PotentialCollision* colls );
 
     //! Transfer the interpolated solution data back to destination mesh
     void transferSolution( const std::vector< SolutionData >& soln );
@@ -107,7 +108,9 @@ class TransferDetails : public CBase_TransferDetails {
     int m_numsent;
     //! The number of messages received by the dest mesh
     int m_numreceived;
-    //! Called once the transfer is complete (m_numsent == m_numreceived)
+   //! Set to nonzero once source mesh is notified that source side is done
+    int m_srcnotified = 0;
+    //! Called once the transfer is complete
     CkCallback m_donecb;
 
     //! Initialize dest mesh solution with background data