Split ZVertexSoA in a run-time sized PortableMultiCollection

DQM/SiPixelHeterogeneous/plugins/SiPixelCompareVertices.cc

@@ -9,20 +9,19 @@
 //
 // Author: Suvankar Roy Chowdhury
 //
-#include "FWCore/Framework/interface/Frameworkfwd.h"
+
+#include "DQMServices/Core/interface/DQMEDAnalyzer.h"
+#include "DQMServices/Core/interface/DQMStore.h"
+#include "DQMServices/Core/interface/MonitorElement.h"
+#include "DataFormats/BeamSpot/interface/BeamSpot.h"
+#include "DataFormats/Common/interface/Handle.h"
+#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
 #include "FWCore/Framework/interface/Event.h"
+#include "FWCore/Framework/interface/Frameworkfwd.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/Utilities/interface/InputTag.h"
-#include "DataFormats/Common/interface/Handle.h"
-// DQM Histograming
-#include "DQMServices/Core/interface/MonitorElement.h"
-#include "DQMServices/Core/interface/DQMEDAnalyzer.h"
-#include "DQMServices/Core/interface/DQMStore.h"
-#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
-#include "CUDADataFormats/Vertex/interface/ZVertexSoAHeterogeneousHost.h"
-#include "DataFormats/BeamSpot/interface/BeamSpot.h"
 
 // TODO: change class name to SiPixelCompareVerticesSoA when CUDA code is removed
 class SiPixelCompareVertices : public DQMEDAnalyzer {
@@ -81,7 +80,7 @@ void SiPixelCompareVertices::analyzeSeparate(U tokenRef, V tokenTar, const edm::
       out << "reference vertices not found; ";
     }
     if (not vsoaHandleTar) {
-      out << "Refget vertices not found; ";
+      out << "target vertices not found; ";
     }
     out << "the comparison will not run.";
     return;
@@ -94,8 +93,8 @@ void SiPixelCompareVertices::analyzeSeparate(U tokenRef, V tokenTar, const edm::
 
   auto bsHandle = iEvent.getHandle(tokenBeamSpot_);
   float x0 = 0., y0 = 0., z0 = 0., dxdz = 0., dydz = 0.;
-  if (!bsHandle.isValid()) {
-    edm::LogWarning("SiPixelCompareVertices") << "No beamspot found. returning vertexes with (0,0,Z) ";
+  if (not bsHandle.isValid()) {
+    edm::LogWarning("SiPixelCompareVertices") << "No beamspot found, returning vertexes with (0,0,Z).";
   } else {
     const reco::BeamSpot& bs = *bsHandle;
     x0 = bs.x0();
@@ -112,7 +111,7 @@ void SiPixelCompareVertices::analyzeSeparate(U tokenRef, V tokenTar, const edm::
     auto yc = y0 + dydz * zc;
     zc += z0;
 
-    auto ndofRef = vsoaRef.view()[sic].ndof();
+    auto ndofRef = vsoaRef.template view<reco::ZVertexTracksSoA>()[sic].ndof();
     auto chi2Ref = vsoaRef.view()[sic].chi2();
 
     const int32_t notFound = -1;
@@ -138,7 +137,7 @@ void SiPixelCompareVertices::analyzeSeparate(U tokenRef, V tokenTar, const edm::
     auto xg = x0 + dxdz * zg;
     auto yg = y0 + dydz * zg;
     zg += z0;
-    auto ndofTar = vsoaTar.view()[closestVtxidx].ndof();
+    auto ndofTar = vsoaTar.template view<reco::ZVertexTracksSoA>()[closestVtxidx].ndof();
     auto chi2Tar = vsoaTar.view()[closestVtxidx].chi2();
 
     hx_->Fill(xc - x0, xg - x0);

DQM/SiPixelHeterogeneous/plugins/SiPixelMonitorVertexSoAAlpaka.cc

@@ -67,7 +67,9 @@ void SiPixelMonitorVertexSoAAlpaka::analyze(const edm::Event& iEvent, const edm:
   }
 
   auto const& vsoa = *vsoaHandle;
-  int nVertices = vsoa.view().nvFinal();
+  auto vtx_view = vsoa.view<reco::ZVertexSoA>();
+  auto trk_view = vsoa.view<reco::ZVertexTracksSoA>();
+  int nVertices = vtx_view.nvFinal();
   auto bsHandle = iEvent.getHandle(tokenBeamSpot_);
   float x0 = 0., y0 = 0., z0 = 0., dxdz = 0., dydz = 0.;
   if (!bsHandle.isValid()) {
@@ -82,19 +84,19 @@ void SiPixelMonitorVertexSoAAlpaka::analyze(const edm::Event& iEvent, const edm:
   }
 
   for (int iv = 0; iv < nVertices; iv++) {
-    auto si = vsoa.view()[iv].sortInd();
-    auto z = vsoa.view()[si].zv();
+    auto si = vtx_view[iv].sortInd();
+    auto z = vtx_view[si].zv();
     auto x = x0 + dxdz * z;
     auto y = y0 + dydz * z;
 
     z += z0;
     hx->Fill(x);
     hy->Fill(y);
     hz->Fill(z);
-    auto ndof = vsoa.view()[si].ndof();
-    hchi2->Fill(vsoa.view()[si].chi2());
-    hchi2oNdof->Fill(vsoa.view()[si].chi2() / ndof);
-    hptv2->Fill(vsoa.view()[si].ptv2());
+    auto ndof = trk_view[si].ndof();
+    hchi2->Fill(vtx_view[si].chi2());
+    hchi2oNdof->Fill(vtx_view[si].chi2() / ndof);
+    hptv2->Fill(vtx_view[si].ptv2());
     hntrks->Fill(ndof + 1);
   }
   hnVertex->Fill(nVertices);

DataFormats/VertexSoA/README.md

@@ -1,45 +1,32 @@
 # Vertex Portable Data Formats
 
-`DataFormat`s meant to be used on Host (CPU) or Device (GPU) for
-storing information about vertices created during the Pixel-local Reconstruction
-chain. It stores data in an SoA manner. It contains the data that was previously
-contained in the deprecated `ZVertexSoA` class. 
+`DataFormat`s meant to be used on Host (CPU) or Device (GPUs) for storing
+information about reconstructed pixel vertices in Structure of Array (SoA)
+format.
 
-The host format is inheriting from `DataFormats/Common/interface/PortableHostCollection.h`,
-while the device format is inheriting from `DataFormats/Common/interface/PortableDeviceCollection.h`
+The host collection is an instantiation of `PortableHostMultiCollection`, while
+the device collection is an instantiation of `PortableDeviceMultiCollection`.
 
-Both formats use the same SoA Layout (`ZVertexLayout`) which is generated
-via the `GENERATE_SOA_LAYOUT` macro in the `ZVertexUtilities.h` file.
+Both collections use two SoA layouts (`ZVertexLayout` and `ZVertexTracksLayout`)
+with different number of elements, defined at run-time.
+The layouts are defined by the `GENERATE_SOA_LAYOUT` macro in
+`DataFormats/VertexSoA/interface/ZVertexSoA.h`.
 
-## Notes
 
-- Initially, `ZVertexSoA` had distinct array sizes for each attribute (e.g. `zv` was `MAXVTX` elements 
-long, `ndof` was `MAXTRACKS` elements long). All columns are now of uniform `MAXTRACKS` size, 
-meaning that there will be some wasted space (appx. 190kB). 
-- Host and Device classes should **not** be created via inheritance, as they're done here,
-but via composition. See [this discussion](https://github.com/cms-sw/cmssw/pull/40465#discussion_r1066039309).
-
-## ZVertexHeterogeneousHost
+## `ZVertexHost`
 
 The version of the data format to be used for storing vertex data on the CPU. 
 Instances of this class are to be used for:
+  - having a place to copy data to host from device, which is usually taken care
+    of automatically by the framework;
+  - running host-side algorithms using data stored in an SoA manner.
 
-- Having a place to copy data to host from device, via `cudaMemcpy`, or
-- Running host-side algorithms using data stored in an SoA manner.
 
-## ZVertexHeterogeneousDevice
+## `ZVertexDevice`
 
 The version of the data format to be used for storing vertex data on the GPU.
-
-Instances of `ZVertexHeterogeneousDevice` are to be created on host and be
-used on device only. To do so, the instance's `view()` method is to be called
-to pass a `View` to any kernel launched. Accessing data from the `view()` is not
-possible on the host side.
-
-## Utilities
-
-Apart from `ZVertexLayout`, `ZVertexUtilities.h` also contains
-a collection of methods which were originally
-defined as class methods inside the `ZVertexSoA` class
-which have been adapted to operate on `View` instances, so that they are callable
-from within `__global__` kernels, on both CPU and CPU. 
+Instances of `ZVertexDevice` are created on the host, and can only be used only
+on the device. To do so, the instance's `view()` or `const_view()` methods are
+called, and the resulting `View` or `ConstView` are passed to a kernel launch.
+The data from the instance, `view()` or `const_view()` is not accessible on the
+host side.

DataFormats/VertexSoA/interface/ZVertexDevice.h

@@ -4,23 +4,12 @@
 #include <cstdint>
 
 #include <alpaka/alpaka.hpp>
+
 #include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
-#include "DataFormats/VertexSoA/interface/ZVertexDefinitions.h"
 #include "DataFormats/VertexSoA/interface/ZVertexHost.h"
 #include "DataFormats/Portable/interface/PortableDeviceCollection.h"
 
-template <int32_t S, typename TDev>
-class ZVertexDeviceSoA : public PortableDeviceCollection<reco::ZVertexLayout<>, TDev> {
-public:
-  ZVertexDeviceSoA() = default;  // necessary for ROOT dictionaries
-
-  // Constructor which specifies the SoA size
-  template <typename TQueue>
-  explicit ZVertexDeviceSoA(TQueue queue) : PortableDeviceCollection<reco::ZVertexLayout<>, TDev>(S, queue) {}
-};
-
-using namespace ::zVertex;
 template <typename TDev>
-using ZVertexDevice = ZVertexDeviceSoA<MAXTRACKS, TDev>;
+using ZVertexDevice = PortableDeviceMultiCollection<TDev, reco::ZVertexSoA, reco::ZVertexTracksSoA>;
 
 #endif  // DataFormats_VertexSoA_interface_ZVertexDevice_h

DataFormats/VertexSoA/interface/ZVertexHost.h

@@ -5,25 +5,10 @@
 
 #include <alpaka/alpaka.hpp>
 
-#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
-#include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
-#include "DataFormats/VertexSoA/interface/ZVertexDefinitions.h"
 #include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
-template <int32_t S>
-class ZVertexHostSoA : public PortableHostCollection<reco::ZVertexSoA> {
-public:
-  ZVertexHostSoA() = default;
-
-  // Constructor which specifies the queue
-  template <typename TQueue>
-  explicit ZVertexHostSoA(TQueue queue) : PortableHostCollection<reco::ZVertexSoA>(S, queue) {}
-
-  // Constructor which specifies the DevHost
-  explicit ZVertexHostSoA(alpaka_common::DevHost const& host) : PortableHostCollection<reco::ZVertexSoA>(S, host) {}
-};
-
-//using namespace ::zVertex;
-using ZVertexHost = ZVertexHostSoA<zVertex::MAXTRACKS>;
+using ZVertexHost = PortableHostCollection2<reco::ZVertexSoA, reco::ZVertexTracksSoA>;
 
 #endif  // DataFormats_VertexSoA_ZVertexHost_H

DataFormats/VertexSoA/interface/ZVertexSoA.h

@@ -10,20 +10,29 @@
 namespace reco {
 
   GENERATE_SOA_LAYOUT(ZVertexLayout,
-                      SOA_COLUMN(int16_t, idv),
-                      SOA_COLUMN(float, zv),
-                      SOA_COLUMN(float, wv),
-                      SOA_COLUMN(float, chi2),
-                      SOA_COLUMN(float, ptv2),
-                      SOA_COLUMN(int32_t, ndof),
-                      SOA_COLUMN(uint16_t, sortInd),
-                      SOA_SCALAR(uint32_t, nvFinal))
+                      SOA_COLUMN(float, zv),          // output z-posistion of found vertices
+                      SOA_COLUMN(float, wv),          // output weight (1/error^2) on the above
+                      SOA_COLUMN(float, chi2),        // vertices chi2
+                      SOA_COLUMN(float, ptv2),        // vertices pt^2
+                      SOA_COLUMN(uint16_t, sortInd),  // sorted index (by pt2)  ascending
+                      SOA_SCALAR(uint32_t, nvFinal))  // the number of vertices
+
+  GENERATE_SOA_LAYOUT(ZVertexTracksLayout,
+                      SOA_COLUMN(int16_t, idv),   // vertex index for each associated (original) track
+                                                  // (-1 == not associate)
+                      SOA_COLUMN(int32_t, ndof))  // vertices number of dof
+                                                  // FIXME: reused as workspace for the number of nearest neighbours
 
   // Common types for both Host and Device code
   using ZVertexSoA = ZVertexLayout<>;
   using ZVertexSoAView = ZVertexSoA::View;
   using ZVertexSoAConstView = ZVertexSoA::ConstView;
 
+  // Common types for both Host and Device code
+  using ZVertexTracksSoA = ZVertexTracksLayout<>;
+  using ZVertexTracksSoAView = ZVertexTracksSoA::View;
+  using ZVertexTracksSoAConstView = ZVertexTracksSoA::ConstView;
+
   ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE void init(ZVertexSoAView &vertices) { vertices.nvFinal() = 0; }
 
 }  // namespace reco

DataFormats/VertexSoA/interface/alpaka/ZVertexSoACollection.h

@@ -4,13 +4,13 @@
 #include <cstdint>
 
 #include <alpaka/alpaka.hpp>
+
 #include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
-#include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
-#include "DataFormats/VertexSoA/interface/ZVertexDefinitions.h"
-#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
 #include "DataFormats/VertexSoA/interface/ZVertexDevice.h"
-#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
+#include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
 #include "HeterogeneousCore/AlpakaInterface/interface/CopyToHost.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
@@ -19,21 +19,6 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
 }  // namespace ALPAKA_ACCELERATOR_NAMESPACE
 
-namespace cms::alpakatools {
-  template <typename TDevice>
-  struct CopyToHost<ZVertexDevice<TDevice>> {
-    template <typename TQueue>
-    static auto copyAsync(TQueue& queue, ZVertexDevice<TDevice> const& deviceData) {
-      ZVertexHost hostData(queue);
-      alpaka::memcpy(queue, hostData.buffer(), deviceData.buffer());
-#ifdef GPU_DEBUG
-      printf("ZVertexSoACollection: I'm copying to host.\n");
-#endif
-      return hostData;
-    }
-  };
-}  // namespace cms::alpakatools
-
 ASSERT_DEVICE_MATCHES_HOST_COLLECTION(ZVertexSoACollection, ZVertexHost);
 
 #endif  // DataFormats_VertexSoA_interface_ZVertexSoACollection_h

DataFormats/VertexSoA/src/classes.cc

@@ -1,4 +1,4 @@
 #include "DataFormats/Portable/interface/PortableHostCollectionReadRules.h"
-#include "DataFormats/VertexSoA/interface/ZVertexSoA.h"
+#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
 
-SET_PORTABLEHOSTCOLLECTION_READ_RULES(PortableHostCollection<reco::ZVertexSoA>);
+SET_PORTABLEHOSTMULTICOLLECTION_READ_RULES(ZVertexHost);

DataFormats/VertexSoA/src/classes_def.xml

@@ -1,8 +1,18 @@
 <lcgdict>
-  <class name="reco::ZVertexSoA"/>
-  <class name="PortableHostCollection<reco::ZVertexSoA>"/>
-  <class name="ZVertexHost" ClassVersion="3">
-    <version ClassVersion="3" checksum="1989784241"/>
-  </class>
+  <!-- SoA classes -->
+  <!-- FIXME using the ZVertexSoA and ZVertexTracksSoA type aliases here and below does not work -->
+  <class name="reco::ZVertexLayout<128, false>"/>
+  <class name="reco::ZVertexTracksLayout<128, false>"/>
+
+  <!-- Recursive templates (with no data) ensuring we have one CollectionLeaf<index, type> for each layout in the collection -->
+  <class name="portablecollection::CollectionImpl<0, reco::ZVertexLayout<128, false>, reco::ZVertexTracksLayout<128, false>>"/>
+  <class name="portablecollection::CollectionImpl<1, reco::ZVertexTracksLayout<128, false>>"/>
+
+  <!-- Recursive templates implementing the association of indices and layouts, and containing the data -->
+  <class name="portablecollection::CollectionLeaf<0, reco::ZVertexLayout<128, false>>"/>
+  <class name="portablecollection::CollectionLeaf<1, reco::ZVertexTracksLayout<128, false>>"/>
+
+  <!-- Collection declaration for dictionary -->
+  <class name="ZVertexHost"/>
   <class name="edm::Wrapper<ZVertexHost>" splitLevel="0"/>
 </lcgdict>