feat: Initial Arrow conversion and parquet output (#5410)

Adds an Apache Arrow / Parquet output path for the examples framework, so event data (particles, sim hits, tracks) can be written to a columnar, sharded Parquet dataset and read back per-event — from both C++ and Python/pyarrow.

### What

- **New `ActsPluginArrow` plugin** wrapping Apache Arrow. Arrow is linked
    with hidden visibility so none of its symbols leak across the `.so`
    boundary (enforced by exported-symbol lists). Two visibility-exported,
    pybind-friendly handles — `ArrowSchemaHandle` and `ArrowTable` — move
    schemas and tables across library and language boundaries via the Arrow
    **C Data Interface**, giving ABI-safe pyarrow interop without exposing
    arrow's own typeinfo.

- **Parquet I/O (`Examples/Io/Parquet`)**:
    - `ParquetWriter`: sharded dataset, one directory per collection. Events
      are routed to shard files by `event_id`, so each shard owns a disjoint
      event-id range and footer min/max statistics stay tight — letting the
      reader prune to a single fragment per lookup. Row-group buffering bounds
      peak memory; per-collection schemas are validated on write.
    - `ParquetReader`: dataset reader with per-event lookup via filter
      pushdown + footer-statistics pruning, and added-column schema evolution
      against an optional target schema.

- **Output converters (`Examples/Io/Arrow`)** for particles, sim hits and
    tracks. Each builds a per-event **nested** table: one row per event, every
    field a `list<T>` whose single element holds that event's values, with
    `event_id` as the outer routing key.

- **Python bindings** for the converters, reader and writer, with the arrow
    schema bridged into Python so the producing and consuming sides share one
    schema handle.

- Wired into `full_chain_odd.py` (new `--output-parquet` flag); covered by
    `test_arrow.py` and an ABI-isolation test (`test_arrow_isolation.py`).

### Why

ACTS examples lacked a standard columnar output. Parquet gives compact, typed, schema-stable files that downstream ML/analysis tooling consumes directly, and the C-Data-interface design lets the same buffers be used from pyarrow with no copies and no arrow-symbol clashes.

Blocked by:
- #5439 
- #5440

Author: paulgessinger

CMakeLists.txt

@@ -76,6 +76,15 @@ option(ACTS_USE_SYSTEM_GBL "Use a system-provided General Broken Lines (GBL) fro
 option(ACTS_USE_SYSTEM_MILLE "Use a system-provided Mille" ON)
 
 option(ACTS_BUILD_PLUGIN_ACTSVG "Build SVG display plugin" OFF)
+option(ACTS_BUILD_PLUGIN_ARROW "Build Apache Arrow/Parquet plugin" OFF)
+option(ACTS_ARROW_ISOLATED
+    "Statically link arrow/parquet into libActsPluginArrow and hide their \
+symbols, so they cannot collide with other arrow consumers in the same \
+process (e.g. pyarrow's bundled libarrow). Disable only if your arrow \
+install is ABI-compatible with every other arrow consumer that will be \
+loaded alongside ACTS."
+    ON
+)
 option(ACTS_BUILD_PLUGIN_DD4HEP "Build DD4hep plugin" OFF)
 option(ACTS_BUILD_PLUGIN_EDM4HEP "Build EDM4hep plugin" OFF)
 option(ACTS_BUILD_PLUGIN_FPEMON "Build FPE monitoring plugin" OFF)
@@ -124,6 +133,7 @@ option(ACTS_BUILD_EXAMPLES_FASTJET "Build FastJet plugin" OFF)
 option(ACTS_BUILD_EXAMPLES_GEANT4 "Build Geant4-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_GNN "Build the GNN example code" OFF)
 option(ACTS_BUILD_EXAMPLES_HASHING "Build Hashing-based code in the examples" OFF)
+option(ACTS_BUILD_EXAMPLES_PARQUET "Build Arrow/Parquet-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_PODIO "Build Podio-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_PYTHIA8 "Build Pythia8-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_PYTHON_BINDINGS "[Deprecated] Build python bindings and enables the examples" OFF)
@@ -182,6 +192,8 @@ set_option_if(
     OR
     ACTS_BUILD_EXAMPLES_HASHING
     OR
+    ACTS_BUILD_EXAMPLES_PARQUET
+    OR
     ACTS_BUILD_EXAMPLES_PODIO
     OR
     ACTS_BUILD_EXAMPLES_PYTHIA8
@@ -200,6 +212,7 @@ set_option_if(
 )
 set_option_if(ACTS_BUILD_PLUGIN_EDM4HEP ACTS_BUILD_EXAMPLES_EDM4HEP)
 set_option_if(ACTS_BUILD_EXAMPLES_PODIO ACTS_BUILD_EXAMPLES_EDM4HEP)
+set_option_if(ACTS_BUILD_PLUGIN_ARROW ACTS_BUILD_EXAMPLES_PARQUET)
 set_option_if(ACTS_BUILD_PLUGIN_GEANT4 ACTS_BUILD_EXAMPLES_GEANT4)
 set_option_if(
     ACTS_BUILD_PLUGIN_ROOT
@@ -336,6 +349,7 @@ set(_acts_covfie_version 0.15.2)
 set(_acts_vecmem_version 1.24.0)
 set(_acts_annoy_version 1.17.3)
 set(_acts_fastjet_version 3.4.1)
+set(_acts_arrow_version 23.0.0)
 
 # Help with compiler flags discovery
 include(ActsFunctions)
@@ -582,6 +596,11 @@ endif()
 if(ACTS_BUILD_PLUGIN_GEANT4)
     find_package(Geant4 ${_acts_geant4_version} REQUIRED CONFIG COMPONENTS gdml)
 endif()
+if(ACTS_BUILD_PLUGIN_ARROW)
+    find_package(Arrow ${_acts_arrow_version} REQUIRED CONFIG)
+    find_package(Parquet ${_acts_arrow_version} REQUIRED CONFIG)
+    find_package(ArrowDataset ${_acts_arrow_version} REQUIRED CONFIG)
+endif()
 
 if(ACTS_SETUP_VECMEM)
     if(ACTS_USE_SYSTEM_VECMEM)

Examples/Io/Arrow/CMakeLists.txt

@@ -0,0 +1,51 @@
+add_library(
+    ActsExamplesIoArrow_obj
+    OBJECT
+    src/ArrowParticleOutputConverter.cpp
+    src/ArrowSimHitOutputConverter.cpp
+    src/ArrowTrackOutputConverter.cpp
+)
+
+target_include_directories(
+    ActsExamplesIoArrow_obj
+    PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        # Genex (not target_link_libraries) for Plugin and sibling OBJECT
+        # donor: a link-level dep on ActsPluginArrow would create a cycle
+        # since it absorbs this OBJECT lib's .o files.
+        $<TARGET_PROPERTY:Acts::ExamplesIoParquet,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:Acts::PluginArrow,INTERFACE_INCLUDE_DIRECTORIES>
+)
+
+target_link_libraries(
+    ActsExamplesIoArrow_obj
+    PRIVATE Acts::ExamplesFramework Acts::ArrowLinkage Acts::ParquetLinkage
+)
+
+set_target_properties(
+    ActsExamplesIoArrow_obj
+    PROPERTIES POSITION_INDEPENDENT_CODE ON
+)
+if(ACTS_ARROW_ISOLATED)
+    set_target_properties(
+        ActsExamplesIoArrow_obj
+        PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES
+    )
+endif()
+
+target_sources(
+    ActsPluginArrow
+    PRIVATE $<TARGET_OBJECTS:ActsExamplesIoArrow_obj>
+)
+
+acts_add_library(ExamplesIoArrow INTERFACE)
+target_include_directories(
+    ActsExamplesIoArrow
+    INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+)
+target_link_libraries(
+    ActsExamplesIoArrow
+    INTERFACE Acts::ExamplesIoParquet Acts::PluginArrow Acts::ExamplesFramework
+)
+
+acts_compile_headers(ExamplesIoArrow GLOB include/**/*.hpp)

Examples/Io/Arrow/include/ActsExamples/Io/Arrow/ArrowParticleOutputConverter.hpp

@@ -0,0 +1,59 @@
+// This file is part of the ACTS project.
+//
+// Copyright (C) 2016 CERN for the benefit of the ACTS project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#pragma once
+
+#include "ActsExamples/EventData/SimParticle.hpp"
+#include "ActsExamples/Framework/DataHandle.hpp"
+#include "ActsExamples/Io/Parquet/ArrowOutputConverter.hpp"
+#include "ActsPlugins/Arrow/ArrowUtil.hpp"
+#include "ActsPlugins/Arrow/Export.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace ActsExamples {
+
+/// Convert a @c SimParticleContainer to an @c arrow::Table.
+///
+/// The output table has one row per particle with columns for id, PDG code,
+/// charge, mass, and the initial-state four-momentum / four-position. The
+/// table is placed on the whiteboard under the configured key; the
+/// @c ParquetWriter picks it up from there and stamps the @c event_id column.
+class ACTS_ARROW_EXPORT ArrowParticleOutputConverter final
+    : public ArrowOutputConverter {
+ public:
+  struct Config {
+    /// Input @c SimParticleContainer on the whiteboard.
+    std::string inputParticles;
+    /// Output whiteboard key for the resulting @c arrow::Table.
+    std::string outputTable = "particles";
+  };
+
+  explicit ArrowParticleOutputConverter(
+      const Config& cfg, std::unique_ptr<const Acts::Logger> logger = nullptr);
+
+  ~ArrowParticleOutputConverter() override;
+
+  const Config& config() const { return m_cfg; }
+
+  std::vector<std::string> collections() const override;
+
+ private:
+  ProcessCode execute(const AlgorithmContext& ctx) const override;
+
+  Config m_cfg;
+
+  ReadDataHandle<SimParticleContainer> m_inputParticles{this, "InputParticles"};
+
+  WriteDataHandle<ActsPlugins::ArrowUtil::ArrowTable> m_outputTable{
+      this, "OutputTable"};
+};
+
+}  // namespace ActsExamples

Examples/Io/Arrow/include/ActsExamples/Io/Arrow/ArrowSimHitOutputConverter.hpp

@@ -0,0 +1,107 @@
+// This file is part of the ACTS project.
+//
+// Copyright (C) 2016 CERN for the benefit of the ACTS project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#pragma once
+
+#include "Acts/Geometry/GeometryIdentifier.hpp"
+#include "ActsExamples/EventData/Cluster.hpp"
+#include "ActsExamples/EventData/SimHit.hpp"
+#include "ActsExamples/EventData/SimParticle.hpp"
+#include "ActsExamples/EventData/TruthMatching.hpp"
+#include "ActsExamples/Framework/DataHandle.hpp"
+#include "ActsExamples/Io/Parquet/ArrowOutputConverter.hpp"
+#include "ActsPlugins/Arrow/ArrowUtil.hpp"
+#include "ActsPlugins/Arrow/Export.hpp"
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace ActsExamples {
+
+/// Convert a @c SimHitContainer to an @c arrow::Table.
+///
+/// The output table has one row per event with list-valued columns. Hits are
+/// emitted in @c SimHitContainer iteration order, so the row index of a hit
+/// inside the per-event list equals its @c SimHitIndex; downstream tables
+/// (e.g. tracks) can therefore reference hits by that index.
+///
+/// When @c inputClusters and @c inputSimHitMeasurementsMap are both provided,
+/// the precomputed digitized cluster position (@c Cluster::globalPosition) of
+/// the matched measurement is written into @c x,y,z. Clusters have a one-to-one
+/// relation with measurements, so the @c SimHitMeasurementsMap (keyed by
+/// @c SimHitIndex, valued by measurement index) doubles as a sim-hit → cluster
+/// map. Otherwise those columns are filled with NaN. The truth position is
+/// always written into @c true_x,true_y,true_z.
+class ACTS_ARROW_EXPORT ArrowSimHitOutputConverter final
+    : public ArrowOutputConverter {
+ public:
+  struct Config {
+    /// Input @c SimHitContainer on the whiteboard.
+    std::string inputSimHits;
+    /// Optional input particle container used to resolve the hit's particle
+    /// barcode to a row index in the corresponding parquet table. Must be the
+    /// same container the @c ArrowParticleOutputConverter consumes for that
+    /// table — leaving it empty forces the unmatched sentinel.
+    std::string inputParticles;
+    /// Optional cluster container. Required (together with the map below) to
+    /// fill the digitized @c x,y,z columns from @c Cluster::globalPosition;
+    /// otherwise those are NaN. Clusters are indexed one-to-one with
+    /// measurements.
+    std::string inputClusters;
+    /// Optional sim-hit → measurement(s) inverse map; keyed by @c SimHitIndex.
+    /// Because clusters and measurements share indices, the values double as
+    /// cluster indices.
+    std::string inputSimHitMeasurementsMap;
+    /// Output whiteboard key for the resulting @c arrow::Table.
+    std::string outputTable = "simhits";
+    /// Resolves the @c detector subsystem id for a given hit's geometry id.
+    /// Defaults to reading the geometry id's @c extra byte; users can swap
+    /// in any custom mapping (e.g. by volume or by surface lookup) when the
+    /// geometry-construction side hasn't stamped @c extra yet.
+    std::function<std::uint8_t(Acts::GeometryIdentifier)> detectorResolver =
+        [](Acts::GeometryIdentifier gid) {
+          return static_cast<std::uint8_t>(gid.extra());
+        };
+  };
+
+  explicit ArrowSimHitOutputConverter(
+      const Config& cfg, std::unique_ptr<const Acts::Logger> logger = nullptr);
+
+  /// Build a resolver from a volume-id -> detector-id lookup table.
+  ///
+  /// This returns a pure C++ callable so Python can configure the mapping
+  /// once without paying a Python callback roundtrip for each hit.
+  static std::function<std::uint8_t(Acts::GeometryIdentifier)>
+  makeVolumeIdDetectorResolver(
+      const std::unordered_map<std::uint32_t, std::uint8_t>& volumeToDetector,
+      std::uint8_t defaultValue = 255);
+
+  const Config& config() const { return m_cfg; }
+
+  std::vector<std::string> collections() const override;
+
+ private:
+  ProcessCode execute(const AlgorithmContext& ctx) const override;
+
+  Config m_cfg;
+
+  ReadDataHandle<SimHitContainer> m_inputSimHits{this, "InputSimHits"};
+  ReadDataHandle<SimParticleContainer> m_inputParticles{this, "InputParticles"};
+  ReadDataHandle<ClusterContainer> m_inputClusters{this, "InputClusters"};
+  ReadDataHandle<SimHitMeasurementsMap> m_inputSimHitMeasurementsMap{
+      this, "InputSimHitMeasurementsMap"};
+
+  WriteDataHandle<ActsPlugins::ArrowUtil::ArrowTable> m_outputTable{
+      this, "OutputTable"};
+};
+
+}  // namespace ActsExamples

Examples/Io/Arrow/include/ActsExamples/Io/Arrow/ArrowTrackOutputConverter.hpp

@@ -0,0 +1,83 @@
+// This file is part of the ACTS project.
+//
+// Copyright (C) 2016 CERN for the benefit of the ACTS project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#pragma once
+
+#include "ActsExamples/EventData/Index.hpp"
+#include "ActsExamples/EventData/SimParticle.hpp"
+#include "ActsExamples/EventData/Track.hpp"
+#include "ActsExamples/EventData/TruthMatching.hpp"
+#include "ActsExamples/Framework/DataHandle.hpp"
+#include "ActsExamples/Io/Parquet/ArrowOutputConverter.hpp"
+#include "ActsPlugins/Arrow/ArrowUtil.hpp"
+#include "ActsPlugins/Arrow/Export.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace ActsExamples {
+
+/// Convert a @c ConstTrackContainer to an @c arrow::Table.
+///
+/// The output table has one row per event with list-valued columns for the
+/// perigee parameters (d0, z0, phi, theta, qop), the majority truth particle
+/// id, the per-track measurement (hit) indices, and a running track index.
+/// The @c ParquetWriter stamps the @c event_id column.
+class ACTS_ARROW_EXPORT ArrowTrackOutputConverter final
+    : public ArrowOutputConverter {
+ public:
+  struct Config {
+    /// Input @c ConstTrackContainer on the whiteboard.
+    std::string inputTracks;
+    /// Optional input track-to-particle matching on the whiteboard. If empty,
+    /// @c majority_particle_id is filled with the unmatched sentinel.
+    std::string inputTrackParticleMatching;
+    /// Particle container used to resolve the matched truth particle's row
+    /// index in the corresponding parquet table. Must be the same container
+    /// the @c ArrowParticleOutputConverter consumes for that table — leaving
+    /// it empty disables index resolution and forces the unmatched sentinel.
+    std::string inputParticles;
+    /// Optional measurement → sim-hit map. When set, each track-state's
+    /// measurement index is translated to one or more sim-hit indices (the
+    /// row indices of the corresponding hits parquet table); without it,
+    /// @c hit_ids is left empty so consumers don't mistake measurement
+    /// indices for sim-hit indices.
+    std::string inputMeasurementSimHitsMap;
+    /// Output whiteboard key for the resulting @c arrow::Table.
+    std::string outputTable = "tracks";
+    /// If false, the @c t (perigee time) column is still in the schema but
+    /// every cell is written as null. Lets downstream readers consume a
+    /// stable schema regardless of whether the producer carried time info.
+    bool writeTime = true;
+  };
+
+  explicit ArrowTrackOutputConverter(
+      const Config& cfg, std::unique_ptr<const Acts::Logger> logger = nullptr);
+
+  const Config& config() const { return m_cfg; }
+
+  std::vector<std::string> collections() const override;
+
+ private:
+  ProcessCode execute(const AlgorithmContext& ctx) const override;
+
+  Config m_cfg;
+
+  ReadDataHandle<ConstTrackContainer> m_inputTracks{this, "InputTracks"};
+  ReadDataHandle<TrackParticleMatching> m_inputTrackParticleMatching{
+      this, "InputTrackParticleMatching"};
+  ReadDataHandle<SimParticleContainer> m_inputParticles{this, "InputParticles"};
+  ReadDataHandle<MeasurementSimHitsMap> m_inputMeasurementSimHitsMap{
+      this, "InputMeasurementSimHitsMap"};
+
+  WriteDataHandle<ActsPlugins::ArrowUtil::ArrowTable> m_outputTable{
+      this, "OutputTable"};
+};
+
+}  // namespace ActsExamples