diff --git a/CMakeLists.txt b/CMakeLists.txt index ab4d090919..cc38dc7078 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -315,6 +315,9 @@ set(CORE_SOURCE src/backend/Writable.cpp src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp) +if(openPMD_HAVE_MPI) + set(CORE_SOURCE ${CORE_SOURCE} src/MPISeries.cpp) # TODO: MPI_SOURCE and own target +endif() set(IO_SOURCE src/IO/AbstractIOHandlerHelper.cpp src/IO/DummyIOHandler.cpp diff --git a/NEWS.rst b/NEWS.rst index bb3123822a..3962368a7f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,27 @@ Upgrade Guide ============= +0.12.0-alpha +------------ + +MPI-parallel C++ functionality has its own include ```` and ``MPISeries`` class now. +The old ``Series(std::string, AccessType, MPI_Comm)`` constructor has been deprecated and will be removed in a future version. +New usage for MPI-parallel I/O: + +.. code-block:: cpp + + #include // new include + + using namespace io = openPMD; + + // ... + auto series = io::MPISeries( // instead of io::Series + "path/to/data_files_%T.h5", + io::AccessType::READ_ONLY, + MPI_COMM_WORLD + ); + + 0.11.0-alpha ------------ diff --git a/docs/source/details/doxygen.rst b/docs/source/details/doxygen.rst index 42fe946570..ae96dab40e 100644 --- a/docs/source/details/doxygen.rst +++ b/docs/source/details/doxygen.rst @@ -14,7 +14,8 @@ Public Headers =========================== ======================================================= Include Description =========================== ======================================================= -```` Public facade header (serial and MPI) +```` Public facade header (serial) +```` Public facade header (MPI) ```` Optional :ref:`benchmark ` helpers =========================== ======================================================= diff --git a/include/openPMD/MPI.hpp b/include/openPMD/MPI.hpp new file mode 100644 index 0000000000..afa4e5342a --- /dev/null +++ b/include/openPMD/MPI.hpp @@ -0,0 +1,27 @@ +/* Copyright 2020 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +// IWYU pragma: begin_exports +#include "openPMD/MPISeries.hpp" + +#include "openPMD/openPMD.hpp" +// IWYU pragma: end_exports diff --git a/include/openPMD/MPISeries.hpp b/include/openPMD/MPISeries.hpp new file mode 100644 index 0000000000..50c857643c --- /dev/null +++ b/include/openPMD/MPISeries.hpp @@ -0,0 +1,56 @@ +/* Copyright 2017-2020 Fabian Koller, Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/config.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/IO/AccessType.hpp" + +#if openPMD_HAVE_MPI +# include +#else +# error "Including requires an MPI-enabled installation" +#endif + +#include + + +namespace openPMD +{ + /** @brief Root level of the openPMD hierarchy for MPI-parallel I/O. + * + * Entry point and common link between all iterations of particle and mesh data. + * + * @see Series + */ + class MPISeries : public Series + { + //friend class Iteration; + //friend class Series; + + public: + MPISeries(std::string const& filepath, + AccessType at, + MPI_Comm comm); + + ~MPISeries(); + }; // MPISeries +} // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 358b412db3..3c3f7110a1 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -22,6 +22,7 @@ #include "openPMD/config.hpp" #include "openPMD/auxiliary/Deprecated.hpp" +#include "openPMD/auxiliary/ParsedInput.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" @@ -41,6 +42,9 @@ #ifndef OPENPMD_private # define OPENPMD_private private #endif +#ifndef OPENPMD_protected +# define OPENPMD_protected protected +#endif namespace openPMD @@ -58,6 +62,7 @@ class Series : public Attributable public: #if openPMD_HAVE_MPI + OPENPMDAPI_DEPRECATED("Please include and use the class MPISeries for MPI-parallel I/O.") Series(std::string const& filepath, AccessType at, MPI_Comm comm); @@ -252,10 +257,19 @@ class Series : public Attributable Container< Iteration, uint64_t > iterations; +OPENPMD_protected: + /** This constructor is only for derived Series-types + * + * calling this will skip the creation of an ioHandler + */ + Series(); + + std::unique_ptr< auxiliary::ParsedInput > parseInput(std::string); + void init(std::shared_ptr< AbstractIOHandler >, std::unique_ptr< auxiliary::ParsedInput >); + + std::shared_ptr< IterationEncoding > m_iterationEncoding; + OPENPMD_private: - struct ParsedInput; - std::unique_ptr< ParsedInput > parseInput(std::string); - void init(std::shared_ptr< AbstractIOHandler >, std::unique_ptr< ParsedInput >); void initDefaults(); void flushFileBased(); void flushGroupBased(); @@ -268,7 +282,6 @@ class Series : public Attributable static constexpr char const * const BASEPATH = "/data/%T/"; - std::shared_ptr< IterationEncoding > m_iterationEncoding; std::shared_ptr< std::string > m_name; std::shared_ptr< Format > m_format; diff --git a/include/openPMD/auxiliary/ParsedInput.hpp b/include/openPMD/auxiliary/ParsedInput.hpp new file mode 100644 index 0000000000..4e6ba37605 --- /dev/null +++ b/include/openPMD/auxiliary/ParsedInput.hpp @@ -0,0 +1,44 @@ +/* Copyright 2020 Axel Huebl, Fabian Koller + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IterationEncoding.hpp" +#include "openPMD/IO/Format.hpp" + +#include + + +namespace openPMD +{ +namespace auxiliary +{ + struct ParsedInput + { + std::string path; + std::string name; + Format format; + IterationEncoding iterationEncoding; + std::string filenamePrefix; + std::string filenamePostfix; + int filenamePadding; + }; // ParsedInput +} // namespace backend +} // namespace openPMD diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index fa411eda60..bd718a26e4 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -75,6 +75,7 @@ class Container : public Attributable friend class Iteration; friend class ParticleSpecies; friend class Series; + friend class MPISeries; public: using key_type = typename InternalContainer::key_type; diff --git a/src/MPISeries.cpp b/src/MPISeries.cpp new file mode 100644 index 0000000000..71ab234cca --- /dev/null +++ b/src/MPISeries.cpp @@ -0,0 +1,66 @@ +/* Copyright 2017-2020 Fabian Koller, Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include "openPMD/MPISeries.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/IterationEncoding.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandlerHelper.hpp" + +#include +#include +#include +#include + + +namespace openPMD +{ + MPISeries::MPISeries( + std::string const& filepath, + AccessType at, + MPI_Comm comm + ) + { + iterations = Container< Iteration, uint64_t >(); + m_iterationEncoding = std::make_shared< IterationEncoding >(); + + auto input = parseInput(filepath); + auto handler = createIOHandler(input->path, at, input->format, comm); + init(handler, std::move(input)); + } + + MPISeries::~MPISeries() + { + // we must not throw in a destructor + try + { + flush(); + } + catch( std::exception const & ex ) + { + std::cerr << "[~MPISeries] An error occurred: " << ex.what() << std::endl; + } + catch( ... ) + { + std::cerr << "[~MPISeries] An error occurred." << std::endl; + } + } +} // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index f03b9d3a58..5daa9ef793 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -68,17 +68,6 @@ std::string cleanFilename(std::string const& filename, Format f); std::function< std::tuple< bool, int >(std::string const&) > matcher(std::string const& prefix, int padding, std::string const& postfix, Format f); -struct Series::ParsedInput -{ - std::string path; - std::string name; - Format format; - IterationEncoding iterationEncoding; - std::string filenamePrefix; - std::string filenamePostfix; - int filenamePadding; -}; //ParsedInput - #if openPMD_HAVE_MPI Series::Series(std::string const& filepath, AccessType at, @@ -102,6 +91,10 @@ Series::Series(std::string const& filepath, init(handler, std::move(input)); } +Series::Series() +{ +} + Series::~Series() { flush(); @@ -361,10 +354,10 @@ Series::flush() IOHandler->flush(); } -std::unique_ptr< Series::ParsedInput > +std::unique_ptr< auxiliary::ParsedInput > Series::parseInput(std::string filepath) { - std::unique_ptr< Series::ParsedInput > input{new Series::ParsedInput}; + std::unique_ptr< auxiliary::ParsedInput > input{new auxiliary::ParsedInput}; #ifdef _WIN32 if( auxiliary::contains(filepath, '/') ) @@ -462,7 +455,7 @@ Series::parseInput(std::string filepath) void Series::init(std::shared_ptr< AbstractIOHandler > ioHandler, - std::unique_ptr< Series::ParsedInput > input) + std::unique_ptr< auxiliary::ParsedInput > input) { m_writable->IOHandler = ioHandler; IOHandler = m_writable->IOHandler.get();