CLI Tools: openpmd-ls (#574)

Start adding command line interface (CLI) tools.

The first tool is `openpmd-ls`, which has the same function as `ls`
but specific for querying an openPMD data series.

CLI argument parsing and tool functionality will be improved in the
future.

Author: ax3l

CHANGELOG.rst

@@ -26,6 +26,7 @@ Features
 - ``Series::setSoftware()`` add second argument for version #657
 - free standing functions to query the API version and feature variants at runtime #665
 - expose ``determineFormat`` and ``suffix`` functions #684
+- CLI: add ``openpmd-ls`` tool #574
 
 Bug Fixes
 """""""""

CMakeLists.txt

@@ -91,7 +91,8 @@ mark_as_advanced(BUILD_SHARED_LIBS)
 include(CTest)
 # automatically defines: BUILD_TESTING, default is ON
 
-option(BUILD_EXAMPLES "Build the examples" ON)
+option(BUILD_CLI_TOOLS "Build the command line tools" ON)
+option(BUILD_EXAMPLES  "Build the examples" ON)
 
 
 # Dependencies ################################################################
@@ -312,7 +313,8 @@ set(CORE_SOURCE
         src/backend/PatchRecord.cpp
         src/backend/PatchRecordComponent.cpp
         src/backend/Writable.cpp
-        src/benchmark/mpi/OneDimensionalBlockSlicer.cpp)
+        src/benchmark/mpi/OneDimensionalBlockSlicer.cpp
+        src/helper/list_series.cpp)
 set(IO_SOURCE
         src/IO/AbstractIOHandlerHelper.cpp
         src/IO/DummyIOHandler.cpp
@@ -525,6 +527,7 @@ if(openPMD_HAVE_PYTHON)
         src/binding/python/Container.cpp
         src/binding/python/Dataset.cpp
         src/binding/python/Datatype.cpp
+        src/binding/python/Helper.cpp
         src/binding/python/Iteration.cpp
         src/binding/python/IterationEncoding.cpp
         src/binding/python/Mesh.cpp
@@ -588,6 +591,10 @@ set(openPMD_TEST_NAMES
     SerialIO
     ParallelIO
 )
+# command line tools
+set(openPMD_CLI_TOOL_NAMES
+    ls
+)
 # examples
 set(openPMD_EXAMPLE_NAMES
     1_structure
@@ -653,6 +660,13 @@ if(BUILD_TESTING)
     endforeach()
 endif()
 
+if(BUILD_CLI_TOOLS)
+    foreach(toolname ${openPMD_CLI_TOOL_NAMES})
+        add_executable(openpmd-${toolname} src/cli/${toolname}.cpp)
+        target_link_libraries(openpmd-${toolname} PRIVATE openPMD)
+    endforeach()
+endif()
+
 if(BUILD_EXAMPLES)
     foreach(examplename ${openPMD_EXAMPLE_NAMES})
         if(${examplename} MATCHES ".+parallel$")
@@ -761,6 +775,12 @@ if(openPMD_HAVE_ADIOS1)
         openPMD.ADIOS1.Serial openPMD.ADIOS1.Parallel)
 endif()
 
+if(BUILD_CLI_TOOLS)
+    foreach(toolname ${openPMD_CLI_TOOL_NAMES})
+        list(APPEND openPMD_INSTALL_TARGET_NAMES openpmd-${toolname})
+    endforeach()
+endif()
+
 install(TARGETS ${openPMD_INSTALL_TARGET_NAMES}
     EXPORT openPMDTargets
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -919,6 +939,25 @@ if(BUILD_EXAMPLES)
     endif()
 endif()
 
+# Command Line Tools ##################################################
+#
+if(BUILD_CLI_TOOLS)
+    # all tools must provide a "--help"
+    foreach(toolname ${openPMD_CLI_TOOL_NAMES})
+        list(APPEND openPMD_INSTALL_TARGET_NAMES openpmd-${toolname})
+        add_test(NAME CLI.help.${toolname}
+            COMMAND openpmd-${toolname} --help
+            WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
+        )
+    endforeach()
+    if(openPMD_HAVE_HDF5 AND EXISTS "${openPMD_BINARY_DIR}/samples/git-sample/")
+        add_test(NAME CLI.ls
+            COMMAND openpmd-ls ../samples/git-sample/data%08T.h5
+            WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
+        )
+    endif()
+endif()
+
 # Python Examples
 # Current examples all use HDF5, elaborate if other backends are used
 if(openPMD_HAVE_PYTHON AND openPMD_HAVE_HDF5)

Dockerfile

@@ -139,6 +139,7 @@ RUN        ls -hal /usr/local/lib/python3.7/dist-packages/
 # RUN        objdump -x /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so | grep RPATH
 RUN        ldd /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so
 RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
+#RUN        openpmd-ls --help
 #RUN        echo "* soft core 100000" >> /etc/security/limits.conf && \
 #           python3 -c "import openpmd_api as api;"; \
 #           gdb -ex bt -c core
@@ -167,6 +168,7 @@ RUN        python3 --version \
            && python3 -m pip install -U pip \
            && python3 -m pip install openPMD_api-*-cp36-cp36m-manylinux2010_x86_64.whl
 RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
+#RUN        openpmd-ls --help
 
 # test in fresh env: Debian:Stretch + Python 3.5
 FROM       debian:stretch
@@ -179,6 +181,7 @@ RUN        python3 --version \
            && python3 -m pip install -U pip \
            && python3 -m pip install openPMD_api-*-cp35-cp35m-manylinux2010_x86_64.whl
 RUN        python3 -c "import openpmd_api as api; print(api.__version__); print(api.variants)"
+#RUN        openpmd-ls --help
 
 
 # copy binary artifacts (wheels)

README.md

@@ -248,8 +248,8 @@ You can only build a static or a shared library at a time.
 By default, the `Release` version is built.
 In order to build with debug symbols, pass `-DCMAKE_BUILD_TYPE=Debug` to your `cmake` command.
 
-By default, tests and examples are built.
-In order to skip building those, pass `-DBUILD_TESTING=OFF` or `-DBUILD_EXAMPLES=OFF` to your `cmake` command.
+By default, tests, examples and command line tools are built.
+In order to skip building those, pass `-DBUILD_TESTING=OFF`, `-DBUILD_EXAMPLES=OFF`, or `-DBUILD_CLI_TOOLS=OFF` to your `cmake` command.
 
 ## Linking to your project
 

docs/source/dev/buildoptions.rst

@@ -75,8 +75,8 @@ CMake Option                      Values      Installs Library       Version
 ================================= =========== ======== ============= ========
 
 
-Tests
------
+Tests, Examples and Command Line Tools
+--------------------------------------
 
-By default, tests and examples are built.
-In order to skip building those, pass ``-DBUILD_TESTING=OFF`` or ``-DBUILD_EXAMPLES=OFF`` to your ``cmake`` command.
+By default, tests, examples and command line tools are built.
+In order to skip building those, pass ``-DBUILD_TESTING=OFF``, ``-DBUILD_EXAMPLES=OFF``, or ``-DBUILD_CLI_TOOLS=OFF`` to your ``cmake`` command.

docs/source/dev/repostructure.rst

@@ -19,10 +19,22 @@ Directory Structure
   * set ``-I`` here
   * prefixed with project name
 
+  * ``auxiliary/``
+
+    * internal auxiliary functionality
+
+  * ``helper/``, ``benchmark/``
+
+    * user-facing helper functionality
+
 * ``src/``
 
   * C++ source files
 
+  * ``cli/``
+
+    * user-facing command line tools
+
 * ``lib/``
 
     * ``python/``

docs/source/index.rst

@@ -92,7 +92,8 @@ Utilities
    :maxdepth: 1
    :hidden:
 
-   utilities/benchmark.rst
+   utilities/cli
+   utilities/benchmark
 
 Backends
 --------

docs/source/utilities/cli.rst

@@ -0,0 +1,14 @@
+.. _utilities-cli:
+
+Command Line Tools
+==================
+
+openPMD-api installs command line tools alongside the main library.
+These terminal-focused tools help to quickly explore, manage or manipulate openPMD data series.
+
+``openpmd-ls``
+--------------
+
+List information about an openPMD data series.
+See ``openpmd-ls --help`` for help.
+

include/openPMD/Series.hpp

@@ -227,17 +227,23 @@ class Series : public Attributable
     Series& setIterationFormat(std::string const& iterationFormat);
 
     /**
-     * @return  String of a pattern for file names.
+     * @return String of a pattern for file names.
      */
     std::string name() const;
+
     /** Set the pattern for file names.
      *
      * @param   name    String of the pattern for file names. Must include iteration regex <CODE>\%T</CODE> for fileBased data.
      * @return  Reference to modified series.
      */
     Series& setName(std::string const& name);
 
-    /** The currently used backend */
+    /** The currently used backend
+     *
+     * @see AbstractIOHandler::backendName()
+     *
+     * @return String of a pattern for data backend.
+     */
     std::string backend() const;
 
     /** Execute all required remaining IO operations to write or read data.

include/openPMD/helper/list_series.hpp

@@ -0,0 +1,47 @@
+/* Copyright 2019-2020 Axel Huebl
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "openPMD/Series.hpp"
+
+#include <ostream>
+#include <iostream>
+
+
+namespace openPMD
+{
+namespace helper
+{
+    /** List information about an openPMD data series
+     *
+     * @param series a openPMD data path as in Series::Series
+     * @param longer write more information
+     * @param out    an output stream to write textual information to
+     * @return reference to out as output stream, e.g. to pass the stream on via `operator<<`
+     */
+    std::ostream &
+    listSeries(
+        Series const & series,
+        bool const longer = false,
+        std::ostream & out = std::cout
+    );
+} // helper
+} // openPMD