ADIOS1: read dataset transform from JSON

Author: franzpoeschel

include/openPMD/Error.hpp

@@ -3,6 +3,7 @@
 #include <exception>
 #include <string>
 #include <utility>
+#include <vector>
 
 namespace openPMD
 {
@@ -62,5 +63,13 @@ namespace error
     public:
         WrongAPIUsage( std::string what );
     };
+
+    class BackendConfigSchema : public Error
+    {
+    public:
+        std::vector< std::string > errorLocation;
+
+        BackendConfigSchema( std::vector< std::string >, std::string what );
+    };
 }
 }

include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp

@@ -27,6 +27,7 @@
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/DerefDynamicCast.hpp"
+#include "openPMD/auxiliary/JSON.hpp"
 #include "openPMD/auxiliary/Memory.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
@@ -89,11 +90,15 @@ namespace openPMD
         std::unordered_map< std::shared_ptr< std::string >, ADIOS_FILE* > m_openReadFileHandles;
         std::unordered_map< ADIOS_FILE*, std::vector< ADIOS_SELECTION* > > m_scheduledReads;
         std::unordered_map< int64_t, std::unordered_map< std::string, Attribute > > m_attributeWrites;
+        // config options
+        std::string m_defaultTransform;
         /**
          * Call this function to get adios file id for a Writable. Will create one if does not exist
          * @return  returns an adios file id. 
          */	  
         int64_t GetFileHandle(Writable*);
+
+        void initJson( json::TracingJSON );
     }; // ParallelADIOS1IOHandlerImpl
 } // openPMD
 

include/openPMD/auxiliary/JSON.hpp

@@ -23,6 +23,8 @@
 
 #include "openPMD/config.hpp"
 
+#include "openPMD/auxiliary/Option.hpp"
+
 #include <nlohmann/json.hpp>
 
 #if openPMD_HAVE_MPI
@@ -182,8 +184,9 @@ namespace json
 
     nlohmann::json & lowerCase( nlohmann::json & );
 
-    std::string asStringDynamic( nlohmann::json const & );
+    auxiliary::Option< std::string > asStringDynamic( nlohmann::json const & );
 
-    std::string asLowerCaseStringDynamic( nlohmann::json const & );
+    auxiliary::Option< std::string >
+    asLowerCaseStringDynamic( nlohmann::json const & );
 } // namespace json
 } // namespace openPMD

src/Error.cpp

@@ -1,5 +1,7 @@
 #include "openPMD/Error.hpp"
 
+#include <sstream>
+
 namespace openPMD
 {
 const char * Error::what() const noexcept
@@ -20,5 +22,31 @@ namespace error
         : Error( "Wrong API usage: " + what )
     {
     }
+
+    static std::string concatVector(
+        std::vector< std::string > const & vec,
+        std::string const & intersperse = "." )
+    {
+        if( vec.empty() )
+        {
+            return "";
+        }
+        std::stringstream res;
+        res << vec[ 0 ];
+        for( size_t i = 1; i < vec.size(); ++i )
+        {
+            res << intersperse << vec[ i ];
+        }
+        return res.str();
+    }
+
+    BackendConfigSchema::BackendConfigSchema(
+        std::vector< std::string > errorLocation_in, std::string what )
+        : Error(
+              "Wrong JSON schema at index '" + concatVector( errorLocation ) +
+              "': " + std::move( what ) )
+        , errorLocation( std::move( errorLocation_in ) )
+    {
+    }
 }
 }

src/IO/ADIOS/ADIOS1IOHandler.cpp

@@ -45,7 +45,9 @@ namespace openPMD
 
 ADIOS1IOHandlerImpl::ADIOS1IOHandlerImpl(AbstractIOHandler* handler, json::TracingJSON json)
         : Base_t(handler)
-{ }
+{
+    initJson( std::move( json ) );
+}
 
 ADIOS1IOHandlerImpl::~ADIOS1IOHandlerImpl()
 {

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -22,6 +22,7 @@
 #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
 
 #include "openPMD/Datatype.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
@@ -140,8 +141,18 @@ ADIOS2IOHandlerImpl::init( json::TracingJSON cfg )
             if( !engineTypeConfig.is_null() )
             {
                 // convert to string
-                m_engineType =
+                auto maybeEngine =
                     json::asLowerCaseStringDynamic( engineTypeConfig );
+                if( maybeEngine.has_value() )
+                {
+                    m_engineType = std::move( maybeEngine.get() );
+                }
+                else
+                {
+                    throw error::BackendConfigSchema(
+                        {"adios2", "engine", "type"},
+                        "Must be convertible to string type." );
+                }
             }
         }
         auto operators = getOperators();
@@ -2334,8 +2345,18 @@ namespace detail
                 for( auto it = params.json().begin(); it != params.json().end();
                      it++ )
                 {
-                    m_IO.SetParameter(
-                        it.key(), json::asStringDynamic( it.value() ) );
+                    auto maybeString = json::asStringDynamic( it.value() );
+                    if( maybeString.has_value() )
+                    {
+                        m_IO.SetParameter(
+                            it.key(), std::move( maybeString.get() ) );
+                    }
+                    else
+                    {
+                        throw error::BackendConfigSchema(
+                            {"adios2", "engine", "parameters", it.key() },
+                            "Must be convertible to string type." );
+                    }
                     alreadyConfigured.emplace( it.key() );
                 }
             }

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -23,6 +23,8 @@
 
 #if openPMD_HAVE_ADIOS1
 
+#include "openPMD/auxiliary/JSON.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp"
 #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp"
 
@@ -468,6 +470,33 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createPath(Writable* writable,
     }
 }
 
+static auxiliary::Option< std::string > datasetTransform(
+    json::TracingJSON config )
+{
+    using ret_t = auxiliary::Option< std::string >;
+    if( !config.json().contains( "dataset" ) )
+    {
+        return ret_t{};
+    }
+    config = config[ "dataset" ];
+    if( !config.json().contains( "transform" ) )
+    {
+        return ret_t{};
+    }
+    config = config[ "transform" ];
+    auto maybeRes = json::asStringDynamic( config.json() );
+    if( maybeRes.has_value() )
+    {
+        return std::move( maybeRes.get() );
+    }
+    else
+    {
+        throw error::BackendConfigSchema(
+            { "adios1", "dataset", "transform" },
+            "Key must convertible to type string." );
+    }
+}
+
 template< typename ChildClass >
 void
 CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable,
@@ -521,7 +550,31 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable,
                               chunkOffsetParam.c_str());
         VERIFY(id != 0, "[ADIOS1] Internal error: Failed to define ADIOS variable during Dataset creation");
 
-        std::string transform = ""; // @todo read from JSON
+        std::string transform = "";
+        {
+            json::TracingJSON options =
+                json::parseOptions( parameters.options );
+            auto maybeTransform = datasetTransform( options );
+            if( maybeTransform.has_value() )
+            {
+                transform = maybeTransform.get();
+            }
+
+            auto shadow = options.invertShadow();
+            if( shadow.size() > 0 )
+            {
+                std::cerr << "Warning: parts of the JSON configuration for "
+                             "ADIOS1 dataset '"
+                          << name << "' remain unused:\n"
+                          << shadow << std::endl;
+            }
+        }
+        // Fallback: global option
+        if( transform.empty() )
+        {
+            transform = m_defaultTransform;
+        }
+
         if( !transform.empty() )
         {
             int status;
@@ -1700,9 +1753,24 @@ CommonADIOS1IOHandlerImpl< ChildClass >::listAttributes(Writable* writable,
     }
 }
 
+template< typename ChildClass >
+void CommonADIOS1IOHandlerImpl< ChildClass >::initJson(
+    json::TracingJSON config )
+{
+    if( !config.json().contains( "adios1" ) )
+    {
+        return;
+    }
+    auto maybeTransform = datasetTransform( config[ "adios1" ] );
+    if( maybeTransform.has_value() )
+    {
+        m_defaultTransform = std::move( maybeTransform.get() );
+    }
+}
+
 template class CommonADIOS1IOHandlerImpl< ADIOS1IOHandlerImpl >;
 #if openPMD_HAVE_MPI
 template class CommonADIOS1IOHandlerImpl< ParallelADIOS1IOHandlerImpl >;
 #endif // opepnPMD_HAVE_MPI
 #endif // openPMD_HAVE_ADIOS1
-}
+}

src/IO/ADIOS/ParallelADIOS1IOHandler.cpp

@@ -49,6 +49,7 @@ ParallelADIOS1IOHandlerImpl::ParallelADIOS1IOHandlerImpl(AbstractIOHandler* hand
     int status = MPI_SUCCESS;
     status = MPI_Comm_dup(comm, &m_mpiComm);
     VERIFY(status == MPI_SUCCESS, "[ADIOS1] Internal error: Failed to duplicate MPI communicator");
+    initJson( std::move( json ) );
 }
 
 ParallelADIOS1IOHandlerImpl::~ParallelADIOS1IOHandlerImpl()

src/auxiliary/JSON.cpp

@@ -232,7 +232,8 @@ namespace json
         return json;
     }
 
-    std::string asStringDynamic( nlohmann::json const & value )
+    auxiliary::Option< std::string >
+    asStringDynamic( nlohmann::json const & value )
     {
         if( value.is_string() )
         {
@@ -248,14 +249,20 @@ namespace json
         }
         else if( value.is_boolean() )
         {
-            return value.get< bool >() ? "1" : "0";
+            return std::string( value.get< bool >() ? "1" : "0" );
         }
-        throw std::runtime_error( "JSON: Cannot convert value to string." );
+        return auxiliary::Option< std::string >{};
     }
 
-    std::string asLowerCaseStringDynamic( nlohmann::json const & value )
+    auxiliary::Option< std::string >
+    asLowerCaseStringDynamic( nlohmann::json const & value )
     {
-        return auxiliary::lowerCase( asStringDynamic( value ) );
+        auto maybeString = asStringDynamic( value );
+        if( maybeString.has_value() )
+        {
+            auxiliary::lowerCase( maybeString.get() );
+        }
+        return maybeString;
     }
 } // namespace json
 } // namespace openPMD