Skip to content

Commit 2ee1ab4

Browse files
franzpoeschelfranzpoeschel
committed
ADIOS1: read dataset transform from JSON
1 parent b79191a commit 2ee1ab4

File tree

9 files changed

+157
-13
lines changed

9 files changed

+157
-13
lines changed

‎include/openPMD/Error.hpp

+9
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <exception>
44
#include <string>
55
#include <utility>
6+
#include <vector>
67

78
namespace openPMD
89
{
@@ -62,5 +63,13 @@ namespace error
6263
public:
6364
WrongAPIUsage( std::string what );
6465
};
66+
67+
class BackendConfigSchema : public Error
68+
{
69+
public:
70+
std::vector< std::string > errorLocation;
71+
72+
BackendConfigSchema( std::vector< std::string >, std::string what );
73+
};
6574
}
6675
}

‎include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp

+5
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "openPMD/IO/AbstractIOHandler.hpp"
2828
#include "openPMD/auxiliary/Filesystem.hpp"
2929
#include "openPMD/auxiliary/DerefDynamicCast.hpp"
30+
#include "openPMD/auxiliary/JSON.hpp"
3031
#include "openPMD/auxiliary/Memory.hpp"
3132
#include "openPMD/auxiliary/StringManip.hpp"
3233
#include "openPMD/IO/AbstractIOHandlerImpl.hpp"
@@ -89,11 +90,15 @@ namespace openPMD
8990
std::unordered_map< std::shared_ptr< std::string >, ADIOS_FILE* > m_openReadFileHandles;
9091
std::unordered_map< ADIOS_FILE*, std::vector< ADIOS_SELECTION* > > m_scheduledReads;
9192
std::unordered_map< int64_t, std::unordered_map< std::string, Attribute > > m_attributeWrites;
93+
// config options
94+
std::string m_defaultTransform;
9295
/**
9396
* Call this function to get adios file id for a Writable. Will create one if does not exist
9497
* @return returns an adios file id.
9598
*/
9699
int64_t GetFileHandle(Writable*);
100+
101+
void initJson( json::TracingJSON );
97102
}; // ParallelADIOS1IOHandlerImpl
98103
} // openPMD
99104

‎include/openPMD/auxiliary/JSON.hpp

+5-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
#include "openPMD/config.hpp"
2525

26+
#include "openPMD/auxiliary/Option.hpp"
27+
2628
#include <nlohmann/json.hpp>
2729

2830
#if openPMD_HAVE_MPI
@@ -182,8 +184,9 @@ namespace json
182184

183185
nlohmann::json & lowerCase( nlohmann::json & );
184186

185-
std::string asStringDynamic( nlohmann::json const & );
187+
auxiliary::Option< std::string > asStringDynamic( nlohmann::json const & );
186188

187-
std::string asLowerCaseStringDynamic( nlohmann::json const & );
189+
auxiliary::Option< std::string >
190+
asLowerCaseStringDynamic( nlohmann::json const & );
188191
} // namespace json
189192
} // namespace openPMD

‎src/Error.cpp

+28
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "openPMD/Error.hpp"
22

3+
#include <sstream>
4+
35
namespace openPMD
46
{
57
const char * Error::what() const noexcept
@@ -20,5 +22,31 @@ namespace error
2022
: Error( "Wrong API usage: " + what )
2123
{
2224
}
25+
26+
static std::string concatVector(
27+
std::vector< std::string > const & vec,
28+
std::string const & intersperse = "." )
29+
{
30+
if( vec.empty() )
31+
{
32+
return "";
33+
}
34+
std::stringstream res;
35+
res << vec[ 0 ];
36+
for( size_t i = 1; i < vec.size(); ++i )
37+
{
38+
res << intersperse << vec[ i ];
39+
}
40+
return res.str();
41+
}
42+
43+
BackendConfigSchema::BackendConfigSchema(
44+
std::vector< std::string > errorLocation_in, std::string what )
45+
: Error(
46+
"Wrong JSON schema at index '" + concatVector( errorLocation ) +
47+
"': " + std::move( what ) )
48+
, errorLocation( std::move( errorLocation_in ) )
49+
{
50+
}
2351
}
2452
}

‎src/IO/ADIOS/ADIOS1IOHandler.cpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@ namespace openPMD
4545

4646
ADIOS1IOHandlerImpl::ADIOS1IOHandlerImpl(AbstractIOHandler* handler, json::TracingJSON json)
4747
: Base_t(handler)
48-
{ }
48+
{
49+
initJson( std::move( json ) );
50+
}
4951

5052
ADIOS1IOHandlerImpl::~ADIOS1IOHandlerImpl()
5153
{

‎src/IO/ADIOS/ADIOS2IOHandler.cpp

+24-3
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
2323

2424
#include "openPMD/Datatype.hpp"
25+
#include "openPMD/Error.hpp"
2526
#include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
2627
#include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
2728
#include "openPMD/auxiliary/Environment.hpp"
@@ -140,8 +141,18 @@ ADIOS2IOHandlerImpl::init( json::TracingJSON cfg )
140141
if( !engineTypeConfig.is_null() )
141142
{
142143
// convert to string
143-
m_engineType =
144+
auto maybeEngine =
144145
json::asLowerCaseStringDynamic( engineTypeConfig );
146+
if( maybeEngine.has_value() )
147+
{
148+
m_engineType = std::move( maybeEngine.get() );
149+
}
150+
else
151+
{
152+
throw error::BackendConfigSchema(
153+
{"adios2", "engine", "type"},
154+
"Must be convertible to string type." );
155+
}
145156
}
146157
}
147158
auto operators = getOperators();
@@ -2334,8 +2345,18 @@ namespace detail
23342345
for( auto it = params.json().begin(); it != params.json().end();
23352346
it++ )
23362347
{
2337-
m_IO.SetParameter(
2338-
it.key(), json::asStringDynamic( it.value() ) );
2348+
auto maybeString = json::asStringDynamic( it.value() );
2349+
if( maybeString.has_value() )
2350+
{
2351+
m_IO.SetParameter(
2352+
it.key(), std::move( maybeString.get() ) );
2353+
}
2354+
else
2355+
{
2356+
throw error::BackendConfigSchema(
2357+
{"adios2", "engine", "parameters", it.key() },
2358+
"Must be convertible to string type." );
2359+
}
23392360
alreadyConfigured.emplace( it.key() );
23402361
}
23412362
}

‎src/IO/ADIOS/CommonADIOS1IOHandler.cpp

+70-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
#if openPMD_HAVE_ADIOS1
2525

26+
#include "openPMD/auxiliary/JSON.hpp"
27+
#include "openPMD/Error.hpp"
2628
#include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp"
2729
#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp"
2830

@@ -468,6 +470,33 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createPath(Writable* writable,
468470
}
469471
}
470472

473+
static auxiliary::Option< std::string > datasetTransform(
474+
json::TracingJSON config )
475+
{
476+
using ret_t = auxiliary::Option< std::string >;
477+
if( !config.json().contains( "dataset" ) )
478+
{
479+
return ret_t{};
480+
}
481+
config = config[ "dataset" ];
482+
if( !config.json().contains( "transform" ) )
483+
{
484+
return ret_t{};
485+
}
486+
config = config[ "transform" ];
487+
auto maybeRes = json::asStringDynamic( config.json() );
488+
if( maybeRes.has_value() )
489+
{
490+
return std::move( maybeRes.get() );
491+
}
492+
else
493+
{
494+
throw error::BackendConfigSchema(
495+
{ "adios1", "dataset", "transform" },
496+
"Key must convertible to type string." );
497+
}
498+
}
499+
471500
template< typename ChildClass >
472501
void
473502
CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable,
@@ -521,7 +550,31 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable,
521550
chunkOffsetParam.c_str());
522551
VERIFY(id != 0, "[ADIOS1] Internal error: Failed to define ADIOS variable during Dataset creation");
523552

524-
std::string transform = ""; // @todo read from JSON
553+
std::string transform = "";
554+
{
555+
json::TracingJSON options =
556+
json::parseOptions( parameters.options );
557+
auto maybeTransform = datasetTransform( options );
558+
if( maybeTransform.has_value() )
559+
{
560+
transform = maybeTransform.get();
561+
}
562+
563+
auto shadow = options.invertShadow();
564+
if( shadow.size() > 0 )
565+
{
566+
std::cerr << "Warning: parts of the JSON configuration for "
567+
"ADIOS1 dataset '"
568+
<< name << "' remain unused:\n"
569+
<< shadow << std::endl;
570+
}
571+
}
572+
// Fallback: global option
573+
if( transform.empty() )
574+
{
575+
transform = m_defaultTransform;
576+
}
577+
525578
if( !transform.empty() )
526579
{
527580
int status;
@@ -1700,9 +1753,24 @@ CommonADIOS1IOHandlerImpl< ChildClass >::listAttributes(Writable* writable,
17001753
}
17011754
}
17021755

1756+
template< typename ChildClass >
1757+
void CommonADIOS1IOHandlerImpl< ChildClass >::initJson(
1758+
json::TracingJSON config )
1759+
{
1760+
if( !config.json().contains( "adios1" ) )
1761+
{
1762+
return;
1763+
}
1764+
auto maybeTransform = datasetTransform( config[ "adios1" ] );
1765+
if( maybeTransform.has_value() )
1766+
{
1767+
m_defaultTransform = std::move( maybeTransform.get() );
1768+
}
1769+
}
1770+
17031771
template class CommonADIOS1IOHandlerImpl< ADIOS1IOHandlerImpl >;
17041772
#if openPMD_HAVE_MPI
17051773
template class CommonADIOS1IOHandlerImpl< ParallelADIOS1IOHandlerImpl >;
17061774
#endif // opepnPMD_HAVE_MPI
17071775
#endif // openPMD_HAVE_ADIOS1
1708-
}
1776+
}

‎src/IO/ADIOS/ParallelADIOS1IOHandler.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ ParallelADIOS1IOHandlerImpl::ParallelADIOS1IOHandlerImpl(AbstractIOHandler* hand
4949
int status = MPI_SUCCESS;
5050
status = MPI_Comm_dup(comm, &m_mpiComm);
5151
VERIFY(status == MPI_SUCCESS, "[ADIOS1] Internal error: Failed to duplicate MPI communicator");
52+
initJson( std::move( json ) );
5253
}
5354

5455
ParallelADIOS1IOHandlerImpl::~ParallelADIOS1IOHandlerImpl()

‎src/auxiliary/JSON.cpp

+12-5
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,8 @@ namespace json
232232
return json;
233233
}
234234

235-
std::string asStringDynamic( nlohmann::json const & value )
235+
auxiliary::Option< std::string >
236+
asStringDynamic( nlohmann::json const & value )
236237
{
237238
if( value.is_string() )
238239
{
@@ -248,14 +249,20 @@ namespace json
248249
}
249250
else if( value.is_boolean() )
250251
{
251-
return value.get< bool >() ? "1" : "0";
252+
return std::string( value.get< bool >() ? "1" : "0" );
252253
}
253-
throw std::runtime_error( "JSON: Cannot convert value to string." );
254+
return auxiliary::Option< std::string >{};
254255
}
255256

256-
std::string asLowerCaseStringDynamic( nlohmann::json const & value )
257+
auxiliary::Option< std::string >
258+
asLowerCaseStringDynamic( nlohmann::json const & value )
257259
{
258-
return auxiliary::lowerCase( asStringDynamic( value ) );
260+
auto maybeString = asStringDynamic( value );
261+
if( maybeString.has_value() )
262+
{
263+
auxiliary::lowerCase( maybeString.get() );
264+
}
265+
return maybeString;
259266
}
260267
} // namespace json
261268
} // namespace openPMD

0 commit comments

Comments
 (0)