Merge pull request #5001 from vicentebolea/backport-from-master

backport from master

Author: vicentebolea

REUSE.toml

@@ -102,6 +102,12 @@ precedence = "aggregate"
 SPDX-FileCopyrightText = "mingw-w64 contributors"
 SPDX-License-Identifier = "LicenseRef-public-domain"
 
+[[annotations]]
+path = "source/adios2/toolkit/derived/parser/pregen-source/*"
+precedence = "aggregate"
+SPDX-FileCopyrightText = "Oak Ridge National Laboratory and Contributors"
+SPDX-License-Identifier = "Apache-2.0"
+
 [[annotations]]
 path = ["thirdparty/CMakeLists.txt", "thirdparty/atl/CMakeLists.txt", "thirdparty/dill/CMakeLists.txt", "thirdparty/enet/CMakeLists.txt", "thirdparty/EVPath/CMakeLists.txt", "thirdparty/ffs/CMakeLists.txt", "thirdparty/GTest/CMakeLists.txt", "thirdparty/KWSys/CMakeLists.txt", "thirdparty/mingw-w64/CMakeLists.txt", "thirdparty/nanobind/CMakeLists.txt", "thirdparty/nlohmann_json/CMakeLists.txt", "thirdparty/perfstubs/CMakeLists.txt", "thirdparty/pugixml/CMakeLists.txt", "thirdparty/yaml-cpp/CMakeLists.txt"]
 precedence = "aggregate"

source/adios2/CMakeLists.txt

@@ -149,7 +149,7 @@ set(maybe_adios2_core_derived)
 if(ADIOS2_HAVE_Derived_Variable)
   target_sources(adios2_core PRIVATE
     core/VariableDerived.cpp
-    toolkit/derived/Expression.cpp
+    toolkit/derived/ExprCodeStream.cpp
     toolkit/derived/Function.cpp)
   set_target_properties(adios2_core PROPERTIES
     INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
@@ -182,17 +182,18 @@ if(ADIOS2_HAVE_Derived_Variable)
   elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM")
     SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
   elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
-    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
+    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/ExprCodeStream.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
   endif()
 
   add_library(adios2_core_derived
     ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
     ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
     toolkit/derived/parser/ASTDriver.cpp
-    toolkit/derived/parser/ASTNode.cpp)
+    toolkit/derived/parser/ASTNode.cpp
+    toolkit/derived/parser/ASTToExprNode.cpp)
   set_target_properties(adios2_core_derived PROPERTIES
     VISIBILITY_INLINES_HIDDEN ON
-    INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
+    INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source>;$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
     EXPORT_NAME core_derived
     OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)
   target_link_libraries(adios2_core PRIVATE adios2_core_derived)

source/adios2/core/IO.cpp

@@ -7,6 +7,8 @@
 #include "IO.h"
 #include "IO.tcc"
 
+#include <cstdlib>
+#include <iostream>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -948,9 +950,9 @@ VariableDerived &IO::DefineDerivedVariable(const std::string &name, const std::s
         }
     }
 
-    derived::Expression derived_exp(exp_string);
-    std::vector<std::string> var_list = derived_exp.VariableNameList();
-    bool isConstant = true;
+    // Parse expression string into ExprNode tree
+    derived::ExprNode exprTree = detail::ParseToExprNode(exp_string);
+    std::vector<std::string> var_list = derived::VariableNameList(exprTree);
     std::map<std::string, DataType> name_to_type;
     std::map<std::string, std::tuple<Dims, Dims, Dims>> name_to_dims;
     // check correctness for the variable names and types within the expression
@@ -963,22 +965,32 @@ VariableDerived &IO::DefineDerivedVariable(const std::string &name, const std::s
                                                      " in defining the derived variable " + name);
         DataType var_type = InquireVariableType(var_name);
         name_to_type.insert({var_name, var_type});
-        if ((itVariable->second)->IsConstantDims() == false)
-            isConstant = false;
         name_to_dims.insert({var_name,
                              {(itVariable->second)->m_Start, (itVariable->second)->m_Count,
                               (itVariable->second)->m_Shape}});
     }
-    // set the type of the expression and check correcness
-    DataType expressionType = derived_exp.GetType(name_to_type);
-    // set the initial shape of the expression and check correcness
-    derived_exp.SetDims(name_to_dims);
-    // std::cout << "Derived variable " << name << ": PASS : initial variable dimensions are valid"
-    //          << std::endl;
-
-    // create derived variable with the expression
+
+    // Build the expression code stream: GenerateCode -> ResolveTypes -> ConstantFold -> PlanBuffers
+    derived::ExprCodeStream codeStream = derived::GenerateCode(exprTree);
+    derived::ResolveTypes(codeStream, name_to_type);
+    derived::ConstantFold(codeStream);
+    derived::PlanBuffers(codeStream);
+    codeStream.ExprString = exp_string;
+    DataType expressionType = codeStream.OutputType;
+
+    {
+        static bool verbose = (getenv("DerivedVerbose") != nullptr);
+        if (verbose)
+            std::cerr << derived::DumpCodeStream(codeStream);
+    }
+
+    // Compute output dims before construction (VariableBase validates in InitShapeType)
+    auto outDims = derived::GetDims(codeStream, name_to_dims);
+
     auto itVariablePair = m_VariablesDerived.emplace(
-        name, std::make_unique<VariableDerived>(name, derived_exp, expressionType, isConstant,
+        name, std::make_unique<VariableDerived>(name, std::move(exprTree), std::move(codeStream),
+                                                exp_string, expressionType, std::get<2>(outDims),
+                                                std::get<0>(outDims), std::get<1>(outDims), false,
                                                 varType, name_to_type));
     VariableDerived &variable = static_cast<VariableDerived &>(*itVariablePair.first->second);
 

source/adios2/core/VariableDerived.cpp

@@ -12,27 +12,32 @@ namespace adios2
 namespace core
 {
 
-VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expression expr,
-                                 const DataType exprType, const bool isConstant,
-                                 const DerivedVarType varType,
+VariableDerived::VariableDerived(const std::string &name, adios2::derived::ExprNode exprTree,
+                                 adios2::derived::ExprCodeStream codeStream,
+                                 const std::string &exprString, const DataType exprType,
+                                 const Dims &shape, const Dims &start, const Dims &count,
+                                 const bool isConstant, const DerivedVarType varType,
                                  const std::map<std::string, DataType> nameToType)
-: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),
-               expr.GetCount(), isConstant),
-  m_DerivedType(varType), m_NameToType(nameToType), m_Expr(expr)
+: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), shape, start, count, isConstant),
+  m_DerivedType(varType), m_NameToType(nameToType), m_ExprTree(std::move(exprTree)),
+  m_CodeStream(std::move(codeStream)), m_ExprString(exprString)
 {
     if (varType != DerivedVarType::StoreData)
         m_WriteData = false;
 }
 
 DerivedVarType VariableDerived::GetDerivedType() { return m_DerivedType; }
 
-std::vector<std::string> VariableDerived::VariableNameList() { return m_Expr.VariableNameList(); }
+std::vector<std::string> VariableDerived::VariableNameList() { return m_CodeStream.InputVarNames; }
+
 void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims)
 {
-    m_Expr.SetDims(NameToDims);
-    m_Shape = m_Expr.GetShape();
-    m_Start = m_Expr.GetStart();
-    m_Count = m_Expr.GetCount();
+    auto outDims = adios2::derived::GetDims(m_CodeStream, NameToDims);
+    m_Shape = std::get<2>(outDims);
+    m_Start = std::get<0>(outDims);
+    m_Count = std::get<1>(outDims);
+    if (!m_Shape.empty())
+        m_ShapeID = ShapeID::GlobalArray;
 }
 
 std::vector<std::tuple<void *, Dims, Dims>>
@@ -80,7 +85,7 @@ VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInf
         inputData.insert({variable.first, varData});
     }
     std::vector<adios2::derived::DerivedData> outputData =
-        m_Expr.ApplyExpression(numBlocks, inputData);
+        adios2::derived::Execute(m_CodeStream, numBlocks, inputData);
 
     std::vector<std::tuple<void *, Dims, Dims>> blockData;
     for (size_t i = 0; i < numBlocks; i++)
@@ -115,7 +120,7 @@ VariableDerived::CreateEmptyData(std::map<std::string, std::unique_ptr<MinVarInf
             nameToDims.insert({variable.first, varDims});
         }
 
-        auto outputDims = m_Expr.GetDims(nameToDims);
+        auto outputDims = adios2::derived::GetDims(m_CodeStream, nameToDims);
 
         blockData.push_back({nullptr, std::get<0>(outputDims), std::get<1>(outputDims)});
     }

source/adios2/core/VariableDerived.h

@@ -8,7 +8,8 @@
 #define ADIOS2_CORE_VARIABLE_DERIVED_H_
 
 #include "adios2/core/VariableBase.h"
-#include "adios2/toolkit/derived/Expression.h"
+#include "adios2/toolkit/derived/ExprCodeStream.h"
+#include "adios2/toolkit/derived/ExprNode.h"
 
 namespace adios2
 {
@@ -28,9 +29,14 @@ class VariableDerived : public VariableBase
                     size_t numBlocks);
 
 public:
-    adios2::derived::Expression m_Expr;
-    VariableDerived(const std::string &name, adios2::derived::Expression expr,
-                    const DataType exprType, const bool isConstant, const DerivedVarType varType,
+    adios2::derived::ExprNode m_ExprTree;
+    adios2::derived::ExprCodeStream m_CodeStream;
+    std::string m_ExprString;
+
+    VariableDerived(const std::string &name, adios2::derived::ExprNode exprTree,
+                    adios2::derived::ExprCodeStream codeStream, const std::string &exprString,
+                    const DataType exprType, const Dims &shape, const Dims &start,
+                    const Dims &count, const bool isConstant, const DerivedVarType varType,
                     const std::map<std::string, DataType> nameToType);
     ~VariableDerived() = default;
 

source/adios2/engine/bp5/BP5Reader.cpp

@@ -25,6 +25,7 @@
 
 #include "BP5Reader.h"
 #include "BP5Reader.tcc"
+#include "adios2/toolkit/derived/ExprNode.h"
 
 using TP = std::chrono::high_resolution_clock::time_point;
 #define NOW() std::chrono::high_resolution_clock::now();
@@ -1508,8 +1509,8 @@ std::string BP5Reader::VariableExprStr(const VariableBase &Var)
     char *expPtr = m_BP5Deserializer->VariableExprStr(Var);
     if (expPtr != nullptr)
     {
-        derived::Expression expr(expPtr);
-        return expr.toStringExpr();
+        derived::ExprNode node = detail::ParseToExprNode(expPtr);
+        return derived::ToStringExpr(node);
     }
 #endif
     std::string noDerive("");

source/adios2/toolkit/derived/DerivedData.h

@@ -19,13 +19,15 @@ struct DerivedData
     Dims Start;
     Dims Count;
     DataType Type;
+    bool IsScalar = false; // true = Data points to one element, broadcast by compute functions
 };
 
 struct ExprData
 {
     std::vector<DerivedData> Data;
-    std::vector<std::string> Const;
     DataType OutType;
+    void *Output = nullptr; // pre-allocated output buffer
+    size_t OutputSize = 0;  // number of elements in output
 };
 }
 }