Merge pull request #6 from dneg/v0.0.3

Version 0.0.3 - Novemember 13, 2018

Author: Idclip

openvdb_ax/CHANGES.md

@@ -1,6 +1,49 @@
 OpenVDB AX Version History
 ==========================
 
+Version 0.0.3 - November 13, 2018
+
+    New features:
+    - Introduced new $ syntax and back-end logic for fast lookups into AX Custom
+      Data. This syntax works identically to attribute access syntax with type
+      specifiers expected to be present when accessing non float data. For
+      literal path names, this can be up to x20 faster than using the lookup()
+      functions.
+    - New External Variable AST node, Code Generation/Visitor methods and
+      internal intrinsic functions for processing Custom Data lookups using $
+      syntax.
+    - Added a --print-ast command to the vdb_ax binary to output the built AST
+      from the provided code snippet.
+
+    Improvements:
+    - Removed the uint64_t LLVM Type specialization as all back-end values are
+      signed.
+    - Introduced a uintptr_t LLVM Type specialization for accessing pointers
+      to typed data.
+    - Consolidated attribute type checking with external variable type checking
+      in Compiler.cc into verifyTypedAccesses().
+    - Minor re-work to the PointDefaultModifier for conversion of default
+      attributes.
+
+    API Changes:
+    - ax::print() now takes a const ast::Tree reference.
+
+    Houdini:
+    - External variable $ syntax can now be used to access parameters on the AX
+      SOP e.g. ch("parm") -> $parm
+    - Known path lookups using supported Houdini channel functions (e.g. ch())
+      or AX lookup functions are optimized using the new external variable
+      syntax for speed gains of up to x20.
+    - Added optimised access to all available floating points Global and Local
+      Node HScript variables such as $F, $T etc.
+    - Back-ticks are no longer supported as the code is evaluated as a raw
+      string. This is to support optimizations for ch() and Houdini HScript $
+      parameters. These can be embedded without the need to re-compile the
+      string parameter, even if they are time dependent.
+    - Verbified the AX SOP so that it can be used in compile blocks and as a
+      verb through python.
+
+
 Version 0.0.2 - October 8, 2018
 
     Improvements:

openvdb_ax/CMakeLists.txt

@@ -195,6 +195,7 @@ SET ( TEST_SOURCE_FILES
   test/frontend/TestConditionalStatementNode.cc
   test/frontend/TestCrementNode.cc
   test/frontend/TestDeclareLocalNode.cc
+  test/frontend/TestExternalVariableNode.cc
   test/frontend/TestFunctionCallNode.cc
   test/frontend/TestKeywordNode.cc
   test/frontend/TestLocalValueNode.cc
@@ -207,7 +208,6 @@ SET ( TEST_SOURCE_FILES
   test/integration/TestAssign.cc
   test/integration/TestBinary.cc
   test/integration/TestCast.cc
-  test/integration/TestChannelExpressions.cc
   test/integration/TestDeclare.cc
   test/integration/TestEditGroups.cc
   test/integration/TestEmpty.cc

openvdb_ax/Makefile

@@ -318,6 +318,7 @@ TEST_SRC_NAMES := \
     test/frontend/TestConditionalStatementNode.cc \
     test/frontend/TestCrementNode.cc \
     test/frontend/TestDeclareLocalNode.cc \
+    test/frontend/TestExternalVariableNode.cc \
     test/frontend/TestFunctionCallNode.cc \
     test/frontend/TestKeywordNode.cc \
     test/frontend/TestLocalValueNode.cc \
@@ -331,7 +332,6 @@ TEST_SRC_NAMES := \
     test/integration/TestAssign.cc \
     test/integration/TestBinary.cc \
     test/integration/TestCast.cc \
-    test/integration/TestChannelExpressions.cc \
     test/integration/TestDeclare.cc \
     test/integration/TestEditGroups.cc \
     test/integration/TestEmpty.cc \

openvdb_ax/ast/AST.cc

@@ -139,6 +139,11 @@ void Attribute::accept(Visitor& visitor) const
     visitor.visit(*this);
 }
 
+void ExternalVariable::accept(Visitor& visitor) const
+{
+    visitor.visit(*this);
+}
+
 void AttributeValue::accept(Visitor& visitor) const
 {
     mAttribute->accept(visitor);
@@ -356,6 +361,14 @@ Variable* Attribute::accept(Modifier& visitor)
     return nullptr;
 }
 
+Variable* ExternalVariable::accept(Modifier& visitor)
+{
+    if (Variable* node = visitor.visit(*this)) {
+        return node;
+    }
+    return nullptr;
+}
+
 Expression* AttributeValue::accept(Modifier& visitor)
 {
     if (Expression* node = visitor.visit(*this)) {

openvdb_ax/ast/AST.h

@@ -76,6 +76,7 @@ struct Cast;
 struct Variable;
 struct Attribute;
 struct AttributeValue;
+struct ExternalVariable;
 struct DeclareLocal;
 struct Local;
 struct LocalValue;
@@ -415,6 +416,26 @@ struct Attribute : public Variable
     const bool mTypeInferred;
 };
 
+struct ExternalVariable : public Variable
+{
+    using Ptr = std::shared_ptr<ExternalVariable>;
+    using UniquePtr = std::unique_ptr<ExternalVariable>;
+
+    ExternalVariable(const std::string& name, const std::string& type)
+        : Variable(name)
+        , mType(type) {}
+    ExternalVariable(const ExternalVariable& other)
+        : Variable(other.mName)
+        , mType(other.mType) {}
+    ~ExternalVariable() override = default;
+
+    void accept(Visitor& visitor) const override final;
+    Variable* accept(Modifier& visitor) override final;
+    ExternalVariable* copy() const override final { return new ExternalVariable(*this); }
+
+    const Name mType;
+};
+
 struct AttributeValue : public Expression
 {
     using Ptr = std::shared_ptr<AttributeValue>;
@@ -641,6 +662,7 @@ struct Visitor
     inline virtual void visit(const Return& node) {};
     inline virtual void visit(const Attribute& node) {};
     inline virtual void visit(const AttributeValue& node) {};
+    inline virtual void visit(const ExternalVariable& node) {};
     inline virtual void visit(const DeclareLocal& node) {};
     inline virtual void visit(const Local& node) {};
     inline virtual void visit(const LocalValue& node) {};
@@ -675,6 +697,7 @@ struct Modifier
     inline virtual Expression* visit(Return& node) { return nullptr; };
     inline virtual Variable*   visit(Attribute& node) { return nullptr; };
     inline virtual Expression* visit(AttributeValue& node) { return nullptr; };
+    inline virtual Variable*   visit(ExternalVariable& node) { return nullptr; };
     inline virtual Variable*   visit(DeclareLocal& node) { return nullptr; };
     inline virtual Variable*   visit(Local& node) { return nullptr; };
     inline virtual Expression* visit(LocalValue& node) { return nullptr; };

openvdb_ax/ast/PrintTree.cc

@@ -62,6 +62,7 @@ struct PrintVisitor : public ast::Visitor
     void visit(const ast::FunctionCall& node) override;
     void visit(const ast::Attribute& node) override;
     void visit(const ast::AttributeValue& node) override;
+    void visit(const ast::ExternalVariable& node) override;
     void visit(const ast::DeclareLocal& node) override;
     void visit(const ast::Local& node) override;
     void visit(const ast::LocalValue& node) override;
@@ -225,6 +226,12 @@ void PrintVisitor::visit(const ast::VectorPack& node)
     mOs << "VectorPack " << std::endl;
 }
 
+void PrintVisitor::visit(const ast::ExternalVariable& node)
+{
+    printIndent();
+    mOs << "External Variable " << node.mName << std::endl;
+}
+
 template <typename T>
 void PrintVisitor::visitValue(const ast::Value<T>& node)
 {
@@ -245,7 +252,7 @@ void PrintVisitor::printIndent()
 ////////////////////////////////////////////////////////////////////////////////
 
 
-void print(ast::Tree& tree, std::ostream& os)
+void print(const ast::Tree& tree, std::ostream& os)
 {
     PrintVisitor visitor(os);
     tree.accept(visitor);

openvdb_ax/ast/PrintTree.h

@@ -48,7 +48,7 @@ struct Tree;
 /// @brief Writes a debug printout of a Tree object into a target stream
 /// @param tree Tree to print
 /// @param os   Stream to write into
-void print(ast::Tree& tree, std::ostream& os = std::cout);
+void print(const ast::Tree& tree, std::ostream& os = std::cout);
 
 } // namespace ast
 } // namespace ax

openvdb_ax/cmd/openvdb_ax/main.cc

@@ -30,6 +30,7 @@
 
 #include <openvdb_ax/ast/AST.h>
 #include <openvdb_ax/ast/Scanners.h>
+#include <openvdb_ax/ast/PrintTree.h>
 #include <openvdb_ax/codegen/FunctionRegistry.h>
 #include <openvdb_ax/compiler/Compiler.h>
 #include <openvdb_ax/compiler/PointExecutable.h>
@@ -57,6 +58,7 @@ struct ProgOptions
     std::string mInputVDBFile = "";
     std::string mOutputVDBFile = "";
     bool mVerbose = false;
+    bool mPrintAST = false;
 };
 
 void
@@ -70,6 +72,7 @@ usage [[noreturn]] (int exitStatus = EXIT_FAILURE)
 "    -f file.txt       execute text file containing a code snippet on the input.vdb file\n" <<
 "    -v                verbose (print timing and diagnostics)\n" <<
 "    --list-functions  list all available functions, their signatures and their documentation\n" <<
+"    --print-ast       print the abstract syntax tree generated for point and volume execution\n" <<
 "Warning:\n" <<
 "     Providing the same file-path to both input.vdb and output.vdb arguments will overwrite\n" <<
 "     the file. If no output file is provided, the input.vdb will be processed but will remain\n" <<
@@ -205,6 +208,8 @@ main(int argc, char *argv[])
                 initializer.initializeCompiler();
                 printFunctions(std::cout);
                 return EXIT_SUCCESS;
+            } else if (parser.check(i, "--print-ast", 0)) {
+                options.mPrintAST = true;
             } else if (arg == "-h" || arg == "-help" || arg == "--help") {
                 usage(EXIT_SUCCESS);
             } else {
@@ -259,6 +264,14 @@ main(int argc, char *argv[])
     initializer.initializeCompiler();
     openvdb::ax::Compiler::Ptr compiler = openvdb::ax::Compiler::create();
 
+    // parse
+
+    const openvdb::ax::ast::Tree::ConstPtr syntaxTree =
+        openvdb::ax::ast::parse(options.mInputCode.c_str());
+    if (options.mPrintAST) {
+        openvdb::ax::ast::print(*syntaxTree);
+    }
+
     // Execute on PointDataGrids
 
     bool executeOnPoints = false;
@@ -276,9 +289,6 @@ main(int argc, char *argv[])
         openvdb::ax::CustomData::Ptr customData = openvdb::ax::CustomData::create();
         PointExecutable::Ptr pointExecutable;
 
-        const openvdb::ax::ast::Tree::ConstPtr syntaxTree =
-            openvdb::ax::ast::parse(options.mInputCode.c_str());
-
         if (options.mVerbose) std::cout << "OpenVDB PointDataGrids Found" << std::endl;
         std::vector<std::string> warnings;
 
@@ -348,7 +358,7 @@ main(int argc, char *argv[])
         try {
             if (options.mVerbose) std::cout << "  Compiling for Volume VDB Grid...";
             volumeExecutable =
-                compiler->compile<VolumeExecutable>(options.mInputCode, customData, &warnings);
+                compiler->compile<VolumeExecutable>(*syntaxTree, customData, &warnings);
         } catch (std::exception& e) {
             OPENVDB_LOG_FATAL("Compilation error!");
             OPENVDB_LOG_FATAL("Errors:");

openvdb_ax/codegen/ComputeGenerator.cc

@@ -987,6 +987,24 @@ ComputeGenerator::visit(const ast::Value<ValueType>& node)
     mValues.push(store);
 }
 
+void ComputeGenerator::visit(const ast::ExternalVariable& node)
+{
+    const std::string globalName = getGlobalExternalAccess(node.mName, node.mType);
+    llvm::Value* ptrToAddress = this->globals().get(globalName);
+
+    if (!ptrToAddress) {
+        ptrToAddress = llvm::cast<llvm::GlobalVariable>
+            (mModule.getOrInsertGlobal(globalName, LLVMType<uintptr_t>::get(mContext)));
+        this->globals().insert(globalName, ptrToAddress);
+    }
+
+    llvm::Type* type = llvmTypeFromName(node.mType, mContext);
+    llvm::Value* address = mBuilder.CreateLoad(ptrToAddress);
+    llvm::Value* value = mBuilder.CreateIntToPtr(address, type->getPointerTo(0));
+
+    mValues.push(value);
+}
+
 void ComputeGenerator::visit(const ast::Tree& node)
 {
     assert(mBlocks.size() == 1);

openvdb_ax/codegen/ComputeGenerator.h

@@ -127,6 +127,7 @@ struct ComputeGenerator : public ast::Visitor
     void visit(const ast::Cast& node) override;
     void visit(const ast::DeclareLocal& node) override;
     void visit(const ast::Local& node) override;
+    void visit(const ast::ExternalVariable& node) override;
 
     /// @brief VectorUnpack pushes a single llvm::Value onto the value
     /// stack which represents a pointer to an element of a vector

openvdb_ax/codegen/FunctionRegistry.cc

@@ -185,13 +185,13 @@ void insertStandardFunctions(FunctionRegistry& registry)
 
     // indirect internals
 
-    registry.insert("internal_cross", CrossProd::Internal::create, false);
-    registry.insert("internal_normalize", Normalize::Internal::create, false);
-    registry.insert("internal_addtogroup", AddToGroup::Internal::create, false);
-    registry.insert("internal_ingroup", InGroup::Internal::create, false);
-    registry.insert("internal_removefromgroup", RemoveFromGroup::Internal::create, false);
-    registry.insert("internal_lookupf", LookupFloat::Internal::create, false);
-    registry.insert("internal_lookupvec3f", LookupVec3f::Internal::create, false);
+    registry.insert("internal_cross", CrossProd::Internal::create, true);
+    registry.insert("internal_normalize", Normalize::Internal::create, true);
+    registry.insert("internal_addtogroup", AddToGroup::Internal::create, true);
+    registry.insert("internal_ingroup", InGroup::Internal::create, true);
+    registry.insert("internal_removefromgroup", RemoveFromGroup::Internal::create, true);
+    registry.insert("internal_lookupf", LookupFloat::Internal::create, true);
+    registry.insert("internal_lookupvec3f", LookupVec3f::Internal::create, true);
 
     // volume functions