[hdSt] Early parallel MaterialX codegen, launched by a scene index

Author: ppenenko

pxr/imaging/hd/sceneIndexAdapterSceneDelegate.cpp

@@ -1308,29 +1308,37 @@ _ToMaterialNetworkMap(
 }
 
 VtValue
-HdSceneIndexAdapterSceneDelegate::GetMaterialResource(SdfPath const & id)
+HdSceneIndexAdapterSceneDelegate::GetMaterialResourceFromSceneIndexPrim(
+    HdSceneIndexPrim& prim, const TfTokenVector& renderContexts)
 {
-    TRACE_FUNCTION();
-    HF_MALLOC_TAG_FUNCTION();
-    HdSceneIndexPrim prim = _GetInputPrim(id);
-
     HdMaterialSchema matSchema = HdMaterialSchema::GetFromParent(
             prim.dataSource);
     if (!matSchema.IsDefined()) {
         return VtValue();
     }
 
     // Query for a material network to match the requested render contexts
-    const TfTokenVector renderContexts =
-        GetRenderIndex().GetRenderDelegate()->GetMaterialRenderContexts();
-    HdMaterialNetworkSchema netSchema = matSchema.GetMaterialNetwork(renderContexts);
+    HdMaterialNetworkSchema netSchema =
+        matSchema.GetMaterialNetwork(renderContexts);
     if (!netSchema.IsDefined()) {
         return VtValue();
     }
 
     return VtValue(_ToMaterialNetworkMap(netSchema, renderContexts));
 }
 
+VtValue
+HdSceneIndexAdapterSceneDelegate::GetMaterialResource(SdfPath const & id)
+{
+    TRACE_FUNCTION();
+    HF_MALLOC_TAG_FUNCTION();
+    
+    HdSceneIndexPrim prim = _GetInputPrim(id);
+    return GetMaterialResourceFromSceneIndexPrim(
+        prim,
+        GetRenderIndex().GetRenderDelegate()->GetMaterialRenderContexts());
+}
+
 static
 TfTokenVector
 _ToTokenVector(const std::vector<std::string> &strings)

pxr/imaging/hd/sceneIndexAdapterSceneDelegate.h

@@ -133,6 +133,9 @@ class HdSceneIndexAdapterSceneDelegate
     // ------------------------------------------------------------------------
     // Material API
 
+    static HD_API VtValue GetMaterialResourceFromSceneIndexPrim(
+        HdSceneIndexPrim& prim, const TfTokenVector& renderContexts);
+
     SdfPath GetMaterialId(SdfPath const &id) override;
     VtValue GetMaterialResource(SdfPath const &id) override;
     HdIdVectorSharedPtr GetCoordSysBindings(SdfPath const &id) override;

pxr/imaging/hdSt/CMakeLists.txt

@@ -26,6 +26,7 @@ if (${PXR_ENABLE_MATERIALX_SUPPORT})
     list(APPEND optionalPrivateClasses
          materialXFilter
          materialXShaderGen
+         materialXSyncSceneIndex
     )
 endif()
 if (PXR_ENABLE_PTEX_SUPPORT)

pxr/imaging/hdSt/material.cpp

@@ -19,6 +19,11 @@
 #include "pxr/imaging/hdSt/tokens.h"
 #include "pxr/imaging/hdSt/materialParam.h"
 
+#ifdef PXR_MATERIALX_SUPPORT_ENABLED
+#include "pxr/imaging/hdSt/materialXSyncSceneIndex.h"
+#include "pxr/imaging/hdSt/renderDelegate.h"
+#endif
+
 #include "pxr/imaging/hd/changeTracker.h"
 #include "pxr/imaging/hd/tokens.h"
 
@@ -172,8 +177,8 @@ HdStMaterial::_ProcessTextureDescriptors(
 /* virtual */
 void
 HdStMaterial::Sync(HdSceneDelegate *sceneDelegate,
-                 HdRenderParam   *renderParam,
-                 HdDirtyBits     *dirtyBits)
+                   HdRenderParam   *renderParam,
+                   HdDirtyBits     *dirtyBits)
 {
     HD_TRACE_FUNCTION();
     HF_MALLOC_TAG_FUNCTION();
@@ -191,6 +196,19 @@ HdStMaterial::Sync(HdSceneDelegate *sceneDelegate,
 
     bool markBatchesDirty = false;
 
+#ifdef PXR_MATERIALX_SUPPORT_ENABLED
+    {
+        HdStRenderDelegate* stormDelegate = static_cast<HdStRenderDelegate*>(
+            sceneDelegate->GetRenderIndex().GetRenderDelegate());
+
+        if (HdSt_MaterialXSyncSceneIndex* sceneIndex =
+            stormDelegate->GetMaterialXSyncSceneIndex()) {
+
+            sceneIndex->Wait();
+        }
+    }
+#endif
+
     std::string fragmentSource;
     std::string displacementSource;
     std::string volumeSource;
@@ -201,18 +219,19 @@ HdStMaterial::Sync(HdSceneDelegate *sceneDelegate,
 
     VtValue vtMat = sceneDelegate->GetMaterialResource(GetId());
     if (vtMat.IsHolding<HdMaterialNetworkMap>()) {
+
         HdMaterialNetworkMap const& hdNetworkMap =
             vtMat.UncheckedGet<HdMaterialNetworkMap>();
         if (!hdNetworkMap.terminals.empty() && !hdNetworkMap.map.empty()) {
-            _networkProcessor.ProcessMaterialNetwork(GetId(), hdNetworkMap,
+            _hdStMaterialNetwork.ProcessMaterialNetwork(GetId(), hdNetworkMap,
                                                     resourceRegistry.get());
-            fragmentSource = _networkProcessor.GetFragmentCode();
-            volumeSource = _networkProcessor.GetVolumeCode();
-            displacementSource = _networkProcessor.GetDisplacementCode();
-            materialMetadata = _networkProcessor.GetMetadata();
-            materialTag = _networkProcessor.GetMaterialTag();
-            params = _networkProcessor.GetMaterialParams();
-                textureDescriptors = _networkProcessor.GetTextureDescriptors();
+            fragmentSource = _hdStMaterialNetwork.GetFragmentCode();
+            volumeSource = _hdStMaterialNetwork.GetVolumeCode();
+            displacementSource = _hdStMaterialNetwork.GetDisplacementCode();
+            materialMetadata = _hdStMaterialNetwork.GetMetadata();
+            materialTag = _hdStMaterialNetwork.GetMaterialTag();
+            params = _hdStMaterialNetwork.GetMaterialParams();
+                textureDescriptors = _hdStMaterialNetwork.GetTextureDescriptors();
         }
     }
 

pxr/imaging/hdSt/material.h

@@ -8,9 +8,11 @@
 #define PXR_IMAGING_HD_ST_MATERIAL_H
 
 #include "pxr/pxr.h"
+
 #include "pxr/imaging/hdSt/api.h"
 #include "pxr/imaging/hdSt/materialNetwork.h"
 #include "pxr/imaging/hdSt/shaderCode.h"
+
 #include "pxr/imaging/hd/material.h"
 #include "pxr/imaging/hf/perfLog.h"
 
@@ -123,7 +125,7 @@ class HdStMaterial final: public HdMaterial
     TfToken _materialTag;
     size_t _textureHash;
 
-    HdStMaterialNetwork _networkProcessor;
+    HdStMaterialNetwork _hdStMaterialNetwork;
 };
 
 inline bool HdStMaterial::HasPtex() const

pxr/imaging/hdSt/materialNetwork.cpp

@@ -159,8 +159,8 @@ _GetGlslfxForTerminal(
     }
 }
 
-static HdMaterialNode2 const*
-_GetTerminalNode(
+HdMaterialNode2 const*
+HdSt_GetTerminalNode(
     HdMaterialNetwork2 const& network,
     TfToken const& terminalName,
     SdfPath * terminalNodePath)
@@ -1100,57 +1100,85 @@ HdStMaterialNetwork::ProcessMaterialNetwork(
 {
     HD_TRACE_FUNCTION();
 
-    _fragmentSource.clear();
-    _displacementSource.clear();
-    _materialMetadata.clear();
-    _materialParams.clear();
-    _textureDescriptors.clear();
-    _materialTag = HdStMaterialTagTokens->defaultMaterialTag;
+    bool isVolume = false;
+
+    auto filterTask = std::make_shared<HdSt_MaterialFilterTask>();
+    filterTask->hdNetwork =
+        HdConvertToHdMaterialNetwork2(hdNetworkMap, &isVolume);
 
     // The fragment source comes from the 'surface' network or the
     // 'volume' network.
-    bool isVolume = false;
     HdMaterialNetwork2 surfaceNetwork =
         HdConvertToHdMaterialNetwork2(hdNetworkMap, &isVolume);
     const TfToken &terminalName = (isVolume) ? HdMaterialTerminalTokens->volume 
                                             : HdMaterialTerminalTokens->surface;
 
-    SdfPath surfTerminalPath;
-    if (HdMaterialNode2 const* surfTerminal = 
-            _GetTerminalNode(surfaceNetwork, terminalName, &surfTerminalPath)) {
+    filterTask->terminalNode = 
+        HdSt_GetTerminalNode(
+            filterTask->hdNetwork,
+            terminalName,
+            &filterTask->terminalNodePath);
+
+    if (!filterTask->terminalNode) {
+        return;
+    }
+
+    ProcessFilterTask(materialId, filterTask, isVolume, resourceRegistry);
+}
+
+void
+HdStMaterialNetwork::ProcessFilterTask(
+    SdfPath const& materialId,
+    HdSt_MaterialFilterTaskSharedPtr filterTask,
+    bool isVolume,
+    HdStResourceRegistry *resourceRegistry)
+{
+    HD_TRACE_FUNCTION();
+
+    _fragmentSource.clear();
+    _displacementSource.clear();
+    _materialMetadata.clear();
+    _materialParams.clear();
+    _textureDescriptors.clear();
+
+    _materialTag = HdStMaterialTagTokens->defaultMaterialTag;
+
+    if (!filterTask || !filterTask->terminalNode) {
+        return;
+    }
 
 #ifdef PXR_MATERIALX_SUPPORT_ENABLED
-        if (!isVolume) {
-            _materialXGfx = HdSt_ApplyMaterialXFilter(&surfaceNetwork, materialId,
-                                      *surfTerminal, surfTerminalPath,
-                                      &_materialParams, resourceRegistry);
-        }
+    if (!isVolume) {
+        _materialXGfx = HdSt_ApplyMaterialXFilter(filterTask, materialId,
+                                    &_materialParams, resourceRegistry);
+    }
 #endif
-        // Extract the glslfx and metadata for surface/volume.
-        _GetGlslfxForTerminal(_surfaceGfx, &_surfaceGfxHash,
-                              surfTerminal->nodeTypeId, resourceRegistry);
-        if (_surfaceGfx) {
-
-            // If the glslfx file is not valid we skip parsing the network.
-            // This produces no fragmentSource which means Storm's material
-            // will use the fallback shader.
-            if (_surfaceGfx->IsValid()) {
-                
-                _fragmentSource = _surfaceGfx->GetSurfaceSource();
-                _volumeSource = _surfaceGfx->GetVolumeSource();
-
-                _materialMetadata = _surfaceGfx->GetMetadata();
-                _materialTag = _GetMaterialTag(_materialMetadata, *surfTerminal);
-                _GatherMaterialParams(surfaceNetwork, *surfTerminal,
-                                      &_materialParams, &_textureDescriptors, 
-                                      _materialTag);
-
-                // OSL networks have a displacement network in hdNetworkMap
-                // under terminal: HdMaterialTerminalTokens->displacement.
-                // For Storm however we expect the displacement shader to be
-                // provided via the surface glslfx / terminal.
-                _displacementSource = _surfaceGfx->GetDisplacementSource();
-            }
+    // Extract the glslfx and metadata for surface/volume.
+    _GetGlslfxForTerminal(_surfaceGfx, &_surfaceGfxHash,
+                    filterTask->terminalNode->nodeTypeId, resourceRegistry);
+    if (_surfaceGfx) {
+
+        // If the glslfx file is not valid we skip parsing the network.
+        // This produces no fragmentSource which means Storm's material
+        // will use the fallback shader.
+        if (_surfaceGfx->IsValid()) {
+            _fragmentSource = _surfaceGfx->GetSurfaceSource();
+            _volumeSource = _surfaceGfx->GetVolumeSource();
+
+            _materialMetadata = _surfaceGfx->GetMetadata();
+
+            _materialTag = _GetMaterialTag(
+                _surfaceGfx->GetMetadata(), *filterTask->terminalNode);
+            _GatherMaterialParams(
+                filterTask->hdNetwork, *filterTask->terminalNode,
+                &_materialParams, &_textureDescriptors, 
+                _materialTag);
+
+            // OSL networks have a displacement network in hdNetworkMap
+            // under terminal: HdMaterialTerminalTokens->displacement.
+            // For Storm however we expect the displacement shader to be
+            // provided via the surface glslfx / terminal.
+            _displacementSource = _surfaceGfx->GetDisplacementSource();
         }
     }
 }

pxr/imaging/hdSt/materialNetwork.h

@@ -20,9 +20,57 @@
 
 PXR_NAMESPACE_OPEN_SCOPE
 
+class SdrRegistry;
 class HdStResourceRegistry;
 using HioGlslfxSharedPtr = std::shared_ptr<class HioGlslfx>;
 using HdSt_MaterialParamVector = std::vector<class HdSt_MaterialParam>;
+struct HdSt_MaterialFilterTask;
+using HdSt_MaterialFilterTaskSharedPtr =
+    std::shared_ptr<HdSt_MaterialFilterTask>;
+
+extern HdMaterialNode2 const*
+HdSt_GetTerminalNode(
+    HdMaterialNetwork2 const& network,
+    TfToken const& terminalName,
+    SdfPath * terminalNodePath);
+
+/// Encapsulates the input data for MaterialX codegen as well as metadata
+/// necessary for completing `HdStMaterial::Sync` based on the result of the
+/// codegen.
+/// This object can either live on the stack, if MaterialX codegen happens
+/// synchronously, or on the heap, owned by the respective Sprim, if the
+/// codegen happens in parallel tasks.
+///
+struct ARCH_EXPORT_TYPE HdSt_MaterialFilterTask final
+{
+    HdMaterialNetwork2 hdNetwork;
+    HdMaterialNode2 const* terminalNode = nullptr;  // pointer to a node
+                                                    // in the above network
+    SdfPath terminalNodePath;   // path to the above node
+
+#ifdef PXR_MATERIALX_SUPPORT_ENABLED
+    // Stores the mappings between the node paths in the original
+    // HdMaterialNetwork to the corresponding anonymized node paths
+    using OrigToAnonSdfPathMap =
+        std::unordered_map<SdfPath, SdfPath, SdfPath::Hash>;
+    OrigToAnonSdfPathMap origToAnonSdfPathMap;
+
+    /// Build the `anonNetwork`, equivalent to the given hdNetwork but anonymized
+    /// and stripped of non-topological parameters to better re-use the generated
+    /// shader.
+    /// Returns the hash of the anonymized network.
+    size_t BuildAnonymizedMaterialNetwork(
+        HdMaterialNetwork2* anonNetwork);
+
+    void AddFallbackDomeLightTextureNode();
+
+    void AddMaterialXParams(
+        MaterialX::Shader const& mxShader,
+        HdSt_MaterialParamVector* materialParams);
+
+    bool IsMaterialX(SdrRegistry* sdrRegistry) const;
+#endif
+};
 
 /// \class HdStMaterialNetwork
 ///
@@ -37,6 +85,14 @@ class HdStMaterialNetwork final
     HDST_API
     ~HdStMaterialNetwork();
 
+    /// Process the necessary network information cached in the filter task.
+    HDST_API
+    void ProcessFilterTask(
+        SdfPath const& materialId,
+        HdSt_MaterialFilterTaskSharedPtr filterTask,
+        bool isVolume,
+        HdStResourceRegistry *resourceRegistry);
+
     /// Process a material network topology and extract all the information we
     /// need from it.
     HDST_API
@@ -67,7 +123,7 @@ class HdStMaterialNetwork final
     struct TextureDescriptor
     {
         // Name by which the texture will be accessed, i.e., the name
-        // of the accesor for thexture will be HdGet_name(...).
+        // of the accessor for the texture will be HdGet_name(...).
         // It is generated from the input name the corresponding texture
         // node is connected to.
         TfToken name;