Merge branch 'RB-10.6'

Author: johnhaddon

.github/workflows/main.yml

@@ -33,7 +33,8 @@ jobs:
           linux-gcc11,
           linux-debug-gcc11,
           windows,
-          windows-debug
+          windows-debug,
+          macos-arm64
         ]
 
         include:
@@ -76,6 +77,15 @@ jobs:
             publish: false
             jobs: 4
 
+          - name: macos-arm64
+            os: macos-14
+            buildType: RELEASE
+            options: .github/workflows/main/options.posix
+            dependenciesURL: https://github.com/GafferHQ/dependencies/releases/download/10.0.0/gafferDependencies-10.0.0-macos-arm64.tar.gz
+            tests: testCore testCorePython testScene testImage testAlembic testUSD testVDB
+            publish: true
+            jobs: 3
+
     runs-on: ${{ matrix.os }}
 
     container: ${{ matrix.containerImage }}

.github/workflows/main/options.posix

@@ -63,4 +63,5 @@ ENV_VARS_TO_IMPORT = "PATH CI"
 
 if platform.system() == "Darwin" :
 	os.environ["DYLD_FRAMEWORK_PATH"] = libs
-	ENV_VARS_TO_IMPORT += " DYLD_FRAMEWORK_PATH"
+	os.environ["PYTHONHOME"] = libs + "/Python.framework/Versions/" + pythonABIVersion
+	ENV_VARS_TO_IMPORT += " DYLD_FRAMEWORK_PATH PYTHONHOME"

Changes

@@ -3,10 +3,46 @@
 
 
 
-10.6.x.x (relative to 10.6.0.1)
+10.6.x.x (relative to 10.6.1.0)
 ========
 
+Improvements
+------------
+
+- ShaderNetwork : Optimised applySubstitutions() for cases where not all shaders need them.
+- CurvesPrimitiveEvaluator : Added accessors for getting interpolation coefficients and data indices from Result.
+
+Fixes
+-----
+
+- ShaderNetwork::hashSubstitutions : Fixed shader string substitutions that are escaped with backslashes (ie. `myText\<substitutionHandledByArnold\>`).
+
+10.6.1.0 (relative to 10.6.0.2)
+========
+
+Improvements
+------------
+
+- ShaderNetworkAlgo : Added support for RenderMan's spline convention in `expandSplines()` and `collapseSplines()`.
+
+Fixes
+-----
+
+- ConfigLoader : Fixed UnicodeDecodeErrors in non-UTF8 locales.
+- USDScene : Fixed identifiers for exported RenderMan shaders, by omitting the `ri:` type prefix.
+
+10.6.0.2 (relative to 10.6.0.1)
+========
 
+Fixes
+-----
+
+- DisplayDriverServer : Fixed to support IPv4-only environments.
+
+API
+---
+
+- TypeIds : Claimed range for AtomsGaffer.
 
 10.6.0.1 (relative to 10.6.0.0)
 ========
@@ -53,10 +89,23 @@ Breaking Changes
 - Removed support for `IECORE_RTLD_GLOBAL` environment variable.
 - SmoothSkinningData : Removed, along with all associated Ops and Parameters.
 
-10.5.x.x (relative to 10.5.15.3)
+10.5.x.x (relative to 10.5.15.4)
+=======
+
+
+
+10.5.15.4 (relative to 10.5.15.3)
 ========
 
+Improvements
+-----
+
+- IECoreUSD: Added support for root level tags (reading/writing) to our IECoreUSD::SceneCacheData plugin.
+
+Fixes
+-----
 
+- IECoreUSD: Fixed crash when using invalid file path with IECoreUSD::SceneCacheFileFormat.
 
 10.5.15.3 (relative to 10.5.15.2)
 =========

contrib/IECoreUSD/src/IECoreUSD/SceneCacheData.cpp

@@ -217,7 +217,7 @@ void SceneCacheData::addReference( ConstSceneInterfacePtr scene, SpecData& spec,
 	addValueClip( spec, times, actives, linkFileName, linkRootPath.GetText() );
 }
 
-void SceneCacheData::addInternalRoot( TfTokenVector children )
+void SceneCacheData::addInternalRoot( TfTokenVector children, IECoreScene::ConstSceneInterfacePtr scene )
 {
 	// add transform for internal root.
 	SdfPath internalRootPath = SdfPath::AbsoluteRootPath().AppendChild( SceneCacheDataAlgo::internalRootNameToken() );
@@ -237,6 +237,26 @@ void SceneCacheData::addInternalRoot( TfTokenVector children )
 	// steal root children
 	internalRootSpec.fields.push_back( FieldValuePair( SdfChildrenKeys->PrimChildren, children ) );
 
+	// add collection
+	FieldValuePair propertyChildren;
+	propertyChildren.first = SdfChildrenKeys->PropertyChildren;
+	TfTokenVector properties;
+
+	// we don't want to keep children tags in this case.
+	m_collections.clear();
+
+	SceneInterface::NameList tags;
+	scene->readTags( tags );
+	for ( auto& tag : tags )
+	{
+		m_collections[tag].push_back( internalRootPath );
+	}
+
+	addCollections( internalRootSpec, properties, internalRootPath );
+
+	propertyChildren.second = properties;
+	internalRootSpec.fields.push_back( propertyChildren );
+
 	m_data[internalRootPath] = internalRootSpec;
 }
 
@@ -344,7 +364,7 @@ void SceneCacheData::loadSceneIntoCache( ConstSceneInterfacePtr scene )
 		// end timecode
 		spec.fields.push_back( FieldValuePair( SdfFieldKeys->EndTimeCode, lastFrame ) );
 
-		addInternalRoot( children );
+		addInternalRoot( children, scene );
 
 		// add internal root as single child
 		children.clear();

contrib/IECoreUSD/src/IECoreUSD/SceneCacheData.h

@@ -179,7 +179,7 @@ class SceneCacheData : public SdfAbstractData
 	void addCollections( SpecData& spec, TfTokenVector& properties, const SdfPath& primPath );
 	void addReference( IECoreScene::ConstSceneInterfacePtr scene, SpecData& spec, TfTokenVector& children );
 	void addValueClip( SpecData& spec, const VtVec2dArray times, const VtVec2dArray actives, const std::string& assetPath, const std::string& primPath);
-	void addInternalRoot( TfTokenVector children );
+	void addInternalRoot( TfTokenVector children, IECoreScene::ConstSceneInterfacePtr scene );
 
 	VtValue getTimeSampleMap( const SdfPath& path, const TfToken& field, const VtValue& value ) const;
 

contrib/IECoreUSD/src/IECoreUSD/SceneCacheFileFormat.cpp

@@ -175,6 +175,11 @@ bool UsdSceneCacheFileFormat::WriteToFile( const SdfLayer& layer, const std::str
 	SceneInterfacePtr outScene;
 
 	outScene = SdfFileFormatSharedSceneWriters::get( filePath );
+	if ( !outScene )
+	{
+		IECore::msg( IECore::Msg::Error, "UsdSceneCacheFileFormat::WriteToFile", boost::format( "Invalid file path \"%s\" for layer \"%s\"." ) % filePath % layer.GetIdentifier() );
+		return false;
+	}
 
 	SceneInterface::NameList childNames;
 	usdScene->childNames( childNames );
@@ -370,6 +375,18 @@ void UsdSceneCacheFileFormat::writeLocation(
 			}
 		}
 	}
+	// internal root is mapped to the SceneInterface root '/'
+	else
+	{
+		SceneInterface::NameList tags;
+		inChild->readTags( tags );
+		// round trip internal tag name
+		for ( auto& tag : tags )
+		{
+			tag = SceneCacheDataAlgo::fromInternalName( tag );
+		}
+		outChild->writeTags( tags );
+	}
 
 	// recursion
 	SceneInterface::NameList grandChildNames;

contrib/IECoreUSD/src/IECoreUSD/SdfFileFormatSharedSceneWriters.cpp

@@ -35,6 +35,7 @@
 #include "SdfFileFormatSharedSceneWriters.h"
 
 #include "IECore/LRUCache.h"
+#include "IECore/MessageHandler.h"
 
 using namespace IECore;
 using namespace IECoreScene;
@@ -61,7 +62,15 @@ class Cache : public SceneLRUCache
 
 		static SceneInterfacePtr fileCacheGetter( const std::string &fileName, size_t &cost )
 		{
-			SceneInterfacePtr result = SceneInterface::create( fileName, IECore::IndexedIO::Write );
+			SceneInterfacePtr result = nullptr;
+			try
+			{
+				result = SceneInterface::create( fileName, IECore::IndexedIO::Write );
+			}
+			catch ( ... )
+			{
+				IECore::msg( IECore::Msg::Error, "SdfFileFormatSharedSceneWriters::SceneLRUCache", boost::format( "Unable to open file path \"%s\" for writing IndexedIo data." ) % fileName );
+			}
 			cost = 1;
 			return result;
 		}

contrib/IECoreUSD/src/IECoreUSD/ShaderAlgo.cpp

@@ -309,15 +309,33 @@ pxr::UsdShadeConnectableAPI createShaderPrim( const IECoreScene::Shader *shader,
 	{
 		throw IECore::Exception( "Could not create shader at " + path.GetAsString() );
 	}
+
 	const std::string type = shader->getType();
+
 	std::string typePrefix;
-	size_t typeColonPos = type.find( ":" );
-	if( typeColonPos != std::string::npos )
+	if( boost::starts_with( shader->getName(), "Pxr" ) || boost::starts_with( shader->getName(), "Lama" ) )
 	{
-		typePrefix = type.substr( 0, typeColonPos ) + ":";
-		if( typePrefix == "ai:" )
+		// Leave the type prefix empty. This should be the default, but we are currently only doing this
+		// for a small number of shaders that we can be completely confident require it, in order to
+		// preserve backwards compatibility.
+	}
+	else
+	{
+		size_t typeColonPos = type.find( ":" );
+		if( typeColonPos != std::string::npos )
 		{
-			typePrefix = "arnold:";
+			// According to our current understanding, this is almost completely wrong. Renderer's like
+			// PRMan won't accept shaders with type prefixes, and Arnold apparently requires all shaders
+			// to be prefixed with "arnold:", including OSL. This code prefixes OSL shaders with "osl:",
+			// which fails in all renderers we're aware of - we're keeping this behaviour for now for
+			// backwards compatibility reasons.
+			typePrefix = type.substr( 0, typeColonPos ) + ":";
+
+			// This is the one case that actually works
+			if( typePrefix == "ai:" )
+			{
+				typePrefix = "arnold:";
+			}
 		}
 	}
 	usdShader.SetShaderId( pxr::TfToken( typePrefix + shader->getName() ) );

contrib/IECoreUSD/test/IECoreUSD/SceneCacheFileFormatTest.py

@@ -320,6 +320,7 @@ def testTagsLoadedAsCollections( self ):
 		# includes
 		fileName = os.path.join( self.temporaryDirectory(), "testUSDTags.scc" )
 		m = IECoreScene.SceneCache( fileName, IECore.IndexedIO.OpenMode.Write )
+		m.writeTags( ["geoId:chessy"] )
 		t = m.createChild( "t" )
 		s = t.createChild( "s" )
 		t.writeTags( ["t1", "all", "asset-(12)"] )
@@ -330,6 +331,16 @@ def testTagsLoadedAsCollections( self ):
 		stage = pxr.Usd.Stage.Open( fileName )
 		root = stage.GetPseudoRoot()
 
+		tagInternalRootPrim = root.GetPrimAtPath( f"/{IECoreUSD.SceneCacheDataAlgo.internalRootName()}" )
+		self.assertEqual(
+			tagInternalRootPrim.GetRelationship(
+				"collection:{}:includes".format(
+					IECoreUSD.SceneCacheDataAlgo.toInternalName( "geoId:chessy" )
+				)
+			).GetTargets(),
+			[ pxr.Sdf.Path( f"/{IECoreUSD.SceneCacheDataAlgo.internalRootName()}" ) ]
+		)
+
 		tagPrim = root.GetPrimAtPath( "/{}/t".format( IECoreUSD.SceneCacheDataAlgo.internalRootName() ) )
 		self.assertTrue( tagPrim )
 
@@ -346,6 +357,10 @@ def testTagsLoadedAsCollections( self ):
 		stage.Export( exportPath )
 
 		scene = IECoreScene.SharedSceneInterfaces.get( exportPath )
+		# check root tags
+		self.assertTrue( "geoId:chessy" in scene.readTags() )
+
+		# check children tags
 		for tag, paths in tags.items():
 			for path in paths:
 				child = scene.scene( IECoreScene.SceneInterface.stringToPath( path ) )
@@ -939,6 +954,17 @@ def testSceneWrite( self ):
 		stage.Export( exportPath )
 		self.assertTrue( os.path.exists( exportPath ) )
 
+        # invalid path
+		invalidExportPath = os.path.join( self.temporaryDirectory(), "invalid", "invalid.scc" )
+		with IECore.CapturingMessageHandler() as mh :
+			stage.Export( invalidExportPath )
+
+		self.assertEqual( len( mh.messages ), 2 )
+		self.assertEqual( mh.messages[0].level, IECore.Msg.Level.Error )
+		self.assertEqual( mh.messages[0].context, "SdfFileFormatSharedSceneWriters::SceneLRUCache" )
+		self.assertEqual( mh.messages[1].level, IECore.Msg.Level.Error )
+		self.assertEqual( mh.messages[1].context, "UsdSceneCacheFileFormat::WriteToFile" )
+
 		# root
 		layer = pxr.Sdf.Layer.FindOrOpen( linkFileName )
 

contrib/IECoreUSD/test/IECoreUSD/USDSceneTest.py

@@ -3055,6 +3055,13 @@ def testShaders( self ) :
 		) )
 		manualComponentNetwork.setOutput( IECoreScene.ShaderNetwork.Parameter( "dest", "" ) )
 
+
+		rmanSurface = IECoreScene.Shader( "PxrFoo", "osl:surface", surface.parameters )
+		rmanNetwork = IECoreScene.ShaderNetwork()
+		rmanNetwork.addShader( "out", rmanSurface )
+		rmanNetwork.setOutput( IECoreScene.ShaderNetwork.Parameter( "out", "" ) )
+
+
 		writerRoot = IECoreScene.SceneInterface.create( fileName, IECore.IndexedIO.OpenMode.Write )
 		shaderLocation = writerRoot.createChild( "shaderLocation" )
 		shaderLocation.writeAttribute( "ai:surface", oneShaderNetwork, 0 )
@@ -3063,6 +3070,7 @@ def testShaders( self ) :
 		shaderLocation.writeAttribute( "complex:surface", complexNetwork, 0 )
 		shaderLocation.writeAttribute( "componentConnection:surface", componentConnectionNetwork, 0 )
 		shaderLocation.writeAttribute( "manualComponent:surface", manualComponentNetwork, 0 )
+		shaderLocation.writeAttribute( "ri:surface", rmanNetwork, 0 )
 
 		shaderLocation.writeAttribute( "volume", oneShaderNetwork, 0 ) # USD supports shaders without a prefix
 
@@ -3121,11 +3129,26 @@ def testShaders( self ) :
 		self.assertEqual( textureUsd.GetInput( "filename" ).Get(), "sometexture.tx" )
 
 
+
+
+		riShaderSource = mat.GetOutput( "ri:surface" ).GetConnectedSource()
+		self.assertEqual( riShaderSource[1], "DEFAULT_OUTPUT" )
+		riShaderUsd = pxr.UsdShade.Shader( riShaderSource[0].GetPrim() )
+		self.assertEqual( riShaderUsd.GetShaderId(), "PxrFoo" )
+		self.assertEqual( riShaderUsd.GetInput( "c" ).Get(), "42" )
+		self.assertEqual( riShaderUsd.GetInput( "a" ).Get(), 42.0 )
+		self.assertEqual( riShaderUsd.GetInput( "g" ).Get(), 5 )
+		self.assertEqual( riShaderUsd.GetInput( "g_Knots" ).Get(), pxr.Vt.FloatArray( [0, 0, 10, 20, 20] ) )
+		self.assertEqual( riShaderUsd.GetInput( "g_Colors" ).Get(), pxr.Vt.Vec3fArray(
+			[pxr.Gf.Vec3f( 1 ), pxr.Gf.Vec3f( 1 ), pxr.Gf.Vec3f( 2 ), pxr.Gf.Vec3f( 0 ), pxr.Gf.Vec3f( 0 )]
+		) )
+		self.assertEqual( riShaderUsd.GetInput( "g_Interpolation" ).Get(), 'linear' )
+
 		# Read via SceneInterface, and check that we've round-tripped successfully.
 
 		root = IECoreScene.SceneInterface.create( fileName, IECore.IndexedIO.OpenMode.Read )
 
-		self.assertEqual( set( root.child( "shaderLocation" ).attributeNames() ), set( ['ai:disp_map', 'ai:surface', 'complex:surface', 'volume', 'componentConnection:surface', 'manualComponent:surface' ] ) )
+		self.assertEqual( set( root.child( "shaderLocation" ).attributeNames() ), set( ['ai:disp_map', 'ai:surface', 'complex:surface', 'volume', 'componentConnection:surface', 'manualComponent:surface', "ri:surface" ] ) )
 
 		self.assertEqual( root.child( "shaderLocation" ).readAttribute( "ai:surface", 0 ).outputShader().parameters, oneShaderNetwork.outputShader().parameters )
 		self.assertEqual( root.child( "shaderLocation" ).readAttribute( "ai:surface", 0 ).outputShader(), oneShaderNetwork.outputShader() )
@@ -3150,6 +3173,13 @@ def testShaders( self ) :
 		self.assertEqual( root.child( "shaderLocation" ).readAttribute( "componentConnection:surface", 0 ), componentConnectionNetwork )
 		self.assertEqual( root.child( "shaderLocation" ).readAttribute( "manualComponent:surface", 0 ), manualComponentNetwork )
 
+		rmanSurfaceWithoutTypePrefix = IECoreScene.Shader( "PxrFoo", "surface", surface.parameters )
+		rmanNetworkWithoutTypePrefix = IECoreScene.ShaderNetwork()
+		rmanNetworkWithoutTypePrefix.addShader( "out", rmanSurfaceWithoutTypePrefix )
+		rmanNetworkWithoutTypePrefix.setOutput( IECoreScene.ShaderNetwork.Parameter( "out", "" ) )
+
+		self.assertEqual( root.child( "shaderLocation" ).readAttribute( "ri:surface", 0 ), rmanNetworkWithoutTypePrefix )
+
 	def testManyShaders( self ) :
 
 		# Write shaders