Merge pull request #2800 from alicevision/dev/specific_rez_package_version_in_plugin

[core/submitters] Freeze rez plugins dependencies versions in subprocess and farm submitter

Author: fabiencastan

meshroom/core/plugins.py

@@ -151,6 +151,9 @@ def __init__(self, folder: str, configEnv: dict[str: str]):
 class RezProcessEnv(ProcessEnv):
     """
     """
+    
+    REZ_DELIMITER_PATTERN = re.compile(r"-|==|>=|>|<=|<")
+    
     def __init__(self, folder: str, configEnv: dict[str: str], uri: str = ""):
         if not uri:
             raise RuntimeError("Missing name of the Rez environment needs to be provided.")
@@ -164,34 +167,39 @@ def resolveRezSubrequires(self) -> list[str]:
         environment (if this package is defined).
         """
         subrequires = os.environ.get(f"{self.uri.upper()}_SUBREQUIRES", "").split(os.pathsep)
-        packages = []
+        if not subrequires:
+            return []
 
+        packages = []
         # Packages that are resolved in the current environment
         currentEnvPackages = []
-        if "REZ_REQUEST" in os.environ:
-            resolvedPackages = os.getenv("REZ_RESOLVE", "").split()
-            resolvedVersions = {}
+        resolvedVersions = {}
+        if "REZ_USED_RESOLVE" in os.environ:
+            resolvedPackages = os.getenv("REZ_USED_RESOLVE", "").split()
             for package in resolvedPackages:
                 if package.startswith("~"):
                     continue
-                version = package.split("-")
-                resolvedVersions[version[0]] = version[1]
-            currentEnvPackages = [package + "-" + resolvedVersions[package] for package in resolvedVersions.keys()]
+                currentEnvPackages.append(package)
+                name, version = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1)
+                resolvedVersions[name] = version
         logging.debug("Packages in the current environment: " + ", ".join(currentEnvPackages))
 
         # Take packages with the set versions for those which have one, and try to take packages in the current
         # environment (if they are resolved in it)
         for package in subrequires:
-            if "-" in package:
-                packages.append(package)
-            else:
-                definedInParentEnv = False
-                for p in currentEnvPackages:
-                    if p.startswith(package + "-"):
-                        packages.append(p)
-                        definedInParentEnv = True
-                        break
-                if not definedInParentEnv:
+            packageTuple = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1)
+            match len(packageTuple):
+                case 1:
+                    # Only the package name in the subrequires.
+                    # Search for a corresponding version in the parent environment.
+                    packageName = packageTuple[0]
+                    parentResolvedVersion = resolvedVersions.get(packageName)
+                    if parentResolvedVersion:
+                        packages.append(f"{packageName}=={parentResolvedVersion}")
+                    else:
+                        packages.append(package)
+                case 2:
+                    # The subrequires ask for a specific version
                     packages.append(package)
 
         logging.debug("Packages for the execution environment: " + ", ".join(packages))

meshroom/submitters/simpleFarmSubmitter.py

@@ -4,6 +4,7 @@
 import json
 import logging
 import getpass
+import re
 
 import simpleFarm
 from meshroom.core.desc import Level
@@ -21,30 +22,35 @@ class SimpleFarmSubmitter(BaseSubmitter):
     environment = {}
     ENGINE = ''
     DEFAULT_TAGS = {'prod': ''}
+    REZ_DELIMITER_PATTERN = re.compile(r"-|==|>=|>|<=|<")
 
     def __init__(self, parent=None):
         super().__init__(name='SimpleFarm', parent=parent)
         self.engine = os.environ.get('MESHROOM_SIMPLEFARM_ENGINE', 'tractor')
         self.share = os.environ.get('MESHROOM_SIMPLEFARM_SHARE', 'vfx')
         self.prod = os.environ.get('PROD', 'mvg')
-        if 'REZ_REQUEST' in os.environ:
-            packages = os.environ.get('REZ_REQUEST', '').split()
+        if 'REZ_USED_REQUEST' in os.environ:
+            requestPackages = os.environ.get('REZ_USED_REQUEST', '').split()
             resolvedPackages = os.environ.get('REZ_RESOLVE', '').split()
             resolvedVersions = {}
             for r in resolvedPackages:
                 # remove implicit packages
                 if r.startswith('~'):
                     continue
                 # logging.info('REZ: {}'.format(str(r)))
-                v = r.split('-')
+                name, version = self.REZ_DELIMITER_PATTERN.split(r, maxsplit=1)
                 # logging.info('    v: {}'.format(str(v)))
-                if len(v) >= 2:
-                    resolvedVersions[v[0]] = v[1]
-            for p in packages:
+                resolvedVersions[name] = version
+            requestPackageNames = set()  # Use set to remove duplicates
+            for p in requestPackages:
                 if p.startswith('~'):
                     continue
-                v = p.split('-')
-                self.reqPackages.append('-'.join([v[0], resolvedVersions[v[0]]]))
+                v = self.REZ_DELIMITER_PATTERN.split(p, maxsplit=1)
+                requestPackageNames.add(v[0])
+            for p in requestPackageNames:
+                # Use "==" to guarantee that the job uses the exact same version
+                # as the environment where Meshroom was launched.
+                self.reqPackages.append(f"{p}=={resolvedVersions[p]}")
             logging.debug(f'REZ Packages: {str(self.reqPackages)}')
         elif 'REZ_MESHROOM_VERSION' in os.environ:
             self.reqPackages = [f"meshroom-{os.environ.get('REZ_MESHROOM_VERSION', '')}"]