Refactor usage of pkg_resources (#1338)

Author: mrDzurb

ads/common/model_artifact.py

@@ -971,10 +971,11 @@ def install_requirements(self, conflict_strategy=ConflictStrategy.IGNORE):
             IGNORE: Use the installed version in  case of a conflict.
             UPDATE: Force update dependency to the version required by model artifact in case of conflict.
         """
-        import pkg_resources
+        from importlib.metadata import PackageNotFoundError
+        from importlib.metadata import version as get_version
 
-        importlib.reload(pkg_resources)
-        from pkg_resources import DistributionNotFound, VersionConflict
+        from packaging.markers import default_environment
+        from packaging.requirements import Requirement
 
         if self.version.split(".")[0] not in ["0", "1"] and os.path.exists(
             Path(os.path.join(self.artifact_dir), "requirements.txt")
@@ -1001,19 +1002,43 @@ def install_requirements(self, conflict_strategy=ConflictStrategy.IGNORE):
             )
 
         version_conflicts = {}
+        env = default_environment()
         for requirement in requirements:
+            req_line = requirement.strip()
+            if not req_line or req_line.startswith("#"):
+                continue
+            # Skip include or index options lines
+            if req_line.startswith(("-", "--")):
+                continue
             try:
-                pkg_resources.require(requirement)
-            except VersionConflict as vc:
+                req = Requirement(req_line)
+            except Exception:
+                # If the requirement line cannot be parsed, attempt to install it as-is.
+                pip_install(req_line)
+                continue
+
+            # Evaluate environment markers, if any
+            if req.marker and not req.marker.evaluate(environment=env):
+                continue
+
+            package_name = req.name
+            spec = req.specifier  # SpecifierSet
+
+            try:
+                installed_version = get_version(package_name)
+            except PackageNotFoundError:
+                # Not installed; install the requirement as written
+                pip_install(req_line)
+                continue
+
+            if spec and installed_version not in spec:
                 if conflict_strategy == ConflictStrategy.UPDATE:
-                    pip_install("%s%s" % (vc.req.name, vc.req.specifier), "-U")
+                    pip_install(f"{package_name}{spec}", "-U")
                 elif conflict_strategy == ConflictStrategy.IGNORE:
-                    version_conflicts[
-                        "%s==%s" % (vc.dist.key, vc.dist.parsed_version)
-                    ] = "%s%s" % (vc.req.name, vc.req.specifier)
-            except DistributionNotFound:
-                pip_install(requirement)
-                # distributions_not_found.add('%s%s' % (dnf.req.name, dnf.req.specifier))
+                    version_conflicts[f"{package_name}=={installed_version}"] = (
+                        f"{package_name}{spec}"
+                    )
+
         if len(version_conflicts) > 0:
             print(
                 "\033[93m"

ads/common/model_export_util.py

@@ -226,7 +226,8 @@ def prepare_generic_model(
     max_col_num = kwargs.get("max_col_num", utils.DATA_SCHEMA_MAX_COL_NUM)
     artifact_type_generic = progress is not None
 
-    from pkg_resources import DistributionNotFound, get_distribution
+    from importlib.metadata import PackageNotFoundError
+    from importlib.metadata import version as get_version
 
     from ads.common.model import ADSModel
 
@@ -261,26 +262,22 @@ def prepare_generic_model(
             # before we request versions we want to check if fdk installed by user
             # and provide support in error message, if not installed
             try:
-                get_distribution("fdk")
-            except Exception as e:
-                if isinstance(e, DistributionNotFound):
-                    error_message = (
-                        "fdk library not installed in current environment, it is required "
-                        "for deployment with fn. Install fdk with 'pip install fdk'."
-                    )
-                    logger.error(str(error_message))
-                    raise
+                get_version("fdk")
+            except PackageNotFoundError:
+                error_message = (
+                    "fdk library not installed in current environment, it is required "
+                    "for deployment with fn. Install fdk with 'pip install fdk'."
+                )
+                logger.error(str(error_message))
+                raise
             else:
                 required_fn_libs = get_function_config()["requires"]["functions"]
-                [
-                    model_libs.update({lib: get_distribution(lib).version})
-                    for lib in required_fn_libs
-                ]
+                [model_libs.update({lib: get_version(lib)}) for lib in required_fn_libs]
                 required_model_libs = get_function_config()["requires"][
                     kwargs.get("serializer", "default")
                 ]
                 [
-                    model_libs.update({lib: get_distribution(lib).version})
+                    model_libs.update({lib: get_version(lib)})
                     for lib in required_model_libs
                 ]
                 utils.generate_requirement_file(

ads/common/utils.py

@@ -936,7 +936,7 @@ def extract_lib_dependencies_from_model(model) -> dict:
     -------
     Dict: A dictionary of library dependencies.
     """
-    from pkg_resources import get_distribution
+    from importlib.metadata import version as pkg_version
 
     module_versions = {}
     modules_to_include = set(
@@ -948,7 +948,7 @@ def extract_lib_dependencies_from_model(model) -> dict:
         if mod not in module_ignore:
             try:
                 mod_name = lib_translator.get(mod, mod)
-                module_versions[mod_name] = get_distribution(mod_name).version
+                module_versions[mod_name] = pkg_version(mod_name)
             except:
                 pass
     return module_versions

pyproject.toml

@@ -82,6 +82,7 @@ dependencies = [
   "pydantic>=2.6.3",
   "tenacity",
   "httpx",
+  "packaging>=21.0",
 ]
 
 [project.optional-dependencies]