Handle yum not accepting --setopt=varsdir=

bocekm · bocekm · commit 2d069797ed3a · 2025-03-18T21:58:15.000+01:00
The --setopt=varsdir= option was introduced in dnf. For that reason we
can't pass to yum a directory with yum variable files and instead we
need to ensure that all necessary yum vars are defined in /etc/yum/vars.
diff --git a/convert2rhel/actions/pre_ponr_changes/backup_system.py b/convert2rhel/actions/pre_ponr_changes/backup_system.py
@@ -24,8 +24,7 @@
 from convert2rhel.logger import LOG_DIR, root_logger
 from convert2rhel.pkghandler import VERSIONLOCK_FILE_PATH
 from convert2rhel.redhatrelease import os_release_file, system_release_file
-from convert2rhel.repo import DEFAULT_DNF_VARS_DIR, DEFAULT_YUM_REPOFILE_DIR, DEFAULT_YUM_VARS_DIR
-from convert2rhel.systeminfo import system_info
+from convert2rhel.repo import DEFAULT_YUM_REPOFILE_DIR
 from convert2rhel.toolopts import tool_opts
 from convert2rhel.utils import warn_deprecated_env
 from convert2rhel.utils.rpm import PRE_RPM_VA_LOG_FILENAME
@@ -100,40 +99,6 @@ def run(self):
             backup.backup_control.push(restorable_file)
 
 
-class BackupYumVariables(actions.Action):
-    id = "BACKUP_YUM_VARIABLES"
-
-    def run(self):
-        """Backup varsdir folder in /etc/{yum,dnf}/vars so the variables can be restored on rollback."""
-        logger.task("Backup variables")
-
-        super(BackupYumVariables, self).run()
-
-        logger.info("Backing up variables files from {}.".format(DEFAULT_YUM_VARS_DIR))
-        self._backup_variables(path=DEFAULT_YUM_VARS_DIR)
-
-        if system_info.version.major >= 8:
-            logger.info("Backing up variables files from {}.".format(DEFAULT_DNF_VARS_DIR))
-            self._backup_variables(path=DEFAULT_DNF_VARS_DIR)
-
-    def _backup_variables(self, path):
-        """Helper internal function to backup the variables.
-
-        :param path: The path for the original variable.
-        :type path: str
-        """
-        variable_files_backed_up = False
-
-        for variable in os.listdir(path):
-            variable_path = os.path.join(path, variable)
-            restorable_file = RestorableFile(variable_path)
-            backup.backup_control.push(restorable_file)
-            variable_files_backed_up = True
-
-        if not variable_files_backed_up:
-            logger.info("No variables files backed up.")
-
-
 class BackupPackageFiles(actions.Action):
     id = "BACKUP_PACKAGE_FILES"
     # BACKUP_PACKAGE_FILES should be the last one
diff --git a/convert2rhel/actions/pre_ponr_changes/handle_packages.py b/convert2rhel/actions/pre_ponr_changes/handle_packages.py
@@ -77,6 +77,7 @@ class RemoveSpecialPackages(actions.Action):
         "BACKUP_REPOSITORY",
         "BACKUP_PACKAGE_FILES",
         "BACKUP_REDHAT_RELEASE",
+        "BACKUP_YUM_VARIABLES",
     )
 
     def run(self):
diff --git a/convert2rhel/actions/pre_ponr_changes/yum_variables.py b/convert2rhel/actions/pre_ponr_changes/yum_variables.py
@@ -0,0 +1,133 @@
+# Copyright(C) 2025 Red Hat, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+__metaclass__ = type
+
+import os
+import shutil
+
+from convert2rhel import actions
+from convert2rhel import backup
+from convert2rhel.backup.files import InstalledFile, RestorableFile
+from convert2rhel.logger import root_logger
+from convert2rhel.pkghandler import get_files_owned_by_package, get_packages_to_remove
+from convert2rhel.repo import DEFAULT_DNF_VARS_DIR, DEFAULT_YUM_VARS_DIR
+from convert2rhel.systeminfo import system_info
+from convert2rhel.toolopts.config import loggerinst
+
+logger = root_logger.getChild(__name__)
+
+
+class BackUpYumVariables(actions.Action):
+    id = "BACKUP_YUM_VARIABLES"
+    # We don't make a distinction between /etc/yum/vars/ and /etc/yum/vars/ in this Action. Wherever the files are we
+    # back them up.
+    yum_var_dirs = [DEFAULT_DNF_VARS_DIR, DEFAULT_YUM_VARS_DIR]
+
+    def run(self):
+        """Back up yum variable files in /etc/{yum,dnf}/vars/ owned by packages that are known to install these yum
+        variable files (such as system-release). We back them up to be able to restore them right after we remove these
+        packages. We need to restore the variable files because we use repofiles also installed by these packages and
+        yum does not allow specifying a custom directory with yum variable files. This functionality came later with dnf
+        however we apply the same approach to both yum and dnf for the sake of code simplicity.
+        """
+        logger.task("Back up yum variables")
+
+        super(BackUpYumVariables, self).run()
+
+        logger.debug("Getting a list of files owned by packages affecting variables in .repo files.")
+        yum_var_affecting_pkgs = get_packages_to_remove(system_info.repofile_pkgs)
+        yum_var_filepaths = self._get_yum_var_files_owned_by_pkgs(
+            [pkg_obj.nevra.name for pkg_obj in yum_var_affecting_pkgs]
+        )
+
+        self._back_up_var_files(yum_var_filepaths)
+
+    def _get_yum_var_files_owned_by_pkgs(self, pkg_names):
+        """Get paths of yum var files owned by the packages passed to the method."""
+        pkg_owned_files = set()
+        for pkg in pkg_names:
+            pkg_owned_files.union(get_files_owned_by_package(pkg))  # using set() and union() to get unique paths
+
+        # Out of all the files owned by the packages get just those in yum/dnf var dirs
+        yum_var_filepaths = [path for path in pkg_owned_files if os.path.dirname(path) in self.yum_var_dirs]
+
+        return yum_var_filepaths
+
+    def _back_up_var_files(self, paths):
+        """Back up yum variable files.
+
+        :param paths: Paths to the variable files to back up
+        :type paths: list[str]
+        """
+        logger.info("Backing up variables files from {}.".format(" and ".join(self.yum_var_dirs)))
+        if not paths:
+            logger.info("No variables files backed up.")
+
+        for filepath in paths:
+            restorable_file = RestorableFile(filepath)
+            backup.backup_control.push(restorable_file)
+
+
+class RestoreYumVarFiles(actions.Action):
+    id = "RESTORE_YUM_VAR_FILES"
+    dependencies = ("REMOVE_SPECIAL_PACKAGES",)
+
+    def run(self):
+        """Right after removing packages that own yum variable files in the REMOVE_SPECIAL_PACKAGES Action, in this
+        Action we restore these files to /etc/{yum,dnf}/vars/ so that yum can use them when accessing the original
+        vendor repositories (which are backed up in a temporary folder and passed to yum through the --setopt=reposdir=
+        option).
+        The ideal solution would be to use the --setopt=varsdir= option also for the temporary folder where yum variable
+        files are backed up however the option was only introduced in dnf so it's not available in RHEL 7 and its
+        derivatives. For the sake of using just one approach to simplify the codebase, we are restoring the yum variable
+        files no matter the package manager.
+        We use the backup controller to record that we've restored the variable files meaning that upon rollback the
+        files get removed. As part of the rollback we also install beck the packages that include these files so they'll
+        be present.
+        TODO: These restored variable files should not be present after a successful conversion. One option is to
+        enhance the backup controller to indicate that a certain activity should be rolled back not only during a rollback
+        but also after a successful conversion. With such a flag we would add a new post-conversion Action to run the
+        backup controller restoration but only for the activities recorded with this flag.
+        """
+        super(RestoreYumVarFiles, self).run()
+
+        backed_up_yum_var_dirs = backup.get_backed_up_yum_var_dirs()
+        loggerinst.task("Restoring yum variable files")
+        loggerinst.info(
+            "In a previous step we removed a package that might have come with yum variables and in case we"
+            " need to access {} repositories (e.g. when installing dependencies of subscription-manager) we"
+            " need these yum variables available.".format(system_info.name)
+        )
+        for orig_yum_var_dir in backed_up_yum_var_dirs:
+            for backed_up_yum_var_filepath in os.listdir(backed_up_yum_var_dirs[orig_yum_var_dir]):
+                try:
+                    shutil.copy2(backed_up_yum_var_filepath, orig_yum_var_dir)
+                    logger.debug("Copied {} from backup to {}.".format(backed_up_yum_var_filepath, orig_yum_var_dir))
+                except (OSError, IOError) as err:
+                    # IOError for py2 and OSError for py3
+                    # Not being able to restore the yum variables might or might not cause problems down the road. No
+                    # need to stop the conversion because of that. The warning message below should be enough of a clue
+                    # for resolving subsequent yum errors.
+                    logger.warning(
+                        "Couldn't copy {} to {}. Error: {}".format(
+                            backed_up_yum_var_filepath, orig_yum_var_dir, err.strerror
+                        )
+                    )
+                    return
+                restored_file = InstalledFile(
+                    os.path.join(orig_yum_var_dir, os.path.basename(backed_up_yum_var_filepath))
+                )
+                backup.backup_control.push(restored_file)
diff --git a/convert2rhel/backup/__init__.py b/convert2rhel/backup/__init__.py
@@ -24,7 +24,7 @@
 import six
 
 from convert2rhel.logger import root_logger
-from convert2rhel.repo import DEFAULT_YUM_REPOFILE_DIR
+from convert2rhel.repo import DEFAULT_YUM_REPOFILE_DIR, DEFAULT_YUM_VARS_DIR, DEFAULT_DNF_VARS_DIR
 from convert2rhel.utils import TMP_DIR
 
 
@@ -43,6 +43,19 @@ def get_backedup_system_repos():
     return backedup_reposdir
 
 
+def get_backed_up_yum_var_dirs():
+    """Get folders where we've backed up yum and dnf variables inside our backup structure.
+
+    :returns dict: Keys are the original dir paths, values are paths to the dir with backed up yum/dnf variable files
+    """
+
+    yum_var_dirs = {
+        DEFAULT_YUM_VARS_DIR: hashlib.md5(DEFAULT_YUM_VARS_DIR.encode()).hexdigest(),
+        DEFAULT_DNF_VARS_DIR: hashlib.md5(DEFAULT_DNF_VARS_DIR.encode()).hexdigest(),
+    }
+    return yum_var_dirs
+
+
 class BackupController:
     """
     Controls backup and restore for all restorable types.
diff --git a/convert2rhel/backup/certs.py b/convert2rhel/backup/certs.py
@@ -118,8 +118,9 @@ def enable(self):
             try:
                 files.mkdir_p(self._target_cert_dir)
                 shutil.copy2(self._source_cert_path, self._target_cert_dir)
-            except OSError as err:
-                logger.critical_no_exit("OSError({0}): {1}".format(err.errno, err.strerror))
+            except (OSError, IOError) as err:
+                # IOError for py2 and OSError for py3
+                logger.critical_no_exit("Error({0}): {1}".format(err.errno, err.strerror))
                 raise exceptions.CriticalError(
                     id_="FAILED_TO_INSTALL_CERTIFICATE",
                     title="Failed to install certificate.",
diff --git a/convert2rhel/backup/files.py b/convert2rhel/backup/files.py
@@ -208,3 +208,44 @@ def restore(self):
             logger.info("File {filepath} removed".format(filepath=self.filepath))
 
             super(MissingFile, self).restore()
+
+
+class InstalledFile(RestorableChange):
+    """
+    A file we plant on the system during the conversion. It can either be removed on a rollback or after a successful
+    conversion, depending on what purpose the planted file serves.
+    """
+
+    def __init__(self, filepath):
+        super(InstalledFile, self).__init__()
+        self.filepath = filepath
+
+    def enable(self):
+        if self.enabled:
+            return
+
+        logger.info("Marking file {filepath} as installed on the system.".format(filepath=self.filepath))
+        super(InstalledFile, self).enable()
+
+    def restore(self):
+        """Remove the file if it was installed during the conversion.
+
+        .. warning::
+            Exceptions are not handled and left for handling by the calling code.
+
+        :raises OSError: When the removal of the file fails.
+        :raises IOError: When the removal of the file fails.
+        """
+        if not self.enabled:
+            return
+
+        logger.task("Remove {filepath} installed during the conversion".format(filepath=self.filepath))
+
+        if not os.path.isfile(self.filepath):
+            logger.info("File {filepath} wasn't installed during conversion.".format(filepath=self.filepath))
+        else:
+            # Possible exceptions will be handled in the BackupController
+            os.remove(self.filepath)
+            logger.info("File {filepath} removed.".format(filepath=self.filepath))
+
+            super(InstalledFile, self).restore()
diff --git a/convert2rhel/data/7/x86_64/configs/amazon-2-x86_64.cfg b/convert2rhel/data/7/x86_64/configs/amazon-2-x86_64.cfg
@@ -1,6 +1,6 @@
 [system_info]
 
-# The GPG Key IDs used for signing packages of the source OS. They are available at https://cdn.amazonlinux.com/_assets/11CF1F95C87F5B1A.asc
+# The GPG Key ID used for signing packages of the source OS. It is available at https://cdn.amazonlinux.com/_assets/11CF1F95C87F5B1A.asc.
 # Delimited by whitespace(s).
 gpg_key_ids = 11cf1f95c87f5b1a
 
@@ -53,4 +53,3 @@ releasever=
 # Some kernel modules move from kernel modules to kernel core. Instead of inhibiting the conversion with a message
 # that such a module is not available in RHEL and thus is unsupported, we ignore it.
 kmods_to_ignore =
-  kernel/drivers/input/ff-memless.ko.xz
diff --git a/convert2rhel/pkghandler.py b/convert2rhel/pkghandler.py
@@ -292,6 +292,16 @@ def get_installed_pkgs_w_different_key_id(key_ids, name="*"):
     return [pkg for pkg in pkgs_w_key_ids if pkg.key_id not in key_ids and pkg.nevra.name != "gpg-pubkey"]
 
 
+@utils.run_as_child_process
+def get_files_owned_by_package(installed_pkg_name):
+    """Get a list of files that are owned by an installed package."""
+    output, ret_code = utils.run_subprocess(["/usr/bin/rpm", "-ql", installed_pkg_name])
+    if ret_code != 0:
+        logger.warning("Failed to list files for package {0}: {1}".format(installed_pkg_name, output))
+        return []
+    return output.decode("utf-8").splitlines()
+
+
 @utils.run_as_child_process
 def format_pkg_info(pkgs, disable_repos=None):
     """Format package information.
diff --git a/convert2rhel/pkgmanager/handlers/yum/__init__.py b/convert2rhel/pkgmanager/handlers/yum/__init__.py
@@ -17,8 +17,6 @@
 
 __metaclass__ = type
 
-import hashlib
-import os
 import re
 
 from convert2rhel import backup, exceptions, pkgmanager, utils
@@ -27,7 +25,6 @@
 from convert2rhel.pkghandler import get_system_packages_for_replacement
 from convert2rhel.pkgmanager.handlers.base import TransactionHandlerBase
 from convert2rhel.pkgmanager.handlers.yum.callback import PackageDownloadCallback, TransactionDisplayCallback
-from convert2rhel.repo import DEFAULT_YUM_VARS_DIR
 from convert2rhel.systeminfo import system_info
 from convert2rhel.utils import remove_pkgs
 
@@ -74,15 +71,13 @@ def _resolve_yum_problematic_dependencies(output):
             "\n".join(packages_to_remove),
         )
         backedup_reposdir = backup.get_backedup_system_repos()
-        backedup_yum_varsdir = os.path.join(backup.BACKUP_DIR, hashlib.md5(DEFAULT_YUM_VARS_DIR.encode()).hexdigest())
 
         backup.backup_control.push(
             RestorablePackage(
                 pkgs=packages_to_remove,
                 reposdir=backedup_reposdir,
                 set_releasever=True,
                 custom_releasever=system_info.version.major,
-                varsdir=backedup_yum_varsdir,
             )
         )
         remove_pkgs(pkgs_to_remove=packages_to_remove, critical=True)
diff --git a/convert2rhel/subscription.py b/convert2rhel/subscription.py
@@ -17,7 +17,6 @@
 
 __metaclass__ = type
 
-import hashlib
 import json
 import os
 import re
@@ -33,7 +32,6 @@
 from convert2rhel.backup.packages import RestorablePackageSet
 from convert2rhel.logger import root_logger
 from convert2rhel.redhatrelease import os_release_file
-from convert2rhel.repo import DEFAULT_DNF_VARS_DIR, DEFAULT_YUM_VARS_DIR
 from convert2rhel.systeminfo import system_info
 from convert2rhel.toolopts import tool_opts
 from convert2rhel.utils.subscription import _should_subscribe
@@ -610,8 +608,6 @@ def install_rhel_subscription_manager(pkgs_to_install):
         shortcoming by installing the certificate.
     """
     backedup_reposdir = backup.get_backedup_system_repos()
-    varsdir = DEFAULT_YUM_VARS_DIR if system_info.version.major == 7 else DEFAULT_DNF_VARS_DIR
-    backedup_varsdir = os.path.join(backup.BACKUP_DIR, hashlib.md5(varsdir.encode()).hexdigest())
 
     setopts = []
     # Oracle Linux 7 needs to set the exclude option to avoid installing the
@@ -632,7 +628,6 @@ def install_rhel_subscription_manager(pkgs_to_install):
 
     reposdir = [os.path.dirname(client_tools_repofile_path), backedup_reposdir]
     setopts.append("reposdir={}".format(",".join(reposdir)))
-    setopts.append("varsdir={}".format(backedup_varsdir))
     installed_pkg_set = RestorablePackageSet(
         pkgs_to_install,
         custom_releasever=system_info.version.major,
diff --git a/convert2rhel/unit_tests/actions/pre_ponr_changes/backup_system_test.py b/convert2rhel/unit_tests/actions/pre_ponr_changes/backup_system_test.py
diff --git a/convert2rhel/unit_tests/actions/pre_ponr_changes/handle_packages_test.py b/convert2rhel/unit_tests/actions/pre_ponr_changes/handle_packages_test.py
diff --git a/convert2rhel/unit_tests/backup/certs_test.py b/convert2rhel/unit_tests/backup/certs_test.py
diff --git a/convert2rhel/unit_tests/pkgmanager/handlers/yum/yum_test.py b/convert2rhel/unit_tests/pkgmanager/handlers/yum/yum_test.py
diff --git a/tests/integration/tier0/non-destructive/single-yum-transaction-validation/test_single_yum_transaction_validation.py b/tests/integration/tier0/non-destructive/single-yum-transaction-validation/test_single_yum_transaction_validation.py

Original file line number	Diff line number	Diff line change
`@@ -77,6 +77,7 @@ class RemoveSpecialPackages(actions.Action):`
`77`	`77`	`"BACKUP_REPOSITORY",`
`78`	`78`	`"BACKUP_PACKAGE_FILES",`
`79`	`79`	`"BACKUP_REDHAT_RELEASE",`
	`80`	`+ "BACKUP_YUM_VARIABLES",`
`80`	`81`	`)`
`81`	`82`
`82`	`83`	`def run(self):`