Copy files with same destination in single rsync

Author: martinhoyer

tmt/steps/execute/__init__.py

@@ -133,8 +133,11 @@ class ScriptTemplate(Script):
     context: dict[str, str]
 
     _rendered_script_path: Optional[Path] = None
+    _delete_on_exit: bool = True  # Flag to control cleanup
 
     def __enter__(self) -> Path:
+        # Ensure flag is reset for each use
+        self._delete_on_exit = True
         with NamedTemporaryFile(mode='w', delete=False) as rendered_script:
             rendered_script.write(
                 render_template_file(
@@ -147,8 +150,19 @@ def __enter__(self) -> Path:
         return self._rendered_script_path
 
     def __exit__(self, *args: object) -> None:
-        assert self._rendered_script_path
-        os.unlink(self._rendered_script_path)
+        # Only cleanup if the flag is set
+        if self._delete_on_exit:
+            self.cleanup()
+
+    def cleanup(self) -> None:
+        """Explicitly cleans up the rendered script file."""
+        if self._rendered_script_path and self._rendered_script_path.exists():
+            self._rendered_script_path.unlink()
+            self._rendered_script_path = None
+
+    def keep_rendered_file(self) -> None:
+        """Prevents automatic deletion on __exit__."""
+        self._delete_on_exit = False
 
 
 def effective_scripts_dest_dir(default: Path = DEFAULT_SCRIPTS_DEST_DIR) -> Path:
@@ -794,28 +808,107 @@ def prepare_tests(self, guest: Guest, logger: tmt.log.Logger) -> list[TestInvoca
 
     def prepare_scripts(self, guest: "tmt.steps.provision.Guest") -> None:
         """
-        Prepare additional scripts for testing
-        """
-
-        # Make sure scripts directory exists
-        command = Command("mkdir", "-p", f"{guest.scripts_path}")
+        Prepare additional scripts for testing by copying them to the guest.
 
+        Scripts destined for the default guest scripts path are pushed in
+        a single batch operation. Scripts with custom destination paths are
+        handled individually. Aliases are created as symbolic links after
+        the scripts are pushed.
+        """
+        # Ensure the main default scripts directory exists on the guest
+        scripts_dest_dir = guest.scripts_path
+        mkdir_command = Command("mkdir", "-p", f"{scripts_dest_dir}")
         if not guest.facts.is_superuser:
-            command = Command("sudo") + command
-
-        guest.execute(command)
+            mkdir_command = Command("sudo") + mkdir_command
+        guest.execute(mkdir_command)
+
+        # Group scripts by their target directory on the guest
+        # Key: Target directory path on guest
+        # Value: List of source paths (local paths) to copy into that directory
+        scripts_by_target_dir: dict[Path, list[Path]] = {}
+        # Store aliases: key is the target path on guest, value is list of alias names
+        # Aliases are created in the default scripts_dest_dir
+        aliases_to_create: dict[Path, list[str]] = {}
+        # Keep track of ScriptTemplate instances for later cleanup
+        templates_to_cleanup: list[ScriptTemplate] = []
 
-        # Install all scripts on guest
         for script in self.scripts:
+            if not script.enabled(guest):
+                continue
+
             with script as source:
-                for filename in [script.source_filename, *script.aliases]:
-                    if script.enabled(guest):
-                        guest.push(
-                            source=source,
-                            destination=script.destination_path or guest.scripts_path / filename,
-                            options=["-p", "--chmod=755"],
-                            superuser=guest.facts.is_superuser is not True,
-                        )
+                # If it's a template, prevent immediate cleanup and track it
+                if isinstance(script, ScriptTemplate):
+                    script.keep_rendered_file()
+                    templates_to_cleanup.append(script)
+                    # Use the actual rendered path as the source
+                    source_path = script._rendered_script_path
+                    assert source_path is not None  # Should be set by __enter__
+                else:
+                    source_path = source  # Use the path from __enter__ directly
+
+                destination_path = script.destination_path
+                target_dir: Path
+                final_target_path_on_guest: Path
+
+                if destination_path is None:
+                    # Default destination directory
+                    target_dir = scripts_dest_dir
+                    final_target_path_on_guest = target_dir / script.source_filename
+                    # Store aliases associated with the final target path in the default dir
+                    if script.aliases:
+                        aliases_to_create[final_target_path_on_guest] = script.aliases
+                else:
+                    # Custom destination path - treat it as the final file path
+                    target_dir = destination_path.parent
+                    final_target_path_on_guest = destination_path
+                    # Ensure the custom parent directory exists
+                    mkdir_parent_cmd = Command("mkdir", "-p", f"{target_dir}")
+                    if not guest.facts.is_superuser:
+                        mkdir_parent_cmd = Command("sudo") + mkdir_parent_cmd
+                    guest.execute(mkdir_parent_cmd)
+
+                # Add the source path to the list for its target directory
+                if target_dir not in scripts_by_target_dir:
+                    scripts_by_target_dir[target_dir] = []
+                scripts_by_target_dir[target_dir].append(source_path)
+
+        # Push script batches grouped by target directory
+        for target_dir, source_paths in scripts_by_target_dir.items():
+            if not source_paths:
+                continue
+
+            self.debug(f"Pushing script batch ({len(source_paths)} files) to {target_dir}")
+            guest.push(
+                source=source_paths,
+                destination=target_dir,
+                options=["-s", "-p", "--chmod=755"],
+                superuser=guest.facts.is_superuser is not True,
+            )
+
+        # Create aliases on the guest (only within the default scripts_dest_dir)
+        full_alias_command = ShellScript("")  # Start with an empty script
+        alias_count = 0
+        for target_path, aliases in aliases_to_create.items():
+            for alias in aliases:
+                link_path = scripts_dest_dir / alias
+                # Use absolute path for symlink target for simplicity/robustness
+                single_alias_cmd = ShellScript(
+                    f"ln -sf {target_path.as_posix()} {link_path.as_posix()}"
+                )
+                if alias_count == 0:
+                    full_alias_command = single_alias_cmd
+                else:
+                    full_alias_command &= single_alias_cmd  # Use the '&' operator (__and__)
+                alias_count += 1
+
+        if alias_count > 0:
+            self.debug("Creating script aliases on guest.")
+            guest.execute(full_alias_command, friendly_command="Create script aliases")
+
+        # Cleanup rendered templates now that they've been pushed
+        for template_script in templates_to_cleanup:
+            template_script.cleanup()
 
     def _tmt_report_results_filepath(self, invocation: TestInvocation) -> Path:
         """

tmt/steps/provision/__init__.py

@@ -1607,13 +1607,19 @@ def execute(
 
     def push(
         self,
-        source: Optional[Path] = None,
+        source: Optional[Union[Path, list[Path]]] = None,
         destination: Optional[Path] = None,
         options: Optional[list[str]] = None,
         superuser: bool = False,
     ) -> None:
         """
-        Push files to the guest
+        Push files or directories to the guest.
+
+        :param source: Path or list of paths on the local machine to push.
+                       If None, the plan workdir is pushed.
+        :param destination: Path on the guest where to push. If None, defaults to '/'.
+        :param options: List of rsync options to use.
+        :param superuser: If True, run rsync with sudo on the guest.
         """
 
         raise NotImplementedError
@@ -2416,21 +2422,19 @@ def execute(
 
     def push(
         self,
-        source: Optional[Path] = None,
+        source: Optional[Union[Path, list[Path]]] = None,
         destination: Optional[Path] = None,
         options: Optional[list[str]] = None,
         superuser: bool = False,
     ) -> None:
         """
-        Push files to the guest
-
-        By default the whole plan workdir is synced to the same location
-        on the guest. Use the 'source' and 'destination' to sync custom
-        location and the 'options' parameter to modify default options
-        which are '-Rrz --links --safe-links --delete'.
+        Push files or directories to the guest using rsync.
 
-        Set 'superuser' if rsync command has to run as root or passwordless
-        sudo on the Guest (e.g. pushing to r/o destination)
+        :param source: Path or list of paths on the local machine to push.
+                       If None, the plan workdir is pushed.
+        :param destination: Path on the guest where to push. If None, defaults to '/'.
+        :param options: List of rsync options to use.
+        :param superuser: If True, run rsync with sudo on the guest.
         """
 
         # Abort if guest is unavailable
@@ -2441,43 +2445,47 @@ def push(
         options = options or DEFAULT_RSYNC_PUSH_OPTIONS
         if destination is None:
             destination = Path("/")
+
+        sources: list[Path]
+        log_message: str
+
         if source is None:
             # FIXME: cast() - https://github.com/teemtee/tmt/issues/1372
             parent = cast(Provision, self.parent)
-
             assert parent.plan.workdir is not None
+            sources = [parent.plan.workdir]
+            log_message = f"Push workdir to guest '{self.primary_address}'."
+        elif isinstance(source, Path):
+            sources = [source]
+            log_message = f"Copy '{source}' to '{destination}' on the guest."
+        else:  # source is a list of Paths
+            sources = source
+            log_message = f"Copy {len(sources)} files/dirs to '{destination}' on the guest."
 
-            source = parent.plan.workdir
-            self.debug(f"Push workdir to guest '{self.primary_address}'.")
-        else:
-            self.debug(f"Copy '{source}' to '{destination}' on the guest.")
+        self.debug(log_message)
 
         def rsync() -> None:
-            """
-            Run the rsync command
-            """
-
-            # In closure, mypy has hard times to reason about the state of used variables.
-            assert options
-            assert source
-            assert destination
+            """Run the rsync command"""
+            assert options is not None
+            assert sources is not None
+            assert destination is not None
 
             cmd = ['rsync']
             if superuser and self.user != 'root':
                 cmd += ['--rsync-path', 'sudo rsync']
 
-            self._run_guest_command(
-                Command(
-                    *cmd,
-                    *options,
-                    "-e",
-                    self._ssh_command.to_element(),
-                    source,
-                    f"{self._ssh_guest}:{destination}",
-                ),
-                silent=True,
+            # Construct the rsync command with potentially multiple sources
+            rsync_cmd = Command(
+                *cmd,
+                *options,
+                "-e",
+                self._ssh_command.to_element(),
+                *[str(s) for s in sources],  # Unpack all source paths
+                f"{self._ssh_guest}:{destination}",
             )
 
+            self._run_guest_command(rsync_cmd, silent=True)
+
         # Try to push twice, check for rsync after the first failure
         try:
             rsync()

tmt/steps/provision/local.py

@@ -160,7 +160,7 @@ def reboot(
 
     def push(
         self,
-        source: Optional[Path] = None,
+        source: Optional[Union[Path, list[Path]]] = None,
         destination: Optional[Path] = None,
         options: Optional[list[str]] = None,
         superuser: bool = False,

tmt/steps/provision/podman.py

@@ -452,7 +452,7 @@ def execute(
 
     def push(
         self,
-        source: Optional[Path] = None,
+        source: Optional[Union[Path, list[Path]]] = None,
         destination: Optional[Path] = None,
         options: Optional[list[str]] = None,
         superuser: bool = False,
@@ -483,16 +483,15 @@ def push(
         # to the container. If running in toolbox, make sure to copy from the toolbox
         # container instead of localhost.
         if source and destination:
+            sources = source if isinstance(source, list) else [source]
             container_name: Optional[str] = None
             if self.parent.plan.my_run.runner.facts.is_toolbox:
                 container_name = self.parent.plan.my_run.runner.facts.toolbox_container_name
-            self.podman(
-                Command(
-                    "cp",
-                    f"{container_name}:{source}" if container_name else source,
-                    f"{self.container}:{destination}",
-                )
-            )
+
+            for src in sources:
+                source_spec = f"{container_name}:{src}" if container_name else str(src)
+                dest_spec = f"{self.container}:{destination}"
+                self.podman(Command("cp", source_spec, dest_spec))
 
     def pull(
         self,