diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 3a84cc61f43..694ceeb2e1e 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -93,7 +93,7 @@ jobs: test-type: ${{ matrix.test-type }} tests: test/${{ matrix.test-type }} - linux_docker_tests: + linux_runner_tests: needs: build_container runs-on: ubuntu-latest strategy: @@ -101,7 +101,7 @@ jobs: # Use modern versions due to docker incompatibility with python <3.8 python-version: ${{ github.event_name != 'pull_request' && fromJson('["3.13", "3.9"]') || fromJson('["3.10"]') }} - name: Docker Runner Tests (${{ matrix.python-version }}) + name: Runner Tests (${{ matrix.python-version }}) steps: - name: Checkout code uses: actions/checkout@v4 @@ -124,11 +124,25 @@ jobs: pip install -r conans/requirements_dev.txt pip install -r conans/requirements_server.txt pip install -r conans/requirements_runner.txt + sudo apt-get update && sudo apt-get install -y openssh-server - - name: Run tests + - name: Install OpenSSH Server + run: sudo apt-get update && sudo apt-get install -y openssh-server + + - name: Configure SSH Daemon, Generate SSH Keys & Configure Access + run: | + sudo sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + sudo sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config + sudo service ssh restart || sudo systemctl restart ssh + mkdir -p ~/.ssh + ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + chmod 600 ~/.ssh/authorized_keys + + - name: Run runner tests uses: ./.github/actions/test-coverage with: python-version: ${{ matrix.python-version }} - test-type: docker - tests: '-m docker_runner -rs' + test-type: runners + tests: '-m docker_runner -m ssh_runner -rs' workers: 1 diff --git a/conan/api/subapi/command.py b/conan/api/subapi/command.py index 272b5e09962..0a763ec1e45 100644 --- a/conan/api/subapi/command.py +++ b/conan/api/subapi/command.py @@ -52,7 +52,7 @@ def get_runner(profile_host): raise ConanException(f"Invalid runner configuration. 'type' must be defined") runner_instances_map = { 'docker': DockerRunner, - # 'ssh': SSHRunner, + 'ssh': SSHRunner, # 'wsl': WSLRunner, } try: diff --git a/conan/internal/model/profile.py b/conan/internal/model/profile.py index 54aa126d1f9..7f4e37c98c8 100644 --- a/conan/internal/model/profile.py +++ b/conan/internal/model/profile.py @@ -1,6 +1,7 @@ import copy from collections import OrderedDict, defaultdict +from conan.errors import ConanException from conan.tools.env.environment import ProfileEnvironment from conan.internal.model.conf import ConfDefinition from conan.internal.model.options import Options @@ -149,6 +150,11 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) + + runner_type = self.runner.get("type") + other_runner_type = other.runner.get("type") + if runner_type and other_runner_type and runner_type != other_runner_type: + raise ConanException(f"Found different runner types in profile composition ({runner_type} and {other_runner_type})") self.runner.update(other.runner) current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires} diff --git a/conan/internal/runner/output.py b/conan/internal/runner/output.py index a1732aa9e5d..7b2e0ec8684 100644 --- a/conan/internal/runner/output.py +++ b/conan/internal/runner/output.py @@ -10,3 +10,8 @@ def _write_message(self, msg, fg=None, bg=None, newline=True): for line in msg.splitlines(): super()._write_message(self._prefix, Color.BLACK, Color.BRIGHT_YELLOW, newline=False) super()._write_message(line, fg, bg, newline) + + @property + def padding(self): + return len(self._prefix) + 1 + diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py index b6fb0ab0083..558afc958c2 100644 --- a/conan/internal/runner/ssh.py +++ b/conan/internal/runner/ssh.py @@ -1,116 +1,106 @@ +import argparse from pathlib import Path import pathlib import tempfile +from conan.api.conan_api import ConanAPI from conan.api.output import Color, ConanOutput from conan.errors import ConanException import os +import re from io import BytesIO import sys -def ssh_info(msg, error=False): - fg=Color.BRIGHT_MAGENTA - if error: - fg=Color.BRIGHT_RED - ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) - ConanOutput().status(f'| {msg} |', fg=fg) - ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) +from conan import conan_version +from conan.internal.conan_app import ConanApp +from conan.internal.model.version import Version +from conan.internal.model.profile import Profile +from conan.internal.runner.output import RunnerOutput -class SSHRunner: - def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): - from paramiko.config import SSHConfig - from paramiko.client import SSHClient +class SSHRunner: + def __init__( + self, + conan_api: ConanAPI, + command: str, + host_profile: Profile, + build_profile: Profile, + args: argparse.Namespace, + raw_args: list[str], + ): self.conan_api = conan_api self.command = command self.host_profile = host_profile self.build_profile = build_profile - self.remote_host_profile = None - self.remote_build_profile = None - self.remote_python_command = None - self.remote_create_dir = None - self.remote_is_windows = None self.args = args self.raw_args = raw_args - self.ssh_config = None - self.remote_workspace = None - self.remote_conan = None - self.remote_conan_home = None - if host_profile.runner.get('use_ssh_config', False): - ssh_config_file = Path.home() / ".ssh" / "config" - ssh_config = SSHConfig.from_file(open(ssh_config_file)) - hostname = host_profile.runner.get("host") # TODO: this one is required - if ssh_config and ssh_config.lookup(hostname): - hostname = ssh_config.lookup(hostname)['hostname'] + try: + hostname = self._create_ssh_connection() + except Exception as e: + raise ConanException(f"Error creating SSH connection: {e}") + self.output = ConanOutput() + self.output.set_warnings_as_errors(True) + self.runner_output = RunnerOutput(hostname) + self.output.status(f"Connected to {hostname}", fg=Color.BRIGHT_MAGENTA) + self.remote_conn = RemoteConnection(self.client, self.runner_output) + + def run(self): + self._ensure_runner_environment() + self._copy_profiles() + self._copy_working_conanfile_path() + self._remote_create() + def _create_ssh_connection(self) -> str: + try: + from paramiko.config import SSHConfig + from paramiko.client import SSHClient, AutoAddPolicy + except ImportError: + raise ConanException("Paramiko is required for SSH runner, try installing it with 'pip install paramiko'") + + hostname = self.host_profile.runner.get("host") + if not hostname: + raise ConanException("Host not specified in runner.ssh configuration") + configfile = self.host_profile.runner.get("configfile", False) + if configfile and configfile not in ["False", "false", "0"]: + if configfile in ["True", "true", "1"]: + ssh_config_file = Path.home() / ".ssh" / "config" + else: + ssh_config_file = Path(configfile) + if not ssh_config_file.exists(): + raise ConanException(f"SSH config file not found at {ssh_config_file}") + ssh_config = SSHConfig.from_file(open(ssh_config_file)) + if ssh_config and ssh_config.lookup(hostname): + hostname = ssh_config.lookup(hostname)['hostname'] self.client = SSHClient() + self.client.set_missing_host_key_policy(AutoAddPolicy()) # Auto accept unknown keys self.client.load_system_host_keys() self.client.connect(hostname) + return hostname - - def run(self, use_cache=True): - ssh_info('Got to SSHRunner.run(), doing nothing') - - self.ensure_runner_environment() - self.copy_working_conanfile_path() - - raw_args = self.raw_args - raw_args[raw_args.index(self.args.path)] = self.remote_create_dir - raw_args = " ".join(raw_args) - - _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath - remote_json_output = _Path(self.remote_create_dir).joinpath("conan_create.json").as_posix() - command = f"{self.remote_conan} create {raw_args} --format json > {remote_json_output}" - - ssh_info(f"Remote command: {command}") - - stdout, _ = self._run_command(command) - first_line = True - while not stdout.channel.exit_status_ready(): - line = stdout.channel.recv(1024) - if first_line and self.remote_is_windows: - # Avoid clearing and moving the cursor when the remote server is Windows - # https://github.com/PowerShell/Win32-OpenSSH/issues/1738#issuecomment-789434169 - line = line.replace(b"\x1b[2J\x1b[m\x1b[H",b"") - sys.stdout.buffer.write(line) - sys.stdout.buffer.flush() - first_line = False - - if stdout.channel.recv_exit_status() == 0: - self.update_local_cache(remote_json_output) - - # self.client.close() - def ensure_runner_environment(self): - has_python3_command = False - python_is_python3 = False - - _, _stdout, _stderr = self.client.exec_command("python3 --version") - has_python3_command = _stdout.channel.recv_exit_status() == 0 - if not has_python3_command: - _, _stdout, _stderr = self.client.exec_command("python --version") - if _stdout.channel.recv_exit_status() == 0 and "Python 3" in _stdout.read().decode(): - python_is_python3 = True - - python_command = "python" if python_is_python3 else "python3" - self.remote_python_command = python_command - - if not has_python3_command and not python_is_python3: - raise ConanException("Unable to locate working Python 3 executable in remote SSH environment") + def _ensure_runner_environment(self): + # Check python3 is available in remote host + if self.remote_conn.run_command("python3 --version", "Checking python3 version").success: + self.remote_python_command = "python3" + else: + result = self.remote_conn.run_command("python --version", "Checking python version") + if result.success and "Python 3" in result.stdout: + self.remote_python_command = "python" + else: + self.output.error("Unable to locate Python 3 executable in remote SSH environment") # Determine if remote host is Windows - _, _stdout, _ = self.client.exec_command(f'{python_command} -c "import os; print(os.name)"') - if _stdout.channel.recv_exit_status() != 0: - raise ConanException("Unable to determine remote OS type") - is_windows = _stdout.read().decode().strip() == "nt" - self.remote_is_windows = is_windows + result = self.remote_conn.run_command(f'{self.remote_python_command} -c "import os; print(os.name)"', "Checking remote OS type") + if not result.success: + self.output.error("Unable to determine remote OS type") + self.is_remote_windows = result.stdout == "nt" # Get remote user home folder - _, _stdout, _ = self.client.exec_command(f'{python_command} -c "from pathlib import Path; print(Path.home())"') - if _stdout.channel.recv_exit_status() != 0: - raise ConanException("Unable to determine remote home user folder") - home_folder = _stdout.read().decode().strip() + result = self.remote_conn.run_command(f'{self.remote_python_command} -c "from pathlib import Path; print(Path.home())"', "Checking remote home folder") + if not result.success: + self.output.error("Unable to determine remote home user folder") + home_folder = result.stdout # Expected remote paths remote_folder = Path(home_folder) / ".conan2remote" @@ -119,92 +109,97 @@ def ensure_runner_environment(self): remote_conan_home = Path(home_folder) / ".conan2remote" / "conanhome" remote_conan_home = remote_conan_home.as_posix().replace("\\", "/") self.remote_conan_home = remote_conan_home - ssh_info(f"Remote workfolder: {remote_folder}") # Ensure remote folders exist for folder in [remote_folder, remote_conan_home]: - _, _stdout, _stderr = self.client.exec_command(f"""{python_command} -c "import os; os.makedirs('{folder}', exist_ok=True)""") - if _stdout.channel.recv_exit_status() != 0: - ssh_info(f"Error creating remote folder: {_stderr.read().decode()}") - raise ConanException(f"Unable to create remote workfolder at {folder}") + if not self.remote_conn.run_command(f'{self.remote_python_command} -c "import os; os.makedirs(\'{folder}\', exist_ok=True)"', f"Checking {folder} folder exists").success: + self.output.error(f"Unable to create remote workfolder at {folder}: {result.stderr}") + + # TODO: allow multiple venv given the client side conan version + requested_conan_version = "dev" if conan_version.pre == "dev" else str(conan_version) conan_venv = remote_folder + "/venv" - if is_windows: + if self.is_remote_windows: conan_cmd = remote_folder + "/venv/Scripts/conan.exe" else: conan_cmd = remote_folder + "/venv/bin/conan" - ssh_info(f"Expected remote conan home: {remote_conan_home}") - ssh_info(f"Expected remote conan command: {conan_cmd}") - # Check if remote Conan executable exists, otherwise invoke pip inside venv - sftp = self.client.open_sftp() - try: - sftp.stat(conan_cmd) - has_remote_conan = True - except FileNotFoundError: - has_remote_conan = False - finally: - sftp.close() + has_remote_conan = self.remote_conn.check_file_exists(conan_cmd) + + if self.is_remote_windows: + python_command = remote_folder + "/venv" + "/Scripts" + "/python.exe" + else: + python_command = remote_folder + "/venv" + "/bin" + "/python" if not has_remote_conan: - _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m venv {conan_venv}") - if _stdout.channel.recv_exit_status() != 0: - ssh_info(f"Unable to create remote venv: {_stderr.read().decode().strip()}") + self.output.verbose("Creating remote venv") + result = self.remote_conn.run_command(f"{self.remote_python_command} -m venv {conan_venv}", "Creating remote venv") + if not result.success: + self.output.error(f"Unable to create remote venv: {result.stderr}") + self._install_conan_remotely(python_command, requested_conan_version) + else: + version = self.remote_conn.run_command(f"{conan_cmd} --version", "Checking conan version", verbose=True).stdout + remote_conan_version = Version(version[version.rfind(" ")+1:]) + if requested_conan_version == "dev" and remote_conan_version.bump(1) == str(conan_version).replace("-dev", ""): + pass + elif remote_conan_version != requested_conan_version: + self.output.verbose(f"Remote Conan version mismatch: {remote_conan_version} != {requested_conan_version}") + self._install_conan_remotely(python_command, requested_conan_version) - if is_windows: - python_command = remote_folder + "/venv" + "/Scripts" + "/python.exe" - else: - python_command = remote_folder + "/venv" + "/bin" + "/python" + if not self.remote_conn.run_command(f"{conan_cmd} remote update conancenter --url='https://center2.conan.io'", "Updating conancenter remote").success: + self.output.error(f"Unable to update conancenter remote: {result.stderr}") + + self._create_remote_conan_wrapper(remote_conan_home, remote_folder, conan_cmd) - _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper") - if _stdout.channel.recv_exit_status() != 0: - # Note: this may fail on windows - ssh_info(f"Unable to install conan in venv: {_stderr.read().decode().strip()}") + def _install_conan_remotely(self, python_command: str, version: str): + self.output.verbose(f"Installing conan version: {version}") + # Note: this may fail on windows + result = self.remote_conn.run_command(f"{python_command} -m pip install conan{f'=={version}' if version != 'dev' else ' --upgrade'}", "Installing conan") + if not result.success: + self.output.error(f"Unable to install conan in venv: {result.stderr}") + def _create_remote_conan_wrapper(self, remote_conan_home: str, remote_folder: str, conan_cmd: str): + + # Create conan wrapper with proper environment variables remote_env = { 'CONAN_HOME': remote_conan_home, - 'CONAN_RUNNER_ENVIRONMENT': "1" + 'CONAN_RUNNER_ENVIRONMENT': "1" # This env will prevent conan (running remotely) to start an infinite remote call } - if is_windows: + if self.is_remote_windows: # Wrapper script with environment variables preset - env_lines = "\n".join([f"set {k}={v}" for k,v in remote_env.items()]) - conan_bat_contents = f"""@echo off\n{env_lines}\n{conan_cmd} %*\n""" - conan_bat = remote_folder + "/conan.bat" - try: - sftp = self.client.open_sftp() - sftp.putfo(BytesIO(conan_bat_contents.encode()), conan_bat) - except: - raise ConanException("unable to set up Conan remote script") - finally: - sftp.close() - - self.remote_conan = conan_bat - _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} config home") - ssh_info(f"Remote conan config home returned: {_stdout.read().decode().strip()}") - _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} profile detect --force") - self._copy_profiles() + env_lines = "\n".join([f"set {k}={v}" for k, v in remote_env.items()]) + conan_wrapper_contents = f"""@echo off\n{env_lines}\n{conan_cmd} %*\n""" + else: + env_lines = "\n".join([f"export {k}={v}" for k, v in remote_env.items()]) + conan_wrapper_contents = f"""{env_lines}\n{conan_cmd} $@\n""" + self.remote_conan = self.remote_conn.create_remote_script(conan_wrapper_contents, remote_folder + "/conan", self.is_remote_windows) + self.remote_conn.run_command(f"{self.remote_conan} config home", "Checking conan home", verbose=True) + if not self.remote_conn.run_command(f"{self.remote_conan} profile detect --force", "Detecting remote profile").success: + self.output.error("Error creating default profile in remote machine") def _copy_profiles(self): - sftp = self.client.open_sftp() - - # TODO: very questionable choices here - try: - profiles = { - self.args.profile_host[0]: self.host_profile.dumps(), - self.args.profile_build[0]: self.build_profile.dumps() - } - - for name, contents in profiles.items(): - dest_filename = self.remote_conan_home + f"/profiles/{name}" - sftp.putfo(BytesIO(contents.encode()), dest_filename) - except: - raise ConanException("Unable to copy profiles to remote") - finally: - sftp.close() - - def copy_working_conanfile_path(self): + self.output.status("Copying profiles and recipe to host...", fg=Color.BRIGHT_MAGENTA) + if not self.remote_conan_home: + raise ConanException("Remote Conan home folder not set") + remote_profile_path = Path(self.remote_conan_home) / "profiles" + # If profile path does not exist, create the folder to avoid errors + if not self.remote_conn.check_file_exists(remote_profile_path.as_posix()): + self.remote_conn.mkdir(remote_profile_path.as_posix()) + # Iterate over all profiles and copy using sftp + for profile in set(self.args.profile_host + (self.args.profile_build or [])): + dest_filename = remote_profile_path / profile + profile_path = self.conan_api.profiles.get_path(profile) + self.output.verbose(f"Copying profile '{profile}': {profile_path} -> {dest_filename}") + self.remote_conn.put(profile_path, dest_filename.as_posix()) + if not self.args.profile_build: + dest_filename = remote_profile_path / "default" # in remote use "default" profile + default_build_profile = self.conan_api.profiles.get_default_build() + self.output.verbose(f"Copying default profile: {default_build_profile} -> {dest_filename}") + self.remote_conn.put(default_build_profile, dest_filename.as_posix()) + + def _copy_working_conanfile_path(self): resolved_path = Path(self.args.path).resolve() if resolved_path.is_file(): resolved_path = resolved_path.parent @@ -213,60 +208,190 @@ def copy_working_conanfile_path(self): return ConanException("Error determining conanfile directory") # Create temporary destination directory - temp_dir_create_cmd = f"""{self.remote_python_command} -c "import tempfile; print(tempfile.mkdtemp(dir='{self.remote_workspace}'))""" - _, _stdout, _ = self.client.exec_command(temp_dir_create_cmd) - if _stdout.channel.recv_exit_status() != 0: - raise ConanException("Unable to create remote temporary directory") - self.remote_create_dir = _stdout.read().decode().strip().replace("\\", '/') - + temp_dir_create_cmd = f"""{self.remote_python_command} -c "import tempfile; print(tempfile.mkdtemp(dir='{self.remote_workspace}'))" """ + result = self.remote_conn.run_command(temp_dir_create_cmd, "Creating remote temporary directory") + if not result.success or not result.stdout: + self.output.error(f"Unable to create remote temporary directory: {result.stderr}") + self.remote_create_dir = result.stdout.replace("\\", '/') + _Path = pathlib.PureWindowsPath if self.is_remote_windows else pathlib.PurePath + + # Base case: Conanfile located at the root of the project + remote_tmp_dir = _Path(self.remote_create_dir) + + # Check if recipe defines a root folder + app = ConanApp(self.conan_api) + remotes = self.conan_api.remotes.list(self.args.remote) if not self.args.no_remote else [] + conanfile = app.loader.load_consumer(resolved_path / "conanfile.py", remotes=remotes) + if hasattr(conanfile, "layout"): + try: + conanfile.layout() + if conanfile.folders.root: + root_relative_path = Path(conanfile.folders.root) + tmp = (resolved_path / root_relative_path).resolve() + extra_path = resolved_path.relative_to(tmp) + self.remote_create_dir = (remote_tmp_dir / extra_path).as_posix() + resolved_path = tmp + except Exception: + pass # Copy current folder to destination using sftp - _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath - sftp = self.client.open_sftp() for root, dirs, files in os.walk(resolved_path.as_posix()): relative_root = Path(root).relative_to(resolved_path) + dirs[:] = [d for d in dirs if d not in {'venv', '.venv'}] # Filter out virtualenv folders for dir in dirs: - dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(dir).as_posix() - sftp.mkdir(dst) + dst = remote_tmp_dir.joinpath(relative_root).joinpath(dir).as_posix() + self.remote_conn.mkdir(dst) for file in files: orig = os.path.join(root, file) - dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(file).as_posix() - sftp.put(orig, dst) - sftp.close() + dst = remote_tmp_dir.joinpath(relative_root).joinpath(file).as_posix() + self.remote_conn.put(orig, dst) - def _run_command(self, command): - ''' Run a command in an SSH session. - When requesting a pseudo-terminal from the server, - ensure we pass width and height that matches the current - terminal - ''' - channel = self.client.get_transport().open_session() - if sys.stdout.isatty(): - width, height = os.get_terminal_size() - channel.get_pty(width=width, height=height) + def _remote_create(self): + raw_args = self.raw_args + raw_args[raw_args.index(self.args.path)] = self.remote_create_dir + raw_args = " ".join(raw_args) + raw_args = raw_args.replace("&", '"&"').replace("*", '"*"') - channel.exec_command(command) + _Path = pathlib.PureWindowsPath if self.is_remote_windows else pathlib.PurePath + remote_json_output = _Path(self.remote_create_dir).joinpath("conan_create.json").as_posix() + conan_create_cmd = f'{self.remote_conan} create {raw_args} --format json > {remote_json_output}' + script_path = _Path(self.remote_create_dir).joinpath("conan_create").as_posix() + script_path = self.remote_conn.create_remote_script(conan_create_cmd, script_path, self.is_remote_windows) - stdout = channel.makefile("r") - stderr = channel.makefile("r") - return stdout, stderr + if self.remote_conn.run_interactive_command(script_path, self.is_remote_windows, f"Running conan create {raw_args}"): + self._update_local_cache(remote_json_output) - def update_local_cache(self, json_result): - # ('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json' - _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + self.client.close() + + def _update_local_cache(self, json_result: str): + _Path = pathlib.PureWindowsPath if self.is_remote_windows else pathlib.PurePath pkg_list_json = _Path(self.remote_create_dir).joinpath("pkg_list.json").as_posix() - pkg_list_command = f"{self.remote_conan} list --graph={json_result} --graph-binaries=build --format=json > {pkg_list_json}" + # List every package (built or cached) because local cache could have been deleted + pkg_list_command = f"{self.remote_conan} list --graph={json_result} --format=json > {pkg_list_json}" _, stdout, _ = self.client.exec_command(pkg_list_command) if stdout.channel.recv_exit_status() != 0: raise ConanException("Unable to generate remote package list") conan_cache_tgz = _Path(self.remote_create_dir).joinpath("cache.tgz").as_posix() + self.output.status("Retrieving remote artifacts into local cache...", fg=Color.BRIGHT_MAGENTA) + self.output.verbose("Remote cache tgz: " + conan_cache_tgz) cache_save_command = f"{self.remote_conan} cache save --list {pkg_list_json} --file {conan_cache_tgz}" _, stdout, _ = self.client.exec_command(cache_save_command) if stdout.channel.recv_exit_status() != 0: raise ConanException("Unable to save remote conan cache state") - sftp = self.client.open_sftp() with tempfile.TemporaryDirectory() as tmp: local_cache_tgz = os.path.join(tmp, 'cache.tgz') - sftp.get(conan_cache_tgz, local_cache_tgz) - package_list = self.conan_api.cache.restore(local_cache_tgz) + self.remote_conn.get(conan_cache_tgz, local_cache_tgz) + self.output.verbose("Retrieved local cache: " + local_cache_tgz) + self.conan_api.cache.restore(local_cache_tgz) + + +class RemoteConnection: + def __init__(self, client, runner_output: RunnerOutput): + from paramiko.client import SSHClient + self.client: SSHClient = client + self.runner_output = runner_output + + def put(self, src: str, dst: str) -> None: + try: + sftp = self.client.open_sftp() + sftp.put(src, dst) + sftp.close() + except IOError as e: + self.runner_output.error(f"Unable to copy {src} to {dst}:\n{e}") + + def get(self, src: str, dst: str) -> None: + try: + sftp = self.client.open_sftp() + sftp.get(src, dst) + sftp.close() + except IOError as e: + self.runner_output.error(f"Unable to copy from remote {src} to {dst}:\n{e}") + + def mkdir(self, folder: str) -> None: + sftp = self.client.open_sftp() + sftp.mkdir(folder) + sftp.close() + + def check_file_exists(self, file: str) -> bool: + try: + sftp = self.client.open_sftp() + sftp.stat(file) + sftp.close() + return True + except FileNotFoundError: + return False + + def create_remote_script(self, script: str, script_path: str, is_remote_windows: bool) -> str: + script_path += ".bat" if is_remote_windows else ".sh" + try: + sftp = self.client.open_sftp() + sftp.putfo(BytesIO(script.encode()), script_path) + sftp.chmod(script_path, 0o755) + sftp.close() + except Exception as e: + self.runner_output.error(f"Unable to create remote script in {script_path}:\n{e}") + return script_path + + class RunResult: + def __init__(self, success, stdout, stderr): + self.success = success + self.stdout = stdout + self.stderr = stderr + + def run_command(self, command: str, friendly_command: str = "", verbose: bool = False) -> RunResult: + _, stdout, stderr = self.client.exec_command(command) + log = self.runner_output.status if verbose else self.runner_output.verbose + log(f'{friendly_command}...', fg=Color.BLUE) + self.runner_output.debug(f'$ {command}') + result = RemoteConnection.RunResult(stdout.channel.recv_exit_status() == 0, + stdout.read().decode().strip(), + stderr.read().decode().strip()) + log(f"{result.stdout}") + return result + + def run_interactive_command(self, command: str, is_remote_windows: bool, friendly_command: str = "") -> bool: + ''' Run a command in an SSH session. + When requesting a pseudo-terminal from the server, + ensure we pass width and height that matches the current + terminal + :return: True if the command succeeded + ''' + self.runner_output.status(f'{friendly_command}...', fg=Color.BLUE) + self.runner_output.debug(f"$ {command}") + channel = self.client.get_transport().open_session() + if sys.stdout.isatty(): + width, height = os.get_terminal_size() + else: + width, height = 80, 24 + width -= self.runner_output.padding + channel.get_pty(width=width, height=height) + + channel.exec_command(command) + stdout = channel.makefile("r") + first_line = True + + cursor_movement_pattern = re.compile(r'(\x1b\[(\d+);(\d+)H)') + + def remove_cursor_movements(data): + """Replace cursor movements with newline if column is 1, or empty otherwise.""" + def replace_cursor_match(match): + column = int(match.group(3)) + if column == 1: + return "\n" # Replace with newline if column is 1 + return "" # Otherwise, replace with empty string + return cursor_movement_pattern.sub(replace_cursor_match, data) + + while not stdout.channel.exit_status_ready(): + line = stdout.channel.recv(2048) + if first_line and is_remote_windows: + # Avoid clearing and moving the cursor when the remote server is Windows + # https://github.com/PowerShell/Win32-OpenSSH/issues/1738#issuecomment-789434169 + line = line.replace(b"\x1b[2J\x1b[m\x1b[H", b"").replace(b"\r\n", b"") + first_line = False + # This is the easiest and better working approach but not testable + # sys.stdout.buffer.write(line) + # sys.stdout.buffer.flush() + line = remove_cursor_movements(line.replace(b'\r', b'').decode(errors='ignore').strip()) + self.runner_output.status(line) + return stdout.channel.recv_exit_status() == 0 diff --git a/pytest.ini b/pytest.ini index bf8bb6be5d0..d2b4ad7296b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,6 +3,7 @@ norecursedirs = '.*', 'dist', 'CVS', '_darcs', '{arch}', '*.egg', 'venv', 'asset testpaths = 'test' markers = docker_runner: Mark tests that require Docker to run. + ssh_runner: Mark tests that require SSH to run. artifactory_ready: These tests can be run against a full Artifactory filterwarnings = ignore:'cgi' is deprecated:DeprecationWarning diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index eaee5d18528..c4a0aef9d21 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -2,17 +2,22 @@ import os import pytest import docker +from pathlib import Path from conan.test.utils.tools import TestClient from conan.test.assets.cmake import gen_cmakelists from conan.test.assets.sources import gen_function_h, gen_function_cpp +def docker_from_env(): + try: + return docker.from_env() + except Exception: + return docker.DockerClient(base_url=f'unix://{os.path.expanduser("~")}/.rd/docker.sock', version='auto') # Rancher + + def docker_skip(test_image='ubuntu:22.04'): try: - try: - docker_client = docker.from_env() - except: - docker_client = docker.DockerClient(base_url=f'unix://{os.path.expanduser("~")}/.rd/docker.sock', version='auto') # Rancher + docker_client = docker_from_env() if test_image: docker_client.images.pull(test_image) except docker.errors.DockerException: @@ -459,7 +464,7 @@ def test_create_docker_runner_from_configfile_with_args(): client = TestClient() # Ensure the network exists - docker_client = docker.from_env() + docker_client = docker_from_env() docker_client.networks.create("my-network") configfile = textwrap.dedent(f""" @@ -550,6 +555,45 @@ def test_create_docker_runner_default_build_profile(): assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out +@pytest.mark.docker_runner +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +def test_create_docker_runner_profile_composition(): + """ + Tests the ``conan create . `` with profile composition + """ + client = TestClient() + + profile = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + image=conan-runner-ninja-test + """) + + profile_extension = textwrap.dedent(f"""\ + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_ninja")} + build_context={conan_base_path()} + cache=copy + remove=True + """) + client.save({"profile": profile, "profile_extension": profile_extension}) + client.run("new cmake_lib -d name=pkg -d version=2.0") + client.run("create . -pr:h profile -pr:h profile_extension") + + assert "[100%] Built target example" in client.out + assert "Restore: pkg/2.0 in pkgc6abef0178849" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + @pytest.mark.docker_runner @pytest.mark.skipif(docker_skip(), reason="Only docker running") @@ -645,3 +689,280 @@ def package(self): assert "Restore: pkg/1.0" in client.out assert "Removing container" in client.out + + +def ssh_skip(): + import platform + try: + import paramiko + except ImportError: + return True + return platform.system() != "Linux" + +@pytest.mark.ssh_runner +@pytest.mark.skipif(ssh_skip(), reason="SSH environment have to be configured") +def test_create_ssh_runner_only_host(): + """ + Tests the ``conan create . `` with ssh runner using only host + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + + profile_host = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=localhost + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=1.0") + client.run("create . -pr:h host -pr:b build") + + assert "[100%] Built target example" in client.out + assert "Restore: pkg/1.0 in pkgc8bc87152b946" in client.out + assert "Restore: pkg/1.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/1.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + +@pytest.mark.ssh_runner +@pytest.mark.skipif(ssh_skip(), reason="SSH environment have to be configured") +def test_create_ssh_runner_with_config(): + """ + Tests the ``conan create . `` with ssh config file + """ + client = TestClient() + + ssh_config = textwrap.dedent(f"""\ + Host local-machine + HostName localhost + """) + client.save({"ssh_config": ssh_config}) + + profile_build = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + + profile_host = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=local-machine + configfile={os.path.join(client.current_folder, 'ssh_config')} + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=2.0") + client.run("create . -pr:h host -pr:b build") + + assert "[100%] Built target example" in client.out + assert "Restore: pkg/2.0 in pkgc6abef0178849" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + + client.save({"config": ssh_config}, path=Path.home() / ".ssh") + profile_host = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=local-machine + # Let the runner find default config file + configfile=True + """) + client.save({"host": profile_host}) + client.run("create . -pr:h host -pr:b build") + + assert "[100%] Built target example" in client.out + assert "Restore: pkg/2.0 in pkgc6abef0178849" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + +@pytest.mark.ssh_runner +@pytest.mark.skipif(ssh_skip(), reason="SSH environment have to be configured") +def test_create_ssh_runner_default_profile(): + """ + Tests the ``conan create . `` without build profile + """ + client = TestClient() + + profile_host = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=localhost + """) + + client.save({"host": profile_host}) + client.run("new cmake_lib -d name=pkg -d version=2.0") + client.run("create . -pr:h host -vverbose") + + assert "Copying default profile: " in client.out + assert "[100%] Built target example" in client.out + assert "Restore: pkg/2.0 in pkgc6abef0178849" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/2.0:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + +@pytest.mark.ssh_runner +@pytest.mark.skipif(ssh_skip(), reason="SSH environment have to be configured") +def test_create_ssh_runner_in_subfolder(): + client = TestClient() + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import load, copy + from conan.tools.cmake import CMake + + class Pkg(ConanFile): + name = "pkg" + version = "1.0" + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeToolchain" + + def layout(self): + self.folders.root = ".." + self.folders.source = "." + self.folders.build = "build" + + def export_sources(self): + folder = os.path.join(self.recipe_folder, "..") + copy(self, "*.txt", folder, self.export_sources_folder) + copy(self, "src/*.cpp", folder, self.export_sources_folder) + copy(self, "include/*.h", folder, self.export_sources_folder) + + def source(self): + cmake_file = load(self, "CMakeLists.txt") + + def build(self): + path = os.path.join(self.source_folder, "CMakeLists.txt") + cmake_file = load(self, path) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + """) + + header = textwrap.dedent(""" + #pragma once + void hello(); + """) + source = textwrap.dedent(""" + #include + void hello() { + std::cout << "Hello!" << std::endl; + } + """) + + cmakelist = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(pkg CXX) + add_library(pkg src/hello.cpp) + target_include_directories(pkg PUBLIC include) + set_target_properties(pkg PROPERTIES PUBLIC_HEADER "include/hello.h") + install(TARGETS pkg) + + """) + + profile_host = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=localhost + """) + + client.save({"conan/conanfile.py": conanfile, + "conan/host": profile_host, + "include/hello.h": header, + "src/hello.cpp": source, + "CMakeLists.txt": cmakelist}) + + with client.chdir("conan"): + client.run("create . -pr:h host -vverbose") + + assert "Restore: pkg/1.0" in client.out + + +def test_create_multiple_runner_types(): + """ + Tests the ``conan create . `` without multiple host profiles defining different runner types + """ + client = TestClient() + + profile_host_ssh = textwrap.dedent(f"""\ + [settings] + arch={{{{ detect_api.detect_arch() }}}} + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=ssh + host=localhost + """) + + profile_host_docker = textwrap.dedent(f"""\ + [runner] + type=docker + dockerfile=/path + image=conan-runner-default-test + cache=shared + remove=True + """) + client.save({"host_ssh": profile_host_ssh, "host_docker": profile_host_docker}) + client.run("new cmake_lib -d name=pkg -d version=2.0") + client.run("create . -pr:h host_docker -pr:h host_ssh -vverbose", assert_error=True) + assert "Found different runner types in profile composition (docker and ssh)" in client.out diff --git a/test/integration/conanfile/runner_test.py b/test/integration/conanfile/runner_test.py index 28ce9c9b99b..917249da87a 100644 --- a/test/integration/conanfile/runner_test.py +++ b/test/integration/conanfile/runner_test.py @@ -59,3 +59,4 @@ def source(self): client.save({"conanfile.py": conanfile}) client.run("source .") assert 'conanfile.py: Buffer got stderr msgs Hello' in client.out +