From 238fbe6fd2a7c56058c40a87e382ae19e3a34f8b Mon Sep 17 00:00:00 2001 From: Lucas RAVAGNIER Date: Wed, 26 Aug 2026 17:56:41 +0200 Subject: [PATCH] add SSH feature coverage for the OpenSSH update Add regression/feature tests for the new OpenSSH package (post-quantum KEX, FIDO/security-key support, scp/sftp behavior, ssh_config.d inclusion) - test_ssh_config_include.py: extends the sshd_config.d include regression test with the client-side ssh_config.d equivalent. - test_ssh_algorithms.py: hardcoded, exhaustive coverage of the KEX/cipher/MAC/host-key algorithms the package supports, so any future change to that list has to be a conscious update of this file. - test_ssh_post_quantum.py: checks the hybrid post-quantum KEX is negotiated by default. - test_ssh_fido.py: checks FIDO/security-key (sk-*) support is advertised and wired in. - test_ssh_file_transfer.py: checks scp (SFTP and legacy protocols) and sftp file transfers. Signed-off-by: Lucas RAVAGNIER --- tests/system/test_ssh_algorithms.py | 186 +++++++++++++++++++++++ tests/system/test_ssh_config_include.py | 46 ++++++ tests/system/test_ssh_fido.py | 52 +++++++ tests/system/test_ssh_file_transfer.py | 73 +++++++++ tests/system/test_ssh_post_quantum.py | 28 ++++ tests/system/test_sshd_config_include.py | 28 ---- 6 files changed, 385 insertions(+), 28 deletions(-) create mode 100644 tests/system/test_ssh_algorithms.py create mode 100644 tests/system/test_ssh_config_include.py create mode 100644 tests/system/test_ssh_fido.py create mode 100644 tests/system/test_ssh_file_transfer.py create mode 100644 tests/system/test_ssh_post_quantum.py delete mode 100644 tests/system/test_sshd_config_include.py diff --git a/tests/system/test_ssh_algorithms.py b/tests/system/test_ssh_algorithms.py new file mode 100644 index 000000000..7cf5ddd67 --- /dev/null +++ b/tests/system/test_ssh_algorithms.py @@ -0,0 +1,186 @@ +import pytest + +from contextlib import nullcontext + +from lib.commands import SSHCommandFailed, ssh +from lib.host import Host + +# Exhaustive coverage of the KEX/cipher/MAC/host-key algorithms OpenSSH +# supports, hardcoded from `ssh -Q kex|cipher|mac|key` on the reference +# build (openssh-9.9p1-27.2.xcpng8.3). The lists are deliberately not +# queried live: the point of test_*_list_matches_hardcoded is to fail the +# day the compiled-in algorithm support changes, forcing a conscious +# update of this file instead of the change going unnoticed. +# +# Requirements: +# - an XCP-ng host (--hosts) >= 8.3, with the OpenSSH 9.9p1 + +# algorithm -> enabled by default (sshd -T) on the reference build +KEX_ALGORITHMS = { + # hybrid post-quantum, new in this OpenSSH update + "mlkem1024nistp384-sha384": True, + "mlkem768x25519-sha256": True, + "mlkem768nistp256-sha256": True, + "sntrup761x25519-sha512": True, + "sntrup761x25519-sha512@openssh.com": True, + # classical, enabled + "curve25519-sha256": True, + "curve25519-sha256@libssh.org": True, + "ecdh-sha2-nistp521": True, + "ecdh-sha2-nistp384": True, + "ecdh-sha2-nistp256": True, + "diffie-hellman-group16-sha512": True, + "diffie-hellman-group18-sha512": True, + # compiled in, but disabled + "diffie-hellman-group1-sha1": False, + "diffie-hellman-group14-sha1": False, + "diffie-hellman-group14-sha256": False, + "diffie-hellman-group-exchange-sha1": False, + "diffie-hellman-group-exchange-sha256": False, +} + +CIPHERS = { + "chacha20-poly1305@openssh.com": True, + "aes256-gcm@openssh.com": True, + "aes128-gcm@openssh.com": True, + "aes256-ctr": True, + "aes128-ctr": True, + "3des-cbc": False, + "aes128-cbc": False, + "aes192-cbc": False, + "aes256-cbc": False, + "aes192-ctr": False, +} + +MACS = { + "hmac-sha2-512-etm@openssh.com": True, + "hmac-sha2-256-etm@openssh.com": True, + "umac-128-etm@openssh.com": True, + "hmac-sha2-512": True, + "hmac-sha2-256": True, + "umac-128@openssh.com": True, + "hmac-sha1": False, + "hmac-sha1-96": False, + "hmac-md5": False, + "hmac-md5-96": False, + "umac-64@openssh.com": False, + "hmac-sha1-etm@openssh.com": False, + "hmac-sha1-96-etm@openssh.com": False, + "hmac-md5-etm@openssh.com": False, + "hmac-md5-96-etm@openssh.com": False, + "umac-64-etm@openssh.com": False, +} + +# Server host identity algorithms actually exercisable end to end: the host +# only carries one host key per type (ed25519, ecdsa on nistp256, rsa), so +# only algorithms with matching key material are forced here. Certificate +# variants would need a CA, and the sk-* types from `ssh -Q key` are not +# host identity keys at all (they're for user authentication via a +# hardware security key, see test_ssh_fido.py) so they don't belong in a +# HostKeyAlgorithms test. +HOSTKEY_ALGORITHMS = { + "ssh-ed25519": True, + "ecdsa-sha2-nistp256": True, + "rsa-sha2-256": True, + "rsa-sha2-512": True, + "ssh-rsa": False, # raw SHA-1 RSA signature, disabled by crypto-policy +} + +# Full compiled-in key type support (`ssh -Q key`), used only to detect if +# that list ever drifts (e.g. FIDO/security-key support silently regressing, +# as it did between the previous and this OpenSSH build). +ALL_COMPILED_KEY_TYPES = { + "ssh-ed25519", + "ssh-ed25519-cert-v01@openssh.com", + "sk-ssh-ed25519@openssh.com", + "sk-ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "sk-ecdsa-sha2-nistp256@openssh.com", + "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ssh-rsa", + "ssh-rsa-cert-v01@openssh.com", +} + +def _assert_list_matches(host: Host, query: str, expected: set) -> None: + actual = {line for line in host.ssh(f"ssh -Q {query}").splitlines() if line and ' ' not in line} + assert actual == expected, ( + f"`ssh -Q {query}` no longer matches the hardcoded list in this test file " + f"(missing={sorted(expected - actual)}, new={sorted(actual - expected)}); " + "update the hardcoded list after reviewing the change" + ) + +def test_kex_algorithms_list_matches_hardcoded(host: Host) -> None: + _assert_list_matches(host, "kex", set(KEX_ALGORITHMS)) + +def test_ciphers_list_matches_hardcoded(host: Host) -> None: + _assert_list_matches(host, "cipher", set(CIPHERS)) + +def test_macs_list_matches_hardcoded(host: Host) -> None: + _assert_list_matches(host, "mac", set(MACS)) + +def test_key_types_list_matches_hardcoded(host: Host) -> None: + _assert_list_matches(host, "key", ALL_COMPILED_KEY_TYPES) + +def _force_algorithm(host: Host, option: str, algo: str, *, extra_options: list[str] = []) -> None: + # multiplexing must be off: a shared control connection would reuse the + # algorithm negotiated by whichever call created it, silently ignoring + # the -o option on every later call. + ssh(host.hostname_or_ip, 'true', options=['-o', f'{option}={algo}'] + extra_options, multiplexing=False) + +# mlkem768nistp256-sha256 and mlkem1024nistp384-sha384 aren't part of +# upstream OpenSSH (see openssh-10.0-mlkem-nist.patch): they're carried by +# RHEL-family builds (RHEL, CentOS, Alma, and this XCP-ng build) for FIPS +# compliance, but not by other builds, including the machine running these +# tests. There's no second RHEL-family host available to interoperate with +# either, so unlike every other algorithm here, these two are exercised in +# loopback on the host itself (which does understand its own names). That +# can't reach an authenticated session (root has no key to log into +# itself), so instead we assert the exact requested algorithm was the one +# negotiated. +SPECIFIC_KEX_NAMES = {"mlkem768nistp256-sha256", "mlkem1024nistp384-sha384"} + +def _negotiated_kex_in_loopback(host: Host, algo: str) -> str: + # the trailing "; true" keeps the remote command's exit code at 0 + # (the inner loopback ssh fails at authentication, not at key exchange) + # so we can just inspect its output instead of juggling SSHCommandFailed. + output = host.ssh( + "ssh -v -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " + f"-o ControlMaster=no -o KexAlgorithms={algo} localhost true 2>&1; true" + ) + for line in output.splitlines(): + if line.startswith("debug1: kex: algorithm:"): + return line.rsplit(':', 1)[1].strip() + pytest.fail(f"could not find the negotiated KEX algorithm in loopback ssh -v output:\n{output}") + +@pytest.mark.parametrize("algo,enabled", KEX_ALGORITHMS.items(), ids=list(KEX_ALGORITHMS)) +def test_kex_algorithm(host: Host, algo: str, enabled: bool) -> None: + if algo in SPECIFIC_KEX_NAMES: + assert enabled + assert _negotiated_kex_in_loopback(host, algo) == algo + return + with pytest.raises(SSHCommandFailed) if not enabled else nullcontext(): + _force_algorithm(host, "KexAlgorithms", algo) + +@pytest.mark.parametrize("algo,enabled", CIPHERS.items(), ids=list(CIPHERS)) +def test_cipher_algorithm(host: Host, algo: str, enabled: bool) -> None: + with pytest.raises(SSHCommandFailed) if not enabled else nullcontext(): + _force_algorithm(host, "Ciphers", algo) + +@pytest.mark.parametrize("algo,enabled", MACS.items(), ids=list(MACS)) +def test_mac_algorithm(host: Host, algo: str, enabled: bool) -> None: + # MACs are only negotiated for non-AEAD ciphers (AEAD ciphers like the + # default chacha20-poly1305/aes-gcm have their MAC built in, making the + # MACs option moot), so a non-AEAD cipher is pinned to force the point. + non_aead_cipher = ['-o', 'Ciphers=aes256-ctr'] + with pytest.raises(SSHCommandFailed) if not enabled else nullcontext(): + _force_algorithm(host, "MACs", algo, extra_options=non_aead_cipher) + +@pytest.mark.parametrize("algo,enabled", HOSTKEY_ALGORITHMS.items(), ids=list(HOSTKEY_ALGORITHMS)) +def test_hostkey_algorithm(host: Host, algo: str, enabled: bool) -> None: + with pytest.raises(SSHCommandFailed) if not enabled else nullcontext(): + _force_algorithm(host, "HostKeyAlgorithms", algo) diff --git a/tests/system/test_ssh_config_include.py b/tests/system/test_ssh_config_include.py new file mode 100644 index 000000000..e5c585ad8 --- /dev/null +++ b/tests/system/test_ssh_config_include.py @@ -0,0 +1,46 @@ +import random + +from lib.common import Defer +from lib.host import Host + +# Regression test for the ssh client and sshd silently ignoring drop-in +# configuration files (missing "Include" directive in the packaged config). +# Covers both /etc/ssh/ssh_config.d/*.conf (client) and +# /etc/ssh/sshd_config.d/*.conf (server). +# +# Each test creates a drop-in with a directive not set elsewhere, then uses +# `ssh -G` or `sshd -T` to verify that the corresponding directory is included. +# +# Requirements: +# - an XCP-ng host (--hosts) >= 8.2 +# - the ssh_config.d test additionally requires the OpenSSH security update: +# the Include directive isn't in the previous package's ssh_config at all + +SSH_CONFIG_D = "/etc/ssh/ssh_config.d" +SSHD_CONFIG_D = "/etc/ssh/sshd_config.d" + +def test_ssh_config_d_is_included(host: Host, defer: Defer) -> None: + marker = str(random.randint(10000, 99999)) + dropin = f"{SSH_CONFIG_D}/99-xcp-ng-tests-include-check.conf" + + host.ssh(f"echo 'ConnectTimeout {marker}' > {dropin}") + defer(lambda: host.ssh(f"rm -f {dropin}")) + + effective_config = host.ssh("ssh -G localhost") + assert f"connecttimeout {marker}" in effective_config, ( + f"drop-in {dropin} was not picked up by ssh -G: " + f"{SSH_CONFIG_D} seems to be ignored" + ) + +def test_sshd_config_d_is_included(host: Host, defer: Defer) -> None: + marker = host.ssh('mktemp') + dropin = f"{SSHD_CONFIG_D}/99-xcp-ng-tests-include-check.conf" + + host.ssh(f"echo 'Banner {marker}' > {dropin}") + defer(lambda: host.ssh(f"rm -f {dropin}")) + + effective_config = host.ssh("/usr/sbin/sshd -T") + assert f"banner {marker}" in effective_config, ( + f"drop-in {dropin} was not picked up by sshd -T: " + f"{SSHD_CONFIG_D} seems to be ignored" + ) diff --git a/tests/system/test_ssh_fido.py b/tests/system/test_ssh_fido.py new file mode 100644 index 000000000..b78ed6bb7 --- /dev/null +++ b/tests/system/test_ssh_fido.py @@ -0,0 +1,52 @@ +import pytest + +from lib.commands import SSHCommandFailed +from lib.common import Defer +from lib.host import Host + +# FIDO/security-key (sk-*) support is new in this OpenSSH update. +# +# We have no physical FIDO token to enroll a real sk key end to end, so +# these tests prove the feature is compiled in and wired up as far as +# possible without hardware: +# - the sk-* algorithms are advertised by the client and accepted by sshd +# for pubkey authentication; +# - `ssh-keygen -t ed25519-sk` reaches actual hardware detection and fails +# with "device not found" rather than "unknown key type", proving the +# key type itself is recognized and libfido2 support is wired in. +# +# Requirements: +# - an XCP-ng host (--hosts) >= 8.3, with the OpenSSH 9.9p1 + +SK_KEY_TYPES = { + "sk-ssh-ed25519@openssh.com", + "sk-ssh-ed25519-cert-v01@openssh.com", + "sk-ecdsa-sha2-nistp256@openssh.com", + "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com", +} + +def test_fido_key_types_advertised_by_client(host: Host) -> None: + supported = set(host.ssh("ssh -Q key").splitlines()) + missing = SK_KEY_TYPES - supported + assert not missing, f"ssh client doesn't advertise expected FIDO key types: {missing}" + +def test_fido_key_types_accepted_by_server(host: Host) -> None: + output = host.ssh("sshd -T | grep '^pubkeyacceptedalgorithms '") + # look for the matching line specifically: some hosts print unrelated + # lines on every ssh session (e.g. a root-login warning) ahead of it + line = next(line for line in output.splitlines() if line.startswith("pubkeyacceptedalgorithms ")) + accepted = set(line.split(" ", 1)[1].split(',')) + missing = {"sk-ssh-ed25519@openssh.com", "sk-ecdsa-sha2-nistp256@openssh.com"} - accepted + assert not missing, f"sshd doesn't accept expected FIDO key types for pubkey auth: {missing}" + +def test_sk_key_enrollment_reaches_hardware_detection(host: Host, defer: Defer) -> None: + """ Without a token, enrollment must fail at the hardware-detection stage, not earlier. """ + keyfile = host.ssh("mktemp -u") + defer(lambda: host.ssh(f"rm -f {keyfile} {keyfile}.pub")) + + with pytest.raises(SSHCommandFailed) as exc_info: + host.ssh(f"timeout 8 ssh-keygen -t ed25519-sk -f {keyfile} -N '' -O no-touch-required") + + error = str(exc_info.value) + assert "device not found" in error, f"expected a hardware-detection failure, got: {error}" + assert "unknown key type" not in error diff --git a/tests/system/test_ssh_file_transfer.py b/tests/system/test_ssh_file_transfer.py new file mode 100644 index 000000000..efaed2b95 --- /dev/null +++ b/tests/system/test_ssh_file_transfer.py @@ -0,0 +1,73 @@ +import pytest + +import hashlib +import os +import tempfile +from pathlib import Path + +from lib.commands import local_cmd, sftp +from lib.common import Defer +from lib.host import Host + +from typing import Literal, TypeAlias, get_args + +# OpenSSH 9.0 switched scp's default wire protocol from the legacy SCP +# protocol to SFTP ('-O' restores the old protocol, still shipped for +# compatibility). Both paths, plus the sftp client itself, are exercised +# here end to end (upload then download, content checked with a checksum) +# since SFTP-based transfers are exactly what caused trouble in the past. +# +# Requirements: +# - an XCP-ng host (--hosts) >= 8.2 + +REMOTE_DIR = "/tmp" + +Protocol: TypeAlias = Literal['sftp', 'legacy'] + +def _sha256(path: Path) -> str: + with open(path, 'rb') as f: + return hashlib.sha256(f.read()).hexdigest() + +def _make_random_local_file(defer: Defer, size: int = 1_000_000) -> Path: + with tempfile.NamedTemporaryFile(prefix="xcpng-tests-ssh-transfer-", delete=False) as f: + defer(lambda: os.remove(f.name)) + f.write(os.urandom(size)) + return Path(f.name) + +def _scp(src: str, dest: str, protocol: Protocol) -> None: + opts = ['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', '-o', 'LogLevel=ERROR'] + if protocol == 'legacy': + opts.append('-O') + local_cmd(['scp'] + opts + [src, dest]) + +@pytest.mark.parametrize("protocol", get_args(Protocol)) +def test_scp_roundtrip(host: Host, defer: Defer, protocol: Protocol) -> None: + local_src = _make_random_local_file(defer) + remote_path = f"{REMOTE_DIR}/{local_src.name}" + defer(lambda: host.ssh(f"rm -f {remote_path}")) + + _scp(str(local_src), f"root@{host.hostname_or_ip}:{remote_path}", protocol) + remote_sha256 = host.ssh(f"sha256sum {remote_path}").split()[0] + assert remote_sha256 == _sha256(local_src), "uploaded file content differs from the original" + + local_dst = local_src.with_name(local_src.name + ".download") + defer(lambda: os.remove(local_dst)) + _scp(f"root@{host.hostname_or_ip}:{remote_path}", str(local_dst), protocol) + assert _sha256(local_dst) == _sha256(local_src), "downloaded file content differs from the original" + +def test_sftp_batch_roundtrip(host: Host, defer: Defer) -> None: + local_src = _make_random_local_file(defer) + remote_path = f"{REMOTE_DIR}/{local_src.name}" + defer(lambda: host.ssh(f"rm -f {remote_path}")) + + put_res = sftp(host.hostname_or_ip, [f"put {local_src} {remote_path}", "bye"]) + assert put_res.returncode == 0, put_res.stdout.decode() + remote_sha256 = host.ssh(f"sha256sum {remote_path}").split()[0] + assert remote_sha256 == _sha256(local_src), "uploaded file content differs from the original" + + local_dst = local_src.with_name(local_src.name + ".download") + defer(lambda: os.remove(local_dst)) + get_res = sftp(host.hostname_or_ip, [f"get {remote_path} {local_dst}", "bye"]) + assert get_res.returncode == 0, get_res.stdout.decode() + assert _sha256(local_dst) == _sha256(local_src), "downloaded file content differs from the original" diff --git a/tests/system/test_ssh_post_quantum.py b/tests/system/test_ssh_post_quantum.py new file mode 100644 index 000000000..83f5f4227 --- /dev/null +++ b/tests/system/test_ssh_post_quantum.py @@ -0,0 +1,28 @@ +import pytest + +from lib.commands import ssh_with_result +from lib.host import Host + +# Hybrid post-quantum key exchange (ML-KEM, e.g. mlkem768x25519-sha256) is +# new in OpenSSH 9.9p1 (reference build openssh-9.9p1-27.2.xcpng8.3); the +# OpenSSH build shipped before it had no ML-KEM support. +# This test proves it is actually negotiated by default, not just compiled +# in and available on request (see test_ssh_algorithms.py for per-algorithm +# coverage). +# +# Requirements: +# - an XCP-ng host (--hosts) >= 8.3, with openssh >= 9.9p1 installed + +def _negotiated_kex_algorithm(host: Host) -> str: + # multiplexing must be off, otherwise a shared control connection could + # be reused and its (already negotiated) algorithm silently returned. + result = ssh_with_result(host.hostname_or_ip, 'true', options=['-v'], multiplexing=False) + for line in result.ssherr.splitlines(): + if line.startswith('debug1: kex: algorithm:'): + return line.rsplit(':', 1)[1].strip() + pytest.fail(f"could not find the negotiated KEX algorithm in ssh -v output:\n{result.ssherr}") + +def test_pq_kex_preferred_by_default(host: Host) -> None: + """ The default KEX negotiated must be the hybrid post-quantum one. """ + algo = _negotiated_kex_algorithm(host) + assert algo.startswith("mlkem"), f"expected a hybrid post-quantum KEX by default, got {algo}" diff --git a/tests/system/test_sshd_config_include.py b/tests/system/test_sshd_config_include.py deleted file mode 100644 index 0d9f3cfd0..000000000 --- a/tests/system/test_sshd_config_include.py +++ /dev/null @@ -1,28 +0,0 @@ -from lib.common import Defer -from lib.host import Host - -# Regression test for sshd silently ignoring /etc/ssh/sshd_config.d/*.conf -# drop-in files (missing "Include" directive in the packaged sshd_config). -# -# A drop-in file is created, setting a directive (a pre-auth Banner), that -# isn't set anywhere else. `sshd -T` dumps the configuration as sshd itself -# would apply it, so finding the directive there proves the drop-in directory -# is actually included. -# -# Requirements: -# - an XCP-ng host (--hosts) >= 8.2 - -SSHD_CONFIG_D = "/etc/ssh/sshd_config.d" - -def test_sshd_config_d_is_included(host: Host, defer: Defer) -> None: - marker = host.ssh('mktemp') - dropin = f"{SSHD_CONFIG_D}/99-xcp-ng-tests-include-check.conf" - - host.ssh(f"echo 'Banner {marker}' > {dropin}") - defer(lambda: host.ssh(f"rm -f {dropin}")) - - effective_config = host.ssh("/usr/sbin/sshd -T") - assert f"banner {marker}" in effective_config, ( - f"drop-in {dropin} was not picked up by sshd -T: " - f"{SSHD_CONFIG_D} seems to be ignored" - )