add SSH feature coverage for the OpenSSH update

tests/system/test_ssh_algorithms.py

@@ -0,0 +1,186 @@
+import pytest
+
+from contextlib import nullcontext
+
+from lib.commands import SSHCommandFailed, ssh
+from lib.host import Host
+
+# Exhaustive coverage of the KEX/cipher/MAC/host-key algorithms OpenSSH
+# supports, hardcoded from `ssh -Q kex|cipher|mac|key` on the reference
+# build (openssh-9.9p1-27.2.xcpng8.3). The lists are deliberately not
+# queried live: the point of test_*_list_matches_hardcoded is to fail the
+# day the compiled-in algorithm support changes, forcing a conscious
+# update of this file instead of the change going unnoticed.
+#
+# Requirements:
+# - an XCP-ng host (--hosts) >= 8.3, with the OpenSSH 9.9p1
+
+# algorithm -> enabled by default (sshd -T) on the reference build
+KEX_ALGORITHMS = {
+    # hybrid post-quantum, new in this OpenSSH update
+    "mlkem1024nistp384-sha384": True,
+    "mlkem768x25519-sha256": True,
+    "mlkem768nistp256-sha256": True,
+    "sntrup761x25519-sha512": True,
+    "sntrup761x25519-sha512@openssh.com": True,
+    # classical, enabled
+    "curve25519-sha256": True,
+    "curve25519-sha256@libssh.org": True,
+    "ecdh-sha2-nistp521": True,
+    "ecdh-sha2-nistp384": True,
+    "ecdh-sha2-nistp256": True,
+    "diffie-hellman-group16-sha512": True,
+    "diffie-hellman-group18-sha512": True,
+    # compiled in, but disabled
+    "diffie-hellman-group1-sha1": False,
+    "diffie-hellman-group14-sha1": False,
+    "diffie-hellman-group14-sha256": False,
+    "diffie-hellman-group-exchange-sha1": False,
+    "diffie-hellman-group-exchange-sha256": False,
+}
+
+CIPHERS = {
+    "chacha20-poly1305@openssh.com": True,
+    "aes256-gcm@openssh.com": True,
+    "aes128-gcm@openssh.com": True,
+    "aes256-ctr": True,
+    "aes128-ctr": True,
+    "3des-cbc": False,
+    "aes128-cbc": False,
+    "aes192-cbc": False,
+    "aes256-cbc": False,
+    "aes192-ctr": False,
+}
+
+MACS = {
+    "hmac-sha2-512-etm@openssh.com": True,
+    "hmac-sha2-256-etm@openssh.com": True,
+    "umac-128-etm@openssh.com": True,
+    "hmac-sha2-512": True,
+    "hmac-sha2-256": True,
+    "umac-128@openssh.com": True,
+    "hmac-sha1": False,
+    "hmac-sha1-96": False,
+    "hmac-md5": False,
+    "hmac-md5-96": False,
+    "umac-64@openssh.com": False,
+    "hmac-sha1-etm@openssh.com": False,
+    "hmac-sha1-96-etm@openssh.com": False,
+    "hmac-md5-etm@openssh.com": False,
+    "hmac-md5-96-etm@openssh.com": False,
+    "umac-64-etm@openssh.com": False,
+}
+
+# Server host identity algorithms actually exercisable end to end: the host
+# only carries one host key per type (ed25519, ecdsa on nistp256, rsa), so
+# only algorithms with matching key material are forced here. Certificate
+# variants would need a CA, and the sk-* types from `ssh -Q key` are not
+# host identity keys at all (they're for user authentication via a
+# hardware security key, see test_ssh_fido.py) so they don't belong in a
+# HostKeyAlgorithms test.
+HOSTKEY_ALGORITHMS = {
+    "ssh-ed25519": True,
+    "ecdsa-sha2-nistp256": True,
+    "rsa-sha2-256": True,
+    "rsa-sha2-512": True,
+    "ssh-rsa": False,  # raw SHA-1 RSA signature, disabled by crypto-policy
+}
+
+# Full compiled-in key type support (`ssh -Q key`), used only to detect if
+# that list ever drifts (e.g. FIDO/security-key support silently regressing,
+# as it did between the previous and this OpenSSH build).
+ALL_COMPILED_KEY_TYPES = {
+    "ssh-ed25519",
+    "ssh-ed25519-cert-v01@openssh.com",
+    "sk-ssh-ed25519@openssh.com",
+    "sk-ssh-ed25519-cert-v01@openssh.com",
+    "ecdsa-sha2-nistp256",
+    "ecdsa-sha2-nistp256-cert-v01@openssh.com",
+    "ecdsa-sha2-nistp384",
+    "ecdsa-sha2-nistp384-cert-v01@openssh.com",
+    "ecdsa-sha2-nistp521",
+    "ecdsa-sha2-nistp521-cert-v01@openssh.com",
+    "sk-ecdsa-sha2-nistp256@openssh.com",
+    "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com",
+    "ssh-rsa",
+    "ssh-rsa-cert-v01@openssh.com",
+}
+
+def _assert_list_matches(host: Host, query: str, expected: set) -> None:
+    actual = {line for line in host.ssh(f"ssh -Q {query}").splitlines() if line and ' ' not in line}
+    assert actual == expected, (
+        f"`ssh -Q {query}` no longer matches the hardcoded list in this test file "
+        f"(missing={sorted(expected - actual)}, new={sorted(actual - expected)}); "
+        "update the hardcoded list after reviewing the change"
+    )
+
+def test_kex_algorithms_list_matches_hardcoded(host: Host) -> None:
+    _assert_list_matches(host, "kex", set(KEX_ALGORITHMS))
+
+def test_ciphers_list_matches_hardcoded(host: Host) -> None:
+    _assert_list_matches(host, "cipher", set(CIPHERS))
+
+def test_macs_list_matches_hardcoded(host: Host) -> None:
+    _assert_list_matches(host, "mac", set(MACS))
+
+def test_key_types_list_matches_hardcoded(host: Host) -> None:
+    _assert_list_matches(host, "key", ALL_COMPILED_KEY_TYPES)
+
+def _force_algorithm(host: Host, option: str, algo: str, *, extra_options: list[str] = []) -> None:
+    # multiplexing must be off: a shared control connection would reuse the
+    # algorithm negotiated by whichever call created it, silently ignoring
+    # the -o option on every later call.
+    ssh(host.hostname_or_ip, 'true', options=['-o', f'{option}={algo}'] + extra_options, multiplexing=False)
+
+# mlkem768nistp256-sha256 and mlkem1024nistp384-sha384 aren't part of
+# upstream OpenSSH (see openssh-10.0-mlkem-nist.patch): they're carried by
+# RHEL-family builds (RHEL, CentOS, Alma, and this XCP-ng build) for FIPS
+# compliance, but not by other builds, including the machine running these
+# tests. There's no second RHEL-family host available to interoperate with
+# either, so unlike every other algorithm here, these two are exercised in
+# loopback on the host itself (which does understand its own names). That
+# can't reach an authenticated session (root has no key to log into
+# itself), so instead we assert the exact requested algorithm was the one
+# negotiated.
+SPECIFIC_KEX_NAMES = {"mlkem768nistp256-sha256", "mlkem1024nistp384-sha384"}
+
+def _negotiated_kex_in_loopback(host: Host, algo: str) -> str:
+    # the trailing "; true" keeps the remote command's exit code at 0
+    # (the inner loopback ssh fails at authentication, not at key exchange)
+    # so we can just inspect its output instead of juggling SSHCommandFailed.
+    output = host.ssh(
+        "ssh -v -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
+        f"-o ControlMaster=no -o KexAlgorithms={algo} localhost true 2>&1; true"
+    )
+    for line in output.splitlines():
+        if line.startswith("debug1: kex: algorithm:"):
+            return line.rsplit(':', 1)[1].strip()
+    pytest.fail(f"could not find the negotiated KEX algorithm in loopback ssh -v output:\n{output}")
+
+@pytest.mark.parametrize("algo,enabled", KEX_ALGORITHMS.items(), ids=list(KEX_ALGORITHMS))
+def test_kex_algorithm(host: Host, algo: str, enabled: bool) -> None:
+    if algo in SPECIFIC_KEX_NAMES:
+        assert enabled
+        assert _negotiated_kex_in_loopback(host, algo) == algo
+        return
+    with pytest.raises(SSHCommandFailed) if not enabled else nullcontext():
+        _force_algorithm(host, "KexAlgorithms", algo)
+
+@pytest.mark.parametrize("algo,enabled", CIPHERS.items(), ids=list(CIPHERS))
+def test_cipher_algorithm(host: Host, algo: str, enabled: bool) -> None:
+    with pytest.raises(SSHCommandFailed) if not enabled else nullcontext():
+        _force_algorithm(host, "Ciphers", algo)
+
+@pytest.mark.parametrize("algo,enabled", MACS.items(), ids=list(MACS))
+def test_mac_algorithm(host: Host, algo: str, enabled: bool) -> None:
+    # MACs are only negotiated for non-AEAD ciphers (AEAD ciphers like the
+    # default chacha20-poly1305/aes-gcm have their MAC built in, making the
+    # MACs option moot), so a non-AEAD cipher is pinned to force the point.
+    non_aead_cipher = ['-o', 'Ciphers=aes256-ctr']
+    with pytest.raises(SSHCommandFailed) if not enabled else nullcontext():
+        _force_algorithm(host, "MACs", algo, extra_options=non_aead_cipher)
+
+@pytest.mark.parametrize("algo,enabled", HOSTKEY_ALGORITHMS.items(), ids=list(HOSTKEY_ALGORITHMS))
+def test_hostkey_algorithm(host: Host, algo: str, enabled: bool) -> None:
+    with pytest.raises(SSHCommandFailed) if not enabled else nullcontext():
+        _force_algorithm(host, "HostKeyAlgorithms", algo)

tests/system/test_ssh_config_include.py

@@ -0,0 +1,46 @@
+import random
+
+from lib.common import Defer
+from lib.host import Host
+
+# Regression test for the ssh client and sshd silently ignoring drop-in
+# configuration files (missing "Include" directive in the packaged config).
+# Covers both /etc/ssh/ssh_config.d/*.conf (client) and
+# /etc/ssh/sshd_config.d/*.conf (server).
+#
+# Each test creates a drop-in with a directive not set elsewhere, then uses
+# `ssh -G` or `sshd -T` to verify that the corresponding directory is included.
+#
+# Requirements:
+# - an XCP-ng host (--hosts) >= 8.2
+# - the ssh_config.d test additionally requires the OpenSSH security update:
+#   the Include directive isn't in the previous package's ssh_config at all
+
+SSH_CONFIG_D = "/etc/ssh/ssh_config.d"
+SSHD_CONFIG_D = "/etc/ssh/sshd_config.d"
+
+def test_ssh_config_d_is_included(host: Host, defer: Defer) -> None:
+    marker = str(random.randint(10000, 99999))
+    dropin = f"{SSH_CONFIG_D}/99-xcp-ng-tests-include-check.conf"
+
+    host.ssh(f"echo 'ConnectTimeout {marker}' > {dropin}")
+    defer(lambda: host.ssh(f"rm -f {dropin}"))
+
+    effective_config = host.ssh("ssh -G localhost")
+    assert f"connecttimeout {marker}" in effective_config, (
+        f"drop-in {dropin} was not picked up by ssh -G: "
+        f"{SSH_CONFIG_D} seems to be ignored"
+    )
+
+def test_sshd_config_d_is_included(host: Host, defer: Defer) -> None:
+    marker = host.ssh('mktemp')
+    dropin = f"{SSHD_CONFIG_D}/99-xcp-ng-tests-include-check.conf"
+
+    host.ssh(f"echo 'Banner {marker}' > {dropin}")
+    defer(lambda: host.ssh(f"rm -f {dropin}"))
+
+    effective_config = host.ssh("/usr/sbin/sshd -T")
+    assert f"banner {marker}" in effective_config, (
+        f"drop-in {dropin} was not picked up by sshd -T: "
+        f"{SSHD_CONFIG_D} seems to be ignored"
+    )

tests/system/test_ssh_fido.py

@@ -0,0 +1,52 @@
+import pytest
+
+from lib.commands import SSHCommandFailed
+from lib.common import Defer
+from lib.host import Host
+
+# FIDO/security-key (sk-*) support is new in this OpenSSH update.
+#
+# We have no physical FIDO token to enroll a real sk key end to end, so
+# these tests prove the feature is compiled in and wired up as far as
+# possible without hardware:
+# - the sk-* algorithms are advertised by the client and accepted by sshd
+#   for pubkey authentication;
+# - `ssh-keygen -t ed25519-sk` reaches actual hardware detection and fails
+#   with "device not found" rather than "unknown key type", proving the
+#   key type itself is recognized and libfido2 support is wired in.
+#
+# Requirements:
+# - an XCP-ng host (--hosts) >= 8.3, with the OpenSSH 9.9p1
+
+SK_KEY_TYPES = {
+    "sk-ssh-ed25519@openssh.com",
+    "sk-ssh-ed25519-cert-v01@openssh.com",
+    "sk-ecdsa-sha2-nistp256@openssh.com",
+    "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com",
+}
+
+def test_fido_key_types_advertised_by_client(host: Host) -> None:
+    supported = set(host.ssh("ssh -Q key").splitlines())
+    missing = SK_KEY_TYPES - supported
+    assert not missing, f"ssh client doesn't advertise expected FIDO key types: {missing}"
+
+def test_fido_key_types_accepted_by_server(host: Host) -> None:
+    output = host.ssh("sshd -T | grep '^pubkeyacceptedalgorithms '")
+    # look for the matching line specifically: some hosts print unrelated
+    # lines on every ssh session (e.g. a root-login warning) ahead of it
+    line = next(line for line in output.splitlines() if line.startswith("pubkeyacceptedalgorithms "))
+    accepted = set(line.split(" ", 1)[1].split(','))
+    missing = {"sk-ssh-ed25519@openssh.com", "sk-ecdsa-sha2-nistp256@openssh.com"} - accepted
+    assert not missing, f"sshd doesn't accept expected FIDO key types for pubkey auth: {missing}"
+
+def test_sk_key_enrollment_reaches_hardware_detection(host: Host, defer: Defer) -> None:
+    """ Without a token, enrollment must fail at the hardware-detection stage, not earlier. """
+    keyfile = host.ssh("mktemp -u")
+    defer(lambda: host.ssh(f"rm -f {keyfile} {keyfile}.pub"))
+
+    with pytest.raises(SSHCommandFailed) as exc_info:
+        host.ssh(f"timeout 8 ssh-keygen -t ed25519-sk -f {keyfile} -N '' -O no-touch-required")
+
+    error = str(exc_info.value)
+    assert "device not found" in error, f"expected a hardware-detection failure, got: {error}"
+    assert "unknown key type" not in error

tests/system/test_ssh_file_transfer.py

@@ -0,0 +1,73 @@
+import pytest
+
+import hashlib
+import os
+import tempfile
+from pathlib import Path
+
+from lib.commands import local_cmd, sftp
+from lib.common import Defer
+from lib.host import Host
+
+from typing import Literal, TypeAlias, get_args
+
+# OpenSSH 9.0 switched scp's default wire protocol from the legacy SCP
+# protocol to SFTP ('-O' restores the old protocol, still shipped for
+# compatibility). Both paths, plus the sftp client itself, are exercised
+# here end to end (upload then download, content checked with a checksum)
+# since SFTP-based transfers are exactly what caused trouble in the past.
+#
+# Requirements:
+# - an XCP-ng host (--hosts) >= 8.2
+
+REMOTE_DIR = "/tmp"
+
+Protocol: TypeAlias = Literal['sftp', 'legacy']
+
+def _sha256(path: Path) -> str:
+    with open(path, 'rb') as f:
+        return hashlib.sha256(f.read()).hexdigest()
+
+def _make_random_local_file(defer: Defer, size: int = 1_000_000) -> Path:
+    with tempfile.NamedTemporaryFile(prefix="xcpng-tests-ssh-transfer-", delete=False) as f:
+        defer(lambda: os.remove(f.name))
+        f.write(os.urandom(size))
+    return Path(f.name)
+
+def _scp(src: str, dest: str, protocol: Protocol) -> None:
+    opts = ['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no',
+            '-o', 'UserKnownHostsFile=/dev/null', '-o', 'LogLevel=ERROR']
+    if protocol == 'legacy':
+        opts.append('-O')
+    local_cmd(['scp'] + opts + [src, dest])
+
+@pytest.mark.parametrize("protocol", get_args(Protocol))
+def test_scp_roundtrip(host: Host, defer: Defer, protocol: Protocol) -> None:
+    local_src = _make_random_local_file(defer)
+    remote_path = f"{REMOTE_DIR}/{local_src.name}"
+    defer(lambda: host.ssh(f"rm -f {remote_path}"))
+
+    _scp(str(local_src), f"root@{host.hostname_or_ip}:{remote_path}", protocol)
+    remote_sha256 = host.ssh(f"sha256sum {remote_path}").split()[0]
+    assert remote_sha256 == _sha256(local_src), "uploaded file content differs from the original"
+
+    local_dst = local_src.with_name(local_src.name + ".download")
+    defer(lambda: os.remove(local_dst))
+    _scp(f"root@{host.hostname_or_ip}:{remote_path}", str(local_dst), protocol)
+    assert _sha256(local_dst) == _sha256(local_src), "downloaded file content differs from the original"
+
+def test_sftp_batch_roundtrip(host: Host, defer: Defer) -> None:
+    local_src = _make_random_local_file(defer)
+    remote_path = f"{REMOTE_DIR}/{local_src.name}"
+    defer(lambda: host.ssh(f"rm -f {remote_path}"))
+
+    put_res = sftp(host.hostname_or_ip, [f"put {local_src} {remote_path}", "bye"])
+    assert put_res.returncode == 0, put_res.stdout.decode()
+    remote_sha256 = host.ssh(f"sha256sum {remote_path}").split()[0]
+    assert remote_sha256 == _sha256(local_src), "uploaded file content differs from the original"
+
+    local_dst = local_src.with_name(local_src.name + ".download")
+    defer(lambda: os.remove(local_dst))
+    get_res = sftp(host.hostname_or_ip, [f"get {remote_path} {local_dst}", "bye"])
+    assert get_res.returncode == 0, get_res.stdout.decode()
+    assert _sha256(local_dst) == _sha256(local_src), "downloaded file content differs from the original"

tests/system/test_ssh_post_quantum.py

@@ -0,0 +1,28 @@
+import pytest
+
+from lib.commands import ssh_with_result
+from lib.host import Host
+
+# Hybrid post-quantum key exchange (ML-KEM, e.g. mlkem768x25519-sha256) is
+# new in OpenSSH 9.9p1 (reference build openssh-9.9p1-27.2.xcpng8.3); the
+# OpenSSH build shipped before it had no ML-KEM support.
+# This test proves it is actually negotiated by default, not just compiled
+# in and available on request (see test_ssh_algorithms.py for per-algorithm
+# coverage).
+#
+# Requirements:
+# - an XCP-ng host (--hosts) >= 8.3, with openssh >= 9.9p1 installed
+
+def _negotiated_kex_algorithm(host: Host) -> str:
+    # multiplexing must be off, otherwise a shared control connection could
+    # be reused and its (already negotiated) algorithm silently returned.
+    result = ssh_with_result(host.hostname_or_ip, 'true', options=['-v'], multiplexing=False)
+    for line in result.ssherr.splitlines():
+        if line.startswith('debug1: kex: algorithm:'):
+            return line.rsplit(':', 1)[1].strip()
+    pytest.fail(f"could not find the negotiated KEX algorithm in ssh -v output:\n{result.ssherr}")
+
+def test_pq_kex_preferred_by_default(host: Host) -> None:
+    """ The default KEX negotiated must be the hybrid post-quantum one. """
+    algo = _negotiated_kex_algorithm(host)
+    assert algo.startswith("mlkem"), f"expected a hybrid post-quantum KEX by default, got {algo}"