mtls: add test

Author: flobz

README.md

@@ -113,7 +113,10 @@ Run hawkBit docker container:
 $ docker pull hawkbit/hawkbit-update-server
 $ docker run -d --name hawkbit -p 8080:8080 hawkbit/hawkbit-update-server \
     --hawkbit.server.security.dos.filter.enabled=false \
-    --hawkbit.server.security.dos.maxStatusEntriesPerAction=-1
+    --hawkbit.server.security.dos.maxStatusEntriesPerAction=-1 \
+    --hawkbit.dmf.rabbitmq.enabled=false \
+    --server.forward-headers-strategy=NATIVE \
+    --hawkbit.artifact.url.protocols.download-http.protocol=<https or http>
 ```
 
 Run test suite:

test/conftest.py

@@ -9,7 +9,8 @@
 import pytest
 
 from hawkbit_mgmt import HawkbitMgmtTestClient, HawkbitError
-from helper import run_pexpect, available_port
+from helper import run_pexpect, available_port, run
+from mtls_conf import MtlsConfig
 
 def pytest_addoption(parser):
     """Register custom argparse-style options."""
@@ -103,8 +104,11 @@ def _adjust_config(options={'client': {}}, remove={}, add_trailing_space=False):
                 adjusted_config.set(section, key, value)
 
         # remove
-        for section, option in remove.items():
-            adjusted_config.remove_option(section, option)
+        for section, options in remove.items():
+            if type(options) is str:
+                options = [options]
+            for option in options:
+                adjusted_config.remove_option(section, option)
 
         # add trailing space
         if add_trailing_space:
@@ -209,6 +213,24 @@ def rauc_dbus_install_success(rauc_bundle):
     assert proc.terminate(force=True)
     proc.expect(pexpect.EOF)
 
+@pytest.fixture
+def rauc_dbus_install_success_mtls(rauc_bundle,tmp_path_factory):
+    """
+    Creates a RAUC D-Bus dummy interface on the SessionBus mimicing a successful installation on
+    InstallBundle().
+    """
+    import pexpect
+
+    proc = run_pexpect(f'{sys.executable} -m rauc_dbus_dummy --tmp-dir {tmp_path_factory.getbasetemp()} --mtls {rauc_bundle}',
+                      cwd=os.path.dirname(__file__))
+    proc.expect('Interface published')
+
+    yield
+
+    assert proc.isalive()
+    assert proc.terminate(force=True)
+    proc.expect(pexpect.EOF)
+
 @pytest.fixture
 def rauc_dbus_install_failure(rauc_bundle):
     """
@@ -242,11 +264,26 @@ def nginx_config(tmp_path_factory):
 
 http {{
     access_log /dev/null;
+    map $ssl_client_s_dn $ssl_client_s_dn_cn {{
+        default "";
+        ~CN=(?<CN>[^,]+) $CN;
+    }}
 
+    {server}
+}}
+  """
+    http_server = """
     server {{
         listen {port};
         listen [::]:{port};
-
+        
+        client_body_temp_path {tmp_dir};
+        proxy_temp_path {tmp_dir};
+        fastcgi_temp_path {tmp_dir};
+        uwsgi_temp_path {tmp_dir};
+        scgi_temp_path {tmp_dir};
+        {server_options}
+        
         location / {{
             proxy_pass http://localhost:8080;
             {location_options}
@@ -258,12 +295,63 @@ def nginx_config(tmp_path_factory):
             sub_filter_once off;
         }}
     }}
-}}"""
-
-    def _nginx_config(port, location_options):
-        proxy_config = tmp_path_factory.mktemp('nginx') / 'nginx.conf'
-        location_options = ( f'{key} {value};' for key, value in location_options.items())
-        proxy_config_str = config_template.format(port=port, location_options=" ".join(location_options))
+    """
+    mtls_server = """
+      server {{
+        listen {port} ssl;
+        listen [::]:{port} ssl;
+
+        client_body_temp_path {tmp_dir};
+        proxy_temp_path {tmp_dir};
+        fastcgi_temp_path {tmp_dir};
+        uwsgi_temp_path {tmp_dir};
+        scgi_temp_path {tmp_dir};
+        server_name localhost;
+        {server_options}
+     
+        
+        ssl_verify_client      optional;
+        ssl_verify_depth 3; 
+        # For devices that is using device integration API, 
+        # Mutual TLS is required.
+        location ~*/.*/controller/ {{
+            if ($ssl_client_verify != SUCCESS) {{
+               return 403;
+            }}
+            
+            proxy_pass http://localhost:8080;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto https;
+            proxy_set_header X-Forwarded-Port {port};
+            proxy_set_header X-Forwarded-Protocol https;
+  
+            # Client certificate Common Name and Issuer Hash is required
+            # for auth in hawkbit. 
+            proxy_set_header X-Ssl-Client-Cn $ssl_client_s_dn_cn;
+    
+            # These are required for clients to upload and download software. 
+            proxy_request_buffering off;
+            client_max_body_size 1000m;
+            {location_options}
+        }}
+    }}
+    """
+    def dict_to_nginx_option(option:dict):
+        if not option:
+            return ""
+        option = ( f'{key} {value};' for key, value in option.items())
+        return " ".join(option)
+    def _nginx_config(port, location_options, server_options, mtls):
+        nginx_tmp_dir=tmp_path_factory.mktemp('nginx')
+        server_config = mtls_server if mtls else http_server
+        server_config_str = server_config.format(port=port,location_options=dict_to_nginx_option(location_options),
+                                                 server_options=dict_to_nginx_option(server_options),
+                                                 tmp_dir=nginx_tmp_dir)
+
+        proxy_config_str = config_template.format(port=port,server=server_config_str, tmp_dir=nginx_tmp_dir)
+        proxy_config = nginx_tmp_dir / 'nginx.conf'
         proxy_config.write_text(proxy_config_str)
         return proxy_config
 
@@ -272,18 +360,16 @@ def _nginx_config(port, location_options):
 @pytest.fixture(scope='session')
 def nginx_proxy(nginx_config):
     """
-    Runs an nginx rate liming proxy, limiting download speeds to 70 KB/s. HTTP requests are
+    Runs an nginx proxy. HTTP requests are
     forwarded to port 8080 (default port of the docker hawkBit instance). Returns the port the
-    proxy is running on. This port can be set in the rauc-hawkbit-updater config to rate limit its
-    HTTP requests.
+    proxy is running on. This port can be set in the rauc-hawkbit-updater config.
     """
     import pexpect
 
     procs = []
-
-    def _nginx_proxy(options):
+    def _nginx_proxy(options, server_options=None, mtls=False):
         port = available_port()
-        proxy_config = nginx_config(port, options)
+        proxy_config = nginx_config(port, options, server_options, mtls)
 
         try:
             proc = run_pexpect(f'nginx -c {proxy_config} -p .', timeout=None)
@@ -332,3 +418,40 @@ def partial_download_port(nginx_proxy):
         'limit_rate': '70k',
     }
     return nginx_proxy(location_options)
+
+@pytest.fixture
+def mtls_download_port(nginx_proxy, mtls_certificates):
+    """
+    Runs an nginx proxy. HTTPS requests are forwarded to port 8080
+    (default port of the docker hawkBit instance). Returns the port the proxy is running on. This
+    port can be set in the rauc-hawkbit-updater config to test partial downloads.
+    """
+    mtls_cert = mtls_certificates()
+    hash_issuer = mtls_cert.get_issuer_hash()
+    location_options = {"proxy_set_header X-Ssl-Issuer-Hash-1": hash_issuer}
+    server_options = {
+        "ssl_certificate": mtls_cert.ca_cert,
+        "ssl_certificate_key": mtls_cert.ca_key,
+        "ssl_client_certificate": mtls_cert.ca_cert
+    }
+    return nginx_proxy(location_options, server_options, mtls=True)
+
+
+
+@pytest.fixture
+def mtls_config(tmp_path_factory):
+    return MtlsConfig(tmp_path_factory.getbasetemp())
+
+@pytest.fixture
+def mtls_certificates(mtls_config, tmp_path_factory, hawkbit_target_added):
+    """
+    Generate CA cert and key if they don't exist and also generate specific client cert and key
+    for the Hawkbit controller id as Common Name.
+    """
+    def _mtls_certificates():
+        out, err, exitcode = run(f'{os.path.dirname(__file__)}/gen_key.sh {mtls_config.certs_dir} {hawkbit_target_added}', timeout=20)
+        assert exitcode == 0
+        assert mtls_config.ca_cert_exist()
+        assert mtls_config.client_cert_exist()
+        return mtls_config
+    return _mtls_certificates

test/gen_key.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+CERT_DIR="${1}"
+CONTROLLER_ID="${2}"
+CA_CERT="root-ca.crt"
+CA_KEY="root-ca.key"
+CA_CSR="root-csr.pem"
+CERT_CONFIG='
+  [client]
+  basicConstraints = CA:FALSE
+  nsCertType = client, email
+  nsComment = "Local Test Client Certificate"
+  subjectKeyIdentifier = hash
+  authorityKeyIdentifier = keyid,issuer
+  keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
+  extendedKeyUsage = clientAuth
+'
+
+mkdir -p ${CERT_DIR}
+cd ${CERT_DIR}
+if [ ! -f "${CA_CERT}" ]; then
+  echo "Development CA"
+  openssl req -newkey rsa -keyout ${CA_KEY} -out ${CA_CSR} -subj "/O=Test/CN=localhost" --nodes
+  openssl req -x509 -sha256 -new -nodes -key ${CA_KEY} -days 3650 -out ${CA_CERT} -subj '/CN=localhost'
+fi
+if [ -n "${CONTROLLER_ID}" ]; then
+  openssl genrsa -out "client.key" 4096
+  openssl req -new -key "client.key" -out "client.csr" -sha256 -subj "/CN=${CONTROLLER_ID}"
+
+  openssl x509 -req -days 750 -in "client.csr" -sha256 -CA ${CA_CERT} -CAkey ${CA_KEY} -CAcreateserial -out "client.crt" -extensions client -extfile <(printf ${CERT_CONFIG})
+  openssl x509 -in client.crt -issuer_hash -noout > issuer_hash.txt
+fi

test/mtls_conf.py

@@ -0,0 +1,20 @@
+import os
+
+
+class MtlsConfig:
+    def __init__(self, cert_dir_location):
+        self.certs_dir= str(cert_dir_location) + "/certs/"
+        self.ca_cert= self.certs_dir + "root-ca.crt"
+        self.ca_key= self.certs_dir + "root-ca.key"
+        self.ca_csr= self.certs_dir + "root-csr.pem"
+        self.client_cert= self.certs_dir + "client.crt"
+        self.client_key= self.certs_dir + "client.key"
+        self.issuer_hash = self. certs_dir + "issuer_hash.txt"
+    def client_cert_exist(self):
+        return os.path.isfile(self.client_cert)
+
+    def ca_cert_exist(self):
+        return os.path.isfile(self.ca_cert)
+
+    def get_issuer_hash(self):
+        return open(self.issuer_hash, "r").readline().strip()

test/rauc_dbus_dummy.py

@@ -10,6 +10,8 @@
 from pydbus.generic import signal
 import requests
 
+from mtls_conf import MtlsConfig
+
 
 class Installer:
     """
@@ -28,13 +30,15 @@ class Installer:
     Completed = signal()
     PropertiesChanged = signal()
 
-    def __init__(self, bundle, completed_code=0):
+    def __init__(self, bundle, mtls,tmp_path, completed_code=0):
         self._bundle = bundle
         self._completed_code = completed_code
 
         self._operation = 'idle'
         self._last_error = ''
         self._progress = 0, '', 1
+        self._mtls = mtls
+        self.tmp_path = tmp_path
 
     def InstallBundle(self, source, args):
         def mimic_install():
@@ -103,7 +107,7 @@ def _get_bundle_sha1(bundle):
         return sha1.hexdigest()
 
     @staticmethod
-    def _get_http_bundle_sha1(url, auth_header):
+    def _get_http_bundle_sha1(url, auth_header, cert, verify):
         """Download file from URL using HTTP range requests and compute its sha1 checksum."""
         sha1 = hashlib.sha1()
         headers = auth_header
@@ -112,7 +116,7 @@ def _get_http_bundle_sha1(url, auth_header):
         offset = 0
         while True:
             headers['Range'] = f'bytes={offset}-{offset + range_size - 1}'
-            r = requests.get(url, headers=headers)
+            r = requests.get(url, headers=headers, cert=cert, verify=verify)
             try:
                 r.raise_for_status()
                 sha1.update(r.content)
@@ -130,17 +134,24 @@ def _check_install_requirements(self, source, args):
         Check that required headers are set, bundle is accessible (HTTP or locally) and its
         checksum matches.
         """
+        headers = {}
+        verify = False
+        if self._mtls:
+            mtls_conf = MtlsConfig(self.tmp_path)
+            cert = (mtls_conf.client_cert, mtls_conf.client_key)
+        else:
+            cert = None
         if 'http-headers' in args:
-            assert len(args['http-headers']) == 1
-
-            [auth_header] = args['http-headers']
-            key, value = auth_header.split(': ', maxsplit=1)
-            http_bundle_sha1 = self._get_http_bundle_sha1(source, {key: value})
+            if len(args['http-headers']) == 1:
+                [auth_header] = args['http-headers']
+                headers = dict([auth_header.split(': ', maxsplit=1)])
+            elif not self._mtls:
+                raise Exception("No headers in args")
+            verify = args['tls-no-verify'] is False
+        if source.startswith("http"):
+            http_bundle_sha1 = self._get_http_bundle_sha1(source, headers, cert, verify=verify)
             assert http_bundle_sha1 == self._get_bundle_sha1(self._bundle)
 
-            # assume ssl_verify=false is set in test setup
-            assert args['tls-no-verify'] is True
-
         else:
             # check bundle checksum matches expected checksum
             assert self._get_bundle_sha1(source) == self._get_bundle_sha1(self._bundle)
@@ -193,11 +204,15 @@ def BootSlot(self):
     parser.add_argument('bundle', help='Expected RAUC bundle')
     parser.add_argument('--completed-code', type=int, default=0,
                         help='Code to emit as D-Bus Completed signal')
+    parser.add_argument('--tmp-dir', type=str, default=None,
+                        help='Test tmp dir')
+    parser.add_argument('--mtls', action='store_true',
+                        help='Use MTLS protocols')
     args = parser.parse_args()
 
     loop = GLib.MainLoop()
     bus = SessionBus()
-    installer = Installer(args.bundle, args.completed_code)
+    installer = Installer(args.bundle, args.mtls, args.tmp_dir, args.completed_code)
     with bus.publish('de.pengutronix.rauc', ('/', installer)):
         print('Interface published')
         loop.run()