feat(madmail): add test-madmail command and E2E tests, with some selected madmail tests run by default

Author: hpk42

src/cmlxc/cli.py

@@ -1,7 +1,7 @@
 """cmlxc -- Manage local chatmail relay containers via Incus.
 
 Standard workflow:
-init -> deploy-cmdeploy/deploy-madmail -> test-cmdeploy/test-mini.
+init -> deploy-cmdeploy/deploy-madmail -> test-cmdeploy/test-madmail/test-mini.
 """
 
 import argparse
@@ -289,6 +289,56 @@ def test_cmdeploy_cmd(args, out):
     return driver.run_tests(second_domain=second_domain)
 
 
+# -------------------------------------------------------------------
+# test-madmail
+# -------------------------------------------------------------------
+
+
+def test_madmail_cmd_options(parser):
+    _add_test_relay_args(parser)
+    parser.add_argument(
+        "--cool",
+        action="store_true",
+        help="Minimal colored output (pass --cool to madmail test suite).",
+    )
+    parser.add_argument(
+        "--simple",
+        action="store_true",
+        default=True,
+        help="Run only simpler tests (1-6). Enabled by default.",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_false",
+        dest="simple",
+        help="Run all tests (disables --simple).",
+    )
+
+
+def test_madmail_cmd(args, out):
+    """Run madmail E2E tests inside the builder container."""
+    ix = Incus(out)
+    ct = ix.get_running_relay(args.relay)
+    driver = MadmailDriver(ct, out)
+    if not driver.check_init():
+        return 1
+
+    if not driver.get_builder():
+        return 1
+
+    second_domain = None
+    if args.relay2:
+        ct2 = ix.get_running_relay(args.relay2)
+        drv2 = MadmailDriver(ct2, out)
+        second_domain = drv2.get_test_domain_or_ip()
+
+    return driver.run_tests(
+        second_domain=second_domain,
+        cool=args.cool,
+        simple=args.simple,
+    )
+
+
 # -------------------------------------------------------------------
 # minitest
 # -------------------------------------------------------------------
@@ -523,6 +573,7 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False):
 SUBCOMMANDS = [
     ("init", init_cmd, init_cmd_options),
     ("test-cmdeploy", test_cmdeploy_cmd, test_cmdeploy_cmd_options),
+    ("test-madmail", test_madmail_cmd, test_madmail_cmd_options),
     ("test-mini", test_mini_cmd, test_mini_cmd_options),
     ("status", status_cmd, status_cmd_options),
     ("start", start_cmd, start_cmd_options),

src/cmlxc/container.py

@@ -448,7 +448,11 @@ def install_deps(self):
                     deltachat-rpc-client \
                     deltachat-rpc-server \
                     imap-tools \
-                    requests
+                    requests \
+                    python-dotenv \
+                    cryptography
+                # python-dotenv and cryptography are needed by
+                # the madmail E2E test suite (test-madmail).
             fi
         """)
 
@@ -526,7 +530,15 @@ def init_ssh(self):
         )
         self.bash("chown root:root /root/.ssh/id_localchat")
         self.bash("chmod 600 /root/.ssh/id_localchat")
-        self.bash("echo 'Include /root/.ssh/config.d/*' > /root/.ssh/config")
+        self.bash(
+            "printf 'Include /root/.ssh/config.d/*\\n\\n"
+            "Host *\\n"
+            "  IdentityFile /root/.ssh/id_localchat\\n"
+            "  StrictHostKeyChecking accept-new\\n"
+            "  UserKnownHostsFile /dev/null\\n"
+            "  LogLevel ERROR\\n'"
+            " > /root/.ssh/config"
+        )
 
     def write_relay_ssh_config(self, ct):
         """Write an ssh-config.d file for a single relay container."""

src/cmlxc/driver_madmail.py

@@ -97,6 +97,48 @@ def on_init_relay(self, repo_path):
                 if check is None:
                     raise SetupError("admin-web build produced no index.html")
 
+    def run_tests(self, second_domain=None, cool=False, simple=False):
+        """Execute the madmail E2E test suite against relays."""
+        test_src = f"{self.get_git_main_path()}/tests/deltachat-test"
+
+        with self.out.section("test-madmail"):
+            # Symlink the built maddy binary into the test directory so
+            # tests that spawn a local server can find it at build/maddy.
+            self.bld_ct.bash(
+                f"mkdir -p {test_src}/build"
+                f" && ln -sf {self.repo_path}/build/maddy {test_src}/build/maddy"
+            )
+
+            relay1 = self.get_test_domain_or_ip()
+            rpc = "/root/minitest-venv/bin/deltachat-rpc-server"
+            env_exports = (
+                f"export REMOTE1={relay1} REMOTE2={relay1} RPC_SERVER_PATH={rpc}"
+            )
+            if second_domain:
+                env_exports = (
+                    f"export REMOTE1={relay1} REMOTE2={second_domain}"
+                    f" RPC_SERVER_PATH={rpc}"
+                )
+
+            cool_flag = " --cool" if cool else ""
+            if simple:
+                test_flags = "--test-1 --test-2 --test-3 --test-4 --test-5 --test-6"
+            else:
+                test_flags = "--all"
+
+            cmd = (
+                f"incus exec {self.bld_ct.name} --"
+                f" bash -c '"
+                f"{env_exports} &&"
+                f" source /root/minitest-venv/bin/activate &&"
+                f" cd {test_src} &&"
+                f" python main.py {test_flags}{cool_flag}'"
+            )
+            ret = self.out.shell(cmd)
+            if ret:
+                self.out.red(f"test-madmail failed (exit {ret})")
+            return ret
+
     def get_test_domain_or_ip(self):
         if not self.ct.ipv4:
             self.ct.wait_ready()
@@ -178,8 +220,7 @@ def print_admin_info(out, ct, ip):
         status = ct.bash("madmail admin-web status", check=False) or ""
         enabled, path = _parse_admin_web_status(status)
         if enabled and path:
-            out.print(f"admin web: https://{ip}{path}/")
-            out.print(f"admin API: https://{ip}/api/admin")
+            out.print(f"admin-url: https://{ip}{path}/")
         else:
             out.print("admin-url: disabled")
     except Exception:

src/relay_minitest/test_relay.py

@@ -1,6 +1,11 @@
+import imaplib
 import ipaddress
+import smtplib
+import ssl
+from email.mime.text import MIMEText
 
 import imap_tools
+import pytest
 import requests
 
 
@@ -27,6 +32,18 @@ def test_one_on_one(self, cmfactory, lp):
         msg2 = ac2.wait_for_incoming_msg()
         assert msg2.get_snapshot().text == "message0"
 
+    def test_send_dot(self, cmfactory, lp):
+        """Test that a single dot is properly escaped in SMTP protocol"""
+        ac1, ac2 = cmfactory.get_online_accounts(2)
+        chat = cmfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("ac1: sending single dot message")
+        chat.send_text(".")
+
+        lp.sec("ac2: wait for receive")
+        msg2 = ac2.wait_for_incoming_msg()
+        assert msg2.get_snapshot().text == "."
+
 
 class TestMultiRelay:
     """Tests that use two different chatmail relays."""
@@ -73,3 +90,44 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
     msgs = list(mailbox.fetch(mark_seen=False))
     assert msgs, "expected at least one message"
     assert public_ip not in msgs[0].obj.as_string()
+
+
+def test_unencrypted_rejection(cmsetup, lp):
+    """Test that unencrypted messages are rejected by the relay."""
+    lp.sec("creating users")
+    u1, u2 = cmsetup.gen_users(2)
+
+    lp.sec("sending unencrypted mail via SMTP")
+    msg = MIMEText("unencrypted")
+    msg["Subject"] = "test"
+    msg["From"] = u1.addr
+    msg["To"] = u2.addr
+
+    try:
+        u1.smtp.sendmail(u1.addr, [u2.addr], msg.as_string())
+        pytest.fail("Unencrypted message was accepted!")
+    except smtplib.SMTPDataError as e:
+        assert e.smtp_code == 523
+    except smtplib.SMTPRecipientsRefused as e:
+        for addr, (code, msg) in e.recipients.items():
+            assert code == 523
+
+
+def test_login_domain_validation(maildomain, lp):
+    """Test that IMAP LOGIN validates the domain part for JIT creation."""
+    lp.sec("attempting login with invalid domain")
+    # We use raw imaplib because cmsetup.gen_users() would fail at credential generation
+    # or handle the error differently.
+    user = f"invalid@{maildomain}.invalid"
+
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    with imaplib.IMAP4_SSL(maildomain, ssl_context=ctx) as conn:
+        try:
+            conn.login(user, "password")
+            pytest.fail("Login with invalid domain was accepted!")
+        except imaplib.IMAP4.error as e:
+            # Most relays return [AUTHENTICATIONFAILED] or similar
+            assert "FAILED" in str(e) or "Invalid" in str(e)

tests/fullrun.py

@@ -1,6 +1,6 @@
 """Functional test — exercises the complete cmlxc workflow in the live system.
 
-Uses and destroys ``fulltest0`` and ``fulltest1`` containers.
+Uses and destroys ``fulltest0``, ``fulltest1`` and ``fulltest-mad0`` containers.
 
 Run with::
 
@@ -19,6 +19,7 @@
 
 CT0 = "fulltest0"
 CT1 = "fulltest1"
+CT_MAD = "fulltest-mad0"
 
 
 def cmlxc(*args):
@@ -41,7 +42,7 @@ def _module_setup():
 
     yield
     subprocess.run(
-        ["cmlxc", "destroy", CT0, CT1],
+        ["cmlxc", "destroy", CT0, CT1, CT_MAD],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
         check=False,
@@ -96,12 +97,16 @@ def test_destroy():
 
 
 def test_mad_deploy():
-    cmlxc("deploy-madmail", "--source", "@main", "--ipv4-only", CT0)
+    cmlxc("deploy-madmail", "--source", "@main", "--ipv4-only", CT_MAD)
 
 
 def test_mini_madmail():
-    cmlxc("test-mini", CT0)
+    cmlxc("test-mini", CT_MAD)
+
+
+def test_madmail():
+    cmlxc("test-madmail", CT_MAD)
 
 
 def test_destroy_madmail():
-    cmlxc("destroy", CT0)
+    cmlxc("destroy", CT_MAD)