reenable the Version Negotiation test

README.md

@@ -59,7 +59,6 @@ Implementations that implement [qlog](https://github.com/quiclog/internet-drafts
 The Interop Runner implements the following test cases. Unless noted otherwise, test cases use HTTP/0.9 for file transfers. More test cases will be added in the future, to test more protocol features. The name in parentheses is the value of the `TESTCASE` environment variable passed into your Docker container.
 
 * **Version Negotiation** (`versionnegotiation`): Tests that a server sends a Version Negotiation packet in response to an unknown QUIC version number. The client should start a connection using an unsupported version number (it can use a reserved version number to do so), and should abort the connection attempt when receiving the Version Negotiation packet.
-Currently disabled due to #20.
 
 * **Handshake** (`handshake`): Tests the successful completion of the handshake. The client is expected to establish a single QUIC connection to the server and download one or multiple small files. Servers should not send a Retry packet in this test case.
 

docker-compose.yml

@@ -31,19 +31,19 @@ services:
     environment:
       - CRON=$CRON
       - ROLE=server
-      - SERVER_PARAMS=$SERVER_PARAMS
       - SSLKEYLOGFILE=/logs/keys.log
       - QLOGDIR=/logs/qlog/
       - TESTCASE=$TESTCASE_SERVER
-    depends_on:
-      - sim
     cap_add: 
       - NET_ADMIN
     ulimits:
       memlock: 67108864
     networks:
       rightnet:
         ipv4_address: 193.167.100.100
+    extra_hosts:
+      - "sim:193.167.100.2"
+      - "client:193.167.0.100"
 
   client:
     image: $CLIENT
@@ -57,13 +57,10 @@ services:
     environment:
       - CRON=$CRON
       - ROLE=client
-      - CLIENT_PARAMS=$CLIENT_PARAMS
       - SSLKEYLOGFILE=/logs/keys.log
       - QLOGDIR=/logs/qlog/
       - TESTCASE=$TESTCASE_CLIENT
       - REQUESTS=$REQUESTS
-    depends_on:
-      - sim
     cap_add: 
       - NET_ADMIN
     ulimits:
@@ -72,6 +69,7 @@ services:
       leftnet:
         ipv4_address: 193.167.0.100
     extra_hosts:
+      - "sim:193.167.0.2"
       - "server:193.167.100.100"
 
   iperf_server:
@@ -81,8 +79,6 @@ services:
     tty: true
     environment:
       - ROLE=server
-    depends_on:
-      - sim
     cap_add: 
       - NET_ADMIN
     networks:
@@ -98,8 +94,6 @@ services:
     tty: true
     environment:
       - ROLE=client
-    depends_on:
-      - sim
     cap_add: 
       - NET_ADMIN
     networks:

docker.py

@@ -0,0 +1,91 @@
+import io
+import logging
+import shutil
+import subprocess
+import threading
+
+
+class DockerRunner:
+    _containers = None
+    _cond = None
+    _timeout = 0  # in seconds
+    _expired = False
+
+    def __init__(self, timeout: int):
+        self._containers = []
+        self._cond = threading.Condition()
+        self._timeout = timeout
+
+    def add_container(self, name: str, env: dict):
+        self._containers.append({"name": name, "env": env})
+
+    def _run_container(self, cmd: str, env: dict, name: str):
+        self._execute(cmd, env, name)
+        with self._cond:
+            logging.debug("%s container returned.", name)
+            self._cond.notify()
+
+    def _execute(self, cmd: str, env: dict = {}, name: str = ""):
+        p = subprocess.Popen(
+            cmd.split(" "),
+            bufsize=1,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
+        for line in p.stdout:
+            ll = ""
+            if name:
+                ll = name + ": "
+            ll = ll + line.rstrip()
+            logging.debug(ll)
+
+    def _run_timer(self):
+        logging.debug("Timer expired. Stopping all containers.")
+        self._expired = True
+        with self._cond:
+            self._cond.notify()
+
+    def run(self) -> (str, bool):  # returns if the timer expired
+        # also log to a string, so we can parse the output later
+        output_string = io.StringIO()
+        output_stream = logging.StreamHandler(output_string)
+        output_stream.setLevel(logging.DEBUG)
+        logging.getLogger().addHandler(output_stream)
+
+        threads = []
+        # Start all containers (in separate threads)
+        docker_compose = shutil.which("docker-compose")
+        for e in self._containers:
+            t = threading.Thread(
+                target=self._run_container,
+                kwargs={
+                    "cmd": docker_compose + " up " + e["name"],
+                    "env": e["env"],
+                    "name": e["name"],
+                },
+            )
+            t.start()
+            threads.append(t)
+        # set a timer
+        timer = threading.Timer(self._timeout, self._run_timer)
+        timer.start()
+
+        # Wait for the first container to exit.
+        # Then stop all other docker containers.
+        with self._cond:
+            self._cond.wait()
+            names = [x["name"] for x in self._containers]
+            self._execute(
+                shutil.which("docker-compose") + " stop -t 5 " + " ".join(names)
+            )
+        # wait for all threads to finish
+        for t in threads:
+            t.join()
+        timer.cancel()
+
+        output = output_string.getvalue()
+        output_string.close()
+        logging.getLogger().removeHandler(output_stream)
+        return output, self._expired

interop.py

@@ -16,6 +16,7 @@
 from termcolor import colored
 
 import testcases
+from docker import DockerRunner
 from result import TestResult
 from testcases import Perspective
 
@@ -122,46 +123,48 @@ def _check_impl_is_compliant(self, name: str) -> bool:
 
         # check that the client is capable of returning UNSUPPORTED
         logging.debug("Checking compliance of %s client", name)
-        cmd = (
-            "CERTS=./certs" + " "
-            "TESTCASE_CLIENT=" + random_string(6) + " "
-            "SERVER_LOGS=/dev/null "
-            "CLIENT_LOGS=" + client_log_dir.name + " "
-            "WWW=" + www_dir.name + " "
-            "DOWNLOADS=" + downloads_dir.name + " "
-            'SCENARIO="simple-p2p --delay=15ms --bandwidth=10Mbps --queue=25" '
-            "CLIENT=" + self._implementations[name]["image"] + " "
-            "docker-compose up --timeout 0 --abort-on-container-exit -V sim client"
-        )
-        output = subprocess.run(
-            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-        )
-        if not self._is_unsupported(output.stdout.splitlines()):
+        env_sim = {"SCENARIO": "simple-p2p --delay=15ms --bandwidth=10Mbps --queue=25"}
+        env_client = {
+            "CERTS": "./certs",
+            "TESTCASE_CLIENT": random_string(6),
+            "DOWNLOADS": downloads_dir.name,
+            "CLIENT_LOGS": client_log_dir.name,
+            "CLIENT": self._implementations[name]["image"],
+        }
+        env = {}
+        env.update(env_sim)
+        env.update(env_client)
+        r = DockerRunner(5)
+        r.add_container("client", env)
+        r.add_container("sim", env)
+        output, expired = r.run()
+        if expired or not self._is_unsupported(output.splitlines()):
             logging.error("%s client not compliant.", name)
-            logging.debug("%s", output.stdout.decode("utf-8"))
+            logging.debug("%s", output)
             self.compliant[name] = False
             return False
         logging.debug("%s client compliant.", name)
 
         # check that the server is capable of returning UNSUPPORTED
         logging.debug("Checking compliance of %s server", name)
         server_log_dir = tempfile.TemporaryDirectory(dir="/tmp", prefix="logs_server_")
-        cmd = (
-            "CERTS=./certs" + " "
-            "TESTCASE_SERVER=" + random_string(6) + " "
-            "SERVER_LOGS=" + server_log_dir.name + " "
-            "CLIENT_LOGS=/dev/null "
-            "WWW=" + www_dir.name + " "
-            "DOWNLOADS=" + downloads_dir.name + " "
-            "SERVER=" + self._implementations[name]["image"] + " "
-            "docker-compose up -V server"
-        )
-        output = subprocess.run(
-            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-        )
-        if not self._is_unsupported(output.stdout.splitlines()):
+        env_server = {
+            "CERTS": "./certs",
+            "TESTCASE_SERVER": random_string(6),
+            "SERVER_LOGS": server_log_dir.name,
+            "WWW": www_dir.name,
+            "SERVER": self._implementations[name]["image"],
+        }
+        env = {}
+        env.update(env_sim)
+        env.update(env_server)
+        r = DockerRunner(5)
+        r.add_container("server", env)
+        r.add_container("sim", env)
+        output, expired = r.run()
+        if expired or not self._is_unsupported(output.splitlines()):
             logging.error("%s server not compliant.", name)
-            logging.debug("%s", output.stdout.decode("utf-8"))
+            logging.debug("%s", output)
             self.compliant[name] = False
             return False
         logging.debug("%s server compliant.", name)
@@ -330,73 +333,53 @@ def _run_test(
             server_keylog_file=server_log_dir.name + "/keys.log",
         )
         print(
-            "Server: "
-            + server
-            + ". Client: "
-            + client
-            + ". Running test case: "
-            + str(testcase)
+            "Server: {}. Client: {}. Running test: {}".format(
+                server, client, str(testcase)
+            )
         )
 
         reqs = " ".join(["https://server:443/" + p for p in testcase.get_paths()])
         logging.debug("Requests: %s", reqs)
-        params = (
-            "CERTS=./certs" + " "
-            "TESTCASE_SERVER=" + testcase.testname(Perspective.SERVER) + " "
-            "TESTCASE_CLIENT=" + testcase.testname(Perspective.CLIENT) + " "
-            "WWW=" + testcase.www_dir() + " "
-            "DOWNLOADS=" + testcase.download_dir() + " "
-            "SERVER_LOGS=" + server_log_dir.name + " "
-            "CLIENT_LOGS=" + client_log_dir.name + " "
-            'SCENARIO="{}" '
-            "CLIENT=" + self._implementations[client]["image"] + " "
-            "SERVER=" + self._implementations[server]["image"] + " "
-            'REQUESTS="' + reqs + '" '
-        ).format(testcase.scenario())
-        params += " ".join(testcase.additional_envs())
-        containers = "sim client server " + " ".join(testcase.additional_containers())
-        cmd = (
-            params
-            + " docker-compose up --abort-on-container-exit --timeout 1 "
-            + containers
-        )
-        logging.debug("Command: %s", cmd)
+        r = DockerRunner(timeout=testcase.timeout())
+        env_server = {
+            "CERTS": "./certs",
+            "TESTCASE_SERVER": testcase.testname(Perspective.SERVER),
+            "WWW": testcase.www_dir(),
+            "SERVER_LOGS": server_log_dir.name,
+            "SERVER": self._implementations[server]["image"],
+        }
+        env_client = {
+            "CERTS": "./certs",
+            "TESTCASE_CLIENT": testcase.testname(Perspective.CLIENT),
+            "DOWNLOADS": testcase.download_dir(),
+            "CLIENT_LOGS": client_log_dir.name,
+            "CLIENT": self._implementations[client]["image"],
+            "REQUESTS": reqs,
+        }
+        env_sim = {"SCENARIO": testcase.scenario()}
+        env = {}
+        env.update(env_server)
+        env.update(env_client)
+        env.update(env_sim)
+        r.add_container(name="server", env=env)
+        r.add_container(name="client", env=env)
+        r.add_container(name="sim", env=env)
+        print(testcase.additional_containers())
+        for c in testcase.additional_containers():
+            print("adding", c)
+            r.add_container(name=c, env=testcase.additional_envs())
+        output, expired = r.run()
 
         status = TestResult.FAILED
-        output = ""
-        expired = False
-        try:
-            r = subprocess.run(
-                cmd,
-                shell=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                timeout=testcase.timeout(),
-            )
-            output = r.stdout
-        except subprocess.TimeoutExpired as ex:
-            output = ex.stdout
-            expired = True
-
-        logging.debug("%s", output.decode("utf-8"))
-
-        if expired:
-            logging.debug("Test failed: took longer than %ds.", testcase.timeout())
-            r = subprocess.run(
-                "docker-compose stop " + containers,
-                shell=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                timeout=60,
-            )
-            logging.debug("%s", r.stdout.decode("utf-8"))
 
         # copy the pcaps from the simulator
         self._copy_logs("sim", sim_log_dir)
         self._copy_logs("client", client_log_dir)
         self._copy_logs("server", server_log_dir)
 
-        if not expired:
+        if expired:
+            logging.debug("Test failed: took longer than %ds.", testcase.timeout())
+        else:
             lines = output.splitlines()
             if self._is_unsupported(lines):
                 status = TestResult.UNSUPPORTED