[SDFAB-488] Use PyPy interpreter to analyze INT report PCAP to decrease runtime test completion (#378)

* Implement PyPy compiler for `int_traffic_trace` to decrease runtime test
completion
* Install PyPy and `ptf` PyPy module in Dockerfile
* Add temporary comments in relevant files for debugging Python
interpreter (CPython or PyPy)

* Enable PyPy for parsing linerate PTF pcap file results
* Test time improved in `int_traffic_trace` tests from ~28 minutes each
to ~10 minutes each
* Clean int_single_flow to match current linerate tests, added simple
result verification
* Changed Dockerfile to install PyPy

* Clean `xnt.py`

* Add copyright and licensing information

* Address comments
* Remove test.py and run code in python command
* Save dictionary in command to pickle, and depickle in function

* Address comments
* Remove obsolete Dockerfile dependencies
* Move new pcap function to `xnt.py`
* Update analyze pcap functions in test files
* Add more verification to `int_single_flow.py`

* Remove reduntant code from `pypy_analyze_int_report_pcap`

* Add FIXME to `int_single_flow` runtime INT report mismatch

* Remove unused param PTF_VER from Dockerfile

* Add comment explaining new Dockerfile packages

Author: Darius

ptf/Dockerfile

@@ -83,9 +83,9 @@ ARG P4RUNTIME_SHELL_VER
 ARG UNITTEST_XML_REPORTING_VER
 
 ENV RUNTIME_DEPS \
-	python3 \
-	python3-pip \
-	python3-setuptools \
+    python3 \
+    python3-pip \
+    python3-setuptools \
     git
 
 ENV PIP_DEPS \
@@ -132,7 +132,11 @@ ENV RUNTIME_DEPS \
     python3-dev \
     build-essential \
     python3-pip \
-    netbase
+    wget \
+    netbase \
+    # below packages are necessary dependencies for installing scipy in PyPy
+    libatlas-base-dev \
+    gfortran
 
 RUN apt update && apt install -y $RUNTIME_DEPS
 RUN pip3 install --no-cache-dir scipy==1.5.4 numpy==1.19.4 matplotlib==3.3.3 pyyaml==5.4.1
@@ -144,6 +148,14 @@ COPY --from=proto-deps /output /usr/lib/python3.8/dist-packages
 COPY --from=ptf-deps /python_output /
 # Install custom scapy version from TRex
 RUN cd ${TREX_EXT_LIBS}/scapy-${SCAPY_VER}/ && python3 setup.py install
+# Build pypy from source for traffic trace tests
+RUN cd /opt/ && \
+    wget -nv https://downloads.python.org/pypy/pypy3.7-v7.3.5-linux64.tar.bz2 && \
+    tar xf pypy3.7-v7.3.5-linux64.tar.bz2 && \
+    ln -s /opt/pypy3.7-v7.3.5-linux64/bin/pypy /usr/local/bin/pypy
+RUN pypy -m ensurepip && \
+    pypy -mpip install -U pip wheel
+RUN pypy -mpip install --no-cache-dir six scipy==1.5.4 numpy==1.19.4 matplotlib==3.3.3 pyyaml==5.4.1 scapy==2.4.5
 RUN ldconfig
 
 ENTRYPOINT []

ptf/tests/common/xnt.py

@@ -6,6 +6,8 @@
 import logging
 import os
 from os.path import abspath, exists, splitext
+from subprocess import Popen
+import pickle
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -347,6 +349,29 @@ def analyze_report_pcap(pcap_file: str, total_flows_from_trace: int = 0) -> dict
     return results
 
 
+def pypy_analyze_int_report_pcap(pcap_file: str, total_flows: int = 0) -> dict:
+    code = "import pickle\n" \
+           "from xnt import analyze_report_pcap\n"
+
+    if total_flows:
+        code += f"result = analyze_report_pcap('{pcap_file}', {total_flows})\n"
+    else:
+        code += f"result = analyze_report_pcap('{pcap_file}')\n"
+
+    # pickle dictionary from analyze_report_pcap
+    code += "with open('trace.pickle', 'wb') as handle:\n" \
+            "    pickle.dump(result, handle, protocol=pickle.HIGHEST_PROTOCOL)"
+
+    cmd = ["pypy", "-c", code]
+    p = Popen(cmd)
+    p.wait()
+
+    with open('trace.pickle', 'rb') as handle:
+        result = pickle.load(handle)
+
+    return result
+
+
 def plot_histogram_and_cdf(report_plot_file, valid_report_irgs):
     if exists(report_plot_file):
         os.remove(report_plot_file)

ptf/tests/linerate/int_single_flow.py

@@ -9,15 +9,17 @@
 from trex_stl_lib.api import STLPktBuilder, STLStream, STLTXCont
 from trex_test import TRexTest
 from trex_utils import list_port_status
-from xnt import analyze_report_pcap
+from xnt import pypy_analyze_int_report_pcap
 
 TRAFFIC_MULT = "40gbpsl1"
 TEST_DURATION = 10
-CAPTURE_LIMIT = 20
+CAPTURE_LIMIT = 30
 
-SENDER_PORTS = [0]
-RECEIVER_PORTS = [1]
-INT_COLLECTOR_PORTS = [2]
+MIN_FLOW_REPORTS = 28
+
+SENDER_PORT = 0
+RECEIVER_PORT = 1
+INT_COLLECTOR_PORT = 2
 
 
 @group("int")
@@ -50,28 +52,50 @@ def doRunTest(
 
         # Define traffic to be sent
         stream = STLStream(packet=STLPktBuilder(pkt=pkt, vm=[]), mode=STLTXCont())
-        self.trex_client.add_streams(stream, ports=SENDER_PORTS)
+        self.trex_client.add_streams(stream, ports=[SENDER_PORT])
 
         # Capture INT packets
-        self.trex_client.set_service_mode(ports=INT_COLLECTOR_PORTS, enabled=True)
+        self.trex_client.set_service_mode(ports=[INT_COLLECTOR_PORT], enabled=True)
         capture = self.trex_client.start_capture(
-            rx_ports=INT_COLLECTOR_PORTS,
+            rx_ports=[INT_COLLECTOR_PORT],
             limit=CAPTURE_LIMIT,
             bpf_filter="udp and dst port 32766",
         )
 
         # Start sending stateless traffic
-        self.trex_client.start(ports=SENDER_PORTS, mult=mult, duration=TEST_DURATION)
-        self.trex_client.wait_on_traffic(ports=SENDER_PORTS)
+        self.trex_client.start(ports=[SENDER_PORT], mult=mult, duration=TEST_DURATION)
+        self.trex_client.wait_on_traffic(ports=[SENDER_PORT])
 
         output = "/tmp/int-single-flow-{}.pcap".format(
             datetime.now().strftime("%Y%m%d-%H%M%S")
         )
         self.trex_client.stop_capture(capture["id"], output)
-        analyze_report_pcap(output)
-        list_port_status(self.trex_client.get_stats())
 
-        # TODO: parse data and verify results
+        results = pypy_analyze_int_report_pcap(output)
+        port_stats = self.trex_client.get_stats()
+
+        sent_packets = port_stats[SENDER_PORT]["opackets"]
+        recv_packets = port_stats[RECEIVER_PORT]["ipackets"]
+
+        list_port_status(port_stats)
+
+        """
+        Verify the following:
+        - Packet loss: No packets were dropped during the test
+        - INT reports: 1 INT report per second per flow was generated
+        """
+        self.failIf(
+            sent_packets != recv_packets,
+            f"Didn't receive all packets; sent {sent_packets}, received {recv_packets}",
+        )
+
+        # FIXME: Although duration specified is 10, test in reality runs for 29-30 seconds, and
+        # thus generates 28-29 INT flow reports
+        local_reports = results["local_reports"]
+        self.failIf(
+            local_reports < MIN_FLOW_REPORTS,
+            f"Flow reports generated for ~30 second test should be at least 28, was {local_reports}",
+        )
 
     def runTest(self):
         # TODO: iterate all possible parameters of test

ptf/tests/linerate/int_traffic_trace.py

@@ -8,7 +8,7 @@
 from ptf.testutils import group
 from trex_test import TRexTest
 from trex_utils import list_port_status
-from xnt import analyze_report_pcap
+from xnt import pypy_analyze_int_report_pcap
 
 TRAFFIC_SPEEDUP = 1.0
 TEST_DURATION = 60
@@ -77,7 +77,7 @@ def runTest(self):
             datetime.now().strftime("%Y%m%d-%H%M%S")
         )
         self.trex_client.stop_capture(capture["id"], output)
-        results = analyze_report_pcap(output, TOTAL_FLOWS)
+        results = pypy_analyze_int_report_pcap(output, TOTAL_FLOWS)
 
         port_stats = self.trex_client.get_stats()
         sent_packets = port_stats[SENDER_PORT]["opackets"]
@@ -161,7 +161,7 @@ def runTest(self):
             datetime.now().strftime("%Y%m%d-%H%M%S")
         )
         self.trex_client.stop_capture(capture["id"], output)
-        results = analyze_report_pcap(output, TOTAL_FLOWS)
+        results = pypy_analyze_int_report_pcap(output, TOTAL_FLOWS)
 
         port_stats = self.trex_client.get_stats()
         sent_packets = port_stats[SENDER_PORT]["opackets"]