Merge pull request #333 from KimiNewt/support-v3-json

Support v3 json

Author: KimiNewt

src/pyshark/capture/capture.py

@@ -3,14 +3,15 @@
 import threading
 import subprocess
 import concurrent.futures
+from distutils.version import LooseVersion
 
 import logbook
 import sys
 
 from logbook import StreamHandler
 
 from pyshark.tshark.tshark import get_process_path, get_tshark_display_filter_flag, \
-    tshark_supports_json, TSharkVersionException
+    tshark_supports_json, TSharkVersionException, get_tshark_version
 from pyshark.tshark.tshark_json import packet_from_json_packet
 from pyshark.tshark.tshark_xml import packet_from_xml_packet, psml_structure_from_xml
 
@@ -67,6 +68,7 @@ def __init__(self, display_filter=None, only_summaries=False, eventloop=None,
         self._log = logbook.Logger(self.__class__.__name__, level=self.DEFAULT_LOG_LEVEL)
         self._closed = False
         self._custom_parameters = custom_parameters
+        self._tshark_version = None
 
         if include_raw and not use_json:
             raise RawMustUseJsonException("use_json must be True if include_raw")
@@ -158,25 +160,40 @@ def _setup_eventloop(self):
         if os.name == 'posix' and isinstance(threading.current_thread(), threading._MainThread):
             asyncio.get_child_watcher().attach_loop(self.eventloop)
 
-    @classmethod
-    def _get_json_separator(cls):
-        return ("}%s%s  ," % (os.linesep, os.linesep)).encode()
+    def _get_json_separators(self):
+        """"Returns the separators between packets in a JSON output
 
-    @classmethod
-    def _extract_packet_json_from_data(cls, data, got_first_packet=True):
+        Returns a tuple of (packet_separator, end_of_file_separator, characters_to_disregard).
+        The latter variable being the number of characters to ignore in order to pass the packet (i.e. extra newlines,
+        commas, parenthesis).
+        """
+        if LooseVersion(self._tshark_version) >= LooseVersion("3.0.0"):
+            return ("%s  },%s" % (os.linesep, os.linesep)).encode(), ("}%s]" % os.linesep).encode(), (
+                    1 + len(os.linesep))
+        else:
+            return ("}%s%s  ," % (os.linesep, os.linesep)).encode(), ("}%s%s]" % (os.linesep, os.linesep)).encode(), 1
+
+    def _extract_packet_json_from_data(self, data, got_first_packet=True):
         tag_start = 0
         if not got_first_packet:
             tag_start = data.find(b"{")
             if tag_start == -1:
                 return None, data
-        closing_tag = cls._get_json_separator()
-        tag_end = data.find(closing_tag)
+        packet_separator, end_separator, end_tag_strip_length = self._get_json_separators()
+        found_separator = None
+
+        tag_end = data.find(packet_separator)
         if tag_end == -1:
-            closing_tag = ("}%s%s]" % (os.linesep, os.linesep)).encode()
-            tag_end = data.find(closing_tag)
-        if tag_end != -1:
-            # Include closing parenthesis but not comma
-            tag_end += len(closing_tag) - 1
+            # Not end of packet, maybe it has end of entire file?
+            tag_end = data.find(end_separator)
+            if tag_end != -1:
+                found_separator = end_separator
+        else:
+            # Found a single packet, just add the separator without extras
+            found_separator = packet_separator
+
+        if found_separator:
+            tag_end += len(found_separator) - end_tag_strip_length
             return data[tag_start:tag_end], data[tag_end + 1:]
         return None, data
 
@@ -229,7 +246,8 @@ def _packets_from_tshark_sync(self, packet_count=None, existing_process=None):
                 if packet_count and packets_captured >= packet_count:
                     break
         finally:
-            self.eventloop.run_until_complete(self._cleanup_subprocess(tshark_process))
+            if tshark_process in self._running_processes:
+                self.eventloop.run_until_complete(self._cleanup_subprocess(tshark_process))
 
     def apply_on_packets(self, callback, timeout=None, packet_count=None):
         """
@@ -357,7 +375,9 @@ async def _get_tshark_process(self, packet_count=None, stdin=None):
         """
         if self.use_json:
             output_type = 'json'
-            if not tshark_supports_json(self.tshark_path):
+            if not self._tshark_version:
+                self._tshark_version = get_tshark_version(self.tshark_path)
+            if not tshark_supports_json(self._tshark_version):
                 raise TSharkVersionException("JSON only supported on Wireshark >= 2.2.0")
         else:
             output_type = 'psml' if self._only_summaries else 'pdml'

src/pyshark/capture/inmem_capture.py

@@ -4,7 +4,7 @@
 import struct
 import time
 import warnings
-
+from distutils.version import LooseVersion
 
 from pyshark.capture.capture import Capture, StopCapture
 
@@ -74,9 +74,17 @@ async def _get_tshark_process(self, packet_count=None):
 
         return proc
 
-    @classmethod
-    def _get_json_separator(cls):
-        return ("}%s%s" % (os.linesep, os.linesep)).encode()
+    def _get_json_separators(self):
+        """"Returns the separators between packets in a JSON output
+
+        Returns a tuple of (packet_separator, end_of_file_separator, characters_to_disregard).
+        The latter variable being the number of characters to ignore in order to pass the packet (i.e. extra newlines,
+        commas, parenthesis).
+        """
+        if LooseVersion(self._tshark_version) >= LooseVersion("3.0.0"):
+            return ("%s  }" % os.linesep).encode(), ("}%s]" % os.linesep).encode(), 0
+        else:
+            return ("}%s%s" % (os.linesep, os.linesep)).encode(), ("}%s%s]" % (os.linesep, os.linesep)).encode(), 1
 
     def _write_packet(self, packet):
         # Write packet header

src/pyshark/tshark/tshark.py

@@ -77,8 +77,7 @@ def get_tshark_version(tshark_path=None):
     return version_string
 
 
-def tshark_supports_json(tshark_path=None):
-    tshark_version = get_tshark_version(tshark_path)
+def tshark_supports_json(tshark_version):
     return LooseVersion(tshark_version) >= LooseVersion("2.2.0")
 
 

src/setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name="pyshark",
-    version="0.4.2.2",
+    version="0.4.2.3",
     packages=find_packages(),
     package_data={'': ['*.ini', '*.pcapng']},
     install_requires=['lxml', 'py', 'logbook'],

tests/conftest.py

@@ -43,4 +43,9 @@ def simple_summary_capture(request):
     """
     A capture already full of packets
     """
-    return make_test_capture(request, only_summaries=True)
+    return make_test_capture(request, only_summaries=True)
+
+
+@pytest.fixture(params=[True, False])
+def simple_xml_and_json_capture(request):
+    return make_test_capture(request, use_json=request.param)

tests/test_basic_parsing.py

@@ -1,42 +1,42 @@
-def test_count_packets(simple_capture):
+def test_count_packets(simple_xml_and_json_capture):
     """Test to make sure the right number of packets are read from a known
        capture"""
-    packet_count = sum(1 for _ in simple_capture)
+    packet_count = sum(1 for _ in simple_xml_and_json_capture)
     assert packet_count == 24
 
 
-def test_sum_lengths(simple_capture):
+def test_sum_lengths(simple_xml_and_json_capture):
     """Test to make sure that the right packet length is being read from
        tshark's output by comparing the aggregate length of all packets
        to a known value"""
-    total_length = sum(int(packet.length) for packet in simple_capture)
+    total_length = sum(int(packet.length) for packet in simple_xml_and_json_capture)
     assert total_length == 2178
 
 
-def test_layers(simple_capture):
+def test_layers(simple_xml_and_json_capture):
     """Test to make sure the correct protocols are reported for known
        packets"""
     packet_indexes = (0, 5, 6, 13, 14, 17, 23)
-    test_values = [simple_capture[i].highest_layer for i in packet_indexes]
+    test_values = [simple_xml_and_json_capture[i].highest_layer for i in packet_indexes]
     known_values = ['DNS', 'DNS', 'ICMP', 'ICMP', 'TCP', 'HTTP', 'TCP']
     assert test_values == known_values
 
 
-def test_ethernet(simple_capture):
+def test_ethernet(simple_xml_and_json_capture):
     """Test to make sure Ethernet fields are being read properly by comparing
        packet dissection results to known values"""
-    packet = simple_capture[0]
+    packet = simple_xml_and_json_capture[0]
     test_values = packet.eth.src, packet.eth.dst
     known_values = ('00:00:bb:10:20:10', '00:00:bb:02:04:01')
     assert test_values == known_values
 
 
-def test_icmp(simple_capture):
+def test_icmp(simple_xml_and_json_capture):
     """Test to make sure ICMP fields are being read properly by comparing
        packet dissection results to known values"""
-    packet = simple_capture[11]
+    packet = simple_xml_and_json_capture[11]
     # The value returned by tshark is locale-dependent.
     # Depending on the locale, a comma can be used instead of a dot
     # as decimal separator.
     resptime = packet.icmp.resptime.replace(',', '.')
-    assert resptime == '1.667'
+    assert resptime == '1.667'