New distribution [0.15.0]

 * bumped version to 0.15.0
 * fixed issue #37 & #40 as we updated deps on DictDumper
 * merged protocol parsing & constructing logics
 * revised TCP flow tracing & several interfaces
 * added a full API doc as hosted on https://pypcapkit.jarryshaw.me
 * minor bugfix all over the whole project

Author: JarryShaw

.vscode/settings.json

@@ -31,6 +31,7 @@
         "Sval",
         "TLVs",
         "Topt",
+        "Tpng",
         "Xcode",
         "ackn",
         "acnm",
@@ -116,6 +117,7 @@
         "gbhdr",
         "genc",
         "gpid",
+        "gprof",
         "hdrs",
         "hend",
         "hexbuf",
@@ -300,6 +302,7 @@
         "usec",
         "utopt",
         "verf",
+        "vfunc",
         "vihl",
         "vinfo",
         "vlen",

Makefile

@@ -10,6 +10,11 @@ version  = $(shell cat setup.py | grep "^__version__" | sed "s/__version__ = '\(
 # commit message
 message  ?= ""
 
+profile:
+	pipenv run python -m cProfile -o /tmp/parse_pcap.pstats $@
+	gprof2dot -f pstats /tmp/parse_pcap.pstats | dot -Tpng -o /tmp/parse_pcap.png
+	open /tmp/parse_pcap.png
+
 clean: clean-pyc clean-misc clean-pypi
 const: update-const
 #dist: dist-pypi dist-upload

Pipfile.lock

@@ -70,9 +70,16 @@ extracts parametres from a PCAP file.
       TCP flow tracing flag (as the ``trace`` parameter).
 
    .. attribute:: _flag_v
-      :type: bool
+      :type: Union[bool, Callable[[pcapkit.foundation.extraction.Extractor, pcapkit.protocols.pcap.frame.Frame]]]
+
+      A :obj:`bool` value or a function takes the :class:`Extract` instance and current parsed frame (depends on
+      the engine selected) as parameters to print verbose output information (as the ``verbose`` parameter).
+
+   .. attribute:: _vfunc
+      :type: Union[NotImplemented, Callable[[pcapkit.foundation.extraction.Extractor, pcapkit.protocols.pcap.frame.Frame]]]
 
-      Verbose output flag (as the ``verbose`` parameter).
+      If the ``verbose`` parameter is a callable, then it will be assigned as :attr:`self._vfunc <Extractor._vfunc>`;
+      otherwise, it keeps :obj:`NotImplemented` as a placeholder and has specific function for each engine.
 
    .. attribute:: _frnum
       :type: int

doc/img/profile.png

@@ -102,7 +102,13 @@ def __init__(self, fname, *, protocol, byteorder=sys.byteorder, nanosecond=False
             **kwargs: arbitrary keyword arguments
 
         """
+        #: int: Frame counter.
+        self._fnum = 1
+        #: bool: Nanosecond-resolution file flag.
         self._nsec = nanosecond
+        #: Union[pcapkit.const.reg.linktype.LinkType, enum.IntEnum, str, int]: Data link type.
+        self._link = protocol
+
         super().__init__(fname, protocol=protocol, byteorder=byteorder,
                          nanosecond=nanosecond, **kwargs)
 
@@ -155,6 +161,9 @@ def _append_value(self, value, file, name):  # pylint: disable=unused-argument
         packet = Frame(
             packet=value.packet,
             nanosecond=self._nsec,
+            num=self._fnum,
+            proto=self._link,
             **value.get('frame_info', value),
         ).data
         file.write(packet)
+        self._fnum += 1

doc/sphinx/source/foundation/extraction.rst

@@ -558,7 +558,10 @@ def __init__(self,
 
             files (bool): if split each frame into different files
             nofile (bool): if no output file is to be dumped
-            verbose (bool): if print verbose output information
+            verbose (Union[bool, Callable[[pcapkit.foundation.extraction.Extractor,
+                pcapkit.protocol.pcap.frame.Frame]]]): a :obj:`bool` value or a function takes the :class:`Extract`
+                instance and current parsed frame (depends on engine selected) as parameters to print verbose output
+                information
 
             engine (Optional[Literal['default', 'pcapkit', 'dpkt', 'scapy', 'pyshark', 'server', 'pipeline']]):
                 extraction engine to be used
@@ -601,7 +604,13 @@ def __init__(self,
         self._flag_m = False            # multiprocessing flag
         self._flag_q = nofile           # no output flag
         self._flag_t = trace            # trace flag
-        self._flag_v = verbose          # verbose output flag
+        self._flag_v = bool(verbose)    # verbose output flag
+
+        # verbose callback function
+        if isinstance(verbose, bool):
+            self._vfunc = NotImplemented
+        else:
+            self._vfunc = verbose
 
         self._frnum = 0                 # frame number
         self._frame = list()            # frame record
@@ -881,7 +890,10 @@ def _default_read_frame(self, *, frame=None, mpkit=None):
 
         # verbose output
         if self._flag_v:
-            print(f' - Frame {self._frnum:>3d}: {frame.protochain}')
+            if self._vfunc is NotImplemented:
+                print(f' - Frame {self._frnum:>3d}: {frame.protochain}')
+            else:
+                self._vfunc(self, frame)
 
         # write plist
         frnum = f'Frame {self._frnum}'
@@ -989,7 +1001,10 @@ def _scapy_read_frame(self):
         self._frnum += 1
         self._proto = packet2chain(packet)
         if self._flag_v:
-            print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            if self._vfunc is NotImplemented:
+                print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            else:
+                self._vfunc(self, packet)
 
         # write plist
         frnum = f'Frame {self._frnum}'
@@ -1102,7 +1117,10 @@ def _dpkt_read_frame(self):
         self._frnum += 1
         self._proto = packet2chain(packet)
         if self._flag_v:
-            print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            if self._vfunc is NotImplemented:
+                print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            else:
+                self._vfunc(self, packet)
 
         # write plist
         frnum = f'Frame {self._frnum}'
@@ -1213,7 +1231,10 @@ def _pyshark_read_frame(self):
         self._frnum = int(packet.number)
         self._proto = packet.frame_info.protocols
         if self._flag_v:
-            print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            if self._vfunc is NotImplemented:
+                print(f' - Frame {self._frnum:>3d}: {self._proto}')
+            else:
+                self._vfunc(self, packet)
 
         # write plist
         frnum = f'Frame {self._frnum}'

pcapkit/dumpkit/__init__.py

@@ -66,7 +66,7 @@
         |--> (Info) data ...
 
 """
-import copy
+import ipaddress
 import pathlib
 import sys
 import warnings
@@ -155,7 +155,36 @@ def make_fout(fout='./tmp', fmt='pcap'):
             else:
                 raise FileExists(*error.args).with_traceback(error.__traceback__) from None
 
-        return output, fmt
+        class DictDumper(output):
+            """Customised :class:`~dictdumper.dumper.Dumper` object."""
+
+            def object_hook(self, o):
+                """Convert content for function call.
+
+                Args:
+                    o (:obj:`Any`): object to convert
+
+                Returns:
+                    :obj:`Any`: the converted object
+
+                """
+                import enum
+                import aenum
+
+                if isinstance(o, (enum.IntEnum, aenum.IntEnum)):
+                    return dict(
+                        enum=type(o).__name__,
+                        desc=o.__doc__,
+                        name=o.name,
+                        value=o.value,
+                    )
+                if isinstance(o, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
+                    return str(o)
+                if isinstance(o, Info):
+                    return o.info2dict()
+                return super().object_hook(o)
+
+        return DictDumper, fmt
 
     def dump(self, packet):
         """Dump frame to output files.
@@ -168,8 +197,7 @@ def dump(self, packet):
         output = self.trace(packet, check=False, output=True)
 
         # dump files
-        output(packet['frame'], name=f"Frame {packet['index']}",
-               byteorder=self._endian, nanosecond=self._nnsecd)
+        output(packet['frame'], name=f"Frame {packet['index']}")
 
     def trace(self, packet, *, check=True, output=False):
         """Trace packets.
@@ -216,8 +244,8 @@ def trace(self, packet, *, check=True, output=False):
         if BUFID not in self._buffer:
             label = f'{info.src}_{info.srcport}-{info.dst}_{info.dstport}-{info.timestamp}'  # pylint: disable=E1101
             self._buffer[BUFID] = dict(
-                fpout=self._foutio(f'{self._fproot}/{label}.{self._fdpext}',
-                                   protocol=info.protocol),  # pylint: disable=E1101
+                fpout=self._foutio(fname=f'{self._fproot}/{label}.{self._fdpext}', protocol=info.protocol,  # pylint: disable=E1101
+                                   byteorder=self._endian, nanosecond=self._nnsecd),
                 index=list(),
                 label=label,
             )
@@ -251,13 +279,10 @@ def submit(self):
         self._newflg = False
         ret = list()
         for buf in self._buffer.values():
-            buf = copy.deepcopy(buf)
-            if self._fdpext:
-                buf['fpout'] = f"{self._fproot}/{buf['label']}.{self._fdpext}"
-            else:
-                del buf['fpout']
-            buf['index'] = tuple(buf['index'])
-            ret.append(Info(buf))
+            lbl = buf['label']
+            ret.append(Info(fpout=f"{self._fproot}/{lbl}.{self._fdpext}" if self._fdpext else NotImplemented,
+                            index=tuple(buf['index']),
+                            label=lbl,))
         ret += self._stream
         return tuple(ret)
 

pcapkit/foundation/extraction.py

@@ -4,7 +4,7 @@
 import sys
 
 # version string
-__version__ = '0.15.0rc1'
+__version__ = '0.15.0'
 
 # README
 with open('README.md', encoding='utf-8') as file:

pcapkit/foundation/traceflow.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+import pcapkit
+
+json = pcapkit.extract(fin='../sample/http.pcap', fout='../sample/http', format='json', files=True,
+                       store=True, verbose=True, ip=True, tcp=True, strict=False, trace=True,
+                       trace_format='json', trace_fout='../sample/trace')

setup.py

@@ -11,7 +11,7 @@ def test():
                            store=False, nofile=True, engine=engine)
 
 
-for engine in {'default', 'pyshark', 'scapy', 'dpkt', 'pipline', 'server'}:
+for engine in ['default', 'dpkt', 'scapy', 'pyshark', 'pipline', 'server']:
     profiler = cProfile.Profile()
     profiler.runcall(test)
 

test/test_api.py

@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import os
-import pprint
+#import pprint
 import textwrap
-import time
 
-import chardet
 import pcapkit
 
 os.system('> ../sample/out')

test/test_profile.py

@@ -5,5 +5,5 @@
 import pcapkit
 
 trace = pcapkit.extract(fin='../sample/http.pcap', nofile=True, verbose=True,
-                        trace=True, trace_format='pcap', trace_fout='../sample/trace')
+                        trace=True, trace_format='json', trace_fout='../sample/trace')
 pprint.pprint(trace.trace)