New distribution [0.14.4]

* included vendor scripts to distro
* seperate CLI deps in setup.py
* added vendor deps in setup.py
* linted source files (except protocols/ipsuite)
* revised impl of vendor scripts
* updated const enums accordingly
* some other minor changes

Author: JarryShaw

.gitignore

@@ -1,4 +1,7 @@
 
+pcapkit-*-tempdir/
+pcapkit-temp.html
+
 out
 *.js
 !out.js

.vscode/settings.json

@@ -1,30 +1,92 @@
 {
     "python.pythonPath": ".venv/bin/python",
     "cSpell.words": [
+        "Chksum",
         "DEVMODE",
         "ECDSA",
         "INET",
+        "NPROCESSORS",
+        "PROTO",
+        "RCVBT",
         "SIOP",
+        "Serv",
+        "aftermathmp",
         "aktau",
+        "bctr",
         "bufid",
+        "ccecho",
+        "ccnew",
+        "chkreq",
         "cmmd",
         "cmmt",
         "distro",
+        "dlink",
+        "dport",
         "dscp",
+        "dtgram",
+        "echore",
+        "emojize",
+        "eool",
+        "exeng",
+        "exlyr",
+        "expkg",
+        "exptl",
+        "extmp",
+        "fdpext",
+        "fext",
         "fileng",
+        "fout",
+        "foutio",
+        "fpout",
+        "fproot",
+        "frnum",
         "gaocegege",
+        "gbhdr",
+        "hdrs",
+        "httpv",
+        "ifile",
+        "ifnm",
+        "ilnp",
+        "inarp",
         "ipsuite",
         "lrfc",
+        "mpbuf",
+        "mpfdp",
+        "mpfrm",
+        "mpkit",
+        "mpmng",
+        "mpprc",
+        "mprsm",
+        "mpsvc",
+        "newflg",
+        "nnsec",
+        "nnsecd",
+        "nofile",
+        "nsec",
+        "ofile",
+        "ofnm",
+        "padn",
         "plen",
+        "pocsp",
         "prio",
         "protos",
         "pval",
+        "pycapfile",
         "pyenv",
+        "pypcap",
+        "reasm",
         "renm",
+        "sackpmt",
+        "sched",
+        "splt",
         "srcport",
+        "strflg",
         "subd",
+        "subn",
         "sufs",
+        "sysconf",
         "tbitem",
-        "tbody"
+        "tbody",
+        "vinfo"
     ]
 }

MANIFEST.in

@@ -3,16 +3,20 @@ global-exclude .DS_Store
 global-include *.md
 global-include *.py
 
+prune .github
 prune .venv
+prune .vscode
+
 prune archive
 prune devel
+prune doc
 prune release
+prune sample
 prune src
 prune test
 prune testbench
 
 include README.md
 include LICENSE
 
-prune pcapkit/protocols/*/NotImplemented
-# prune pcapkit/vendor
+exclude pcapkit/protocols/*/NotImplemented

Makefile

@@ -15,7 +15,8 @@ const: update-const
 dist: dist-pypi dist-upload
 release: release-master
 pipenv: update-pipenv
-update: update-const update-date
+# update: update-const update-date
+update: update-const
 
 pypi:
 	DIR=release $(MAKE) dist-prep dist

Pipfile

@@ -21,13 +21,14 @@ emoji = "*"
 tbtrim = ">=0.2.1"
 
 [dev-packages]
-pypcapkit = {editable = true, path = "."}
+pypcapkit = {editable = true,path = "."}
 pyshark = "*"
 dpkt = "*"
 scapy = "*"
+pathlib2 = "*"
 bs4 = "*"
 html5lib = "*"
-requests = "*"
+requests = {extras = ["socks"],version = "*"}
 ipython = "*"
 autopep8 = "*"
 pylint = "*"

Pipfile.lock

@@ -6,7 +6,7 @@
     from distutils.core import setup
 
 # version string
-__version__ = '0.14.3'
+__version__ = '0.14.4'
 
 # README
 with open('README.md', encoding='utf-8') as file:
@@ -32,14 +32,23 @@
         'dictdumper>=0.7.0.post1',  # for formatted output
         'chardet',                  # for bytes decode
         'aenum',                    # for const types
-        'emoji',                    # for CLI display
         'tbtrim>=0.2.1',            # for refined exceptions
     ],
     extras_require={
-        'all': ['dpkt', 'scapy', 'pyshark'],
+        'all': [
+            'emoji',
+            'dpkt', 'scapy', 'pyshark',
+            'requests[socks]', 'bs4', 'html5lib',
+        ],
+        # for CLI display
+        'cli': ['emoji'],
+        # for normal users
         'DPKT': ['dpkt'],
         'Scapy': ['scapy'],
         'PyShark': ['pyshark'],
+        # for developers
+        'vendor': ['requests[socks]', 'bs4', 'html5lib'],
+        # version compatibility
         ':python_version == "3.4"': ['pathlib2>=2.3.2'],
     },
     # py_modules = ['pcapkit'],
@@ -63,8 +72,8 @@
         'pcapkit.const.ipv6',
         'pcapkit.const.ipx',
         'pcapkit.const.mh',
-        'pcapkit.const.misc',
         'pcapkit.const.ospf',
+        'pcapkit.const.reg',
         'pcapkit.const.tcp',
         'pcapkit.const.vlan',
         'pcapkit.corekit',
@@ -95,8 +104,8 @@
         'pcapkit.vendor.ipv6',
         'pcapkit.vendor.ipx',
         'pcapkit.vendor.mh',
-        'pcapkit.vendor.misc',
         'pcapkit.vendor.ospf',
+        'pcapkit.vendor.reg',
         'pcapkit.vendor.tcp',
         'pcapkit.vendor.vlan',
     ],

setup.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=wrong-import-position, unused-import, unused-wildcard-import, bad-continuation
 """stream pcap file extractor
 
 `pcapkit` is an independent open source library, using only
@@ -53,17 +54,17 @@
     dump utilities for `pcapkit` implementation
 
 """
-# pylint: disable=wrong-import-position, unused-import, unused-wildcard-import
 import os
 import warnings
 
 import tbtrim
 
 from pcapkit.utilities.exceptions import DEVMODE, BaseError
+from pcapkit.utilities.warnings import DevModeWarning
 
 # set up sys.excepthook
 if DEVMODE:
-    warnings.showwarning('development mode enabled', RuntimeWarning,
+    warnings.showwarning('development mode enabled', DevModeWarning,
                          filename=__file__, lineno=0,
                          line=f"PCAPKIT_DEVMODE={os.environ['PCAPKIT_DEVMODE']}")
 else:
@@ -72,7 +73,7 @@
                          exception=BaseError, strict=False)
 
 # All Reference
-import pcapkit.__all__ as all  # pylint: disable=redefined-builtin
+import pcapkit.all
 
 # Interface
 from pcapkit.interface import *
@@ -111,13 +112,13 @@
     'TREE', 'JSON', 'PLIST', 'PCAP',                        # Format Macros
     'LINK', 'INET', 'TRANS', 'APP', 'RAW',                  # Layer Macros
     'DPKT', 'Scapy', 'PyShark', 'MPServer', 'MPPipeline', 'PCAPKit',
-                                                            # Engine Macros  # pylint: disable=bad-continuation
+                                                            # Engine Macros
     'NoPayload',                                            # No Payload
     'Raw',                                                  # Raw Packet
     'ARP', 'Ethernet', 'L2TP', 'OSPF', 'RARP', 'VLAN',      # Link Layer
     'AH', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',             # Internet Layer
     'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_Opts', 'IPv6_Route', 'MH',
-                                                            # IPv6 Extension Header  # pylint: disable=bad-continuation
+                                                            # IPv6 Extension Header
     'TCP', 'UDP',                                           # Transport Layer
     'FTP', 'HTTP',                                          # Application Layer
 ]

src/__init__.py

@@ -20,7 +20,7 @@
 
 
 def get_parser():
-    parser = argparse.ArgumentParser(prog='pcapkit',
+    parser = argparse.ArgumentParser(prog='pcapkit-cli',
                                      description='PCAP file extractor and formatted dumper')
     parser.add_argument('-V', '--version', action='version', version=__version__)
     parser.add_argument('fin', metavar='input-file-name',

src/__main__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=unused-import, unused-wildcard-import, bad-continuation
 """index for the library
 
 `pcapkit` has defined various and numerous functions and
@@ -7,6 +8,8 @@
 contains all things from `pcapkit`.
 
 """
+import pcapkit.const as const
+import pcapkit.vendor as vendor
 from pcapkit.corekit import *
 from pcapkit.dumpkit import *
 from pcapkit.foundation import *
@@ -15,9 +18,15 @@
 from pcapkit.protocols import *
 from pcapkit.reassembly import *
 from pcapkit.toolkit import *
-from pcapkit.utilities import *
+from pcapkit.utilities import *  # pylint: disable=redefined-builtin
 
 __all__ = [
+    # pcapkit.const
+    'const',
+
+    # pcapkit.vendor
+    'vendor',
+
     # pcapkit.corekit
     'Info',                                                 # Info Class
     'ProtoChain',                                           # ProtoChain