New distribution [0.14.5]

 * revised deps
 * added vendor CLI
 * delayed vendor import

Author: JarryShaw

.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Module",
+            "type": "python",
+            "request": "launch",
+            "module": "pcapkit"
+        }
+    ]
+}

.vscode/settings.json

@@ -28,6 +28,7 @@
         "emojize",
         "eool",
         "exeng",
+        "exlayer",
         "exlyr",
         "expkg",
         "exptl",
@@ -43,6 +44,8 @@
         "gaocegege",
         "gbhdr",
         "hdrs",
+        "hexbuf",
+        "hexlst",
         "httpv",
         "ifile",
         "ifnm",
@@ -80,13 +83,16 @@
         "sched",
         "splt",
         "srcport",
+        "strbuf",
         "strflg",
+        "strlst",
         "subd",
         "subn",
         "sufs",
         "sysconf",
         "tbitem",
         "tbody",
         "vinfo"
-    ]
+    ],
+    "restructuredtext.confPath": "${workspaceFolder}/docs/conf.py"
 }

Makefile

@@ -51,9 +51,9 @@ clean-pipenv:
 clean-pypi:
 	cd $(DIR)
 	mkdir -p sdist eggs wheels
-	find dist -iname '*.egg' -exec mv {} eggs \;
-	find dist -iname '*.whl' -exec mv {} wheels \;
-	find dist -iname '*.tar.gz' -exec mv {} sdist \;
+	[ -d dist ] && find dist -iname '*.egg' -exec mv {} eggs \; || true
+	[ -d dist ] && find dist -iname '*.whl' -exec mv {} wheels \; || true
+	[ -d dist ] && find dist -iname '*.tar.gz' -exec mv {} sdist \; || true
 	rm -rf build dist *.egg-info
 
 # update pipenv
@@ -85,7 +85,7 @@ dist-pypi: clean-pypi dist-pypi-new dist-pypi-old
 dist-pypi-new:
 	set -ex
 	cd $(DIR)
-	~/.pyenv/versions/3.8-dev/bin/python3.8 setup.py bdist_egg bdist_wheel --python-tag='cp38'
+	~/.pyenv/versions/3.8.0/bin/python3.8 setup.py bdist_egg bdist_wheel --python-tag='cp38'
 	~/.pyenv/versions/3.7.4/bin/python3.7 setup.py bdist_egg bdist_wheel --python-tag='cp37'
 	~/.pyenv/versions/3.6.9/bin/python3.6 setup.py bdist_egg bdist_wheel --python-tag='cp36'
 	~/.pyenv/versions/pypy3.6-7.1.1/bin/pypy3 setup.py bdist_egg bdist_wheel --python-tag='pp36'

doc/pcap/Frame.md

@@ -8,7 +8,7 @@ Reference: [Libpcap File Format](https://wiki.wireshark.org/Development/LibpcapF
 
 # PCAP Frame Header
 
-## Description
+## Description
 
  
 
@@ -41,7 +41,7 @@ class Frame(file, *, num, proto, nanosecond, **kwrags)
   2. `pcapkit.protocols.protocol.Protocol`
   3. `pcapkit.protocols.pcap.frame.Frame`
 
-### Data Format
+### Data Format
 
 ```
 (Info) Frame

doc/pcap/Header.md

@@ -10,7 +10,7 @@ Reference: [Libpcap File Format](https://wiki.wireshark.org/Development/LibpcapF
 
 # PCAP Global Header
 
-## Description
+## Description
 
   This header starts the `libpcap` file and will be followed by the first packet header:
 

setup.py

@@ -6,7 +6,7 @@
     from distutils.core import setup
 
 # version string
-__version__ = '0.14.4'
+__version__ = '0.14.5'
 
 # README
 with open('README.md', encoding='utf-8') as file:
@@ -38,7 +38,7 @@
         'all': [
             'emoji',
             'dpkt', 'scapy', 'pyshark',
-            'requests[socks]', 'bs4', 'html5lib',
+            'requests[socks]', 'bs4[html5lib]',
         ],
         # for CLI display
         'cli': ['emoji'],
@@ -47,14 +47,15 @@
         'Scapy': ['scapy'],
         'PyShark': ['pyshark'],
         # for developers
-        'vendor': ['requests[socks]', 'bs4', 'html5lib'],
+        'vendor': ['requests[socks]', 'bs4[html5lib]'],
         # version compatibility
         ':python_version == "3.4"': ['pathlib2>=2.3.2'],
     },
     # py_modules = ['pcapkit'],
     entry_points={
         'console_scripts': [
             'pcapkit-cli = pcapkit.__main__:main',
+            'pcapkit-vendor = pcapkit.vendor.__main__:main',
         ]
     },
     # packages=setuptools.find_namespace_packages(
@@ -109,6 +110,9 @@
         'pcapkit.vendor.tcp',
         'pcapkit.vendor.vlan',
     ],
+    package_dir={
+        'pcapkit': 'src',
+    },
     package_data={
         '': [
             'LICENSE',

src/all.py

@@ -9,7 +9,7 @@
 
 """
 import pcapkit.const as const
-import pcapkit.vendor as vendor
+# import pcapkit.vendor as vendor
 from pcapkit.corekit import *
 from pcapkit.dumpkit import *
 from pcapkit.foundation import *
@@ -24,8 +24,8 @@
     # pcapkit.const
     'const',
 
-    # pcapkit.vendor
-    'vendor',
+    # # pcapkit.vendor
+    # 'vendor',
 
     # pcapkit.corekit
     'Info',                                                 # Info Class

src/utilities/warnings.py

@@ -7,10 +7,10 @@
     # UserWarning
     'BaseWarning',
     # ImportWarning
-    'FormatWarning', 'EngineWarning',
+    'FormatWarning', 'EngineWarning', 'InvalidVendorWarning',
     # RuntimeWarning
     'FileWarning', 'LayerWarning', 'ProtocolWarning', 'AttributeWarning',
-    'DevModeWarning', 'VendorRequestWarning',
+    'DevModeWarning', 'VendorRequestWarning', 'VendorRuntimeWarning',
     # ResourceWarning
     'DPKTWarning', 'ScapyWarning', 'PySharkWarning'
 ]
@@ -41,6 +41,10 @@ class EngineWarning(BaseWarning, ImportWarning):
     """Unsupported extraction engine."""
 
 
+class InvalidVendorWarning(BaseWarning, ImportWarning):
+    """Vendor CLI invalid updater."""
+
+
 ##############################################################################
 # RuntimeWarning session.
 ##############################################################################
@@ -70,6 +74,10 @@ class VendorRequestWarning(BaseWarning, RuntimeWarning):
     """Vendor request connection failed."""
 
 
+class VendorRuntimeWarning(BaseWarning, RuntimeWarning):
+    """Vendor failed during runtime."""
+
+
 ##############################################################################
 # ResourceWarning session.
 ##############################################################################

src/vendor/__main__.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""CLI for web crawlers."""
+
+import argparse
+import importlib
+import multiprocessing
+import sys
+import warnings
+
+import pcapkit.vendor as vendor_module
+from pcapkit.utilities.warnings import InvalidVendorWarning, VendorRuntimeWarning
+from pcapkit.vendor import __all__ as vendor_all
+
+# version string
+__version__ = '0.14.5'
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(prog='pcapkit-vendor',
+                                     description='update constant enumerations')
+    parser.add_argument('-V', '--version', action='version', version=__version__)
+    parser.add_argument('target', action='store', nargs=argparse.REMAINDER,
+                        help='update targets, supply none to update all')
+    return parser
+
+
+def run(vendor):
+    print(vendor.__doc__)
+    try:
+        vendor()
+    except Exception as error:
+        warnings.warn(error, VendorRuntimeWarning)
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    target_list = list()
+    for target in args.target:
+        try:
+            module = importlib.import_module(f'pcapkit.vendor.{target}')
+            target_list.extend(getattr(module, name) for name in module.__all__)
+        except ImportError:
+            warn = warnings.formatwarning(f'invalid vendor updater: {target}', InvalidVendorWarning,
+                                          __file__, 0, ' '.join(sys.argv))
+            print(warn, file=sys.stderr)
+
+    if not target_list:
+        target_list.extend(getattr(vendor_module, name) for name in vendor_all)
+
+    with multiprocessing.Pool() as pool:
+        pool.map(run, target_list)
+
+
+if __name__ == "__main__":
+    sys.exit(main())