Merge pull request #3368 from trailofbits/3367-windows-readline

Adds support for readline on Windows

Author: ESultanik

.github/workflows/pythonpublish.yml

@@ -22,8 +22,6 @@ jobs:
         python-version: '3.x'
     - name: Install dependencies
       run: |
-        curl -LO https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/0.9/kaitai-struct-compiler_0.9_all.deb
-        sudo apt-get install ./kaitai-struct-compiler_0.9_all.deb
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
     - name: Build and publish

.github/workflows/tests.yml

@@ -12,10 +12,12 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        os: [ubuntu-latest] # windows-latest, macos-latest,
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v2
@@ -25,10 +27,8 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install Python Dependencies
       run: |
-        curl -LO https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/0.9/kaitai-struct-compiler_0.9_all.deb
-        sudo apt-get install ./kaitai-struct-compiler_0.9_all.deb
         python -m pip install --upgrade pip
         pip install setuptools
         pip install .[dev]

file

@@ -0,0 +1 @@
+kaitai-struct-compiler-0.9

kaitai_struct_formats

@@ -4,18 +4,32 @@
 This module automates compilation of Kaitai Struct definitions into Python code.
 
 This script is called from PolyFile's setup.py to compile the entire Kaitai Struct format library at build time.
-Therefore, this script should always be self-contained and not require any dependencies outside of the Python standard
+Therefore, this script should always be self-contained and not require any dependencies other than the Python standard
 library.
 
 """
+from io import BytesIO
 import json
+import os
 from pathlib import Path
 import shutil
+import stat
 import subprocess
-from typing import Iterable, List, Union
+import sys
+from typing import Iterable, List, Optional, Union
+from urllib.request import urlopen
+from zipfile import ZipFile
 
 
-KAITAI_COMPILER_NAME: str = "kaitai-struct-compiler"
+if os.name == "nt":
+    KAITAI_COMPILER_NAME: str = "kaitai-struct-compiler.bat"
+else:
+    KAITAI_COMPILER_NAME = "kaitai-struct-compiler"
+
+
+COMPILER_DIR = Path(__file__).absolute().parent / "kaitai-struct-compiler-0.9"
+COMPILER_BIN_DIR = COMPILER_DIR / "bin"
+COMPILER_BIN = COMPILER_BIN_DIR / KAITAI_COMPILER_NAME
 
 
 class KaitaiError(RuntimeError):
@@ -31,10 +45,6 @@ def __str__(self):
         return f"{self.ksy_file}: {super().__str__()}"
 
 
-def has_kaitai_compiler() -> bool:
-    return shutil.which(KAITAI_COMPILER_NAME) is not None
-
-
 class CompiledKSY:
     def __init__(self, class_name: str, python_path: Union[str, Path], dependencies: Iterable["CompiledKSY"] = ()):
         self.class_name: str = class_name
@@ -48,19 +58,46 @@ def __repr__(self):
                f"dependencies={self.dependencies!r})"
 
 
-def compile(ksy_path: Union[str, Path], output_directory: Union[str, Path]) -> CompiledKSY:
+def install_compiler():
+    resp = urlopen("https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/0.9/"
+                   "kaitai-struct-compiler-0.9.zip")
+    zipfile = ZipFile(BytesIO(resp.read()))
+    COMPILER_DIR.mkdir(exist_ok=True)
+    zipfile.extractall(COMPILER_DIR.parent)
+    if COMPILER_BIN.exists():
+        COMPILER_BIN.chmod(COMPILER_BIN.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        sys.stderr.write(f"Installed the Kaitai Struct Compiler to {COMPILER_BIN}\n")
+
+
+def compiler_path(auto_install: bool = True) -> Optional[Path]:
+    if COMPILER_BIN.exists():
+        return COMPILER_BIN
+    global_path = shutil.which(KAITAI_COMPILER_NAME)
+    if global_path is not None:
+        return Path(global_path)
+    if not auto_install:
+        return None
+    install_compiler()
+    return compiler_path(auto_install=False)
+
+
+def compile(ksy_path: Union[str, Path], output_directory: Union[str, Path], auto_install: bool = True) -> CompiledKSY:
     """Returns the list of compiled KSYs; the original spec being first, followed by its dependencies"""
-    if not has_kaitai_compiler():
-        raise KaitaiError(f"{KAITAI_COMPILER_NAME} not found! Please make sure it is in your PATH")
+    compiler = compiler_path(auto_install=auto_install)
+    if compiler is None:
+        raise KaitaiError(f"{KAITAI_COMPILER_NAME} not found! Please make sure it is in your PATH. "
+                          f"See https://kaitai.io/#download")
+
+    # sys.stderr.write(f"Using Kaitai Struct Compiler: {compiler!s}\n")
 
     if not isinstance(output_directory, Path):
         output_directory = Path(output_directory)
 
     output_directory.mkdir(parents=True, exist_ok=True)
 
     cmd = [
-        KAITAI_COMPILER_NAME, "--target", "python", "--outdir", str(output_directory), str(ksy_path),
-        "--debug", "--ksc-json-output", "-I", Path.cwd(), "--python-package", "polyfile.kaitai.parsers"
+        str(compiler), "--target", "python", "--outdir", str(output_directory), str(ksy_path),
+        "--debug", "--ksc-json-output", "-I", str(Path.cwd()), "--python-package", "polyfile.kaitai.parsers"
     ]
 
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -96,7 +133,12 @@ def compile(ksy_path: Union[str, Path], output_directory: Union[str, Path]) -> C
 
 if __name__ == "__main__":
     import argparse
-    import sys
+
+    if len(sys.argv) == 2 and sys.argv[1] == "--install":
+        if compiler_path() is None:
+            sys.exit(1)
+        else:
+            sys.exit(0)
 
     parser = argparse.ArgumentParser(description="A Kaitai Struct to Python compiler")
     parser.add_argument("KSY_PATH", type=str, help="path to the Kaitai Struct definition file")

polyfile/kaitai/.gitignore

@@ -39,7 +39,7 @@ class CompressedResource(KaitaiStruct):
     
     
     .. seealso::
-       Source - https://github.com/dgelessus/python-rsrcfork/tree/master/rsrcfork/compress
+       Source - https://github.com/dgelessus/python-rsrcfork/tree/f891a6e/src/rsrcfork/compress
     """
     SEQ_FIELDS = ["header", "compressed_data"]
     def __init__(self, _io, _parent=None, _root=None):

polyfile/kaitai/compiler.py

@@ -40,7 +40,7 @@ class Dcmp0(KaitaiStruct):
     with the main difference that it operates mostly on single bytes rather than two-byte units.
     
     .. seealso::
-       Source - https://github.com/dgelessus/python-rsrcfork/tree/master/rsrcfork/compress/dcmp0.py
+       Source - https://github.com/dgelessus/python-rsrcfork/blob/f891a6e/src/rsrcfork/compress/dcmp0.py
     """
     SEQ_FIELDS = ["chunks"]
     def __init__(self, _io, _parent=None, _root=None):

polyfile/kaitai/parsers/compressed_resource.py

@@ -37,7 +37,7 @@ class Dcmp1(KaitaiStruct):
     such as text.
     
     .. seealso::
-       Source - https://github.com/dgelessus/python-rsrcfork/tree/master/rsrcfork/compress/dcmp1.py
+       Source - https://github.com/dgelessus/python-rsrcfork/blob/f891a6e/src/rsrcfork/compress/dcmp1.py
     """
     SEQ_FIELDS = ["chunks"]
     def __init__(self, _io, _parent=None, _root=None):

polyfile/kaitai/parsers/dcmp_0.py

@@ -31,7 +31,7 @@ class Dcmp2(KaitaiStruct):
     to differentiate literals and table references.
     
     .. seealso::
-       Source - https://github.com/dgelessus/python-rsrcfork/tree/master/rsrcfork/compress/dcmp2.py
+       Source - https://github.com/dgelessus/python-rsrcfork/blob/f891a6e/src/rsrcfork/compress/dcmp2.py
     """
     SEQ_FIELDS = ["custom_lookup_table", "data", "last_byte"]
     def __init__(self, len_decompressed, header_parameters_with_io, _io, _parent=None, _root=None):