Provide PyOxidizer binaries

.github/workflows/pyoxidizer.yml

@@ -0,0 +1,183 @@
+name: Build PyOxidizer Binary
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Tag to build
+        required: true
+
+jobs:
+  build-linux-gnu:
+    name: Build Linux (GNU)
+    runs-on: ubuntu-latest
+    container: quay.io/pypa/manylinux2014_x86_64
+
+    steps:
+      # We use older versions of the checkout and cache actions, those work with the LIBC version of this container
+      # We also can't use the setup-python action for this reason - it depends on a newer LIBC
+      - uses: actions/checkout@v3
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pyoxidizer/
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: linux-cache
+
+      - name: Install dependencies
+        run: /opt/python/cp310-cp310/bin/pip install pyoxidizer dissect
+
+      - name: Build binary
+        run: |
+          mkdir -p build/lib/dissect/target/plugins
+          /opt/python/cp310-cp310/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py
+          /opt/python/cp310-cp310/bin/pyoxidizer build --release --target-triple x86_64-unknown-linux-gnu --var flavor standalone --var version ${{ github.event.inputs.tag }}
+          strip build/x86_64-unknown-linux-gnu/release/install/acquire
+
+      - name: Verify binary
+        run: build/x86_64-unknown-linux-gnu/release/install/acquire-x86_64-unknown-linux-gnu --help
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: acquire-linux
+          path: build/x86_64-unknown-linux-gnu/release/install/*
+
+  build-linux-musl:
+    name: Build Linux (musl)
+    runs-on: ubuntu-latest
+    container: quay.io/pypa/manylinux2014_x86_64
+    env:
+      musl_version: 1.2.4
+
+    steps:
+      # We use older versions of the checkout and cache actions, those work with the LIBC version of this container
+      # We also can't use the setup-python action for this reason - it depends on a newer LIBC
+      - uses: actions/checkout@v3
+
+      - name: Setup Rust
+        run: |
+          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
+          echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pyoxidizer/
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: linux-musl-cache
+
+      - name: Setup musl
+        run: |
+          curl -L https://www.musl-libc.org/releases/musl-${{ env.musl_version }}.tar.gz | tar -xzf -
+          cd musl-*
+          ./configure --exec-prefix=/usr/local
+          make
+          make install
+
+      - name: Install forked PyOxidizer
+        run: cargo install --git https://github.com/fox-it/pyoxidizer --branch esxi-compatibility pyoxidizer --force --locked
+
+      - name: Install dependencies
+        run: /opt/python/cp310-cp310/bin/pip install dissect
+
+      - name: Build binary
+        run: |
+          mkdir -p build/lib/dissect/target/plugins
+          /opt/python/cp310-cp310/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py
+          pyoxidizer build --release --target-triple x86_64-unknown-linux-musl --var flavor standalone --var version ${{ github.event.inputs.tag }}
+          strip build/x86_64-unknown-linux-musl/release/install/acquire
+
+      - name: Verify binary
+        run: build/x86_64-unknown-linux-musl/release/install/acquire-x86_64-unknown-linux-musl --help
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: acquire-linux-musl
+          path: build/x86_64-unknown-linux-musl/release/install/*
+
+  build-windows:
+    name: Build Windows
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~\AppData\Local\pyoxidizer\
+            ~\.cargo\bin\
+            ~\.cargo\registry\index\
+            ~\.cargo\registry\cache\
+            ~\.cargo\git\db\
+          key: windows-cache
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install pyoxidizer dissect
+
+      - name: Build binary
+        run: |
+          mkdir -p build/lib/dissect/target/plugins
+          target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py
+          pyoxidizer build --release --target-triple x86_64-pc-windows-msvc --var flavor standalone_static --var version ${{ github.event.inputs.tag }}
+          strip build/x86_64-pc-windows-msvc/release/install/acquire.exe
+
+      - name: Verify binary
+        run: build/x86_64-pc-windows-msvc/release/install/acquire-x86_64-pc-windows-msvc.exe --help
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: acquire-windows
+          path: build/x86_64-pc-windows-msvc/release/install/*
+
+  build-macos:
+    name: Build macOS
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/pyoxidizer/
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: macos-cache
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install pyoxidizer dissect
+
+      - name: Build binary
+        run: |
+          mkdir -p build/lib/dissect/target/plugins
+          target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py
+          pyoxidizer build --release --target-triple x86_64-apple-darwin --var flavor standalone --var version ${{ github.event.inputs.tag }}
+          pyoxidizer build --release --target-triple aarch64-apple-darwin --var flavor standalone --var version ${{ github.event.inputs.tag }}
+
+          mkdir -p build/universal2-apple-darwin/release/install
+          cp -r build/x86_64-apple-darwin/release/install/* build/universal2-apple-darwin/release/install/
+          lipo -create build/x86_64-apple-darwin/release/install/acquire build/aarch64-apple-darwin/release/install/acquire -output build/universal2-apple-darwin/release/install/acquire-universal2-apple-darwin
+          strip build/universal2-apple-darwin/release/install/acquire-universal2-apple-darwin
+
+      - name: Verify binary
+        run: build/universal2-apple-darwin/release/install/acquire-universal2-apple-darwin --help
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: acquire-macos
+          path: build/universal2-apple-darwin/release/install/*

acquire/acquire.py

@@ -48,6 +48,7 @@
 from acquire.uploaders.plugin import UploaderPlugin, upload_files_using_uploader
 from acquire.uploaders.plugin_registry import UploaderRegistry
 from acquire.utils import (
+    VERSION,
     check_and_set_acquire_args,
     check_and_set_log_args,
     create_argument_parser,
@@ -67,19 +68,13 @@
 
     from dissect.target.filesystem import Filesystem
 
-try:
-    from acquire.version import version
-except ImportError:
-    version = "0.0.dev"
-
 try:
     # Injected by pystandalone builder
     from acquire.config import CONFIG
 except ImportError:
     CONFIG = defaultdict(lambda: None)
 
 
-VERSION = version
 ACQUIRE_BANNER = rf"""
                        _
   __ _  ___ __ _ _   _(_)_ __ ___

acquire/utils.py

@@ -28,6 +28,11 @@
 
     from acquire.uploaders.plugin_registry import UploaderRegistry
 
+try:
+    from acquire.version import version as VERSION
+except ImportError:
+    VERSION = "0.0.dev"
+
 
 class StrEnum(str, Enum):
     """Sortable and serializible string-based enum"""
@@ -189,6 +194,7 @@ def create_argument_parser(profiles: dict, volatile: dict, modules: dict) -> arg
             parser.add_argument(*args, **kwargs)
 
     parser.add_argument("-v", "--verbose", action="count", default=3, help="increase output verbosity")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}")
     return parser
 
 

pyoxidizer.bzl

@@ -0,0 +1,107 @@
+def make_exe():
+    dist = default_python_distribution(flavor=VARS["flavor"], python_version="3.10")
+
+    policy = dist.make_python_packaging_policy()
+    policy.bytecode_optimize_level_two = True
+    policy.file_scanner_classify_files = True
+    policy.resources_location = "in-memory"
+
+    python_config = dist.make_python_interpreter_config()
+    python_config.oxidized_importer = True
+    python_config.filesystem_importer = False
+    python_config.run_module = "acquire.acquire"
+
+    exe = dist.to_python_executable(
+        name="acquire-" + BUILD_TARGET_TRIPLE,
+        packaging_policy=policy,
+        config=python_config,
+    )
+    exe.windows_runtime_dlls_mode = "when-present"
+
+    # The default dependency list of acquire doesn't include enough, and full includes some that are hard to package
+    pip_args = [
+        "acquire==" + VARS["version"],
+        "dissect.cstruct",
+        "dissect.eventlog",
+        "dissect.evidence",
+        "dissect.extfs",
+        "dissect.fat",
+        "dissect.ffs",
+        "dissect.hypervisor",
+        "dissect.ntfs",
+        "dissect.regf",
+        "dissect.sql",
+        "dissect.squashfs",
+        "dissect.target",
+        "dissect.util",
+        "dissect.vmfs",
+        "dissect.volume",
+        "dissect.xfs",
+        "minio",
+    ]
+
+    # If you want to build acquire from the local source directory, uncomment this and remove "acquire" from pip_args
+    # exe.add_python_resources(exe.read_package_root(CWD, ["acquire"]))
+
+    # Lie about our platform to get cross-compilation to work (msgpack fails to download otherwise)
+    if BUILD_TARGET_TRIPLE == "x86_64-pc-windows-msvc":
+        pip_args += ["--platform", "win_amd64"]
+    elif BUILD_TARGET_TRIPLE == "i686-pc-windows-msvc":
+        pip_args += ["--platform", "win32"]
+    elif BUILD_TARGET_TRIPLE == "x86_64-unknown-linux-musl":
+        pip_args += ["--platform", "manylinux2014_x86_64"]
+
+    # Use pip_download for all the dependencies
+    for resource in exe.pip_download(pip_args):
+        # Discard msgpack's extension, it has a pure Python fallback
+        if resource.name == "msgpack._cmsgpack":
+            continue
+
+        # The crypto portions of minio aren't needed for normal usage
+        if resource.name == "_cffi_backend" or resource.name.startswith("_argon2_cffi_bindings"):
+            continue
+
+        # Discard pycryptodome fully for the time being, unsure how to make it play nicely
+        if resource.name.startswith("Crypto"):
+            continue
+
+        exe.add_python_resource(resource)
+
+    # Add the _pluginlist.py "overlay"
+    # This is created by the CI, if you want to build manually, be sure to generate it:
+    # mkdir -p build/lib/dissect/target/plugins/ && target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py
+    exe.add_python_resources(exe.read_package_root("build/lib", ["dissect"]))
+
+    # If you want to add your own configuration customizations, you can put them in here
+    # 'arguments' allows you to override specific arguments by default, e.g. ['--compress']
+    # 'public_key' allows you to include a PEM encoded RSA public key for output encryption
+    # NOTE: pycryptodome is not currently packaged in this PyOxidizer configuration, so output encryption is unavailable
+    # 'upload' allows you to configure upload credentials
+    # Example AWS S3 configuration: {'mode': 'cloud', 'endpoint': 's3.amazonaws.com', 'access_id': '', 'access_key': '', 'bucket': ''}
+    exe.add_python_resource(
+        exe.make_python_module_source(
+            "acquire.config",
+            "CONFIG = {'arguments': [], 'public_key': '', 'upload': {}}",
+            False
+        )
+    )
+
+    return exe
+
+
+def make_embedded_resources(exe):
+    return exe.to_embedded_resources()
+
+
+def make_install(exe):
+    files = FileManifest()
+    files.add_python_resource(".", exe)
+
+    return files
+
+
+register_target("exe", make_exe)
+register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
+register_target("install", make_install, depends=["exe"], default=True)
+
+resolve_targets()