Merge pull request #3416 from trailofbits/importlib-for-resource-loading

Switches to using importlib for resource loading

Author: ESultanik

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest] # windows-latest, macos-latest,
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     runs-on: ${{ matrix.os }}
 

polyfile/kaitai/parser.py

@@ -1,23 +1,30 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
+from importlib import resources
 import importlib.util
 import inspect
 from io import BufferedReader, BytesIO
 import json
 from pathlib import Path
+import sys
 from typing import Any, Dict, Iterator, List, Optional, Set, Type, Union
 
+from . import parsers
 from .compiler import CompiledKSY
 from ..fileutils import FileStream
 
 from kaitaistruct import KaitaiStruct
 
-PARSER_DIR: Path = Path(__file__).absolute().parent / "parsers"
-MANIFEST_FILE: Path = PARSER_DIR / "manifest.json"
+with resources.path(parsers, "manifest.json") as manifest_path:
+    PARSER_DIR: Path = manifest_path.parent
+if sys.version_info >= (3, 9):
+    with (resources.files(parsers) / "manifest.json").open("r") as f:
+        MANIFEST: Dict[str, Dict[str, Any]] = json.load(f)
+else:
+    with resources.open_text(parsers, "manifest.json") as f:
+        MANIFEST = json.load(f)
 
-with open(MANIFEST_FILE, "r") as f:
-    MANIFEST: Dict[str, Dict[str, Any]] = json.load(f)
 COMPILED_INFO_BY_KSY: Dict[str, CompiledKSY] = {
     ksy_path: CompiledKSY(
         class_name=component["class_name"],

polyfile/magic.py

@@ -13,6 +13,7 @@
 import csv
 from datetime import datetime
 from enum import Enum, IntFlag
+from importlib import resources
 from io import StringIO
 import json
 import logging
@@ -34,6 +35,8 @@
 from .logger import getStatusLogger, TRACE
 from .repl import ANSIColor, ANSIWriter
 
+from . import magic_defs
+
 
 if sys.version_info < (3, 9):
     from typing import Pattern
@@ -43,12 +46,27 @@
 
 log = getStatusLogger("libmagic")
 
-DEFS_DIR: Path = Path(__file__).absolute().parent / "magic_defs"
+
+if sys.version_info < (3, 11):
+    def get_resource_path(name: str) -> Path:
+        with resources.path(magic_defs, name) as path:
+            return path
+
+    def get_resource_contents(package):
+        return resources.contents(package)
+else:
+    def get_resource_path(name: str) -> Path:
+        with resources.as_file(resources.files(magic_defs).joinpath(name)) as f:
+            return f
+
+    def get_resource_contents(package):
+        return (resource.name for resource in resources.files(package).iterdir() if resource.is_file())
+
 
 MAGIC_DEFS: List[Path] = [
-    path
-    for path in DEFS_DIR.glob("*")
-    if path.name not in ("COPYING", "magic.mgc") and not path.name.startswith(".")
+    get_resource_path(resource_name)
+    for resource_name in get_resource_contents(magic_defs)
+    if resource_name not in ("COPYING", "magic.mgc", "__pycache__") and not resource_name.startswith(".")
 ]
 
 

polyfile/magic_defs/__init__.py

@@ -20,7 +20,7 @@
     long_description_content_type="text/markdown",
     url='https://github.com/trailofbits/polyfile',
     author='Trail of Bits',
-    version="0.5.3",
+    version="0.5.4",
     packages=find_packages(exclude=("tests",)),
     python_requires='>=3.8',
     install_requires=[