Add support for detecting archives by signature

Detect installable archive type by "magic number" file signatures if an
extension is not present.

This fixes downloads from private Github releases, which have no file
extension.

Author: bennettgoble

autobuild/autobuild_tool_install.py

@@ -23,6 +23,7 @@
 from autobuild import autobuild_base, common, configfile
 from autobuild.autobuild_tool_source_environment import get_enriched_environment
 from autobuild.hash_algorithms import verify_hash
+from autobuild import filetype
 
 logger = logging.getLogger('autobuild.install')
 
@@ -439,12 +440,15 @@ def _default_metadata_for_package(package_file: str, package = None):
 
 
 def open_archive(filename: str) -> tarfile.TarFile | zipfile.ZipFile:
-    if filename.endswith(".tar.zst"):
+    f_type = filetype.detect_archive_type(filename)
+
+    if f_type == filetype.ArchiveType.ZST:
         return common.ZstdTarFile(filename, "r")
-    elif filename.endswith(".zip"):
+
+    if f_type == filetype.ArchiveType.ZIP:
         return zipfile.ZipFile(filename, "r")
-    else:
-        return tarfile.open(filename, "r")
+
+    return tarfile.open(filename, "r")
 
 
 class ExtractPackageResults:

autobuild/filetype.py

@@ -0,0 +1,49 @@
+"""Utilities for detecting file types"""
+
+class ArchiveType:
+    GZ = "gz"
+    BZ2 = "bz2"
+    ZIP = "zip"
+    ZST = "zst"
+
+
+# File signatures used for sniffing archive type
+# https://www.garykessler.net/library/file_sigs.html
+_ARCHIVE_MAGIC_NUMBERS = {
+    b"\x1f\x8b\x08": ArchiveType.GZ,
+    b"\x42\x5a\x68": ArchiveType.BZ2,
+    b"\x50\x4b\x03\x04": ArchiveType.ZIP,
+    b"\x28\xb5\x2f\xfd": ArchiveType.ZST,
+}
+
+_ARCHIVE_MAGIC_NUMBERS_MAX = max(len(x) for x in _ARCHIVE_MAGIC_NUMBERS)
+
+
+def _archive_type_from_signature(filename: str):
+    """Sniff archive type using file signature"""
+    with open(filename, "rb") as f:
+        head = f.read(_ARCHIVE_MAGIC_NUMBERS_MAX)
+        for magic, f_type in _ARCHIVE_MAGIC_NUMBERS.items():
+            if head.startswith(magic):
+                return f_type 
+    return None
+
+
+def _archive_type_from_extension(filename: str):
+    if filename.endswith(".tar.gz"):
+        return ArchiveType.GZ
+    if filename.endswith(".tar.bz2"):
+        return ArchiveType.BZ2
+    if filename.endswith(".tar.zst"):
+        return ArchiveType.ZST
+    if filename.endswith(".zip"):
+        return ArchiveType.ZIP
+    return None
+
+
+def detect_archive_type(filename: str):
+    """Given a filename, detect its ArchiveType using file extension and signature."""
+    f_type = _archive_type_from_extension(filename)
+    if f_type:
+        return f_type
+    return _archive_type_from_signature(filename)

tests/data/archive.tar.bz2

@@ -0,0 +1,32 @@
+import shutil
+from os import path
+from pathlib import Path
+from tests.basetest import temp_dir
+
+import pytest
+from autobuild import filetype
+
+
+_DATA_DIR = Path(__file__).parent / "data"
+
+_ARCHIVE_TEST_CASES = (
+    (path.join(_DATA_DIR, "archive.tar.bz2"), filetype.ArchiveType.BZ2),
+    (path.join(_DATA_DIR, "archive.tar.gz"), filetype.ArchiveType.GZ),
+    (path.join(_DATA_DIR, "archive.tar.zst"), filetype.ArchiveType.ZST),
+    (path.join(_DATA_DIR, "archive.zip"), filetype.ArchiveType.ZIP),
+)
+
+
+@pytest.mark.parametrize("filename,expected_type", _ARCHIVE_TEST_CASES)
+def test_detect_from_extension(filename, expected_type):
+    f_type = filetype.detect_archive_type(filename)
+    assert f_type == expected_type
+
+
+@pytest.mark.parametrize("filename,expected_type", _ARCHIVE_TEST_CASES)
+def test_detect_from_signature(filename, expected_type):
+    with temp_dir() as dir:
+        filename_no_ext = str(Path(dir) / "archive")
+        shutil.copyfile(filename, filename_no_ext)
+        f_type = filetype.detect_archive_type(filename_no_ext)
+        assert f_type == expected_type