Move to different approach: tar encapsulation respecting extensions

Author: perseoGI

conan/api/subapi/cache.py

@@ -181,13 +181,13 @@ def restore(self, path):
         cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf)
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
 
-        compression_plugin = self.conan_api.config.compression_plugin
-        if compression_plugin:
-            compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder,
-                                           conf=self.conan_api.config.global_conf)
-        else:
-            with open(path, mode='rb') as file_handler:
-                tar_extract(file_handler, cache_folder)
+        with open(path, mode="rb") as file_handler:
+            tar_extract(
+                fileobj=file_handler,
+                destination_dir=cache_folder,
+                compression_plugin=self.conan_api.config.compression_plugin,
+                conf=self.conan_api.config.global_conf,
+            )
 
         # Retrieve the package list from the already extracted archive
         pkglist_path = os.path.join(cache_folder, "pkglist.json")

conan/internal/api/uploader.py

@@ -4,6 +4,7 @@
 import shutil
 import tarfile
 import time
+from pathlib import Path
 
 from conan.internal.conan_app import ConanApp
 from conan.api.output import ConanOutput
@@ -12,7 +13,7 @@
 from conan.errors import ConanException
 from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
                                   EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
-from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
+from conan.internal.util.files import (COMPRESSED_PLUGIN_TAR_NAME, clean_dirty, is_dirty, gather_files, remove,
                                        set_dirty_context_manager, mkdir, human_size)
 
 UPLOAD_POLICY_FORCE = "force-upload"
@@ -274,13 +275,26 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
 def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None):
     tgz_path = os.path.join(dest_dir, name)
     if compression_plugin:
-        compression_plugin.tar_compress(
-            archive_path=tgz_path,
+        t1 = time.time()
+        compressed_path = compression_plugin.tar_compress(
+            archive_path=os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME),
             files=files,
             recursive=recursive,
             conf=conf,
             ref=ref,
         )
+        ConanOutput().debug(f"{name} compressed in {time.time() - t1} time in plugin")
+        ConanOutput(scope=str(ref or "")).info(f"Compressing {compressed_path}")
+        t1 = time.time()
+        ConanOutput().debug(f"Wrapping {compressed_path} in {name}")
+        with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
+            tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0)
+            tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive)
+            tgz.close()
+        ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time")
+        # Only remove wrapped if it is different from the tgz_path
+        if compressed_path != os.path.basename(tgz_path):
+            remove(compressed_path)
         return tgz_path
 
     t1 = time.time()

conan/internal/rest/remote_manager.py

@@ -304,12 +304,13 @@ def uncompress_file(src_path, dest_folder, scope="", config_api=None):
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
 
-        if config_api and config_api.compression_plugin:
-            config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder,
-                                                      conf=config_api.global_conf)
-        else:
-            with open(src_path, mode='rb') as file_handler:
-                tar_extract(file_handler, dest_folder)
+        with open(src_path, mode='rb') as file_handler:
+            tar_extract(
+                fileobj=file_handler,
+                destination_dir=dest_folder,
+                compression_plugin=config_api.compression_plugin if config_api and config_api.compression_plugin else None,
+                conf=config_api.global_conf if config_api else None
+            )
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

conan/internal/util/files.py

@@ -1,4 +1,6 @@
 import errno
+from pathlib import Path
+import tempfile
 import gzip
 import hashlib
 import os
@@ -11,10 +13,13 @@
 
 from contextlib import contextmanager
 
+from conan.api.output import ConanOutput
 from conan.errors import ConanException
 
 _DIRTY_FOLDER = ".dirty"
 
+# Name (without extension) of the tar file to be created by the compression plugin
+COMPRESSED_PLUGIN_TAR_NAME = "__conan_plugin_compressed_contents__"
 
 def set_dirty(folder):
     dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER
@@ -256,20 +261,43 @@ def mkdir(path):
     os.makedirs(path)
 
 
-def tar_extract(fileobj, destination_dir):
-    try:
-        the_tar = tarfile.open(fileobj=fileobj)
-        # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to
-        # "could not change modification time", with time=0
-        # the_tar.errorlevel = 2  # raise exception if any error
-        the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
-        the_tar.extractall(path=destination_dir)
-        the_tar.close()
-    except tarfile.ReadError:
-        raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}. The file compression is not recogniced.\n"
-                             "This file could have been compressed using a `compression` plugin.\n"
+def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None):
+    if compression_plugin:
+        _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf)
+        return
+    the_tar = tarfile.open(fileobj=fileobj)
+    # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to
+    # "could not change modification time", with time=0
+    # the_tar.errorlevel = 2  # raise exception if any error
+    the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
+    the_tar.extractall(path=destination_dir)
+    the_tar.close()
+    if Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*"):
+        raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n"
+                             "This file has been compressed using a `compression` plugin.\n"
                              "If your organization uses this plugin, ensure it is correctly installed on your environment.")
 
+def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf):
+    """First remove tar.gz wrapper and then call the plugin to extract"""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        t1 = time.time()
+        the_tar = tarfile.open(fileobj=fileobj)
+        the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
+        the_tar.extractall(path=temp_dir)
+        # Check if the tar was compressed with the compression plugin by checking the existence of
+        # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin)
+        if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")):
+            # Get the only extracted file: the plugin tar
+            plugin_tar_path = os.path.join(temp_dir, the_tar.getnames()[0])
+            the_tar.close()
+            ConanOutput().debug(f"Unwrapped in {time.time() - t1} time")
+            t1 = time.time()
+            compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf)
+            ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin")
+        else:
+            # The tar was not compressed using the plugin, copy files to destination
+            from conan.tools.files import copy
+            copy(None, pattern="*", src=temp_dir, dst=destination_dir)
 
 def merge_directories(src, dst):
     from conan.tools.files import copy

test/integration/extensions/test_compression_plugin.py

@@ -1,7 +1,7 @@
 import os
 import textwrap
 
-from conan.internal.util.files import tar_extract
+from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, tar_extract
 from conan.test.assets.genconanfile import GenConanfile
 from conan.test.utils.tools import TestClient
 
@@ -26,7 +26,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs):
         }
     )
     c.run("create .")
-    c.run("cache save 'pkg/*'", assert_error=True)
+    c.run("cache save 'pkg/*:*'", assert_error=True)
     assert (
         "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions"
         in c.out
@@ -48,13 +48,15 @@ def test_compression_plugin_correctly_load():
 
         # xz compression
         def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs):
+            archive_path += ".xz"
             name = os.path.basename(archive_path)
             ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)")
             compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None
             kwargs = {"preset": compresslevel} if compresslevel else {}
             with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz:
                 for filename, abs_path in sorted(files.items()):
                     tgz.add(abs_path, filename, recursive=True)
+            return archive_path
 
         def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs):
             ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)")
@@ -75,24 +77,25 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs):
         }
     )
     c.run("create .")
-    c.run("cache save 'pkg/*'")
-    assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out
+    c.run("cache save 'pkg/*:*'")
+    print(c.out)
+    assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out
     c.run("remove pkg/* -c")
     c.run("cache restore conan_cache_save.tgz")
-    assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out
+    assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out
     c.run("list pkg/1.0")
     assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out
 
     # Remove pre existing tgz to force a recompression
     c.run("remove pkg/* -c")
     c.run("create .")
     # Check the plugin is also used on remote interactions
-    c.run("upload * -r=default -c")
-    assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out
+    c.run("upload *:* -r=default -c")
+    assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out
     assert "pkg/1.0: Uploading recipe" in c.out
     c.run("remove pkg/* -c")
     c.run("download 'pkg/*' -r=default")
-    assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out
+    assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out
 
 
 def test_compression_plugin_tar_not_compatible_with_builtin():
@@ -111,6 +114,7 @@ def test_compression_plugin_tar_not_compatible_with_builtin():
         # zip compression
         def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs):
             # compress files using zipfile library taking into account recursive
+            archive_path += ".zip"
             name = os.path.basename(archive_path)
             compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None
             ConanOutput().info(f"Compressing {name} using compression plugin (zip)")
@@ -121,6 +125,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs):
                         zipf.write(abs_path, arcname)
                     else:
                         zipf.write(abs_path, filename)
+            return archive_path
 
         def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs):
             # extract tar using zipfile library
@@ -139,13 +144,13 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs):
         }
     )
     c.run("create .")
-    c.run("cache save 'pkg/*'")
+    c.run("cache save 'pkg/*:*'")
     c.run("remove pkg/* -c")
     os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py"))
     c.run("cache restore conan_cache_save.tgz", assert_error=True)
     assert (
-        "Error while extracting conan_cache_save.tgz. The file compression is not recogniced.\n"
-        "This file could have been compressed using a `compression` plugin.\n"
+        "Error while extracting conan_cache_save.tgz.\n"
+        "This file has been compressed using a `compression` plugin.\n"
         "If your organization uses this plugin, ensure it is correctly installed on your environment."
     ) in c.out
 
@@ -160,13 +165,15 @@ def test_compress_in_subdirectory():
         from conan.api.output import ConanOutput
         def tar_compress(archive_path, files, recursive, *args, **kwargs):
             # compress files using tarfile putting all content in a `conan/` subfolder
+            archive_path += ".tgz"
             name = os.path.basename(archive_path)
             ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder")
             with open(archive_path, "wb") as tgz_handle:
                 tgz = tarfile.open(name, "w", fileobj=tgz_handle)
                 for filename, abs_path in sorted(files.items()):
                     tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive)
                 tgz.close()
+            return archive_path
 
         def tar_extract(archive_path, dest_dir, *args, **kwargs):
             ConanOutput().info(f"Decompressing {archive_path} in conan subfolder")
@@ -189,12 +196,6 @@ def tar_extract(archive_path, dest_dir, *args, **kwargs):
         }
     )
     c.run("create .")
-    c.run("cache save 'pkg/*'")
+    c.run("cache save 'pkg/*:*'")
     c.run("remove pkg/* -c")
     c.run("cache restore conan_cache_save.tgz")
-    with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler:
-        dest_dir = os.path.join(c.cache_folder, "extracted")
-        tar_extract(file_handler, dest_dir)
-    assert os.listdir(dest_dir) == ["conan"]
-    assert os.path.exists(os.path.join(dest_dir, "conan", "pkglist.json"))
-