Feature/builtin compression (#19337)

* Add support for zstd compression of binary packages

This change adds zstd support to conan in the following ways:
1. The person or build running `conan upload` can set a config value
   core.upload:compression_format = zstd
   to upload binaries using zstd instead of gzip.
2. The zstd compression is done entirely in Python using a combination
   of tarfile and python-zstandard. Then the file is uploaded as normal.
3. When downloading packages, if a .tar.zst file is encountered, the
   extraction code uses tarfile and python-zstandard to extract.

I chose python-zstandard as the library because that is what urllib3 uses.

* Switch to include python-zstandard in the package requirements

Because zstd decompression is expected to just work if the server has a
.tar.zst file, I am including zstandard in requirements.txt.
https://python-zstandard.readthedocs.io/en/latest/projectinfo.html#state-of-project
recommends that we "Pin the package version to prevent unwanted breakage when this
change occurs!", although I doubt that much will change before an eventual 1.0.

* Add a test case to cover zstd compress and decompress

* Downgrade to 0.20.0 to fix CI

CI is unable to find 0.21.0

* Two small improvements

1. Change requirements.txt to allow either zstandard 0.20 or 0.21. That
   prevents a downgrade for people who already have 0.21 installed, while
   also allowing CI to find 0.20.
2. Move compressformat parameter earlier in compress_files() function.
   It made a bit more sense to have it earlier; as long as consumers are
   correctly using positional kwargs, it shouldn't break anyone.

* Address review feedback

* Add file missed by merge

* Fix typo in parameter which broke tests

* A few more small fixes in hopes of unbreaking the build

1. Fix bad merge causing uploader.py change to still refer to `self._app.cache.new_config`, when now we are supposed to use `self._global_conf`.
2. Change two output calls in uploader.py to only output the package file basename to be consistent with other existing log lines.
3. Use double quotes instead of single quotes to be more consistent with existing code.

* Some more improvements

1. Downgrade bufsize to 32KB because that performs well for compression and
   decompression. The values don't need to be the same, but it happened to be
   the best value in both compression and decompression tests.
2. Use a context manager for stream_reader as I do for stream_writer.
3. Add some comments about the bufsize value.

* Address some of the review feedback

Still need to do some testing though.

* Flush zstd frames around every 128MB

* Fix DeprecationWarning

Newer Python has this warning:

DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives
and reject files or modify their metadata. Use the filter argument to control
this behavior

* wip

* wip

* review

* compression for cache save/restore too

* fix unit test

* fix tests

* fix tests

* fix save/restore with Path

* last review

* fix tests

---------

Co-authored-by: Adam Gross <grossag@vmware.com>

Author: memsharded

conan/api/subapi/cache.py

@@ -6,13 +6,14 @@
 
 from conan.api.model import PackagesList
 from conan.api.output import ConanOutput
-from conan.internal.api.uploader import compress_files
+from conan.internal.api.uploader import compress_files, get_compress_level
 from conan.internal.cache.cache import PkgCache
 from conan.internal.cache.conan_reference_layout import (EXPORT_SRC_FOLDER, EXPORT_FOLDER,
                                                          SRC_FOLDER, METADATA,
                                                          DOWNLOAD_EXPORT_FOLDER)
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.cache.integrity_check import IntegrityChecker
+from conan.internal.paths import COMPRESSIONS
 from conan.internal.rest.download_cache import DownloadCache
 from conan.errors import ConanException
 from conan.api.model import PkgReference
@@ -148,7 +149,11 @@ def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None:
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
         out = ConanOutput()
         mkdir(os.path.dirname(tgz_path))
-        compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int)
+        tgz_name = os.path.basename(tgz_path)
+        compressformat = next((e for e in COMPRESSIONS if tgz_name.endswith(e)), None)
+        if not compressformat:
+            raise ConanException(f"Unsupported compression format for {tgz_name}")
+        compresslevel = get_compress_level(compressformat, global_conf)
         tar_files: dict[str, str] = {}  # {path_in_tar: abs_path}
 
         for ref, packages in package_list.items():
@@ -191,9 +196,9 @@ def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None:
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path),
-                       compresslevel, recursive=True)
+        compress_files(tar_files, tgz_name, os.path.dirname(tgz_path), compresslevel, recursive=True)
         remove(pkglist_path)
+        ConanOutput().success(f"Created cache save file: {tgz_path}")
 
     def restore(self, path) -> PackagesList:
         if not os.path.isfile(path):

conan/internal/api/uploader.py

@@ -2,6 +2,7 @@
 import gzip
 import os
 import shutil
+import sys
 import tarfile
 import time
 
@@ -10,8 +11,8 @@
 from conan.internal.source import retrieve_exports_sources
 from conan.internal.errors import NotFoundException
 from conan.errors import ConanException
-from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
-                                  EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
+from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \
+    EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME
 from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
                                        set_dirty_context_manager, mkdir, human_size)
 
@@ -80,10 +81,37 @@ def _check_upstream_package(self, pref, prev_bundle, remote, force):
                 prev_bundle["upload"] = False
 
 
+def get_compress_level(compressformat, global_conf):
+    if compressformat == "xz":
+        msg = ("The 'xz' compression is experimental. "
+               "Consumers using older Conan versions will not be able to install these packages. "
+               "Feedback is welcome, please report any issues as GitHub tickets.")
+        ConanOutput().warning(msg, warn_tag="experimental")
+    elif compressformat == "zst":
+        msg = ("The 'zst' compression is experimental. "
+               "Consumers installing packages created with this format must use Python >= 3.14. "
+               "Consumers using older Conan or Python versions will not be able to install these "
+               "packages. Feedback is welcome, please report any issues as GitHub tickets.")
+        ConanOutput().warning(msg, warn_tag="experimental")
+
+    if compressformat == "zst" and sys.version_info.minor < 14:
+        raise ConanException("The 'core.upload:compression_format=zst' is only for Python>=3.14")
+    compresslevel = global_conf.get("core:compresslevel", check_type=int)
+    if compresslevel is None and compressformat == "gz":
+        compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int)
+        # do not deprecate yet core.gzip:compresslevel, wait a bit to stabilize core:compresslevel
+    return compresslevel
+
+
 class PackagePreparator:
     def __init__(self, app: ConanApp, global_conf):
         self._app = app
         self._global_conf = global_conf
+        compressformat = self._global_conf.get("core.upload:compression_format", default="gz",
+                                               choices=COMPRESSIONS)
+        compresslevel = get_compress_level(compressformat, global_conf)
+        self._compressformat = compressformat
+        self._compresslevel = compresslevel
 
     def prepare(self, pkg_list, enabled_remotes):
         local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"])
@@ -128,14 +156,6 @@ def _prepare_recipe(self, ref, ref_bundle, conanfile, remotes):
     def _compress_recipe_files(self, layout, ref):
         download_export_folder = layout.download_export()
 
-        output = ConanOutput(scope=str(ref))
-        for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
-            tgz_path = os.path.join(download_export_folder, f)
-            if is_dirty(tgz_path):
-                output.warning("Removing %s, marked as dirty" % f)
-                os.remove(tgz_path)
-                clean_dirty(tgz_path)
-
         export_folder = layout.export()
         files, symlinked_folders = gather_files(export_folder)
         files.update(symlinked_folders)
@@ -159,18 +179,13 @@ def _compress_recipe_files(self, layout, ref):
         files.pop(CONANFILE)
         files.pop(CONAN_MANIFEST)
 
-        def add_tgz(tgz_name, tgz_files):
-            tgz = os.path.join(download_export_folder, tgz_name)
-            if os.path.isfile(tgz):
-                result[tgz_name] = tgz
-            elif tgz_files:
-                compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-                tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
-                result[tgz_name] = tgz
-
-        add_tgz(EXPORT_TGZ_NAME, files)
-        add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files)
+        if files:
+            comp = self._compressed_file(EXPORT_FILE_NAME, files, download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
+        if src_files:
+            comp = self._compressed_file(EXPORT_SOURCES_FILE_NAME, src_files,
+                                         download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
         return result
 
     def _prepare_package(self, pref, prev_bundle):
@@ -181,14 +196,39 @@ def _prepare_package(self, pref, prev_bundle):
         cache_files = self._compress_package_files(pkg_layout, pref)
         prev_bundle["files"] = cache_files
 
+    def _compressed_file(self, filename, files, download_folder, ref):
+        output = ConanOutput(scope=str(ref))
+
+        # Check if there is some existing compressed file already
+        matches = []
+        for extension in COMPRESSIONS:
+            file_name = filename + extension
+            package_file = os.path.join(download_folder, file_name)
+            if is_dirty(package_file):
+                output.warning(f"Removing {file_name}, marked as dirty")
+                os.remove(package_file)
+                clean_dirty(package_file)
+            if os.path.isfile(package_file):
+                matches.append(file_name)
+        if len(matches) > 1:
+            raise ConanException(f"{ref}: Multiple package files found for {filename}: {matches}")
+        if len(matches) == 1:
+            existing = matches[0]
+            if not existing.endswith(self._compressformat):
+                output.info(f"Existing {existing} compressed file, "
+                            f"keeping it, not using '{self._compressformat}' format")
+            return existing
+
+        file_name = filename + self._compressformat
+        package_file = os.path.join(download_folder, file_name)
+        compressed_path = compress_files(files, file_name, download_folder,
+                                         compresslevel=self._compresslevel, scope=str(ref))
+        assert compressed_path == package_file
+        assert os.path.exists(package_file)
+        return file_name
+
     def _compress_package_files(self, layout, pref):
-        output = ConanOutput(scope=str(pref))
         download_pkg_folder = layout.download_package()
-        package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
-        if is_dirty(package_tgz):
-            output.warning("Removing %s, marked as dirty" % PACKAGE_TGZ_NAME)
-            os.remove(package_tgz)
-            clean_dirty(package_tgz)
 
         # Get all the files in that directory
         # existing package
@@ -209,15 +249,8 @@ def _compress_package_files(self, layout, pref):
         files.pop(CONANINFO)
         files.pop(CONAN_MANIFEST)
 
-        if not os.path.isfile(package_tgz):
-            tgz_files = {f: path for f, path in files.items()}
-            compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-            tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
-            assert tgz_path == package_tgz
-            assert os.path.exists(package_tgz)
-
-        return {PACKAGE_TGZ_NAME: package_tgz,
+        compressed_file = self._compressed_file(PACKAGE_FILE_NAME, files, download_pkg_folder, pref)
+        return {compressed_file: os.path.join(download_pkg_folder, compressed_file),
                 CONANINFO: os.path.join(download_pkg_folder, CONANINFO),
                 CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)}
 
@@ -282,21 +315,36 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False):
     t1 = time.time()
-    # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
-    if ref:
-        ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+
+    out = ConanOutput(scope=scope)
+    out.info(f"Compressing {name}")
+
+    if name.endswith("zst"):
+        with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar:  # noqa Py314 only
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        out.debug(f"{name} compressed in {time.time() - t1} time")
+        return tgz_path
+
+    if name.endswith("xz"):
+        # The default to PAX_FORMAT in case of Python 3.7
+        with tarfile.open(tgz_path, "w:xz", preset=compresslevel, format=tarfile.PAX_FORMAT) as tar:
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        out.debug(f"{name} compressed in {time.time() - t1} time")
+        return tgz_path
+
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):
             # recursive is False by default in case it is a symlink to a folder
             tgz.add(abs_path, filename, recursive=recursive)
         tgz.close()
 
-    duration = time.time() - t1
-    ConanOutput().debug(f"{name} compressed in {duration} time")
+    out.debug(f"{name} compressed in {time.time() - t1} time")
     return tgz_path
 
 

conan/internal/model/conf.py

@@ -60,8 +60,11 @@
     "core.net.http:cacert_path": "Path containing a custom Cacert file",
     "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)",
     "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded",
-    # Gzip compression
+    # Compression for `conan upload`
+    "core.upload:compression_format": "The compression format used when uploading Conan packages. "
+                                      "Possible values: 'zst', 'xz', 'gz' (default=gz)",
     "core.gzip:compresslevel": "The Gzip compression level for Conan artifacts (default=9)",
+    "core:compresslevel": "The compression level for Conan artifacts (default zstd=3, gz=9)",
     # Excluded from revision_mode = "scm" dirty and Git().is_dirty() checks
     "core.scm:excluded": "List of excluded patterns for builtin git dirty checks",
     "core.scm:local_url": "By default allows to store local folders as remote url, but not upload them. Use 'allow' for allowing upload and 'block' to completely forbid it",

conan/internal/model/manifest.py

@@ -1,7 +1,8 @@
 import os
 from collections import defaultdict
 
-from conan.internal.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
+from conan.internal.paths import CONAN_MANIFEST, COMPRESSIONS, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \
+    EXPORT_SOURCES_FILE_NAME
 from conan.internal.util.dates import timestamp_now, timestamp_to_str
 from conan.internal.util.files import load, md5, md5sum, save, gather_files
 
@@ -91,8 +92,10 @@ def create(cls, folder, exports_sources_folder=None):
         """
         files, _ = gather_files(folder)
         # The folders symlinks are discarded for the manifest
-        for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME):
-            files.pop(f, None)
+        for f in (PACKAGE_FILE_NAME, EXPORT_FILE_NAME, EXPORT_SOURCES_FILE_NAME):
+            for e in COMPRESSIONS:
+                files.pop(f + e, None)
+        files.pop(CONAN_MANIFEST, None)
 
         file_dict = {}
         for name, filepath in files.items():

conan/internal/paths.py

@@ -86,7 +86,8 @@ def _user_home_from_conanrc_file():
 CONANFILE_TXT = "conanfile.txt"
 CONAN_MANIFEST = "conanmanifest.txt"
 CONANINFO = "conaninfo.txt"
-PACKAGE_TGZ_NAME = "conan_package.tgz"
-EXPORT_TGZ_NAME = "conan_export.tgz"
-EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz"
+PACKAGE_FILE_NAME = "conan_package.t"
+EXPORT_FILE_NAME = "conan_export.t"
+EXPORT_SOURCES_FILE_NAME = "conan_sources.t"
+COMPRESSIONS = "gz", "xz", "zst"
 DATA_YML = "conandata.yml"

conan/internal/rest/remote_manager.py

@@ -1,11 +1,14 @@
 import os
 import shutil
+import sys
+
 from collections import namedtuple
 from typing import List
 
 from requests.exceptions import ConnectionError
 
 from conan.api.model import LOCAL_RECIPES_INDEX
+from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME
 from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex
 from conan.api.model import Remote
 from conan.api.output import ConanOutput
@@ -17,7 +20,6 @@
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
 from conan.internal.util.files import rmdir, human_size
-from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
 from conan.internal.util.files import mkdir, tar_extract
 
 
@@ -86,7 +88,8 @@ def get_recipe(self, ref, remote, metadata=None):
             self._cache.remove_recipe_layout(layout)
             raise
         export_folder = layout.export()
-        tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
+        export_file = next((f for f in zipped_files if f.startswith(EXPORT_FILE_NAME)), None)
+        tgz_file = zipped_files.pop(export_file, None)
 
         if tgz_file:
             uncompress_file(tgz_file, export_folder, scope=str(ref))
@@ -132,7 +135,8 @@ def get_recipe_sources(self, ref, layout, remote):
             return
 
         self._signer.verify(ref, download_folder, files=zipped_files)
-        tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
+        # Only 1 file is guaranteed
+        tgz_file = next(iter(zipped_files.values()))
         uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
 
     def get_package(self, pref, remote, metadata=None):
@@ -178,12 +182,15 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
                                              metadata, only_metadata=False)
             zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
             # quick server package integrity check:
-            for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"):
+            for f in (CONANINFO, CONAN_MANIFEST):
                 if f not in zipped_files:
                     raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}")
+
+            # This is guaranteed to exists, otherwise RestClient would have raised already
+            package_file = next(f for f in zipped_files if PACKAGE_FILE_NAME in f)
             self._signer.verify(pref, download_pkg_folder, zipped_files)
 
-            tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
+            tgz_file = zipped_files.pop(package_file)
             package_folder = layout.package()
             uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
@@ -337,6 +344,9 @@ def _call_remote(self, remote, method, *args, **kwargs):
 
 
 def uncompress_file(src_path, dest_folder, scope=None):
+    if sys.version_info.minor < 14 and src_path.endswith("zst"):
+        raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', "
+                             f"unsupported for Python<3.14 ")
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB