Feature/builtin compression

conan/api/subapi/cache.py

@@ -6,13 +6,14 @@
 
 from conan.api.model import PackagesList
 from conan.api.output import ConanOutput
-from conan.internal.api.uploader import compress_files
+from conan.internal.api.uploader import compress_files, get_compress_level
 from conan.internal.cache.cache import PkgCache
 from conan.internal.cache.conan_reference_layout import (EXPORT_SRC_FOLDER, EXPORT_FOLDER,
                                                          SRC_FOLDER, METADATA,
                                                          DOWNLOAD_EXPORT_FOLDER)
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.cache.integrity_check import IntegrityChecker
+from conan.internal.paths import COMPRESSIONS
 from conan.internal.rest.download_cache import DownloadCache
 from conan.errors import ConanException
 from conan.api.model import PkgReference
@@ -148,7 +149,11 @@ def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None:
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
         out = ConanOutput()
         mkdir(os.path.dirname(tgz_path))
-        compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int)
+        tgz_name = os.path.basename(tgz_path)
+        compressformat = next((e for e in COMPRESSIONS if tgz_name.endswith(e)), None)
+        if not compressformat:
+            raise ConanException(f"Unsupported compression format for {tgz_name}")
+        compresslevel = get_compress_level(compressformat, global_conf)
         tar_files: dict[str, str] = {}  # {path_in_tar: abs_path}
 
         for ref, packages in package_list.items():
@@ -191,9 +196,9 @@ def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None:
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path),
-                       compresslevel, recursive=True)
+        compress_files(tar_files, tgz_name, os.path.dirname(tgz_path), compresslevel, recursive=True)
         remove(pkglist_path)
+        ConanOutput().success(f"Created cache save file: {tgz_path}")
 
     def restore(self, path) -> PackagesList:
         if not os.path.isfile(path):

conan/internal/api/uploader.py

@@ -2,6 +2,7 @@
 import gzip
 import os
 import shutil
+import sys
 import tarfile
 import time
 
@@ -10,8 +11,8 @@
 from conan.internal.source import retrieve_exports_sources
 from conan.internal.errors import NotFoundException
 from conan.errors import ConanException
-from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
-                                  EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
+from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \
+    EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME
 from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
                                        set_dirty_context_manager, mkdir, human_size)
 
@@ -80,10 +81,37 @@ def _check_upstream_package(self, pref, prev_bundle, remote, force):
                 prev_bundle["upload"] = False
 
 
+def get_compress_level(compressformat, global_conf):
+    if compressformat == "xz":
+        msg = ("The 'xz' compression is experimental. "
+               "Consumers using older Conan versions will not be able to install these packages. "
+               "Feedback is welcome, please report any issues as GitHub tickets.")
+        ConanOutput().warning(msg, warn_tag="experimental")
+    elif compressformat == "zst":
+        msg = ("The 'zst' compression is experimental. "
+               "Consumers installing packages created with this format must use Python >= 3.14. "
+               "Consumers using older Conan or Python versions will not be able to install these "
+               "packages. Feedback is welcome, please report any issues as GitHub tickets.")
+        ConanOutput().warning(msg, warn_tag="experimental")
+
+    if compressformat == "zst" and sys.version_info.minor < 14:
+        raise ConanException("The 'core.upload:compression_format=zst' is only for Python>=3.14")
+    compresslevel = global_conf.get("core:compresslevel", check_type=int)
+    if compresslevel is None and compressformat == "gz":
+        compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int)
+        # do not deprecate yet core.gzip:compresslevel, wait a bit to stabilize core:compresslevel
+    return compresslevel
+
+
 class PackagePreparator:
     def __init__(self, app: ConanApp, global_conf):
         self._app = app
         self._global_conf = global_conf
+        compressformat = self._global_conf.get("core.upload:compression_format", default="gz",
+                                               choices=COMPRESSIONS)
+        compresslevel = get_compress_level(compressformat, global_conf)
+        self._compressformat = compressformat
+        self._compresslevel = compresslevel
 
     def prepare(self, pkg_list, enabled_remotes):
         local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"])
@@ -128,14 +156,6 @@ def _prepare_recipe(self, ref, ref_bundle, conanfile, remotes):
     def _compress_recipe_files(self, layout, ref):
         download_export_folder = layout.download_export()
 
-        output = ConanOutput(scope=str(ref))
-        for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
-            tgz_path = os.path.join(download_export_folder, f)
-            if is_dirty(tgz_path):
-                output.warning("Removing %s, marked as dirty" % f)
-                os.remove(tgz_path)
-                clean_dirty(tgz_path)
-
         export_folder = layout.export()
         files, symlinked_folders = gather_files(export_folder)
         files.update(symlinked_folders)
@@ -159,18 +179,13 @@ def _compress_recipe_files(self, layout, ref):
         files.pop(CONANFILE)
         files.pop(CONAN_MANIFEST)
 
-        def add_tgz(tgz_name, tgz_files):
-            tgz = os.path.join(download_export_folder, tgz_name)
-            if os.path.isfile(tgz):
-                result[tgz_name] = tgz
-            elif tgz_files:
-                compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-                tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
-                result[tgz_name] = tgz
-
-        add_tgz(EXPORT_TGZ_NAME, files)
-        add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files)
+        if files:
+            comp = self._compressed_file(EXPORT_FILE_NAME, files, download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
+        if src_files:
+            comp = self._compressed_file(EXPORT_SOURCES_FILE_NAME, src_files,
+                                         download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
         return result
 
     def _prepare_package(self, pref, prev_bundle):
@@ -181,14 +196,39 @@ def _prepare_package(self, pref, prev_bundle):
         cache_files = self._compress_package_files(pkg_layout, pref)
         prev_bundle["files"] = cache_files
 
+    def _compressed_file(self, filename, files, download_folder, ref):
+        output = ConanOutput(scope=str(ref))
+
+        # Check if there is some existing compressed file already
+        matches = []
+        for extension in COMPRESSIONS:
+            file_name = filename + extension
+            package_file = os.path.join(download_folder, file_name)
+            if is_dirty(package_file):
+                output.warning(f"Removing {file_name}, marked as dirty")
+                os.remove(package_file)
+                clean_dirty(package_file)
+            if os.path.isfile(package_file):
+                matches.append(file_name)
+        if len(matches) > 1:
+            raise ConanException(f"{ref}: Multiple package files found for {filename}: {matches}")
+        if len(matches) == 1:
+            existing = matches[0]
+            if not existing.endswith(self._compressformat):
+                output.info(f"Existing {existing} compressed file, "
+                            f"keeping it, not using '{self._compressformat}' format")
+            return existing
+
+        file_name = filename + self._compressformat
+        package_file = os.path.join(download_folder, file_name)
+        compressed_path = compress_files(files, file_name, download_folder,
+                                         compresslevel=self._compresslevel, scope=str(ref))
+        assert compressed_path == package_file
+        assert os.path.exists(package_file)
+        return file_name
+
     def _compress_package_files(self, layout, pref):
-        output = ConanOutput(scope=str(pref))
         download_pkg_folder = layout.download_package()
-        package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
-        if is_dirty(package_tgz):
-            output.warning("Removing %s, marked as dirty" % PACKAGE_TGZ_NAME)
-            os.remove(package_tgz)
-            clean_dirty(package_tgz)
 
         # Get all the files in that directory
         # existing package
@@ -209,15 +249,8 @@ def _compress_package_files(self, layout, pref):
         files.pop(CONANINFO)
         files.pop(CONAN_MANIFEST)
 
-        if not os.path.isfile(package_tgz):
-            tgz_files = {f: path for f, path in files.items()}
-            compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-            tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
-            assert tgz_path == package_tgz
-            assert os.path.exists(package_tgz)
-
-        return {PACKAGE_TGZ_NAME: package_tgz,
+        compressed_file = self._compressed_file(PACKAGE_FILE_NAME, files, download_pkg_folder, pref)
+        return {compressed_file: os.path.join(download_pkg_folder, compressed_file),
                 CONANINFO: os.path.join(download_pkg_folder, CONANINFO),
                 CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)}
 
@@ -282,21 +315,36 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, compresslevel=None, scope=None, recursive=False):
     t1 = time.time()
-    # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
-    if ref:
-        ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+
+    out = ConanOutput(scope=scope)
+    out.info(f"Compressing {name}")
+
+    if name.endswith("zst"):
+        with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar:  # noqa Py314 only
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        out.debug(f"{name} compressed in {time.time() - t1} time")
+        return tgz_path
+
+    if name.endswith("xz"):
+        # The default to PAX_FORMAT in case of Python 3.7
+        with tarfile.open(tgz_path, "w:xz", preset=compresslevel, format=tarfile.PAX_FORMAT) as tar:
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        out.debug(f"{name} compressed in {time.time() - t1} time")
+        return tgz_path
+
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):
             # recursive is False by default in case it is a symlink to a folder
             tgz.add(abs_path, filename, recursive=recursive)
         tgz.close()
 
-    duration = time.time() - t1
-    ConanOutput().debug(f"{name} compressed in {duration} time")
+    out.debug(f"{name} compressed in {time.time() - t1} time")
     return tgz_path
 
 

conan/internal/model/conf.py

@@ -60,8 +60,11 @@
     "core.net.http:cacert_path": "Path containing a custom Cacert file",
     "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)",
     "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded",
-    # Gzip compression
+    # Compression for `conan upload`
+    "core.upload:compression_format": "The compression format used when uploading Conan packages. "
+                                      "Possible values: 'zst', 'xz', 'gz' (default=gz)",
     "core.gzip:compresslevel": "The Gzip compression level for Conan artifacts (default=9)",
+    "core:compresslevel": "The compression level for Conan artifacts (default zstd=3, gz=9)",
     # Excluded from revision_mode = "scm" dirty and Git().is_dirty() checks
     "core.scm:excluded": "List of excluded patterns for builtin git dirty checks",
     "core.scm:local_url": "By default allows to store local folders as remote url, but not upload them. Use 'allow' for allowing upload and 'block' to completely forbid it",

conan/internal/model/manifest.py

@@ -1,7 +1,8 @@
 import os
 from collections import defaultdict
 
-from conan.internal.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
+from conan.internal.paths import CONAN_MANIFEST, COMPRESSIONS, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \
+    EXPORT_SOURCES_FILE_NAME
 from conan.internal.util.dates import timestamp_now, timestamp_to_str
 from conan.internal.util.files import load, md5, md5sum, save, gather_files
 
@@ -91,8 +92,10 @@ def create(cls, folder, exports_sources_folder=None):
         """
         files, _ = gather_files(folder)
         # The folders symlinks are discarded for the manifest
-        for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME):
-            files.pop(f, None)
+        for f in (PACKAGE_FILE_NAME, EXPORT_FILE_NAME, EXPORT_SOURCES_FILE_NAME):
+            for e in COMPRESSIONS:
+                files.pop(f + e, None)
+        files.pop(CONAN_MANIFEST, None)
 
         file_dict = {}
         for name, filepath in files.items():

conan/internal/paths.py

@@ -86,7 +86,8 @@ def _user_home_from_conanrc_file():
 CONANFILE_TXT = "conanfile.txt"
 CONAN_MANIFEST = "conanmanifest.txt"
 CONANINFO = "conaninfo.txt"
-PACKAGE_TGZ_NAME = "conan_package.tgz"
-EXPORT_TGZ_NAME = "conan_export.tgz"
-EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz"
+PACKAGE_FILE_NAME = "conan_package.t"
+EXPORT_FILE_NAME = "conan_export.t"
+EXPORT_SOURCES_FILE_NAME = "conan_sources.t"
+COMPRESSIONS = "gz", "xz", "zst"
 DATA_YML = "conandata.yml"

conan/internal/rest/remote_manager.py

@@ -1,11 +1,14 @@
 import os
 import shutil
+import sys
+
 from collections import namedtuple
 from typing import List
 
 from requests.exceptions import ConnectionError
 
 from conan.api.model import LOCAL_RECIPES_INDEX
+from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME
 from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex
 from conan.api.model import Remote
 from conan.api.output import ConanOutput
@@ -17,7 +20,6 @@
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
 from conan.internal.util.files import rmdir, human_size
-from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
 from conan.internal.util.files import mkdir, tar_extract
 
 
@@ -86,7 +88,8 @@ def get_recipe(self, ref, remote, metadata=None):
             self._cache.remove_recipe_layout(layout)
             raise
         export_folder = layout.export()
-        tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
+        export_file = next((f for f in zipped_files if f.startswith(EXPORT_FILE_NAME)), None)
+        tgz_file = zipped_files.pop(export_file, None)
 
         if tgz_file:
             uncompress_file(tgz_file, export_folder, scope=str(ref))
@@ -132,7 +135,8 @@ def get_recipe_sources(self, ref, layout, remote):
             return
 
         self._signer.verify(ref, download_folder, files=zipped_files)
-        tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
+        # Only 1 file is guaranteed
+        tgz_file = next(iter(zipped_files.values()))
         uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
 
     def get_package(self, pref, remote, metadata=None):
@@ -178,12 +182,15 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
                                              metadata, only_metadata=False)
             zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
             # quick server package integrity check:
-            for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"):
+            for f in (CONANINFO, CONAN_MANIFEST):
                 if f not in zipped_files:
                     raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}")
+
+            # This is guaranteed to exists, otherwise RestClient would have raised already
+            package_file = next(f for f in zipped_files if PACKAGE_FILE_NAME in f)
             self._signer.verify(pref, download_pkg_folder, zipped_files)
 
-            tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
+            tgz_file = zipped_files.pop(package_file)
             package_folder = layout.package()
             uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
@@ -337,6 +344,9 @@ def _call_remote(self, remote, method, *args, **kwargs):
 
 
 def uncompress_file(src_path, dest_folder, scope=None):
+    if sys.version_info.minor < 14 and src_path.endswith("zst"):
+        raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', "
+                             f"unsupported for Python<3.14 ")
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB