Merge pull request #1569 from pierluigilenoci/feat/zstd-compression-level

Add --compress-debug-sections=zstd:N for configurable zstd level

Author: rui314

docs/mold.md

@@ -506,9 +506,12 @@ but as `-o magic`.
 * `--no-build-id`:
   Synonym for `--build-id=none`.
 
-* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zstd` | `none` ]:
+* `--compress-debug-sections`=[ `zlib` | `zlib-gabi` | `zstd` | `zstd:`_N_ | `none` ]:
   Compress DWARF debug info (`.debug_*` sections) using the zlib or zstd
-  compression algorithm. `zlib-gabi` is an alias for `zlib`.
+  compression algorithm. `zlib-gabi` is an alias for `zlib`. `zstd:`_N_
+  selects the zstd compression level, where _N_ is between 1 and 22.
+  Higher levels achieve better compression ratios but are slower.
+  `zstd` is equivalent to `zstd:3`.
 
 * `--defsym`=_symbol_=_value_:
   Define _symbol_ as an alias for _value_.

lib/compress.cc

@@ -139,22 +139,21 @@ void ZlibCompressor::write_to(u8 *buf) {
   *(ub32 *)(end - 4) = checksum;
 }
 
-static std::span<u8> zstd_compress(std::span<u8> input) {
+static std::span<u8> zstd_compress(std::span<u8> input, int level) {
   i64 bufsize = ZSTD_COMPRESSBOUND(input.size());
   u8 *buf = new u8[bufsize];
-  int level = 3; // compression level; must be between 1 to 22
   size_t sz = ZSTD_compress(buf, bufsize, input.data(), input.size(), level);
   assert(!ZSTD_isError(sz));
   return {buf, sz};
 }
 
-ZstdCompressor::ZstdCompressor(u8 *buf, i64 size) {
+ZstdCompressor::ZstdCompressor(u8 *buf, i64 size, i64 level) {
   std::vector<std::span<u8>> inputs = split(std::span(buf, size));
   shards.resize(inputs.size());
 
   // Compress each shard
   tbb::parallel_for((i64)0, (i64)inputs.size(), [&](i64 i) {
-    shards[i] = zstd_compress(inputs[i]);
+    shards[i] = zstd_compress(inputs[i], level);
   });
 
   compressed_size = 0;

lib/lib.h

@@ -674,7 +674,7 @@ class ZlibCompressor : public Compressor {
 
 class ZstdCompressor : public Compressor {
 public:
-  ZstdCompressor(u8 *buf, i64 size);
+  ZstdCompressor(u8 *buf, i64 size, i64 level = 3);
   void write_to(u8 *buf) override;
 };
 

src/cmdline.cc

@@ -89,7 +89,7 @@ static const char helpmsg[] = R"(
   --color-diagnostics=[auto,always,never]
                               Use colors in diagnostics
   --color-diagnostics         Alias for --color-diagnostics=always
-  --compress-debug-sections [none,zlib,zlib-gabi,zstd]
+  --compress-debug-sections [none,zlib,zlib-gabi,zstd,zstd:1,...,zstd:22]
                               Compress .debug_* sections
   --dc                        Ignored
   --dependency-file=FILE      Write Makefile-style dependency rules to FILE
@@ -1039,14 +1039,22 @@ std::vector<std::string> parse_nonpositional_args(Context<E> &ctx) {
     } else if (read_flag("zero-to-bss")) {
       ctx.arg.zero_to_bss = true;
     } else if (read_arg("compress-debug-sections")) {
-      if (arg == "zlib" || arg == "zlib-gabi")
+      if (arg == "zlib" || arg == "zlib-gabi") {
         ctx.arg.compress_debug_sections = ELFCOMPRESS_ZLIB;
-      else if (arg == "zstd")
+      } else if (arg == "zstd") {
         ctx.arg.compress_debug_sections = ELFCOMPRESS_ZSTD;
-      else if (arg == "none")
+      } else if (arg.starts_with("zstd:")) {
+        ctx.arg.compress_debug_sections = ELFCOMPRESS_ZSTD;
+        i64 level = parse_number(ctx, "compress-debug-sections", arg.substr(5));
+        if (level < 1 || level > 22)
+          Fatal(ctx) << "invalid --compress-debug-sections argument: " << arg
+                     << " (zstd level must be between 1 and 22)";
+        ctx.arg.compress_debug_sections_level = level;
+      } else if (arg == "none") {
         ctx.arg.compress_debug_sections = ELFCOMPRESS_NONE;
-      else
+      } else {
         Fatal(ctx) << "invalid --compress-debug-sections argument: " << arg;
+      }
     } else if (read_arg("wrap")) {
       ctx.arg.wrap.insert(arg);
     } else if (read_flag("omagic") || read_flag("N")) {

src/mold.h

@@ -2420,6 +2420,7 @@ struct Context {
     bool z_text = false;
     bool zero_to_bss = false;
     i64 compress_debug_sections = ELFCOMPRESS_NONE;
+    i64 compress_debug_sections_level = 3;
     i64 filler = -1;
     i64 spare_dynamic_tags = 5;
     i64 spare_program_headers = 0;

src/output-chunks.cc

@@ -2863,7 +2863,8 @@ CompressedSection<E>::CompressedSection(Context<E> &ctx, Chunk<E> &chunk) {
   if (ctx.arg.compress_debug_sections == ELFCOMPRESS_ZLIB)
     compressor.reset(new ZlibCompressor(buf.get(), chunk.shdr.sh_size));
   else
-    compressor.reset(new ZstdCompressor(buf.get(), chunk.shdr.sh_size));
+    compressor.reset(new ZstdCompressor(buf.get(), chunk.shdr.sh_size,
+                                        ctx.arg.compress_debug_sections_level));
 
   // Compute header field values
   chdr.ch_type = ctx.arg.compress_debug_sections;

test/compress-debug-sections-zstd-level.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+# arm-linux-gnueabihf-objcopy crashes on x86-64
+[[ $MACHINE = arm* ]] && skip
+[ $MACHINE = riscv32 ] && skip
+
+command -v zstdcat >& /dev/null || skip
+
+cat <<EOF | $CC -c -g -o $t/a.o -xc -
+#include <stdio.h>
+
+int main() {
+  printf("Hello world\n");
+  return 0;
+}
+EOF
+
+# Test zstd:1 (lowest level)
+$CC -B. -o $t/exe1 $t/a.o -Wl,--compress-debug-sections=zstd:1
+$OBJCOPY --dump-section .debug_info=$t/debug_info1 $t/exe1
+dd if=$t/debug_info1 of=$t/debug_info1.zstd bs=24 skip=1 status=none
+zstdcat $t/debug_info1.zstd > /dev/null
+
+# Test zstd:19 (high level)
+$CC -B. -o $t/exe19 $t/a.o -Wl,--compress-debug-sections=zstd:19
+$OBJCOPY --dump-section .debug_info=$t/debug_info19 $t/exe19
+dd if=$t/debug_info19 of=$t/debug_info19.zstd bs=24 skip=1 status=none
+zstdcat $t/debug_info19.zstd > /dev/null