Merge pull request #2515 from facebook/dev

ZStandard v1.4.9

Author: felixhandte

.circleci/config.yml

@@ -31,9 +31,9 @@ jobs:
           command: |
             make gnu90build; make clean
             make gnu99build; make clean
-            make ppc64build; make clean
-            make ppcbuild  ; make clean
-            make armbuild  ; make clean
+            make ppc64build V=1; make clean
+            make ppcbuild   V=1; make clean
+            make armbuild   V=1; make clean
             make -C tests test-legacy test-longmatch; make clean
             make -C lib libzstd-nomt; make clean
   # This step is only run on release tags.

.cirrus.yml

@@ -2,7 +2,7 @@ task:
   name: FreeBSD (shortest)
   freebsd_instance:
     matrix:
-      image_family: freebsd-12-1
+      image_family: freebsd-12-2
       # The stable 11.3 image causes "Agent is not responding" so use a snapshot
       image_family: freebsd-11-3-snap
   install_script: pkg install -y gmake coreutils

.github/workflows/generic-dev.yml

@@ -41,18 +41,17 @@ jobs:
     - uses: actions/checkout@v2
     - name: make check on 32-bit
       run: |
-        make libc6install
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-m32 -O1 -fstack-protector" make check V=1
 
-  gcc-6-7-libzstd:
+  gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + gcc-7 + libzstdmt compilation
+    - name: gcc-7 + libzstdmt compilation
       run: |
-        make gcc6install gcc7install
-        CC=gcc-6 CFLAGS=-Werror make -j all
-        make clean
+        make gcc7install
         CC=gcc-7 CFLAGS=-Werror make -j all
         make clean
         LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
@@ -71,30 +70,28 @@ jobs:
         make gcc8install
         CC=gcc-8 CFLAGS="-Werror" make -j all
         make clean
-        CC=gcc-8 make -j uasan-test-zstd </dev/null
+        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
-  gcc-6-asan-ubsan-testzstd-32bit:
+  gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
+    - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
-        make gcc6install libc6install
-        CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
+        make libc6install
+        CFLAGS="-Werror -m32" make -j all32
         make clean
-        CC=gcc-6 make -j uasan-test-zstd32
+        make -j uasan-test-zstd32 V=1
 
-  clang-38-msan-testzstd:
-    runs-on: ubuntu-16.04 # fails on 18.04
+  clang-msan-testzstd:
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: clang-3.8 + MSan + Test Zstd
+    - name: clang + MSan + Test Zstd
       run: |
-        # make clang38install (doesn't work)
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
         sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
+        sudo apt-get install clang
+        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
 
     # Note : external libraries must be turned off when using MSAN tests,
     # because they are not msan-instrumented,
@@ -120,26 +117,14 @@ jobs:
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
-  gcc-6-asan-ubsan-fuzz32:
+  gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: gcc-6 + ASan + UBSan + Fuzz Test 32bit
-      run: |
-        make gcc6install libc6install
-        CC=gcc-6 CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
-
-  clang-38-msan-fuzz:
-    runs-on: ubuntu-16.04 # fails on 18.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: clang-3.8 + MSan + Fuzz Test
+    - name: ASan + UBSan + Fuzz Test 32bit
       run: |
-        # make clang38install (doesn't work)
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
-        sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make clean msan-fuzztest
+        make libc6install
+        CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
   asan-ubsan-msan-regression:
     runs-on: ubuntu-latest
@@ -205,21 +190,25 @@ jobs:
         tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
         shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
 
-  icc:
-    name: icc-check
-    runs-on: ubuntu-latest
-    steps:
-    - name: install icc
-      run: |
-        export DEBIAN_FRONTEND=noninteractive
-        sudo apt-get -qqq update
-        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-        sudo apt-get update
-        sudo apt-get install -y intel-basekit intel-hpckit
-    - uses: actions/checkout@v2
-    - name: make check
-      run: |
-        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
+# For reference : icc tests
+# icc tests are currently failing on Github Actions, likely to issues during installation stage
+# To be fixed later
+#
+#  icc:
+#    name: icc-check
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: install icc
+#      run: |
+#        export DEBIAN_FRONTEND=noninteractive
+#        sudo apt-get -qqq update
+#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
+#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+#        sudo apt-get update
+#        sudo apt-get install -y intel-basekit intel-hpckit
+#    - uses: actions/checkout@v2
+#    - name: make check
+#      run: |
+#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

.github/workflows/generic-release.yml

@@ -34,24 +34,21 @@ jobs:
         make -C tests test-zbuff
 
   tsan:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: thread sanitizer
       run: |
-        sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
-        sudo apt-get update
-        sudo apt-get install clang-3.8
-        CC=clang-3.8 make tsan-test-zstream
-        CC=clang-3.8 make tsan-fuzztest
+        CC=clang make tsan-test-zstream
+        CC=clang make tsan-fuzztest
 
   zlib-wrapper:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/checkout@v2
     - name: zlib wrapper test
       run: |
-        make gpp6install valgrindinstall
+        make valgrindinstall
         make -C zlibWrapper test
         make -C zlibWrapper valgrindTest
 

.github/workflows/main.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sanitizer: [address, undefined, memory]
+        sanitizer: [address, undefined]
     steps:
     - name: Build Fuzzers (${{ matrix.sanitizer }})
       id: build

.travis.yml

@@ -32,11 +32,6 @@ matrix:
       script:
         - make check
 
-    - name: make test (complete)
-      script:
-        # DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
-        - DEVNULLRIGHTS=test make test
-
     - name: Minimal Decompressor Macros    # ~5mn
       script:
         - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"

CHANGELOG

@@ -1,3 +1,29 @@
+v1.4.9  (Mar 1, 2021)
+bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
+bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
+api: Add (De)Compression Tracing Functionality (#2482, @terrelln)
+api: Support References to Multiple DDicts (#2446, @senhuang42)
+api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
+perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
+perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
+perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
+cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
+cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
+cli: Improve Help Message (#2500, @senhuang42)
+tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
+tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat)
+tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte)
+build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973)
+build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt)
+build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime)
+build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka)
+build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln)
+build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg)
+build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973)
+contrib: Add `recover_directory` Program (#2473, @terrelln)
+doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42)
+doc: Fix Typos (#2459, @ThomasWaldmann)
+
 v1.4.8  (Dec 18, 2020)
 hotfix: wrong alignment of an internal buffer
 

Makefile

@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
@@ -48,7 +48,7 @@ allmost: allzstd zlibwrapper
 
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
-allzstd: lib-all
+allzstd: lib
 	$(Q)$(MAKE) -C $(PRGDIR) all
 	$(Q)$(MAKE) -C $(TESTDIR) all
 
@@ -57,9 +57,8 @@ all32:
 	$(MAKE) -C $(PRGDIR) zstd32
 	$(MAKE) -C $(TESTDIR) all32
 
-.PHONY: lib lib-release libzstd.a
-lib-all : lib
-lib lib-release lib-all :
+.PHONY: lib lib-release
+lib lib-release :
 	$(Q)$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
@@ -225,10 +224,10 @@ aarch64build: clean
 	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 ppcbuild: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
 
 ppc64build: clean
-	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
 
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
@@ -287,12 +286,11 @@ uasanregressiontest:
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
 
-# run UBsan with -fsanitize-recover=signed-integer-overflow
-# due to a bug in UBsan when doing pointer subtraction
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+# run UBsan with -fsanitize-recover=pointer-overflow
+# this only works with recent compilers such as gcc 8+
 
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
@@ -310,10 +308,10 @@ asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

appveyor.yml

@@ -26,7 +26,7 @@
     - COMPILER: "clang"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
       BUILD:    "true"
 
     - COMPILER: "gcc"
@@ -204,7 +204,7 @@
     - COMPILER: "clang"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
+      SCRIPT:   "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
 
     - COMPILER: "visual"
       HOST:     "visual"

build/VS2008/zstd/zstd.vcproj

@@ -480,6 +480,10 @@
 				RelativePath="..\..\..\programs\zstdcli.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\programs\zstdcli_trace.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
 				>