Skip to content

Commit e4558ff

Browse files
felixhandtefelixhandte
authored
Merge pull request #2515 from facebook/dev
ZStandard v1.4.9
2 parents 97a3da1 + 74d65ea commit e4558ff

File tree

229 files changed

+2548
-824
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

229 files changed

+2548
-824
lines changed

‎.circleci/config.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ jobs:
3131
command: |
3232
make gnu90build; make clean
3333
make gnu99build; make clean
34-
make ppc64build; make clean
35-
make ppcbuild ; make clean
36-
make armbuild ; make clean
34+
make ppc64build V=1; make clean
35+
make ppcbuild V=1; make clean
36+
make armbuild V=1; make clean
3737
make -C tests test-legacy test-longmatch; make clean
3838
make -C lib libzstd-nomt; make clean
3939
# This step is only run on release tags.

‎.cirrus.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ task:
22
name: FreeBSD (shortest)
33
freebsd_instance:
44
matrix:
5-
image_family: freebsd-12-1
5+
image_family: freebsd-12-2
66
# The stable 11.3 image causes "Agent is not responding" so use a snapshot
77
image_family: freebsd-11-3-snap
88
install_script: pkg install -y gmake coreutils

‎.github/workflows/generic-dev.yml

+42-53
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,17 @@ jobs:
4141
- uses: actions/checkout@v2
4242
- name: make check on 32-bit
4343
run: |
44-
make libc6install
44+
sudo apt update
45+
APT_PACKAGES="gcc-multilib" make apt-install
4546
CFLAGS="-m32 -O1 -fstack-protector" make check V=1
4647
47-
gcc-6-7-libzstd:
48+
gcc-7-libzstd:
4849
runs-on: ubuntu-latest
4950
steps:
5051
- uses: actions/checkout@v2
51-
- name: gcc-6 + gcc-7 + libzstdmt compilation
52+
- name: gcc-7 + libzstdmt compilation
5253
run: |
53-
make gcc6install gcc7install
54-
CC=gcc-6 CFLAGS=-Werror make -j all
55-
make clean
54+
make gcc7install
5655
CC=gcc-7 CFLAGS=-Werror make -j all
5756
make clean
5857
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
@@ -71,30 +70,28 @@ jobs:
7170
make gcc8install
7271
CC=gcc-8 CFLAGS="-Werror" make -j all
7372
make clean
74-
CC=gcc-8 make -j uasan-test-zstd </dev/null
73+
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
7574
76-
gcc-6-asan-ubsan-testzstd-32bit:
75+
gcc-asan-ubsan-testzstd-32bit:
7776
runs-on: ubuntu-latest
7877
steps:
7978
- uses: actions/checkout@v2
80-
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode
79+
- name: ASan + UBSan + Test Zstd, 32bit mode
8180
run: |
82-
make gcc6install libc6install
83-
CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
81+
make libc6install
82+
CFLAGS="-Werror -m32" make -j all32
8483
make clean
85-
CC=gcc-6 make -j uasan-test-zstd32
84+
make -j uasan-test-zstd32 V=1
8685
87-
clang-38-msan-testzstd:
88-
runs-on: ubuntu-16.04 # fails on 18.04
86+
clang-msan-testzstd:
87+
runs-on: ubuntu-latest
8988
steps:
9089
- uses: actions/checkout@v2
91-
- name: clang-3.8 + MSan + Test Zstd
90+
- name: clang + MSan + Test Zstd
9291
run: |
93-
# make clang38install (doesn't work)
94-
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
9592
sudo apt-get update
96-
sudo apt-get install clang-3.8
97-
CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
93+
sudo apt-get install clang
94+
CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
9895
9996
# Note : external libraries must be turned off when using MSAN tests,
10097
# because they are not msan-instrumented,
@@ -120,26 +117,14 @@ jobs:
120117
make gcc8install
121118
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
122119
123-
gcc-6-asan-ubsan-fuzz32:
120+
gcc-asan-ubsan-fuzz32:
124121
runs-on: ubuntu-latest
125122
steps:
126123
- uses: actions/checkout@v2
127-
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit
128-
run: |
129-
make gcc6install libc6install
130-
CC=gcc-6 CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
131-
132-
clang-38-msan-fuzz:
133-
runs-on: ubuntu-16.04 # fails on 18.04
134-
steps:
135-
- uses: actions/checkout@v2
136-
- name: clang-3.8 + MSan + Fuzz Test
124+
- name: ASan + UBSan + Fuzz Test 32bit
137125
run: |
138-
# make clang38install (doesn't work)
139-
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
140-
sudo apt-get update
141-
sudo apt-get install clang-3.8
142-
CC=clang-3.8 make clean msan-fuzztest
126+
make libc6install
127+
CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
143128
144129
asan-ubsan-msan-regression:
145130
runs-on: ubuntu-latest
@@ -205,21 +190,25 @@ jobs:
205190
tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz
206191
shellcheck-v0.7.1/shellcheck --shell=sh --severity=warning --exclude=SC2010 tests/playTests.sh
207192
208-
icc:
209-
name: icc-check
210-
runs-on: ubuntu-latest
211-
steps:
212-
- name: install icc
213-
run: |
214-
export DEBIAN_FRONTEND=noninteractive
215-
sudo apt-get -qqq update
216-
sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
217-
sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
218-
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
219-
sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
220-
sudo apt-get update
221-
sudo apt-get install -y intel-basekit intel-hpckit
222-
- uses: actions/checkout@v2
223-
- name: make check
224-
run: |
225-
make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
193+
# For reference : icc tests
194+
# icc tests are currently failing on Github Actions, likely to issues during installation stage
195+
# To be fixed later
196+
#
197+
# icc:
198+
# name: icc-check
199+
# runs-on: ubuntu-latest
200+
# steps:
201+
# - name: install icc
202+
# run: |
203+
# export DEBIAN_FRONTEND=noninteractive
204+
# sudo apt-get -qqq update
205+
# sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
206+
# sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
207+
# sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
208+
# sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
209+
# sudo apt-get update
210+
# sudo apt-get install -y intel-basekit intel-hpckit
211+
# - uses: actions/checkout@v2
212+
# - name: make check
213+
# run: |
214+
# make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

‎.github/workflows/generic-release.yml

+4-7
Original file line numberDiff line numberDiff line change
@@ -34,24 +34,21 @@ jobs:
3434
make -C tests test-zbuff
3535
3636
tsan:
37-
runs-on: ubuntu-16.04
37+
runs-on: ubuntu-latest
3838
steps:
3939
- uses: actions/checkout@v2
4040
- name: thread sanitizer
4141
run: |
42-
sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main"
43-
sudo apt-get update
44-
sudo apt-get install clang-3.8
45-
CC=clang-3.8 make tsan-test-zstream
46-
CC=clang-3.8 make tsan-fuzztest
42+
CC=clang make tsan-test-zstream
43+
CC=clang make tsan-fuzztest
4744
4845
zlib-wrapper:
4946
runs-on: ubuntu-16.04
5047
steps:
5148
- uses: actions/checkout@v2
5249
- name: zlib wrapper test
5350
run: |
54-
make gpp6install valgrindinstall
51+
make valgrindinstall
5552
make -C zlibWrapper test
5653
make -C zlibWrapper valgrindTest
5754

‎.github/workflows/main.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ jobs:
66
strategy:
77
fail-fast: false
88
matrix:
9-
sanitizer: [address, undefined, memory]
9+
sanitizer: [address, undefined]
1010
steps:
1111
- name: Build Fuzzers (${{ matrix.sanitizer }})
1212
id: build

‎.travis.yml

-5
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,6 @@ matrix:
3232
script:
3333
- make check
3434

35-
- name: make test (complete)
36-
script:
37-
# DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
38-
- DEVNULLRIGHTS=test make test
39-
4035
- name: Minimal Decompressor Macros # ~5mn
4136
script:
4237
- make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"

‎CHANGELOG

+26
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,29 @@
1+
v1.4.9 (Mar 1, 2021)
2+
bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
3+
bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)
4+
api: Add (De)Compression Tracing Functionality (#2482, @terrelln)
5+
api: Support References to Multiple DDicts (#2446, @senhuang42)
6+
api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
7+
perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
8+
perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
9+
perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
10+
cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
11+
cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
12+
cli: Improve Help Message (#2500, @senhuang42)
13+
tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
14+
tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat)
15+
tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte)
16+
build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973)
17+
build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt)
18+
build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime)
19+
build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka)
20+
build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln)
21+
build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg)
22+
build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973)
23+
contrib: Add `recover_directory` Program (#2473, @terrelln)
24+
doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42)
25+
doc: Fix Typos (#2459, @ThomasWaldmann)
26+
127
v1.4.8 (Dec 18, 2020)
228
hotfix: wrong alignment of an internal buffer
329

‎Makefile

+11-13
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# ################################################################
2-
# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
2+
# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
33
# All rights reserved.
44
#
55
# This source code is licensed under both the BSD-style license (found in the
@@ -48,7 +48,7 @@ allmost: allzstd zlibwrapper
4848

4949
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
5050
.PHONY: allzstd
51-
allzstd: lib-all
51+
allzstd: lib
5252
$(Q)$(MAKE) -C $(PRGDIR) all
5353
$(Q)$(MAKE) -C $(TESTDIR) all
5454

@@ -57,9 +57,8 @@ all32:
5757
$(MAKE) -C $(PRGDIR) zstd32
5858
$(MAKE) -C $(TESTDIR) all32
5959

60-
.PHONY: lib lib-release libzstd.a
61-
lib-all : lib
62-
lib lib-release lib-all :
60+
.PHONY: lib lib-release
61+
lib lib-release :
6362
$(Q)$(MAKE) -C $(ZSTDDIR) $@
6463

6564
.PHONY: zstd zstd-release
@@ -225,10 +224,10 @@ aarch64build: clean
225224
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
226225

227226
ppcbuild: clean
228-
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
227+
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
229228

230229
ppc64build: clean
231-
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
230+
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
232231

233232
armfuzz: clean
234233
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
@@ -287,12 +286,11 @@ uasanregressiontest:
287286
msanregressiontest:
288287
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
289288

290-
# run UBsan with -fsanitize-recover=signed-integer-overflow
291-
# due to a bug in UBsan when doing pointer subtraction
292-
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
289+
# run UBsan with -fsanitize-recover=pointer-overflow
290+
# this only works with recent compilers such as gcc 8+
293291

294292
usan: clean
295-
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror"
293+
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
296294

297295
asan: clean
298296
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
@@ -310,10 +308,10 @@ asan32: clean
310308
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
311309

312310
uasan: clean
313-
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror"
311+
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
314312

315313
uasan-%: clean
316-
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
314+
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
317315

318316
tsan-%: clean
319317
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests

‎appveyor.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
- COMPILER: "clang"
2727
HOST: "mingw"
2828
PLATFORM: "x64"
29-
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
29+
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
3030
BUILD: "true"
3131

3232
- COMPILER: "gcc"
@@ -204,7 +204,7 @@
204204
- COMPILER: "clang"
205205
HOST: "mingw"
206206
PLATFORM: "x64"
207-
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
207+
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"
208208

209209
- COMPILER: "visual"
210210
HOST: "visual"

‎build/VS2008/zstd/zstd.vcproj

+4
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,10 @@
480480
RelativePath="..\..\..\programs\zstdcli.c"
481481
>
482482
</File>
483+
<File
484+
RelativePath="..\..\..\programs\zstdcli_trace.c"
485+
>
486+
</File>
483487
<File
484488
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
485489
>

‎build/VS2010/zstd/zstd.vcxproj

+1
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
<ClCompile Include="..\..\..\programs\dibio.c" />
6464
<ClCompile Include="..\..\..\programs\fileio.c" />
6565
<ClCompile Include="..\..\..\programs\zstdcli.c" />
66+
<ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
6667
</ItemGroup>
6768
<ItemGroup>
6869
<ClInclude Include="..\..\..\lib\common\pool.h" />

‎build/cmake/lib/CMakeLists.txt

+5-16
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
# in the COPYING file in the root directory of this source tree).
88
# ################################################################
99

10-
project(libzstd)
10+
project(libzstd C)
1111

1212
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
1313
option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
@@ -137,7 +137,7 @@ endif ()
137137
if (UNIX OR MINGW)
138138
# pkg-config
139139
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
140-
set(EXEC_PREFIX "\\$$\{prefix}")
140+
set(EXEC_PREFIX "\${prefix}")
141141
set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
142142
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
143143
set(VERSION "${zstd_VERSION}")
@@ -149,24 +149,13 @@ if (UNIX OR MINGW)
149149
string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
150150

151151
if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
152-
set(INCLUDEDIR_PREFIX "\\$$\{prefix}")
152+
set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
153153
endif()
154154
if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
155-
set(LIBDIR_PREFIX "\\$$\{exec_prefix}")
155+
set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
156156
endif()
157157

158-
add_custom_target(libzstd.pc ALL
159-
${CMAKE_COMMAND}
160-
-DIN=${LIBRARY_DIR}/libzstd.pc.in
161-
-DOUT="libzstd.pc"
162-
-DPREFIX="${PREFIX}"
163-
-DEXEC_PREFIX="${EXEC_PREFIX}"
164-
-DINCLUDEDIR="${INCLUDEDIR_PREFIX}${INCLUDEDIR_SUFFIX}"
165-
-DLIBDIR="${LIBDIR_PREFIX}${LIBDIR_SUFFIX}"
166-
-DVERSION="${VERSION}"
167-
-P ${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake
168-
COMMENT "Creating pkg-config file")
169-
158+
configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
170159
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
171160
endif ()
172161

‎build/cmake/lib/pkgconfig.cmake

-1
This file was deleted.

0 commit comments

Comments
 (0)