Skip to content

Commit a488ba1

Browse files
authored
Zstd 1.5.0 Release
Zstd 1.5.0 Release
2 parents e4558ff + 0671808 commit a488ba1

File tree

269 files changed

+6329
-3058
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

269 files changed

+6329
-3058
lines changed

.circleci/config.yml

-27
Original file line numberDiff line numberDiff line change
@@ -36,33 +36,6 @@ jobs:
3636
make armbuild V=1; make clean
3737
make -C tests test-legacy test-longmatch; make clean
3838
make -C lib libzstd-nomt; make clean
39-
# This step is only run on release tags.
40-
# It publishes the source tarball as artifacts and if the GITHUB_TOKEN
41-
# environment variable is set it will publish the source tarball to the
42-
# tagged release.
43-
publish-github-release:
44-
docker:
45-
- image: fbopensource/zstd-circleci-primary:0.0.1
46-
environment:
47-
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
48-
steps:
49-
- checkout
50-
- run:
51-
name: Publish
52-
command: |
53-
export VERSION=$(echo $CIRCLE_TAG | tail -c +2)
54-
export ZSTD_VERSION=zstd-$VERSION
55-
git archive $CIRCLE_TAG --prefix $ZSTD_VERSION/ --format tar \
56-
-o $ZSTD_VERSION.tar
57-
sha256sum $ZSTD_VERSION.tar > $ZSTD_VERSION.tar.sha256
58-
zstd -19 $ZSTD_VERSION.tar
59-
sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
60-
gzip -k -9 $ZSTD_VERSION.tar
61-
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
62-
mkdir -p $CIRCLE_ARTIFACTS
63-
cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS
64-
- store_artifacts:
65-
path: /tmp/circleci-artifacts
6639
# This step should only be run in a cron job
6740
regression-test:
6841
docker:

.github/workflows/generic-dev.yml

+14-3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ jobs:
3030

3131
test:
3232
runs-on: ubuntu-latest
33+
env:
34+
DEVNULLRIGHTS: 1
35+
READFROMBLOCKDEVICE: 1
3336
steps:
3437
- uses: actions/checkout@v2
3538
- name: make test
@@ -55,14 +58,12 @@ jobs:
5558
CC=gcc-7 CFLAGS=-Werror make -j all
5659
make clean
5760
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
58-
make -C tests zbufftest-dll
5961
6062
# candidate test (to check) : underlink test
6163
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
62-
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
6364

6465
gcc-8-asan-ubsan-testzstd:
65-
runs-on: ubuntu-16.04 # fails on 18.04
66+
runs-on: ubuntu-latest
6667
steps:
6768
- uses: actions/checkout@v2
6869
- name: gcc-8 + ASan + UBSan + Test Zstd
@@ -126,6 +127,16 @@ jobs:
126127
make libc6install
127128
CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
128129
130+
clang-msan-fuzz:
131+
runs-on: ubuntu-latest
132+
steps:
133+
- uses: actions/checkout@v2
134+
- name: clang + MSan + Fuzz Test
135+
run: |
136+
sudo apt-get update
137+
sudo apt-get install clang
138+
CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
139+
129140
asan-ubsan-msan-regression:
130141
runs-on: ubuntu-latest
131142
steps:

.github/workflows/generic-release.yml

-8
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,6 @@ jobs:
2525
make test
2626
# make -c lib all (need to fix. not working right now)
2727
28-
zbuff:
29-
runs-on: ubuntu-16.04
30-
steps:
31-
- uses: actions/checkout@v2
32-
- name: zbuff test
33-
run: |
34-
make -C tests test-zbuff
35-
3628
tsan:
3729
runs-on: ubuntu-latest
3830
steps:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: publish-release-artifacts
2+
3+
on:
4+
release:
5+
types:
6+
- created
7+
8+
jobs:
9+
publish-release-artifacts:
10+
runs-on: ubuntu-latest
11+
if: startsWith(github.ref, 'refs/tags/')
12+
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v2
16+
17+
- name: Archive
18+
env:
19+
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
20+
RELEASE_SIGNING_KEY_PASSPHRASE: ${{ secrets.RELEASE_SIGNING_KEY_PASSPHRASE }}
21+
run: |
22+
# compute file name
23+
export TAG="$(echo "$GITHUB_REF" | sed -n 's_^refs/tags/__p')"
24+
if [ -z "$TAG" ]; then
25+
echo "action must be run on a tag. GITHUB_REF is not a tag: $GITHUB_REF"
26+
exit 1
27+
fi
28+
# Attempt to extract "1.2.3" from "v1.2.3" to maintain artifact name backwards compat.
29+
# Otherwise, degrade to using full tag.
30+
export VERSION="$(echo "$TAG" | sed 's_^v\([0-9]\+\.[0-9]\+\.[0-9]\+\)$_\1_')"
31+
export ZSTD_VERSION="zstd-$VERSION"
32+
33+
# archive
34+
git archive $TAG \
35+
--prefix $ZSTD_VERSION/ \
36+
--format tar \
37+
-o $ZSTD_VERSION.tar
38+
39+
# Do the rest of the work in a sub-dir so we can glob everything we want to publish.
40+
mkdir artifacts/
41+
mv $ZSTD_VERSION.tar artifacts/
42+
cd artifacts/
43+
44+
# compress
45+
zstd -k -19 $ZSTD_VERSION.tar
46+
gzip -k -9 $ZSTD_VERSION.tar
47+
48+
# we only publish the compressed tarballs
49+
rm $ZSTD_VERSION.tar
50+
51+
# hash
52+
sha256sum $ZSTD_VERSION.tar.zst > $ZSTD_VERSION.tar.zst.sha256
53+
sha256sum $ZSTD_VERSION.tar.gz > $ZSTD_VERSION.tar.gz.sha256
54+
55+
# sign
56+
if [ -n "$RELEASE_SIGNING_KEY" ]; then
57+
export GPG_BATCH_OPTS="--batch --no-use-agent --pinentry-mode loopback --no-tty --yes"
58+
echo "$RELEASE_SIGNING_KEY" | gpg $GPG_BATCH_OPTS --import
59+
gpg $GPG_BATCH_OPTS --armor --sign --sign-with [email protected] --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.zst.sig $ZSTD_VERSION.tar.zst
60+
gpg $GPG_BATCH_OPTS --armor --sign --sign-with [email protected] --detach-sig --passphrase "$RELEASE_SIGNING_KEY_PASSPHRASE" --output $ZSTD_VERSION.tar.gz.sig $ZSTD_VERSION.tar.gz
61+
fi
62+
63+
- name: Publish
64+
65+
env:
66+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
67+
with:
68+
args: artifacts/*

.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,5 @@ googletest/
5050
*.code-workspace
5151
compile_commands.json
5252
.clangd
53+
perf.data
54+
perf.data.old

.travis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ matrix:
125125
# meson dedicated test
126126
- name: Xenial (Meson + clang) # ~15mn
127127
if: branch = release
128-
dist: xenial
128+
dist: bionic
129129
language: cpp
130130
compiler: clang
131131
install:

CHANGELOG

+56
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,59 @@
1+
v1.5.0 (May 11, 2021)
2+
api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
3+
`ZSTD_defaultCLevel()`
4+
`ZSTD_getDictID_fromCDict()`
5+
api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42)
6+
`ZSTD_compress_advanced()`
7+
`ZSTD_compress_usingCDict_advanced()`
8+
`ZSTD_compressBegin_advanced()`
9+
`ZSTD_compressBegin_usingCDict_advanced()`
10+
`ZSTD_initCStream_srcSize()`
11+
`ZSTD_initCStream_usingDict()`
12+
`ZSTD_initCStream_usingCDict()`
13+
`ZSTD_initCStream_advanced()`
14+
`ZSTD_initCStream_usingCDict_advanced()`
15+
`ZSTD_resetCStream()`
16+
api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973)
17+
perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln)
18+
perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42)
19+
perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973)
20+
perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln)
21+
bug: Improve setting permissions of created files (#2525, @felixhandte)
22+
bug: Fix large dictionary non-determinism (#2607, @terrelln)
23+
bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln)
24+
bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte)
25+
bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42)
26+
bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink)
27+
bug: Fix UBSAN error in decompression (#2625, @terrelln)
28+
bug: Fix superblock compression divide by zero bug (#2592, @senhuang42)
29+
bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone)
30+
doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln)
31+
doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize)
32+
doc: Add style guide docs for open source contributors (#2626, @Cyan4973)
33+
tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42)
34+
tests: Better test coverage of index reduction (#2603, @terrelln)
35+
tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42)
36+
tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42)
37+
build: Dynamic library built multithreaded by default (#2584, @senhuang42)
38+
build: Move `zstd_errors.h` and `zdict.h` to `lib/` root (#2597, @terrelln)
39+
build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973)
40+
build: Single file library build script moved to `build/` directory (#2618, @felixhandte)
41+
build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42)
42+
build: Fixed Meson build (#2548, @SupervisedThinking @kloczek)
43+
build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson)
44+
build: Detect presence of `md5` on Darwin (#2609, @felixhandte)
45+
build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann)
46+
cli: `--progress` flag added to always display progress bar (#2595, @senhuang42)
47+
cli: Allow reading from block devices with `--force` (#2613, @felixhandte)
48+
cli: Fix CLI filesize display bug (#2550, @Cyan4973)
49+
cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973)
50+
contrib: Various fixes for linux kernel patch (#2539, @terrelln)
51+
contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42)
52+
contrib: Seekable format - New seek table-only API (#2113 #2518, @mdittmer @Cyan4973)
53+
contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng)
54+
contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat)
55+
misc: Automatically published release tarballs available on Github (#2535, @felixhandte)
56+
157
v1.4.9 (Mar 1, 2021)
258
bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte)
359
bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln)

CONTRIBUTING.md

+98
Original file line numberDiff line numberDiff line change
@@ -399,7 +399,105 @@ disclosure of security bugs. In those cases, please go through the process
399399
outlined on that page and do not file a public issue.
400400
401401
## Coding Style
402+
It's a pretty long topic, which is difficult to summarize in a single paragraph.
403+
As a rule of thumbs, try to imitate the coding style of
404+
similar lines of codes around your contribution.
405+
The following is a non-exhaustive list of rules employed in zstd code base:
406+
407+
### C90
408+
This code base is following strict C90 standard,
409+
with 2 extensions : 64-bit `long long` types, and variadic macros.
410+
This rule is applied strictly to code within `lib/` and `programs/`.
411+
Sub-project in `contrib/` are allowed to use other conventions.
412+
413+
### C++ direct compatibility : symbol mangling
414+
All public symbol declarations must be wrapped in `extern “C” { … }`,
415+
so that this project can be compiled as C++98 code,
416+
and linked into C++ applications.
417+
418+
### Minimal Frugal
419+
This design requirement is fundamental to preserve the portability of the code base.
420+
#### Dependencies
421+
- Reduce dependencies to the minimum possible level.
422+
Any dependency should be considered “bad” by default,
423+
and only tolerated because it provides a service in a better way than can be achieved locally.
424+
The only external dependencies this repository tolerates are
425+
standard C libraries, and in rare cases, system level headers.
426+
- Within `lib/`, this policy is even more drastic.
427+
The only external dependencies allowed are `<assert.h>`, `<stdlib.h>`, `<string.h>`,
428+
and even then, not directly.
429+
In particular, no function shall ever allocate on heap directly,
430+
and must use instead `ZSTD_malloc()` and equivalent.
431+
Other accepted non-symbol headers are `<stddef.h>` and `<limits.h>`.
432+
- Within the project, there is a strict hierarchy of dependencies that must be respected.
433+
`programs/` is allowed to depend on `lib/`, but only its public API.
434+
Within `lib/`, `lib/common` doesn't depend on any other directory.
435+
`lib/compress` and `lib/decompress` shall not depend on each other.
436+
`lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`.
437+
#### Resources
438+
- Functions in `lib/` must use very little stack space,
439+
several dozens of bytes max.
440+
Everything larger must use the heap allocator,
441+
or require a scratch buffer to be emplaced manually.
442+
443+
### Naming
444+
* All public symbols are prefixed with `ZSTD_`
445+
+ private symbols, with a scope limited to their own unit, are free of this restriction.
446+
However, since `libzstd` source code can be amalgamated,
447+
each symbol name must attempt to be (and remain) unique.
448+
Avoid too generic names that could become ground for future collisions.
449+
This generally implies usage of some form of prefix.
450+
* For symbols (functions and variables), naming convention is `PREFIX_camelCase`.
451+
+ In some advanced cases, one can also find :
452+
- `PREFIX_prefix2_camelCase`
453+
- `PREFIX_camelCase_extendedQualifier`
454+
* Multi-words names generally consist of an action followed by object:
455+
- for example : `ZSTD_createCCtx()`
456+
* Prefer positive actions
457+
- `goBackward` rather than `notGoForward`
458+
* Type names (`struct`, etc.) follow similar convention,
459+
except that they are allowed and even invited to start by an Uppercase letter.
460+
Example : `ZSTD_CCtx`, `ZSTD_CDict`
461+
* Macro names are all Capital letters.
462+
The same composition rules (`PREFIX_NAME_QUALIFIER`) apply.
463+
* File names are all lowercase letters.
464+
The convention is `snake_case`.
465+
File names **must** be unique across the entire code base,
466+
even when they stand in clearly separated directories.
467+
468+
### Qualifiers
469+
* This code base is `const` friendly, if not `const` fanatical.
470+
Any variable that can be `const` (aka. read-only) **must** be `const`.
471+
Any pointer which content will not be modified must be `const`.
472+
This property is then controlled at compiler level.
473+
`const` variables are an important signal to readers that this variable isn’t modified.
474+
Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function.
475+
* If a function must be inlined, mention it explicitly,
476+
using project's own portable macros, such as `FORCE_INLINE_ATTR`,
477+
defined in `lib/common/compiler.h`.
478+
479+
### Debugging
480+
* **Assertions** are welcome, and should be used very liberally,
481+
to control any condition the code expects for its correct execution.
482+
These assertion checks will be run in debug builds, and disabled in production.
483+
* For traces, this project provides its own debug macros,
484+
in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`.
485+
486+
### Code documentation
487+
* Avoid code documentation that merely repeats what the code is already stating.
488+
Whenever applicable, prefer employing the code as the primary way to convey explanations.
489+
Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`.
490+
Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`.
491+
* At declaration level, the documentation explains how to use the function or variable
492+
and when applicable why it's needed, of the scenarios where it can be useful.
493+
* At implementation level, the documentation explains the general outline of the algorithm employed,
494+
and when applicable why this specific choice was preferred.
495+
496+
### General layout
402497
* 4 spaces for indentation rather than tabs
498+
* Code documentation shall directly precede function declaration or implementation
499+
* Function implementations and its code documentation should be preceded and followed by an empty line
500+
403501
404502
## License
405503
By contributing to Zstandard, you agree that your contributions will be licensed

0 commit comments

Comments
 (0)