Generate #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate | |
| permissions: | |
| contents: write | |
| on: | |
| push: | |
| workflow_dispatch: | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 | |
| with: | |
| persist-credentials: false | |
| - name: Download and extract Zig | |
| run: | | |
| curl -L -o zig.tar.xz https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz | |
| tar -xJf zig.tar.xz | |
| mv zig-linux-x86_64-0.14.0 zig | |
| echo "$GITHUB_WORKSPACE/zig" >> $GITHUB_PATH | |
| - name: Download and build bzip2 0.9.0c (from 1998) | |
| run: | | |
| curl -L -o bzip2.tar.gz https://github.com/libarchive/bzip2/archive/refs/tags/bzip2-0.9.0c.tar.gz | |
| tar -xzf bzip2.tar.gz | |
| mv bzip2-bzip2-0.9.0c bzip2-0.9.0c | |
| sed -i 's|ar clq|zig ar clq|' bzip2-0.9.0c/Makefile | |
| cd bzip2-0.9.0c | |
| make CC=gcc -j"$(nproc)" | |
| mkdir -p $GITHUB_WORKSPACE/bin | |
| cp bzip2 $GITHUB_WORKSPACE/bin/bzip2-0.9.0c | |
| echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH | |
| $GITHUB_WORKSPACE/bin/bzip2-0.9.0c --version | |
| - name: Download and build zstd 0.7.4 | |
| run: | | |
| curl -L -o zstd.tar.gz https://github.com/facebook/zstd/archive/refs/tags/v0.7.4.tar.gz | |
| tar -xzf zstd.tar.gz | |
| cd zstd-0.7.4 | |
| make -j"$(nproc)" | |
| mkdir -p $GITHUB_WORKSPACE/bin | |
| cp zstd $GITHUB_WORKSPACE/bin/zstd-0.7.4 | |
| echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH | |
| $GITHUB_WORKSPACE/bin/zstd-0.7.4 --version | |
| - name: Download and build zstd 0.6.1 | |
| run: | | |
| curl -L -o zstd.tar.gz https://github.com/facebook/zstd/archive/refs/tags/v0.6.1.tar.gz | |
| tar -xzf zstd.tar.gz | |
| cd zstd-0.6.1 | |
| make -j"$(nproc)" | |
| mkdir -p $GITHUB_WORKSPACE/bin | |
| cp programs/zstd $GITHUB_WORKSPACE/bin/zstd-0.6.1 | |
| echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH | |
| $GITHUB_WORKSPACE/bin/zstd-0.6.1 --version | |
| - name: Download and build zstd 0.5.1 | |
| run: | | |
| curl -L -o zstd.tar.gz https://github.com/facebook/zstd/archive/refs/tags/v0.5.1.tar.gz | |
| tar -xzf zstd.tar.gz | |
| cd zstd-0.5.1 | |
| make -j"$(nproc)" | |
| mkdir -p $GITHUB_WORKSPACE/bin | |
| cp programs/zstd $GITHUB_WORKSPACE/bin/zstd-0.5.1 | |
| echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH | |
| $GITHUB_WORKSPACE/bin/zstd-0.5.1 --version | |
| - name: Download and build zlib-ng (because it exposes compression level 0) | |
| run: | | |
| curl -L -o zlib-ng.tar.gz https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.2.4.tar.gz | |
| tar -xzf zlib-ng.tar.gz | |
| cd zlib-ng-2.2.4 | |
| ./configure | |
| make -j"$(nproc)" | |
| mkdir -p $GITHUB_WORKSPACE/bin | |
| cp minigzip $GITHUB_WORKSPACE/bin/minigzip | |
| echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH | |
| $GITHUB_WORKSPACE/bin/minigzip --help | |
| - name: Install zstd | |
| run: | | |
| sudo apt-get install -y zstd | |
| zstd --version | |
| - name: Install bzip2 | |
| run: | | |
| sudo apt-get install -y bzip2 | |
| bzip2 --version | |
| - name: Generate zstd dict of `data/*.md` | |
| run: | | |
| zstd --train data/*.md -B128 -o compression-corpus.zstd | |
| zstd-0.7.4 --train data/*.md -B128 -o compression-corpus-0.7.4.zstd | |
| zstd-0.6.1 --train data/*.md -B128 -o compression-corpus-0.6.1.zstd | |
| zstd-0.5.1 --train data/*.md -B128 -o compression-corpus-0.5.1.zstd | |
| - name: Compress files in data folder | |
| run: | | |
| mkdir -p compressed | |
| rm -f compressed/* | |
| for file in data/*; do | |
| for level in 0 1 2 3 4 5 6 7 8 9; do | |
| if [ "$level" -ne 0 ]; then | |
| bzip2 -k -$level "$file" -c > "compressed/$(basename "$file").bzip2-$level.bz2" | |
| bzip2-0.9.0c -k -$level "$file" -c > "compressed/$(basename "$file").bzip2-0.9.0-$level.bz2" | |
| bzip2-0.9.0c -k -$level "$file" -c -s > "compressed/$(basename "$file").bzip2-0.9.0-$level-small.bz2" | |
| fi | |
| minigzip -k -c -$level "$file" > "compressed/$(basename "$file").gzip-$level.gz" | |
| minigzip -k -c -$level -f "$file" > "compressed/$(basename "$file").gzip-filtered-$level.gz" | |
| minigzip -k -c -$level -h "$file" > "compressed/$(basename "$file").gzip-huffman-$level.gz" | |
| minigzip -k -c -$level -R "$file" > "compressed/$(basename "$file").gzip-rle-$level.gz" | |
| minigzip -k -c -$level -F "$file" > "compressed/$(basename "$file").gzip-fixed-$level.gz" | |
| done | |
| # Zstandard compression levels (1 to 19) | |
| for level in $(seq 1 19); do | |
| zstd -$level -c "$file" > "compressed/$(basename "$file").zstd-$level.zst" | |
| zstd -$level --long=27 -c "$file" > "compressed/$(basename "$file").zstd-long27-$level.zst" | |
| done | |
| for level in 1 5 10 19; do | |
| zstd-0.7.4 -$level -c "$file" > "compressed/$(basename "$file").zstd-0.7.4-$level.zst" | |
| zstd-0.6.1 -$level -c "$file" > "compressed/$(basename "$file").zstd-0.6.1-$level.zst" | |
| zstd-0.5.1 -$level -c "$file" > "compressed/$(basename "$file").zstd-0.5.1-$level.zst" | |
| done | |
| # A couple of files with a custom dictionary. | |
| zstd -10 -c "$file" -D compression-corpus.zstd > "compressed/$(basename "$file").zstd-custom-dict.zst" | |
| zstd-0.7.4 -10 -c "$file" -D compression-corpus-0.7.4.zstd > "compressed/$(basename "$file").zstd-0.7.4-custom-dict.zst" | |
| zstd-0.6.1 -10 -c "$file" -D compression-corpus-0.6.1.zstd > "compressed/$(basename "$file").zstd-0.6.1-custom-dict.zst" | |
| zstd-0.6.1 -10 -c "$file" -D compression-corpus-0.6.1.zstd > "compressed/$(basename "$file").zstd-0.6.1-custom-dict.zst" | |
| # Optional: test --ultra compression modes (levels 20 to 22) | |
| for level in 20 21 22; do | |
| zstd --ultra -$level -c "$file" > "compressed/$(basename "$file").zstd-ultra-$level.zst" | |
| done | |
| done | |
| - name: Copy handpicked files | |
| run: | | |
| cp handpicked/* compressed/ | |
| - name: Create gzip artifact zip | |
| run: | | |
| zip -r gzip-files.zip compressed/*.gz | |
| - name: Create bzip2 artifact zip | |
| run: | | |
| zip -r bzip2-files.zip compressed/*.bz2 | |
| - name: Create zstd artifact zip | |
| run: | | |
| zip -r zstd-files.zip compressed/*.zst compression-corpus.zstd compression-corpus-0.7.4.zstd compression-corpus-0.6.1.zstd compression-corpus-0.5.1.zstd | |
| - name: Upload gzip artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gzip-files | |
| path: gzip-files.zip | |
| - name: Upload bzip2 artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bzip2-files | |
| path: bzip2-files.zip | |
| - name: Upload zstd artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: zstd-files | |
| path: zstd-files.zip | |
| - name: Set tag | |
| id: set_tag | |
| run: | | |
| echo "RELEASE_TAG=$(date +'%Y-%m-%d-%H%M%S')" >> $GITHUB_ENV | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| - name: Create GitHub release | |
| uses: softprops/action-gh-release@v1 | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| with: | |
| tag_name: ${{ env.RELEASE_TAG }} | |
| files: | | |
| gzip-files.zip | |
| bzip2-files.zip | |
| zstd-files.zip |