Skip to content

karchive, kimageformats, kcodecs: remove libzip dependency, fix build, add lz compression support and update homepage urls #13308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion projects/karchive/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ RUN apt-get update && apt-get install --yes cmake make autoconf automake autopoi
RUN git clone --depth 1 https://github.com/madler/zlib.git
RUN git clone --depth 1 https://github.com/facebook/zstd.git
RUN git clone --depth 1 https://github.com/openssl/openssl.git
RUN git clone --depth 1 https://github.com/nih-at/libzip.git
RUN wget https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
RUN git clone https://github.com/tukaani-project/xz.git
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qtbase.git
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qttools.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/extra-cmake-modules.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/karchive.git
COPY build.sh karchive_fuzzer.cc $SRC/
Expand Down
15 changes: 7 additions & 8 deletions projects/karchive/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,6 @@ cd $SRC/zlib
./configure --static
make install -j$(nproc)

# Build libzip
cd $SRC/libzip
cmake . -DBUILD_SHARED_LIBS=OFF
make install -j$(nproc)

# Build bzip2
# Inspired from ../bzip2/build
cd $SRC
Expand Down Expand Up @@ -81,16 +76,20 @@ cd $SRC/qtbase
cmake --build . --parallel $(nproc)
cmake --install .

# Build qttools
cd $SRC/qttools
cmake . -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr
cmake --build . --parallel $(nproc)
cmake --install .

# Build karchive
cd $SRC
cd karchive
cd $SRC/karchive
rm -rf poqm
cmake . -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF
make install -j$(nproc)

# Build karchive_fuzzer
$CXX $CXXFLAGS -fPIC -std=c++17 $SRC/karchive_fuzzer.cc -o $OUT/karchive_fuzzer -I /usr/include/QtCore/ -I /usr/local/include/KF6/KArchive -lQt6Core -lm -lQt6BundledPcre2 -ldl -lpthread $LIB_FUZZING_ENGINE /usr/local/lib/libzip.a /usr/local/lib/libz.a -lKF6Archive /usr/local/lib/libbz2.a -llzma /usr/local/lib/libzstd.a /usr/local/lib64/libcrypto.a
$CXX $CXXFLAGS -fPIC -std=c++17 $SRC/karchive_fuzzer.cc -o $OUT/karchive_fuzzer -I /usr/include/QtCore/ -I /usr/local/include/KF6/KArchive -lQt6Core -lm -lQt6BundledPcre2 -ldl -lpthread $LIB_FUZZING_ENGINE /usr/local/lib/libz.a -lKF6Archive /usr/local/lib/libbz2.a -llzma /usr/local/lib/libzstd.a /usr/local/lib64/libcrypto.a

cd $SRC
find . -name "*.gz" -o -name "*.zip" -o -name "*.xz" -o -name "*.tar" -o -name "*.7z" | zip -q $OUT/karchive_fuzzer_seed_corpus.zip -@
2 changes: 2 additions & 0 deletions projects/karchive/karchive_fuzzer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
std::unique_ptr<KCompressionDevice> bzipKD(new KCompressionDevice(&b, false, KCompressionDevice::BZip2));
std::unique_ptr<KCompressionDevice> xzKD(new KCompressionDevice(&b, false, KCompressionDevice::Xz));
std::unique_ptr<KCompressionDevice> zstdKD(new KCompressionDevice(&b, false, KCompressionDevice::Zstd));
std::unique_ptr<KCompressionDevice> lzKD(new KCompressionDevice(&b, false, KCompressionDevice::Lz));

const QVector<KArchive*> handlers = {
new K7Zip(&b),
Expand All @@ -77,6 +78,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
new KTar(bzipKD.get()),
new KTar(xzKD.get()),
new KTar(zstdKD.get()),
new KTar(lzKD.get()),
new KZip(&b),
new KAr(&b)
};
Expand Down
2 changes: 1 addition & 1 deletion projects/karchive/project.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
homepage: https://cgit.kde.org/karchive.git/
homepage: https://invent.kde.org/frameworks/karchive.git/
language: c++
primary_contact: [email protected]
sanitizers:
Expand Down
1 change: 1 addition & 0 deletions projects/kcodecs/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ FROM gcr.io/oss-fuzz-base/base-builder
RUN apt-get update && apt-get install --yes cmake ninja-build
RUN curl -L http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz -O
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qtbase.git
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qttools.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/extra-cmake-modules.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/kcodecs.git
RUN git clone --depth 1 https://gitlab.freedesktop.org/uchardet/uchardet.git
Expand Down
14 changes: 8 additions & 6 deletions projects/kcodecs/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,21 @@ export CFLAGS="${FUZZ_CFLAGS}"
export CXXFLAGS="${FUZZ_CXXFLAGS}"


cd $SRC
cd extra-cmake-modules
cd $SRC/extra-cmake-modules
cmake -DBUILD_TESTING=OFF .
make install

cd $SRC
cd qtbase
cd $SRC/qtbase
./configure -no-glib -qt-libpng -qt-pcre -qt-zlib -opensource -confirm-license -static -no-opengl -no-icu -platform linux-clang-libc++ -debug -prefix /usr -no-feature-gui -no-feature-sql -no-feature-network -no-feature-xml -no-feature-dbus -no-feature-printsupport
cmake --build . --parallel $(nproc)
cmake --install .

cd $SRC
cd kcodecs
cd $SRC/qttools
cmake . -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr
cmake --build . --parallel $(nproc)
cmake --install .

cd $SRC/kcodecs
rm -rf poqm
cmake . -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Debug
make -j$(nproc) VERBOSE=1
Expand Down
2 changes: 1 addition & 1 deletion projects/kcodecs/project.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
homepage: https://cgit.kde.org/kcodecs.git/
homepage: https://invent.kde.org/frameworks/kcodecs.git/
language: c++
primary_contact: [email protected]
sanitizers:
Expand Down
2 changes: 1 addition & 1 deletion projects/kimageformats/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ FROM gcr.io/oss-fuzz-base/base-builder
RUN apt-get update && apt-get install --yes cmake make autoconf automake autopoint libtool wget po4a ninja-build pkgconf
RUN git clone --depth 1 https://github.com/madler/zlib.git
RUN git clone --depth 1 -b v1.5.7 https://github.com/facebook/zstd.git
RUN git clone --depth 1 https://github.com/nih-at/libzip.git
RUN wget https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
RUN git clone https://github.com/tukaani-project/xz.git
RUN git clone --depth 1 --branch=RB-3.3 https://github.com/AcademySoftwareFoundation/openexr.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/extra-cmake-modules.git
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qtbase.git
RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qttools.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/karchive.git
RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/kimageformats.git
RUN git clone --depth 1 -b v3.12.0 https://aomedia.googlesource.com/aom
Expand Down
66 changes: 25 additions & 41 deletions projects/kimageformats/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,15 @@
################################################################################

# build zstd
cd $SRC
cd zstd
cd $SRC/zstd
cmake -S build/cmake -DBUILD_SHARED_LIBS=OFF
make install -j$(nproc)

# Build zlib
cd $SRC
cd zlib
cd $SRC/zlib
./configure --static
make install -j$(nproc)

# Build libzip
cd $SRC
cd libzip
cmake . -DBUILD_SHARED_LIBS=OFF
make install -j$(nproc)

# Build bzip2
# Inspired from ../bzip2/build
cd $SRC
Expand All @@ -54,40 +46,41 @@ export ORIG_CFLAGS="${CFLAGS}"
export ORIG_CXXFLAGS="${CXXFLAGS}"
unset CFLAGS
unset CXXFLAGS
cd $SRC
cd xz
cd $SRC/xz
./autogen.sh --no-po4a --no-doxygen
./configure --enable-static --disable-debug --disable-shared --disable-xz --disable-xzdec --disable-lzmainfo
make install -j$(nproc)
export CFLAGS="${ORIG_CFLAGS}"
export CXXFLAGS="${ORIG_CXXFLAGS}"

cd $SRC
cd qtbase
# Build qt
cd $SRC/qtbase
./configure -no-glib -qt-libpng -qt-pcre -opensource -confirm-license -static -no-opengl -no-icu -platform linux-clang-libc++ -debug -prefix /usr -no-feature-widgets -no-feature-sql -no-feature-network -no-feature-xml -no-feature-dbus -no-feature-printsupport
cmake --build . --parallel $(nproc)
cmake --install .

# Build qttools
cd $SRC/qttools
cmake . -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr
cmake --build . --parallel $(nproc)
cmake --install .

# Build extra-cmake-modules
cd $SRC
cd extra-cmake-modules
cd $SRC/extra-cmake-modules
cmake . -DBUILD_TESTING=OFF
make install -j$(nproc)

cd $SRC
cd karchive
cd $SRC/karchive
rm -rf poqm
cmake . -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local
make install -j$(nproc)

# Build JXRlib
cd $SRC
cd jxrlib
cd $SRC/jxrlib
make -j$(nproc)

# Build LibRaw
cd $SRC
cd LibRaw
cd $SRC/LibRaw
TMP_CFLAGS=$CFLAGS
TMP_CXXFLAGS=$CXXFLAGS
CFLAGS="$CFLAGS -fno-sanitize=function,vptr"
Expand All @@ -101,8 +94,7 @@ CXXFLAGS=$TMP_CXXFLAGS


# Build aom
cd $SRC
cd aom
cd $SRC/aom
mkdir build.libavif
cd build.libavif
extra_libaom_flags='-DAOM_MAX_ALLOCABLE_MEMORY=536870912 -DDO_RANGE_CHECK_CLAMP=1'
Expand All @@ -111,47 +103,41 @@ make -j$(nproc)
make install -j$(nproc)

# Build libavif
cd $SRC
cd $SRC/libavif
ln -s "$SRC/aom" "$SRC/libavif/ext/"
cd libavif
mkdir build
cd build
CFLAGS="$CFLAGS -fPIC" cmake -DBUILD_SHARED_LIBS=OFF -DAVIF_ENABLE_WERROR=OFF -DAVIF_CODEC_AOM=LOCAL -DAVIF_LIBYUV=OFF ..
make -j$(nproc)

# Build libde265
cd $SRC
cd libde265
cd $SRC/libde265
cmake -DBUILD_SHARED_LIBS=OFF -DDISABLE_SSE=ON .
make -j$(nproc)
make install -j$(nproc)

# Build openjpeg
cd $SRC
cd openjpeg
cd $SRC/openjpeg
mkdir build
cd build
cmake -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_CODEC=OFF ..
make -j$(nproc)
make install -j$(nproc)

# build openh264
cd $SRC
cd openh264
cd $SRC/openh264
make USE_ASM=No BUILDTYPE=Debug install-static -j$(nproc)

# Build openexr
cd $SRC
cd openexr
cd $SRC/openexr
mkdir _build
cd _build
cmake -DBUILD_SHARED_LIBS=OFF ..
make -j$(nproc)
make install -j$(nproc)

# Build libheif
cd $SRC
cd libheif
cd $SRC/libheif
#Reduce max width and height to avoid allocating too much memory
sed -i "s/static const int MAX_IMAGE_WIDTH = 32768;/static const int MAX_IMAGE_WIDTH = 8192;/g" libheif/security_limits.h
sed -i "s/static const int MAX_IMAGE_HEIGHT = 32768;/static const int MAX_IMAGE_HEIGHT = 8192;/g" libheif/security_limits.h
Expand All @@ -162,15 +148,13 @@ make -j$(nproc)
make install -j$(nproc)

# Build libjxl
cd $SRC
cd libjxl
cd $SRC/libjxl
mkdir build
cd build
CXXFLAGS="$CXXFLAGS -DHWY_COMPILE_ONLY_SCALAR" cmake -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DJPEGXL_ENABLE_BENCHMARK=OFF -DJPEGXL_ENABLE_DOXYGEN=OFF -DJPEGXL_ENABLE_EXAMPLES=OFF -DJPEGXL_ENABLE_JNI=OFF -DJPEGXL_ENABLE_JPEGLI=OFF -DJPEGXL_ENABLE_JPEGLI_LIBJPEG=OFF -DJPEGXL_ENABLE_MANPAGES=OFF -DJPEGXL_ENABLE_OPENEXR=OFF -DJPEGXL_ENABLE_PLUGINS=OFF -DJPEGXL_ENABLE_SJPEG=OFF -DJPEGXL_ENABLE_SKCMS=ON -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_TOOLS=OFF -DJPEGXL_ENABLE_FUZZERS=OFF ..
make -j$(nproc) jxl jxl_cms jxl_threads

cd $SRC
cd kimageformats
cd $SRC/kimageformats
HANDLER_TYPES="ANIHandler ani
QAVIFHandler avif
QDDSHandler dds
Expand Down Expand Up @@ -202,7 +186,7 @@ echo "$HANDLER_TYPES" | while read class format; do
/usr/libexec/moc $SRC/kimageformats/src/imageformats/$format.cpp -o $format.moc
header=`ls $SRC/kimageformats/src/imageformats/$format*.h`
/usr/libexec/moc $header -o moc_`basename $header .h`.cpp
$CXX $CXXFLAGS -fPIC -DHANDLER=$class -std=c++17 $SRC/kimgio_fuzzer.cc $SRC/kimageformats/src/imageformats/$format.cpp $SRC/kimageformats/src/imageformats/scanlineconverter.cpp $SRC/kimageformats/src/imageformats/microexif.cpp -o $OUT/$fuzz_target_name -DJXL_STATIC_DEFINE -DJXL_THREADS_STATIC_DEFINE -DJXL_CMS_STATIC_DEFINE -DINITGUID -I $SRC/kimageformats/src/imageformats/ -I $SRC/libavif/include/ -I $SRC/libjxl/build/lib/include/ -I $SRC/libjxl/lib/include/ -I /usr/local/include/OpenEXR/ -I /usr/local/include/KF6/KArchive/ -I /usr/local/include/openjpeg-2.5 -I /usr/local/include/Imath -I $SRC/jxrlib/common/include -I $SRC/jxrlib/jxrgluelib -I $SRC/jxrlib/image/sys -I /usr/include/QtCore/ -I /usr/include/QtGui/ -I . $SRC/libavif/build/libavif.a /usr/local/lib/libheif.a /usr/local/lib/libde265.a /usr/local/lib/libopenh264.a $SRC/aom/build.libavif/libaom.a $SRC/libjxl/build/lib/libjxl_threads.a $SRC/libjxl/build/lib/libjxl.a $SRC/libjxl/build/lib/libjxl_cms.a $SRC/libjxl/build/third_party/highway/libhwy.a $SRC/libjxl/build/third_party/brotli/libbrotlidec.a $SRC/libjxl/build/third_party/brotli/libbrotlienc.a $SRC/libjxl/build/third_party/brotli/libbrotlicommon.a -lQt6Gui -lQt6Core -lQt6BundledLibpng -lQt6BundledHarfbuzz -lm -lQt6BundledPcre2 -ldl -lpthread $LIB_FUZZING_ENGINE /usr/local/lib/libzip.a /usr/local/lib/libz.a -lKF6Archive /usr/local/lib/libz.a /usr/local/lib/libraw.a /usr/local/lib/libOpenEXR-3_3.a /usr/local/lib/libIex-3_3.a /usr/local/lib/libImath-3_1.a /usr/local/lib/libIlmThread-3_3.a /usr/local/lib/libOpenEXRCore-3_3.a /usr/local/lib/libOpenEXRUtil-3_3.a /usr/local/lib/libopenjp2.a /usr/local/lib/libzstd.a $SRC/jxrlib/build/libjxrglue.a $SRC/jxrlib/build/libjpegxr.a -llzma /usr/local/lib/libbz2.a -lclang_rt.builtins
$CXX $CXXFLAGS -fPIC -DHANDLER=$class -std=c++17 $SRC/kimgio_fuzzer.cc $SRC/kimageformats/src/imageformats/$format.cpp $SRC/kimageformats/src/imageformats/scanlineconverter.cpp $SRC/kimageformats/src/imageformats/microexif.cpp -o $OUT/$fuzz_target_name -DJXL_STATIC_DEFINE -DJXL_THREADS_STATIC_DEFINE -DJXL_CMS_STATIC_DEFINE -DINITGUID -I $SRC/kimageformats/src/imageformats/ -I $SRC/libavif/include/ -I $SRC/libjxl/build/lib/include/ -I $SRC/libjxl/lib/include/ -I /usr/local/include/OpenEXR/ -I /usr/local/include/KF6/KArchive/ -I /usr/local/include/openjpeg-2.5 -I /usr/local/include/Imath -I $SRC/jxrlib/common/include -I $SRC/jxrlib/jxrgluelib -I $SRC/jxrlib/image/sys -I /usr/include/QtCore/ -I /usr/include/QtGui/ -I . $SRC/libavif/build/libavif.a /usr/local/lib/libheif.a /usr/local/lib/libde265.a /usr/local/lib/libopenh264.a $SRC/aom/build.libavif/libaom.a $SRC/libjxl/build/lib/libjxl_threads.a $SRC/libjxl/build/lib/libjxl.a $SRC/libjxl/build/lib/libjxl_cms.a $SRC/libjxl/build/third_party/highway/libhwy.a $SRC/libjxl/build/third_party/brotli/libbrotlidec.a $SRC/libjxl/build/third_party/brotli/libbrotlienc.a $SRC/libjxl/build/third_party/brotli/libbrotlicommon.a -lQt6Gui -lQt6Core -lQt6BundledLibpng -lQt6BundledHarfbuzz -lm -lQt6BundledPcre2 -ldl -lpthread $LIB_FUZZING_ENGINE /usr/local/lib/libz.a -lKF6Archive /usr/local/lib/libz.a /usr/local/lib/libraw.a /usr/local/lib/libOpenEXR-3_3.a /usr/local/lib/libIex-3_3.a /usr/local/lib/libImath-3_1.a /usr/local/lib/libIlmThread-3_3.a /usr/local/lib/libOpenEXRCore-3_3.a /usr/local/lib/libOpenEXRUtil-3_3.a /usr/local/lib/libopenjp2.a /usr/local/lib/libzstd.a $SRC/jxrlib/build/libjxrglue.a $SRC/jxrlib/build/libjpegxr.a -llzma /usr/local/lib/libbz2.a -lclang_rt.builtins

# -lclang_rt.builtins in the previous line is a temporary workaround to avoid a linker error "undefined reference to __truncsfhf2". Investigate why this is needed here, but not anywhere else, and possibly remove it.

Expand Down
2 changes: 1 addition & 1 deletion projects/kimageformats/project.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
homepage: https://cgit.kde.org/kimageformats.git/
homepage: https://invent.kde.org/frameworks/kimageformats.git/
language: c++
primary_contact: [email protected]
sanitizers:
Expand Down